From d6d1d33ff9861fd513672a41f1caa70ca8808cae Mon Sep 17 00:00:00 2001 From: Matthias Dellweg Date: Tue, 20 Feb 2024 12:10:04 +0100 Subject: [PATCH] Improved cookiecutter templates This change will extract the actual test matrix from the existing file and reproduce it as part of the template. No need to configure it in pyproject.toml anymore. [noissue] --- .github/workflows/test.yml | 28 ++--- .../apply_templates.py | 10 +- cookiecutter/ci/cookiecutter.json | 4 +- cookiecutter/ci/hooks/post_gen_project.py | 17 +++ .../.github/workflows/test.yml | 4 +- .../CHANGES/.TEMPLATE.md | 42 +++++++ .../.gitkeep | 0 cookiecutter/pulp_filter_extension.py | 113 ++++++++++++++++++ pyproject.toml | 21 +--- 9 files changed, 200 insertions(+), 39 deletions(-) rename {.ci/scripts => cookiecutter}/apply_templates.py (72%) mode change 100644 => 100755 create mode 100644 cookiecutter/ci/hooks/post_gen_project.py create mode 100644 cookiecutter/ci/{{ cookiecutter.__project_slug }}/CHANGES/.TEMPLATE.md create mode 100644 cookiecutter/ci/{{ cookiecutter.__project_slug }}/CHANGES/pulp-glue{{ cookiecutter.__app_label_suffix }}/.gitkeep create mode 100644 cookiecutter/pulp_filter_extension.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 273e6fc4a..1a341199e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,24 +16,24 @@ jobs: fail-fast: false matrix: include: - - python: "3.11" - image_tag: "nightly" + - image_tag: "nightly" pulp_api_root: "/relocated/djnd/" - - python: "3.11" - image_tag: "latest" - - python: "3.12" - image_tag: "3.39" - - python: "3.12" - image_tag: "3.28" + python: "3.11" + - image_tag: "latest" + python: "3.11" + - image_tag: "3.39" + python: "3.12" + - image_tag: "3.28" lower_bounds: true - - python: "3.8" - image_tag: "3.22" - - python: "3.9" - image_tag: "3.21" + python: "3.12" + - image_tag: "3.22" + python: "3.8" + - image_tag: "3.21" pulp_api_root: "/relocated/djnd/" - - python: "3.10" - image_tag: "3.18" + python: "3.9" + - image_tag: "3.18" lower_bounds: true + python: "3.10" steps: - uses: "actions/checkout@v4" - uses: "actions/cache@v4" diff --git a/.ci/scripts/apply_templates.py b/cookiecutter/apply_templates.py old mode 100644 new mode 100755 similarity index 72% rename from .ci/scripts/apply_templates.py rename to cookiecutter/apply_templates.py index f3dc59b1f..4febaee6e --- a/.ci/scripts/apply_templates.py +++ b/cookiecutter/apply_templates.py @@ -1,6 +1,10 @@ +#!/bin/env python3 + from pathlib import Path import toml +import yaml + from cookiecutter.main import cookiecutter if __name__ == "__main__": @@ -10,12 +14,16 @@ with open("pyproject.toml") as fp: config = toml.load(fp)["tool"]["pulp_cli_template"] + # CI sections.append("ci") + with open(".github/workflows/test.yml") as fp: + config["test_matrix"] = yaml.safe_load(fp)["jobs"]["test"]["strategy"]["matrix"] + # Docs if config["docs"]: sections.append("docs") - cutter_path = Path(__file__).parent.parent.parent / "cookiecutter" + cutter_path = Path(__file__).parent for section in sections: print(f"Apply {section} template") diff --git a/cookiecutter/ci/cookiecutter.json b/cookiecutter/ci/cookiecutter.json index 499d4aeba..8e6d453d3 100644 --- a/cookiecutter/ci/cookiecutter.json +++ b/cookiecutter/ci/cookiecutter.json @@ -8,10 +8,12 @@ "__project_name": "pulp-cli{{ cookiecutter.__app_label_suffix }}", "__project_slug": "{{ cookiecutter.__project_name | lower | replace(' ', '_') }}", "_copy_without_render": [ + "CHANGES/.TEMPLATE.md", ".github/workflows/collect_changes.yml", ".github/workflows/pr.yml", ".github/workflows/release.yml", ".github/workflows/release_branch.yml", ".github/workflows/pr_checks.yml" - ] + ], + "_extensions": [".pulp_filter_extension.PulpFilterExtension"] } diff --git a/cookiecutter/ci/hooks/post_gen_project.py b/cookiecutter/ci/hooks/post_gen_project.py new file mode 100644 index 000000000..bac343376 --- /dev/null +++ b/cookiecutter/ci/hooks/post_gen_project.py @@ -0,0 +1,17 @@ +# flake8: noqa + +import os +import shutil +import sys + +REMOVE_PATHS = [ + {% if not cookiecutter.glue -%} "CHANGES/pulp-glue{{ cookiecutter.__app_label_suffix }}", {%- endif %} +] + +for path in REMOVE_PATHS: + path = path.strip() + if path and os.path.exists(path): + if os.path.isdir(path): + shutil.rmtree(path) + else: + os.unlink(path) diff --git a/cookiecutter/ci/{{ cookiecutter.__project_slug }}/.github/workflows/test.yml b/cookiecutter/ci/{{ cookiecutter.__project_slug }}/.github/workflows/test.yml index b153b3fc6..7ada46a1b 100644 --- a/cookiecutter/ci/{{ cookiecutter.__project_slug }}/.github/workflows/test.yml +++ b/cookiecutter/ci/{{ cookiecutter.__project_slug }}/.github/workflows/test.yml @@ -14,9 +14,7 @@ jobs: runs-on: "ubuntu-20.04" strategy: fail-fast: false - matrix: - include: - {{ cookiecutter.test_matrix | indent(10) }} + matrix:{{ cookiecutter.test_matrix | to_nice_yaml(level=4, embed_in="dict") }} steps: - uses: "actions/checkout@v4" {%- include "cache_action" %} diff --git a/cookiecutter/ci/{{ cookiecutter.__project_slug }}/CHANGES/.TEMPLATE.md b/cookiecutter/ci/{{ cookiecutter.__project_slug }}/CHANGES/.TEMPLATE.md new file mode 100644 index 000000000..1b9b356fb --- /dev/null +++ b/cookiecutter/ci/{{ cookiecutter.__project_slug }}/CHANGES/.TEMPLATE.md @@ -0,0 +1,42 @@ + +{# TOWNCRIER TEMPLATE #} +{% for section, _ in sections.items() %} +{%- set section_slug = "-" + section|replace(" ", "-")|replace("_", "-")|lower %} +{% if section %}### {{section}} {: #{{versiondata.version}}{{section_slug}} } + +{% else %} +{%- set section_slug = "" %} +{% endif %} + +{% if sections[section] %} +{% for category, val in definitions.items() if category in sections[section]%} +#### {{ definitions[category]['name'] }} {: #{{versiondata.version}}{{section_slug}}-{{category}} } + +{% if definitions[category]['showcontent'] %} +{% for text, values in sections[section][category].items() %} +- {{ text }} +{% if values %} + {{ values|join(',\n ') }} +{% endif %} +{% endfor %} + +{% else %} +- {{ sections[section][category]['']|join(', ') }} + +{% endif %} +{% if sections[section][category]|length == 0 %} +No significant changes. + +{% else %} +{% endif %} + +{% endfor %} +{% else %} +No significant changes. + + +{% endif %} +{% endfor %} +--- + + diff --git a/cookiecutter/ci/{{ cookiecutter.__project_slug }}/CHANGES/pulp-glue{{ cookiecutter.__app_label_suffix }}/.gitkeep b/cookiecutter/ci/{{ cookiecutter.__project_slug }}/CHANGES/pulp-glue{{ cookiecutter.__app_label_suffix }}/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/cookiecutter/pulp_filter_extension.py b/cookiecutter/pulp_filter_extension.py new file mode 100644 index 000000000..716e06aa9 --- /dev/null +++ b/cookiecutter/pulp_filter_extension.py @@ -0,0 +1,113 @@ +import shlex +import typing as t + +from jinja2 import Environment +from jinja2.ext import Extension + + +def _assert_key(key: t.Any) -> str: + assert isinstance(key, str) + assert key.isidentifier() + return key + + +def to_nice_yaml(data: t.Any, level: int = 0, embed_in: str = "") -> str: + """Filter for Jinja 2 templates to render human readable YAML.""" + # Don't even believe this is complete! + # Yes, I have checked pyyaml and ruamel. + # Should I call this markup language "jaml" or "yson"? + + nl = False + if isinstance(data, str): + result = f'"{data}"' + elif data is True: + result = "true" + elif data is False: + result = "false" + elif isinstance(data, int): + result = f"{data}" + elif isinstance(data, list): + if len(data): + nl = embed_in == "dict" + result = ("\n" + " " * level).join( + ("-" + to_nice_yaml(item, level + 1, "list") for item in data) + ) + else: + result = "[]" + elif isinstance(data, dict): + if len(data): + nl = embed_in == "dict" + result = ("\n" + " " * level).join( + ( + f"{_assert_key(key)}:" + to_nice_yaml(value, level + 1, "dict") + for key, value in sorted(data.items()) + ) + ) + else: + result = "{}" + else: + raise NotImplementedError("YAML sucks!") + if nl: + return "\n" + " " * level + result + elif embed_in: + return " " + result + else: + return result + + +def to_camel(name: str) -> str: + return name.title().replace("_", "") + + +def to_dash(name: str) -> str: + return name.replace("_", "-") + + +def to_snake(name: str) -> str: + return name.replace("-", "_") + + +class PulpFilterExtension(Extension): + def __init__(self, environment: Environment): + super().__init__(environment) + environment.filters["camel"] = to_camel + environment.filters["caps"] = str.upper + environment.filters["dash"] = to_dash + environment.filters["snake"] = to_snake + environment.filters["to_nice_yaml"] = to_nice_yaml + environment.filters["shquote"] = shlex.quote + + +try: + import pytest +except ImportError: + pass +else: + + @pytest.mark.parametrize( + "value,level,embed_in,out", + [ + ([], 0, "", "[]"), + ({}, 0, "", "{}"), + ("test", 0, "", '"test"'), + ({}, 1, "dict", " {}"), + ([], 1, "list", " []"), + ({}, 1, "list", " {}"), + ([], 1, "dict", " []"), + ([{}], 0, "", "- {}"), + ([[[]]], 0, "", "- - []"), + ( + {"a": [], "b": "test", "c": [True, False]}, + 2, + "list", + """ a: [] + b: "test" + c: + - true + - false""", + ), + ({"b": 1, "a": 0}, 0, "", "a: 0\nb: 1"), + ], + ) + def test_to_nice_yaml(value, level, embed_in, out): + assert to_nice_yaml(value, level, embed_in) == out diff --git a/pyproject.toml b/pyproject.toml index d590e8d04..0c270082e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,26 +66,6 @@ namespaces = true app_label = "" docs = true translations = true -test_matrix = """ -- python: "3.11" - image_tag: "nightly" - pulp_api_root: "/relocated/djnd/" -- python: "3.11" - image_tag: "latest" -- python: "3.12" - image_tag: "3.39" -- python: "3.12" - image_tag: "3.28" - lower_bounds: true -- python: "3.8" - image_tag: "3.22" -- python: "3.9" - image_tag: "3.21" - pulp_api_root: "/relocated/djnd/" -- python: "3.10" - image_tag: "3.18" - lower_bounds: true -""" [tool.towncrier] filename = "docs/CHANGES.md" @@ -141,6 +121,7 @@ showcontent = false [tool.black] line-length = 100 +exclude = "cookiecutter/ci/hooks" [tool.isort] profile = "black"