diff --git a/.github/workflows/check_config_docs.yaml b/.github/workflows/check_config_docs.yaml index f863f80b..bb88b574 100644 --- a/.github/workflows/check_config_docs.yaml +++ b/.github/workflows/check_config_docs.yaml @@ -9,7 +9,7 @@ jobs: - uses: actions/checkout@v3 - id: common - uses: ghga-de/gh-action-common@v3 + uses: ghga-de/gh-action-common@v4 - name: Check config docs run: | diff --git a/.github/workflows/check_openapi_spec.yaml b/.github/workflows/check_openapi_spec.yaml index 5fad821c..f5db7665 100644 --- a/.github/workflows/check_openapi_spec.yaml +++ b/.github/workflows/check_openapi_spec.yaml @@ -10,7 +10,7 @@ jobs: - uses: actions/checkout@v3 - id: common - uses: ghga-de/gh-action-common@v3 + uses: ghga-de/gh-action-common@v4 - name: Check if openapi.yaml is up to date run: | diff --git a/.github/workflows/check_readme.yaml b/.github/workflows/check_readme.yaml index 9f400d23..7414554d 100644 --- a/.github/workflows/check_readme.yaml +++ b/.github/workflows/check_readme.yaml @@ -9,7 +9,7 @@ jobs: - uses: actions/checkout@v3 - id: common - uses: ghga-de/gh-action-common@v3 + uses: ghga-de/gh-action-common@v4 - name: Check readme run: | diff --git a/.github/workflows/static_code_analysis.yaml b/.github/workflows/static_code_analysis.yaml index 39b9bad4..86a014df 100644 --- a/.github/workflows/static_code_analysis.yaml +++ b/.github/workflows/static_code_analysis.yaml @@ -10,7 +10,7 @@ jobs: - uses: actions/checkout@v3 - id: common - uses: ghga-de/gh-action-common@v3 + uses: ghga-de/gh-action-common@v4 - uses: pre-commit/action@v3.0.0 env: diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 05f474ca..dacc9f50 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -11,7 +11,7 @@ jobs: - uses: actions/checkout@v3 - id: common - uses: ghga-de/gh-action-common@v3 + uses: ghga-de/gh-action-common@v4 - id: pytest run: | diff --git a/readme_generation.md b/.readme_generation/README.md similarity index 66% rename from readme_generation.md rename to .readme_generation/README.md index 84cd618b..ed2a2a79 100644 --- a/readme_generation.md +++ b/.readme_generation/README.md @@ -18,30 +18,30 @@ # Readme Generation -The README file is generated by collecting information from different sources as +The Repository README is generated by collecting information from different sources as outlined in the following. - name: The full name of the package is derived from the remote origin Git repository. - title: A title case representation of the name. - shortname: An abbreviation of the full name. This is derived from the name mentioned - in the [`./pyproject.toml`](./pyproject.toml). + in the [`../pyproject.toml`](../pyproject.toml). - summary: A short 1-2 sentence summary derived from the description in the - [`./pyproject.toml`](./pyproject.toml). + [`../pyproject.toml`](../pyproject.toml). - version: The package version derived from the version specified in the - [`./pyproject.toml`](./pyproject.toml). + [`../pyproject.toml`](../pyproject.toml). - description: A markdown-formatted description of the features and use cases of this - service or package. Obtained from the [`./.description.md`](./.description.md). + service or package. Obtained from the [`./description.md`](./description.md). - design_description: A markdown-formatted description of the overall architecture and - design of the package. Obtained from the [`./.design.md`](./.design.md). + design of the package. Obtained from the [`./design.md`](./design.md). - config_description: A markdown-formatted description of all config parameters. - This is autogenerated from the [`./config_schema.json`](./config_schema.json). + This is autogenerated from the [`../config_schema.json`](../config_schema.json). - openapi_doc: A markdown-formatted description of the HTTP API. This is autogenerated - and links to the [`./openapi.yaml`](./openapi.yaml). If the openapi.yaml is not + and links to the [`../openapi.yaml`](../openapi.yaml). If the openapi.yaml is not this documentation is empty. -The [`./.readme_template.md`](./.readme_template.md) serves as a template where the +The [`./readme_template.md`](./readme_template.md) serves as a template where the above variable can be filled in using Pythons `string.Template` utility from the standard library. -The [`./scripts/update_readme.py`] script can be used to collect all information and -fill it into the template to generate the README file. +The [`../scripts/update_readme.py`](../scripts/update_readme.py) script can be used to +collect all information and fill it into the template to generate the README file. diff --git a/.description.md b/.readme_generation/description.md similarity index 100% rename from .description.md rename to .readme_generation/description.md diff --git a/.design.md b/.readme_generation/design.md similarity index 100% rename from .design.md rename to .readme_generation/design.md diff --git a/.readme_template.md b/.readme_generation/readme_template.md similarity index 100% rename from .readme_template.md rename to .readme_generation/readme_template.md diff --git a/.template/README.md b/.template/README.md new file mode 100644 index 00000000..8a5e7cd6 --- /dev/null +++ b/.template/README.md @@ -0,0 +1,47 @@ + + +# Template File Lists + +This directory contains multiple text files that are listing paths to other files +of this repository. The listed files are affected in different ways by template updates +as explained in the following. + +## `static_files.txt` +The files listed here are synced with their counterparts in the template. They should +never be modified manually. + +## `static_files_ignore.txt` +To opt out of template updates just for individual files declared as static +(e.g. because you would like manually modify them), you may add them to this list. + +## `mandatory_files.txt` +The contents of the files listed here are not synced with the template, however, upon +every template update it is checked that the files exist. You should modify them +manually to the needs of your repository. + +## `mandatory_files_ignore.txt` +To opt out of existence checks for individual files declared as mandatory, you may add +them to this list. + +## `deprecated_files.txt` +Files listed here must not exist in your repository and are automatically deleted upon +a template update. + +## `deprecated_files_ignore.txt` +If you would like to keep files declared as deprecated, you may add them to this list. diff --git a/.deprecated_files b/.template/deprecated_files.txt similarity index 68% rename from .deprecated_files rename to .template/deprecated_files.txt index 1fc912566..79d4c279 100644 --- a/.deprecated_files +++ b/.template/deprecated_files.txt @@ -19,7 +19,21 @@ docs setup.py setup.cfg +requirements-dev-common.in +requirements-dev.in +requirements-dev.txt +requirements.txt .pylintrc .flake8 .editorconfig +.deprecated_files +.deprecated_files_ignore +.mandatory_files +.mandatory_files_ignore +.static_files +.static_files_ignore +.description.md +.design.md +.readme_template.md +.readme_generation.md diff --git a/.deprecated_files_ignore b/.template/deprecated_files_ignore.txt similarity index 100% rename from .deprecated_files_ignore rename to .template/deprecated_files_ignore.txt diff --git a/.mandatory_files b/.template/mandatory_files.txt similarity index 79% rename from .mandatory_files rename to .template/mandatory_files.txt index ba9f6c31..218a58ab 100644 --- a/.mandatory_files +++ b/.template/mandatory_files.txt @@ -13,16 +13,18 @@ tests/fixtures/__init__.py scripts/script_utils/fastapi_app_location.py +.readme_generation/description.md +.readme_generation/design.md + +lock/requirements-dev.in +lock/requirements-dev.txt +lock/requirements.txt + Dockerfile config_schema.json example_config.yaml LICENSE pyproject.toml README.md -requirements-dev.in -requirements-dev.txt -requirements.txt -.description.md -.design.md .pre-commit-config.yaml diff --git a/.mandatory_files_ignore b/.template/mandatory_files_ignore.txt similarity index 100% rename from .mandatory_files_ignore rename to .template/mandatory_files_ignore.txt diff --git a/.static_files b/.template/static_files.txt similarity index 90% rename from .static_files rename to .template/static_files.txt index 1f129ea4..65f9e0c5 100644 --- a/.static_files +++ b/.template/static_files.txt @@ -40,6 +40,14 @@ scripts/README.md example_data/README.md +.template/README.md + +.readme_generation/.readme_template.md +.readme_generation/README.md + +lock/requirements-dev-template.in +lock/README.md + .coveragerc .gitattributes .gitignore @@ -48,7 +56,3 @@ example_data/README.md pytest.ini LICENSE -requirements-dev-common.in - -.readme_template.md -readme_generation.md diff --git a/.static_files_ignore b/.template/static_files_ignore.txt similarity index 100% rename from .static_files_ignore rename to .template/static_files_ignore.txt diff --git a/lock/README.md b/lock/README.md new file mode 100644 index 00000000..fc255b55 --- /dev/null +++ b/lock/README.md @@ -0,0 +1,56 @@ + + +# Lock Files + +This directory contains two lock files locking the dependencies of this microservice: + +The [`./requirements.txt`](./requirements.txt) contains production dependencies. + +The [`./requirements-dev.txt`](./requirements-dev.txt) additionally contains development +dependencies. + +## Sources + +For generating the production lock file, only the dependencies specified in the +[`../pyproject.toml`](../pyproject.toml) are considered as input. + +For generating the development lock file, additionally, the +[`./requirements-dev-template.in`](./requirements-dev-template.in) as well as +the [`./requirements-dev.in`](./requirements-dev.in) are considered. + +The `./requirements-dev-template.in` is automatically updated from the template +repository and should not be manually modified. + +If you require additional dev dependencies not part of the +`./requirements-dev-template.in`, you can add them to the +`./requirements-dev.in`. + +## Update and Upgrade + +The lock files can be updated running the +[`../scripts/update_lock.py`](../scripts/update_lock.py) script. This will keep the +dependency versions in the lockfile unchanged unless they are in conflict with the +the input sources. In that case, the affected dependencies are updated to the latest +versions compatible with the input. + +If you would like to upgrade all dependencies in the lock file to the latest versions +compatible with the input, you can run `../scripts/update_lock.py --upgrade`. + +If you just want to check if the script would do update, you can run +`../scripts/update_lock.py --check`. diff --git a/requirements-dev-common.in b/lock/requirements-dev-template.in similarity index 100% rename from requirements-dev-common.in rename to lock/requirements-dev-template.in diff --git a/requirements-dev.in b/lock/requirements-dev.in similarity index 62% rename from requirements-dev.in rename to lock/requirements-dev.in index dc25ec87..4d48a8fd 100644 --- a/requirements-dev.in +++ b/lock/requirements-dev.in @@ -1,7 +1,7 @@ # requirements for development and testing this service -# common requirements for development and testing --r requirements-dev-common.in +# template requirements for development and testing +-r requirements-dev-template.in # additional requirements can be listed here testcontainers[kafka,mongo]>=3.4.1 diff --git a/requirements-dev.txt b/lock/requirements-dev.txt similarity index 100% rename from requirements-dev.txt rename to lock/requirements-dev.txt diff --git a/requirements.txt b/lock/requirements.txt similarity index 100% rename from requirements.txt rename to lock/requirements.txt diff --git a/scripts/license_checker.py b/scripts/license_checker.py index 5d8be069..b6a718df 100755 --- a/scripts/license_checker.py +++ b/scripts/license_checker.py @@ -64,12 +64,12 @@ ".mypy.ini", ".pytest_cache", ".editorconfig", - ".static_files", - ".static_files_ignore", - ".mandatory_files", - ".mandatory_files_ignore", - ".deprecated_files", - ".deprecated_files_ignore", + ".template/.static_files.txt", + ".template/.static_files_ignore.txt", + ".template/.mandatory_files.txt", + ".template/.mandatory_files_ignore.txt", + ".template/.deprecated_files.txt", + ".template/.deprecated_files_ignore.txt", ] # exclude file by file ending from license header check: diff --git a/scripts/update_hook_revs.py b/scripts/update_hook_revs.py index 3d653f85..6522cbe6 100755 --- a/scripts/update_hook_revs.py +++ b/scripts/update_hook_revs.py @@ -27,7 +27,7 @@ REPO_ROOT_DIR = Path(__file__).parent.parent.resolve() PRE_COMMIT_CFG_PATH = REPO_ROOT_DIR / ".pre-commit-config.yaml" -LOCK_FILE_PATH = REPO_ROOT_DIR / "requirements-dev.txt" +LOCK_FILE_PATH = REPO_ROOT_DIR / "lock" / "requirements-dev.txt" def make_dependency_dict(requirements: list[Requirement]) -> dict[str, str]: diff --git a/scripts/update_lock.py b/scripts/update_lock.py index 825aade0..98516b88 100755 --- a/scripts/update_lock.py +++ b/scripts/update_lock.py @@ -31,11 +31,12 @@ from script_utils import cli, deps REPO_ROOT_DIR = Path(__file__).parent.parent.resolve() +LOCK_DIR = REPO_ROOT_DIR / "lock" PYPROJECT_TOML_PATH = REPO_ROOT_DIR / "pyproject.toml" -DEV_DEPS_PATH = REPO_ROOT_DIR / "requirements-dev.in" -OUTPUT_LOCK_PATH = REPO_ROOT_DIR / "requirements.txt" -OUTPUT_DEV_LOCK_PATH = REPO_ROOT_DIR / "requirements-dev.txt" +DEV_DEPS_PATH = LOCK_DIR / "requirements-dev.in" +OUTPUT_LOCK_PATH = LOCK_DIR / "requirements.txt" +OUTPUT_DEV_LOCK_PATH = LOCK_DIR / "requirements-dev.txt" def fix_temp_dir_comments(file_path: Path): diff --git a/scripts/update_readme.py b/scripts/update_readme.py index 244d0ba3..c0dd4e32 100755 --- a/scripts/update_readme.py +++ b/scripts/update_readme.py @@ -32,9 +32,10 @@ ROOT_DIR = Path(__file__).parent.parent.resolve() PYPROJECT_TOML_PATH = ROOT_DIR / "pyproject.toml" -DESCRIPTION_PATH = ROOT_DIR / ".description.md" -DESIGN_PATH = ROOT_DIR / ".design.md" -README_TEMPLATE_PATH = ROOT_DIR / ".readme_template.md" +README_GENERATION_DIR = ROOT_DIR / ".readme_generation" +DESCRIPTION_PATH = README_GENERATION_DIR / "description.md" +DESIGN_PATH = README_GENERATION_DIR / "design.md" +README_TEMPLATE_PATH = README_GENERATION_DIR / "readme_template.md" CONFIG_SCHEMA_PATH = ROOT_DIR / "config_schema.json" OPENAPI_YAML_REL_PATH = "./openapi.yaml" README_PATH = ROOT_DIR / "README.md" diff --git a/scripts/update_template_files.py b/scripts/update_template_files.py index d3313fd5..fe29d575 100755 --- a/scripts/update_template_files.py +++ b/scripts/update_template_files.py @@ -40,12 +40,20 @@ def run(main_fn): main_fn(check="--check" in sys.argv[1:]) -REPO_ROOT_DIR = Path(__file__).parent.parent.resolve() +REPO_ROOT_DIR = Path(__file__).parent.parent.absolute() + +FILE_LIST_DIR_NAME = ".template" +DEPRECATED_FILES = "deprecated_files" +MANDATORY_FILES = "mandatory_files" +STATIC_FILES = "static_files" -DEPRECATED_FILES = ".deprecated_files" -MANDATORY_FILES = ".mandatory_files" -STATIC_FILES = ".static_files" IGNORE_SUFFIX = "_ignore" + +TEMPLATE_LIST_REL_PATHS = [ + f"{FILE_LIST_DIR_NAME}/{list_name}.txt" + for list_name in [STATIC_FILES, MANDATORY_FILES, DEPRECATED_FILES] +] + RAW_TEMPLATE_URL = ( "https://raw.githubusercontent.com/ghga-de/microservice-repository-template/main/" ) @@ -55,9 +63,14 @@ class ValidationError(RuntimeError): """Raised when files need to be updated.""" +def get_file_list_path(list_name: str, relative: bool = False) -> Path: + """Get the path to the file list of the given name.""" + return Path(REPO_ROOT_DIR / FILE_LIST_DIR_NAME / f"{list_name}.txt") + + def get_file_list(list_name: str) -> list[str]: """Return a list of all file names specified in a given list file.""" - list_path = REPO_ROOT_DIR / list_name + list_path = get_file_list_path(list_name) with open(list_path, encoding="utf8") as list_file: file_list = [ clean_line @@ -212,14 +225,13 @@ def remove_files(files: list[str], check: bool = False) -> bool: def main(check: bool = False): """Update the static files in the service template.""" updated = False - if not check: - update_files([STATIC_FILES], diff=True, check=False) + + print("Template lists...") + if update_files(TEMPLATE_LIST_REL_PATHS, diff=True, check=False): + updated = True print("Static files...") files_to_update = get_file_list(STATIC_FILES) - if check: - files_to_update.append(STATIC_FILES) - files_to_update.extend((MANDATORY_FILES, DEPRECATED_FILES)) if update_files(files_to_update, diff=True, check=check): updated = True