From b7b484d07300be60d0824f222fe1422f9631c72e Mon Sep 17 00:00:00 2001 From: KerstenBreuer Date: Mon, 18 Dec 2023 13:59:21 +0000 Subject: [PATCH 1/9] Moved template file lists (GSI-540) ... into dedicated directory. --- .../.deprecated_files | 6 +++ .../.deprecated_files_ignore | 0 .../.mandatory_files | 0 .../.mandatory_files_ignore | 0 .static_files => .template/.static_files | 0 .../.static_files_ignore | 0 .template/README.md | 47 +++++++++++++++++++ scripts/update_template_files.py | 26 +++++++--- 8 files changed, 72 insertions(+), 7 deletions(-) rename .deprecated_files => .template/.deprecated_files (83%) rename .deprecated_files_ignore => .template/.deprecated_files_ignore (100%) rename .mandatory_files => .template/.mandatory_files (100%) rename .mandatory_files_ignore => .template/.mandatory_files_ignore (100%) rename .static_files => .template/.static_files (100%) rename .static_files_ignore => .template/.static_files_ignore (100%) create mode 100644 .template/README.md diff --git a/.deprecated_files b/.template/.deprecated_files similarity index 83% rename from .deprecated_files rename to .template/.deprecated_files index 1fc912566..d55ff32b 100644 --- a/.deprecated_files +++ b/.template/.deprecated_files @@ -23,3 +23,9 @@ setup.cfg .pylintrc .flake8 .editorconfig +.deprecated_files +.deprecated_files_ignore +.mandatory_files +.mandatory_files_ignore +.static_files +.static_files_ignore diff --git a/.deprecated_files_ignore b/.template/.deprecated_files_ignore similarity index 100% rename from .deprecated_files_ignore rename to .template/.deprecated_files_ignore diff --git a/.mandatory_files b/.template/.mandatory_files similarity index 100% rename from .mandatory_files rename to .template/.mandatory_files diff --git a/.mandatory_files_ignore b/.template/.mandatory_files_ignore similarity index 100% rename from .mandatory_files_ignore rename to .template/.mandatory_files_ignore diff --git a/.static_files b/.template/.static_files similarity index 100% rename from .static_files rename to .template/.static_files diff --git a/.static_files_ignore b/.template/.static_files_ignore similarity index 100% rename from .static_files_ignore rename to .template/.static_files_ignore diff --git a/.template/README.md b/.template/README.md new file mode 100644 index 00000000..df346b27 --- /dev/null +++ b/.template/README.md @@ -0,0 +1,47 @@ + + +# Template File Lists + +This directory contains multiple text files that are listing paths to other files +of this repository. The listed files are affected in different ways by template updates +as explained in the following. + +## `.static_files` +The files listed here are synced with their counterparts in the template. They should +never be modified manually. + +## `.static_files_ignore` +To opt out of template updates just for individual files declared as static +(e.g. because you would like manually modify them), you may add them to this list. + +## `.mandatory_files` +The contents of the files listed here are not synced with the template, however, upon +every template update it is checked that the files exist. You should modify them +manually to the needs of your repository. + +## `.mandatory_files_ignore` +To opt out of existence checks for individual files declared as mandatory, you may add +them to this list. + +## `deprecated_files` +Files listed here must not exist in your repository and are automatically deleted upon +a template update. + +## `.deprecated_files_ignore` +If you would like to keep files declared as deprecated, you may add them to this list. diff --git a/scripts/update_template_files.py b/scripts/update_template_files.py index d3313fd5..04edd49f 100755 --- a/scripts/update_template_files.py +++ b/scripts/update_template_files.py @@ -40,12 +40,20 @@ def run(main_fn): main_fn(check="--check" in sys.argv[1:]) -REPO_ROOT_DIR = Path(__file__).parent.parent.resolve() +REPO_ROOT_DIR = Path(__file__).parent.parent.absolute() +FILE_LIST_DIR_NAME = ".template" DEPRECATED_FILES = ".deprecated_files" MANDATORY_FILES = ".mandatory_files" STATIC_FILES = ".static_files" + IGNORE_SUFFIX = "_ignore" + +TEMPLATE_LIST_REL_PATHS = [ + f"{FILE_LIST_DIR_NAME}/{list_name}" + for list_name in [STATIC_FILES, MANDATORY_FILES, DEPRECATED_FILES] +] + RAW_TEMPLATE_URL = ( "https://raw.githubusercontent.com/ghga-de/microservice-repository-template/main/" ) @@ -55,9 +63,14 @@ class ValidationError(RuntimeError): """Raised when files need to be updated.""" +def get_file_list_path(list_name: str, relative: bool = False) -> Path: + """Get the path to the file list of the given name.""" + return Path(REPO_ROOT_DIR / FILE_LIST_DIR_NAME / list_name) + + def get_file_list(list_name: str) -> list[str]: """Return a list of all file names specified in a given list file.""" - list_path = REPO_ROOT_DIR / list_name + list_path = get_file_list_path(list_name) with open(list_path, encoding="utf8") as list_file: file_list = [ clean_line @@ -212,14 +225,13 @@ def remove_files(files: list[str], check: bool = False) -> bool: def main(check: bool = False): """Update the static files in the service template.""" updated = False - if not check: - update_files([STATIC_FILES], diff=True, check=False) + + print("Template lists...") + if update_files(TEMPLATE_LIST_REL_PATHS, diff=True, check=False): + updated = True print("Static files...") files_to_update = get_file_list(STATIC_FILES) - if check: - files_to_update.append(STATIC_FILES) - files_to_update.extend((MANDATORY_FILES, DEPRECATED_FILES)) if update_files(files_to_update, diff=True, check=check): updated = True From a5e42bb3ba90b33cd214a6e12bb1d3ab0d9e286d Mon Sep 17 00:00:00 2001 From: KerstenBreuer Date: Mon, 18 Dec 2023 14:03:01 +0000 Subject: [PATCH 2/9] fix license checker --- scripts/license_checker.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/license_checker.py b/scripts/license_checker.py index 5d8be069..d065965f 100755 --- a/scripts/license_checker.py +++ b/scripts/license_checker.py @@ -64,12 +64,12 @@ ".mypy.ini", ".pytest_cache", ".editorconfig", - ".static_files", - ".static_files_ignore", - ".mandatory_files", - ".mandatory_files_ignore", - ".deprecated_files", - ".deprecated_files_ignore", + ".template/.static_files", + ".template/.static_files_ignore", + ".template/.mandatory_files", + ".template/.mandatory_files_ignore", + ".template/.deprecated_files", + ".template/.deprecated_files_ignore", ] # exclude file by file ending from license header check: From 8f27482327fe304ff38c979c3a90269e317a435c Mon Sep 17 00:00:00 2001 From: KerstenBreuer Date: Mon, 18 Dec 2023 14:17:30 +0000 Subject: [PATCH 3/9] moved readme generation files to dedicated directory --- .../README.md | 22 +++++++++---------- .../description.md | 0 .design.md => .readme_generation/design.md | 0 .../readme_template.md | 0 .template/.deprecated_files | 4 ++++ .template/.static_files | 8 ++++--- scripts/update_readme.py | 7 +++--- 7 files changed, 24 insertions(+), 17 deletions(-) rename readme_generation.md => .readme_generation/README.md (66%) rename .description.md => .readme_generation/description.md (100%) rename .design.md => .readme_generation/design.md (100%) rename .readme_template.md => .readme_generation/readme_template.md (100%) diff --git a/readme_generation.md b/.readme_generation/README.md similarity index 66% rename from readme_generation.md rename to .readme_generation/README.md index 84cd618b..ed2a2a79 100644 --- a/readme_generation.md +++ b/.readme_generation/README.md @@ -18,30 +18,30 @@ # Readme Generation -The README file is generated by collecting information from different sources as +The Repository README is generated by collecting information from different sources as outlined in the following. - name: The full name of the package is derived from the remote origin Git repository. - title: A title case representation of the name. - shortname: An abbreviation of the full name. This is derived from the name mentioned - in the [`./pyproject.toml`](./pyproject.toml). + in the [`../pyproject.toml`](../pyproject.toml). - summary: A short 1-2 sentence summary derived from the description in the - [`./pyproject.toml`](./pyproject.toml). + [`../pyproject.toml`](../pyproject.toml). - version: The package version derived from the version specified in the - [`./pyproject.toml`](./pyproject.toml). + [`../pyproject.toml`](../pyproject.toml). - description: A markdown-formatted description of the features and use cases of this - service or package. Obtained from the [`./.description.md`](./.description.md). + service or package. Obtained from the [`./description.md`](./description.md). - design_description: A markdown-formatted description of the overall architecture and - design of the package. Obtained from the [`./.design.md`](./.design.md). + design of the package. Obtained from the [`./design.md`](./design.md). - config_description: A markdown-formatted description of all config parameters. - This is autogenerated from the [`./config_schema.json`](./config_schema.json). + This is autogenerated from the [`../config_schema.json`](../config_schema.json). - openapi_doc: A markdown-formatted description of the HTTP API. This is autogenerated - and links to the [`./openapi.yaml`](./openapi.yaml). If the openapi.yaml is not + and links to the [`../openapi.yaml`](../openapi.yaml). If the openapi.yaml is not this documentation is empty. -The [`./.readme_template.md`](./.readme_template.md) serves as a template where the +The [`./readme_template.md`](./readme_template.md) serves as a template where the above variable can be filled in using Pythons `string.Template` utility from the standard library. -The [`./scripts/update_readme.py`] script can be used to collect all information and -fill it into the template to generate the README file. +The [`../scripts/update_readme.py`](../scripts/update_readme.py) script can be used to +collect all information and fill it into the template to generate the README file. diff --git a/.description.md b/.readme_generation/description.md similarity index 100% rename from .description.md rename to .readme_generation/description.md diff --git a/.design.md b/.readme_generation/design.md similarity index 100% rename from .design.md rename to .readme_generation/design.md diff --git a/.readme_template.md b/.readme_generation/readme_template.md similarity index 100% rename from .readme_template.md rename to .readme_generation/readme_template.md diff --git a/.template/.deprecated_files b/.template/.deprecated_files index d55ff32b..41afa2fe 100644 --- a/.template/.deprecated_files +++ b/.template/.deprecated_files @@ -29,3 +29,7 @@ setup.cfg .mandatory_files_ignore .static_files .static_files_ignore +.description.md +.design.md +.readme_template.md +.readme_generation.md diff --git a/.template/.static_files b/.template/.static_files index 1f129ea4..6d0a857b 100644 --- a/.template/.static_files +++ b/.template/.static_files @@ -40,6 +40,11 @@ scripts/README.md example_data/README.md +.template/README.md + +readme_generation/.readme_template.md +readme_generation/README.md + .coveragerc .gitattributes .gitignore @@ -49,6 +54,3 @@ pytest.ini LICENSE requirements-dev-common.in - -.readme_template.md -readme_generation.md diff --git a/scripts/update_readme.py b/scripts/update_readme.py index 244d0ba3..c0dd4e32 100755 --- a/scripts/update_readme.py +++ b/scripts/update_readme.py @@ -32,9 +32,10 @@ ROOT_DIR = Path(__file__).parent.parent.resolve() PYPROJECT_TOML_PATH = ROOT_DIR / "pyproject.toml" -DESCRIPTION_PATH = ROOT_DIR / ".description.md" -DESIGN_PATH = ROOT_DIR / ".design.md" -README_TEMPLATE_PATH = ROOT_DIR / ".readme_template.md" +README_GENERATION_DIR = ROOT_DIR / ".readme_generation" +DESCRIPTION_PATH = README_GENERATION_DIR / "description.md" +DESIGN_PATH = README_GENERATION_DIR / "design.md" +README_TEMPLATE_PATH = README_GENERATION_DIR / "readme_template.md" CONFIG_SCHEMA_PATH = ROOT_DIR / "config_schema.json" OPENAPI_YAML_REL_PATH = "./openapi.yaml" README_PATH = ROOT_DIR / "README.md" From cd16bbb29c7300f42203b11e6a68a8eec35918a7 Mon Sep 17 00:00:00 2001 From: KerstenBreuer Date: Mon, 18 Dec 2023 14:24:20 +0000 Subject: [PATCH 4/9] rename template file lists --- .template/{.deprecated_files => deprecated_files.txt} | 0 ...ecated_files_ignore => deprecated_files_ignore.txt} | 0 .template/{.mandatory_files => mandatory_files.txt} | 0 ...ndatory_files_ignore => mandatory_files_ignore.txt} | 0 .template/{.static_files => static_files.txt} | 0 .../{.static_files_ignore => static_files_ignore.txt} | 0 scripts/update_template_files.py | 10 +++++----- 7 files changed, 5 insertions(+), 5 deletions(-) rename .template/{.deprecated_files => deprecated_files.txt} (100%) rename .template/{.deprecated_files_ignore => deprecated_files_ignore.txt} (100%) rename .template/{.mandatory_files => mandatory_files.txt} (100%) rename .template/{.mandatory_files_ignore => mandatory_files_ignore.txt} (100%) rename .template/{.static_files => static_files.txt} (100%) rename .template/{.static_files_ignore => static_files_ignore.txt} (100%) diff --git a/.template/.deprecated_files b/.template/deprecated_files.txt similarity index 100% rename from .template/.deprecated_files rename to .template/deprecated_files.txt diff --git a/.template/.deprecated_files_ignore b/.template/deprecated_files_ignore.txt similarity index 100% rename from .template/.deprecated_files_ignore rename to .template/deprecated_files_ignore.txt diff --git a/.template/.mandatory_files b/.template/mandatory_files.txt similarity index 100% rename from .template/.mandatory_files rename to .template/mandatory_files.txt diff --git a/.template/.mandatory_files_ignore b/.template/mandatory_files_ignore.txt similarity index 100% rename from .template/.mandatory_files_ignore rename to .template/mandatory_files_ignore.txt diff --git a/.template/.static_files b/.template/static_files.txt similarity index 100% rename from .template/.static_files rename to .template/static_files.txt diff --git a/.template/.static_files_ignore b/.template/static_files_ignore.txt similarity index 100% rename from .template/.static_files_ignore rename to .template/static_files_ignore.txt diff --git a/scripts/update_template_files.py b/scripts/update_template_files.py index 04edd49f..d8237843 100755 --- a/scripts/update_template_files.py +++ b/scripts/update_template_files.py @@ -43,14 +43,14 @@ def run(main_fn): REPO_ROOT_DIR = Path(__file__).parent.parent.absolute() FILE_LIST_DIR_NAME = ".template" -DEPRECATED_FILES = ".deprecated_files" -MANDATORY_FILES = ".mandatory_files" -STATIC_FILES = ".static_files" +DEPRECATED_FILES = "deprecated_files" +MANDATORY_FILES = "mandatory_files" +STATIC_FILES = "static_files" IGNORE_SUFFIX = "_ignore" TEMPLATE_LIST_REL_PATHS = [ - f"{FILE_LIST_DIR_NAME}/{list_name}" + f"{FILE_LIST_DIR_NAME}/{list_name}.txt" for list_name in [STATIC_FILES, MANDATORY_FILES, DEPRECATED_FILES] ] @@ -65,7 +65,7 @@ class ValidationError(RuntimeError): def get_file_list_path(list_name: str, relative: bool = False) -> Path: """Get the path to the file list of the given name.""" - return Path(REPO_ROOT_DIR / FILE_LIST_DIR_NAME / list_name) + return Path(REPO_ROOT_DIR / FILE_LIST_DIR_NAME / f"{list_name}") def get_file_list(list_name: str) -> list[str]: From eba0a0b9c64c798e5c2c116220412b69c70e4a16 Mon Sep 17 00:00:00 2001 From: KerstenBreuer Date: Mon, 18 Dec 2023 14:26:15 +0000 Subject: [PATCH 5/9] add readme design and description to mandatory files --- .template/README.md | 12 ++++++------ .template/mandatory_files.txt | 5 +++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.template/README.md b/.template/README.md index df346b27..8a5e7cd6 100644 --- a/.template/README.md +++ b/.template/README.md @@ -22,26 +22,26 @@ This directory contains multiple text files that are listing paths to other file of this repository. The listed files are affected in different ways by template updates as explained in the following. -## `.static_files` +## `static_files.txt` The files listed here are synced with their counterparts in the template. They should never be modified manually. -## `.static_files_ignore` +## `static_files_ignore.txt` To opt out of template updates just for individual files declared as static (e.g. because you would like manually modify them), you may add them to this list. -## `.mandatory_files` +## `mandatory_files.txt` The contents of the files listed here are not synced with the template, however, upon every template update it is checked that the files exist. You should modify them manually to the needs of your repository. -## `.mandatory_files_ignore` +## `mandatory_files_ignore.txt` To opt out of existence checks for individual files declared as mandatory, you may add them to this list. -## `deprecated_files` +## `deprecated_files.txt` Files listed here must not exist in your repository and are automatically deleted upon a template update. -## `.deprecated_files_ignore` +## `deprecated_files_ignore.txt` If you would like to keep files declared as deprecated, you may add them to this list. diff --git a/.template/mandatory_files.txt b/.template/mandatory_files.txt index ba9f6c31..96392b9f 100644 --- a/.template/mandatory_files.txt +++ b/.template/mandatory_files.txt @@ -13,6 +13,9 @@ tests/fixtures/__init__.py scripts/script_utils/fastapi_app_location.py +.readme_generation/description.md +.readme_generation/design.md + Dockerfile config_schema.json example_config.yaml @@ -23,6 +26,4 @@ requirements-dev.in requirements-dev.txt requirements.txt -.description.md -.design.md .pre-commit-config.yaml From d3fb77b33539c2999e7914f08cb4b31c29b1149c Mon Sep 17 00:00:00 2001 From: KerstenBreuer Date: Mon, 18 Dec 2023 14:52:39 +0000 Subject: [PATCH 6/9] move lock files and dev in requirements into dedicated dir --- .template/deprecated_files.txt | 4 ++ .template/mandatory_files.txt | 7 ++- .template/static_files.txt | 8 ++- lock/README.md | 56 +++++++++++++++++++ .../requirements-dev-template.in | 0 .../requirements-dev.in | 4 +- .../requirements-dev.txt | 0 requirements.txt => lock/requirements.txt | 0 scripts/update_hook_revs.py | 2 +- scripts/update_lock.py | 7 ++- 10 files changed, 76 insertions(+), 12 deletions(-) create mode 100644 lock/README.md rename requirements-dev-common.in => lock/requirements-dev-template.in (100%) rename requirements-dev.in => lock/requirements-dev.in (62%) rename requirements-dev.txt => lock/requirements-dev.txt (100%) rename requirements.txt => lock/requirements.txt (100%) diff --git a/.template/deprecated_files.txt b/.template/deprecated_files.txt index 41afa2fe..79d4c279 100644 --- a/.template/deprecated_files.txt +++ b/.template/deprecated_files.txt @@ -19,6 +19,10 @@ docs setup.py setup.cfg +requirements-dev-common.in +requirements-dev.in +requirements-dev.txt +requirements.txt .pylintrc .flake8 diff --git a/.template/mandatory_files.txt b/.template/mandatory_files.txt index 96392b9f..218a58ab 100644 --- a/.template/mandatory_files.txt +++ b/.template/mandatory_files.txt @@ -16,14 +16,15 @@ scripts/script_utils/fastapi_app_location.py .readme_generation/description.md .readme_generation/design.md +lock/requirements-dev.in +lock/requirements-dev.txt +lock/requirements.txt + Dockerfile config_schema.json example_config.yaml LICENSE pyproject.toml README.md -requirements-dev.in -requirements-dev.txt -requirements.txt .pre-commit-config.yaml diff --git a/.template/static_files.txt b/.template/static_files.txt index 6d0a857b..65f9e0c5 100644 --- a/.template/static_files.txt +++ b/.template/static_files.txt @@ -42,8 +42,11 @@ example_data/README.md .template/README.md -readme_generation/.readme_template.md -readme_generation/README.md +.readme_generation/.readme_template.md +.readme_generation/README.md + +lock/requirements-dev-template.in +lock/README.md .coveragerc .gitattributes @@ -53,4 +56,3 @@ readme_generation/README.md pytest.ini LICENSE -requirements-dev-common.in diff --git a/lock/README.md b/lock/README.md new file mode 100644 index 00000000..fc255b55 --- /dev/null +++ b/lock/README.md @@ -0,0 +1,56 @@ + + +# Lock Files + +This directory contains two lock files locking the dependencies of this microservice: + +The [`./requirements.txt`](./requirements.txt) contains production dependencies. + +The [`./requirements-dev.txt`](./requirements-dev.txt) additionally contains development +dependencies. + +## Sources + +For generating the production lock file, only the dependencies specified in the +[`../pyproject.toml`](../pyproject.toml) are considered as input. + +For generating the development lock file, additionally, the +[`./requirements-dev-template.in`](./requirements-dev-template.in) as well as +the [`./requirements-dev.in`](./requirements-dev.in) are considered. + +The `./requirements-dev-template.in` is automatically updated from the template +repository and should not be manually modified. + +If you require additional dev dependencies not part of the +`./requirements-dev-template.in`, you can add them to the +`./requirements-dev.in`. + +## Update and Upgrade + +The lock files can be updated running the +[`../scripts/update_lock.py`](../scripts/update_lock.py) script. This will keep the +dependency versions in the lockfile unchanged unless they are in conflict with the +the input sources. In that case, the affected dependencies are updated to the latest +versions compatible with the input. + +If you would like to upgrade all dependencies in the lock file to the latest versions +compatible with the input, you can run `../scripts/update_lock.py --upgrade`. + +If you just want to check if the script would do update, you can run +`../scripts/update_lock.py --check`. diff --git a/requirements-dev-common.in b/lock/requirements-dev-template.in similarity index 100% rename from requirements-dev-common.in rename to lock/requirements-dev-template.in diff --git a/requirements-dev.in b/lock/requirements-dev.in similarity index 62% rename from requirements-dev.in rename to lock/requirements-dev.in index dc25ec87..4d48a8fd 100644 --- a/requirements-dev.in +++ b/lock/requirements-dev.in @@ -1,7 +1,7 @@ # requirements for development and testing this service -# common requirements for development and testing --r requirements-dev-common.in +# template requirements for development and testing +-r requirements-dev-template.in # additional requirements can be listed here testcontainers[kafka,mongo]>=3.4.1 diff --git a/requirements-dev.txt b/lock/requirements-dev.txt similarity index 100% rename from requirements-dev.txt rename to lock/requirements-dev.txt diff --git a/requirements.txt b/lock/requirements.txt similarity index 100% rename from requirements.txt rename to lock/requirements.txt diff --git a/scripts/update_hook_revs.py b/scripts/update_hook_revs.py index 3d653f85..6522cbe6 100755 --- a/scripts/update_hook_revs.py +++ b/scripts/update_hook_revs.py @@ -27,7 +27,7 @@ REPO_ROOT_DIR = Path(__file__).parent.parent.resolve() PRE_COMMIT_CFG_PATH = REPO_ROOT_DIR / ".pre-commit-config.yaml" -LOCK_FILE_PATH = REPO_ROOT_DIR / "requirements-dev.txt" +LOCK_FILE_PATH = REPO_ROOT_DIR / "lock" / "requirements-dev.txt" def make_dependency_dict(requirements: list[Requirement]) -> dict[str, str]: diff --git a/scripts/update_lock.py b/scripts/update_lock.py index 825aade0..98516b88 100755 --- a/scripts/update_lock.py +++ b/scripts/update_lock.py @@ -31,11 +31,12 @@ from script_utils import cli, deps REPO_ROOT_DIR = Path(__file__).parent.parent.resolve() +LOCK_DIR = REPO_ROOT_DIR / "lock" PYPROJECT_TOML_PATH = REPO_ROOT_DIR / "pyproject.toml" -DEV_DEPS_PATH = REPO_ROOT_DIR / "requirements-dev.in" -OUTPUT_LOCK_PATH = REPO_ROOT_DIR / "requirements.txt" -OUTPUT_DEV_LOCK_PATH = REPO_ROOT_DIR / "requirements-dev.txt" +DEV_DEPS_PATH = LOCK_DIR / "requirements-dev.in" +OUTPUT_LOCK_PATH = LOCK_DIR / "requirements.txt" +OUTPUT_DEV_LOCK_PATH = LOCK_DIR / "requirements-dev.txt" def fix_temp_dir_comments(file_path: Path): From 84540cf9c163793d6432d577b04e0bf10f84517a Mon Sep 17 00:00:00 2001 From: KerstenBreuer Date: Mon, 18 Dec 2023 15:31:21 +0000 Subject: [PATCH 7/9] update gh action common --- .github/workflows/check_config_docs.yaml | 2 +- .github/workflows/check_openapi_spec.yaml | 2 +- .github/workflows/check_readme.yaml | 2 +- .github/workflows/static_code_analysis.yaml | 2 +- .github/workflows/tests.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/check_config_docs.yaml b/.github/workflows/check_config_docs.yaml index f863f80b..bb88b574 100644 --- a/.github/workflows/check_config_docs.yaml +++ b/.github/workflows/check_config_docs.yaml @@ -9,7 +9,7 @@ jobs: - uses: actions/checkout@v3 - id: common - uses: ghga-de/gh-action-common@v3 + uses: ghga-de/gh-action-common@v4 - name: Check config docs run: | diff --git a/.github/workflows/check_openapi_spec.yaml b/.github/workflows/check_openapi_spec.yaml index 5fad821c..f5db7665 100644 --- a/.github/workflows/check_openapi_spec.yaml +++ b/.github/workflows/check_openapi_spec.yaml @@ -10,7 +10,7 @@ jobs: - uses: actions/checkout@v3 - id: common - uses: ghga-de/gh-action-common@v3 + uses: ghga-de/gh-action-common@v4 - name: Check if openapi.yaml is up to date run: | diff --git a/.github/workflows/check_readme.yaml b/.github/workflows/check_readme.yaml index 9f400d23..7414554d 100644 --- a/.github/workflows/check_readme.yaml +++ b/.github/workflows/check_readme.yaml @@ -9,7 +9,7 @@ jobs: - uses: actions/checkout@v3 - id: common - uses: ghga-de/gh-action-common@v3 + uses: ghga-de/gh-action-common@v4 - name: Check readme run: | diff --git a/.github/workflows/static_code_analysis.yaml b/.github/workflows/static_code_analysis.yaml index 39b9bad4..86a014df 100644 --- a/.github/workflows/static_code_analysis.yaml +++ b/.github/workflows/static_code_analysis.yaml @@ -10,7 +10,7 @@ jobs: - uses: actions/checkout@v3 - id: common - uses: ghga-de/gh-action-common@v3 + uses: ghga-de/gh-action-common@v4 - uses: pre-commit/action@v3.0.0 env: diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 05f474ca..dacc9f50 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -11,7 +11,7 @@ jobs: - uses: actions/checkout@v3 - id: common - uses: ghga-de/gh-action-common@v3 + uses: ghga-de/gh-action-common@v4 - id: pytest run: | From 210b94af2a069017a8f15183ccb8d62408638b71 Mon Sep 17 00:00:00 2001 From: KerstenBreuer Date: Mon, 18 Dec 2023 16:44:08 +0000 Subject: [PATCH 8/9] add pyproject generation --- .github/workflows/check_pyproject.yaml | 16 ++++ .pyproject_generation/pyproject_custom.toml | 19 ++++ .pyproject_generation/pyproject_template.toml | 25 +++++ .template/mandatory_files.txt | 2 + .template/static_files.txt | 5 + pyproject.toml | 30 ++---- scripts/update_pyproject.py | 92 +++++++++++++++++++ 7 files changed, 166 insertions(+), 23 deletions(-) create mode 100644 .github/workflows/check_pyproject.yaml create mode 100644 .pyproject_generation/pyproject_custom.toml create mode 100644 .pyproject_generation/pyproject_template.toml create mode 100755 scripts/update_pyproject.py diff --git a/.github/workflows/check_pyproject.yaml b/.github/workflows/check_pyproject.yaml new file mode 100644 index 00000000..d929c799 --- /dev/null +++ b/.github/workflows/check_pyproject.yaml @@ -0,0 +1,16 @@ +name: Check if the config schema and the example are up to date. + +on: push + +jobs: + static-code-analysis: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - id: common + uses: ghga-de/gh-action-common@v4 + + - name: Check pyproject.toml + run: | + ./scripts/update_pyproject.py --check diff --git a/.pyproject_generation/pyproject_custom.toml b/.pyproject_generation/pyproject_custom.toml new file mode 100644 index 00000000..5f8c2276 --- /dev/null +++ b/.pyproject_generation/pyproject_custom.toml @@ -0,0 +1,19 @@ +[project] +# please adapt to package name +name = "my_microservice" +version = "0.1.0" +description = "My-Microservice - a short description" +dependencies = [ + "typer >= 0.9.0", + "ghga-service-commons[api] >= 1.2.0", + "ghga-event-schemas >= 1.0.0", + "hexkit[akafka,s3,mongodb] >= 1.1.0" +] + +[project.urls] +# please adapt to package name +Repository = "https://github.com/ghga-de/my-microservice" + +[project.scripts] +# please adapt to package name +my-microservice = "my_microservice.__main__:run" diff --git a/.pyproject_generation/pyproject_template.toml b/.pyproject_generation/pyproject_template.toml new file mode 100644 index 00000000..4a62c847 --- /dev/null +++ b/.pyproject_generation/pyproject_template.toml @@ -0,0 +1,25 @@ +[build-system] +requires = ["setuptools>=67.7.2"] +build-backend = "setuptools.build_meta" + +[project] +readme = "README.md" +authors = [ + { name = "German Human Genome Phenome Archive (GHGA)", email = "contact@ghga.de" }, +] +requires-python = ">=3.9" +license = { text = "Apache 2.0" } +classifiers = [ + "Development Status :: 1 - Planning", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "License :: OSI Approved :: Apache Software License", + "Topic :: Internet :: WWW/HTTP :: HTTP Servers", + "Topic :: Software Development :: Libraries", + "Intended Audience :: Developers", +] + +[tool.setuptools.packages.find] +where = ["src"] diff --git a/.template/mandatory_files.txt b/.template/mandatory_files.txt index 218a58ab..660a15ed 100644 --- a/.template/mandatory_files.txt +++ b/.template/mandatory_files.txt @@ -16,6 +16,8 @@ scripts/script_utils/fastapi_app_location.py .readme_generation/description.md .readme_generation/design.md +.pyproject_generation/pyproject_custom.toml + lock/requirements-dev.in lock/requirements-dev.txt lock/requirements.txt diff --git a/.template/static_files.txt b/.template/static_files.txt index 65f9e0c5..ade3194f 100644 --- a/.template/static_files.txt +++ b/.template/static_files.txt @@ -26,12 +26,14 @@ scripts/update_openapi_docs.py scripts/update_readme.py scripts/update_lock.py scripts/update_hook_revs.py +scripts/update_pyproject.py scripts/list_outdated_dependencies.py scripts/README.md .github/workflows/check_config_docs.yaml .github/workflows/check_openapi_spec.yaml .github/workflows/check_readme.yaml +./github/workflows/check_pyproject.yaml .github/workflows/check_template_files.yaml .github/workflows/ci_release.yaml .github/workflows/ci_workflow_dispatch.yaml @@ -45,6 +47,9 @@ example_data/README.md .readme_generation/.readme_template.md .readme_generation/README.md +.pyproject_generation/pyproject_template.toml +.pyproject_generation/README.md + lock/requirements-dev-template.in lock/README.md diff --git a/pyproject.toml b/pyproject.toml index 903da80b..e2e3f337 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,43 +1,27 @@ [build-system] -requires = ["setuptools>=67.7.2"] +requires = [ + "setuptools>=67.7.2", +] build-backend = "setuptools.build_meta" [project] -# please adapt to package name name = "my_microservice" version = "0.1.0" description = "My-Microservice - a short description" -readme = "README.md" -authors = [ - { name = "German Human Genome Phenome Archive (GHGA)", email = "contact@ghga.de" }, -] -requires-python = ">=3.9" -license = { text = "Apache 2.0" } -classifiers = [ - "Development Status :: 1 - Planning", - "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "License :: OSI Approved :: Apache Software License", - "Topic :: Internet :: WWW/HTTP :: HTTP Servers", - "Topic :: Software Development :: Libraries", - "Intended Audience :: Developers", -] dependencies = [ "typer >= 0.9.0", "ghga-service-commons[api] >= 1.2.0", "ghga-event-schemas >= 1.0.0", - "hexkit[akafka,s3,mongodb] >= 1.1.0" + "hexkit[akafka,s3,mongodb] >= 1.1.0", ] [project.urls] -# please adapt to package name Repository = "https://github.com/ghga-de/my-microservice" [project.scripts] -# please adapt to package name my-microservice = "my_microservice.__main__:run" [tool.setuptools.packages.find] -where = ["src"] +where = [ + "src", +] diff --git a/scripts/update_pyproject.py b/scripts/update_pyproject.py new file mode 100755 index 00000000..82aaf60f --- /dev/null +++ b/scripts/update_pyproject.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 + +# Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln +# for the German Human Genome-Phenome Archive (GHGA) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""A script to update the pyproject.toml.""" + +import sys +from pathlib import Path + +import tomli +import tomli_w + +from script_utils import cli + +REPO_ROOT_DIR = Path(__file__).parent.parent.resolve() +PYPROJECT_GENERATION_DIR = REPO_ROOT_DIR / ".pyproject_generation" + +PYPROJECT_TEMPLATE_PATH = PYPROJECT_GENERATION_DIR / "pyproject_template.toml" +PYPROJECT_CUSTOM_PATH = PYPROJECT_GENERATION_DIR / "pyproject_custom.toml" +PYPROJECT_TOML = REPO_ROOT_DIR / "pyproject.toml" + + +def read_template_pyproject() -> dict[str, object]: + """Read the pyproject_template.toml.""" + with open(PYPROJECT_TEMPLATE_PATH, "rb") as file: + return tomli.load(file) + + +def read_custom_pyproject() -> dict[str, object]: + """Read the pyproject_custom.toml.""" + with open(PYPROJECT_CUSTOM_PATH, "rb") as file: + return tomli.load(file) + + +def read_current_pyproject() -> dict[str, object]: + """Read the current pyproject.toml.""" + with open(PYPROJECT_TOML, "rb") as file: + return tomli.load(file) + + +def write_pyproject(pyproject: dict[str, object]) -> None: + """Write the given pyproject dict into the pyproject.toml.""" + with open(PYPROJECT_TOML, "wb") as file: + tomli_w.dump(pyproject, file) + + +def merge_pyprojects(inputs: list[dict[str, object]]) -> dict[str, object]: + """Compile a pyproject dict from the provided input dicts.""" + pyproject = inputs[0] + + for input in inputs[1:]: + pyproject.update(input) + + return pyproject + + +def main(*, check: bool = False): + """Update the pyproject.toml or checks for updates if the check flag is specified.""" + template_pyproject = read_template_pyproject() + custom_pyproject = read_custom_pyproject() + merged_pyproject = merge_pyprojects([template_pyproject, custom_pyproject]) + + if check: + current_pyproject = read_current_pyproject() + + if current_pyproject != merged_pyproject: + cli.echo_failure("The pyproject.toml is not up to date.") + sys.exit(1) + + cli.echo_success("The pyproject.toml is up to date.") + return + + write_pyproject(merged_pyproject) + cli.echo_success("Successfully updated the pyproject.toml.") + + +if __name__ == "__main__": + cli.run(main) From 641375044e8af91be5ef1ff07dff2495aa39ddc1 Mon Sep 17 00:00:00 2001 From: KerstenBreuer Date: Mon, 18 Dec 2023 17:04:31 +0000 Subject: [PATCH 9/9] moved config for ruff, mypy, pytest, and coverage into pyproject toml --- .coveragerc | 5 - .mypy.ini | 12 -- .pyproject_generation/pyproject_template.toml | 82 ++++++++++++++ .ruff.toml | 64 ----------- pyproject.toml | 106 ++++++++++++++++++ pytest.ini | 5 - 6 files changed, 188 insertions(+), 86 deletions(-) delete mode 100644 .coveragerc delete mode 100644 .mypy.ini delete mode 100644 .ruff.toml delete mode 100644 pytest.ini diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index b172375f..00000000 --- a/.coveragerc +++ /dev/null @@ -1,5 +0,0 @@ -[paths] -source = - src - /workspace/src - **/lib/python*/site-packages diff --git a/.mypy.ini b/.mypy.ini deleted file mode 100644 index 10e9574c..00000000 --- a/.mypy.ini +++ /dev/null @@ -1,12 +0,0 @@ -# Global options: - -[mypy] -disable_error_code = import -show_error_codes = True -exclude = (?x)( - build/lib/ - ) -warn_redundant_casts = True -warn_unused_ignores = True -check_untyped_defs = True -no_site_packages = False diff --git a/.pyproject_generation/pyproject_template.toml b/.pyproject_generation/pyproject_template.toml index 4a62c847..c6bb5fc0 100644 --- a/.pyproject_generation/pyproject_template.toml +++ b/.pyproject_generation/pyproject_template.toml @@ -23,3 +23,85 @@ classifiers = [ [tool.setuptools.packages.find] where = ["src"] + +[tool.ruff] +exclude = [ + ".git", + ".devcontainer", + "__pycache__", + "build", + "dist", +] +ignore = [ + "E", # pycodestyle errors + "W", # pycodestyle warnings - pycodestyle covered by black + "PLW", # pylint warnings + "RUF001", # ambiguous unicode character strings + "RUF010", # explicit conversion to string or repr: !s or !r + "RUF012", # mutable class variables need typing.ClassVar annotation + "N818", # Errors need to have Error suffix + "B008", # function call in arg defaults, + "PLR2004", # magic numbers should be constants + "D205", # blank-line-after-summary + "D400", # first doc line ends in period + "D401", # non-imperative-mood + "D107", # missing docstring in __init__ + "D206", # indent-with-spaces (ignored for formatter) + "D300", # triple-single-quotes (ignored for formatter) +] +line-length = 88 +select = [ + "C90", # McCabe Complexity + "F", # pyflakes codes + "I", # isort + "S", # flake8-bandit + "B", # flake8-bugbear + "N", # pep8-naming + "UP", # pyupgrade + "PL", # pylint + "RUF", # ruff + "SIM", # flake8-simplify + "D", # pydocstyle +] +fixable = [ + "UP", # e.g. List -> list + "I", # sort imports + "D", # pydocstyle +] +src = ["src", "tests", "examples", "scripts"] +target-version = "py39" + +[tool.ruff.mccabe] +max-complexity = 10 + +[tool.ruff.per-file-ignores] +"scripts/*" = ["PL", "S", "SIM", "D"] +"tests/*" = ["S", "SIM", "PLR", "B011"] +".devcontainer/*" = ["S", "SIM", "D"] +"examples/*" = ["S", "D"] +"__init__.py" = ["D"] + +[tool.ruff.pydocstyle] +convention = "pep257" + +[tool.mypy] +disable_error_code = "import" +show_error_codes = true +exclude = [ + 'build/lib/', +] +warn_redundant_casts = true +warn_unused_ignores = true +check_untyped_defs = true +no_site_packages = false + +[tool.pytest.ini_options] +minversion = "7.1" +asyncio_mode = "strict" + +[tool.coverage.paths] +source = [ + "src", + "/workspace/src", + "**/lib/python*/site-packages", +] diff --git a/.ruff.toml b/.ruff.toml deleted file mode 100644 index 75a9dbe4..00000000 --- a/.ruff.toml +++ /dev/null @@ -1,64 +0,0 @@ -exclude = [ - ".git", - ".devcontainer", - "__pycache__", - "build", - "dist", -] - -ignore = [ - "E", # pycodestyle errors - "W", # pycodestyle warnings - pycodestyle covered by black - "PLW", # pylint warnings - "RUF001", # ambiguous unicode character strings - "RUF010", # explicit conversion to string or repr: !s or !r - "RUF012", # mutable class variables need typing.ClassVar annotation - "N818", # Errors need to have Error suffix - "B008", # function call in arg defaults, - "PLR2004", # magic numbers should be constants - "D205", # blank-line-after-summary - "D400", # first doc line ends in period - "D401", # non-imperative-mood - "D107", # missing docstring in __init__ - "D206", # indent-with-spaces (ignored for formatter) - "D300", # triple-single-quotes (ignored for formatter) -] - -line-length = 88 - -select = [ - "C90", # McCabe Complexity - "F", # pyflakes codes - "I", # isort - "S", # flake8-bandit - "B", # flake8-bugbear - "N", # pep8-naming - "UP", # pyupgrade - "PL", # pylint - "RUF", # ruff - "SIM", # flake8-simplify - "D", # pydocstyle -] - -fixable = [ - "UP", # e.g. List -> list - "I", # sort imports - "D", # pydocstyle -] - -src = ["src", "tests", "examples", "scripts"] - -target-version = "py39" - -[mccabe] -max-complexity = 10 - -[per-file-ignores] -"scripts/*" = ["PL", "S", "SIM", "D"] -"tests/*" = ["S", "SIM", "PLR", "B011"] -".devcontainer/*" = ["S", "SIM", "D"] -"examples/*" = ["S", "D"] -"__init__.py" = ["D"] - -[pydocstyle] -convention = "pep257" diff --git a/pyproject.toml b/pyproject.toml index e2e3f337..c832b51b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,3 +25,109 @@ my-microservice = "my_microservice.__main__:run" where = [ "src", ] + +[tool.ruff] +exclude = [ + ".git", + ".devcontainer", + "__pycache__", + "build", + "dist", +] +ignore = [ + "E", + "W", + "PLW", + "RUF001", + "RUF010", + "RUF012", + "N818", + "B008", + "PLR2004", + "D205", + "D400", + "D401", + "D107", + "D206", + "D300", +] +line-length = 88 +select = [ + "C90", + "F", + "I", + "S", + "B", + "N", + "UP", + "PL", + "RUF", + "SIM", + "D", +] +fixable = [ + "UP", + "I", + "D", +] +src = [ + "src", + "tests", + "examples", + "scripts", +] +target-version = "py39" + +[tool.ruff.mccabe] +max-complexity = 10 + +[tool.ruff.per-file-ignores] +"scripts/*" = [ + "PL", + "S", + "SIM", + "D", +] +"tests/*" = [ + "S", + "SIM", + "PLR", + "B011", +] +".devcontainer/*" = [ + "S", + "SIM", + "D", +] +"examples/*" = [ + "S", + "D", +] +"__init__.py" = [ + "D", +] + +[tool.ruff.pydocstyle] +convention = "pep257" + +[tool.mypy] +disable_error_code = "import" +show_error_codes = true +exclude = [ + "build/lib/", +] +warn_redundant_casts = true +warn_unused_ignores = true +check_untyped_defs = true +no_site_packages = false + +[tool.pytest.ini_options] +minversion = "7.1" +asyncio_mode = "strict" + +[tool.coverage.paths] +source = [ + "src", + "/workspace/src", + "**/lib/python*/site-packages", +] diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index ed67994e..00000000 --- a/pytest.ini +++ /dev/null @@ -1,5 +0,0 @@ -# pytest.ini - -[pytest] -minversion = 7.1 -asyncio_mode = strict