From 19c60489734c8376c12a997741aa463fda566790 Mon Sep 17 00:00:00 2001 From: Kersten Henrik Breuer Date: Mon, 18 Dec 2023 17:35:52 +0100 Subject: [PATCH 1/3] tidy up root dir (GSI-540) (#186) - move template file lists in dedicated directory - move lock files and .in files into dedicated directory - move files related to readme generation into dedicated directory --- .github/workflows/check_config_docs.yaml | 2 +- .github/workflows/check_openapi_spec.yaml | 2 +- .github/workflows/check_readme.yaml | 2 +- .github/workflows/static_code_analysis.yaml | 2 +- .github/workflows/tests.yaml | 2 +- .../README.md | 22 ++++---- .../description.md | 0 .design.md => .readme_generation/design.md | 0 .../readme_template.md | 0 .template/README.md | 47 ++++++++++++++++ .../deprecated_files.txt | 14 +++++ .../deprecated_files_ignore.txt | 0 .../mandatory_files.txt | 12 ++-- .../mandatory_files_ignore.txt | 0 .static_files => .template/static_files.txt | 12 ++-- .../static_files_ignore.txt | 0 lock/README.md | 56 +++++++++++++++++++ .../requirements-dev-template.in | 0 .../requirements-dev.in | 4 +- .../requirements-dev.txt | 0 requirements.txt => lock/requirements.txt | 0 scripts/license_checker.py | 12 ++-- scripts/update_hook_revs.py | 2 +- scripts/update_lock.py | 7 ++- scripts/update_readme.py | 7 ++- scripts/update_template_files.py | 32 +++++++---- 26 files changed, 187 insertions(+), 50 deletions(-) rename readme_generation.md => .readme_generation/README.md (66%) rename .description.md => .readme_generation/description.md (100%) rename .design.md => .readme_generation/design.md (100%) rename .readme_template.md => .readme_generation/readme_template.md (100%) create mode 100644 .template/README.md rename .deprecated_files => .template/deprecated_files.txt (68%) rename .deprecated_files_ignore => .template/deprecated_files_ignore.txt (100%) rename .mandatory_files => .template/mandatory_files.txt (79%) rename .mandatory_files_ignore => .template/mandatory_files_ignore.txt (100%) rename .static_files => .template/static_files.txt (90%) rename .static_files_ignore => .template/static_files_ignore.txt (100%) create mode 100644 lock/README.md rename requirements-dev-common.in => lock/requirements-dev-template.in (100%) rename requirements-dev.in => lock/requirements-dev.in (62%) rename requirements-dev.txt => lock/requirements-dev.txt (100%) rename requirements.txt => lock/requirements.txt (100%) diff --git a/.github/workflows/check_config_docs.yaml b/.github/workflows/check_config_docs.yaml index f863f80b..bb88b574 100644 --- a/.github/workflows/check_config_docs.yaml +++ b/.github/workflows/check_config_docs.yaml @@ -9,7 +9,7 @@ jobs: - uses: actions/checkout@v3 - id: common - uses: ghga-de/gh-action-common@v3 + uses: ghga-de/gh-action-common@v4 - name: Check config docs run: | diff --git a/.github/workflows/check_openapi_spec.yaml b/.github/workflows/check_openapi_spec.yaml index 5fad821c..f5db7665 100644 --- a/.github/workflows/check_openapi_spec.yaml +++ b/.github/workflows/check_openapi_spec.yaml @@ -10,7 +10,7 @@ jobs: - uses: actions/checkout@v3 - id: common - uses: ghga-de/gh-action-common@v3 + uses: ghga-de/gh-action-common@v4 - name: Check if openapi.yaml is up to date run: | diff --git a/.github/workflows/check_readme.yaml b/.github/workflows/check_readme.yaml index 9f400d23..7414554d 100644 --- a/.github/workflows/check_readme.yaml +++ b/.github/workflows/check_readme.yaml @@ -9,7 +9,7 @@ jobs: - uses: actions/checkout@v3 - id: common - uses: ghga-de/gh-action-common@v3 + uses: ghga-de/gh-action-common@v4 - name: Check readme run: | diff --git a/.github/workflows/static_code_analysis.yaml b/.github/workflows/static_code_analysis.yaml index 39b9bad4..86a014df 100644 --- a/.github/workflows/static_code_analysis.yaml +++ b/.github/workflows/static_code_analysis.yaml @@ -10,7 +10,7 @@ jobs: - uses: actions/checkout@v3 - id: common - uses: ghga-de/gh-action-common@v3 + uses: ghga-de/gh-action-common@v4 - uses: pre-commit/action@v3.0.0 env: diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 05f474ca..dacc9f50 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -11,7 +11,7 @@ jobs: - uses: actions/checkout@v3 - id: common - uses: ghga-de/gh-action-common@v3 + uses: ghga-de/gh-action-common@v4 - id: pytest run: | diff --git a/readme_generation.md b/.readme_generation/README.md similarity index 66% rename from readme_generation.md rename to .readme_generation/README.md index 84cd618b..ed2a2a79 100644 --- a/readme_generation.md +++ b/.readme_generation/README.md @@ -18,30 +18,30 @@ # Readme Generation -The README file is generated by collecting information from different sources as +The Repository README is generated by collecting information from different sources as outlined in the following. - name: The full name of the package is derived from the remote origin Git repository. - title: A title case representation of the name. - shortname: An abbreviation of the full name. This is derived from the name mentioned - in the [`./pyproject.toml`](./pyproject.toml). + in the [`../pyproject.toml`](../pyproject.toml). - summary: A short 1-2 sentence summary derived from the description in the - [`./pyproject.toml`](./pyproject.toml). + [`../pyproject.toml`](../pyproject.toml). - version: The package version derived from the version specified in the - [`./pyproject.toml`](./pyproject.toml). + [`../pyproject.toml`](../pyproject.toml). - description: A markdown-formatted description of the features and use cases of this - service or package. Obtained from the [`./.description.md`](./.description.md). + service or package. Obtained from the [`./description.md`](./description.md). - design_description: A markdown-formatted description of the overall architecture and - design of the package. Obtained from the [`./.design.md`](./.design.md). + design of the package. Obtained from the [`./design.md`](./design.md). - config_description: A markdown-formatted description of all config parameters. - This is autogenerated from the [`./config_schema.json`](./config_schema.json). + This is autogenerated from the [`../config_schema.json`](../config_schema.json). - openapi_doc: A markdown-formatted description of the HTTP API. This is autogenerated - and links to the [`./openapi.yaml`](./openapi.yaml). If the openapi.yaml is not + and links to the [`../openapi.yaml`](../openapi.yaml). If the openapi.yaml is not this documentation is empty. -The [`./.readme_template.md`](./.readme_template.md) serves as a template where the +The [`./readme_template.md`](./readme_template.md) serves as a template where the above variable can be filled in using Pythons `string.Template` utility from the standard library. -The [`./scripts/update_readme.py`] script can be used to collect all information and -fill it into the template to generate the README file. +The [`../scripts/update_readme.py`](../scripts/update_readme.py) script can be used to +collect all information and fill it into the template to generate the README file. diff --git a/.description.md b/.readme_generation/description.md similarity index 100% rename from .description.md rename to .readme_generation/description.md diff --git a/.design.md b/.readme_generation/design.md similarity index 100% rename from .design.md rename to .readme_generation/design.md diff --git a/.readme_template.md b/.readme_generation/readme_template.md similarity index 100% rename from .readme_template.md rename to .readme_generation/readme_template.md diff --git a/.template/README.md b/.template/README.md new file mode 100644 index 00000000..8a5e7cd6 --- /dev/null +++ b/.template/README.md @@ -0,0 +1,47 @@ + + +# Template File Lists + +This directory contains multiple text files that are listing paths to other files +of this repository. The listed files are affected in different ways by template updates +as explained in the following. + +## `static_files.txt` +The files listed here are synced with their counterparts in the template. They should +never be modified manually. + +## `static_files_ignore.txt` +To opt out of template updates just for individual files declared as static +(e.g. because you would like manually modify them), you may add them to this list. + +## `mandatory_files.txt` +The contents of the files listed here are not synced with the template, however, upon +every template update it is checked that the files exist. You should modify them +manually to the needs of your repository. + +## `mandatory_files_ignore.txt` +To opt out of existence checks for individual files declared as mandatory, you may add +them to this list. + +## `deprecated_files.txt` +Files listed here must not exist in your repository and are automatically deleted upon +a template update. + +## `deprecated_files_ignore.txt` +If you would like to keep files declared as deprecated, you may add them to this list. diff --git a/.deprecated_files b/.template/deprecated_files.txt similarity index 68% rename from .deprecated_files rename to .template/deprecated_files.txt index 1fc912566..79d4c279 100644 --- a/.deprecated_files +++ b/.template/deprecated_files.txt @@ -19,7 +19,21 @@ docs setup.py setup.cfg +requirements-dev-common.in +requirements-dev.in +requirements-dev.txt +requirements.txt .pylintrc .flake8 .editorconfig +.deprecated_files +.deprecated_files_ignore +.mandatory_files +.mandatory_files_ignore +.static_files +.static_files_ignore +.description.md +.design.md +.readme_template.md +.readme_generation.md diff --git a/.deprecated_files_ignore b/.template/deprecated_files_ignore.txt similarity index 100% rename from .deprecated_files_ignore rename to .template/deprecated_files_ignore.txt diff --git a/.mandatory_files b/.template/mandatory_files.txt similarity index 79% rename from .mandatory_files rename to .template/mandatory_files.txt index ba9f6c31..218a58ab 100644 --- a/.mandatory_files +++ b/.template/mandatory_files.txt @@ -13,16 +13,18 @@ tests/fixtures/__init__.py scripts/script_utils/fastapi_app_location.py +.readme_generation/description.md +.readme_generation/design.md + +lock/requirements-dev.in +lock/requirements-dev.txt +lock/requirements.txt + Dockerfile config_schema.json example_config.yaml LICENSE pyproject.toml README.md -requirements-dev.in -requirements-dev.txt -requirements.txt -.description.md -.design.md .pre-commit-config.yaml diff --git a/.mandatory_files_ignore b/.template/mandatory_files_ignore.txt similarity index 100% rename from .mandatory_files_ignore rename to .template/mandatory_files_ignore.txt diff --git a/.static_files b/.template/static_files.txt similarity index 90% rename from .static_files rename to .template/static_files.txt index 1f129ea4..65f9e0c5 100644 --- a/.static_files +++ b/.template/static_files.txt @@ -40,6 +40,14 @@ scripts/README.md example_data/README.md +.template/README.md + +.readme_generation/.readme_template.md +.readme_generation/README.md + +lock/requirements-dev-template.in +lock/README.md + .coveragerc .gitattributes .gitignore @@ -48,7 +56,3 @@ example_data/README.md pytest.ini LICENSE -requirements-dev-common.in - -.readme_template.md -readme_generation.md diff --git a/.static_files_ignore b/.template/static_files_ignore.txt similarity index 100% rename from .static_files_ignore rename to .template/static_files_ignore.txt diff --git a/lock/README.md b/lock/README.md new file mode 100644 index 00000000..fc255b55 --- /dev/null +++ b/lock/README.md @@ -0,0 +1,56 @@ + + +# Lock Files + +This directory contains two lock files locking the dependencies of this microservice: + +The [`./requirements.txt`](./requirements.txt) contains production dependencies. + +The [`./requirements-dev.txt`](./requirements-dev.txt) additionally contains development +dependencies. + +## Sources + +For generating the production lock file, only the dependencies specified in the +[`../pyproject.toml`](../pyproject.toml) are considered as input. + +For generating the development lock file, additionally, the +[`./requirements-dev-template.in`](./requirements-dev-template.in) as well as +the [`./requirements-dev.in`](./requirements-dev.in) are considered. + +The `./requirements-dev-template.in` is automatically updated from the template +repository and should not be manually modified. + +If you require additional dev dependencies not part of the +`./requirements-dev-template.in`, you can add them to the +`./requirements-dev.in`. + +## Update and Upgrade + +The lock files can be updated running the +[`../scripts/update_lock.py`](../scripts/update_lock.py) script. This will keep the +dependency versions in the lockfile unchanged unless they are in conflict with the +the input sources. In that case, the affected dependencies are updated to the latest +versions compatible with the input. + +If you would like to upgrade all dependencies in the lock file to the latest versions +compatible with the input, you can run `../scripts/update_lock.py --upgrade`. + +If you just want to check if the script would do update, you can run +`../scripts/update_lock.py --check`. diff --git a/requirements-dev-common.in b/lock/requirements-dev-template.in similarity index 100% rename from requirements-dev-common.in rename to lock/requirements-dev-template.in diff --git a/requirements-dev.in b/lock/requirements-dev.in similarity index 62% rename from requirements-dev.in rename to lock/requirements-dev.in index dc25ec87..4d48a8fd 100644 --- a/requirements-dev.in +++ b/lock/requirements-dev.in @@ -1,7 +1,7 @@ # requirements for development and testing this service -# common requirements for development and testing --r requirements-dev-common.in +# template requirements for development and testing +-r requirements-dev-template.in # additional requirements can be listed here testcontainers[kafka,mongo]>=3.4.1 diff --git a/requirements-dev.txt b/lock/requirements-dev.txt similarity index 100% rename from requirements-dev.txt rename to lock/requirements-dev.txt diff --git a/requirements.txt b/lock/requirements.txt similarity index 100% rename from requirements.txt rename to lock/requirements.txt diff --git a/scripts/license_checker.py b/scripts/license_checker.py index 5d8be069..b6a718df 100755 --- a/scripts/license_checker.py +++ b/scripts/license_checker.py @@ -64,12 +64,12 @@ ".mypy.ini", ".pytest_cache", ".editorconfig", - ".static_files", - ".static_files_ignore", - ".mandatory_files", - ".mandatory_files_ignore", - ".deprecated_files", - ".deprecated_files_ignore", + ".template/.static_files.txt", + ".template/.static_files_ignore.txt", + ".template/.mandatory_files.txt", + ".template/.mandatory_files_ignore.txt", + ".template/.deprecated_files.txt", + ".template/.deprecated_files_ignore.txt", ] # exclude file by file ending from license header check: diff --git a/scripts/update_hook_revs.py b/scripts/update_hook_revs.py index 3d653f85..6522cbe6 100755 --- a/scripts/update_hook_revs.py +++ b/scripts/update_hook_revs.py @@ -27,7 +27,7 @@ REPO_ROOT_DIR = Path(__file__).parent.parent.resolve() PRE_COMMIT_CFG_PATH = REPO_ROOT_DIR / ".pre-commit-config.yaml" -LOCK_FILE_PATH = REPO_ROOT_DIR / "requirements-dev.txt" +LOCK_FILE_PATH = REPO_ROOT_DIR / "lock" / "requirements-dev.txt" def make_dependency_dict(requirements: list[Requirement]) -> dict[str, str]: diff --git a/scripts/update_lock.py b/scripts/update_lock.py index 825aade0..98516b88 100755 --- a/scripts/update_lock.py +++ b/scripts/update_lock.py @@ -31,11 +31,12 @@ from script_utils import cli, deps REPO_ROOT_DIR = Path(__file__).parent.parent.resolve() +LOCK_DIR = REPO_ROOT_DIR / "lock" PYPROJECT_TOML_PATH = REPO_ROOT_DIR / "pyproject.toml" -DEV_DEPS_PATH = REPO_ROOT_DIR / "requirements-dev.in" -OUTPUT_LOCK_PATH = REPO_ROOT_DIR / "requirements.txt" -OUTPUT_DEV_LOCK_PATH = REPO_ROOT_DIR / "requirements-dev.txt" +DEV_DEPS_PATH = LOCK_DIR / "requirements-dev.in" +OUTPUT_LOCK_PATH = LOCK_DIR / "requirements.txt" +OUTPUT_DEV_LOCK_PATH = LOCK_DIR / "requirements-dev.txt" def fix_temp_dir_comments(file_path: Path): diff --git a/scripts/update_readme.py b/scripts/update_readme.py index 244d0ba3..c0dd4e32 100755 --- a/scripts/update_readme.py +++ b/scripts/update_readme.py @@ -32,9 +32,10 @@ ROOT_DIR = Path(__file__).parent.parent.resolve() PYPROJECT_TOML_PATH = ROOT_DIR / "pyproject.toml" -DESCRIPTION_PATH = ROOT_DIR / ".description.md" -DESIGN_PATH = ROOT_DIR / ".design.md" -README_TEMPLATE_PATH = ROOT_DIR / ".readme_template.md" +README_GENERATION_DIR = ROOT_DIR / ".readme_generation" +DESCRIPTION_PATH = README_GENERATION_DIR / "description.md" +DESIGN_PATH = README_GENERATION_DIR / "design.md" +README_TEMPLATE_PATH = README_GENERATION_DIR / "readme_template.md" CONFIG_SCHEMA_PATH = ROOT_DIR / "config_schema.json" OPENAPI_YAML_REL_PATH = "./openapi.yaml" README_PATH = ROOT_DIR / "README.md" diff --git a/scripts/update_template_files.py b/scripts/update_template_files.py index d3313fd5..fe29d575 100755 --- a/scripts/update_template_files.py +++ b/scripts/update_template_files.py @@ -40,12 +40,20 @@ def run(main_fn): main_fn(check="--check" in sys.argv[1:]) -REPO_ROOT_DIR = Path(__file__).parent.parent.resolve() +REPO_ROOT_DIR = Path(__file__).parent.parent.absolute() + +FILE_LIST_DIR_NAME = ".template" +DEPRECATED_FILES = "deprecated_files" +MANDATORY_FILES = "mandatory_files" +STATIC_FILES = "static_files" -DEPRECATED_FILES = ".deprecated_files" -MANDATORY_FILES = ".mandatory_files" -STATIC_FILES = ".static_files" IGNORE_SUFFIX = "_ignore" + +TEMPLATE_LIST_REL_PATHS = [ + f"{FILE_LIST_DIR_NAME}/{list_name}.txt" + for list_name in [STATIC_FILES, MANDATORY_FILES, DEPRECATED_FILES] +] + RAW_TEMPLATE_URL = ( "https://raw.githubusercontent.com/ghga-de/microservice-repository-template/main/" ) @@ -55,9 +63,14 @@ class ValidationError(RuntimeError): """Raised when files need to be updated.""" +def get_file_list_path(list_name: str, relative: bool = False) -> Path: + """Get the path to the file list of the given name.""" + return Path(REPO_ROOT_DIR / FILE_LIST_DIR_NAME / f"{list_name}.txt") + + def get_file_list(list_name: str) -> list[str]: """Return a list of all file names specified in a given list file.""" - list_path = REPO_ROOT_DIR / list_name + list_path = get_file_list_path(list_name) with open(list_path, encoding="utf8") as list_file: file_list = [ clean_line @@ -212,14 +225,13 @@ def remove_files(files: list[str], check: bool = False) -> bool: def main(check: bool = False): """Update the static files in the service template.""" updated = False - if not check: - update_files([STATIC_FILES], diff=True, check=False) + + print("Template lists...") + if update_files(TEMPLATE_LIST_REL_PATHS, diff=True, check=False): + updated = True print("Static files...") files_to_update = get_file_list(STATIC_FILES) - if check: - files_to_update.append(STATIC_FILES) - files_to_update.extend((MANDATORY_FILES, DEPRECATED_FILES)) if update_files(files_to_update, diff=True, check=check): updated = True From d27da9dd0b39b55b3b6c787bd0bdfe6782047366 Mon Sep 17 00:00:00 2001 From: Kersten Henrik Breuer Date: Wed, 20 Dec 2023 09:12:11 +0100 Subject: [PATCH 2/3] move tool config to pyproject.toml (GSI-544) (#187) Moved config for: - ruff - mypy - pytest - coverage --- .coveragerc | 5 - .github/workflows/check_pyproject.yaml | 16 +++ .mypy.ini | 12 -- .pyproject_generation/pyproject_custom.toml | 19 +++ .../pyproject_template.toml | 61 ++++++-- .template/mandatory_files.txt | 2 + .template/static_files.txt | 5 + pyproject.toml | 136 +++++++++++++++--- pytest.ini | 5 - scripts/update_pyproject.py | 92 ++++++++++++ 10 files changed, 299 insertions(+), 54 deletions(-) delete mode 100644 .coveragerc create mode 100644 .github/workflows/check_pyproject.yaml delete mode 100644 .mypy.ini create mode 100644 .pyproject_generation/pyproject_custom.toml rename .ruff.toml => .pyproject_generation/pyproject_template.toml (55%) delete mode 100644 pytest.ini create mode 100755 scripts/update_pyproject.py diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index b172375f..00000000 --- a/.coveragerc +++ /dev/null @@ -1,5 +0,0 @@ -[paths] -source = - src - /workspace/src - **/lib/python*/site-packages diff --git a/.github/workflows/check_pyproject.yaml b/.github/workflows/check_pyproject.yaml new file mode 100644 index 00000000..d929c799 --- /dev/null +++ b/.github/workflows/check_pyproject.yaml @@ -0,0 +1,16 @@ +name: Check if the config schema and the example are up to date. + +on: push + +jobs: + static-code-analysis: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - id: common + uses: ghga-de/gh-action-common@v4 + + - name: Check pyproject.toml + run: | + ./scripts/update_pyproject.py --check diff --git a/.mypy.ini b/.mypy.ini deleted file mode 100644 index 10e9574c..00000000 --- a/.mypy.ini +++ /dev/null @@ -1,12 +0,0 @@ -# Global options: - -[mypy] -disable_error_code = import -show_error_codes = True -exclude = (?x)( - build/lib/ - ) -warn_redundant_casts = True -warn_unused_ignores = True -check_untyped_defs = True -no_site_packages = False diff --git a/.pyproject_generation/pyproject_custom.toml b/.pyproject_generation/pyproject_custom.toml new file mode 100644 index 00000000..5f8c2276 --- /dev/null +++ b/.pyproject_generation/pyproject_custom.toml @@ -0,0 +1,19 @@ +[project] +# please adapt to package name +name = "my_microservice" +version = "0.1.0" +description = "My-Microservice - a short description" +dependencies = [ + "typer >= 0.9.0", + "ghga-service-commons[api] >= 1.2.0", + "ghga-event-schemas >= 1.0.0", + "hexkit[akafka,s3,mongodb] >= 1.1.0" +] + +[project.urls] +# please adapt to package name +Repository = "https://github.com/ghga-de/my-microservice" + +[project.scripts] +# please adapt to package name +my-microservice = "my_microservice.__main__:run" diff --git a/.ruff.toml b/.pyproject_generation/pyproject_template.toml similarity index 55% rename from .ruff.toml rename to .pyproject_generation/pyproject_template.toml index 75a9dbe4..c6bb5fc0 100644 --- a/.ruff.toml +++ b/.pyproject_generation/pyproject_template.toml @@ -1,3 +1,30 @@ +[build-system] +requires = ["setuptools>=67.7.2"] +build-backend = "setuptools.build_meta" + +[project] +readme = "README.md" +authors = [ + { name = "German Human Genome Phenome Archive (GHGA)", email = "contact@ghga.de" }, +] +requires-python = ">=3.9" +license = { text = "Apache 2.0" } +classifiers = [ + "Development Status :: 1 - Planning", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "License :: OSI Approved :: Apache Software License", + "Topic :: Internet :: WWW/HTTP :: HTTP Servers", + "Topic :: Software Development :: Libraries", + "Intended Audience :: Developers", +] + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.ruff] exclude = [ ".git", ".devcontainer", @@ -5,7 +32,6 @@ exclude = [ "build", "dist", ] - ignore = [ "E", # pycodestyle errors "W", # pycodestyle warnings - pycodestyle covered by black @@ -23,9 +49,7 @@ ignore = [ "D206", # indent-with-spaces (ignored for formatter) "D300", # triple-single-quotes (ignored for formatter) ] - line-length = 88 - select = [ "C90", # McCabe Complexity "F", # pyflakes codes @@ -39,26 +63,45 @@ select = [ "SIM", # flake8-simplify "D", # pydocstyle ] - fixable = [ "UP", # e.g. List -> list "I", # sort imports "D", # pydocstyle ] - src = ["src", "tests", "examples", "scripts"] - target-version = "py39" -[mccabe] +[tool.ruff.mccabe] max-complexity = 10 -[per-file-ignores] +[tool.ruff.per-file-ignores] "scripts/*" = ["PL", "S", "SIM", "D"] "tests/*" = ["S", "SIM", "PLR", "B011"] ".devcontainer/*" = ["S", "SIM", "D"] "examples/*" = ["S", "D"] "__init__.py" = ["D"] -[pydocstyle] +[tool.ruff.pydocstyle] convention = "pep257" + +[tool.mypy] +disable_error_code = "import" +show_error_codes = true +exclude = [ + 'build/lib/', +] +warn_redundant_casts = true +warn_unused_ignores = true +check_untyped_defs = true +no_site_packages = false + +[tool.pytest.ini_options] +minversion = "7.1" +asyncio_mode = "strict" + +[tool.coverage.paths] +source = [ + "src", + "/workspace/src", + "**/lib/python*/site-packages", +] diff --git a/.template/mandatory_files.txt b/.template/mandatory_files.txt index 218a58ab..660a15ed 100644 --- a/.template/mandatory_files.txt +++ b/.template/mandatory_files.txt @@ -16,6 +16,8 @@ scripts/script_utils/fastapi_app_location.py .readme_generation/description.md .readme_generation/design.md +.pyproject_generation/pyproject_custom.toml + lock/requirements-dev.in lock/requirements-dev.txt lock/requirements.txt diff --git a/.template/static_files.txt b/.template/static_files.txt index 65f9e0c5..ade3194f 100644 --- a/.template/static_files.txt +++ b/.template/static_files.txt @@ -26,12 +26,14 @@ scripts/update_openapi_docs.py scripts/update_readme.py scripts/update_lock.py scripts/update_hook_revs.py +scripts/update_pyproject.py scripts/list_outdated_dependencies.py scripts/README.md .github/workflows/check_config_docs.yaml .github/workflows/check_openapi_spec.yaml .github/workflows/check_readme.yaml +./github/workflows/check_pyproject.yaml .github/workflows/check_template_files.yaml .github/workflows/ci_release.yaml .github/workflows/ci_workflow_dispatch.yaml @@ -45,6 +47,9 @@ example_data/README.md .readme_generation/.readme_template.md .readme_generation/README.md +.pyproject_generation/pyproject_template.toml +.pyproject_generation/README.md + lock/requirements-dev-template.in lock/README.md diff --git a/pyproject.toml b/pyproject.toml index 903da80b..c832b51b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,43 +1,133 @@ [build-system] -requires = ["setuptools>=67.7.2"] +requires = [ + "setuptools>=67.7.2", +] build-backend = "setuptools.build_meta" [project] -# please adapt to package name name = "my_microservice" version = "0.1.0" description = "My-Microservice - a short description" -readme = "README.md" -authors = [ - { name = "German Human Genome Phenome Archive (GHGA)", email = "contact@ghga.de" }, -] -requires-python = ">=3.9" -license = { text = "Apache 2.0" } -classifiers = [ - "Development Status :: 1 - Planning", - "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "License :: OSI Approved :: Apache Software License", - "Topic :: Internet :: WWW/HTTP :: HTTP Servers", - "Topic :: Software Development :: Libraries", - "Intended Audience :: Developers", -] dependencies = [ "typer >= 0.9.0", "ghga-service-commons[api] >= 1.2.0", "ghga-event-schemas >= 1.0.0", - "hexkit[akafka,s3,mongodb] >= 1.1.0" + "hexkit[akafka,s3,mongodb] >= 1.1.0", ] [project.urls] -# please adapt to package name Repository = "https://github.com/ghga-de/my-microservice" [project.scripts] -# please adapt to package name my-microservice = "my_microservice.__main__:run" [tool.setuptools.packages.find] -where = ["src"] +where = [ + "src", +] + +[tool.ruff] +exclude = [ + ".git", + ".devcontainer", + "__pycache__", + "build", + "dist", +] +ignore = [ + "E", + "W", + "PLW", + "RUF001", + "RUF010", + "RUF012", + "N818", + "B008", + "PLR2004", + "D205", + "D400", + "D401", + "D107", + "D206", + "D300", +] +line-length = 88 +select = [ + "C90", + "F", + "I", + "S", + "B", + "N", + "UP", + "PL", + "RUF", + "SIM", + "D", +] +fixable = [ + "UP", + "I", + "D", +] +src = [ + "src", + "tests", + "examples", + "scripts", +] +target-version = "py39" + +[tool.ruff.mccabe] +max-complexity = 10 + +[tool.ruff.per-file-ignores] +"scripts/*" = [ + "PL", + "S", + "SIM", + "D", +] +"tests/*" = [ + "S", + "SIM", + "PLR", + "B011", +] +".devcontainer/*" = [ + "S", + "SIM", + "D", +] +"examples/*" = [ + "S", + "D", +] +"__init__.py" = [ + "D", +] + +[tool.ruff.pydocstyle] +convention = "pep257" + +[tool.mypy] +disable_error_code = "import" +show_error_codes = true +exclude = [ + "build/lib/", +] +warn_redundant_casts = true +warn_unused_ignores = true +check_untyped_defs = true +no_site_packages = false + +[tool.pytest.ini_options] +minversion = "7.1" +asyncio_mode = "strict" + +[tool.coverage.paths] +source = [ + "src", + "/workspace/src", + "**/lib/python*/site-packages", +] diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index ed67994e..00000000 --- a/pytest.ini +++ /dev/null @@ -1,5 +0,0 @@ -# pytest.ini - -[pytest] -minversion = 7.1 -asyncio_mode = strict diff --git a/scripts/update_pyproject.py b/scripts/update_pyproject.py new file mode 100755 index 00000000..82aaf60f --- /dev/null +++ b/scripts/update_pyproject.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 + +# Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln +# for the German Human Genome-Phenome Archive (GHGA) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""A script to update the pyproject.toml.""" + +import sys +from pathlib import Path + +import tomli +import tomli_w + +from script_utils import cli + +REPO_ROOT_DIR = Path(__file__).parent.parent.resolve() +PYPROJECT_GENERATION_DIR = REPO_ROOT_DIR / ".pyproject_generation" + +PYPROJECT_TEMPLATE_PATH = PYPROJECT_GENERATION_DIR / "pyproject_template.toml" +PYPROJECT_CUSTOM_PATH = PYPROJECT_GENERATION_DIR / "pyproject_custom.toml" +PYPROJECT_TOML = REPO_ROOT_DIR / "pyproject.toml" + + +def read_template_pyproject() -> dict[str, object]: + """Read the pyproject_template.toml.""" + with open(PYPROJECT_TEMPLATE_PATH, "rb") as file: + return tomli.load(file) + + +def read_custom_pyproject() -> dict[str, object]: + """Read the pyproject_custom.toml.""" + with open(PYPROJECT_CUSTOM_PATH, "rb") as file: + return tomli.load(file) + + +def read_current_pyproject() -> dict[str, object]: + """Read the current pyproject.toml.""" + with open(PYPROJECT_TOML, "rb") as file: + return tomli.load(file) + + +def write_pyproject(pyproject: dict[str, object]) -> None: + """Write the given pyproject dict into the pyproject.toml.""" + with open(PYPROJECT_TOML, "wb") as file: + tomli_w.dump(pyproject, file) + + +def merge_pyprojects(inputs: list[dict[str, object]]) -> dict[str, object]: + """Compile a pyproject dict from the provided input dicts.""" + pyproject = inputs[0] + + for input in inputs[1:]: + pyproject.update(input) + + return pyproject + + +def main(*, check: bool = False): + """Update the pyproject.toml or checks for updates if the check flag is specified.""" + template_pyproject = read_template_pyproject() + custom_pyproject = read_custom_pyproject() + merged_pyproject = merge_pyprojects([template_pyproject, custom_pyproject]) + + if check: + current_pyproject = read_current_pyproject() + + if current_pyproject != merged_pyproject: + cli.echo_failure("The pyproject.toml is not up to date.") + sys.exit(1) + + cli.echo_success("The pyproject.toml is up to date.") + return + + write_pyproject(merged_pyproject) + cli.echo_success("Successfully updated the pyproject.toml.") + + +if __name__ == "__main__": + cli.run(main) From 0e9b5e0262218041c58c04e57fcae512d373966a Mon Sep 17 00:00:00 2001 From: Kersten Henrik Breuer Date: Wed, 20 Dec 2023 16:54:31 +0100 Subject: [PATCH 3/3] add a template description (GSI-545) (#188) Is only present on the template and automatically removed for relying service repos. --- .readme_generation/description.md | 25 +---------- .readme_generation/readme_template.md | 5 +-- .readme_generation/template_overview.md | 27 ++++++++++++ README.md | 57 +++++++++++++------------ scripts/update_readme.py | 22 +++++++++- 5 files changed, 80 insertions(+), 56 deletions(-) create mode 100644 .readme_generation/template_overview.md diff --git a/.readme_generation/description.md b/.readme_generation/description.md index ef8eb2b2..2ad6dc4f 100644 --- a/.readme_generation/description.md +++ b/.readme_generation/description.md @@ -1,26 +1,3 @@ -This repo is a template for creating a new microservice. - -The directories, files, and their structure herein are recommendations -from the GHGA Dev Team. - -### Naming Conventions - -The GitHub repository contains only lowercase letters, numbers, and hyphens "-", -e.g.: `my-microservice` - -The Python package (and thus the source repository) contains underscores "_" -instead of hyphens, e.g.: `my_microservice` -However, an abbreviated version is prefered as package name. - -### Adapt to your service - -This is just a template and needs some adaptation to your specific use case. - -Please search for **"please adapt"** comments. They will indicate all locations that -need modification. Once the adaptations are in place, please remove these comments. - -Finally, follow the instructions to generate the README.md described in -[`./readme_generation.md`](./readme_generation.md). Please also adapt this markdown file -by providing an overview of the feature of the package. +Here you should provide a short summary of the purpose of this microservice. diff --git a/.readme_generation/readme_template.md b/.readme_generation/readme_template.md index 10e3e612..98a00c09 100644 --- a/.readme_generation/readme_template.md +++ b/.readme_generation/readme_template.md @@ -1,6 +1,5 @@ - -[![tests](https://github.com/ghga-de/$name/actions/workflows/tests.yaml/badge.svg)](https://github.com/ghga-de/$name/actions/workflows/tests.yaml) -[![Coverage Status](https://coveralls.io/repos/github/ghga-de/$name/badge.svg?branch=main)](https://coveralls.io/github/ghga-de/$name?branch=main) +[![tests](https://github.com/ghga-de/$repo_name/actions/workflows/tests.yaml/badge.svg)](https://github.com/ghga-de/$repo_name/actions/workflows/tests.yaml) +[![Coverage Status](https://coveralls.io/repos/github/ghga-de/$repo_name/badge.svg?branch=main)](https://coveralls.io/github/ghga-de/$repo_name?branch=main) # $title diff --git a/.readme_generation/template_overview.md b/.readme_generation/template_overview.md new file mode 100644 index 00000000..6e148ef3 --- /dev/null +++ b/.readme_generation/template_overview.md @@ -0,0 +1,27 @@ +# Microservice Repository Template + +This is a template for GitHub repositories containing one Python-based microservice (optimal for a multirepository setup). + +It features: + +- *Continuous Templation* - A continuous update-delivery mechanism for templated repositories +- A [devcontainer](https://containers.dev/)-based fully-configured development environment for vscode +- Tight linting and formatting using [Ruff](https://docs.astral.sh/ruff/) +- Static type checking using [mypy](https://www.mypy-lang.org/) +- Security scanning using [bandit](https://bandit.readthedocs.io/en/latest/) +- A structure for automated tests using [pytest](https://docs.pytest.org/en/7.4.x/) +- Dependency locking using [pip-tools](https://github.com/jazzband/pip-tools) +- Git hooks checking linting and formatting before committing using [pre-commit](https://pre-commit.com/) +- Automatic container-building and publishing to [Docker Hub](https://hub.docker.com/) +- GitHub Actions for automating or checking all of the above + +It is worth emphasizing the first point, this template is not just a one-time kickstart for your project +but repositories created using this template will continue receiving updates as the template evolves. +For further details, please look at the explanation in [.template/README.md](/.template/README.md). + +Please also refer to [.readme_generation/README.md](/.readme_generation/README.md) for details on how +to adapt this readme. + +Here the intro to the template stops and the actual template for the readme of the microservice starts: + +--- diff --git a/README.md b/README.md index 3238efd4..3cc84fe1 100644 --- a/README.md +++ b/README.md @@ -1,54 +1,57 @@ - -[![tests](https://github.com/ghga-de/microservice-repository-template/actions/workflows/tests.yaml/badge.svg)](https://github.com/ghga-de/microservice-repository-template/actions/workflows/tests.yaml) -[![Coverage Status](https://coveralls.io/repos/github/ghga-de/microservice-repository-template/badge.svg?branch=main)](https://coveralls.io/github/ghga-de/microservice-repository-template?branch=main) - # Microservice Repository Template -My-Microservice - a short description +This is a template for GitHub repositories containing one Python-based microservice (optimal for a multirepository setup). -## Description +It features: - +- *Continuous Templation* - A continuous update-delivery mechanism for templated repositories +- A [devcontainer](https://containers.dev/)-based fully-configured development environment for vscode +- Tight linting and formatting using [Ruff](https://docs.astral.sh/ruff/) +- Static type checking using [mypy](https://www.mypy-lang.org/) +- Security scanning using [bandit](https://bandit.readthedocs.io/en/latest/) +- A structure for automated tests using [pytest](https://docs.pytest.org/en/7.4.x/) +- Dependency locking using [pip-tools](https://github.com/jazzband/pip-tools) +- Git hooks checking linting and formatting before committing using [pre-commit](https://pre-commit.com/) +- Automatic container-building and publishing to [Docker Hub](https://hub.docker.com/) +- GitHub Actions for automating or checking all of the above -This repo is a template for creating a new microservice. +It is worth emphasizing the first point, this template is not just a one-time kickstart for your project +but repositories created using this template will continue receiving updates as the template evolves. +For further details, please look at the explanation in [.template/README.md](/.template/README.md). -The directories, files, and their structure herein are recommendations -from the GHGA Dev Team. +Please also refer to [.readme_generation/README.md](/.readme_generation/README.md) for details on how +to adapt this readme. -### Naming Conventions +Here the intro to the template stops and the actual template for the readme of the microservice starts: -The GitHub repository contains only lowercase letters, numbers, and hyphens "-", -e.g.: `my-microservice` +--- +[![tests](https://github.com/ghga-de/microservice-repository-template/actions/workflows/tests.yaml/badge.svg)](https://github.com/ghga-de/microservice-repository-template/actions/workflows/tests.yaml) +[![Coverage Status](https://coveralls.io/repos/github/ghga-de/microservice-repository-template/badge.svg?branch=main)](https://coveralls.io/github/ghga-de/microservice-repository-template?branch=main) -The Python package (and thus the source repository) contains underscores "_" -instead of hyphens, e.g.: `my_microservice` -However, an abbreviated version is prefered as package name. +# My Microservice -### Adapt to your service +My-Microservice - a short description -This is just a template and needs some adaptation to your specific use case. +## Description -Please search for **"please adapt"** comments. They will indicate all locations that -need modification. Once the adaptations are in place, please remove these comments. + -Finally, follow the instructions to generate the README.md described in -[`./readme_generation.md`](./readme_generation.md). Please also adapt this markdown file -by providing an overview of the feature of the package. +Here you should provide a short summary of the purpose of this microservice. ## Installation We recommend using the provided Docker container. -A pre-build version is available at [docker hub](https://hub.docker.com/repository/docker/ghga/microservice-repository-template): +A pre-build version is available at [docker hub](https://hub.docker.com/repository/docker/ghga/my-microservice): ```bash -docker pull ghga/microservice-repository-template:0.1.0 +docker pull ghga/my-microservice:0.1.0 ``` Or you can build the container yourself from the [`./Dockerfile`](./Dockerfile): ```bash # Execute in the repo's root dir: -docker build -t ghga/microservice-repository-template:0.1.0 . +docker build -t ghga/my-microservice:0.1.0 . ``` For production-ready deployment, we recommend using Kubernetes, however, @@ -56,7 +59,7 @@ for simple use cases, you could execute the service using docker on a single server: ```bash # The entrypoint is preconfigured: -docker run -p 8080:8080 ghga/microservice-repository-template:0.1.0 --help +docker run -p 8080:8080 ghga/my-microservice:0.1.0 --help ``` If you prefer not to use containers, you may install the service from source: diff --git a/scripts/update_readme.py b/scripts/update_readme.py index c0dd4e32..b8273b02 100755 --- a/scripts/update_readme.py +++ b/scripts/update_readme.py @@ -33,6 +33,7 @@ ROOT_DIR = Path(__file__).parent.parent.resolve() PYPROJECT_TOML_PATH = ROOT_DIR / "pyproject.toml" README_GENERATION_DIR = ROOT_DIR / ".readme_generation" +TEMPLATE_OVERVIEW_PATH = README_GENERATION_DIR / "template_overview.md" DESCRIPTION_PATH = README_GENERATION_DIR / "description.md" DESIGN_PATH = README_GENERATION_DIR / "design.md" README_TEMPLATE_PATH = README_GENERATION_DIR / "readme_template.md" @@ -59,6 +60,7 @@ class PackageHeader(BaseModel): class PackageName(BaseModel): """The name of a package and it's different representations.""" + repo_name: str = Field(..., description="The name of the repo") name: str = Field(..., description="The full name of the package in spinal case.") title: str = Field(..., description="The name of the package formatted as title.") @@ -119,9 +121,21 @@ def read_package_name() -> PackageName: raise RuntimeError("The name of the git origin could not be resolved.") git_origin_name = stdout.decode("utf-8").strip() - return PackageName( - name=spinalcase(git_origin_name), title=titlecase(git_origin_name) + repo_name = spinalcase(git_origin_name) + name = ( + "my-microservice" + if repo_name == "microservice-repository-template" + else repo_name ) + title = titlecase(name) + + return PackageName(repo_name=repo_name, name=name, title=title) + + +def read_template_overview() -> str: + """Read the template_overview.""" + + return TEMPLATE_OVERVIEW_PATH.read_text() def read_package_description() -> str: @@ -205,6 +219,10 @@ def main(check: bool = False) -> None: details = get_package_details() readme_content = generate_single_readme(details=details) + if details.repo_name == "microservice-repository-template": + template_overview = read_template_overview() + readme_content = template_overview + readme_content + if check: if README_PATH.read_text() != readme_content: echo_failure("README.md is not up to date.")