diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 6e871fa..90ee325 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -2,7 +2,7 @@ // https://github.com/microsoft/vscode-dev-containers/tree/v0.177.0/containers/python-3-postgres // Update the VARIANT arg in docker-compose.yml to pick a Python version: 3, 3.8, 3.7, 3.6 { - "name": "metadata-artifact-search-service", + "name": "${localWorkspaceFolderBasename}", "dockerComposeFile": "docker-compose.yml", "service": "app", "workspaceFolder": "/workspace", @@ -42,6 +42,9 @@ "python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle", "python.linting.pylintPath": "/usr/local/py-utils/bin/pylint", "python.testing.pytestPath": "/usr/local/py-utils/bin/pytest", + "python.testing.pytestArgs": [ + "--profile" + ], "python.testing.unittestEnabled": false, "python.testing.pytestEnabled": true, "editor.formatOnSave": true, @@ -80,12 +83,13 @@ "postCreateCommand": "dev_install", // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. "remoteUser": "vscode", + "containerEnv": { + // for testcontainers to connect to the docker host: + "TC_HOST": "host.docker.internal", + "DOCKER_HOST": "unix:///var/run/docker.sock" + }, "features": { - "ghcr.io/devcontainers/features/docker-in-docker:2": { - "version": "latest", - "enableNonRootDocker": "true", - "moby": true, - "azureDnsAutoDetection": false - } + // details can be found here: https://github.com/devcontainers/features/tree/main/src/docker-outside-of-docker + "ghcr.io/devcontainers/features/docker-outside-of-docker:1": {} } } diff --git a/.mandatory_files b/.mandatory_files index c0b51fb..20b6c56 100644 --- a/.mandatory_files +++ b/.mandatory_files @@ -6,7 +6,6 @@ # may differ from that of the template repository. .devcontainer/dev_launcher -.devcontainer/devcontainer.json .devcontainer/docker-compose.yml .devcontainer/Dockerfile diff --git a/.readme_template.md b/.readme_template.md index dd8b184..229e739 100644 --- a/.readme_template.md +++ b/.readme_template.md @@ -11,6 +11,7 @@ $summary $description ## Installation + We recommend using the provided Docker container. A pre-build version is available at [docker hub](https://hub.docker.com/repository/docker/ghga/$name): @@ -42,6 +43,7 @@ $shortname --help ``` ## Configuration + ### Parameters The service requires the following configuration parameters: @@ -77,19 +79,20 @@ $openapi_doc $design_description ## Development + For setting up the development environment, we rely on the -[devcontainer feature](https://code.visualstudio.com/docs/remote/containers) of vscode +[devcontainer feature](https://code.visualstudio.com/docs/remote/containers) of VS Code in combination with Docker Compose. -To use it, you have to have Docker Compose as well as vscode with its "Remote - Containers" +To use it, you have to have Docker Compose as well as VS Code with its "Remote - Containers" extension (`ms-vscode-remote.remote-containers`) installed. -Then open this repository in vscode and run the command -`Remote-Containers: Reopen in Container` from the vscode "Command Palette". +Then open this repository in VS Code and run the command +`Remote-Containers: Reopen in Container` from the VS Code "Command Palette". This will give you a full-fledged, pre-configured development environment including: - infrastructural dependencies of the service (databases, etc.) -- all relevant vscode extensions pre-installed -- pre-configured linting and auto-formating +- all relevant VS Code extensions pre-installed +- pre-configured linting and auto-formatting - a pre-configured debugger - automatic license-header insertion @@ -101,9 +104,11 @@ if you update dependencies in the [`./pyproject.toml`](./pyproject.toml) or the [`./requirements-dev.txt`](./requirements-dev.txt), please run it again. ## License + This repository is free to use and modify according to the [Apache 2.0 License](./LICENSE). -## Readme Generation -This readme is autogenerate, please see [`readme_generation.md`](./readme_generation.md) +## README Generation + +This README file is auto-generated, please see [`readme_generation.md`](./readme_generation.md) for details. diff --git a/.static_files b/.static_files index e3c8f76..6983902 100644 --- a/.static_files +++ b/.static_files @@ -11,6 +11,7 @@ .devcontainer/dev_install .devcontainer/license_header.txt .devcontainer/Dockerfile +.devcontainer/devcontainer.json scripts/script_utils/__init__.py scripts/script_utils/cli.py diff --git a/README.md b/README.md index 9f02235..ec18739 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ For more information see the OpenAPI spec linked below. ## Installation + We recommend using the provided Docker container. A pre-build version is available at [docker hub](https://hub.docker.com/repository/docker/ghga/mass): @@ -59,31 +60,90 @@ mass --help ``` ## Configuration + ### Parameters The service requires the following configuration parameters: - **`searchable_classes`** *(object)*: A collection of searchable_classes with facetable properties. Can contain additional properties. - - **Additional Properties**: Refer to *[#/definitions/SearchableClass](#definitions/SearchableClass)*. + - **Additional Properties**: Refer to *[#/$defs/SearchableClass](#$defs/SearchableClass)*. - **`resource_change_event_topic`** *(string)*: Name of the event topic used to track resource deletion and upsertion events. + + Examples: + + ```json + "searchable_resource" + ``` + + - **`resource_deletion_event_type`** *(string)*: The type to use for events with deletion instructions. + + Examples: + + ```json + "searchable_resource_deleted" + ``` + + - **`resource_upsertion_event_type`** *(string)*: The type to use for events with upsert instructions. + + Examples: + + ```json + "searchable_resource_upserted" + ``` + + - **`service_name`** *(string)*: Default: `"mass"`. - **`service_instance_id`** *(string)*: A string that uniquely identifies this instance across all instances of this service. A globally unique Kafka client ID will be created by concatenating the service_name and the service_instance_id. + + Examples: + + ```json + "germany-bw-instance-001" + ``` + + - **`kafka_servers`** *(array)*: A list of connection strings to connect to Kafka bootstrap servers. - **Items** *(string)* + + Examples: + + ```json + [ + "localhost:9092" + ] + ``` + + - **`db_connection_str`** *(string, format: password)*: MongoDB connection string. Might include credentials. For more information see: https://naiveskill.com/mongodb-connection-string/. + + Examples: + + ```json + "mongodb://localhost:27017" + ``` + + - **`db_name`** *(string)*: Name of the database located on the MongoDB server. + + Examples: + + ```json + "my-database" + ``` + + - **`host`** *(string)*: IP of the host. Default: `"127.0.0.1"`. - **`port`** *(integer)*: Port to expose the server on the specified host. Default: `8080`. @@ -100,36 +160,83 @@ The service requires the following configuration parameters: - **`docs_url`** *(string)*: Path to host the swagger documentation. This is relative to the specified host and port. Default: `"/docs"`. -- **`cors_allowed_origins`** *(array)*: A list of origins that should be permitted to make cross-origin requests. By default, cross-origin requests are not allowed. You can use ['*'] to allow any origin. +- **`cors_allowed_origins`**: A list of origins that should be permitted to make cross-origin requests. By default, cross-origin requests are not allowed. You can use ['*'] to allow any origin. Default: `null`. - - **Items** *(string)* + - **Any of** -- **`cors_allow_credentials`** *(boolean)*: Indicate that cookies should be supported for cross-origin requests. Defaults to False. Also, cors_allowed_origins cannot be set to ['*'] for credentials to be allowed. The origins must be explicitly specified. + - *array* -- **`cors_allowed_methods`** *(array)*: A list of HTTP methods that should be allowed for cross-origin requests. Defaults to ['GET']. You can use ['*'] to allow all standard methods. + - **Items** *(string)* - - **Items** *(string)* + - *null* -- **`cors_allowed_headers`** *(array)*: A list of HTTP request headers that should be supported for cross-origin requests. Defaults to []. You can use ['*'] to allow all headers. The Accept, Accept-Language, Content-Language and Content-Type headers are always allowed for CORS requests. - - **Items** *(string)* + Examples: + + ```json + [ + "https://example.org", + "https://www.example.org" + ] + ``` + + +- **`cors_allow_credentials`**: Indicate that cookies should be supported for cross-origin requests. Defaults to False. Also, cors_allowed_origins cannot be set to ['*'] for credentials to be allowed. The origins must be explicitly specified. Default: `null`. + + - **Any of** + + - *boolean* + + - *null* + + + Examples: + + ```json + [ + "https://example.org", + "https://www.example.org" + ] + ``` -## Definitions +- **`cors_allowed_methods`**: A list of HTTP methods that should be allowed for cross-origin requests. Defaults to ['GET']. You can use ['*'] to allow all standard methods. Default: `null`. -- **`FacetLabel`** *(object)*: Contains the key and corresponding user-friendly name for a facet. + - **Any of** - - **`key`** *(string, required)*: The raw facet key, such as study.type. + - *array* - - **`name`** *(string)*: The user-friendly name for the facet. Default: `""`. + - **Items** *(string)* -- **`SearchableClass`** *(object)*: Represents a searchable artifact or resource type. + - *null* - - **`description`** *(string, required)*: A brief description of the resource type. - - **`facetable_properties`** *(array, required)*: A list of of the facetable properties for the resource type. + Examples: + + ```json + [ + "*" + ] + ``` + + +- **`cors_allowed_headers`**: A list of HTTP request headers that should be supported for cross-origin requests. Defaults to []. You can use ['*'] to allow all headers. The Accept, Accept-Language, Content-Language and Content-Type headers are always allowed for CORS requests. Default: `null`. + + - **Any of** + + - *array* + + - **Items** *(string)* + + - *null* + + + Examples: + + ```json + [] + ``` - - **Items**: Refer to *[#/definitions/FacetLabel](#definitions/FacetLabel)*. ### Usage: @@ -185,19 +292,20 @@ Typical sequence of events is as follows: ## Development + For setting up the development environment, we rely on the -[devcontainer feature](https://code.visualstudio.com/docs/remote/containers) of vscode +[devcontainer feature](https://code.visualstudio.com/docs/remote/containers) of VS Code in combination with Docker Compose. -To use it, you have to have Docker Compose as well as vscode with its "Remote - Containers" +To use it, you have to have Docker Compose as well as VS Code with its "Remote - Containers" extension (`ms-vscode-remote.remote-containers`) installed. -Then open this repository in vscode and run the command -`Remote-Containers: Reopen in Container` from the vscode "Command Palette". +Then open this repository in VS Code and run the command +`Remote-Containers: Reopen in Container` from the VS Code "Command Palette". This will give you a full-fledged, pre-configured development environment including: - infrastructural dependencies of the service (databases, etc.) -- all relevant vscode extensions pre-installed -- pre-configured linting and auto-formating +- all relevant VS Code extensions pre-installed +- pre-configured linting and auto-formatting - a pre-configured debugger - automatic license-header insertion @@ -209,9 +317,11 @@ if you update dependencies in the [`./pyproject.toml`](./pyproject.toml) or the [`./requirements-dev.txt`](./requirements-dev.txt), please run it again. ## License + This repository is free to use and modify according to the [Apache 2.0 License](./LICENSE). -## Readme Generation -This readme is autogenerate, please see [`readme_generation.md`](./readme_generation.md) +## README Generation + +This README file is auto-generated, please see [`readme_generation.md`](./readme_generation.md) for details. diff --git a/config_schema.json b/config_schema.json index 52b80f3..113a8fe 100644 --- a/config_schema.json +++ b/config_schema.json @@ -1,122 +1,145 @@ { - "title": "ModSettings", + "$defs": { + "FacetLabel": { + "description": "Contains the key and corresponding user-friendly name for a facet", + "properties": { + "key": { + "description": "The raw facet key, such as study.type", + "title": "Key", + "type": "string" + }, + "name": { + "default": "", + "description": "The user-friendly name for the facet", + "title": "Name", + "type": "string" + } + }, + "required": [ + "key" + ], + "title": "FacetLabel", + "type": "object" + }, + "SearchableClass": { + "description": "Represents a searchable artifact or resource type", + "properties": { + "description": { + "description": "A brief description of the resource type", + "title": "Description", + "type": "string" + }, + "facetable_properties": { + "description": "A list of of the facetable properties for the resource type", + "items": { + "$ref": "#/$defs/FacetLabel" + }, + "title": "Facetable Properties", + "type": "array" + } + }, + "required": [ + "description", + "facetable_properties" + ], + "title": "SearchableClass", + "type": "object" + } + }, + "additionalProperties": false, "description": "Modifies the orginal Settings class provided by the user", - "type": "object", "properties": { "searchable_classes": { - "title": "Searchable Classes", - "description": "A collection of searchable_classes with facetable properties", - "env_names": [ - "mass_searchable_classes" - ], - "type": "object", "additionalProperties": { - "$ref": "#/definitions/SearchableClass" - } + "$ref": "#/$defs/SearchableClass" + }, + "description": "A collection of searchable_classes with facetable properties", + "title": "Searchable Classes", + "type": "object" }, "resource_change_event_topic": { - "title": "Resource Change Event Topic", "description": "Name of the event topic used to track resource deletion and upsertion events", - "example": "searchable_resource", - "env_names": [ - "mass_resource_change_event_topic" + "examples": [ + "searchable_resource" ], + "title": "Resource Change Event Topic", "type": "string" }, "resource_deletion_event_type": { - "title": "Resource Deletion Event Type", "description": "The type to use for events with deletion instructions", - "example": "searchable_resource_deleted", - "env_names": [ - "mass_resource_deletion_event_type" + "examples": [ + "searchable_resource_deleted" ], + "title": "Resource Deletion Event Type", "type": "string" }, "resource_upsertion_event_type": { - "title": "Resource Upsertion Event Type", "description": "The type to use for events with upsert instructions", - "example": "searchable_resource_upserted", - "env_names": [ - "mass_resource_upsertion_event_type" + "examples": [ + "searchable_resource_upserted" ], + "title": "Resource Upsertion Event Type", "type": "string" }, "service_name": { - "title": "Service Name", "default": "mass", - "env_names": [ - "mass_service_name" - ], + "title": "Service Name", "type": "string" }, "service_instance_id": { - "title": "Service Instance Id", "description": "A string that uniquely identifies this instance across all instances of this service. A globally unique Kafka client ID will be created by concatenating the service_name and the service_instance_id.", - "example": "germany-bw-instance-001", - "env_names": [ - "mass_service_instance_id" + "examples": [ + "germany-bw-instance-001" ], + "title": "Service Instance Id", "type": "string" }, "kafka_servers": { - "title": "Kafka Servers", "description": "A list of connection strings to connect to Kafka bootstrap servers.", - "example": [ - "localhost:9092" + "examples": [ + [ + "localhost:9092" + ] ], - "env_names": [ - "mass_kafka_servers" - ], - "type": "array", "items": { "type": "string" - } + }, + "title": "Kafka Servers", + "type": "array" }, "db_connection_str": { - "title": "Db Connection Str", "description": "MongoDB connection string. Might include credentials. For more information see: https://naiveskill.com/mongodb-connection-string/", - "example": "mongodb://localhost:27017", - "env_names": [ - "mass_db_connection_str" + "examples": [ + "mongodb://localhost:27017" ], + "format": "password", + "title": "Db Connection Str", "type": "string", - "writeOnly": true, - "format": "password" + "writeOnly": true }, "db_name": { - "title": "Db Name", "description": "Name of the database located on the MongoDB server.", - "example": "my-database", - "env_names": [ - "mass_db_name" + "examples": [ + "my-database" ], + "title": "Db Name", "type": "string" }, "host": { - "title": "Host", - "description": "IP of the host.", "default": "127.0.0.1", - "env_names": [ - "mass_host" - ], + "description": "IP of the host.", + "title": "Host", "type": "string" }, "port": { - "title": "Port", - "description": "Port to expose the server on the specified host", "default": 8080, - "env_names": [ - "mass_port" - ], + "description": "Port to expose the server on the specified host", + "title": "Port", "type": "integer" }, "log_level": { - "title": "Log Level", - "description": "Controls the verbosity of the log.", "default": "info", - "env_names": [ - "mass_log_level" - ], + "description": "Controls the verbosity of the log.", "enum": [ "critical", "error", @@ -125,105 +148,119 @@ "debug", "trace" ], + "title": "Log Level", "type": "string" }, "auto_reload": { - "title": "Auto Reload", - "description": "A development feature. Set to `True` to automatically reload the server upon code changes", "default": false, - "env_names": [ - "mass_auto_reload" - ], + "description": "A development feature. Set to `True` to automatically reload the server upon code changes", + "title": "Auto Reload", "type": "boolean" }, "workers": { - "title": "Workers", - "description": "Number of workers processes to run.", "default": 1, - "env_names": [ - "mass_workers" - ], + "description": "Number of workers processes to run.", + "title": "Workers", "type": "integer" }, "api_root_path": { - "title": "Api Root Path", - "description": "Root path at which the API is reachable. This is relative to the specified host and port.", "default": "/", - "env_names": [ - "mass_api_root_path" - ], + "description": "Root path at which the API is reachable. This is relative to the specified host and port.", + "title": "Api Root Path", "type": "string" }, "openapi_url": { - "title": "Openapi Url", - "description": "Path to get the openapi specification in JSON format. This is relative to the specified host and port.", "default": "/openapi.json", - "env_names": [ - "mass_openapi_url" - ], + "description": "Path to get the openapi specification in JSON format. This is relative to the specified host and port.", + "title": "Openapi Url", "type": "string" }, "docs_url": { - "title": "Docs Url", - "description": "Path to host the swagger documentation. This is relative to the specified host and port.", "default": "/docs", - "env_names": [ - "mass_docs_url" - ], + "description": "Path to host the swagger documentation. This is relative to the specified host and port.", + "title": "Docs Url", "type": "string" }, "cors_allowed_origins": { - "title": "Cors Allowed Origins", - "description": "A list of origins that should be permitted to make cross-origin requests. By default, cross-origin requests are not allowed. You can use ['*'] to allow any origin.", - "example": [ - "https://example.org", - "https://www.example.org" + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } ], - "env_names": [ - "mass_cors_allowed_origins" + "default": null, + "description": "A list of origins that should be permitted to make cross-origin requests. By default, cross-origin requests are not allowed. You can use ['*'] to allow any origin.", + "examples": [ + [ + "https://example.org", + "https://www.example.org" + ] ], - "type": "array", - "items": { - "type": "string" - } + "title": "Cors Allowed Origins" }, "cors_allow_credentials": { - "title": "Cors Allow Credentials", - "description": "Indicate that cookies should be supported for cross-origin requests. Defaults to False. Also, cors_allowed_origins cannot be set to ['*'] for credentials to be allowed. The origins must be explicitly specified.", - "example": [ - "https://example.org", - "https://www.example.org" + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } ], - "env_names": [ - "mass_cors_allow_credentials" + "default": null, + "description": "Indicate that cookies should be supported for cross-origin requests. Defaults to False. Also, cors_allowed_origins cannot be set to ['*'] for credentials to be allowed. The origins must be explicitly specified.", + "examples": [ + [ + "https://example.org", + "https://www.example.org" + ] ], - "type": "boolean" + "title": "Cors Allow Credentials" }, "cors_allowed_methods": { - "title": "Cors Allowed Methods", - "description": "A list of HTTP methods that should be allowed for cross-origin requests. Defaults to ['GET']. You can use ['*'] to allow all standard methods.", - "example": [ - "*" + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } ], - "env_names": [ - "mass_cors_allowed_methods" + "default": null, + "description": "A list of HTTP methods that should be allowed for cross-origin requests. Defaults to ['GET']. You can use ['*'] to allow all standard methods.", + "examples": [ + [ + "*" + ] ], - "type": "array", - "items": { - "type": "string" - } + "title": "Cors Allowed Methods" }, "cors_allowed_headers": { - "title": "Cors Allowed Headers", + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, "description": "A list of HTTP request headers that should be supported for cross-origin requests. Defaults to []. You can use ['*'] to allow all headers. The Accept, Accept-Language, Content-Language and Content-Type headers are always allowed for CORS requests.", - "example": [], - "env_names": [ - "mass_cors_allowed_headers" + "examples": [ + [] ], - "type": "array", - "items": { - "type": "string" - } + "title": "Cors Allowed Headers" } }, "required": [ @@ -236,52 +273,6 @@ "db_connection_str", "db_name" ], - "additionalProperties": false, - "definitions": { - "FacetLabel": { - "title": "FacetLabel", - "description": "Contains the key and corresponding user-friendly name for a facet", - "type": "object", - "properties": { - "key": { - "title": "Key", - "description": "The raw facet key, such as study.type", - "type": "string" - }, - "name": { - "title": "Name", - "description": "The user-friendly name for the facet", - "default": "", - "type": "string" - } - }, - "required": [ - "key" - ] - }, - "SearchableClass": { - "title": "SearchableClass", - "description": "Represents a searchable artifact or resource type", - "type": "object", - "properties": { - "description": { - "title": "Description", - "description": "A brief description of the resource type", - "type": "string" - }, - "facetable_properties": { - "title": "Facetable Properties", - "description": "A list of of the facetable properties for the resource type", - "type": "array", - "items": { - "$ref": "#/definitions/FacetLabel" - } - } - }, - "required": [ - "description", - "facetable_properties" - ] - } - } + "title": "ModSettings", + "type": "object" } \ No newline at end of file diff --git a/example_config.yaml b/example_config.yaml index 0c74e59..26e56c8 100644 --- a/example_config.yaml +++ b/example_config.yaml @@ -26,6 +26,6 @@ searchable_classes: name: Study Type - key: study.project.alias name: Project Alias -service_instance_id: '1' +service_instance_id: '001' service_name: mass workers: 1 diff --git a/openapi.yaml b/openapi.yaml index 65ad204..324179b 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -146,9 +146,11 @@ components: title: Filters type: array limit: + anyOf: + - type: integer + - type: 'null' description: Limit the results to this number title: Limit - type: integer query: default: '' description: The keyword search for the query @@ -195,6 +197,7 @@ components: - descending - relevance title: SortOrder + type: string SortingParameter: description: Represents a combination of a field to sort and the sort order properties: @@ -235,7 +238,7 @@ components: info: title: FastAPI version: 0.1.0 -openapi: 3.0.2 +openapi: 3.1.0 paths: /health: get: diff --git a/scripts/update_config_docs.py b/scripts/update_config_docs.py index b1be57a..d0a6e1e 100755 --- a/scripts/update_config_docs.py +++ b/scripts/update_config_docs.py @@ -28,7 +28,11 @@ from typing import Any import yaml -from pydantic import BaseSettings + +try: + from pydantic_settings import BaseSettings +except ImportError: # fallback for pydantic v1 + from pydantic import BaseSettings # type: ignore [no-redef] from script_utils.cli import echo_failure, echo_success, run @@ -77,14 +81,16 @@ def get_schema() -> str: """Returns a JSON schema generated from a Config class.""" config = get_dev_config() - return config.schema_json(indent=2) + return config.schema_json(indent=2) # change eventually to .model_json_schema(...) def get_example() -> str: """Returns an example config YAML.""" config = get_dev_config() - normalized_config_dict = json.loads(config.json()) + normalized_config_dict = json.loads( + config.json() # change eventually to .model_dump_json() + ) return yaml.dump(normalized_config_dict) # pyright: ignore