diff --git a/application-templates/django-app/Dockerfile b/application-templates/django-app/Dockerfile index f40e9560..05dbd520 100644 --- a/application-templates/django-app/Dockerfile +++ b/application-templates/django-app/Dockerfile @@ -7,11 +7,11 @@ ENV APP_DIR=/app WORKDIR ${APP_DIR} COPY frontend/package.json ${APP_DIR} -COPY frontend/package-lock.json ${APP_DIR} -RUN npm ci +COPY frontend/yarn.lock ${APP_DIR} +RUN yarn install --frozen-lockfile --timeout 60000 COPY frontend ${APP_DIR} -RUN npm run build +RUN yarn build ##### diff --git a/application-templates/django-app/api/genapi.sh b/application-templates/django-app/api/genapi.sh index aab1d30d..cc5737b4 100644 --- a/application-templates/django-app/api/genapi.sh +++ b/application-templates/django-app/api/genapi.sh @@ -1,6 +1,4 @@ #!/bin/bash -fastapi-codegen --input openapi.yaml --output app -t templates && mv app/main.py ../backend/ && mv app/models.py ../backend/openapi/ -rm -rf app - -echo Generated new models and main.py +ROOT_PATH=$(realpath "$(dirname "$BASH_SOURCE")/../../..") +harness-generate servers --app-name "__APP_NAME__" "$ROOT_PATH" \ No newline at end of file diff --git a/docs/dev.md b/docs/dev.md index 28081862..cdd91c98 100644 --- a/docs/dev.md +++ b/docs/dev.md @@ -80,6 +80,7 @@ The code is organized around the idea that there is a module by artifact that ca deployment-cli-tools ├── ch_cli_tools │   ├── codefresh.py # Code Fresh configuration generation +│ ├── common_types.py # Commmon classes needed across multiple scripts/modules │   ├── helm.py # Helm chart files generation │   ├── __init__.py # Defines logging level and some global constants │   ├── models.py # Currently empty file @@ -106,23 +107,59 @@ First the skeleton of the application is generated (the directories, basic files The following code fragment from the `harness-application` script shows how the skeleton is produced: ```python -if "django-app" in args.templates and "webapp" not in templates: - templates = ["base", "webapp"] + templates +def main(): + # ... + + templates = normalize_templates(templates) + + if TemplateType.WEBAPP in templates: + handle_webapp_template(app_name, app_path) + + if TemplateType.SERVER in templates: + handle_server_template(app_path) + for template_name in templates: - if template_name == 'server': - with tempfile.TemporaryDirectory() as tmp_dirname: - copymergedir(os.path.join(CH_ROOT, APPLICATION_TEMPLATE_PATH, template_name), tmp_dirname) # <1> - merge_configuration_directories(app_path, tmp_dirname) - generate_server(app_path, tmp_dirname) - for base_path in (CH_ROOT, os.getcwd()): - template_path = os.path.join(base_path, APPLICATION_TEMPLATE_PATH, template_name) - if os.path.exists(template_path): - merge_configuration_directories(template_path, app_path) # <1> + merge_template_directories(template_name, app_path) + +# ... + +def normalize_templates(templates): + normalized_templates = list(templates) + + if TemplateType.DJANGO_APP in normalized_templates and TemplateType.WEBAPP not in normalized_templates: + django_app_index = normalized_templates.index(TemplateType.DJANGO_APP) + normalized_templates.insert(django_app_index, TemplateType.WEBAPP) + + has_database_template = any(template in TemplateType.database_templates() for template in normalized_templates) + if TemplateType.DJANGO_APP in normalize_templates and not has_database_template: + django_app_index = normalized_templates.index(TemplateType.DJANGO_APP) + normalized_templates.insert(django_app_index, TemplateType.DB_POSTGRES) + + return normalized_templates + +# ... + +def handle_server_template(app_path): + with tempfile.TemporaryDirectory() as tmp_dirname: + tmp_path = pathlib.Path(tmp_dirname) + server_template_path = pathlib.Path(CH_ROOT)/APPLICATION_TEMPLATE_PATH/TemplateType.SERVER + + copymergedir(server_template_path, tmp_path) + merge_configuration_directories(app_path, tmp_path) + generate_server(app_path, tmp_path) + +#... + +def merge_template_directories(template_name, app_path): + for base_path in (pathlib.Path(CH_ROOT), pathlib.Path.cwd()): + template_path = base_path/APPLICATION_TEMPLATE_PATH/template_name + if template_path.exists(): + merge_configuration_directories(template_path, app_path) ``` -First, if `django-app` is defined as a template for the application, and the `webapp` template is not set, then `base` and `webapp` are added to the list of templates. -Then, depending on the template name, a template directory is merged with the code of the application that will be developed (if it exists), as seen in `<1>`. -The templates for each type of application is described by the constant `APPLICATION_TEPLATE_PATH` and points to [`application-templates`](../application-templates/). +First, if `django-app` is defined as a template for the application, and the `webapp` template is not set and/or there is no database template, then `webapp` and/or `db-postgres` are added to the list of templates (using the `TemplateType` string enum). +Then, depending on the template name, a template directory is merged with the code of the application that will be developed (if it exists). +The templates for each type of application is described by the constant `APPLICATION_TEMPLATE_PATH` and points to [`application-templates`](../application-templates/). Based on the name of the template used for the application generation, the actual template with the same name is searched in this path, and copied/merged in the application target folder. The constant, as well as many other constants, are located in [`cloudharness_utils.constants`](../libraries/cloudharness-utils/cloudharness_utils/constants.py). This file is part of the CloudHarness runtime. @@ -143,12 +180,14 @@ Those constants defines several aspects of CloudHarness. For example, we can see there what base Docker image will be considered depending on what's configured for your application, where will be located the deployment files, from where the applications to generate/pick should be generated, where are located the templates for each kind of generation target, as well as where the configuration for codefresh should be looked for. Once the skeleton of the application is generated considering some templates, the code of the REST API is generated from the OpenAPI specification. -The generation relies on two functions: `generate_server` and `generate_fastapi_server` and `generate_ts_client`. +The generation relies on the functions: `generate_server` and `generate_fastapi_server` and `generate_ts_client`. Those functions are defined in the [`openapi.py`](../tools/deployment-cli-tools/ch_cli_tools/openapi.py) module. -This module and those functions use `openapi-generator-cli` to generate the code for the backend and/or the frontend. +This module and those functions use `openapi-generator-cli` and `fastapi-codegen` to generate the code for the backend and/or the frontend. With this generation, and depending on the templates used, some fine tuning or performed in the code/files generated. For example, some placeholders are replaced depending on the name of the application, or depending on the module in which the application is generated. +As final steps a `.ch-manifest` file is created in the root of the application which contains details about the app name and templates used in generation for use by [`harness-generate`](../tools/deployment-cli-tools/harness-generate) and `harness-generate` is run to ensure all server stubs and client code is in place. + #### How to extend it? Here is some scenarios that would need to modify or impact this part of CloudHarness: @@ -165,35 +204,41 @@ Here is some scenarios that would need to modify or impact this part of CloudHar ### Generation of the base application skeleton The (re-)generation REST API is obtain through the [`harness-generate`](../tools/deployment-cli-tools/harness-generate) command. -The command parses the name of the application, gets the necessary dependencies (the java OpenAPI generator cli), and generates the REST model, the servers stubs and well as the clients code from the OpenAPI specifications. +The command parses the `.ch-manifest` file (inferring and creating one if needed), gets the necessary dependencies (the java OpenAPI generator cli), and generates the REST model, the servers stubs and well as the clients code from the OpenAPI specifications. -The generation of the REST model is done by the `generate_model(...)` function, the generation of the server stub is done by the `generate_servers(...)` function, while the clients generation is done by the `generate_clients(...)` function. +The generation of the REST model is done by the `generate_model(...)` function, the generation of the server stub is done by either the `generate_servers(...)` function, while the clients generation is done by the `generate_clients(...)` function. All of these functions are located in the `harness-generate` script. Under the hood, the `generate_servers(...)` function uses the `generate_fastapi_server(...)` and the `generate_server(...)` function that are defined in the [`openapi.py`](../tools/deployment-cli-tools/ch_cli_tools/openapi.py) module. -The generation of one type of servers over another one is bound to the existence of a `genapi.sh` file: +The generation of one type of servers over another one is based on the template used for generation (if the manifest does not exist, the template is inferred by the existance/non-existance of the `genapi.sh` file): ```python -def generate_servers(root_path, interactive=False): +def generate_servers(root_path, should_generate, app_name): # ... - if os.path.exists(os.path.join(application_root, "api", "genapi.sh")): - # fastapi server --> use the genapi.sh script - generate_fastapi_server(application_root) - else: - generate_server(application_root) + for openapi_file in openapi_files: + #... + + if TemplateType.DJANGO_APP in manifest.templates: + generate_fastapi_server(app_path) + + if TemplateType.FLASK_SERVER in manifest.templates: + generate_server(app_path) ``` The `generate_clients(...)` function also uses `generate_python_client(...)` and `generate_ts_client(...)` from the [`openapi.py`](../tools/deployment-cli-tools/ch_cli_tools/openapi.py) module. -The `generate_ts_client(...)` function is called only if there is folder named `frontend` in the application directory structure: +The `generate_ts_client(...)` function is called only if the manifest templates contains `webapp` (if the manifest does not exist then the use of `webapp` is inferred by the existance/non-existance of a `frontend` directory in the application directory structure), and flags can be used to limit generation to just python or typescript clients: ```python def generate_clients(root_path, client_lib_name=LIB_NAME, interactive=False): # ... - app_dir = os.path.dirname(os.path.dirname(openapi_file)) - generate_python_client(app_name, openapi_file, - client_src_path, lib_name=client_lib_name) - if os.path.exists(os.path.join(app_dir, 'frontend')): - generate_ts_client(openapi_file) + for openapi_file in openapi_files: + #... + + if ClientType.PYTHON_CLIENT in client_types: + generate_python_client(manifest.app_name, openapi_file, client_src_path, lib_name=client_lib_name) + + if TemplateType.WEBAPP in manifest.templates and ClientType.TS_CLIENT in client_types: + generate_ts_client(openapi_file) ``` ### Generation of the application deployment files diff --git a/tools/deployment-cli-tools/ch_cli_tools/common_types.py b/tools/deployment-cli-tools/ch_cli_tools/common_types.py new file mode 100644 index 00000000..63f1893d --- /dev/null +++ b/tools/deployment-cli-tools/ch_cli_tools/common_types.py @@ -0,0 +1,48 @@ +from dataclasses import dataclass +from typing import Union + + +try: + from enum import StrEnum +except ImportError: + from strenum import StrEnum + + +class TemplateType(StrEnum): + BASE = 'base' + FLASK_SERVER = 'flask-server' + WEBAPP = 'webapp' + DB_POSTGRES = 'db-postgres' + DB_NEO4J = 'db-neo4j' + DB_MONGO = 'db-mongo' + DJANGO_APP = 'django-app' + SERVER = 'server' + + @classmethod + def database_templates(cls): + return [cls.DB_POSTGRES, cls.DB_NEO4J, cls.DB_MONGO] + + +@dataclass +class CloudHarnessManifest(): + app_name: str + version: str + inferred: bool + templates: list[str] + + @classmethod + def from_dict(cls, data: dict) -> 'CloudHarnessManifest': + return cls( + app_name=data['app-name'], + version=data['version'], + inferred=data['inferred'], + templates=data['templates'], + ) + + def to_dict(self) -> dict: + return { + 'app-name': self.app_name, + 'version': self.version, + 'inferred': self.inferred, + 'templates': [str(template) for template in self.templates], + } diff --git a/tools/deployment-cli-tools/ch_cli_tools/openapi.py b/tools/deployment-cli-tools/ch_cli_tools/openapi.py index 4df7b883..7de31d8c 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/openapi.py +++ b/tools/deployment-cli-tools/ch_cli_tools/openapi.py @@ -2,9 +2,11 @@ import json import logging import os +import pathlib import shutil import subprocess import sys +from typing import Optional import urllib.request from os.path import dirname as dn, join @@ -19,22 +21,53 @@ OPENAPI_GEN_URL = 'https://repo1.maven.org/maven2/org/openapitools/openapi-generator-cli/7.7.0/openapi-generator-cli-7.7.0.jar' -def generate_server(app_path, overrides_folder=""): +def generate_server(app_path: pathlib.Path, overrides_folder: Optional[pathlib.Path] = None) -> None: get_dependencies() - openapi_dir = os.path.join(app_path, 'api') - openapi_file = glob.glob(os.path.join(openapi_dir, '*.yaml'))[0] - out_name = f"backend" if not os.path.exists( - f"{app_path}/server") else f"server" - out_path = f"{app_path}/{out_name}" - command = f"java -jar {CODEGEN} generate -i {openapi_file} -g python-flask -o {out_path} " \ - f"-c {openapi_dir}/config.json " + \ - (f"-t {overrides_folder}" if overrides_folder else "") - os.system(command) + openapi_directory = app_path / 'api' + openapi_file = next(openapi_directory.glob('*.yaml')) -def generate_fastapi_server(app_path): - command = f"cd {app_path}/api && bash genapi.sh" - os.system(command) + server_path = app_path / 'server' + backend_path = app_path / 'backend' + out_path = server_path if server_path.exists() else backend_path + + command = [ + 'java', '-jar', CODEGEN, 'generate', + '-i', openapi_file, + '-g', 'python-flask', + '-o', out_path, + '-c', openapi_directory / 'config.json', + ] + if overrides_folder: + command += ['-t', overrides_folder] + + subprocess.run(command) + + +def generate_fastapi_server(app_path: pathlib.Path) -> None: + api_directory = app_path / 'api' + backend_directory = app_path / 'backend' + temp_directory = api_directory / 'app' + + command = [ + 'fastapi-codegen', + '--input', api_directory / 'openapi.yaml', + '--output', temp_directory, + '-t', api_directory / 'templates', + ] + subprocess.run(command) + + source_main = temp_directory / 'main.py' + destination_main = backend_directory / 'main.py' + source_main.replace(destination_main) + + source_models = temp_directory / 'models.py' + destination_models = backend_directory / 'openapi' / 'models.py' + source_models.replace(destination_models) + + temp_directory.rmdir() + + logging.info('Generated new models and main.py') def generate_model(base_path=ROOT): diff --git a/tools/deployment-cli-tools/ch_cli_tools/utils.py b/tools/deployment-cli-tools/ch_cli_tools/utils.py index 85fcb5bb..3f3f5659 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/utils.py +++ b/tools/deployment-cli-tools/ch_cli_tools/utils.py @@ -1,8 +1,10 @@ +import contextlib +import pathlib import socket import glob import subprocess -from typing import Any +from typing import Any, Union import requests import os from functools import cache @@ -174,17 +176,18 @@ def replaceindir(root_src_dir, source, replace): if not any(file_.endswith(ext) for ext in REPLACE_TEXT_FILES_EXTENSIONS): continue - src_file = join(src_dir, file_) + src_file = pathlib.Path(src_dir) / file_ replace_in_file(src_file, source, replace) -def replace_in_file(src_file, source, replace): - if src_file.endswith('.py') or basename(src_file) == 'Dockerfile': - replace = to_python_module(replace) - with fileinput.FileInput(src_file, inplace=True) as file: +def replace_in_file(src_file: pathlib.Path, source: str, replacement: str) -> None: + if src_file.name.endswith('.py') or src_file.name == 'Dockerfile': + replacement = to_python_module(replacement) + + with fileinput.input(src_file, inplace=True) as file: try: for line in file: - print(line.replace(source, replace), end='') + print(line.replace(source, replacement), end='') except UnicodeDecodeError: pass @@ -205,29 +208,28 @@ def replace_value(value: Any) -> Any: } -def copymergedir(root_src_dir, root_dst_dir): +def copymergedir(source_root_directory: pathlib.Path, destination_root_directory: pathlib.Path) -> None: """ Does copy and merge (shutil.copytree requires that the destination does not exist) - :param root_src_dir: - :param root_dst_dir: + :param source_root_directory: + :param destination_root_directory: :return: """ - logging.info('Copying directory %s to %s', root_src_dir, root_dst_dir) - for src_dir, dirs, files in os.walk(root_src_dir): + logging.info(f'Copying directory {source_root_directory} to {destination_root_directory}') + + for source_directory, _, files in source_root_directory.walk(): + + destination_directory = destination_root_directory / source_directory.relative_to(source_root_directory) + destination_directory.mkdir(parents=True, exist_ok=True) + + for file in files: + source_file = source_directory / file + destination_file = destination_directory / file - dst_dir = src_dir.replace(root_src_dir, root_dst_dir, 1) - if not exists(dst_dir): - os.makedirs(dst_dir) - for file_ in files: - src_file = join(src_dir, file_) - dst_file = join(dst_dir, file_) - if exists(dst_file): - os.remove(dst_file) try: - shutil.copy(src_file, dst_dir) + source_file.replace(destination_file) except: - logging.warning("Error copying file %s to %s.", - src_file, dst_dir) + logging.warning(f'Error copying file {source_file} to {destination_file}.') def movedircontent(root_src_dir, root_dst_dir): @@ -256,52 +258,69 @@ def movedircontent(root_src_dir, root_dst_dir): shutil.rmtree(root_src_dir) -def merge_configuration_directories(source, dest): - if source == dest: +def merge_configuration_directories(source: Union[str, pathlib.Path], destination: Union[str, pathlib.Path]) -> None: + source_path, destination_path = pathlib.Path(source), pathlib.Path(destination) + + if source_path == destination_path: return - if not exists(source): - logging.warning( - "Trying to merge the not existing directory: %s", source) + + if not source_path.exists(): + logging.warning("Trying to merge the not existing directory: %s", source) return - if not exists(dest): - shutil.copytree( - source, dest, ignore=shutil.ignore_patterns(*EXCLUDE_PATHS)) + + if not destination_path.exists(): + shutil.copytree(source_path, destination_path, ignore=shutil.ignore_patterns(*EXCLUDE_PATHS)) return - for src_dir, dirs, files in os.walk(source): - if any(path in src_dir for path in EXCLUDE_PATHS): + for source_directory, _, files in source_path.walk(): + _merge_configuration_directory(source_path, destination_path, source_directory, files) + + +def _merge_configuration_directory( + source: pathlib.Path, + destination: pathlib.Path, + source_directory: pathlib.Path, + files: list[str] +) -> None: + if any(path in str(source_directory) for path in EXCLUDE_PATHS): + return + + destination_directory = destination / source_directory.relative_to(source) + destination_directory.mkdir(exist_ok=True) + + non_build_files = (file for file in files if file not in BUILD_FILENAMES) + + for file_name in non_build_files: + source_file_path = source_directory / file_name + destination_file_path = destination_directory / file_name + + _merge_configuration_file(source_file_path, destination_file_path) + + +def _merge_configuration_file(source_file_path: pathlib.Path, destination_file_path: pathlib.Path) -> None: + if not exists(destination_file_path): + shutil.copy2(source_file_path, destination_file_path) + return + + merge_operations = [ + (file_is_yaml, merge_yaml_files), + (file_is_json, merge_json_files), + ] + + for can_merge_file, merge_files in merge_operations: + if not can_merge_file(source_file_path.name): continue - dst_dir = src_dir.replace(source, dest, 1) - if not exists(dst_dir): - os.makedirs(dst_dir) - for fname in files: - if fname in BUILD_FILENAMES: - continue - fpath = join(src_dir, fname) - frel = relpath(fpath, start=source) - fdest = join(dest, frel) - if not exists(fdest): - shutil.copy2(fpath, fdest) - elif file_is_yaml(fpath): - - try: - merge_yaml_files(fpath, fdest) - logging.info( - f"Merged/overridden file content of {fdest} with {fpath}") - except Exception as e: - logging.warning(f"Overwriting file {fdest} with {fpath}") - shutil.copy2(fpath, fdest) - elif file_is_json(fpath): - try: - merge_json_files(fpath, fdest) - logging.info( - f"Merged/overridden file content of {fdest} with {fpath}") - except Exception as e: - logging.warning(f"Overwriting file {fdest} with {fpath}") - shutil.copy2(fpath, fdest) - else: - logging.warning(f"Overwriting file {fdest} with {fpath}") - shutil.copy2(fpath, fdest) + + try: + merge_files(source_file_path, destination_file_path) + logging.info(f'Merged/overridden file content of {destination_file_path} with {source_file_path}') + except: + break + + return + + logging.warning(f'Overwriting file {destination_file_path} with {source_file_path}') + shutil.copy2(source_file_path, destination_file_path) def merge_yaml_files(fname, fdest): @@ -456,3 +475,13 @@ def get_git_commit_hash(path): ['git', 'rev-parse', '--short', 'HEAD'], cwd=path).decode("utf-8").strip() except: return None + + +def load_yaml(yaml_file: pathlib.Path) -> dict: + with yaml_file.open('r') as file: + return yaml.load(file) + + +def save_yaml(yaml_file: pathlib.Path, data: dict) -> None: + with yaml_file.open('w') as file: + yaml.dump(data, file) diff --git a/tools/deployment-cli-tools/harness-application b/tools/deployment-cli-tools/harness-application index d3601f7a..c79fee3f 100644 --- a/tools/deployment-cli-tools/harness-application +++ b/tools/deployment-cli-tools/harness-application @@ -3,24 +3,20 @@ import json import pathlib import sys -import os import re import shutil import tempfile import subprocess import logging import argparse +from typing import Union from ch_cli_tools import CH_ROOT from cloudharness_utils.constants import APPLICATION_TEMPLATE_PATH from ch_cli_tools.openapi import generate_server, generate_fastapi_server, APPLICATIONS_SRC_PATH, generate_ts_client -from ch_cli_tools.utils import merge_configuration_directories, replaceindir, replace_in_file, \ +from ch_cli_tools.utils import merge_configuration_directories, replaceindir, replace_in_file, save_yaml, \ to_python_module, copymergedir, get_json_template, replace_in_dict - -try: - from enum import StrEnum -except ImportError: - from strenum import StrEnum +from ch_cli_tools.common_types import CloudHarnessManifest, TemplateType # Only allow lowercased alphabetical characters separated by "-". name_pattern = re.compile("[a-z]+((-)?[a-z])?") @@ -28,25 +24,13 @@ name_pattern = re.compile("[a-z]+((-)?[a-z])?") PLACEHOLDER = '__APP_NAME__' -class TemplateType(StrEnum): - BASE = 'base' - FLASK_SERVER = 'flask-server' - WEBAPP = 'webapp' - DB_POSTGRES = 'db-postgres' - DB_NEO4J = 'db-neo4j' - DB_MONGO = 'db-mongo' - DJANGO_APP = 'django-app' - SERVER = 'server' - - -def main(): +def main() -> None: app_name, templates = get_command_line_arguments() - app_path = os.path.join(APPLICATIONS_SRC_PATH, app_name) - os.makedirs(app_path, exist_ok=True) + app_path = pathlib.Path(APPLICATIONS_SRC_PATH) / app_name + app_path.mkdir(exist_ok=True) - if TemplateType.DJANGO_APP in templates and TemplateType.WEBAPP not in templates: - templates = [TemplateType.BASE, TemplateType.WEBAPP] + templates + templates = normalize_templates(templates) if TemplateType.WEBAPP in templates: handle_webapp_template(app_name, app_path) @@ -60,7 +44,7 @@ def main(): if TemplateType.FLASK_SERVER in templates: handle_flask_server_template(app_path) - replace_in_file(os.path.join(app_path, 'api/config.json'), PLACEHOLDER, to_python_module(app_name)) + replace_in_file(app_path / 'api' / 'config.json', PLACEHOLDER, to_python_module(app_name)) if TemplateType.DJANGO_APP in templates: handle_django_app_template(app_name, app_path) @@ -68,7 +52,10 @@ def main(): replaceindir(app_path, PLACEHOLDER, app_name) if TemplateType.WEBAPP in templates: - handle_webapp_template_cleanup(app_path) + handle_webapp_template_post_merge(app_path) + + create_manifest_file(app_path, app_name, templates) + call_harness_generate(app_path, app_name) def get_command_line_arguments() -> tuple[str, list[str]]: @@ -112,50 +99,78 @@ def get_command_line_arguments() -> tuple[str, list[str]]: return args.name, args.templates -def handle_webapp_template(app_name: str, app_path: str) -> None: - if os.path.exists(os.path.join(app_path, 'frontend')): - shutil.rmtree(os.path.join(app_path, 'frontend')) - cmd = ["yarn", "create", "vite", app_name, "--template", "react-ts"] - logging.info(f"Running command: {' '.join(cmd)}") - subprocess.run(cmd, cwd=app_path) - shutil.move(os.path.join(app_path, app_name), os.path.join(app_path, 'frontend')) - generate_ts_client(openapi_file=os.path.join(app_path, 'api/openapi.yaml')) +def normalize_templates(templates: list[str]) -> list[str]: + normalized_templates = list(templates) + if TemplateType.DJANGO_APP in normalized_templates and TemplateType.WEBAPP not in normalized_templates: + django_app_index = normalized_templates.index(TemplateType.DJANGO_APP) + normalized_templates.insert(django_app_index, TemplateType.WEBAPP) + + has_database_template = any(template in TemplateType.database_templates() for template in normalized_templates) + if TemplateType.DJANGO_APP in normalized_templates and not has_database_template: + django_app_index = normalized_templates.index(TemplateType.DJANGO_APP) + normalized_templates.insert(django_app_index, TemplateType.DB_POSTGRES) + + return normalized_templates + + +def handle_webapp_template(app_name: str, app_path: pathlib.Path) -> None: + frontend_path = app_path / 'frontend' + + if frontend_path.exists(): + shutil.rmtree(frontend_path) + + create_vite_skaffold(app_name, app_path, frontend_path) + + +def create_vite_skaffold(app_name: str, app_path: pathlib.Path, frontend_path: pathlib.Path) -> None: + command = ['yarn', 'create', 'vite', app_name, '--template', 'react-ts'] + logging.info(f'Running command: {" ".join(command)}') + subprocess.run(command, cwd=app_path) + shutil.move(app_path / app_name, frontend_path) + + +def install_frontend_dependencies(frontend_path: pathlib.Path) -> None: + command = ['yarn', 'install'] + logging.info(f'Running command: {" ".join(command)}') + subprocess.run(command, cwd=frontend_path) -def handle_webapp_template_cleanup(app_path: str) -> None: - try: - os.remove(os.path.join(app_path, 'backend', 'Dockerfile')) - except FileNotFoundError: - # backend dockerfile not found, continue - pass +def handle_webapp_template_post_merge(app_path: pathlib.Path) -> None: + backend_dockerfile_path = app_path / 'backend' / 'Dockerfile' + backend_dockerfile_path.unlink(missing_ok=True) -def handle_server_template(app_path: str) -> None: + install_frontend_dependencies(app_path / 'frontend') + generate_ts_client(openapi_file=app_path / 'api' / 'openapi.yaml') + + +def handle_server_template(app_path: pathlib.Path) -> None: with tempfile.TemporaryDirectory() as tmp_dirname: - copymergedir(os.path.join(CH_ROOT, APPLICATION_TEMPLATE_PATH, TemplateType.SERVER), tmp_dirname) - merge_configuration_directories(app_path, tmp_dirname) - generate_server(app_path, tmp_dirname) + tmp_path = pathlib.Path(tmp_dirname) + server_template_path = pathlib.Path(CH_ROOT) / APPLICATION_TEMPLATE_PATH / TemplateType.SERVER + copymergedir(server_template_path, tmp_path) + merge_configuration_directories(app_path, tmp_path) + generate_server(app_path, tmp_path) -def handle_flask_server_template(app_path: str) -> None: + +def handle_flask_server_template(app_path: pathlib.Path) -> None: generate_server(app_path) -def handle_django_app_template(app_name: str, app_path: str) -> None: - replace_in_file(os.path.join(app_path, 'api/templates/main.jinja2'), PLACEHOLDER, to_python_module(app_name)) +def handle_django_app_template(app_name: str, app_path: pathlib.Path) -> None: + python_app_name = to_python_module(app_name) + + api_path = app_path / 'api' + replace_in_file(api_path / 'templates' / 'main.jinja2', PLACEHOLDER, python_app_name) + replace_in_file(api_path / 'genapi.sh', PLACEHOLDER, app_name) generate_fastapi_server(app_path) - replace_in_file( - os.path.join(app_path, 'deploy/values.yaml'), - f"{PLACEHOLDER}:{PLACEHOLDER}", - f"{to_python_module(app_name)}:{to_python_module(app_name)}" - ) - replace_in_file(os.path.join(app_path, "dev-setup.sh"), PLACEHOLDER, app_name) + + replace_in_file(app_path / 'deploy' / 'values.yaml', f'{PLACEHOLDER}:{PLACEHOLDER}', f'{python_app_name}:{python_app_name}') + replace_in_file(app_path / 'dev-setup.sh', PLACEHOLDER, app_name) create_django_app_vscode_debug_configuration(app_name) - try: - os.remove(os.path.join(app_path, 'backend', "__APP_NAME__", "__main__.py")) - except FileNotFoundError: - # backend dockerfile not found, continue - pass + + (app_path / 'backend' / '__APP_NAME__' / '__main__.py').unlink(missing_ok=True) def create_django_app_vscode_debug_configuration(app_name: str): @@ -179,10 +194,30 @@ def create_django_app_vscode_debug_configuration(app_name: str): json.dump(launch_config, f, indent=2, sort_keys=True) -def merge_template_directories(template_name: str, app_path: str) -> None: - for base_path in (CH_ROOT, os.getcwd()): - template_path = os.path.join(base_path, APPLICATION_TEMPLATE_PATH, template_name) - if os.path.exists(template_path): +def create_manifest_file(app_path: pathlib.Path, app_name: str, templates: list[Union[str, TemplateType]]) -> None: + manifest_file = app_path / '.ch-manifest' + manifest = CloudHarnessManifest( + app_name=app_name, + version='1', + inferred=False, + templates=[str(template) for template in templates], + ) + + logging.info('Creating manifest file') + save_yaml(manifest_file, manifest.to_dict()) + + +def call_harness_generate(app_path: pathlib.Path, app_name: str): + logging.info('Running initial harness generate...') + root_path = app_path.parent.parent + command = ['harness-generate', 'all', '--ts-only', '--app-name', app_name, root_path] + subprocess.run(command) + + +def merge_template_directories(template_name: str, app_path: pathlib.Path) -> None: + for base_path in (pathlib.Path(CH_ROOT), pathlib.Path.cwd()): + template_path = base_path / APPLICATION_TEMPLATE_PATH / template_name + if template_path.exists(): merge_configuration_directories(template_path, app_path) diff --git a/tools/deployment-cli-tools/harness-generate b/tools/deployment-cli-tools/harness-generate index 12d40e8a..c51edf25 100644 --- a/tools/deployment-cli-tools/harness-generate +++ b/tools/deployment-cli-tools/harness-generate @@ -1,176 +1,346 @@ #!/usr/bin/env python -import glob -import os +import argparse +from dataclasses import dataclass +import enum +import functools +import operator +import pathlib import shutil -import sys import logging +from typing import Callable, Optional +from ruamel.yaml.error import YAMLError from ch_cli_tools.openapi import LIB_NAME, generate_python_client, generate_server, generate_fastapi_server, \ get_dependencies, generate_ts_client, generate_model -from ch_cli_tools.utils import copymergedir +from ch_cli_tools.utils import copymergedir, load_yaml, save_yaml +from ch_cli_tools.common_types import CloudHarnessManifest, TemplateType -HERE = os.path.dirname(os.path.realpath(__file__)) -ROOT = os.path.dirname(HERE) +def main(): + args = get_command_line_arguments() + get_dependencies() + + root_path = args.path.absolute() + should_generate = should_generate_interactive if args.is_interactive else lambda _: True + + if args.generate_models: + generate_models(root_path, should_generate) + + if args.generate_servers: + generate_servers(root_path, should_generate, args.app_name) + + if args.generate_clients: + assert args.client_name is not None + generate_clients(root_path, should_generate, args.app_name, args.client_name, args.client_types) + + +class ClientType(enum.Flag): + TS_CLIENT = enum.auto() + PYTHON_CLIENT = enum.auto() + + @classmethod + def all(cls): + return functools.reduce(operator.or_, cls) + + +class GenerationMode(enum.Flag): + CLIENTS = enum.auto() + MODELS = enum.auto() + SERVERS = enum.auto() + + @classmethod + def all(cls): + return functools.reduce(operator.or_, cls) + + +@dataclass(frozen=True) +class CommandLineArguments: + path: pathlib.Path + app_name: Optional[str] + is_interactive: bool + generation_mode: GenerationMode + client_name: Optional[str] = None + client_types: ClientType = ClientType.all() + + @property + def generate_models(self): + return GenerationMode.MODELS in self.generation_mode + + @property + def generate_servers(self): + return GenerationMode.SERVERS in self.generation_mode + + @property + def generate_clients(self): + return GenerationMode.CLIENTS in self.generation_mode + + +def get_command_line_arguments() -> CommandLineArguments: + parser = argparse.ArgumentParser(description='Walk filesystem inside ./applications create application scaffolding.') + + common_arguments = argparse.ArgumentParser(add_help=False) + common_arguments.add_argument('path', metavar='path', nargs='?', default=pathlib.Path.cwd(), type=pathlib.Path, + help='Base path of the application.') + common_arguments.add_argument('-i', '--interactive', dest='is_interactive', action="store_true", + help='Asks before generate') + common_arguments.add_argument('-a', '--app-name', dest='app_name', action="store", default=None, + help='Generate only for a specific application') + + clients_arguments = argparse.ArgumentParser(add_help=False) + clients_arguments.add_argument('-cn', '--client-name', dest='client_name', action='store', default=LIB_NAME, + help='specify image registry prefix') + client_type_group = clients_arguments.add_mutually_exclusive_group(required=False) + client_type_group.add_argument('-t', '--ts-only', dest='client_types', action='store_const', const=ClientType.TS_CLIENT, + help='Generate only typescript clients') + client_type_group.add_argument('-p', '--python-only', dest='client_types', action='store_const', const=ClientType.PYTHON_CLIENT, + help='Generate only python clients') + clients_arguments.set_defaults(client_types=ClientType.all()) + + subparsers = parser.add_subparsers(title='generation modes', required=True) + + all_parser = subparsers.add_parser('all', parents=[common_arguments, clients_arguments], + help='Generate models, server stubs and client libraries') + all_parser.set_defaults(generation_mode=GenerationMode.all()) + + models_parser = subparsers.add_parser('models', parents=[common_arguments], + help='Generate only model library') + models_parser.set_defaults(generation_mode=GenerationMode.MODELS) -def get_openapi_file_paths(root_path): - return [path for path in glob.glob(root_path + '/applications/*/api/*.yaml')] + servers_parser = subparsers.add_parser('servers', parents=[common_arguments], + help='Generate only server stubs') + servers_parser.set_defaults(generation_mode=GenerationMode.SERVERS) + clients_parser = subparsers.add_parser('clients', parents=[common_arguments, clients_arguments], + help='Generate only client libraries') + clients_parser.set_defaults(generation_mode=GenerationMode.CLIENTS) -def get_application_paths(openapi_files): - return [os.path.basename(os.path.dirname(os.path.dirname(path))) for path in openapi_files] + args = parser.parse_args() + return CommandLineArguments(**args.__dict__) -def generate_servers(root_path, interactive=False, server=None): + +def should_generate_interactive(resource: str) -> bool: + user_input = input(f'Do you want to generate {resource}? [Y/n] ').casefold() + + return user_input == 'y' + + +def generate_models( + root_path: pathlib.Path, + should_generate: Callable[[str], bool], +) -> None: + """ + Generates the main model + """ + library_models_path = root_path / 'libraries' / 'models' + + if not library_models_path.exists(): + return + + if not should_generate('the main model'): + return + + generate_model() + + +def generate_servers( + root_path: pathlib.Path, + should_generate: Callable[[str], bool], + app_name: Optional[str], +) -> None: """ Generates server stubs """ openapi_files = get_openapi_file_paths(root_path) - modules = get_application_paths(openapi_files) - for i in range(len(modules)): - if not interactive or input("Do you want to generate " + openapi_files[i] + "? [Y/n]").upper() != 'N': - openapi_file = openapi_files[i] - application_root = os.path.dirname(os.path.dirname(openapi_file)) - appname = os.path.basename(application_root) - if server and server != appname: - continue - if os.path.exists(os.path.join(application_root, "api", "genapi.sh")): - # fastapi server --> use the genapi.sh script - generate_fastapi_server(application_root) - else: - generate_server(application_root) - - -def aggregate_packages(client_src_path, lib_name=LIB_NAME): - DOCS_PATH = os.path.join(client_src_path, 'docs') - TEST_PATH = os.path.join(client_src_path, 'test') - README = os.path.join(client_src_path, 'README.md') - REQUIREMENTS = os.path.join(client_src_path, 'requirements.txt') - TEST_REQUIREMENTS = os.path.join(client_src_path, 'test-requirements.txt') - - if not os.path.exists(DOCS_PATH): - os.makedirs(DOCS_PATH) - if not os.path.exists(TEST_PATH): - os.makedirs(TEST_PATH) - if os.path.exists(README): - os.remove(README) - if os.path.exists(REQUIREMENTS): - os.remove(REQUIREMENTS) - if os.path.exists(TEST_REQUIREMENTS): - os.remove(TEST_REQUIREMENTS) - - req_lines_seen = set() - test_req_lines_seen = set() - - for MODULE_TMP_PATH in glob.glob(client_src_path + '/tmp-*'): - module = MODULE_TMP_PATH.split( - f'{lib_name}/tmp-')[-1].replace('-', '_') - - # Moves package - - code_dest_dir = os.path.join(client_src_path, lib_name, module) - copymergedir(os.path.join(MODULE_TMP_PATH, - lib_name, module), code_dest_dir) - copymergedir(f"{MODULE_TMP_PATH}/{lib_name}.{module}", - code_dest_dir) # Fixes a a bug with nested packages - - # Adds Docs - module_doc_path = os.path.join(DOCS_PATH, module) - if not os.path.exists(module_doc_path): - os.mkdir(module_doc_path) - copymergedir(f"{client_src_path}/tmp-{module}/docs", module_doc_path) - - # Adds Tests - module_test_path = os.path.join(client_src_path, 'test', module) - copymergedir(os.path.join(MODULE_TMP_PATH, 'test'), module_test_path) - - # Merges Readme - readme_file = f"{MODULE_TMP_PATH}/README.md" - if not os.path.exists(readme_file): - logging.warning("Readme file not found: %s.", readme_file) - continue - with open(README, 'a+') as outfile: - with open(readme_file) as infile: - filedata = infile.read() - fd = filedata.replace('docs/', f'docs/{module}/') - outfile.write(fd) - # Merges Requirements - # FIXME: Different package versions will remain in the output file + for openapi_file in openapi_files: + app_path = openapi_file.parent.parent + manifest = get_manifest(app_path) - requirements_file = f"{MODULE_TMP_PATH}/requirements.txt" - outfile = open(REQUIREMENTS, "a+") - for line in open(requirements_file, "r"): - if line not in req_lines_seen: - outfile.write(line) - req_lines_seen.add(line) - outfile.close() + if app_name and manifest.app_name != app_name: + continue - # Merges Test Requirements - # FIXME: Different package versions will remain in the output file - test_requirements_file = f"{MODULE_TMP_PATH}/test-requirements.txt" - outfile = open(TEST_REQUIREMENTS, "a+") - for line in open(test_requirements_file, "r"): - if line not in test_req_lines_seen: - outfile.write(line) - test_req_lines_seen.add(line) - outfile.close() + if not should_generate(f'server stubs for {openapi_file}'): + continue + + if TemplateType.DJANGO_APP in manifest.templates: + generate_fastapi_server(app_path) - # Removes Tmp Files - shutil.rmtree(MODULE_TMP_PATH) + if TemplateType.FLASK_SERVER in manifest.templates: + generate_server(app_path) -def generate_clients(root_path, client_lib_name=LIB_NAME, interactive=False): +def generate_clients( + root_path: pathlib.Path, + should_generate: Callable[[str], bool], + app_name: Optional[str], + client_lib_name: str, + client_types: ClientType, +) -> None: """ Generates client stubs """ - if interactive and input("Do you want to generate client libraries? [Y/n]").upper() == 'N': + if not should_generate('client libraries'): return openapi_files = get_openapi_file_paths(root_path) - applications = get_application_paths(openapi_files) - - client_src_path = os.path.join( - root_path, 'libraries/client', client_lib_name) - for i in range(len(applications)): - app_name = applications[i] - openapi_file = openapi_files[i] - app_dir = os.path.dirname(os.path.dirname(openapi_file)) - generate_python_client(app_name, openapi_file, - client_src_path, lib_name=client_lib_name) - if os.path.exists(os.path.join(app_dir, 'frontend')): + client_src_path = root_path / 'libraries' / 'client' / client_lib_name + + for openapi_file in openapi_files: + app_path = openapi_file.parent.parent + manifest = get_manifest(app_path) + + if app_name and manifest.app_name != app_name: + continue + + if ClientType.PYTHON_CLIENT in client_types: + generate_python_client(manifest.app_name, openapi_file, client_src_path, lib_name=client_lib_name) + + if TemplateType.WEBAPP in manifest.templates and ClientType.TS_CLIENT in client_types: generate_ts_client(openapi_file) aggregate_packages(client_src_path, client_lib_name) -if __name__ == "__main__": - import argparse - - parser = argparse.ArgumentParser( - description='Walk filesystem inside ./applications create application scaffolding.') - parser.add_argument('path', metavar='path', default=ROOT, type=str, - help='Base path of the application.') - parser.add_argument('-cn', '--client-name', dest='client_name', action="store", default=LIB_NAME, - help='Specify image registry prefix') - parser.add_argument('-i', '--interactive', dest='interactive', action="store_true", - help='Asks before generate') - parser.add_argument('-s', '--server', dest='server', action="store", - help='Generate only a specific server (provide application name) stubs', default=()) - parser.add_argument('-c', '--clients', dest='clients', action="store_true", - help='Generate only client libraries') - parser.add_argument('-m', '--models', dest='models', action="store_true", - help='Generate only model library') - args, unknown = parser.parse_known_args(sys.argv[1:]) - - root_path = os.path.join(os.getcwd(), args.path) if not os.path.isabs( - args.path) else args.path +def get_openapi_file_paths(root_path: pathlib.Path) -> list[pathlib.Path]: + return [path for path in root_path.glob('applications/*/api/*.yaml')] - get_dependencies() - if args.models and os.path.exists(os.path.join(root_path, "libraries/models")) and (not args.interactive or input("Do you want to generate the main model? [Y/n]").upper() != 'N'): - generate_model() - if not (args.clients or args.models) or args.server: - generate_servers(root_path, interactive=args.interactive, server=args.server) - if not (args.server or args.models) or args.clients: - generate_clients(root_path, args.client_name, interactive=args.interactive) + +def aggregate_packages(client_source_path: pathlib.Path, lib_name=LIB_NAME): + client_source_path.mkdir(parents=True, exist_ok=True) + + client_docs_path = client_source_path / 'docs' + client_docs_path.mkdir(exist_ok=True) + + client_test_path = client_source_path / 'test' + client_test_path.mkdir(exist_ok=True) + + client_readme_file = client_source_path / 'README.md' + client_readme_file.unlink(missing_ok=True) + + client_requirements_file = client_source_path / 'requirements.txt' + client_requirements_file.unlink(missing_ok=True) + + client_test_requirements_file = client_source_path / 'test-requirements.txt' + client_test_requirements_file.unlink(missing_ok=True) + + requirements_lines_seen = set() + test_requirements_lines_seen = set() + + for temp_module_path in client_source_path.glob('tmp-*/'): + module = ( + temp_module_path + .name + .removeprefix('tmp-') + .replace('-', '_') + ) + + code_destination_directory = client_source_path / lib_name / module + copymergedir(temp_module_path / lib_name / module, code_destination_directory) + copymergedir(temp_module_path / f'{lib_name}.{module}', code_destination_directory) # Fixes a bug with nested packages + + module_docs_path = client_docs_path / module + module_docs_path.mkdir(parents=True, exist_ok=True) + copymergedir(client_source_path / temp_module_path.name / 'docs', module_docs_path) + + module_tests_path = client_source_path / 'test' / module + copymergedir(temp_module_path / 'test', module_tests_path) + + readme_file = temp_module_path / 'README.md' + if not readme_file.exists(): + logging.warning(f'Readme file not found: {readme_file}.') + continue + + with client_readme_file.open('+a') as out_file, readme_file.open('r') as in_file: + file_data = in_file.read() + updated_file_data = file_data.replace('docs/', f'docs/{module}/') + out_file.write(updated_file_data) + + # FIXME: Different package versions will remain in the output file + requirements_file = temp_module_path / 'requirements.txt' + with requirements_file.open('r') as in_file, client_requirements_file.open('+a') as out_file: + unseen_lines = [line for line in in_file if line not in requirements_lines_seen] + requirements_lines_seen.update(unseen_lines) + out_file.writelines(unseen_lines) + + # FIXME: Different package versions will remain in the output file + test_requirements_file = temp_module_path / 'test-requirements.txt' + with test_requirements_file.open('r') as in_file, client_test_requirements_file.open('+a') as out_file: + unseen_lines = [line for line in in_file if line not in test_requirements_lines_seen] + test_requirements_lines_seen.update(unseen_lines) + out_file.writelines(unseen_lines) + + shutil.rmtree(temp_module_path) + + +def get_manifest(app_path: pathlib.Path) -> CloudHarnessManifest: + manifest_file = app_path / '.ch-manifest' + + try: + manifest_data = load_yaml(manifest_file) + manifest = CloudHarnessManifest.from_dict(manifest_data) + except (FileNotFoundError, YAMLError): + logging.info(f'Could not find manifest file {manifest_file}, inferring manifest from app structure...') + manifest = CloudHarnessManifest( + app_name=app_path.name, + version='1', + inferred=True, + templates=infer_templates(app_path), + ) + save_yaml(manifest_file, manifest.to_dict()) + + return manifest + + +def infer_templates(app_path: pathlib.Path) -> list[str]: + templates = [TemplateType.BASE] + + infer_webapp_template(app_path, templates) + infer_server_template(app_path, templates) + infer_database_template(app_path, templates) + + return templates + + +def infer_webapp_template(app_path: pathlib.Path, templates: list[str]) -> None: + frontend_path = app_path / 'frontend' + if frontend_path.exists(): + templates.append(TemplateType.WEBAPP) + + +def infer_server_template(app_path: pathlib.Path, templates: list[str]) -> None: + genapi_path = app_path / 'api' / 'genapi.sh' + + if genapi_path.exists(): + templates.append(TemplateType.DJANGO_APP) + return + + server_path = app_path / 'server' + backend_path = app_path / 'backend' + if server_path.exists() or backend_path.exists(): + templates.append(TemplateType.FLASK_SERVER) + + +def infer_database_template(app_path: pathlib.Path, templates: list[str]) -> None: + values_file = app_path / 'deploy' / 'values.yaml' + + try: + values_data = load_yaml(values_file) + database_config = values_data['harness']['database'] + if not database_config['auto']: + return + + database_type = database_config['type'] + if database_type == 'mongo': + templates.append(TemplateType.DB_MONGO) + if database_type == 'neo4j': + templates.append(TemplateType.DB_NEO4J) + if database_type == 'postgres': + templates.append(TemplateType.DB_POSTGRES) + except (FileNotFoundError, YAMLError, KeyError): + pass + + +if __name__ == "__main__": + main()