From c8b4f8933590779b57bf0d888c4dec452463e012 Mon Sep 17 00:00:00 2001 From: David Boreham Date: Tue, 18 Jul 2023 08:59:07 -0600 Subject: [PATCH] Copy all referenced config files into deployment dir (#452) --- .../docker-compose-fixturenet-optimism.yml | 6 ++-- .../compose/docker-compose-mobymask-app.yml | 4 +-- .../compose/docker-compose-peer-test-app.yml | 2 +- .../docker-compose-watcher-mobymask-v2.yml | 2 +- app/data/config/{ => network}/wait-for-it.sh | 0 app/deployment.py | 5 +-- app/deployment_create.py | 33 +++++++++++++++++-- 7 files changed, 40 insertions(+), 12 deletions(-) rename app/data/config/{ => network}/wait-for-it.sh (100%) diff --git a/app/data/compose/docker-compose-fixturenet-optimism.yml b/app/data/compose/docker-compose-fixturenet-optimism.yml index c1f9bd2b..9d7d99dd 100644 --- a/app/data/compose/docker-compose-fixturenet-optimism.yml +++ b/app/data/compose/docker-compose-fixturenet-optimism.yml @@ -23,7 +23,7 @@ services: command: | "./wait-for-it.sh -h ${CERC_L1_HOST:-$${DEFAULT_CERC_L1_HOST}} -p ${CERC_L1_PORT:-$${DEFAULT_CERC_L1_PORT}} -s -t 60 -- ./run.sh" volumes: - - ../config/wait-for-it.sh:/app/packages/contracts-bedrock/wait-for-it.sh + - ../config/network/wait-for-it.sh:/app/packages/contracts-bedrock/wait-for-it.sh - ../container-build/cerc-optimism-contracts/hardhat-tasks/verify-contract-deployment.ts:/app/packages/contracts-bedrock/tasks/verify-contract-deployment.ts - ../container-build/cerc-optimism-contracts/hardhat-tasks/rekey-json.ts:/app/packages/contracts-bedrock/tasks/rekey-json.ts - ../container-build/cerc-optimism-contracts/hardhat-tasks/send-balance.ts:/app/packages/contracts-bedrock/tasks/send-balance.ts @@ -120,7 +120,7 @@ services: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} CERC_L1_RPC: ${CERC_L1_RPC} volumes: - - ../config/wait-for-it.sh:/wait-for-it.sh + - ../config/network/wait-for-it.sh:/wait-for-it.sh - ../config/fixturenet-optimism/run-op-batcher.sh:/run-op-batcher.sh - l2_accounts:/l2-accounts:ro entrypoint: ["sh", "-c"] @@ -145,7 +145,7 @@ services: CERC_SCRIPT_DEBUG: ${CERC_SCRIPT_DEBUG} CERC_L1_RPC: ${CERC_L1_RPC} volumes: - - ../config/wait-for-it.sh:/wait-for-it.sh + - ../config/network/wait-for-it.sh:/wait-for-it.sh - ../config/fixturenet-optimism/run-op-proposer.sh:/run-op-proposer.sh - l1_deployment:/contracts-bedrock:ro - l2_accounts:/l2-accounts:ro diff --git a/app/data/compose/docker-compose-mobymask-app.yml b/app/data/compose/docker-compose-mobymask-app.yml index 1b4b4f6d..5ad767ab 100644 --- a/app/data/compose/docker-compose-mobymask-app.yml +++ b/app/data/compose/docker-compose-mobymask-app.yml @@ -18,7 +18,7 @@ services: working_dir: /scripts command: ["sh", "mobymask-app-start.sh"] volumes: - - ../config/wait-for-it.sh:/scripts/wait-for-it.sh + - ../config/network/wait-for-it.sh:/scripts/wait-for-it.sh - ../config/watcher-mobymask-v2/mobymask-app-start.sh:/scripts/mobymask-app-start.sh - peers_ids:/peers - mobymask_deployment:/server @@ -50,7 +50,7 @@ services: working_dir: /scripts command: ["sh", "mobymask-app-start.sh"] volumes: - - ../config/wait-for-it.sh:/scripts/wait-for-it.sh + - ../config/network/wait-for-it.sh:/scripts/wait-for-it.sh - ../config/watcher-mobymask-v2/mobymask-app-start.sh:/scripts/mobymask-app-start.sh - peers_ids:/peers - mobymask_deployment:/server diff --git a/app/data/compose/docker-compose-peer-test-app.yml b/app/data/compose/docker-compose-peer-test-app.yml index 87a36228..593ce89a 100644 --- a/app/data/compose/docker-compose-peer-test-app.yml +++ b/app/data/compose/docker-compose-peer-test-app.yml @@ -14,7 +14,7 @@ services: CERC_DENY_MULTIADDRS: ${CERC_DENY_MULTIADDRS} command: ["sh", "test-app-start.sh"] volumes: - - ../config/wait-for-it.sh:/scripts/wait-for-it.sh + - ../config/network/wait-for-it.sh:/scripts/wait-for-it.sh - ../config/watcher-mobymask-v2/test-app-start.sh:/scripts/test-app-start.sh - peers_ids:/peers ports: diff --git a/app/data/compose/docker-compose-watcher-mobymask-v2.yml b/app/data/compose/docker-compose-watcher-mobymask-v2.yml index db3e3a20..bed274c6 100644 --- a/app/data/compose/docker-compose-watcher-mobymask-v2.yml +++ b/app/data/compose/docker-compose-watcher-mobymask-v2.yml @@ -44,7 +44,7 @@ services: CERC_L2_NODE_PORT: ${CERC_L2_NODE_PORT} command: ["sh", "deploy-and-generate-invite.sh"] volumes: - - ../config/wait-for-it.sh:/app/packages/server/wait-for-it.sh + - ../config/network/wait-for-it.sh:/app/packages/server/wait-for-it.sh - ../config/watcher-mobymask-v2/secrets-template.json:/app/packages/server/secrets-template.json - ../config/watcher-mobymask-v2/deploy-and-generate-invite.sh:/app/packages/server/deploy-and-generate-invite.sh - mobymask_deployment:/app/packages/server diff --git a/app/data/config/wait-for-it.sh b/app/data/config/network/wait-for-it.sh similarity index 100% rename from app/data/config/wait-for-it.sh rename to app/data/config/network/wait-for-it.sh diff --git a/app/deployment.py b/app/deployment.py index fa2236aa..ccb11f51 100644 --- a/app/deployment.py +++ b/app/deployment.py @@ -73,13 +73,14 @@ def start(ctx, extra_args): @command.command() +@click.option("--delete-volumes/--preserve-volumes", default=False, help="delete data volumes") @click.argument('extra_args', nargs=-1) # help: command: down @click.pass_context -def down(ctx, extra_args): +def down(ctx, delete_volumes, extra_args): # Get the stack config file name # TODO: add cluster name and env file here ctx.obj = make_deploy_context(ctx) - down_operation(ctx, extra_args, None) + down_operation(ctx, delete_volumes, extra_args) # stop is the preferred alias for down diff --git a/app/deployment_create.py b/app/deployment_create.py index 5a9ddbcd..fb83f39b 100644 --- a/app/deployment_create.py +++ b/app/deployment_create.py @@ -96,6 +96,24 @@ def _fixup_pod_file(pod, spec, compose_dir): pod["volumes"][volume] = new_volume_spec +# Inspect the pod yaml to find config files referenced in subdirectories +# other than the one associated with the pod +def _find_extra_config_dirs(parsed_pod_file, pod): + config_dirs = set() + services = parsed_pod_file["services"] + for service in services: + service_info = services[service] + if "volumes" in service_info: + for volume in service_info["volumes"]: + if ":" in volume: + host_path = volume.split(":")[0] + if host_path.startswith("../config"): + config_dir = host_path.split("/")[2] + if config_dir != pod: + config_dirs.add(config_dir) + return config_dirs + + @click.command() @click.option("--output", required=True, help="Write yaml spec file here") @click.pass_context @@ -146,10 +164,19 @@ def create(ctx, spec_file, deployment_dir): for pod in pods: pod_file_path = os.path.join(_get_compose_file_dir(), f"docker-compose-{pod}.yml") parsed_pod_file = yaml.load(open(pod_file_path, "r")) + extra_config_dirs = _find_extra_config_dirs(parsed_pod_file, pod) + if global_options(ctx).debug: + print(f"extra config dirs: {extra_config_dirs}") _fixup_pod_file(parsed_pod_file, parsed_spec, destination_compose_dir) with open(os.path.join(destination_compose_dir, os.path.basename(pod_file_path)), "w") as output_file: yaml.dump(parsed_pod_file, output_file) # Copy the config files for the pod, if any - source_config_dir = data_dir.joinpath("config", pod) - if os.path.exists(source_config_dir): - copytree(source_config_dir, os.path.join(deployment_dir, "config", pod)) + config_dirs = {pod} + config_dirs = config_dirs.union(extra_config_dirs) + for config_dir in config_dirs: + source_config_dir = data_dir.joinpath("config", config_dir) + if os.path.exists(source_config_dir): + destination_config_dir = os.path.join(deployment_dir, "config", config_dir) + # If the same config dir appears in multiple pods, it may already have been copied + if not os.path.exists(destination_config_dir): + copytree(source_config_dir, destination_config_dir)