diff --git a/src/azul/docker.py b/src/azul/docker.py index bc00a07688..e3b8701b5a 100644 --- a/src/azul/docker.py +++ b/src/azul/docker.py @@ -508,18 +508,7 @@ def resolve_docker_image_for_launch(alias: str) -> str: image with the given alias. The alias is the top level key in the JSON object contained in the environment variable `azul_docker_images`. """ - ref = TagImageRef.parse(config.docker_images[alias]['ref']) - log.info('Resolving image %r %r …', alias, ref) - manifest = get_docker_image_manifest(ref) - # Use image mirrored in ECR (if defined), instead of the upstream registry - ref_to_pull = TagImageRef.parse(config.docker_registry + str(ref)) - # If no mirror registry is configured, both refs will be equal and we will - # pull from the upstream registry. We should pull by digest in that case, - # since the tag might have been altered in the upstream registry. If a - # mirror is configured, we will need to pull the image by its tag because we - # don't track the repository digest of images mirrored to ECR. - if ref == ref_to_pull: - ref_to_pull = ref.with_digest(manifest['digest']) + ref_to_pull, manifest = resolve_docker_image_for_pull(alias) image = pull_docker_image(ref_to_pull) # In either case, the verification below ensures that the image we pulled # has the expected ID. @@ -538,3 +527,25 @@ def resolve_docker_image_for_launch(alias: str) -> str: # Returning the image ID means that the container will be launched using # exactly the image we just pulled and verified. return image.id + + +def resolve_docker_image_for_pull(alias: str + ) -> tuple[TagImageRef, Manifest | ManifestList]: + """ + Return an image reference that can be used to pull the image + with the given alias from the ECR. Also return a JSON structure + that describes the image ID and digest. + """ + ref = TagImageRef.parse(config.docker_images[alias]['ref']) + log.info('Resolving image %r %r …', alias, ref) + # Use image mirrored in ECR (if defined), instead of the upstream registry + ref_to_pull = TagImageRef.parse(config.docker_registry + str(ref)) + manifest = get_docker_image_manifest(ref) + # If no mirror registry is configured, both refs will be equal and we will + # pull from the upstream registry. We should pull by digest in that case, + # since the tag might have been altered in the upstream registry. If a + # mirror is configured, we will need to pull the image by its tag because we + # don't track the repository digest of images mirrored to ECR. + if ref == ref_to_pull: + ref_to_pull = ref.with_digest(manifest['digest']) + return ref_to_pull, manifest diff --git a/terraform/gitlab/gitlab.tf.json.template.py b/terraform/gitlab/gitlab.tf.json.template.py index 830a486bc2..32e6ff732e 100644 --- a/terraform/gitlab/gitlab.tf.json.template.py +++ b/terraform/gitlab/gitlab.tf.json.template.py @@ -18,7 +18,7 @@ aws, ) from azul.docker import ( - resolve_docker_image_for_launch, + resolve_docker_image_for_pull, ) from azul.strings import ( departition, @@ -230,14 +230,13 @@ 'prod': [] } -# Note that a change to the image references here also requires updating -# azul_docker_images in environment.py and redeploying the `shared` TF component -# prior to deploying the `gitlab` component. +# FIXME: Launch GitLab, DinD & runner images using image ID +# https://github.com/DataBiosphere/azul/issues/5960 -clamav_image = resolve_docker_image_for_launch('clamav') -dind_image = resolve_docker_image_for_launch('dind') -gitlab_image = resolve_docker_image_for_launch('gitlab') -runner_image = resolve_docker_image_for_launch('gitlab_runner') +clamav_image, _ = resolve_docker_image_for_pull('clamav') +dind_image, _ = resolve_docker_image_for_pull('dind') +gitlab_image, _ = resolve_docker_image_for_pull('gitlab') +runner_image, _ = resolve_docker_image_for_pull('gitlab_runner') # For instructions on finding the latest CIS-hardened AMI, see # OPERATOR.rst#upgrading-linux-ami @@ -1627,7 +1626,7 @@ def qq(*words): 'ExecStartPre=-/usr/bin/docker rm gitlab-dind', 'ExecStartPre=-/usr/bin/docker network rm gitlab-runner-net', 'ExecStartPre=/usr/bin/docker network create gitlab-runner-net', - 'ExecStartPre=/usr/bin/docker pull ' + dind_image, + 'ExecStartPre=/usr/bin/docker pull ' + str(dind_image), jw( 'ExecStart=/usr/bin/docker', 'run', @@ -1677,7 +1676,7 @@ def qq(*words): '--volume /etc/resolv.conf:/etc/resolv.conf', f'--volume {gitlab_mount}/docker:/var/lib/docker', f'--volume {gitlab_mount}/runner/config:/etc/gitlab-runner', - dind_image + str(dind_image) ), '[Install]', 'WantedBy=multi-user.target', @@ -1701,7 +1700,7 @@ def qq(*words): 'Restart=always', 'ExecStartPre=-/usr/bin/docker stop gitlab', 'ExecStartPre=-/usr/bin/docker rm gitlab', - 'ExecStartPre=/usr/bin/docker pull ' + gitlab_image, + 'ExecStartPre=/usr/bin/docker pull ' + str(gitlab_image), jw( 'ExecStart=/usr/bin/docker', 'run', @@ -1714,7 +1713,7 @@ def qq(*words): f'--volume {gitlab_mount}/config:/etc/gitlab', f'--volume {gitlab_mount}/logs:/var/log/gitlab', f'--volume {gitlab_mount}/data:/var/opt/gitlab', - gitlab_image + str(gitlab_image) ), '[Install]', 'WantedBy=multi-user.target' @@ -1738,7 +1737,7 @@ def qq(*words): 'Restart=always', 'ExecStartPre=-/usr/bin/docker stop gitlab-runner', 'ExecStartPre=-/usr/bin/docker rm gitlab-runner', - 'ExecStartPre=/usr/bin/docker pull ' + runner_image, + 'ExecStartPre=/usr/bin/docker pull ' + str(runner_image), jw( 'ExecStart=/usr/bin/docker', 'run', @@ -1747,7 +1746,7 @@ def qq(*words): f'--volume {gitlab_mount}/runner/config:/etc/gitlab-runner', '--network gitlab-runner-net', '--env DOCKER_HOST=tcp://gitlab-dind:2375', - runner_image + str(runner_image) ), '[Install]', 'WantedBy=multi-user.target' @@ -1771,7 +1770,7 @@ def qq(*words): 'TimeoutStartSec=5min', # `docker pull` may take a long time 'ExecStartPre=-/usr/bin/docker stop clamscan', 'ExecStartPre=-/usr/bin/docker rm clamscan', - 'ExecStartPre=/usr/bin/docker pull ' + clamav_image, + 'ExecStartPre=/usr/bin/docker pull ' + str(clamav_image), jw( 'ExecStart=/usr/bin/docker', 'run', @@ -1780,7 +1779,7 @@ def qq(*words): '--volume /var/run/docker.sock:/var/run/docker.sock', '--volume /:/scan:ro', f'--volume {gitlab_mount}/clamav:/var/lib/clamav:rw', - clamav_image, + str(clamav_image), '/bin/sh', '-c', qq( @@ -1837,7 +1836,7 @@ def qq(*words): 'TimeoutStartSec=5min', # `docker pull` may take a long time 'ExecStartPre=-/usr/bin/docker stop prune-images', 'ExecStartPre=-/usr/bin/docker rm prune-images', - 'ExecStartPre=/usr/bin/docker pull ' + dind_image, + 'ExecStartPre=/usr/bin/docker pull ' + str(dind_image), jw( 'ExecStart=/usr/bin/docker', 'exec', # Execute (as in `docker exec`) … @@ -1892,7 +1891,7 @@ def qq(*words): 'TimeoutStartSec=5min', # `docker pull` may take a long time 'ExecStartPre=-/usr/bin/docker stop registry-garbage-collect', 'ExecStartPre=-/usr/bin/docker rm registry-garbage-collect', - 'ExecStartPre=/usr/bin/docker pull ' + gitlab_image, + 'ExecStartPre=/usr/bin/docker pull ' + str(gitlab_image), jw( 'ExecStart=/usr/bin/docker', 'exec', # Execute (as in `docker exec`) …