Skip to content

Commit

Permalink
Fix: GitLab fails to resolve docker images when pulling (#5960)
Browse files Browse the repository at this point in the history
  • Loading branch information
@achave11-ucsc
achave11-ucsc committed Feb 15, 2024
1 parent 47dbc1a commit 858f6fb
Show file tree
Hide file tree
Showing 2 changed files with 40 additions and 30 deletions.
35 changes: 23 additions & 12 deletions src/azul/docker.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -508,18 +508,7 @@ def resolve_docker_image_for_launch(alias: str) -> str:
image with the given alias. The alias is the top level key in the JSON
object contained in the environment variable `azul_docker_images`.
"""
ref = TagImageRef.parse(config.docker_images[alias]['ref'])
log.info('Resolving image %r %r …', alias, ref)
manifest = get_docker_image_manifest(ref)
# Use image mirrored in ECR (if defined), instead of the upstream registry
ref_to_pull = TagImageRef.parse(config.docker_registry + str(ref))
# If no mirror registry is configured, both refs will be equal and we will
# pull from the upstream registry. We should pull by digest in that case,
# since the tag might have been altered in the upstream registry. If a
# mirror is configured, we will need to pull the image by its tag because we
# don't track the repository digest of images mirrored to ECR.
if ref == ref_to_pull:
ref_to_pull = ref.with_digest(manifest['digest'])
ref_to_pull, manifest = resolve_docker_image_for_pull(alias)
image = pull_docker_image(ref_to_pull)
# In either case, the verification below ensures that the image we pulled
# has the expected ID.
Expand All @@ -538,3 +527,25 @@ def resolve_docker_image_for_launch(alias: str) -> str:
# Returning the image ID means that the container will be launched using
# exactly the image we just pulled and verified.
return image.id


def resolve_docker_image_for_pull(alias: str
) -> tuple[TagImageRef, Manifest | ManifestList]:
"""
Return an image reference that can be used to pull the image
with the given alias from the ECR. Also return a JSON structure
that describes the image ID and digest.
"""
ref = TagImageRef.parse(config.docker_images[alias]['ref'])
log.info('Resolving image %r %r …', alias, ref)
# Use image mirrored in ECR (if defined), instead of the upstream registry
ref_to_pull = TagImageRef.parse(config.docker_registry + str(ref))
manifest = get_docker_image_manifest(ref)
# If no mirror registry is configured, both refs will be equal and we will
# pull from the upstream registry. We should pull by digest in that case,
# since the tag might have been altered in the upstream registry. If a
# mirror is configured, we will need to pull the image by its tag because we
# don't track the repository digest of images mirrored to ECR.
if ref == ref_to_pull:
ref_to_pull = ref.with_digest(manifest['digest'])
return ref_to_pull, manifest
35 changes: 17 additions & 18 deletions terraform/gitlab/gitlab.tf.json.template.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
aws,
)
from azul.docker import (
resolve_docker_image_for_launch,
resolve_docker_image_for_pull,
)
from azul.strings import (
departition,
Expand Down Expand Up @@ -230,14 +230,13 @@
'prod': []
}

# Note that a change to the image references here also requires updating
# azul_docker_images in environment.py and redeploying the `shared` TF component
# prior to deploying the `gitlab` component.
# FIXME: Launch GitLab, DinD & runner images using image ID
# https://github.com/DataBiosphere/azul/issues/5960

clamav_image = resolve_docker_image_for_launch('clamav')
dind_image = resolve_docker_image_for_launch('dind')
gitlab_image = resolve_docker_image_for_launch('gitlab')
runner_image = resolve_docker_image_for_launch('gitlab_runner')
clamav_image, _ = resolve_docker_image_for_pull('clamav')
dind_image, _ = resolve_docker_image_for_pull('dind')
gitlab_image, _ = resolve_docker_image_for_pull('gitlab')
runner_image, _ = resolve_docker_image_for_pull('gitlab_runner')

# For instructions on finding the latest CIS-hardened AMI, see
# OPERATOR.rst#upgrading-linux-ami
Expand Down Expand Up @@ -1627,7 +1626,7 @@ def qq(*words):
'ExecStartPre=-/usr/bin/docker rm gitlab-dind',
'ExecStartPre=-/usr/bin/docker network rm gitlab-runner-net',
'ExecStartPre=/usr/bin/docker network create gitlab-runner-net',
'ExecStartPre=/usr/bin/docker pull ' + dind_image,
'ExecStartPre=/usr/bin/docker pull ' + str(dind_image),
jw(
'ExecStart=/usr/bin/docker',
'run',
Expand Down Expand Up @@ -1677,7 +1676,7 @@ def qq(*words):
'--volume /etc/resolv.conf:/etc/resolv.conf',
f'--volume {gitlab_mount}/docker:/var/lib/docker',
f'--volume {gitlab_mount}/runner/config:/etc/gitlab-runner',
dind_image
str(dind_image)
),
'[Install]',
'WantedBy=multi-user.target',
Expand All @@ -1701,7 +1700,7 @@ def qq(*words):
'Restart=always',
'ExecStartPre=-/usr/bin/docker stop gitlab',
'ExecStartPre=-/usr/bin/docker rm gitlab',
'ExecStartPre=/usr/bin/docker pull ' + gitlab_image,
'ExecStartPre=/usr/bin/docker pull ' + str(gitlab_image),
jw(
'ExecStart=/usr/bin/docker',
'run',
Expand All @@ -1714,7 +1713,7 @@ def qq(*words):
f'--volume {gitlab_mount}/config:/etc/gitlab',
f'--volume {gitlab_mount}/logs:/var/log/gitlab',
f'--volume {gitlab_mount}/data:/var/opt/gitlab',
gitlab_image
str(gitlab_image)
),
'[Install]',
'WantedBy=multi-user.target'
Expand All @@ -1738,7 +1737,7 @@ def qq(*words):
'Restart=always',
'ExecStartPre=-/usr/bin/docker stop gitlab-runner',
'ExecStartPre=-/usr/bin/docker rm gitlab-runner',
'ExecStartPre=/usr/bin/docker pull ' + runner_image,
'ExecStartPre=/usr/bin/docker pull ' + str(runner_image),
jw(
'ExecStart=/usr/bin/docker',
'run',
Expand All @@ -1747,7 +1746,7 @@ def qq(*words):
f'--volume {gitlab_mount}/runner/config:/etc/gitlab-runner',
'--network gitlab-runner-net',
'--env DOCKER_HOST=tcp://gitlab-dind:2375',
runner_image
str(runner_image)
),
'[Install]',
'WantedBy=multi-user.target'
Expand All @@ -1771,7 +1770,7 @@ def qq(*words):
'TimeoutStartSec=5min', # `docker pull` may take a long time
'ExecStartPre=-/usr/bin/docker stop clamscan',
'ExecStartPre=-/usr/bin/docker rm clamscan',
'ExecStartPre=/usr/bin/docker pull ' + clamav_image,
'ExecStartPre=/usr/bin/docker pull ' + str(clamav_image),
jw(
'ExecStart=/usr/bin/docker',
'run',
Expand All @@ -1780,7 +1779,7 @@ def qq(*words):
'--volume /var/run/docker.sock:/var/run/docker.sock',
'--volume /:/scan:ro',
f'--volume {gitlab_mount}/clamav:/var/lib/clamav:rw',
clamav_image,
str(clamav_image),
'/bin/sh',
'-c',
qq(
Expand Down Expand Up @@ -1837,7 +1836,7 @@ def qq(*words):
'TimeoutStartSec=5min', # `docker pull` may take a long time
'ExecStartPre=-/usr/bin/docker stop prune-images',
'ExecStartPre=-/usr/bin/docker rm prune-images',
'ExecStartPre=/usr/bin/docker pull ' + dind_image,
'ExecStartPre=/usr/bin/docker pull ' + str(dind_image),
jw(
'ExecStart=/usr/bin/docker',
'exec', # Execute (as in `docker exec`) …
Expand Down Expand Up @@ -1892,7 +1891,7 @@ def qq(*words):
'TimeoutStartSec=5min', # `docker pull` may take a long time
'ExecStartPre=-/usr/bin/docker stop registry-garbage-collect',
'ExecStartPre=-/usr/bin/docker rm registry-garbage-collect',
'ExecStartPre=/usr/bin/docker pull ' + gitlab_image,
'ExecStartPre=/usr/bin/docker pull ' + str(gitlab_image),
jw(
'ExecStart=/usr/bin/docker',
'exec', # Execute (as in `docker exec`) …
Expand Down

0 comments on commit 858f6fb

Please sign in to comment.