From 69efb922bd87e4e3e48f725aeb91750f5871759a Mon Sep 17 00:00:00 2001 From: Matthew James Briggs Date: Thu, 17 Mar 2022 13:11:52 -0700 Subject: [PATCH] build: bottlerocket-test-tools Move the third party test binaries to their own container image build. --- .github/workflows/build.yaml | 4 +- .gitignore | 1 + Dockerfile | 200 +++------------- Makefile | 23 +- clarify.toml | 94 -------- hashes/eksctl | 3 - hashes/sonobuoy | 3 - tools/CHANGELOG.md | 21 ++ tools/Dockerfile | 213 ++++++++++++++++++ tools/README.md | 23 ++ tools/clarify.toml | 99 ++++++++ .../hashes}/aws-iam-authenticator | 0 tools/hashes/eksctl | 3 + {hashes => tools/hashes}/kubernetes | 0 tools/hashes/sonobuoy | 3 + 15 files changed, 414 insertions(+), 276 deletions(-) delete mode 100644 hashes/eksctl delete mode 100644 hashes/sonobuoy create mode 100644 tools/CHANGELOG.md create mode 100644 tools/Dockerfile create mode 100644 tools/README.md create mode 100644 tools/clarify.toml rename {hashes => tools/hashes}/aws-iam-authenticator (100%) create mode 100644 tools/hashes/eksctl rename {hashes => tools/hashes}/kubernetes (100%) create mode 100644 tools/hashes/sonobuoy diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 9f3b2be9..fab3a45b 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -31,7 +31,9 @@ jobs: runs-on: [ self-hosted, linux, x64 ] steps: - uses: actions/checkout@v3 - - run: make images + # TODO - for most PRs we should use the published container image instead of building each time + - run: make tools -e TOOLS_IMAGE=localtools:latest + - run: make images -e TOOLS_IMAGE=localtools:latest license-check: # A small machine is OK for this independent job. runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 9a807d4d..5f5da7e1 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ /.cargo/* !.cargo/config.toml /bin/kind +*.tar.* diff --git a/Dockerfile b/Dockerfile index 1bd40b81..507c48fe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,9 +3,15 @@ # =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= # Shared build stage used to build the testsys agent binary ARG BUILDER_IMAGE +ARG TOOLS_IMAGE FROM ${BUILDER_IMAGE} as build -ADD ./ /src +COPY ./ /src + +# =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= +# It appears that the syntax `--from=$TOOLS_IMAGE /foo /bar` does not work. As a workaround +# we cache $TOOLS_IMAGE as a build layer. +FROM ${TOOLS_IMAGE} as tools # =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= FROM build as build-go @@ -46,12 +52,6 @@ RUN --mount=type=cache,mode=0777,target=/src/target \ --path . \ --root . -# TODO get licenses for boringtun -# Install boringtun -RUN cargo install boringtun \ - --target ${ARCH}-bottlerocket-linux-musl \ - --root . - # =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= # TODO figure out how to build this in the Bottlerocket SDK # Builds wireguard tools @@ -68,148 +68,6 @@ RUN temp_dir="$(mktemp -d --suffix wireguard-tools-setup)" && \ make && WITH_BASHCOMPLETION=no WITH_SYSTEMDUNITS=no make install && \ rm -rf ${temp_dir} -# =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= -FROM build-go as eksctl-build - -USER root -RUN mkdir -p /usr/share/licenses/eksctl && \ - chown -R builder:builder /usr/share/licenses/eksctl - -ARG EKSCTL_VERSION=0.82.0 -ARG EKSCTL_SOURCE_URL="https://github.com/weaveworks/eksctl/archive/refs/tags/v${EKSCTL_VERSION}.tar.gz" - -ARG GOARCH -ARG EKSCTL_BINARY_URL="https://github.com/weaveworks/eksctl/releases/download/v${EKSCTL_VERSION}/eksctl_Linux_${GOARCH}.tar.gz" - -USER builder -WORKDIR /home/builder/ -RUN mkdir eksctl && curl -L ${EKSCTL_SOURCE_URL} \ - -o eksctl_${EKSCTL_VERSION}.tar.gz && \ - grep eksctl_${EKSCTL_VERSION}.tar.gz \ - /src/hashes/eksctl | sha512sum --check - && \ - tar -xf eksctl_${EKSCTL_VERSION}.tar.gz --strip-components 1 -C eksctl && \ - rm eksctl_${EKSCTL_VERSION}.tar.gz - -WORKDIR /home/builder/eksctl/ -# TODO - restore this with a fix for https://github.com/bottlerocket-os/bottlerocket-test-system/issues/288 -# For reasons not yet understood, this can take an hour or more in certain environments. For now we need -# to skip it until we can figure out what is happening. -#RUN go mod vendor -#RUN cp -p LICENSE /usr/share/licenses/eksctl && \ -# /usr/libexec/tools/bottlerocket-license-scan \ -# --clarify /src/clarify.toml \ -# --spdx-data /usr/libexec/tools/spdx-data \ -# --out-dir /usr/share/licenses/eksctl/vendor \ -# go-vendor ./vendor -RUN curl -L "${EKSCTL_BINARY_URL}" \ - -o eksctl_${EKSCTL_VERSION}_${GOOS}_${GOARCH}.tar.gz && \ - grep eksctl_${EKSCTL_VERSION}_${GOOS}_${GOARCH}.tar.gz \ - /src/hashes/eksctl | sha512sum --check - && \ - tar -xf eksctl_${EKSCTL_VERSION}_${GOOS}_${GOARCH}.tar.gz -C /tmp && \ - rm eksctl_${EKSCTL_VERSION}_${GOOS}_${GOARCH}.tar.gz - -# =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= -FROM build-go as kubernetes-build - -USER root -RUN mkdir -p /usr/share/licenses/kubernetes && \ - chown -R builder:builder /usr/share/licenses/kubernetes - -ARG K8S_VERSION=1.21.6 -ARG K8S_SOURCE_URL="https://github.com/kubernetes/kubernetes/archive/refs/tags/v${K8S_VERSION}.tar.gz" - -ARG GOARCH -ARG KUBEADM_BINARY_URL="https://dl.k8s.io/release/v${K8S_VERSION}/bin/linux/${GOARCH}/kubeadm" - -USER builder -WORKDIR /home/builder/ -RUN mkdir kubernetes && \ - curl -L "${K8S_SOURCE_URL}" -o kubernetes_${K8S_VERSION}.tar.gz && \ - grep kubernetes_${K8S_VERSION}.tar.gz \ - /src/hashes/kubernetes | sha512sum --check - && \ - tar -xf kubernetes_${K8S_VERSION}.tar.gz \ - --strip-components 1 -C kubernetes && \ - rm kubernetes_${K8S_VERSION}.tar.gz - -WORKDIR /home/builder/kubernetes/ -RUN go mod vendor -RUN cp -p LICENSE /usr/share/licenses/kubernetes && \ - /usr/libexec/tools/bottlerocket-license-scan \ - --clarify /src/clarify.toml \ - --spdx-data /usr/libexec/tools/spdx-data \ - --out-dir /usr/share/licenses/kubernetes/vendor \ - go-vendor ./vendor -RUN curl -L ${KUBEADM_BINARY_URL} \ - -o kubeadm_${K8S_VERSION}_${GOOS}_${GOARCH} && \ - grep kubeadm_${K8S_VERSION}_${GOOS}_${GOARCH} \ - /src/hashes/kubernetes | sha512sum --check - && \ - install -m 0755 kubeadm_${K8S_VERSION}_${GOOS}_${GOARCH} /tmp/kubeadm - -# =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= -FROM build-go as sonobuoy-build - -USER root -RUN mkdir -p /usr/share/licenses/sonobuoy && \ - chown -R builder:builder /usr/share/licenses/sonobuoy - -ARG SONOBUOY_VERSION=0.53.2 -ARG SONOBUOY_SOURCE_URL="https://github.com/vmware-tanzu/sonobuoy/archive/refs/tags/v${SONOBUOY_VERSION}.tar.gz" - -ARG GOARCH -ARG SONOBUOY_BINARY_URL="https://github.com/vmware-tanzu/sonobuoy/releases/download/v${SONOBUOY_VERSION}/sonobuoy_${SONOBUOY_VERSION}_linux_${GOARCH}.tar.gz" - -USER builder -WORKDIR /home/builder/ -RUN mkdir sonobuoy && \ - curl -L "${SONOBUOY_SOURCE_URL}" -o sonobuoy_${SONOBUOY_VERSION}.tar.gz && \ - grep sonobuoy_${SONOBUOY_VERSION}.tar.gz \ - /src/hashes/sonobuoy | sha512sum --check - && \ - tar -xf sonobuoy_${SONOBUOY_VERSION}.tar.gz \ - --strip-components 1 -C sonobuoy && \ - rm sonobuoy_${SONOBUOY_VERSION}.tar.gz - -WORKDIR /home/builder/sonobuoy/ -RUN go mod vendor -RUN cp -p LICENSE /usr/share/licenses/sonobuoy && \ - /usr/libexec/tools/bottlerocket-license-scan \ - --clarify /src/clarify.toml \ - --spdx-data /usr/libexec/tools/spdx-data \ - --out-dir /usr/share/licenses/sonobuoy/vendor \ - go-vendor ./vendor -RUN curl -OL ${SONOBUOY_BINARY_URL} && \ - grep sonobuoy_${SONOBUOY_VERSION}_${GOOS}_${GOARCH}.tar.gz \ - /src/hashes/sonobuoy | sha512sum --check - && \ - tar -xf sonobuoy_${SONOBUOY_VERSION}_${GOOS}_${GOARCH}.tar.gz -C /tmp && \ - chmod 0755 /tmp/sonobuoy && \ - rm sonobuoy_${SONOBUOY_VERSION}_${GOOS}_${GOARCH}.tar.gz - -# =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= -FROM build-go as aws-iam-authenticator-build - -USER root -RUN mkdir -p /usr/share/licenses/aws-iam-authenticator && \ - chown -R builder:builder /usr/share/licenses/aws-iam-authenticator - -ARG AWS_IAM_AUTHENTICATOR_VERSION=0.5.3 -ARG AWS_IAM_AUTHENTICATOR_SHA512_SUM=430af9fd04b9a94205a485281fb668f5bc18cdac569de0232fa98e08ebb0e08a8d233537bd3373a5f1e53cf529bc2050aebc34a4a53c8b29a831070e34213210 -ARG AWS_IAM_AUTHENTICATOR_SOURCE_URL="https://cache.bottlerocket.aws/aws-iam-authenticator-${AWS_IAM_AUTHENTICATOR_VERSION}.tar.gz/${AWS_IAM_AUTHENTICATOR_SHA512_SUM}/aws-iam-authenticator-${AWS_IAM_AUTHENTICATOR_VERSION}.tar.gz" - -USER builder -WORKDIR /home/builder/ -RUN mkdir aws-iam-authenticator && \ - curl -L ${AWS_IAM_AUTHENTICATOR_SOURCE_URL} \ - -o aws-iam-authenticator_${AWS_IAM_AUTHENTICATOR_VERSION}.tar.gz && \ - grep aws-iam-authenticator_${AWS_IAM_AUTHENTICATOR_VERSION}.tar.gz \ - /src/hashes/aws-iam-authenticator | sha512sum --check - && \ - tar -xf aws-iam-authenticator_${AWS_IAM_AUTHENTICATOR_VERSION}.tar.gz \ - --strip-components 1 -C aws-iam-authenticator && \ - rm aws-iam-authenticator_${AWS_IAM_AUTHENTICATOR_VERSION}.tar.gz - -WORKDIR /home/builder/aws-iam-authenticator/ -RUN go mod vendor -RUN CGO_ENABLED=0 go build -mod=vendor -o /tmp/aws-iam-authenticator \ - ./cmd/aws-iam-authenticator - # =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= # Builds the EC2 resource agent image FROM scratch as ec2-resource-agent @@ -229,22 +87,20 @@ FROM public.ecr.aws/amazonlinux/amazonlinux:2 as vsphere-vm-resource-agent RUN yum install -y iproute && yum clean all -# Copy govc binary +# Copy govc COPY --from=build /usr/libexec/tools/govc /usr/local/bin/govc -# Copy govc licenses COPY --from=build /usr/share/licenses/govmomi /licenses/govmomi -# Copy kubeadm binary -COPY --from=kubernetes-build /tmp/kubeadm /usr/local/bin/kubeadm -# Copy kubeadm licenses -COPY --from=kubernetes-build /usr/share/licenses/kubernetes /licenses/kubernetes +# Copy kubeadm +COPY --from=tools /kubeadm /usr/local/bin/kubeadm +COPY --from=tools /licenses/kubernetes /licenses/kubernetes # Copy wireguard-tools binaries COPY --from=wireguard-build /usr/bin/wg /usr/bin/wg COPY --from=wireguard-build /usr/bin/wg-quick /usr/bin/wg-quick # Copy boringtun binary -COPY --from=build-src /src/bottlerocket/agents/bin/boringtun /usr/bin/boringtun +COPY --from=tools /boringtun /usr/bin/boringtun # Copy binary COPY --from=build-src /src/bottlerocket/agents/bin/vsphere-vm-resource-agent ./ @@ -257,18 +113,16 @@ ENTRYPOINT ["./vsphere-vm-resource-agent"] # Builds the EKS resource agent image FROM scratch as eks-resource-agent -# Copy eksctl binary -COPY --from=eksctl-build /tmp/eksctl /usr/bin/eksctl -# TODO - restore this with a fix for https://github.com/bottlerocket-os/bottlerocket-test-system/issues/288 -# Copy eksctl licenses -#COPY --from=eksctl-build /usr/share/licenses/eksctl /licenses/eksctl +# Copy eksctl +COPY --from=tools /eksctl /usr/bin/eksctl +COPY --from=tools /licenses/eksctl /licenses/eksctl # Copy CA certificates store COPY --from=build /etc/ssl /etc/ssl COPY --from=build /etc/pki /etc/pki -# Copy binary + +# Copy eks-resource-agent COPY --from=build-src /src/bottlerocket/agents/bin/eks-resource-agent ./ -# Copy licenses COPY --from=build-src /usr/share/licenses/testsys /licenses/testsys ENTRYPOINT ["./eks-resource-agent"] @@ -300,14 +154,14 @@ ENTRYPOINT ["./ecs-test-agent"] # Builds the Sonobuoy test agent image FROM public.ecr.aws/amazonlinux/amazonlinux:2 AS sonobuoy-test-agent ARG ARCH + # TODO remove unzip once aws-cli moves out RUN yum install -y unzip iproute && yum clean all ARG AWS_CLI_URL=https://awscli.amazonaws.com/awscli-exe-linux-${ARCH}.zip -# Copy aws-iam-authenticator binary -COPY --from=aws-iam-authenticator-build /tmp/aws-iam-authenticator /usr/bin/aws-iam-authenticator -# Copy aws-iam-authenticator licenses -COPY --from=aws-iam-authenticator-build /usr/share/licenses/aws-iam-authenticator /licenses/aws-iam-authenticator +# Copy aws-iam-authenticator +COPY --from=tools /aws-iam-authenticator /usr/bin/aws-iam-authenticator +COPY --from=tools /licenses/aws-iam-authenticator /licenses/aws-iam-authenticator # TODO move this out, get hashes, and attribute licenses # Download aws-cli @@ -317,21 +171,19 @@ RUN temp_dir="$(mktemp -d --suffix aws-cli)" && \ ${temp_dir}/aws/install && \ rm -rf ${temp_dir} -# Copy sonobuoy binary -COPY --from=sonobuoy-build /tmp/sonobuoy /usr/bin/sonobuoy -# Copy sonobuoy licenses -COPY --from=sonobuoy-build /usr/share/licenses/sonobuoy /licenses/sonobuoy +# Copy sonobuoy +COPY --from=tools /sonobuoy /usr/bin/sonobuoy +COPY --from=tools /licenses/sonobuoy /licenses/sonobuoy # Copy wireguard-tools COPY --from=wireguard-build /usr/bin/wg /usr/bin/wg COPY --from=wireguard-build /usr/bin/wg-quick /usr/bin/wg-quick # Copy boringtun -COPY --from=build-src /src/bottlerocket/agents/bin/boringtun /usr/bin/boringtun +COPY --from=tools /boringtun /usr/bin/boringtun -# Copy binary +# Copy sonobuoy-test-agent COPY --from=build-src /src/bottlerocket/agents/bin/sonobuoy-test-agent ./ -# Copy licenses COPY --from=build-src /usr/share/licenses/testsys /licenses/testsys ENTRYPOINT ["./sonobuoy-test-agent"] diff --git a/Makefile b/Makefile index 0f72345a..bcc6f84a 100644 --- a/Makefile +++ b/Makefile @@ -8,14 +8,16 @@ TESTSYS_BUILD_GOPROXY ?= direct BOTTLEROCKET_SDK_VERSION = v0.25.1 BOTTLEROCKET_SDK_ARCH = $(TESTSYS_BUILD_HOST_UNAME_ARCH) +BOTTLEROCKET_TOOLS_VERSION ?= v0.1.0 BUILDER_IMAGE = public.ecr.aws/bottlerocket/bottlerocket-sdk-$(BOTTLEROCKET_SDK_ARCH):$(BOTTLEROCKET_SDK_VERSION) +TOOLS_IMAGE ?= public.ecr.aws/bottlerocket/bottlerocket-test-tools:$(BOTTLEROCKET_TOOLS_VERSION) IMAGES = controller sonobuoy-test-agent ec2-resource-agent eks-resource-agent ecs-resource-agent \ migration-test-agent vsphere-vm-resource-agent ecs-test-agent .PHONY: build sdk-openssl example-test-agent example-resource-agent \ - images fetch integ-test show-variables cargo-deny $(IMAGES) + images fetch integ-test show-variables cargo-deny tools $(IMAGES) export DOCKER_BUILDKIT=1 export CARGO_HOME = $(TOP)/.cargo @@ -25,6 +27,8 @@ show-variables: $(info TESTSYS_BUILD_HOST_GOARCH=$(TESTSYS_BUILD_HOST_GOARCH)) $(info TESTSYS_BUILD_HOST_PLATFORM=$(TESTSYS_BUILD_HOST_PLATFORM)) $(info TESTSYS_BUILD_GOPROXY=$(TESTSYS_BUILD_GOPROXY)) + $(info BUILDER_IMAGE=$(BUILDER_IMAGE)) + $(info TOOLS_IMAGE=$(TOOLS_IMAGE)) @echo > /dev/null # Fetches crates from upstream @@ -75,19 +79,36 @@ controller: show-variables fetch docker build $(DOCKER_BUILD_FLAGS) \ --build-arg ARCH="$(TESTSYS_BUILD_HOST_UNAME_ARCH)" \ --build-arg BUILDER_IMAGE="$(BUILDER_IMAGE)" \ + --build-arg TOOLS_IMAGE="$(TOOLS_IMAGE)" \ --tag "controller" \ -f controller/Dockerfile . +# Build the 3rd-party tools that we use in our agent containers. +tools: + docker build $(DOCKER_BUILD_FLAGS) \ + --build-arg ARCH="$(TESTSYS_BUILD_HOST_UNAME_ARCH)" \ + --build-arg BUILDER_IMAGE="$(BUILDER_IMAGE)" \ + --build-arg GOARCH="$(TESTSYS_BUILD_HOST_GOARCH)" \ + --build-arg GOPROXY="$(TESTSYS_BUILD_GOPROXY)" \ + --network=host \ + -f ./tools/Dockerfile \ + -t bottlerocket-test-tools \ + -t $(TOOLS_IMAGE) \ + --progress=plain \ + ./tools + # Build the container image for a testsys agent eks-resource-agent ec2-resource-agent ecs-resource-agent vsphere-vm-resource-agent sonobuoy-test-agent migration-test-agent ecs-test-agent: show-variables fetch docker build $(DOCKER_BUILD_FLAGS) \ --build-arg ARCH="$(TESTSYS_BUILD_HOST_UNAME_ARCH)" \ --build-arg BUILDER_IMAGE="$(BUILDER_IMAGE)" \ + --build-arg TOOLS_IMAGE="$(TOOLS_IMAGE)" \ --build-arg GOARCH="$(TESTSYS_BUILD_HOST_GOARCH)" \ --build-arg GOPROXY="$(TESTSYS_BUILD_GOPROXY)" \ --network=host \ --target $@ \ --tag $@ \ + --progress=plain \ . # TESTSYS_SELFTEST_SKIP_IMAGE_BUILDS - If this is set to a non-zero-length string, the container images will will be diff --git a/clarify.toml b/clarify.toml index 6cc6b5b8..e626a583 100644 --- a/clarify.toml +++ b/clarify.toml @@ -69,97 +69,3 @@ license-files = [ { path = "LICENSE", hash = 0x001c7e6c }, { path = "third-party/chromium/LICENSE", hash = 0x9b209a1a }, ] - -[clarify."github.com/caarlos0/go-reddit/v3"] -expression = "MIT AND BSD-3-Clause" -license-files = [ - { path = "LICENSE", hash = 0x709001e3 }, -] - -[clarify."github.com/JeffAshton/win_pdh"] -expression = "BSD-3-Clause" -license-files = [ - { path = "LICENSE", hash = 0xb221dcc9 }, -] - -[clarify."github.com/daviddengcn/go-colortext"] -expression = "MIT AND BSD-3-Clause" -license-files = [ - { path = "LICENSE", hash = 0x9769fae1 }, -] - -[clarify."github.com/fatih/structtag"] -expression = "BSD-3-Clause" -license-files = [ - { path = "LICENSE", hash = 0x47b38429 }, -] - -[clarify."github.com/ghodss/yaml"] -expression = "MIT AND BSD-3-Clause" -license-files = [ - { path = "LICENSE", hash = 0xcdf3ae00 }, -] - -[clarify."github.com/go-bindata/go-bindata"] -expression = "CC0-1.0" -license-files = [ - { path = "LICENSE", hash = 0x393fafd6 }, -] - -[clarify."github.com/golangci/go-misc"] -expression = "BSD-3-Clause" -license-files = [ - { path = "LICENSE", hash = 0x14727692 }, -] - -[clarify."github.com/heketi/heketi"] -expression = "(Apache-2.0 OR LGPL-3.0-or-later) AND (LGPL-3.0-or-later OR GPL-2.0)" -license-files = [ - { path = "COPYING-GPLV2", hash = 0xeaa66bfd }, - { path = "COPYING-LGPLV3", hash = 0xf0bccb3a }, - { path = "LICENSE", hash = 0x3c4b96d1 }, - { path = "LICENSE-APACHE2", hash = 0x438c8616 }, -] - -[clarify."github.com/miekg/dns"] -expression = "BSD-3-Clause" -license-files = [ - { path = "COPYRIGHT", hash = 0xe41dd36c }, - { path = "LICENSE", hash = 0xfc8f12ff }, -] - -[clarify."github.com/sanathkr/yaml"] -expression = "BSD-3-Clause" -license-files = [ - { path = "LICENSE", hash = 0xcdf3ae00 }, -] - -[clarify."github.com/storageos/go-api"] -expression = "MIT" -license-files = [ - { path = "LICENCE", hash = 0x67a6861e }, -] -skip-files = [ - "licence.go", - "types/licence.go", -] - -[clarify."github.com/vartanbeno/go-reddit/v2"] -expression = "BSD-3-Clause" -license-files = [ - { path = "LICENSE", hash = 0x709001e3 }, -] - -[clarify."honnef.co/go/tools"] -expression = "MIT AND BSD-3-Clause" -license-files = [ - { path = "LICENSE", hash = 0xad378ed2 }, - { path = "LICENSE-THIRD-PARTY", hash = 0xac406a4d }, - { path = "go/ir/LICENSE", hash = 0xe656fb62 }, -] - -[clarify."sigs.k8s.io/yaml"] -expression = "MIT AND BSD-3-Clause" -license-files = [ - { path = "LICENSE", hash = 0xcdf3ae00 }, -] diff --git a/hashes/eksctl b/hashes/eksctl deleted file mode 100644 index fc26983e..00000000 --- a/hashes/eksctl +++ /dev/null @@ -1,3 +0,0 @@ -02d1c2ad2ea53e5b9c622eeec2cb7b4a1416a4abca07b81ba419e7df7b6c94b9dd3375413557c977ce9b8dc5d226dded611bf334e1abbfd8f0318fe86b8eb13f eksctl_0.82.0.tar.gz -154e4b808752a1223c6afee376b59fcfd79b1efdd8d2453366a223779c81cc102da6d2e8b02b4d12c3dac881f071368950dae0712efed078bf57291ac1efb1c9 eksctl_0.82.0_linux_amd64.tar.gz -31d2e17cbc52e35744e54c5c9e7014ac0b3cb328a2f7a038758ac1df2442ffc3737647944ddb6f20cab70c953555e0d79d14c1f88c537cb6a06f3c90e10c5e78 eksctl_0.82.0_linux_arm64.tar.gz diff --git a/hashes/sonobuoy b/hashes/sonobuoy deleted file mode 100644 index 94687e53..00000000 --- a/hashes/sonobuoy +++ /dev/null @@ -1,3 +0,0 @@ -84022c87f8442da52e84460c8fe7493cb2ea7b01ea635abba2c0ed0d9733ee0b876fd1a57cabcd84cf91999ec4732b483b3016830d0b29e4bb4a6389d960388d sonobuoy_0.53.2.tar.gz -5286c36a4e2931b37bc529ef018828257e21b2fd0cca731abf8c61ecaca06e969dcd540b8f74c2add0881344464ab01ad60c7521e302fe4dca2d9f93df471ad8 sonobuoy_0.53.2_linux_amd64.tar.gz -42de0221f153abe3d78f082546cc42c9ee5c0b9c35d91a82bf0e7f09b40dc3e5eb0e235302fb61448efc4be053ce0087bf02bee7ca3f757e966c06f311404b4e sonobuoy_0.53.2_linux_arm64.tar.gz diff --git a/tools/CHANGELOG.md b/tools/CHANGELOG.md new file mode 100644 index 00000000..0c627bc8 --- /dev/null +++ b/tools/CHANGELOG.md @@ -0,0 +1,21 @@ +# Changelog +This changelog documents changes to the `bottlerocket-test-tools` container image. + +The format is inspired by [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +Since this project is only a vessel for packaging a few binary tools, its adherence to +[Semantic Versioning](https://semver.org/spec/v2.0.0.html) is loose at best. + +## [0.1.0] - 2022-05-11 + +Initial version, x86_64-only for now. + +### Contents +- boringtun v0.4.0 +- eksctl 0.82.0 +- kubeadm v1.21.6 +- sonobuoy v0.53.2 + +[Unreleased]: https://github.com/bottlerocket-os/bottlerocket-test-system/compare/tools-v0.1.0...HEAD + +[0.1.0]: https://github.com/bottlerocket-os/bottlerocket-test-system/tree/tools-v0.1.0 diff --git a/tools/Dockerfile b/tools/Dockerfile new file mode 100644 index 00000000..4c7101e9 --- /dev/null +++ b/tools/Dockerfile @@ -0,0 +1,213 @@ +# syntax=docker/dockerfile:1.1.3-experimental +# This Dockfile contains separate targets for each testsys agent +# =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= +# Shared build stage used to build Go binaries. +ARG BUILDER_IMAGE +FROM ${BUILDER_IMAGE} as build-go +USER builder + +ARG GOARCH +ARG GOOS=linux +ARG GOROOT="/usr/libexec/go" +ARG GOPROXY + +ENV PATH="${GOROOT}/bin:${PATH}" +ENV GOPROXY="${GOPROXY}" + +ADD ./hashes /hashes +COPY ./clarify.toml clarify.toml + +# =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= +# Shared build stage used to build Rust binaries. +ARG BUILDER_IMAGE +FROM ${BUILDER_IMAGE} as build-rust +ARG ARCH + +USER root +RUN mkdir -p /home/builder && \ + chown -R builder:builder /home/builder +USER builder +WORKDIR /home/builder + +# We need these environment variables set for building the `openssl-sys` crate +ENV PKG_CONFIG_PATH=/${ARCH}-bottlerocket-linux-musl/sys-root/usr/lib/pkgconfig +ENV PKG_CONFIG_ALLOW_CROSS=1 +ENV CARGO_HOME=/home/builder/.cargo +ENV OPENSSL_STATIC=true + +# =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= +FROM build-rust as boringtun + +ARG ARCH + +# TODO get licenses for boringtun +RUN cargo install boringtun \ + --target ${ARCH}-bottlerocket-linux-musl \ + --root /home/builder + +# =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= +FROM build-go as eksctl-build + +USER root +RUN mkdir -p /usr/share/licenses/eksctl && \ + chown -R builder:builder /usr/share/licenses/eksctl + +ARG EKSCTL_VERSION=0.96.0 +ARG EKSCTL_SOURCE_URL="https://github.com/weaveworks/eksctl/archive/refs/tags/v${EKSCTL_VERSION}.tar.gz" + +ARG GOARCH +ARG EKSCTL_BINARY_URL="https://github.com/weaveworks/eksctl/releases/download/v${EKSCTL_VERSION}/eksctl_Linux_${GOARCH}.tar.gz" + +USER builder +WORKDIR /home/builder/ +RUN mkdir eksctl && curl -L ${EKSCTL_SOURCE_URL} \ + -o eksctl_${EKSCTL_VERSION}.tar.gz && \ + grep eksctl_${EKSCTL_VERSION}.tar.gz \ + /hashes/eksctl | sha512sum --check - && \ + tar -xf eksctl_${EKSCTL_VERSION}.tar.gz --strip-components 1 -C eksctl && \ + rm eksctl_${EKSCTL_VERSION}.tar.gz + +WORKDIR /home/builder/eksctl/ +# TODO - remove this workaround for a bad go.sum hash +# caused by https://github.com/blizzy78/varnamelen/issues/13#issuecomment-1079040934 +# these sed statement replace a hash that has changed. -i.bak makes these compatible with BSD/macOS sed. +RUN \ + sed -i 's,github.com/blizzy78/varnamelen v0.6.1 h1:kttPCLzXFa+0nt++Cw9fb7GrSSM4KkyIAoX/vXsbuqA=,github.com/blizzy78/varnamelen v0.6.1 h1:iYAU/3A6cpfRm2ZI0P/lece4jsc7GEbzsxTu+vBCChQ=,' go.sum ; \ + sed -i 's,github.com/blizzy78/varnamelen v0.6.1/go.mod h1:zy2Eic4qWqjrxa60jG34cfL0VXcSwzUrIx68eJPb4Q8=,github.com/blizzy78/varnamelen v0.6.1/go.mod h1:mGBHm+Uo4e8JnZEKHRoZgVEOQdSBdQfY/x+k4NAXBWA=,' go.sum +RUN go mod vendor +RUN cp -p LICENSE /usr/share/licenses/eksctl && \ + /usr/libexec/tools/bottlerocket-license-scan \ + --clarify /clarify.toml \ + --spdx-data /usr/libexec/tools/spdx-data \ + --out-dir /usr/share/licenses/eksctl/vendor \ + go-vendor ./vendor +RUN curl -L "${EKSCTL_BINARY_URL}" \ + -o eksctl_${EKSCTL_VERSION}_${GOOS}_${GOARCH}.tar.gz && \ + grep eksctl_${EKSCTL_VERSION}_${GOOS}_${GOARCH}.tar.gz \ + /hashes/eksctl | sha512sum --check - && \ + tar -xf eksctl_${EKSCTL_VERSION}_${GOOS}_${GOARCH}.tar.gz -C /tmp && \ + rm eksctl_${EKSCTL_VERSION}_${GOOS}_${GOARCH}.tar.gz + +# =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= +FROM build-go as kubernetes-build + +USER root +RUN mkdir -p /usr/share/licenses/kubernetes && \ + chown -R builder:builder /usr/share/licenses/kubernetes + +ARG K8S_VERSION=1.21.6 +ARG K8S_SOURCE_URL="https://github.com/kubernetes/kubernetes/archive/refs/tags/v${K8S_VERSION}.tar.gz" + +ARG GOARCH +ARG KUBEADM_BINARY_URL="https://dl.k8s.io/release/v${K8S_VERSION}/bin/linux/${GOARCH}/kubeadm" + +USER builder +WORKDIR /home/builder/ +RUN mkdir kubernetes && \ + curl -L "${K8S_SOURCE_URL}" -o kubernetes_${K8S_VERSION}.tar.gz && \ + grep kubernetes_${K8S_VERSION}.tar.gz \ + /hashes/kubernetes | sha512sum --check - && \ + tar -xf kubernetes_${K8S_VERSION}.tar.gz \ + --strip-components 1 -C kubernetes && \ + rm kubernetes_${K8S_VERSION}.tar.gz + +WORKDIR /home/builder/kubernetes/ +RUN go mod vendor +RUN cp -p LICENSE /usr/share/licenses/kubernetes && \ + /usr/libexec/tools/bottlerocket-license-scan \ + --clarify /clarify.toml \ + --spdx-data /usr/libexec/tools/spdx-data \ + --out-dir /usr/share/licenses/kubernetes/vendor \ + go-vendor ./vendor +RUN curl -L ${KUBEADM_BINARY_URL} \ + -o kubeadm_${K8S_VERSION}_${GOOS}_${GOARCH} && \ + grep kubeadm_${K8S_VERSION}_${GOOS}_${GOARCH} \ + /hashes/kubernetes | sha512sum --check - && \ + install -m 0755 kubeadm_${K8S_VERSION}_${GOOS}_${GOARCH} /tmp/kubeadm + +# =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= +FROM build-go as sonobuoy-build + +USER root +RUN mkdir -p /usr/share/licenses/sonobuoy && \ + chown -R builder:builder /usr/share/licenses/sonobuoy + +ARG SONOBUOY_VERSION=0.56.4 +ARG SONOBUOY_SOURCE_URL="https://github.com/vmware-tanzu/sonobuoy/archive/refs/tags/v${SONOBUOY_VERSION}.tar.gz" + +ARG GOARCH +ARG SONOBUOY_BINARY_URL="https://github.com/vmware-tanzu/sonobuoy/releases/download/v${SONOBUOY_VERSION}/sonobuoy_${SONOBUOY_VERSION}_linux_${GOARCH}.tar.gz" + +USER builder +WORKDIR /home/builder/ +RUN mkdir sonobuoy && \ + curl -L "${SONOBUOY_SOURCE_URL}" -o sonobuoy_${SONOBUOY_VERSION}.tar.gz && \ + grep sonobuoy_${SONOBUOY_VERSION}.tar.gz \ + /hashes/sonobuoy | sha512sum --check - && \ + tar -xf sonobuoy_${SONOBUOY_VERSION}.tar.gz \ + --strip-components 1 -C sonobuoy && \ + rm sonobuoy_${SONOBUOY_VERSION}.tar.gz + +WORKDIR /home/builder/sonobuoy/ +RUN go mod vendor +RUN cp -p LICENSE /usr/share/licenses/sonobuoy && \ + /usr/libexec/tools/bottlerocket-license-scan \ + --clarify /clarify.toml \ + --spdx-data /usr/libexec/tools/spdx-data \ + --out-dir /usr/share/licenses/sonobuoy/vendor \ + go-vendor ./vendor +RUN curl -OL ${SONOBUOY_BINARY_URL} && \ + grep sonobuoy_${SONOBUOY_VERSION}_${GOOS}_${GOARCH}.tar.gz \ + /hashes/sonobuoy | sha512sum --check - && \ + tar -xf sonobuoy_${SONOBUOY_VERSION}_${GOOS}_${GOARCH}.tar.gz -C /tmp && \ + chmod 0755 /tmp/sonobuoy && \ + rm sonobuoy_${SONOBUOY_VERSION}_${GOOS}_${GOARCH}.tar.gz + +# =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= +FROM build-go as aws-iam-authenticator-build + +USER root +RUN mkdir -p /usr/share/licenses/aws-iam-authenticator && \ + chown -R builder:builder /usr/share/licenses/aws-iam-authenticator + +ARG AWS_IAM_AUTHENTICATOR_VERSION=0.5.3 +ARG AWS_IAM_AUTHENTICATOR_SHA512_SUM=430af9fd04b9a94205a485281fb668f5bc18cdac569de0232fa98e08ebb0e08a8d233537bd3373a5f1e53cf529bc2050aebc34a4a53c8b29a831070e34213210 +ARG AWS_IAM_AUTHENTICATOR_SOURCE_URL="https://cache.bottlerocket.aws/aws-iam-authenticator-${AWS_IAM_AUTHENTICATOR_VERSION}.tar.gz/${AWS_IAM_AUTHENTICATOR_SHA512_SUM}/aws-iam-authenticator-${AWS_IAM_AUTHENTICATOR_VERSION}.tar.gz" + +USER builder +WORKDIR /home/builder/ +RUN mkdir aws-iam-authenticator && \ + curl -L ${AWS_IAM_AUTHENTICATOR_SOURCE_URL} \ + -o aws-iam-authenticator_${AWS_IAM_AUTHENTICATOR_VERSION}.tar.gz && \ + grep aws-iam-authenticator_${AWS_IAM_AUTHENTICATOR_VERSION}.tar.gz \ + /hashes/aws-iam-authenticator | sha512sum --check - && \ + tar -xf aws-iam-authenticator_${AWS_IAM_AUTHENTICATOR_VERSION}.tar.gz \ + --strip-components 1 -C aws-iam-authenticator && \ + rm aws-iam-authenticator_${AWS_IAM_AUTHENTICATOR_VERSION}.tar.gz + +WORKDIR /home/builder/aws-iam-authenticator/ +RUN go mod vendor +RUN CGO_ENABLED=0 go build -mod=vendor -o /tmp/aws-iam-authenticator \ + ./cmd/aws-iam-authenticator + +# =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= +# Package the binaries for use by other container image builds. +FROM scratch +# aws-iam-authenticator +COPY --from=aws-iam-authenticator-build /tmp/aws-iam-authenticator /aws-iam-authenticator +COPY --from=aws-iam-authenticator-build /usr/share/licenses/aws-iam-authenticator /licenses/aws-iam-authenticator + +#boringtun +COPY --from=boringtun /home/builder/bin/boringtun /boringtun + +# eksctl +COPY --from=eksctl-build /tmp/eksctl /eksctl +COPY --from=eksctl-build /usr/share/licenses/eksctl /licenses/eksctl + +# kubeadm +COPY --from=kubernetes-build /tmp/kubeadm /kubeadm +COPY --from=kubernetes-build /usr/share/licenses/kubernetes /licenses/kubernetes + +# sonobuoy +COPY --from=sonobuoy-build /tmp/sonobuoy /sonobuoy +COPY --from=sonobuoy-build /usr/share/licenses/sonobuoy /licenses/sonobuoy diff --git a/tools/README.md b/tools/README.md new file mode 100644 index 00000000..df1040e2 --- /dev/null +++ b/tools/README.md @@ -0,0 +1,23 @@ +# Bottlerocket Test Tools + +This container image provides a few of the binary tools that we need to use in Bottlerocket test agents. +Included are the binary and the license files, which we can use in downstream container image builds. + +## Building + +From the root of the git repository (i.e. `..`), run `make tools -e TOOLS_IMAGE=bottlerocket-test-tools:mytag`. + +- The built image will always be tagged as `bottlerocket-test-tools:latest`, but it will also be tagged with `TOOLS_IMAGE`. +- If you plan on pushing the image to a remote repo, you can set `TOOLS_IMAGE=my.repo.com/my-bottlerocket-test-tools:v0.1.0` (replace with your values). Then you can `docker push my.repo.com/my-bottlerocket-test-tools:v0.1.0`. + +## Using + +If you have a built version, whether it is local or you have pushed it, you can use it when building the rest of the images in this git repo: + +```shell +make eks-resource-agent -e TOOLS_IMAGE=TOOLS_IMAGE=my.repo.com/my-bottlerocket-test-tools:v0.1.0 +``` + +## Default + +By default, `TOOLS_IMAGE` will reference a specific version tag at `public.ecr.aws/bottlerocket/bottlerocket-test-tools`. diff --git a/tools/clarify.toml b/tools/clarify.toml new file mode 100644 index 00000000..caed207d --- /dev/null +++ b/tools/clarify.toml @@ -0,0 +1,99 @@ +[clarify."github.com/caarlos0/go-reddit/v3"] +expression = "MIT AND BSD-3-Clause" +license-files = [ + { path = "LICENSE", hash = 0x709001e3 }, +] + +[clarify."github.com/dave/dst"] +expression = "MIT AND BSD-3-Clause" +license-files = [ + { path = "LICENSE", hash = 0x8f288482}, +] + +[clarify."github.com/JeffAshton/win_pdh"] +expression = "BSD-3-Clause" +license-files = [ + { path = "LICENSE", hash = 0xb221dcc9 }, +] + +[clarify."github.com/daviddengcn/go-colortext"] +expression = "MIT AND BSD-3-Clause" +license-files = [ + { path = "LICENSE", hash = 0x9769fae1 }, +] + +[clarify."github.com/fatih/structtag"] +expression = "BSD-3-Clause" +license-files = [ + { path = "LICENSE", hash = 0x47b38429 }, +] + +[clarify."github.com/ghodss/yaml"] +expression = "MIT AND BSD-3-Clause" +license-files = [ + { path = "LICENSE", hash = 0xcdf3ae00 }, +] + +[clarify."github.com/go-bindata/go-bindata"] +expression = "CC0-1.0" +license-files = [ + { path = "LICENSE", hash = 0x393fafd6 }, +] + +[clarify."github.com/golangci/go-misc"] +expression = "BSD-3-Clause" +license-files = [ + { path = "LICENSE", hash = 0x14727692 }, +] + +[clarify."github.com/heketi/heketi"] +expression = "(Apache-2.0 OR LGPL-3.0-or-later) AND (LGPL-3.0-or-later OR GPL-2.0)" +license-files = [ + { path = "COPYING-GPLV2", hash = 0xeaa66bfd }, + { path = "COPYING-LGPLV3", hash = 0xf0bccb3a }, + { path = "LICENSE", hash = 0x3c4b96d1 }, + { path = "LICENSE-APACHE2", hash = 0x438c8616 }, +] + +[clarify."github.com/miekg/dns"] +expression = "BSD-3-Clause" +license-files = [ + { path = "COPYRIGHT", hash = 0xe41dd36c }, + { path = "LICENSE", hash = 0xfc8f12ff }, +] + +[clarify."github.com/sanathkr/yaml"] +expression = "BSD-3-Clause" +license-files = [ + { path = "LICENSE", hash = 0xcdf3ae00 }, +] + +[clarify."github.com/storageos/go-api"] +expression = "MIT" +license-files = [ + { path = "LICENCE", hash = 0x67a6861e }, +] +skip-files = [ + "licence.go", + "types/licence.go", +] + +[clarify."github.com/vartanbeno/go-reddit/v2"] +expression = "BSD-3-Clause" +license-files = [ + { path = "LICENSE", hash = 0x709001e3 }, +] + +[clarify."honnef.co/go/tools"] +expression = "MIT AND BSD-3-Clause" +license-files = [ + { path = "LICENSE", hash = 0xad378ed2 }, + { path = "LICENSE-THIRD-PARTY", hash = 0xac406a4d }, + { path = "go/ir/LICENSE", hash = 0xe656fb62 }, +] + +[clarify."sigs.k8s.io/yaml"] +expression = "MIT AND BSD-3-Clause" +license-files = [ + { path = "LICENSE", hash = 0xcdf3ae00 }, +] diff --git a/hashes/aws-iam-authenticator b/tools/hashes/aws-iam-authenticator similarity index 100% rename from hashes/aws-iam-authenticator rename to tools/hashes/aws-iam-authenticator diff --git a/tools/hashes/eksctl b/tools/hashes/eksctl new file mode 100644 index 00000000..c6e27715 --- /dev/null +++ b/tools/hashes/eksctl @@ -0,0 +1,3 @@ +2bbf700ef4d09d2d131d5a1bbade4613a42d53d6198c8da18b8ad1395ea2728dc3a9927fa44e8c71188d654ae5e0d8343b60edc28f36819d0a15b3d7a5178284 eksctl_0.96.0.tar.gz +2a097819c06c547c4df969f75e25939cf9682e971395be46caa9c29e52d2f88be7d8a3b3f1be0abb36a3eb73c3572d49090573ca4750db38ed44bd34bb54607b eksctl_0.96.0_linux_amd64.tar.gz +64c7c217e5e54e417476dae87b410384af9734d96762801dc7ae719111be65f21e112cd0ad0b6a89d6014523f67acea63c109d259b4c207bc5a593e8fbad8ebe eksctl_0.96.0_linux_arm64.tar.gz diff --git a/hashes/kubernetes b/tools/hashes/kubernetes similarity index 100% rename from hashes/kubernetes rename to tools/hashes/kubernetes diff --git a/tools/hashes/sonobuoy b/tools/hashes/sonobuoy new file mode 100644 index 00000000..c440c37a --- /dev/null +++ b/tools/hashes/sonobuoy @@ -0,0 +1,3 @@ +9f2b902b4e117590ff533eb4f8148cfeb89bd17a0dd19b6a900db52a946a35a2e0439fb66a4d65faa0b186942895059f8192ad43fa60037242f957cf2fd48bea sonobuoy_0.56.4.tar.gz +d91d16514b45854cf21243eacac43cd8601da8ab4bd91e97bcf3a14e001451ea1b4fdbbbdcf6fecda6804efcea900822825fe1a6dc4494b357bd833a82a6a729 sonobuoy_0.56.4_linux_amd64.tar.gz +e3c840653a46b00dad593dffa06ea4b1aabad83defd0f38aa468fa3476caac1d1c69df3f9729aa31ff9753f2e59dba81bda658a57fddd3f4d0bac7dc36032a03 sonobuoy_0.56.4_linux_arm64.tar.gz