From 69f4937df8389d6ae8acbb57014d655ed8e41b57 Mon Sep 17 00:00:00 2001 From: Misha Sugakov Date: Thu, 23 May 2024 19:00:03 +0200 Subject: [PATCH 1/7] Remove all but single collector pipeline --- ...pull-request.yaml => collector-build.yaml} | 9 +-- .tekton/collector-push.yaml | 79 ------------------- .tekton/collector-slim-pull-request.yaml | 79 ------------------- .tekton/collector-slim-push.yaml | 79 ------------------- 4 files changed, 3 insertions(+), 243 deletions(-) rename .tekton/{collector-pull-request.yaml => collector-build.yaml} (87%) delete mode 100644 .tekton/collector-push.yaml delete mode 100644 .tekton/collector-slim-pull-request.yaml delete mode 100644 .tekton/collector-slim-push.yaml diff --git a/.tekton/collector-pull-request.yaml b/.tekton/collector-build.yaml similarity index 87% rename from .tekton/collector-pull-request.yaml rename to .tekton/collector-build.yaml index af283c4216..1ba798ad55 100644 --- a/.tekton/collector-pull-request.yaml +++ b/.tekton/collector-build.yaml @@ -9,13 +9,13 @@ metadata: build.appstudio.redhat.com/target_branch: '{{target_branch}}' pipelinesascode.tekton.dev/max-keep-runs: "500" # TODO(ROX-21073): re-enable for all PR branches - pipelinesascode.tekton.dev/on-cel-expression: event == "pull_request" && (source_branch.contains("rhtap") || source_branch.contains("konflux")) + pipelinesascode.tekton.dev/on-cel-expression: (event == "push" && target_branch == "master") || (event == "pull_request" && (source_branch.contains("rhtap") || source_branch.contains("konflux"))) creationTimestamp: null labels: appstudio.openshift.io/application: acs appstudio.openshift.io/component: collector pipelines.appstudio.openshift.io/type: build - name: collector-on-pull-request + name: collector-build namespace: rh-acs-tenant spec: @@ -25,11 +25,10 @@ spec: - name: git-url value: '{{repo_url}}' - name: image-expires-after + # TODO(ROX-20230): make release images not expire. value: '13w' - name: output-image-repo value: quay.io/rhacs-eng/collector - - name: output-tag-suffix - value: '-latest' - name: path-context value: . - name: revision @@ -50,8 +49,6 @@ spec: value: 'true' - name: build-source-image value: 'true' - - name: build-target-stage - value: collector workspaces: - name: workspace diff --git a/.tekton/collector-push.yaml b/.tekton/collector-push.yaml deleted file mode 100644 index 3e8dde63b8..0000000000 --- a/.tekton/collector-push.yaml +++ /dev/null @@ -1,79 +0,0 @@ -apiVersion: tekton.dev/v1 -kind: PipelineRun - -metadata: - annotations: - build.appstudio.openshift.io/repo: https://github.com/stackrox/collector?rev={{revision}} - build.appstudio.redhat.com/commit_sha: '{{revision}}' - build.appstudio.redhat.com/target_branch: '{{target_branch}}' - pipelinesascode.tekton.dev/max-keep-runs: "500" - pipelinesascode.tekton.dev/on-event: '[push]' - pipelinesascode.tekton.dev/on-target-branch: '[master]' - creationTimestamp: null - labels: - appstudio.openshift.io/application: acs - appstudio.openshift.io/component: collector - pipelines.appstudio.openshift.io/type: build - name: collector-on-push - namespace: rh-acs-tenant - -spec: - params: - - name: dockerfile - value: collector/container/konflux.Dockerfile - - name: git-url - value: '{{repo_url}}' - - name: image-expires-after - # TODO(ROX-20230): make release images not expire. - value: '13w' - - name: output-image-repo - value: quay.io/rhacs-eng/collector - - name: output-tag-suffix - value: '-latest' - - name: path-context - value: . - - name: revision - value: '{{revision}}' - - name: rebuild - value: 'true' - # TODO(ROX-20234): Enable hermetic builds - # - name: hermetic - # value: "true" - # No language dependencies are required for collector image. - - name: prefetch-input - value: '' - - name: clone-depth - value: '0' - - name: clone-fetch-tags - value: 'true' - - name: clone-submodules - value: 'true' - - name: build-source-image - value: 'true' - - name: build-target-stage - value: collector - - workspaces: - - name: workspace - volumeClaimTemplate: - metadata: - creationTimestamp: null - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 5Gi - - name: git-auth - secret: - secretName: '{{ git_auth_secret }}' - - name: subscription-manager-activation-key - secret: - secretName: subscription-manager-activation-key - - # The pipeline regularly takes >1h to finish. - timeouts: - pipeline: 1h30m0s - - pipelineRef: - name: collector-component-pipeline diff --git a/.tekton/collector-slim-pull-request.yaml b/.tekton/collector-slim-pull-request.yaml deleted file mode 100644 index 99089b3d78..0000000000 --- a/.tekton/collector-slim-pull-request.yaml +++ /dev/null @@ -1,79 +0,0 @@ -apiVersion: tekton.dev/v1 -kind: PipelineRun - -metadata: - annotations: - build.appstudio.openshift.io/repo: https://github.com/stackrox/collector?rev={{revision}} - build.appstudio.redhat.com/commit_sha: '{{revision}}' - build.appstudio.redhat.com/pull_request_number: '{{pull_request_number}}' - build.appstudio.redhat.com/target_branch: '{{target_branch}}' - pipelinesascode.tekton.dev/max-keep-runs: "500" - # TODO(ROX-21073): re-enable for all PR branches - pipelinesascode.tekton.dev/on-cel-expression: event == "pull_request" && (source_branch.contains("rhtap") || source_branch.contains("konflux")) - creationTimestamp: null - labels: - appstudio.openshift.io/application: acs - appstudio.openshift.io/component: collector-slim - pipelines.appstudio.openshift.io/type: build - name: collector-slim-on-pull-request - namespace: rh-acs-tenant - -spec: - params: - - name: dockerfile - value: collector/container/konflux.Dockerfile - - name: git-url - value: '{{repo_url}}' - - name: image-expires-after - value: '13w' - - name: output-image-repo - value: quay.io/rhacs-eng/collector - - name: output-tag-suffix - value: '-slim' - - name: path-context - value: . - - name: revision - value: '{{revision}}' - - name: rebuild - value: 'true' - # TODO(ROX-20234): Enable hermetic builds - # - name: hermetic - # value: "true" - # No language dependencies are required for collector image. - - name: prefetch-input - value: '' - - name: clone-depth - value: '0' - - name: clone-fetch-tags - value: 'true' - - name: clone-submodules - value: 'true' - - name: build-source-image - value: 'true' - - name: build-target-stage - value: collector-slim - - workspaces: - - name: workspace - volumeClaimTemplate: - metadata: - creationTimestamp: null - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 5Gi - - name: git-auth - secret: - secretName: '{{ git_auth_secret }}' - - name: subscription-manager-activation-key - secret: - secretName: subscription-manager-activation-key - - # The pipeline regularly takes >1h to finish. - timeouts: - pipeline: 1h30m0s - - pipelineRef: - name: collector-component-pipeline diff --git a/.tekton/collector-slim-push.yaml b/.tekton/collector-slim-push.yaml deleted file mode 100644 index d66469eb67..0000000000 --- a/.tekton/collector-slim-push.yaml +++ /dev/null @@ -1,79 +0,0 @@ -apiVersion: tekton.dev/v1 -kind: PipelineRun - -metadata: - annotations: - build.appstudio.openshift.io/repo: https://github.com/stackrox/collector?rev={{revision}} - build.appstudio.redhat.com/commit_sha: '{{revision}}' - build.appstudio.redhat.com/target_branch: '{{target_branch}}' - pipelinesascode.tekton.dev/max-keep-runs: "500" - pipelinesascode.tekton.dev/on-event: '[push]' - pipelinesascode.tekton.dev/on-target-branch: '[master]' - creationTimestamp: null - labels: - appstudio.openshift.io/application: acs - appstudio.openshift.io/component: collector-slim - pipelines.appstudio.openshift.io/type: build - name: collector-slim-on-push - namespace: rh-acs-tenant - -spec: - params: - - name: dockerfile - value: collector/container/konflux.Dockerfile - - name: git-url - value: '{{repo_url}}' - - name: image-expires-after - # TODO(ROX-20230): make release images not expire. - value: '13w' - - name: output-image-repo - value: quay.io/rhacs-eng/collector - - name: output-tag-suffix - value: '-slim' - - name: path-context - value: . - - name: revision - value: '{{revision}}' - - name: rebuild - value: 'true' - # TODO(ROX-20234): Enable hermetic builds - # - name: hermetic - # value: "true" - # No language dependencies are required for collector image. - - name: prefetch-input - value: '' - - name: clone-depth - value: '0' - - name: clone-fetch-tags - value: 'true' - - name: clone-submodules - value: 'true' - - name: build-source-image - value: 'true' - - name: build-target-stage - value: collector-slim - - workspaces: - - name: workspace - volumeClaimTemplate: - metadata: - creationTimestamp: null - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 5Gi - - name: git-auth - secret: - secretName: '{{ git_auth_secret }}' - - name: subscription-manager-activation-key - secret: - secretName: subscription-manager-activation-key - - # The pipeline regularly takes >1h to finish. - timeouts: - pipeline: 1h30m0s - - pipelineRef: - name: collector-component-pipeline From 177582da01a3e7e6164bf599746fad6d29812806 Mon Sep 17 00:00:00 2001 From: Misha Sugakov Date: Thu, 23 May 2024 19:08:11 +0200 Subject: [PATCH 2/7] Add extra `-latest` and `-slim` tags to the output image so that the image is pushed to both expected locations. --- .tekton/collector-component-pipeline.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.tekton/collector-component-pipeline.yaml b/.tekton/collector-component-pipeline.yaml index 39bb8fc989..97653efc2e 100644 --- a/.tekton/collector-component-pipeline.yaml +++ b/.tekton/collector-component-pipeline.yaml @@ -262,6 +262,26 @@ spec: - name: source workspace: workspace + - name: apply-tags + params: + - name: IMAGE + value: $(tasks.build-container.results.IMAGE_URL) + - name: ADDITIONAL_TAGS + value: + - $(tasks.determine-image-tag.results.image-tag)-latest + - $(tasks.determine-image-tag.results.image-tag)-slim + runAfter: + - build-container + taskRef: + params: + - name: name + value: apply-tags + - name: bundle + value: quay.io/redhat-appstudio-tekton-catalog/task-apply-tags:0.1@sha256:29add9a49a2281a3755a9b580d2b9c5cb110231b14cccf8ade2fd7895a9b4b4a + - name: kind + value: task + resolver: bundles + - name: build-source-image params: - name: BINARY_IMAGE From f750d09ab9c2d2013e8af98f8a99b964e8cbaac8 Mon Sep 17 00:00:00 2001 From: Misha Sugakov Date: Thu, 23 May 2024 19:13:17 +0200 Subject: [PATCH 3/7] Fix init task --- .tekton/collector-component-pipeline.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.tekton/collector-component-pipeline.yaml b/.tekton/collector-component-pipeline.yaml index 97653efc2e..984c40851a 100644 --- a/.tekton/collector-component-pipeline.yaml +++ b/.tekton/collector-component-pipeline.yaml @@ -130,11 +130,10 @@ spec: - name: init params: - name: image-url - # We can't provide a real tag because it is not known at this time. - # We still provide a fake tag to the task to comply with the expected input. - # Because 'rebuild' is set to true, this has no effect. - # TODO(ROX-24116): Apply both Konflux-style and StackRox-style tags to containers - value: $(params.output-image-repo):fake-tag + # We can't provide a StackRox-style tag because it is not known at this time (requires cloning source, etc.) + # As a workaround, we still provide a unique tag that's based on a revision to this task to comply with its + # expected input. We later actually add this tag on a built image with apply-tags task. + value: $(params.output-image-repo):konflux-$(params.revision) - name: rebuild value: $(params.rebuild) taskRef: @@ -270,6 +269,7 @@ spec: value: - $(tasks.determine-image-tag.results.image-tag)-latest - $(tasks.determine-image-tag.results.image-tag)-slim + - konflux-$(params.revision) runAfter: - build-container taskRef: From c53ee3bb5af759010ca8f111b199e4460e5eb5f2 Mon Sep 17 00:00:00 2001 From: Misha Sugakov Date: Thu, 23 May 2024 19:19:24 +0200 Subject: [PATCH 4/7] Clean up Dockerfile --- collector/container/konflux.Dockerfile | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/collector/container/konflux.Dockerfile b/collector/container/konflux.Dockerfile index 213b929c20..149e178bd7 100644 --- a/collector/container/konflux.Dockerfile +++ b/collector/container/konflux.Dockerfile @@ -112,7 +112,7 @@ RUN /tmp/.konflux/scripts/subscription-manager-bro.sh register /mnt && \ rpm --root=/mnt --verbose -e --nodeps $(rpm --root=/mnt -qa 'curl' '*rpm*' '*dnf*' '*libsolv*' '*hawkey*' 'yum*') && \ rm -rf /mnt/var/cache/dnf /mnt/var/cache/yum -FROM scratch as collector-common +FROM scratch COPY --from=rpm-implanter-app /mnt / @@ -160,15 +160,6 @@ CMD collector-wrapper.sh \ --collection-method=$COLLECTION_METHOD \ --grpc-server=$GRPC_SERVER -FROM collector-common AS collector-slim - -LABEL \ - com.redhat.component="rhacs-collector-slim-container" \ - io.k8s.display-name="collector-slim" \ - name="rhacs-collector-slim-rhel8" - -FROM collector-common AS collector - LABEL \ com.redhat.component="rhacs-collector-container" \ io.k8s.display-name="collector" \ From f63074089e3c4e1d7591d9e1d0733f3a75eb56fd Mon Sep 17 00:00:00 2001 From: Misha Sugakov Date: Thu, 23 May 2024 19:24:46 +0200 Subject: [PATCH 5/7] Remove `output-tag-suffix` as now unnecessary --- .tekton/collector-component-pipeline.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.tekton/collector-component-pipeline.yaml b/.tekton/collector-component-pipeline.yaml index 984c40851a..82e89805ef 100644 --- a/.tekton/collector-component-pipeline.yaml +++ b/.tekton/collector-component-pipeline.yaml @@ -53,9 +53,6 @@ spec: - description: Output Image Repository name: output-image-repo type: string - - description: Suffix appended to the tag of the output image - name: output-tag-suffix - type: string - default: . description: Path to the source code of an application's component from where to build image. @@ -226,7 +223,7 @@ spec: - name: build-container params: - name: IMAGE - value: $(params.output-image-repo):$(tasks.determine-image-tag.results.image-tag)$(params.output-tag-suffix) + value: $(params.output-image-repo):$(tasks.determine-image-tag.results.image-tag) - name: DOCKERFILE value: $(params.dockerfile) - name: CONTEXT From a99ac1e8c3cc2e19d2714aac0ee4dd5523be2dc4 Mon Sep 17 00:00:00 2001 From: Misha Sugakov Date: Thu, 23 May 2024 19:30:51 +0200 Subject: [PATCH 6/7] Speed up builds by reserving more CPU --- .tekton/collector-build.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.tekton/collector-build.yaml b/.tekton/collector-build.yaml index 1ba798ad55..85c8d0372d 100644 --- a/.tekton/collector-build.yaml +++ b/.tekton/collector-build.yaml @@ -68,6 +68,26 @@ spec: secret: secretName: subscription-manager-activation-key + taskRunSpecs: + - pipelineTaskName: build-container + stepSpecs: + - name: build + # CPU requests are increased to speed up builds compared to the defaults. + # Defaults: https://github.com/redhat-appstudio/build-definitions/blob/main/task/buildah/0.1/buildah.yaml#L126 + computeResources: + limits: + cpu: 4 + requests: + cpu: 4 + - pipelineTaskName: clamav-scan + stepSpecs: + # Provision more CPU to speed up ClamAV scan compared to the defaults. + # https://github.com/redhat-appstudio/build-definitions/blob/main/task/clamav-scan/0.1/clamav-scan.yaml#L48 + - name: extract-and-scan-image + computeResources: + requests: + cpu: 1 + # The pipeline regularly takes >1h to finish. timeouts: pipeline: 1h30m0s From 6eb96a06d6039b93decb067707edbf09686ea08a Mon Sep 17 00:00:00 2001 From: Misha Sugakov Date: Mon, 27 May 2024 18:36:51 +0200 Subject: [PATCH 7/7] Mention when we can stop pushing `-slim` and `-latest` collectors --- .tekton/collector-component-pipeline.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.tekton/collector-component-pipeline.yaml b/.tekton/collector-component-pipeline.yaml index 82e89805ef..4da8a64604 100644 --- a/.tekton/collector-component-pipeline.yaml +++ b/.tekton/collector-component-pipeline.yaml @@ -264,9 +264,10 @@ spec: value: $(tasks.build-container.results.IMAGE_URL) - name: ADDITIONAL_TAGS value: + - konflux-$(params.revision) + # TODO(ROX-24402): don't publish multiple tags when ability to install Slim collector is gone. - $(tasks.determine-image-tag.results.image-tag)-latest - $(tasks.determine-image-tag.results.image-tag)-slim - - konflux-$(params.revision) runAfter: - build-container taskRef: