Feature/tir rm dyn data add helm status

CHANGELOG.md

@@ -11,6 +11,7 @@
 * Add preserve-digests cli option to skopeo copy command in CopyImageStage ([#1166](https://github.com/opendevstack/ods-jenkins-shared-library/issues/1166))
 * Allow registry/image:tag sources in CopyImageStage instead of directly falling back to internal registry ([#1177](https://github.com/opendevstack/ods-jenkins-shared-library/pull/1177))
 * Simplify successor management since now issue links are no longer inherited ([#1116](https://github.com/opendevstack/ods-jenkins-shared-library/pull/1116))
+* TIR - remove dynamic pod data, surface helm status and helm report tables reformatting ([#1143](https://github.com/opendevstack/ods-jenkins-shared-library/pull/1143))
 
 ### Fixed
 * Fix Tailor deployment drifts for D, Q envs ([#1055](https://github.com/opendevstack/ods-jenkins-shared-library/pull/1055))

src/org/ods/component/AbstractDeploymentStrategy.groovy

@@ -14,6 +14,9 @@ abstract class AbstractDeploymentStrategy implements IDeploymentStrategy {
     @Override
     abstract Map<String, List<PodData>> deploy()
 
+    // Fetches original kubernetes revisions of deployment resources.
+    //
+    // returns a two level map with keys resourceKind -> resourceName -> revision (int)
     protected Map<String, Map<String, Integer>> fetchOriginalVersions(Map<String, List<String>> deploymentResources) {
         def originalVersions = [:]
         deploymentResources.each { resourceKind, resourceNames ->

src/org/ods/component/HelmDeploymentStrategy.groovy

@@ -1,37 +1,34 @@
 package org.ods.component
 
-import groovy.json.JsonOutput
 import groovy.transform.TypeChecked
 import groovy.transform.TypeCheckingMode
 import org.ods.services.JenkinsService
 import org.ods.services.OpenShiftService
+import org.ods.util.HelmStatus
 import org.ods.util.ILogger
-import org.ods.util.PipelineSteps
+import org.ods.util.IPipelineSteps
 import org.ods.util.PodData
 
 class HelmDeploymentStrategy extends AbstractDeploymentStrategy {
 
     // Constructor arguments
-    private final Script script
     private final IContext context
     private final OpenShiftService openShift
     private final JenkinsService jenkins
     private final ILogger logger
-
+    private IPipelineSteps steps
     // assigned in constructor
-    private def steps
     private final RolloutOpenShiftDeploymentOptions options
 
     @SuppressWarnings(['AbcMetric', 'CyclomaticComplexity', 'ParameterCount'])
     HelmDeploymentStrategy(
-        def script,
+        IPipelineSteps steps,
         IContext context,
         Map<String, Object> config,
         OpenShiftService openShift,
         JenkinsService jenkins,
         ILogger logger
     ) {
-
         if (!config.selector) {
             config.selector = context.selector
         }
@@ -72,10 +69,9 @@ class HelmDeploymentStrategy extends AbstractDeploymentStrategy {
         if (!config.helmPrivateKeyCredentialsId) {
             config.helmPrivateKeyCredentialsId = "${context.cdProject}-helm-private-key"
         }
-        this.script = script
         this.context = context
         this.logger = logger
-        this.steps = new PipelineSteps(script)
+        this.steps = steps
 
         this.options = new RolloutOpenShiftDeploymentOptions(config)
         this.openShift = openShift
@@ -92,23 +88,19 @@ class HelmDeploymentStrategy extends AbstractDeploymentStrategy {
         // Tag images which have been built in this pipeline from cd project into target project
         retagImages(context.targetProject, getBuiltImages())
 
-        logger.info "Rolling out ${context.componentId} with HELM, selector: ${options.selector}"
+        logger.info("Rolling out ${context.componentId} with HELM, selector: ${options.selector}")
         helmUpgrade(context.targetProject)
-
-        def deploymentResources = openShift.getResourcesForComponent(
-            context.targetProject, DEPLOYMENT_KINDS, options.selector
-        )
-        logger.info("${this.class.name} -- DEPLOYMENT RESOURCES")
-        logger.info(
-            JsonOutput.prettyPrint(
-                JsonOutput.toJson(deploymentResources)))
+        HelmStatus helmStatus = openShift.helmStatus(context.targetProject, options.helmReleaseName)
+        if (logger.debugMode) {
+            def helmStatusMap = helmStatus.toMap()
+            logger.debug("${this.class.name} -- HELM STATUS: ${helmStatusMap}")
+        }
 
         // // FIXME: pauseRollouts is non trivial to determine!
         // // we assume that Helm does "Deployment" that should work for most
         // // cases since they don't have triggers.
         // metadataSvc.updateMetadata(false, deploymentResources)
-        def rolloutData = getRolloutData(deploymentResources) ?: [:]
-        logger.info(JsonOutput.prettyPrint(JsonOutput.toJson(rolloutData)))
+        def rolloutData = getRolloutData(helmStatus)
         return rolloutData
     }
 
@@ -124,7 +116,7 @@ class HelmDeploymentStrategy extends AbstractDeploymentStrategy {
                 options.helmValues['componentId'] = context.componentId
 
                 // we persist the original ones set from outside - here we just add ours
-                Map mergedHelmValues = [:]
+                Map<String, String> mergedHelmValues = [:]
                 mergedHelmValues << options.helmValues
 
                 // we add the global ones - this allows usage in subcharts
@@ -141,13 +133,14 @@ class HelmDeploymentStrategy extends AbstractDeploymentStrategy {
 
                 // deal with dynamic value files - which are env dependent
                 def mergedHelmValuesFiles = []
-                mergedHelmValuesFiles.addAll(options.helmValuesFiles)
 
-                options.helmEnvBasedValuesFiles.each { envValueFile ->
-                    mergedHelmValuesFiles << envValueFile.replace('.env',
-                        ".${context.environment}")
+                def  envConfigFiles = options.helmEnvBasedValuesFiles.collect {filenamePattern ->
+                    filenamePattern.replace('.env.', ".${context.environment}.")
                 }
 
+                mergedHelmValuesFiles.addAll(options.helmValuesFiles)
+                mergedHelmValuesFiles.addAll(envConfigFiles)
+
                 openShift.helmUpgrade(
                     targetProject,
                     options.helmReleaseName,
@@ -168,41 +161,61 @@ class HelmDeploymentStrategy extends AbstractDeploymentStrategy {
     // ]
     @TypeChecked(TypeCheckingMode.SKIP)
     private Map<String, List<PodData>> getRolloutData(
-        Map<String, List<String>> deploymentResources) {
+        HelmStatus helmStatus
+    ) {
+        // TODO This is still needed for several reasons, still pending refactoring in order to
+        //  remove the need for pod data
+        Map<String, List<PodData>> rolloutData = [:]
 
-        def rolloutData = [:]
-        deploymentResources.each { resourceKind, resourceNames ->
-            resourceNames.each { resourceName ->
-                def podData = []
+        Map<String, List<String>> deploymentKinds = helmStatus.resourcesByKind
+                .findAll { kind, res -> kind in DEPLOYMENT_KINDS }
+
+        deploymentKinds.each { kind, names ->
+            names.each { name ->
+                context.addDeploymentToArtifactURIs("${name}-deploymentMean",
+                    [
+                        type: 'helm',
+                        selector: options.selector,
+                        namespace: context.targetProject,
+                        chartDir: options.chartDir,
+                        helmReleaseName: options.helmReleaseName,
+                        helmEnvBasedValuesFiles: options.helmEnvBasedValuesFiles,
+                        helmValuesFiles: options.helmValuesFiles,
+                        helmValues: options.helmValues,
+                        helmDefaultFlags: options.helmDefaultFlags,
+                        helmAdditionalFlags: options.helmAdditionalFlags,
+                        helmStatus: helmStatus.toMap(),
+                    ]
+                )
+                def podDataContext = [
+                    "targetProject=${context.targetProject}",
+                    "selector=${options.selector}",
+                    "name=${name}",
+                ]
+                def msgPodsNotFound = "Could not find 'running' pod(s) for '${podDataContext.join(', ')}'"
+                List<PodData> podData = null
                 for (def i = 0; i < options.deployTimeoutRetries; i++) {
-                    podData = openShift.checkForPodData(context.targetProject, options.selector, resourceName)
-                    if (!podData.isEmpty()) {
+                    podData = openShift.checkForPodData(context.targetProject, options.selector, name)
+                    if (podData) {
                         break
                     }
-                    steps.echo("Could not find 'running' pod(s) with label '${options.selector}' - waiting")
+                    steps.echo("${msgPodsNotFound} - waiting")
                     steps.sleep(12)
                 }
-                context.addDeploymentToArtifactURIs("${resourceName}-deploymentMean",
-                    [
-                        'type': 'helm',
-                        'selector': options.selector,
-                        'chartDir': options.chartDir,
-                        'helmReleaseName': options.helmReleaseName,
-                        'helmEnvBasedValuesFiles': options.helmEnvBasedValuesFiles,
-                        'helmValuesFiles': options.helmValuesFiles,
-                        'helmValues': options.helmValues,
-                        'helmDefaultFlags': options.helmDefaultFlags,
-                        'helmAdditionalFlags': options.helmAdditionalFlags,
-                    ])
-                rolloutData["${resourceKind}/${resourceName}"] = podData
+                if (!podData) {
+                    throw new RuntimeException(msgPodsNotFound)
+                }
+                logger.debug("Helm podData for ${podDataContext.join(', ')}: ${podData}")
+
+                rolloutData["${kind}/${name}"] = podData
                 // TODO: Once the orchestration pipeline can deal with multiple replicas,
-                // update this to store multiple pod artifacts.
+                //  update this to store multiple pod artifacts.
                 // TODO: Potential conflict if resourceName is duplicated between
-                // Deployment and DeploymentConfig resource.
-                context.addDeploymentToArtifactURIs(resourceName, podData[0]?.toMap())
+                //  Deployment and DeploymentConfig resource.
+                context.addDeploymentToArtifactURIs(name, podData[0]?.toMap())
             }
         }
-        rolloutData
+        return rolloutData
     }
 
 }

src/org/ods/component/RolloutOpenShiftDeploymentStage.groovy

@@ -2,9 +2,10 @@ package org.ods.component
 
 import groovy.transform.TypeChecked
 import groovy.transform.TypeCheckingMode
-import org.ods.services.OpenShiftService
 import org.ods.services.JenkinsService
+import org.ods.services.OpenShiftService
 import org.ods.util.ILogger
+import org.ods.util.PipelineSteps
 
 @SuppressWarnings('ParameterCount')
 @TypeChecked
@@ -110,26 +111,27 @@ class RolloutOpenShiftDeploymentStage extends Stage {
         // We have to move everything here,
         // otherwise Jenkins will complain
         // about: "hudson.remoting.ProxyException: CpsCallableInvocation{methodName=fileExists, ..."
-        def isHelmDeployment = steps.fileExists(options.chartDir + '/Chart.yaml')
+        def isHelmDeployment = this.steps.fileExists(options.chartDir + '/Chart.yaml')
         logger.info("isHelmDeployment: ${isHelmDeployment}")
-        def isTailorDeployment = steps.fileExists(options.openshiftDir)
+        def isTailorDeployment = this.steps.fileExists(options.openshiftDir)
 
         if (isTailorDeployment && isHelmDeployment) {
-            steps.error("Must be either a Tailor based deployment or a Helm based deployment")
+            this.steps.error("Must be either a Tailor based deployment or a Helm based deployment")
             throw new IllegalStateException("Must be either a Tailor based deployment or a Helm based deployment")
         }
 
         // Use tailorDeployment in the following cases:
         // (1) We have an openshiftDir
         // (2) We do not have an openshiftDir but neither do we have an indication that it is Helm
+        def steps = new PipelineSteps(script)
         if (isTailorDeployment || (!isHelmDeployment && !isTailorDeployment)) {
-            deploymentStrategy = new TailorDeploymentStrategy(script, context, config, openShift, jenkins, logger)
+            deploymentStrategy = new TailorDeploymentStrategy(steps, context, config, openShift, jenkins, logger)
             String resourcePath = 'org/ods/component/RolloutOpenShiftDeploymentStage.deprecate-tailor.GString.txt'
-            def msg =steps.libraryResource(resourcePath)
+            def msg = this.steps.libraryResource(resourcePath)
             logger.warn(msg)
         }
         if (isHelmDeployment) {
-            deploymentStrategy = new HelmDeploymentStrategy(script, context, config, openShift, jenkins, logger)
+            deploymentStrategy = new HelmDeploymentStrategy(steps, context, config, openShift, jenkins, logger)
         }
         logger.info("deploymentStrategy: ${deploymentStrategy} -- ${deploymentStrategy.class.name}")
         return deploymentStrategy.deploy()

src/org/ods/component/TailorDeploymentStrategy.groovy

@@ -5,25 +5,24 @@ import groovy.transform.TypeCheckingMode
 import org.ods.services.JenkinsService
 import org.ods.services.OpenShiftService
 import org.ods.util.ILogger
-import org.ods.util.PipelineSteps
+import org.ods.util.IPipelineSteps
 import org.ods.util.PodData
 
 class TailorDeploymentStrategy extends AbstractDeploymentStrategy {
 
     // Constructor arguments
-    private final Script script
     private final IContext context
     private final OpenShiftService openShift
     private final JenkinsService jenkins
     private final ILogger logger
+    private final IPipelineSteps steps
 
     // assigned in constructor
-    private def steps
     private final RolloutOpenShiftDeploymentOptions options
 
     @SuppressWarnings(['AbcMetric', 'CyclomaticComplexity', 'ParameterCount'])
     TailorDeploymentStrategy(
-        def script,
+        IPipelineSteps steps,
         IContext context,
         Map<String, Object> config,
         OpenShiftService openShift,
@@ -67,10 +66,9 @@ class TailorDeploymentStrategy extends AbstractDeploymentStrategy {
         if (!config.containsKey('tailorParams')) {
             config.tailorParams = []
         }
-        this.script = script
         this.context = context
         this.logger = logger
-        this.steps = new PipelineSteps(script)
+        this.steps = steps
 
         this.options = new RolloutOpenShiftDeploymentOptions(config)
         this.openShift = openShift

src/org/ods/orchestration/phases/DeployOdsComponent.groovy

@@ -11,6 +11,7 @@ import org.ods.services.GitService
 import org.ods.orchestration.util.DeploymentDescriptor
 import org.ods.orchestration.util.MROPipelineUtil
 import org.ods.orchestration.util.Project
+import org.ods.util.PodData
 
 // Deploy ODS comnponent (code or service) to 'qa' or 'prod'.
 @TypeChecked
@@ -39,7 +40,11 @@ class DeployOdsComponent {
 
             DeploymentDescriptor deploymentDescriptor
             steps.dir(openShiftDir) {
+                // FIXME This is not correct, when deploying on test and prod envs, the deployment descriptor
+                //  read from file contains deploymentMean.namespace=xxx-dev, which doesn't match the target namespace,
+                //  which should end in either -test or -prod
                 deploymentDescriptor = DeploymentDescriptor.readFromFile(steps)
+                logger.debug("DeploymentDescriptor '${openShiftDir}': ${deploymentDescriptor.deployments}")
             }
             if (!repo.data.openshift.deployments) {
                 repo.data.openshift.deployments = [:]
@@ -54,24 +59,31 @@ class DeployOdsComponent {
                     Map deploymentMean = deployment.deploymentMean
                     logger.debug("Helm Config for ${deploymentName} -> ${deploymentMean}")
                     deploymentMean['repoId'] = repo.id
+                    deploymentMean['namespace'] = project.targetProject
 
                     applyTemplates(openShiftDir, deploymentMean)
 
                     def retries = project.environmentConfig?.openshiftRolloutTimeoutRetries ?: 10
-                    def podData = null
+                    def podDataContext = [
+                        "targetProject=${project.targetProject}",
+                        "selector=${deploymentMean.selector}",
+                        "name=${deploymentName}",
+                    ]
+                    def msgPodsNotFound = "Could not find 'running' pod(s) for '${podDataContext.join(', ')}'"
+                    List<PodData> podData = null
                     for (def i = 0; i < retries; i++) {
                         podData = os.checkForPodData(project.targetProject, deploymentMean.selector, deploymentName)
                         if (podData) {
                             break
                         }
-                        steps.echo("Could not find 'running' pod(s) with label '${deploymentMean.selector}' - waiting")
+                        steps.echo("${msgPodsNotFound} - waiting")
                         steps.sleep(12)
                     }
 
                     if (!podData) {
-                        throw new RuntimeException("Could not find 'running' pod(s) with label " +
-                            "'${deploymentMean.selector}'")
+                        throw new RuntimeException(msgPodsNotFound)
                     }
+                    logger.debug("Helm podData for '${podDataContext.join(', ')}': ${podData}")
 
                     // TODO: Once the orchestration pipeline can deal with multiple replicas,
                     // update this to deal with multiple pods.
@@ -227,6 +239,11 @@ class DeployOdsComponent {
                         deploymentMean.helmDefaultFlags,
                         deploymentMean.helmAdditionalFlags,
                         true)
+
+                    def helmStatus = os.helmStatus(project.targetProject, deploymentMean.helmReleaseName)
+                    def helmStatusMap = helmStatus.toMap()
+                    deploymentMean.helmStatus = helmStatusMap
+                    logger.debug("${this.class.name} -- HELM STATUS: ${helmStatusMap}")
                 }
             }
             jenkins.maybeWithPrivateKeyCredentials(secretName) { String pkeyFile ->