feat(web): add a config property that allows front50 to be the source…

… of truth for applications (#1825) * feat(web): add a config property that allows front50 to be the source of truth for applications * fix(web/test): mock exceptions properly in ApplicationServiceSpec by using >> { throw exception } instead of >> exception so spock doesn't try to coerce the exception into the return type of the method, with errors like: ApplicationServiceSpec > when UseFront50AsSourceOfTruth: #checkFront50 and application exists only in clouddriver, but front50 throws an exception > com.netflix.spinnaker.gate.ApplicationServiceSpec.when UseFront50AsSourceOfTruth: true and application exists only in clouddriver, but front50 throws an exception STANDARD_OUT --------------- Test when UseFront50AsSourceOfTruth: #checkFront50 and application exists only in clouddriver, but front50 throws an exception 2024-09-10 19:21:01.226 ERROR --- [pool-5-thread-1] c.n.s.gate.services.ApplicationService : [] Falling back to application cache org.codehaus.groovy.runtime.typehandling.GroovyCastException: Cannot cast object 'java.lang.Exception: fatal exception' with class 'java.lang.Exception' to class 'java.util.Map' at org.codehaus.groovy.runtime.typehandling.DefaultTypeTransformation.continueCastOnSAM(DefaultTypeTransformation.java:404) at org.codehaus.groovy.runtime.typehandling.DefaultTypeTransformation.continueCastOnNumber(DefaultTypeTransformation.java:315) at org.codehaus.groovy.runtime.typehandling.DefaultTypeTransformation.castToType(DefaultTypeTransformation.java:243) at org.spockframework.runtime.GroovyRuntimeUtil.coerce(GroovyRuntimeUtil.java:49) at org.spockframework.mock.response.ConstantResponseGenerator.doRespond(ConstantResponseGenerator.java:35) at org.spockframework.mock.response.SingleResponseGenerator.respond(SingleResponseGenerator.java:32) at org.spockframework.mock.response.ResponseGeneratorChain.respond(ResponseGeneratorChain.java:44) at org.spockframework.mock.runtime.MockInteraction.accept(MockInteraction.java:73) at org.spockframework.mock.runtime.MockInteractionDecorator.accept(MockInteractionDecorator.java:50) at org.spockframework.mock.runtime.InteractionScope$1.accept(InteractionScope.java:57) at org.spockframework.mock.runtime.MockController.handle(MockController.java:40) at org.spockframework.mock.runtime.JavaMockInterceptor.intercept(JavaMockInterceptor.java:86) at org.spockframework.mock.runtime.DynamicProxyMockInterceptorAdapter.invoke(DynamicProxyMockInterceptorAdapter.java:34) at jdk.proxy3/jdk.proxy3.$Proxy96.getApplication(Unknown Source) at com.netflix.spinnaker.gate.services.ApplicationService$Front50ApplicationRetriever$_callWithMdc_closure1.doCall(ApplicationService.groovy:533) at com.netflix.spinnaker.gate.services.ApplicationService$Front50ApplicationRetriever$_callWithMdc_closure1.call(ApplicationService.groovy) at com.netflix.spinnaker.security.AuthenticatedRequest.lambda$wrapCallableForPrincipal$0(AuthenticatedRequest.java:272) at com.netflix.spinnaker.gate.services.ApplicationService$Front50ApplicationRetriever.callWithMdc(ApplicationService.groovy:531) at com.netflix.spinnaker.gate.services.ApplicationService$Front50ApplicationRetriever.callWithMdc(ApplicationService.groovy) at com.netflix.spinnaker.gate.services.MdcWrappedCallable.call(MdcWrappedCallable.java:41) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:833) --------- Co-authored-by: David Byron <[email protected]>
spinnaker · Sep 12, 2024 · bd9b281 · bd9b281
1 parent b787931
commit bd9b281
Show file tree

Hide file tree

Showing 5 changed files with 485 additions and 52 deletions.
diff --git a/...src/main/groovy/com/netflix/spinnaker/gate/config/ApplicationConfigurationProperties.java b/...src/main/groovy/com/netflix/spinnaker/gate/config/ApplicationConfigurationProperties.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2022 Salesforce.com, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.gate.config;
+
+import lombok.Data;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+
+@Data
+@Configuration
+@ConfigurationProperties(prefix = "application")
+public class ApplicationConfigurationProperties {
+  /**
+   * defaults to false. When enabled, this will ignore all the applications that are only known to
+   * clouddriver but are missing from front50. This is done so that we can treat front50 as the
+   * source of truth for applications
+   */
+  private boolean useFront50AsSourceOfTruth;
+}
diff --git a/gate-web/src/main/groovy/com/netflix/spinnaker/gate/config/GateConfig.groovy b/gate-web/src/main/groovy/com/netflix/spinnaker/gate/config/GateConfig.groovy
@@ -73,7 +73,7 @@ import static retrofit.Endpoints.newFixedEndpoint
 @CompileStatic
 @Configuration
 @Slf4j
-@EnableConfigurationProperties([PipelineControllerConfigProperties.class])
+@EnableConfigurationProperties([PipelineControllerConfigProperties, ApplicationConfigurationProperties])
 @Import([PluginsAutoConfiguration, DeckPluginConfiguration, PluginWebConfiguration])
 class GateConfig {
 

diff --git a/gate-web/src/main/groovy/com/netflix/spinnaker/gate/services/ApplicationService.groovy b/gate-web/src/main/groovy/com/netflix/spinnaker/gate/services/ApplicationService.groovy
@@ -16,6 +16,7 @@
 
 package com.netflix.spinnaker.gate.services
 
+import com.netflix.spinnaker.gate.config.ApplicationConfigurationProperties
 import com.netflix.spinnaker.gate.config.Service
 import com.netflix.spinnaker.gate.config.ServiceConfiguration
 import com.netflix.spinnaker.gate.services.internal.ClouddriverService
@@ -37,25 +38,40 @@ import java.util.concurrent.ExecutionException
 import java.util.concurrent.ExecutorService
 import java.util.concurrent.Future
 import java.util.concurrent.atomic.AtomicReference
+import java.util.stream.Collectors
 
 @CompileStatic
 @Component
 @Slf4j
 class ApplicationService {
+  private ServiceConfiguration serviceConfiguration
+  private ClouddriverServiceSelector clouddriverServiceSelector
+  private Front50Service front50Service
+  private ExecutorService executorService
 
-  @Autowired
-  ServiceConfiguration serviceConfiguration
-
-  @Autowired
-  ClouddriverServiceSelector clouddriverServiceSelector
+  private AtomicReference<List<Map>> allApplicationsCache
+  private ApplicationConfigurationProperties applicationConfigurationProperties
 
   @Autowired
-  Front50Service front50Service
-
-  @Autowired
-  ExecutorService executorService
+  ApplicationService(
+    ServiceConfiguration serviceConfiguration,
+    ClouddriverServiceSelector clouddriverServiceSelector,
+    Front50Service front50Service,
+    ExecutorService executorService,
+    ApplicationConfigurationProperties applicationConfigurationProperties
+  ){
+    this.serviceConfiguration = serviceConfiguration
+    this.clouddriverServiceSelector = clouddriverServiceSelector
+    this.front50Service = front50Service
+    this.executorService = executorService
+    this.applicationConfigurationProperties = applicationConfigurationProperties
+    this.allApplicationsCache = new AtomicReference<>([])
+  }
 
-  private AtomicReference<List<Map>> allApplicationsCache = new AtomicReference<>([])
+  // used in tests
+  AtomicReference<List<Map>> getAllApplicationsCache(){
+    this.allApplicationsCache
+  }
 
   @Scheduled(fixedDelayString = '${services.front50.applicationRefreshIntervalMs:5000}')
   void refreshApplicationsCache() {
@@ -78,14 +94,19 @@ class ApplicationService {
    * @return Applications
    */
   List<Map<String, Object>> tick(boolean expandClusterNames = true) {
-    def applicationListRetrievers = buildApplicationListRetrievers(expandClusterNames)
-    List<Future<List<Map>>> futures = executorService.invokeAll(applicationListRetrievers)
     List<List<Map>> all
-    try {
-      all = futures.collect { it.get() }
-    } catch (ExecutionException ee) {
-      throw ee.cause
+    if (applicationConfigurationProperties.useFront50AsSourceOfTruth) {
+      all = getApplicationsWithFront50AsSourceOfTruth(expandClusterNames)
+    } else {
+      def applicationListRetrievers = buildApplicationListRetrievers(expandClusterNames)
+      List<Future<List<Map>>> futures = executorService.invokeAll(applicationListRetrievers)
+      try {
+        all = futures.collect { it.get() }
+      } catch (ExecutionException ee) {
+        throw ee.cause
+      }
     }
+
     List<Map> flat = (List<Map>) all?.flatten()?.toList()
     return mergeApps(flat, serviceConfiguration.getService('front50')).collect {
       it.attributes
@@ -97,19 +118,24 @@ class ApplicationService {
   }
 
   Map getApplication(String name, boolean expand) {
-    def applicationRetrievers = buildApplicationRetrievers(name, expand)
-    def futures = executorService.invokeAll(applicationRetrievers)
     List<Map> applications
-    try {
-      applications = (List<Map>) futures.collect { it.get() }
-    } catch (ExecutionException ee) {
-      throw ee.cause
-    }
-    if (!expand) {
-      def cachedApplication = allApplicationsCache.get().find { name.equalsIgnoreCase(it.name as String) }
-      if (cachedApplication) {
-        // ensure that `cachedApplication` attributes are overridden by any previously fetched metadata from front50
-        applications.add(0, cachedApplication)
+    if (applicationConfigurationProperties.useFront50AsSourceOfTruth) {
+      applications = getApplicationWithFront50AsSourceOfTruth(name, expand)
+    } else {
+      def applicationRetrievers = buildApplicationRetrievers(name, expand)
+      def futures = executorService.invokeAll(applicationRetrievers)
+      try {
+        applications = (List<Map>) futures.collect { it.get() }
+      } catch (ExecutionException ee) {
+        throw ee.cause
+      }
+
+      if (!expand) {
+        def cachedApplication = allApplicationsCache.get().find { name.equalsIgnoreCase(it.name as String) }
+        if (cachedApplication) {
+          // ensure that `cachedApplication` attributes are overridden by any previously fetched metadata from front50
+          applications.add(0, cachedApplication)
+        }
       }
     }
     List<Map> mergedApps = mergeApps(applications, serviceConfiguration.getService('front50'))
@@ -134,18 +160,25 @@ class ApplicationService {
   }
 
   private Collection<Callable<List<Map>>> buildApplicationListRetrievers(boolean expandClusterNames) {
-    return [
-        new Front50ApplicationListRetriever(front50Service, allApplicationsCache),
-        new ClouddriverApplicationListRetriever(clouddriverServiceSelector.select(), allApplicationsCache, expandClusterNames
-    )] as Collection<Callable<List<Map>>>
+    [
+      new Front50ApplicationListRetriever(front50Service, allApplicationsCache) as Callable<List<Map>>,
+      new ClouddriverApplicationListRetriever(
+        clouddriverServiceSelector.select(),
+        allApplicationsCache,
+        expandClusterNames) as Callable<List<Map>>
+    ]
   }
 
   private Collection<Callable<Map>> buildApplicationRetrievers(String applicationName, boolean expand) {
     def retrievers = [
       new Front50ApplicationRetriever(applicationName, front50Service, allApplicationsCache) as Callable<Map>
     ]
     if (expand) {
-      retrievers.add(new ClouddriverApplicationRetriever(applicationName, clouddriverServiceSelector.select()) as Callable<Map>)
+      retrievers.add(
+        new ClouddriverApplicationRetriever(
+          applicationName,
+          clouddriverServiceSelector.select()) as Callable<Map>
+      )
     }
     return retrievers
   }
@@ -228,6 +261,96 @@ class ApplicationService {
     }.flatten().toSet().sort().join(',')
   }
 
+  /**
+   * gets the applications from front50 and clouddriver, and only considers the applications
+   * returned from front50 to be the source of truth. All applications obtained from clouddriver
+   * that are not known to front50 are ignored
+   *
+   * @param expandClusterNames gets passed along to the ClouddriverApplicationListRetriever
+   * @return a list of type List<Map> that contains the responses from front50 and clouddriver
+   */
+  private List<List<Map>> getApplicationsWithFront50AsSourceOfTruth(boolean expandClusterNames) {
+    List<Map> front50Apps, clouddriverApps
+    try {
+      Future<List<Map>> front50future = executorService.submit(
+        new Front50ApplicationListRetriever(front50Service, allApplicationsCache) as Callable<List<Map>>
+      )
+
+      Future<List<Map>> clouddriverFuture = executorService.submit(
+        new ClouddriverApplicationListRetriever(
+          clouddriverServiceSelector.select(),
+          allApplicationsCache,
+          expandClusterNames) as Callable<List<Map>>
+      )
+      // capture the results from both front50 and clouddriver
+      front50Apps = front50future.get()
+      clouddriverApps = clouddriverFuture.get()
+    } catch (ExecutionException ee) {
+      log.error("error occurred when retrieving applications. Error: ", ee)
+      throw ee.cause
+    }
+
+    // get all app names from front50. This becomes our source of truth for known applications
+    Set<String> allFront50AppNames = front50Apps.stream()
+      .map({ it -> it.get("name") as String })
+      .collect(Collectors.toSet())
+
+    // iterate through all the results from clouddriver and only consider those applications that
+    // are known to front50
+    Set<Map> clouddriverAppsKnownToFront50 = clouddriverApps.stream()
+      .filter({ it ->
+        String clouddriverAppName = it.get("name")
+        allFront50AppNames.any { front50AppName -> front50AppName.equalsIgnoreCase(clouddriverAppName) }
+      })
+      .collect(Collectors.toSet())
+
+    List<List<Map>> all = []
+    all.add(front50Apps)
+    all.add(clouddriverAppsKnownToFront50.toList())
+    return all
+  }
+
+  /**
+   * gets the application from front50 first. If it does not contain the application, then processing
+   * stops as there is no need to check clouddriver for the application. Otherwise, depending on
+   * the parameter expand, clouddriver is queried for the application.
+   *
+   * @param name application name
+   * @param expand if true, then clouddriver is queried for the application only if front50 contains
+   *        the application
+   * @return a list of type Map that contains the responses from front50 and/or clouddriver
+   */
+  private List<Map> getApplicationWithFront50AsSourceOfTruth(String name, boolean expand) {
+    Map front50App, clouddriverApp
+    List<Map> result = []
+    try {
+      Future<Map> front50future = executorService.submit(
+        new Front50ApplicationRetriever(name, front50Service, allApplicationsCache) as Callable<Map>
+      )
+      // capture the result from front50
+      front50App = front50future.get()
+      if (front50App) {
+        result.add(front50App)
+        if (expand) {
+          Future<Map> clouddriverFuture = executorService.submit(
+            new ClouddriverApplicationRetriever(name, clouddriverServiceSelector.select()) as Callable<Map>
+          )
+          // capture the result from clouddriver
+          clouddriverApp = clouddriverFuture.get()
+          String clouddriverAppName = clouddriverApp.get("name")
+          if (clouddriverAppName?.equalsIgnoreCase(front50App.get("name") as String)) {
+            result.add(clouddriverApp)
+          }
+        }
+      }
+    } catch (ExecutionException ee) {
+      log.error("error occurred when retrieving application: ${name}. Error: ", ee)
+      throw ee.cause
+    }
+
+    return result
+  }
+
   static class Front50ApplicationListRetriever implements Callable<List<Map>> {
     private final Front50Service front50
     private final AtomicReference<List<Map>> allApplicationsCache