feat(grouping): add grouping (like app) to resources (#156)

* feat(grouping): add grouping (like app) to resources

* pull determineApp into ApplicationUtils

swabbie-aws/src/main/kotlin/com/netflix/spinnaker/swabbie/aws/images/AmazonImageHandler.kt

@@ -19,7 +19,6 @@ package com.netflix.spinnaker.swabbie.aws.images
 import com.netflix.spectator.api.Registry
 import com.netflix.spinnaker.config.SwabbieProperties
 import com.netflix.spinnaker.kork.core.RetrySupport
-import com.netflix.spinnaker.moniker.frigga.FriggaReflectiveNamer
 import com.netflix.spinnaker.swabbie.*
 import com.netflix.spinnaker.swabbie.aws.edda.providers.AmazonImagesUsedByInstancesCache
 import com.netflix.spinnaker.swabbie.aws.edda.providers.AmazonLaunchConfigurationCache
@@ -33,6 +32,7 @@ import com.netflix.spinnaker.swabbie.orca.*
 import com.netflix.spinnaker.swabbie.repository.*
 import com.netflix.spinnaker.swabbie.tagging.TaggingService
 import com.netflix.spinnaker.swabbie.tagging.UpsertImageTagsRequest
+import com.netflix.spinnaker.swabbie.utils.ApplicationUtils
 import net.logstash.logback.argument.StructuredArguments.kv
 import org.springframework.context.ApplicationEventPublisher
 import org.springframework.stereotype.Component
@@ -58,7 +58,7 @@ class AmazonImageHandler(
   private val rules: List<Rule<AmazonImage>>,
   private val imageProvider: ResourceProvider<AmazonImage>,
   private val orcaService: OrcaService,
-  private val applicationsCaches: List<InMemoryCache<Application>>,
+  private val applicationUtils: ApplicationUtils,
   private val taggingService: TaggingService,
   private val taskTrackingRepository: TaskTrackingRepository,
   private val resourceUseTrackingRepository: ResourceUseTrackingRepository,
@@ -82,8 +82,8 @@ class AmazonImageHandler(
   override fun deleteResources(markedResources: List<MarkedResource>, workConfiguration: WorkConfiguration) {
     orcaService.orchestrate(
       OrchestrationRequest(
-        // resources are partitioned based on app name, so find app name from first resource
-        application = resolveApplicationOrNull(markedResources.first()) ?: "swabbie",
+        // resources are partitioned based on grouping, so find the app to use from first resource
+        application = applicationUtils.determineApp(markedResources.first().resource),
         job = listOf(
           OrcaJob(
             type = "deleteImage",
@@ -153,7 +153,7 @@ class AmazonImageHandler(
             tags = tags,
             cloudProvider = "aws",
             cloudProviderType = "aws",
-            application = resolveApplicationOrNull(resource) ?: "swabbie",
+            application = applicationUtils.determineApp(resource.resource),
             description = "$description for image ${resource.uniqueId()}"
           )
         )
@@ -172,11 +172,6 @@ class AmazonImageHandler(
     return taskIds
   }
 
-  private fun resolveApplicationOrNull(markedResource: MarkedResource): String? {
-    val appName = FriggaReflectiveNamer().deriveMoniker(markedResource).app ?: return null
-    return if (applicationsCaches.any { it.contains(appName) }) appName else null
-  }
-
   override fun handles(workConfiguration: WorkConfiguration): Boolean =
     workConfiguration.resourceType == IMAGE && workConfiguration.cloudProvider == AWS && !rules.isEmpty()
 
@@ -276,7 +271,6 @@ class AmazonImageHandler(
           )
         }
       }
-
   }
 
   /**

swabbie-aws/src/main/kotlin/com/netflix/spinnaker/swabbie/aws/loadbalancers/AmazonLoadBalancerHandler.kt

@@ -19,19 +19,17 @@ package com.netflix.spinnaker.swabbie.aws.loadbalancers
 import com.netflix.spectator.api.Registry
 import com.netflix.spinnaker.config.SwabbieProperties
 import com.netflix.spinnaker.kork.core.RetrySupport
-import com.netflix.spinnaker.moniker.frigga.FriggaReflectiveNamer
 import com.netflix.spinnaker.swabbie.*
 import com.netflix.spinnaker.swabbie.aws.autoscalinggroups.AmazonAutoScalingGroup
 import com.netflix.spinnaker.swabbie.events.Action
 import com.netflix.spinnaker.swabbie.exclusions.ResourceExclusionPolicy
-import com.netflix.spinnaker.swabbie.model.MarkedResource
-import com.netflix.spinnaker.swabbie.model.Rule
+import com.netflix.spinnaker.swabbie.model.*
 import com.netflix.spinnaker.swabbie.orca.OrcaService
-import com.netflix.spinnaker.swabbie.model.WorkConfiguration
 import com.netflix.spinnaker.swabbie.notifications.Notifier
 import com.netflix.spinnaker.swabbie.orca.OrcaJob
 import com.netflix.spinnaker.swabbie.orca.OrchestrationRequest
 import com.netflix.spinnaker.swabbie.repository.*
+import com.netflix.spinnaker.swabbie.utils.ApplicationUtils
 import org.springframework.context.ApplicationEventPublisher
 import org.springframework.stereotype.Component
 import java.time.Clock
@@ -55,7 +53,8 @@ class AmazonLoadBalancerHandler(
   private val serverGroupProvider: ResourceProvider<AmazonAutoScalingGroup>,
   private val orcaService: OrcaService,
   private val taskTrackingRepository: TaskTrackingRepository,
-  private val resourceUseTrackingRepository: ResourceUseTrackingRepository
+  private val resourceUseTrackingRepository: ResourceUseTrackingRepository,
+  private val applicationUtils: ApplicationUtils
 ) : AbstractResourceTypeHandler<AmazonElasticLoadBalancer>(
   registry,
   clock,
@@ -82,7 +81,7 @@ class AmazonLoadBalancerHandler(
           log.info("This load balancer is about to be deleted {}", markedResource)
           orcaService.orchestrate(
             OrchestrationRequest(
-              application = FriggaReflectiveNamer().deriveMoniker(markedResource).app,
+              application = applicationUtils.determineApp(resource),
               job = listOf(
                 OrcaJob(
                   type = "deleteLoadBalancer",
@@ -94,7 +93,7 @@ class AmazonLoadBalancerHandler(
                   )
                 )
               ),
-              description = "Cleaning up Load Balancer for ${FriggaReflectiveNamer().deriveMoniker(markedResource).app}"
+              description = "Cleaning up Load Balancer for ${resource.grouping?.value.orEmpty()}"
             )
           ).let { taskResponse ->
             taskTrackingRepository.add(
@@ -112,6 +111,7 @@ class AmazonLoadBalancerHandler(
     }
   }
 
+
   override fun softDeleteResources(markedResources: List<MarkedResource>, workConfiguration: WorkConfiguration) {
     TODO("not implemented")
   }

swabbie-aws/src/main/kotlin/com/netflix/spinnaker/swabbie/aws/model/AmazonResource.kt

@@ -16,7 +16,11 @@
 
 package com.netflix.spinnaker.swabbie.aws.model
 
+import com.netflix.frigga.ami.AppVersion
+import com.netflix.spinnaker.moniker.frigga.FriggaReflectiveNamer
 import com.netflix.spinnaker.swabbie.Dates
+import com.netflix.spinnaker.swabbie.model.Grouping
+import com.netflix.spinnaker.swabbie.model.GroupingType
 import com.netflix.spinnaker.swabbie.model.Resource
 import java.time.LocalDateTime
 import java.time.ZoneId
@@ -25,6 +29,23 @@ abstract class AmazonResource(
   creationDate: String? // ISO_LOCAL_DATE_TIME format
 ) : Resource() {
 
+  override val grouping: Grouping?
+    get() {
+      if (resourceType.contains("image", ignoreCase = true)) {
+        // Images have only packageName, not app, to group by
+        getTagValue("appversion")?.let { AppVersion.parseName(it)?.packageName }?.let { packageName ->
+          return Grouping(packageName, GroupingType.PACKAGE_NAME)
+        }
+        return null
+
+      } else {
+        FriggaReflectiveNamer().deriveMoniker(this).app?.let { app ->
+          Grouping(app, GroupingType.APPLICATION)
+        }
+        return null
+      }
+    }
+
   override val createTs: Long =
     if (!creationDate.isNullOrBlank())
       Dates.toLocalDateTime(creationDate!!)

swabbie-aws/src/main/kotlin/com/netflix/spinnaker/swabbie/aws/securitygroups/AmazonSecurityGroupHandler.kt

@@ -29,6 +29,7 @@ import com.netflix.spinnaker.swabbie.orca.OrcaJob
 import com.netflix.spinnaker.swabbie.orca.OrcaService
 import com.netflix.spinnaker.swabbie.orca.OrchestrationRequest
 import com.netflix.spinnaker.swabbie.repository.*
+import com.netflix.spinnaker.swabbie.utils.ApplicationUtils
 import org.springframework.context.ApplicationEventPublisher
 import org.springframework.stereotype.Component
 import java.time.Clock
@@ -52,7 +53,8 @@ class AmazonSecurityGroupHandler(
   private val securityGroupProvider: ResourceProvider<AmazonSecurityGroup>,
   private val orcaService: OrcaService,
   private val taskTrackingRepository: TaskTrackingRepository,
-  private val resourceUseTrackingRepository: ResourceUseTrackingRepository
+  private val resourceUseTrackingRepository: ResourceUseTrackingRepository,
+  private val applicationUtils: ApplicationUtils
 ) : AbstractResourceTypeHandler<AmazonSecurityGroup>(
   registry,
   clock,
@@ -78,7 +80,7 @@ class AmazonSecurityGroupHandler(
           log.info("This resource is about to be deleted {}", markedResource)
           orcaService.orchestrate(
             OrchestrationRequest(
-              application = FriggaReflectiveNamer().deriveMoniker(markedResource).app,
+              application = applicationUtils.determineApp(resource),
               job = listOf(
                 OrcaJob(
                   type = "deleteSecurityGroup",
@@ -91,7 +93,7 @@ class AmazonSecurityGroupHandler(
                   )
                 )
               ),
-              description = "Cleaning up Security Group for ${FriggaReflectiveNamer().deriveMoniker(markedResource).app}"
+              description = "Cleaning up Security Group for ${resource.grouping?.value.orEmpty()}"
             )
           ).let { taskResponse ->
             taskTrackingRepository.add(

swabbie-aws/src/test/java/com/netflix/spinnaker/swabbie/aws/images/AmazonImageHandlerTest.kt

@@ -40,6 +40,7 @@ import com.netflix.spinnaker.swabbie.orca.OrcaService
 import com.netflix.spinnaker.swabbie.orca.TaskResponse
 import com.netflix.spinnaker.swabbie.repository.*
 import com.netflix.spinnaker.swabbie.tagging.TaggingService
+import com.netflix.spinnaker.swabbie.utils.ApplicationUtils
 import com.nhaarman.mockito_kotlin.*
 import org.junit.jupiter.api.AfterEach
 import org.junit.jupiter.api.Assertions
@@ -63,7 +64,6 @@ object AmazonImageHandlerTest {
   private val imageProvider = mock<ResourceProvider<AmazonImage>>()
   private val instanceProvider = mock<ResourceProvider<AmazonInstance>>()
   private val launchConfigurationProvider = mock<ResourceProvider<AmazonLaunchConfiguration>>()
-  private val applicationsCache = mock<InMemoryCache<Application>>()
   private val taggingService = mock<TaggingService>()
   private val taskTrackingRepository = mock<TaskTrackingRepository>()
   private val resourceUseTrackingRepository = mock<ResourceUseTrackingRepository>()
@@ -73,6 +73,7 @@ object AmazonImageHandlerTest {
   }
   private val launchConfigurationCache = mock<InMemorySingletonCache<AmazonLaunchConfigurationCache>>()
   private val imagesUsedByinstancesCache = mock<InMemorySingletonCache<AmazonImagesUsedByInstancesCache>>()
+  private val applicationUtils = ApplicationUtils(emptyList())
 
   private val subject = AmazonImageHandler(
     clock = clock,
@@ -92,13 +93,13 @@ object AmazonImageHandlerTest {
     retrySupport = RetrySupport(),
     imageProvider = imageProvider,
     orcaService = orcaService,
-    applicationsCaches = listOf(applicationsCache),
     taggingService = taggingService,
     taskTrackingRepository = taskTrackingRepository,
     resourceUseTrackingRepository = resourceUseTrackingRepository,
     swabbieProperties = swabbieProperties,
     launchConfigurationCache = launchConfigurationCache,
-    imagesUsedByinstancesCache = imagesUsedByinstancesCache
+    imagesUsedByinstancesCache = imagesUsedByinstancesCache,
+    applicationUtils = applicationUtils
   )
 
   @BeforeEach

swabbie-core/src/main/kotlin/com/netflix/spinnaker/swabbie/AbstractResourceTypeHandler.kt

@@ -17,13 +17,11 @@
 package com.netflix.spinnaker.swabbie
 
 import com.google.common.collect.Lists
-import com.netflix.frigga.ami.AppVersion
 import com.netflix.spectator.api.Registry
 import com.netflix.spinnaker.config.SwabbieProperties
 import com.netflix.spinnaker.kork.core.RetrySupport
 import com.netflix.spinnaker.kork.lock.LockManager.LockOptions
 import com.netflix.spinnaker.kork.web.exceptions.NotFoundException
-import com.netflix.spinnaker.moniker.frigga.FriggaReflectiveNamer
 import com.netflix.spinnaker.swabbie.events.Action
 import com.netflix.spinnaker.swabbie.events.DeleteResourceEvent
 import com.netflix.spinnaker.swabbie.events.MarkResourceEvent
@@ -605,18 +603,18 @@ abstract class AbstractResourceTypeHandler<T : Resource>(
 
   /**
    * Partitions the list of marked resources to process:
-   * For convenience, partitions are grouped by relevance (here the derived app they are associated with)
+   * For convenience, partitions are grouped by relevance
    */
   fun partitionList(
     markedResources: List<MarkedResource>,
     configuration: WorkConfiguration
   ): List<List<MarkedResource>> {
     val partitions = mutableListOf<List<MarkedResource>>()
     markedResources.groupBy {
-      FriggaReflectiveNamer().deriveMoniker(it.resource).app //todo eb: images don't have an app
+      it.resource.grouping
     }.map {
-      Lists.partition(it.value, configuration.itemsProcessedBatchSize).forEach {
-        partitions.add(it)
+      Lists.partition(it.value, configuration.itemsProcessedBatchSize).forEach { partition ->
+        partitions.add(partition)
       }
     }
 
@@ -737,10 +735,10 @@ abstract class AbstractResourceTypeHandler<T : Resource>(
           } else {
             owner
           }
-          
+
           val notificationContext = mapOf(
             "resourceOwner" to finalOwner,
-            "application" to FriggaReflectiveNamer().deriveMoniker(resources.first()).app, //todo eb: this doesn't work for debs
+            "application" to resources.first().resource.grouping?.value.orEmpty(), //todo eb: this prob shouldn't be called app
             "resources" to resources.map { it.barebones() },
             "configuration" to workConfiguration,
             "resourceType" to workConfiguration.resourceType.formatted(),

swabbie-core/src/main/kotlin/com/netflix/spinnaker/swabbie/agents/ScheduledAgent.kt

@@ -30,6 +30,7 @@ import org.slf4j.LoggerFactory
 import java.time.*
 import java.time.temporal.Temporal
 import java.util.concurrent.Executor
+import java.util.concurrent.ExecutorService
 import java.util.concurrent.Executors
 import java.util.concurrent.TimeUnit
 import javax.annotation.PostConstruct
@@ -76,14 +77,25 @@ abstract class ScheduledAgent(
        } else {
          log.debug("Caches loaded.")
          scheduleSwabbie()
-         startupExecutorService.shutdown()
+         shutdown(startupExecutorService)
        }
      } catch (e: Exception) {
        log.error("Failed while waiting for cache to start in ${javaClass.simpleName}.", e)
      }
     }, 0, 5, TimeUnit.SECONDS)
   }
 
+  private fun shutdown(executorService: ExecutorService) {
+    executorService.shutdown()
+    try {
+      if (!executorService.awaitTermination(800, TimeUnit.MILLISECONDS)) {
+        executorService.shutdownNow()
+      }
+    } catch (e: InterruptedException) {
+      executorService.shutdownNow()
+    }
+  }
+
   private fun scheduleSwabbie() {
     executorService.scheduleWithFixedDelay({
       try {

swabbie-core/src/main/kotlin/com/netflix/spinnaker/swabbie/events/ResourceStateManager.kt

@@ -18,7 +18,6 @@ package com.netflix.spinnaker.swabbie.events
 
 import com.netflix.spectator.api.Id
 import com.netflix.spectator.api.Registry
-import com.netflix.spinnaker.moniker.frigga.FriggaReflectiveNamer
 import com.netflix.spinnaker.swabbie.InMemoryCache
 import com.netflix.spinnaker.swabbie.MetricsSupport
 import com.netflix.spinnaker.swabbie.model.*
@@ -28,6 +27,7 @@ import com.netflix.spinnaker.swabbie.repository.TaskTrackingRepository
 import com.netflix.spinnaker.swabbie.tagging.ResourceTagger
 import com.netflix.spinnaker.swabbie.tagging.TaggingService
 import com.netflix.spinnaker.swabbie.tagging.UpsertImageTagsRequest
+import com.netflix.spinnaker.swabbie.utils.ApplicationUtils
 import net.logstash.logback.argument.StructuredArguments
 import org.slf4j.LoggerFactory
 import org.springframework.beans.factory.annotation.Autowired
@@ -43,7 +43,7 @@ class ResourceStateManager(
   @Autowired(required = false) private val resourceTagger: ResourceTagger?,
   private val taggingService: TaggingService,
   private val taskTrackingRepository: TaskTrackingRepository,
-  private val applicationsCaches: List<InMemoryCache<Application>>
+  private val applicationUtils: ApplicationUtils
 ) : MetricsSupport(registry) {
 
   private val log = LoggerFactory.getLogger(javaClass)
@@ -186,7 +186,7 @@ class ResourceStateManager(
         tags = mapOf("expiration_time" to "never"),
         cloudProvider = "aws",
         cloudProviderType = "aws",
-        application = resolveApplicationOrNull(resource) ?: "swabbie",
+        application = applicationUtils.determineApp(resource.resource),
         description = "Setting `expiration_time` to `never` for image ${resource.uniqueId()}"
       )
     )
@@ -203,10 +203,6 @@ class ResourceStateManager(
     return taskId
   }
 
-  private fun resolveApplicationOrNull(markedResource: MarkedResource): String? {
-    val appName = FriggaReflectiveNamer().deriveMoniker(markedResource).app ?: return null
-    return if (applicationsCaches.any { it.contains(appName) }) appName else null
-  }
 }
 
 internal fun MarkedResource.typeAndName(): String {