feat(lastSeen): adding last seen info to markedResource (#126)

swabbie-aws/src/main/kotlin/com/netflix/spinnaker/swabbie/aws/autoscalinggroups/AmazonAutoScalingGroupHandler.kt

@@ -35,10 +35,7 @@ import com.netflix.spinnaker.swabbie.notifications.Notifier
 import com.netflix.spinnaker.swabbie.orca.OrcaJob
 import com.netflix.spinnaker.swabbie.orca.OrcaService
 import com.netflix.spinnaker.swabbie.orca.OrchestrationRequest
-import com.netflix.spinnaker.swabbie.repository.ResourceStateRepository
-import com.netflix.spinnaker.swabbie.repository.ResourceTrackingRepository
-import com.netflix.spinnaker.swabbie.repository.TaskCompleteEventInfo
-import com.netflix.spinnaker.swabbie.repository.TaskTrackingRepository
+import com.netflix.spinnaker.swabbie.repository.*
 import org.springframework.context.ApplicationEventPublisher
 import org.springframework.stereotype.Component
 import java.time.Clock
@@ -60,7 +57,8 @@ class AmazonAutoScalingGroupHandler(
   private val rules: List<Rule<AmazonAutoScalingGroup>>,
   private val serverGroupProvider: ResourceProvider<AmazonAutoScalingGroup>,
   private val orcaService: OrcaService,
-  private val taskTrackingRepository: TaskTrackingRepository
+  private val taskTrackingRepository: TaskTrackingRepository,
+  private val resourceUseTrackingRepository: ResourceUseTrackingRepository
 ) : AbstractResourceTypeHandler<AmazonAutoScalingGroup>(
   registry,
   clock,
@@ -72,7 +70,8 @@ class AmazonAutoScalingGroupHandler(
   notifiers,
   applicationEventPublisher,
   lockingService,
-  retrySupport
+  retrySupport,
+  resourceUseTrackingRepository
 ) {
 
   override fun softDeleteResources(markedResources: List<MarkedResource>, workConfiguration: WorkConfiguration) {

swabbie-aws/src/main/kotlin/com/netflix/spinnaker/swabbie/aws/images/AmazonImageHandler.kt

@@ -73,7 +73,8 @@ class AmazonImageHandler(
   notifiers,
   applicationEventPublisher,
   lockingService,
-  retrySupport
+  retrySupport,
+  resourceUseTrackingRepository
 ) {
 
   override fun deleteResources(markedResources: List<MarkedResource>, workConfiguration: WorkConfiguration) {
@@ -368,11 +369,12 @@ class AmazonImageHandler(
       log.info("Bypassing seen in use check, since `swabbieProperties.outOfUseThresholdDays` is 0")
       return
     }
-    val unseenImages: Map<String, String> = resourceUseTrackingRepository
+    val usedImages = resourceUseTrackingRepository.getUsed()
+    val unusedAndTracked: Map<String, String> = resourceUseTrackingRepository
       .getUnused()
       .map {
-        it.resourceIdentifier to it.usedByResourceIdentifier
-      }.toMap()
+        it.resourceId to it.usedByResourceId
+     }.toMap()
 
     images.filter {
       NAIVE_EXCLUSION !in it.details &&
@@ -381,10 +383,8 @@ class AmazonImageHandler(
         HAS_SIBLINGS_IN_OTHER_ACCOUNTS !in it.details &&
         IS_BASE_OR_ANCESTOR !in it.details
     }.forEach { image ->
-      if (!unseenImages.containsKey(image.imageId)) {
-        // Image is not in list of unseen, so we've seen it recently
-        //  set that on image
-        // If there are no unseen images, this will set the key on every image.
+      if (!unusedAndTracked.containsKey(image.imageId) && usedImages.contains(image.imageId)) {
+        // Image has not been unused for the outOfUseThreshold, and has been seen before
         image.set(SEEN_IN_USE_RECENTLY, true)
         log.debug("Image {} ({}) has been SEEN_IN_USE_RECENTLY", kv("imageId", image.imageId), image.name)
       }

swabbie-aws/src/main/kotlin/com/netflix/spinnaker/swabbie/aws/loadbalancers/AmazonLoadBalancerHandler.kt

@@ -30,10 +30,7 @@ import com.netflix.spinnaker.swabbie.model.WorkConfiguration
 import com.netflix.spinnaker.swabbie.notifications.Notifier
 import com.netflix.spinnaker.swabbie.orca.OrcaJob
 import com.netflix.spinnaker.swabbie.orca.OrchestrationRequest
-import com.netflix.spinnaker.swabbie.repository.ResourceStateRepository
-import com.netflix.spinnaker.swabbie.repository.ResourceTrackingRepository
-import com.netflix.spinnaker.swabbie.repository.TaskCompleteEventInfo
-import com.netflix.spinnaker.swabbie.repository.TaskTrackingRepository
+import com.netflix.spinnaker.swabbie.repository.*
 import org.springframework.context.ApplicationEventPublisher
 import org.springframework.stereotype.Component
 import java.time.Clock
@@ -55,7 +52,8 @@ class AmazonLoadBalancerHandler(
   private val loadBalancerProvider: ResourceProvider<AmazonElasticLoadBalancer>,
   private val serverGroupProvider: ResourceProvider<AmazonAutoScalingGroup>,
   private val orcaService: OrcaService,
-  private val taskTrackingRepository: TaskTrackingRepository
+  private val taskTrackingRepository: TaskTrackingRepository,
+  private val resourceUseTrackingRepository: ResourceUseTrackingRepository
 ) : AbstractResourceTypeHandler<AmazonElasticLoadBalancer>(
   registry,
   clock,
@@ -67,7 +65,8 @@ class AmazonLoadBalancerHandler(
   notifiers,
   applicationEventPublisher,
   lockingService,
-  retrySupport
+  retrySupport,
+  resourceUseTrackingRepository
 ) {
   override fun deleteResources(
     markedResources: List<MarkedResource>,

swabbie-aws/src/main/kotlin/com/netflix/spinnaker/swabbie/aws/securitygroups/AmazonSecurityGroupHandler.kt

@@ -27,10 +27,7 @@ import com.netflix.spinnaker.swabbie.notifications.Notifier
 import com.netflix.spinnaker.swabbie.orca.OrcaJob
 import com.netflix.spinnaker.swabbie.orca.OrcaService
 import com.netflix.spinnaker.swabbie.orca.OrchestrationRequest
-import com.netflix.spinnaker.swabbie.repository.ResourceStateRepository
-import com.netflix.spinnaker.swabbie.repository.ResourceTrackingRepository
-import com.netflix.spinnaker.swabbie.repository.TaskCompleteEventInfo
-import com.netflix.spinnaker.swabbie.repository.TaskTrackingRepository
+import com.netflix.spinnaker.swabbie.repository.*
 import org.springframework.context.ApplicationEventPublisher
 import org.springframework.stereotype.Component
 import java.time.Clock
@@ -52,7 +49,8 @@ class AmazonSecurityGroupHandler(
   private val rules: List<Rule<AmazonSecurityGroup>>,
   private val securityGroupProvider: ResourceProvider<AmazonSecurityGroup>,
   private val orcaService: OrcaService,
-  private val taskTrackingRepository: TaskTrackingRepository
+  private val taskTrackingRepository: TaskTrackingRepository,
+  private val resourceUseTrackingRepository: ResourceUseTrackingRepository
 ) : AbstractResourceTypeHandler<AmazonSecurityGroup>(
   registry,
   clock,
@@ -64,7 +62,8 @@ class AmazonSecurityGroupHandler(
   notifiers,
   applicationEventPublisher,
   lockingService,
-  retrySupport
+  retrySupport,
+  resourceUseTrackingRepository
 ) {
   override fun deleteResources(
     markedResources: List<MarkedResource>,

swabbie-aws/src/test/java/com/netflix/spinnaker/swabbie/aws/autoscalinggroups/AmazonAutoScalingGroupHandlerTest.kt

@@ -33,6 +33,7 @@ import com.netflix.spinnaker.swabbie.orca.TaskDetailResponse
 import com.netflix.spinnaker.swabbie.orca.TaskResponse
 import com.netflix.spinnaker.swabbie.repository.ResourceStateRepository
 import com.netflix.spinnaker.swabbie.repository.ResourceTrackingRepository
+import com.netflix.spinnaker.swabbie.repository.ResourceUseTrackingRepository
 import com.netflix.spinnaker.swabbie.repository.TaskTrackingRepository
 import com.nhaarman.mockito_kotlin.*
 import org.junit.jupiter.api.AfterEach
@@ -56,6 +57,7 @@ object AmazonAutoScalingGroupHandlerTest {
   private val applicationEventPublisher = mock<ApplicationEventPublisher>()
   private val lockingService = Optional.empty<LockingService>()
   private val orcaService = mock<OrcaService>()
+  private val resourceUseTrackingRepository = mock<ResourceUseTrackingRepository>()
 
   private val subject = AmazonAutoScalingGroupHandler(
     clock = clock,
@@ -75,7 +77,8 @@ object AmazonAutoScalingGroupHandlerTest {
     lockingService = lockingService,
     retrySupport = RetrySupport(),
     serverGroupProvider = serverGroupProvider,
-    orcaService = orcaService
+    orcaService = orcaService,
+    resourceUseTrackingRepository = resourceUseTrackingRepository
   )
 
   @BeforeEach

swabbie-aws/src/test/java/com/netflix/spinnaker/swabbie/aws/images/AmazonImageHandlerTest.kt

@@ -471,6 +471,7 @@ object AmazonImageHandlerTest {
   fun `should not soft delete if the images have been seen recently`() {
     whenever(resourceUseTrackingRepository.getUnused()) doReturn
       emptyList<LastSeenInfo>()
+    whenever(resourceUseTrackingRepository.getUsed()) doReturn setOf("ami-123")
 
     val fifteenDaysAgo = clock.instant().minusSeconds(15 * 24 * 60 * 60).toEpochMilli()
     val thirteenDaysAgo = clock.instant().minusSeconds(13 * 24 * 60 * 60).toEpochMilli()

swabbie-core/src/main/kotlin/com/netflix/spinnaker/swabbie/AbstractResourceTypeHandler.kt

@@ -31,19 +31,11 @@ import com.netflix.spinnaker.swabbie.events.UnMarkResourceEvent
 import com.netflix.spinnaker.swabbie.events.formatted
 import com.netflix.spinnaker.swabbie.exclusions.ResourceExclusionPolicy
 import com.netflix.spinnaker.swabbie.exclusions.shouldExclude
-import com.netflix.spinnaker.swabbie.model.AlwaysCleanRule
-import com.netflix.spinnaker.swabbie.model.MarkedResource
-import com.netflix.spinnaker.swabbie.model.NotificationInfo
-import com.netflix.spinnaker.swabbie.model.OnDemandMarkData
-import com.netflix.spinnaker.swabbie.model.Resource
-import com.netflix.spinnaker.swabbie.model.ResourceEvaluation
-import com.netflix.spinnaker.swabbie.model.ResourceState
-import com.netflix.spinnaker.swabbie.model.Rule
-import com.netflix.spinnaker.swabbie.model.Summary
-import com.netflix.spinnaker.swabbie.model.WorkConfiguration
+import com.netflix.spinnaker.swabbie.model.*
 import com.netflix.spinnaker.swabbie.notifications.Notifier
 import com.netflix.spinnaker.swabbie.repository.ResourceStateRepository
 import com.netflix.spinnaker.swabbie.repository.ResourceTrackingRepository
+import com.netflix.spinnaker.swabbie.repository.ResourceUseTrackingRepository
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
 import org.springframework.context.ApplicationEventPublisher
@@ -68,7 +60,8 @@ abstract class AbstractResourceTypeHandler<T : Resource>(
   private val notifiers: List<Notifier>,
   private val applicationEventPublisher: ApplicationEventPublisher,
   private val lockingService: Optional<LockingService>,
-  private val retrySupport: RetrySupport
+  private val retrySupport: RetrySupport,
+  private val resourceUseTrackingRepository: ResourceUseTrackingRepository
 ) : ResourceTypeHandler<T>, MetricsSupport(registry) {
   protected val log: Logger = LoggerFactory.getLogger(javaClass)
   private val resourceOwnerField: String = "swabbieResourceOwner"
@@ -173,29 +166,16 @@ abstract class AbstractResourceTypeHandler<T : Resource>(
     val resource = getCandidate(resourceId, "", workConfiguration)
       ?: throw NotFoundException("Resource $resourceId not found in namespace ${workConfiguration.namespace}")
 
-    var markedResource = resourceRepository.find(resourceId, workConfiguration.namespace)
-
-    if (markedResource != null) {
-      markedResource = markedResource.copy(
-        resource = resource,
-        summaries = listOf(Summary("Resource marked via the api", AlwaysCleanRule::class.java.simpleName)),
-        namespace = workConfiguration.namespace,
-        resourceOwner = onDemandMarkData.resourceOwner,
-        projectedSoftDeletionStamp = onDemandMarkData.projectedSoftDeletionStamp,
-        projectedDeletionStamp = onDemandMarkData.projectedDeletionStamp,
-        notificationInfo = onDemandMarkData.notificationInfo
-      )
-    } else {
-      markedResource = MarkedResource(
+    var markedResource = MarkedResource(
         resource = resource,
         summaries = listOf(Summary("Resource marked via the api", AlwaysCleanRule::class.java.simpleName)),
         namespace = workConfiguration.namespace,
         resourceOwner = onDemandMarkData.resourceOwner,
         projectedSoftDeletionStamp = onDemandMarkData.projectedSoftDeletionStamp,
         projectedDeletionStamp = onDemandMarkData.projectedDeletionStamp,
-        notificationInfo = onDemandMarkData.notificationInfo
+        notificationInfo = onDemandMarkData.notificationInfo,
+        lastSeenInfo = onDemandMarkData.lastSeenInfo
       )
-    }
 
     log.debug("Marking resource $resourceId in namespace ${workConfiguration.namespace} without checking rules.")
     resourceRepository.upsert(markedResource)
@@ -308,7 +288,8 @@ abstract class AbstractResourceTypeHandler<T : Resource>(
                       namespace = workConfiguration.namespace,
                       resourceOwner = candidate.details[resourceOwnerField] as String,
                       projectedSoftDeletionStamp = softDeletionTimestamp(workConfiguration),
-                      projectedDeletionStamp = deletionTimestamp(workConfiguration)
+                      projectedDeletionStamp = deletionTimestamp(workConfiguration),
+                      lastSeenInfo = resourceUseTrackingRepository.getLastSeenInfo(candidate.resourceId)
                     )
 
                     if (!workConfiguration.dryRun) {
@@ -734,7 +715,7 @@ abstract class AbstractResourceTypeHandler<T : Resource>(
         if (notificationType.equals(Notifier.NotificationType.EMAIL.name, true)) {
           val notificationContext = mapOf(
             "resourceOwner" to owner,
-            "resources" to resources,
+            "resources" to resources.map { it.slim() },
             "configuration" to workConfiguration,
             "resourceType" to workConfiguration.resourceType.formatted(),
             "spinnakerLink" to workConfiguration.notificationConfiguration.resourceUrl,

swabbie-core/src/main/kotlin/com/netflix/spinnaker/swabbie/model/OnDemandMarkData.kt

@@ -1,6 +1,7 @@
 package com.netflix.spinnaker.swabbie.model
 
 import com.netflix.spinnaker.swabbie.notifications.Notifier
+import com.netflix.spinnaker.swabbie.repository.LastSeenInfo
 
 data class OnDemandMarkData(
   var projectedSoftDeletionStamp: Long,
@@ -11,5 +12,6 @@ data class OnDemandMarkData(
     recipient = resourceOwner,
     notificationType = Notifier.NotificationType.EMAIL.name,
     notificationCount = 1
-  )
+  ),
+  var lastSeenInfo: LastSeenInfo? = null
 )

swabbie-core/src/main/kotlin/com/netflix/spinnaker/swabbie/model/Resource.kt

@@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonAnySetter
 import com.fasterxml.jackson.annotation.JsonInclude
 import com.fasterxml.jackson.annotation.JsonTypeName
 import com.netflix.spinnaker.swabbie.exclusions.Excludable
+import com.netflix.spinnaker.swabbie.repository.LastSeenInfo
 import java.time.Clock
 import java.time.Instant
 import java.time.LocalDate
@@ -114,6 +115,7 @@ interface MarkedResourceInterface : Identifiable {
   var markTs: Long?
   var updateTs: Long?
   val resourceOwner: String
+  var lastSeenInfo: LastSeenInfo?
 }
 
 /**
@@ -129,22 +131,24 @@ data class MarkedResource(
   override var notificationInfo: NotificationInfo? = null,
   override var markTs: Long? = null,
   override var updateTs: Long? = null,
-  override var resourceOwner: String = ""
+  override var resourceOwner: String = "",
+  override var lastSeenInfo: LastSeenInfo? = null
 ) : MarkedResourceInterface, Identifiable by resource {
   fun slim(): SlimMarkedResource {
     return SlimMarkedResource(
-      summaries,
-      namespace,
-      projectedSoftDeletionStamp,
-      projectedDeletionStamp,
-      notificationInfo,
-      markTs,
-      updateTs,
-      resourceOwner,
-      resource.resourceId,
-      resource.resourceType,
-      resource.cloudProvider,
-      resource.name
+      summaries = summaries,
+      namespace = namespace,
+      projectedSoftDeletionStamp = projectedSoftDeletionStamp,
+      projectedDeletionStamp = projectedDeletionStamp,
+      notificationInfo = notificationInfo,
+      markTs = markTs,
+      updateTs = updateTs,
+      resourceOwner = resourceOwner,
+      resourceId = resource.resourceId,
+      resourceType = resource.resourceType,
+      cloudProvider = resource.cloudProvider,
+      name = resource.name,
+      lastSeenInfo = lastSeenInfo
     )
   }
 
@@ -165,7 +169,8 @@ data class SlimMarkedResource (
   override val resourceId: String,
   override val resourceType: String,
   override val cloudProvider: String,
-  override val name: String?
+  override val name: String?,
+  override var lastSeenInfo: LastSeenInfo? = null
 ) : MarkedResourceInterface
 
 data class NotificationInfo(

swabbie-core/src/main/kotlin/com/netflix/spinnaker/swabbie/repository/ResourceUseTrackingRepository.kt

@@ -29,8 +29,15 @@ interface ResourceUseTrackingRepository {
    */
   fun getUnused(): List<LastSeenInfo>
 
+  /**
+   * gets all resources that we have seen in use within the threshold
+   */
+  fun getUsed(): Set<String>
+
   fun isUnused(resourceIdentifier: String): Boolean
 
+  fun getLastSeenInfo(resourceIdentifier: String): LastSeenInfo?
+
   /**
    * Returns true if data is present for the whole outOfUseThreshold period.
    * Returns false if data is only present for part of the period,
@@ -40,7 +47,7 @@ interface ResourceUseTrackingRepository {
 }
 
 data class LastSeenInfo(
-  val resourceIdentifier: String,
-  val usedByResourceIdentifier: String,
+  val resourceId: String,
+  val usedByResourceId: String,
   val timeSeen: Long
 )