diff --git a/swabbie-aws/src/main/kotlin/com/netflix/spinnaker/swabbie/aws/ExpiredResourceRule.kt b/swabbie-aws/src/main/kotlin/com/netflix/spinnaker/swabbie/aws/ExpiredResourceRule.kt new file mode 100644 index 00000000..e15908a1 --- /dev/null +++ b/swabbie-aws/src/main/kotlin/com/netflix/spinnaker/swabbie/aws/ExpiredResourceRule.kt @@ -0,0 +1,49 @@ +/* + * Copyright 2019 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.swabbie.aws + +import com.netflix.spinnaker.swabbie.aws.model.AmazonResource +import com.netflix.spinnaker.swabbie.model.Result +import com.netflix.spinnaker.swabbie.model.Rule +import com.netflix.spinnaker.swabbie.model.Summary +import org.springframework.stereotype.Component +import java.time.Clock + +/** + * This rule applies if this amazon resource has expired. + * A resource is expired if it's tagged with the following keys: ("expiration_time", "expires", "ttl") + * Acceptable tag value: a number followed by a suffix such as d (days), w (weeks), m (month), y (year) + * @see com.netflix.spinnaker.swabbie.tagging.TemporalTags.supportedTemporalTagValues + */ + +@Component +class ExpiredResourceRule( + private val clock: Clock +) : Rule { + override fun apply(resource: T): Result { + if (resource.expired(clock)) { + return Result( + Summary( + description = "$${resource.resourceId} has expired. tags: ${resource.tags()}", + ruleName = javaClass.simpleName + ) + ) + } + + return Result(null) + } +} diff --git a/swabbie-aws/src/main/kotlin/com/netflix/spinnaker/swabbie/aws/exclusions/AmazonTagExclusionPolicy.kt b/swabbie-aws/src/main/kotlin/com/netflix/spinnaker/swabbie/aws/exclusions/AmazonTagExclusionPolicy.kt index 77fcade3..905b06ba 100644 --- a/swabbie-aws/src/main/kotlin/com/netflix/spinnaker/swabbie/aws/exclusions/AmazonTagExclusionPolicy.kt +++ b/swabbie-aws/src/main/kotlin/com/netflix/spinnaker/swabbie/aws/exclusions/AmazonTagExclusionPolicy.kt @@ -21,50 +21,35 @@ import com.netflix.spinnaker.config.ExclusionType import com.netflix.spinnaker.swabbie.aws.model.AmazonResource import com.netflix.spinnaker.swabbie.exclusions.Excludable import com.netflix.spinnaker.swabbie.exclusions.ResourceExclusionPolicy +import com.netflix.spinnaker.swabbie.model.BasicTag import org.springframework.stereotype.Component +import java.time.Clock @Component -class AmazonTagExclusionPolicy : ResourceExclusionPolicy { - private val tagsField = "tags" +class AmazonTagExclusionPolicy( + val clock: Clock +) : ResourceExclusionPolicy { override fun getType(): ExclusionType = ExclusionType.Tag override fun apply(excludable: Excludable, exclusions: List): String? { - if (excludable is AmazonResource) { - keysAndValues(exclusions, ExclusionType.Tag) - .let { excludingTags -> - if (tagsField in excludable.details) { - (excludable.details[tagsField] as List>).map { tag -> - tag.keys.find { key -> - excludingTags[key] != null - }?.let { key -> - if (key in TemporalTagExclusionSupplier.temporalTags) { - excludingTags[key]!!.map { target -> - TemporalTagExclusionSupplier - .computeAndCompareAge( - excludable = excludable, - tagValue = tag[key] as String, - target = target - ).let { - when { - it.age == Age.OLDER || it.age == Age.INFINITE -> - return patternMatchMessage(tag[key] as String, excludingTags[key]!!.toSet()) - it.age == Age.YOUNGER -> - return null - else -> { - // no need to check age here. - log.debug("Resource age comparison with {}. Result: {}", excludable.createTs, it) - } - } - } - } - } + if (excludable !is AmazonResource || excludable.tags().isNullOrEmpty()) { + return null + } + + val tags = excludable.tags()!! + // Exclude this resource if it's tagged with a ttl but has not yet expired + val temporalTags = tags.filter(BasicTag::isTemporal) + if (temporalTags.isNotEmpty() && !excludable.expired(clock)) { + val keysAsString = temporalTags.map { it.key }.joinToString { "," } + val valuesAsString = temporalTags.map { it.value }.toString() + return patternMatchMessage(keysAsString, setOf(valuesAsString)) + } - if (excludingTags[key]!!.contains(tag[key] as? String)) { - return patternMatchMessage(tag[key] as String, excludingTags[key]!!.toSet()) - } - } - } - } - } + val configuredKeysAndTargetValues = keysAndValues(exclusions, ExclusionType.Tag) + tags.forEach { tag -> + val target = configuredKeysAndTargetValues[tag.key] ?: emptyList() + if (target.contains(tag.value as String)) { + return patternMatchMessage(tag.key, setOf(tag.value as String)) + } } return null diff --git a/swabbie-aws/src/main/kotlin/com/netflix/spinnaker/swabbie/aws/exclusions/TemporalTagExclusionSupplier.kt b/swabbie-aws/src/main/kotlin/com/netflix/spinnaker/swabbie/aws/exclusions/TemporalTagExclusionSupplier.kt index 7058630d..1da1a798 100644 --- a/swabbie-aws/src/main/kotlin/com/netflix/spinnaker/swabbie/aws/exclusions/TemporalTagExclusionSupplier.kt +++ b/swabbie-aws/src/main/kotlin/com/netflix/spinnaker/swabbie/aws/exclusions/TemporalTagExclusionSupplier.kt @@ -19,98 +19,25 @@ package com.netflix.spinnaker.swabbie.aws.exclusions import com.netflix.spinnaker.config.Attribute import com.netflix.spinnaker.config.Exclusion import com.netflix.spinnaker.config.ExclusionType -import com.netflix.spinnaker.swabbie.aws.model.AmazonResource import com.netflix.spinnaker.swabbie.exclusions.ExclusionsSupplier +import com.netflix.spinnaker.swabbie.tagging.TemporalTags import org.springframework.stereotype.Component -import java.time.Instant -import java.time.LocalDate -import java.time.ZoneId -import java.time.temporal.ChronoUnit @Component class TemporalTagExclusionSupplier : ExclusionsSupplier { - companion object { - fun computeAndCompareAge(excludable: AmazonResource, tagValue: String, target: String): AgeCompareResult { - if (target.startsWith("pattern:")) { - target.split(":").last().toRegex().find(tagValue)?.groupValues?.let { - val unit = it[1] - val suppliedAmountWithoutUnit = it[0].replace(unit, "").toLong() - supportedTemporalUnits[unit]?.between( - Instant.ofEpochMilli(excludable.createTs), - Instant.now() - )?.let { elapsedSinceCreation -> - val computedTargetStamp = LocalDate.now() - .plus(suppliedAmountWithoutUnit, supportedTemporalUnits[unit]) - .atStartOfDay(ZoneId.systemDefault()) - .toInstant() - .toEpochMilli() - - return if (suppliedAmountWithoutUnit >= elapsedSinceCreation) { - AgeCompareResult( - age = Age.OLDER, - suppliedStamp = computedTargetStamp, - comparedStamp = excludable.createTs - ) - } else { - AgeCompareResult( - age = Age.YOUNGER, - suppliedStamp = computedTargetStamp, - comparedStamp = excludable.createTs - ) - } - } - } - } - - return if (tagValue == "never") { - AgeCompareResult( - age = Age.INFINITE, - suppliedStamp = null, - comparedStamp = excludable.createTs - ) - } else { - AgeCompareResult( - age = Age.UNKNOWN, - suppliedStamp = null, - comparedStamp = excludable.createTs - ) - } - } - - val temporalTags = listOf("expiration_time", "expires", "ttl") - private val supportedTemporalUnits = mapOf( - "d" to ChronoUnit.DAYS, - "m" to ChronoUnit.MONTHS, - "y" to ChronoUnit.YEARS, - "w" to ChronoUnit.WEEKS - ) - } - - private val supportedTemporalTagValues = listOf("pattern:^\\d+(d|m|y|w)$", "never") - override fun get(): List { return listOf( Exclusion() .withType(ExclusionType.Tag.toString()) .withAttributes( - temporalTags.map { key -> + TemporalTags.temporalTags.map { key -> Attribute() .withKey(key) .withValue( - supportedTemporalTagValues + TemporalTags.supportedTemporalTagValues ) }.toSet() ) ) } } - -enum class Age { - INFINITE, OLDER, EQUAL, YOUNGER, UNKNOWN -} - -data class AgeCompareResult( - val age: Age, - val suppliedStamp: Long?, - val comparedStamp: Long -) diff --git a/swabbie-aws/src/main/kotlin/com/netflix/spinnaker/swabbie/aws/instances/AmazonInstance.kt b/swabbie-aws/src/main/kotlin/com/netflix/spinnaker/swabbie/aws/instances/AmazonInstance.kt index e0355807..4ae024e3 100644 --- a/swabbie-aws/src/main/kotlin/com/netflix/spinnaker/swabbie/aws/instances/AmazonInstance.kt +++ b/swabbie-aws/src/main/kotlin/com/netflix/spinnaker/swabbie/aws/instances/AmazonInstance.kt @@ -28,7 +28,6 @@ import java.time.ZoneId data class AmazonInstance( val instanceId: String, val imageId: String, - val tags: List>, private val launchTime: Long, override val resourceId: String = instanceId, override val resourceType: String = INSTANCE, @@ -38,9 +37,7 @@ data class AmazonInstance( LocalDateTime.ofInstant(Instant.ofEpochMilli(launchTime), ZoneId.systemDefault()).toString() ) : AmazonResource(creationDate) { fun getAutoscalingGroup(): String? { - return tags - .find { it.containsKey("aws:autoscaling:groupName") } - ?.get("aws:autoscaling:groupName") + return tags()?.find { it.key == "aws:autoscaling:groupName" }?.value as String } override fun equals(other: Any?): Boolean { diff --git a/swabbie-aws/src/main/kotlin/com/netflix/spinnaker/swabbie/aws/model/AmazonResource.kt b/swabbie-aws/src/main/kotlin/com/netflix/spinnaker/swabbie/aws/model/AmazonResource.kt index 3ca493bc..822c7528 100644 --- a/swabbie-aws/src/main/kotlin/com/netflix/spinnaker/swabbie/aws/model/AmazonResource.kt +++ b/swabbie-aws/src/main/kotlin/com/netflix/spinnaker/swabbie/aws/model/AmazonResource.kt @@ -33,7 +33,7 @@ abstract class AmazonResource( get() { if (resourceType.contains("image", ignoreCase = true) || resourceType.contains("snapshot", ignoreCase = true)) { // Images and snapshots have only packageName, not app, to group by - getTagValue("appversion")?.let { AppVersion.parseName(it)?.packageName }?.let { packageName -> + getTagValue("appversion")?.let { AppVersion.parseName(it as String)?.packageName }?.let { packageName -> return Grouping(packageName, GroupingType.PACKAGE_NAME) } return null diff --git a/swabbie-aws/src/test/java/com/netflix/spinnaker/swabbie/aws/ExpiredResourceRuleTest.kt b/swabbie-aws/src/test/java/com/netflix/spinnaker/swabbie/aws/ExpiredResourceRuleTest.kt new file mode 100644 index 00000000..46274cc5 --- /dev/null +++ b/swabbie-aws/src/test/java/com/netflix/spinnaker/swabbie/aws/ExpiredResourceRuleTest.kt @@ -0,0 +1,79 @@ +/* + * Copyright 2019 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.swabbie.aws + +import com.netflix.spinnaker.kork.test.time.MutableClock +import com.netflix.spinnaker.swabbie.aws.autoscalinggroups.AmazonAutoScalingGroup +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import strikt.api.expectThat +import strikt.assertions.isNotNull +import strikt.assertions.isNull +import java.time.Duration +import java.time.Instant + +object ExpiredResourceRuleTest { + private val clock = MutableClock() + private val subject = ExpiredResourceRule(clock) + private val now = Instant.now(clock).toEpochMilli() + private val asg = AmazonAutoScalingGroup( + autoScalingGroupName = "testapp-v001", + instances = listOf( + mapOf("instanceId" to "i-01234") + ), + loadBalancerNames = listOf(), + createdTime = now + ) + + @BeforeEach + fun setup() { + asg.set("tags", null) + } + + @Test + fun `should not apply if resource is not tagged with a ttl`() { + expectThat( + subject.apply(asg).summary + ).isNull() + } + + @Test + fun `should not apply if resource is not expired`() { + val tags = listOf( + mapOf("ttl" to "4d") + ) + + asg.set("tags", tags) + expectThat( + subject.apply(asg).summary + ).isNull() + } + + @Test + fun `should apply if resource is expired`() { + val tags = listOf( + mapOf("ttl" to "2d") + ) + + asg.set("tags", tags) + + clock.incrementBy(Duration.ofDays(3)) + expectThat( + subject.apply(asg).summary + ).isNotNull() + } +} diff --git a/swabbie-aws/src/test/java/com/netflix/spinnaker/swabbie/aws/exclusions/AmazonTagExclusionPolicyTest.kt b/swabbie-aws/src/test/java/com/netflix/spinnaker/swabbie/aws/exclusions/AmazonTagExclusionPolicyTest.kt index 16f1af0f..7a2b6460 100644 --- a/swabbie-aws/src/test/java/com/netflix/spinnaker/swabbie/aws/exclusions/AmazonTagExclusionPolicyTest.kt +++ b/swabbie-aws/src/test/java/com/netflix/spinnaker/swabbie/aws/exclusions/AmazonTagExclusionPolicyTest.kt @@ -22,11 +22,23 @@ import com.natpryce.hamkrest.should.shouldMatch import com.netflix.spinnaker.config.Attribute import com.netflix.spinnaker.config.Exclusion import com.netflix.spinnaker.config.ExclusionType +import com.netflix.spinnaker.kork.test.time.MutableClock import com.netflix.spinnaker.swabbie.aws.model.AmazonResource +import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test +import java.time.Duration +import java.time.Instant import java.time.LocalDateTime object AmazonTagExclusionPolicyTest { + private val clock = MutableClock() + + @BeforeEach + fun setup() { + clock.instant(Instant.now()) + } + + private val subject = AmazonTagExclusionPolicy(clock) @Test fun `should exclude a resource with exclusion tag`() { val exclusions = listOf( @@ -60,7 +72,7 @@ object AmazonTagExclusionPolicyTest { )) resources.filter { - AmazonTagExclusionPolicy().apply(it, exclusions) == null + subject.apply(it, exclusions) == null }.let { filteredResources -> filteredResources.size shouldMatch equalTo(1) filteredResources.first().resourceId shouldMatch equalTo("2") @@ -69,41 +81,30 @@ object AmazonTagExclusionPolicyTest { @Test fun `should exclude a resource based on temporal tags`() { - val tenDays = 10L - val exclusions = listOf( - Exclusion() - .withType(ExclusionType.Tag.toString()) - .withAttributes( - setOf( - Attribute() - .withKey("expiration_time") - .withValue( - listOf("pattern:^\\d+(d|m|y)\$") - ) - ) - ) - ) - + val now = LocalDateTime.now(clock) val resources = listOf( AwsTestResource( id = "1", - creationDate = LocalDateTime.now().minusDays(tenDays).toString() + creationDate = now.toString() ).withDetail( name = "tags", value = listOf( - mapOf("expiration_time" to "${tenDays}d") + mapOf("expiration_time" to "10d") )), AwsTestResource( id = "2", - creationDate = LocalDateTime.now().minusDays(tenDays).toString() + creationDate = now.toString() ).withDetail( name = "tags", value = listOf( - mapOf("expiration_time" to "${tenDays - 1}d") + mapOf("expiration_time" to "9d") ) )) + + clock.incrementBy(Duration.ofDays(10)) + resources.filter { - AmazonTagExclusionPolicy().apply(it, exclusions) == null + subject.apply(it, emptyList()) == null }.let { filteredResources -> filteredResources.size shouldMatch equalTo(1) filteredResources.first().resourceId shouldMatch equalTo("2") diff --git a/swabbie-aws/swabbie-aws.gradle b/swabbie-aws/swabbie-aws.gradle index 1b004391..9e536946 100644 --- a/swabbie-aws/swabbie-aws.gradle +++ b/swabbie-aws/swabbie-aws.gradle @@ -31,4 +31,5 @@ dependencies { implementation "com.fasterxml.jackson.module:jackson-module-kotlin" testImplementation project(":swabbie-test") + testImplementation "com.netflix.spinnaker.kork:kork-test" } diff --git a/swabbie-core/src/main/kotlin/com/netflix/spinnaker/swabbie/model/Resource.kt b/swabbie-core/src/main/kotlin/com/netflix/spinnaker/swabbie/model/Resource.kt index 096e72c1..d4bf22ab 100644 --- a/swabbie-core/src/main/kotlin/com/netflix/spinnaker/swabbie/model/Resource.kt +++ b/swabbie-core/src/main/kotlin/com/netflix/spinnaker/swabbie/model/Resource.kt @@ -23,9 +23,11 @@ import com.fasterxml.jackson.annotation.JsonTypeName import com.netflix.spinnaker.swabbie.exclusions.Excludable import com.netflix.spinnaker.swabbie.notifications.Notifier import com.netflix.spinnaker.swabbie.repository.LastSeenInfo +import com.netflix.spinnaker.swabbie.tagging.TemporalTags import org.slf4j.Logger import org.slf4j.LoggerFactory import java.time.Clock +import java.time.Duration import java.time.Instant import java.time.LocalDate @@ -81,18 +83,40 @@ abstract class Resource : Excludable, Timestamped, HasDetails() { return details.toString() } - fun getTagValue(key: String): String? { - try { - val tags = this.details["tags"] as List> - tags.forEach { tag -> - if (tag.containsKey(key)) { - return tag.getValue(key) - } + fun tags(): List? { + return (details["tags"] as? List>)?.flatMap { + it.entries.map { tag -> + BasicTag(tag.key, tag.value) } - } catch (e: ClassCastException) { - log.warn("Resource {} does not have normal tag format: {}", this.toLog(), this.details["tags"]) } - return null + } + + fun getTagValue(key: String): Any? = tags()?.find { it.key == key }?.value + + fun expired(clock: Clock): Boolean { + return tags()?.any { expired(it, clock) } ?: false + } + + fun expired(temporalTag: BasicTag, clock: Clock): Boolean { + if (temporalTag.value == "never" || !temporalTag.isTemporal()) { + return false + } + + val (amount, unit) = TemporalTags.toTemporalPair(temporalTag) + val ttl = Duration.of(amount, unit).toDays() + val resourceAge = Duration.between(Instant.ofEpochMilli(createTs), Instant.now(clock)) + return resourceAge.toDays() > ttl + } +} + +data class BasicTag( + val key: String, + val value: Any? +) { + fun isTemporal(): Boolean { + return key in TemporalTags.temporalTags && TemporalTags.supportedTemporalTagValues.any { + (value as String).matches((it).toRegex()) + } } } diff --git a/swabbie-core/src/main/kotlin/com/netflix/spinnaker/swabbie/tagging/TemporalTags.kt b/swabbie-core/src/main/kotlin/com/netflix/spinnaker/swabbie/tagging/TemporalTags.kt new file mode 100644 index 00000000..f48b833e --- /dev/null +++ b/swabbie-core/src/main/kotlin/com/netflix/spinnaker/swabbie/tagging/TemporalTags.kt @@ -0,0 +1,45 @@ +/* + * Copyright 2019 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License") + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.swabbie.tagging + +import com.netflix.spinnaker.swabbie.model.BasicTag +import java.time.temporal.ChronoUnit + +class TemporalTags { + companion object { + val supportedTemporalTagValues = listOf("^\\d+(d|m|y|w)$", "never") + val temporalTags = listOf("expiration_time", "expires", "ttl") + + private val supportedTemporalUnits = mapOf( + "d" to ChronoUnit.DAYS, + "m" to ChronoUnit.MONTHS, + "y" to ChronoUnit.YEARS, + "w" to ChronoUnit.WEEKS + ) + + fun toTemporalPair(tag: BasicTag): Pair { + val tagValue = tag.value.toString() + val unit = tagValue.last().toString() + val amount = tagValue.replace(unit, "").toLong() + if (unit !in supportedTemporalUnits) { + return Pair(amount, null) + } + + return Pair(amount, supportedTemporalUnits[unit]) + } + } +}