From be084a308258ddba40e76c6fbd1bbe94f77c2913 Mon Sep 17 00:00:00 2001 From: musketyr Date: Tue, 26 Mar 2024 10:15:22 +0100 Subject: [PATCH 1/6] Upgraded to the latest AWS Libraries --- gradle.properties | 4 ++-- .../micronaut-aws-sdk-dependencies.gradle | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/gradle.properties b/gradle.properties index b5038a89a..ae93c8b77 100644 --- a/gradle.properties +++ b/gradle.properties @@ -21,8 +21,8 @@ group=com.agorapulse micronautVersion = 4.2.0 micronautGradlePluginVersion = 4.2.0 gruVersion = 2.0.5 -awsSdkVersion = 1.12.299 -awsSdk2Version = 2.18.40 +awsSdkVersion = 1.12.686 +awsSdk2Version = 2.25.16 awsLambdaRuntimeVersion=2.5.0 testcontainersVersion = 1.17.3 kordampVersion=0.53.0 diff --git a/platforms/micronaut-aws-sdk-dependencies/micronaut-aws-sdk-dependencies.gradle b/platforms/micronaut-aws-sdk-dependencies/micronaut-aws-sdk-dependencies.gradle index 3dc827d06..84905d94d 100644 --- a/platforms/micronaut-aws-sdk-dependencies/micronaut-aws-sdk-dependencies.gradle +++ b/platforms/micronaut-aws-sdk-dependencies/micronaut-aws-sdk-dependencies.gradle @@ -53,9 +53,10 @@ dependencies { api "software.amazon.awssdk:sns:$project.awsSdk2Version" api "software.amazon.awssdk:sqs:$project.awsSdk2Version" api "software.amazon.awssdk:sts:$project.awsSdk2Version" + api "software.amazon.awssdk:aws-query-protocol:$project.awsSdk2Version" - api group: 'com.amazonaws', name: 'amazon-dax-client', version: '1.0.205044.0' - api group: 'com.amazonaws', name: 'aws-lambda-java-events', version: '2.2.7' + api group: 'com.amazonaws', name: 'amazon-dax-client', version: '1.0.230341.0' + api group: 'com.amazonaws', name: 'aws-lambda-java-events', version: '2.2.9' api group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-cbor', version: project['jackson.datatype.version'] } } From 11b9898c5d3333352a26bf16dd15b5e0d7d6e5ad Mon Sep 17 00:00:00 2001 From: musketyr Date: Tue, 26 Mar 2024 11:20:00 +0100 Subject: [PATCH 2/6] reverted to older version of SDK used by MariaDB drivers --- gradle.properties | 2 +- .../src/test/resources/application-test.yml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index ae93c8b77..2cd8bfe92 100644 --- a/gradle.properties +++ b/gradle.properties @@ -22,7 +22,7 @@ micronautVersion = 4.2.0 micronautGradlePluginVersion = 4.2.0 gruVersion = 2.0.5 awsSdkVersion = 1.12.686 -awsSdk2Version = 2.25.16 +awsSdk2Version = 2.21.1 awsLambdaRuntimeVersion=2.5.0 testcontainersVersion = 1.17.3 kordampVersion=0.53.0 diff --git a/subprojects/micronaut-aws-sdk-sqs/src/test/resources/application-test.yml b/subprojects/micronaut-aws-sdk-sqs/src/test/resources/application-test.yml index cf2f87005..2214b8829 100644 --- a/subprojects/micronaut-aws-sdk-sqs/src/test/resources/application-test.yml +++ b/subprojects/micronaut-aws-sdk-sqs/src/test/resources/application-test.yml @@ -1,6 +1,8 @@ localstack: services: sqs shared: true + env: + DEBUG: 1 aws: sqs: From 8c850ad5c3c3a720dd2720b5cc2eb01903a7344d Mon Sep 17 00:00:00 2001 From: musketyr Date: Tue, 26 Mar 2024 15:07:57 +0100 Subject: [PATCH 3/6] using dynamic SQS URLs --- build.gradle | 1 + .../LocalstackContainerConfiguration.java | 14 +++++++++++++- .../src/test/resources/application-test.yml | 1 - 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index b42e5cd9e..f829637ac 100644 --- a/build.gradle +++ b/build.gradle @@ -197,6 +197,7 @@ subprojects { Project subproject -> testImplementation 'io.micronaut.test:micronaut-test-spock' testRuntimeOnly 'ch.qos.logback:logback-classic:1.2.3' + testRuntimeOnly 'org.yaml:snakeyaml:1.27' } compileJava.options.compilerArgs += '-parameters' diff --git a/subprojects/micronaut-amazon-awssdk-integration-testing/src/main/java/com/agorapulse/micronaut/amazon/awssdk/itest/localstack/LocalstackContainerConfiguration.java b/subprojects/micronaut-amazon-awssdk-integration-testing/src/main/java/com/agorapulse/micronaut/amazon/awssdk/itest/localstack/LocalstackContainerConfiguration.java index c3ea87bda..c36500044 100644 --- a/subprojects/micronaut-amazon-awssdk-integration-testing/src/main/java/com/agorapulse/micronaut/amazon/awssdk/itest/localstack/LocalstackContainerConfiguration.java +++ b/subprojects/micronaut-amazon-awssdk-integration-testing/src/main/java/com/agorapulse/micronaut/amazon/awssdk/itest/localstack/LocalstackContainerConfiguration.java @@ -20,13 +20,19 @@ import io.micronaut.context.annotation.ConfigurationProperties; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; @ConfigurationProperties("localstack") public class LocalstackContainerConfiguration extends AbstractContainerConfiguration { + public static final Map ALWAYS_PRESENT_ENV = Map.of( + "SQS_ENDPOINT_STRATEGY", "dynamic" + ); + public static final String DEFAULT_IMAGE = "localstack/localstack"; - public static final String DEFAULT_TAG = "1.3.0"; + public static final String DEFAULT_TAG = "3.2.0"; public LocalstackContainerConfiguration() { setImage(DEFAULT_IMAGE); @@ -43,4 +49,10 @@ public void setServices(List services) { this.services = services; } + @Override + public Map getEnv() { + Map allEnv = new HashMap<>(super.getEnv()); + ALWAYS_PRESENT_ENV.forEach(allEnv::putIfAbsent); + return allEnv; + } } diff --git a/subprojects/micronaut-aws-sdk-sqs/src/test/resources/application-test.yml b/subprojects/micronaut-aws-sdk-sqs/src/test/resources/application-test.yml index 2214b8829..ff9073e80 100644 --- a/subprojects/micronaut-aws-sdk-sqs/src/test/resources/application-test.yml +++ b/subprojects/micronaut-aws-sdk-sqs/src/test/resources/application-test.yml @@ -3,7 +3,6 @@ localstack: shared: true env: DEBUG: 1 - aws: sqs: tags: From 4d20361c85c64a9053f26b0e1a41bd775eaf7c0f Mon Sep 17 00:00:00 2001 From: musketyr Date: Tue, 26 Mar 2024 15:31:28 +0100 Subject: [PATCH 4/6] fixed tests --- .../src/test/resources/application-test.yml | 2 +- .../amazon/awssdk/s3/SimpleStorageServiceSpec.groovy | 7 ++++++- .../amazon/awssdk/sns/SimpleNotificationServiceSpec.groovy | 2 -- .../src/test/resources/application-test.yml | 2 +- .../micronaut/aws/sns/SimpleNotificationServiceSpec.groovy | 4 ---- 5 files changed, 8 insertions(+), 9 deletions(-) diff --git a/subprojects/micronaut-amazon-awssdk-lambda/src/test/resources/application-test.yml b/subprojects/micronaut-amazon-awssdk-lambda/src/test/resources/application-test.yml index 1fab9e734..a5c731a48 100644 --- a/subprojects/micronaut-amazon-awssdk-lambda/src/test/resources/application-test.yml +++ b/subprojects/micronaut-amazon-awssdk-lambda/src/test/resources/application-test.yml @@ -1,5 +1,5 @@ localstack: - services: lambda + services: lambda,cloudwatchlogs shared: true env: PROVIDER_OVERRIDE_LAMBDA: asf diff --git a/subprojects/micronaut-amazon-awssdk-s3/src/test/groovy/com/agorapulse/micronaut/amazon/awssdk/s3/SimpleStorageServiceSpec.groovy b/subprojects/micronaut-amazon-awssdk-s3/src/test/groovy/com/agorapulse/micronaut/amazon/awssdk/s3/SimpleStorageServiceSpec.groovy index 1721b20ba..c36f48168 100644 --- a/subprojects/micronaut-amazon-awssdk-s3/src/test/groovy/com/agorapulse/micronaut/amazon/awssdk/s3/SimpleStorageServiceSpec.groovy +++ b/subprojects/micronaut-amazon-awssdk-s3/src/test/groovy/com/agorapulse/micronaut/amazon/awssdk/s3/SimpleStorageServiceSpec.groovy @@ -35,6 +35,7 @@ import spock.lang.TempDir import spock.lang.Unroll import jakarta.inject.Inject +import spock.util.concurrent.PollingConditions /** * Tests for SimpleStorageService based on Testcontainers. @@ -147,6 +148,8 @@ class SimpleStorageServiceSpec extends Specification { @Unroll void 'move object created with canned acl #desiredAcl'() { + given: + PollingConditions conditions = new PollingConditions(timeout: 30) when: String newKey = 'mix/moved-' + desiredAcl String oldKey = 'mix/to-be-moved-' + desiredAcl @@ -173,7 +176,9 @@ class SimpleStorageServiceSpec extends Specification { .tagSet() .any { it.key() == 'foo' && it.value() == 'bar' } - !service.exists(oldKey) + conditions.eventually { + !service.exists(oldKey) + } when: GetObjectAclResponse newAcls = amazonS3.getObjectAcl { it.bucket(MY_BUCKET).key(newKey) } diff --git a/subprojects/micronaut-amazon-awssdk-sns/src/test/groovy/com/agorapulse/micronaut/amazon/awssdk/sns/SimpleNotificationServiceSpec.groovy b/subprojects/micronaut-amazon-awssdk-sns/src/test/groovy/com/agorapulse/micronaut/amazon/awssdk/sns/SimpleNotificationServiceSpec.groovy index 56852a0c3..7d908ab84 100644 --- a/subprojects/micronaut-amazon-awssdk-sns/src/test/groovy/com/agorapulse/micronaut/amazon/awssdk/sns/SimpleNotificationServiceSpec.groovy +++ b/subprojects/micronaut-amazon-awssdk-sns/src/test/groovy/com/agorapulse/micronaut/amazon/awssdk/sns/SimpleNotificationServiceSpec.groovy @@ -19,7 +19,6 @@ package com.agorapulse.micronaut.amazon.awssdk.sns import io.micronaut.context.annotation.Property import io.micronaut.test.extensions.spock.annotation.MicronautTest -import spock.lang.PendingFeature import spock.lang.Shared import spock.lang.Specification import spock.lang.Stepwise @@ -125,7 +124,6 @@ class SimpleNotificationServiceSpec extends Specification { service.registerIosSandboxDevice('ANOTHER-IOS-SANDBOX-TOKEN') } - @PendingFeature (reason = 'Needs to be tested with real SNS') void 'register iOS sandbox device with the same token'() { expect: iosSandboxEndpointArn == service.createPlatformEndpoint(configuration.iosSandbox.arn, 'IOS-SANDBOX-TOKEN') diff --git a/subprojects/micronaut-aws-sdk-lambda/src/test/resources/application-test.yml b/subprojects/micronaut-aws-sdk-lambda/src/test/resources/application-test.yml index 1fab9e734..a5c731a48 100644 --- a/subprojects/micronaut-aws-sdk-lambda/src/test/resources/application-test.yml +++ b/subprojects/micronaut-aws-sdk-lambda/src/test/resources/application-test.yml @@ -1,5 +1,5 @@ localstack: - services: lambda + services: lambda,cloudwatchlogs shared: true env: PROVIDER_OVERRIDE_LAMBDA: asf diff --git a/subprojects/micronaut-aws-sdk-sns/src/test/groovy/com/agorapulse/micronaut/aws/sns/SimpleNotificationServiceSpec.groovy b/subprojects/micronaut-aws-sdk-sns/src/test/groovy/com/agorapulse/micronaut/aws/sns/SimpleNotificationServiceSpec.groovy index d4a293e06..072216e14 100644 --- a/subprojects/micronaut-aws-sdk-sns/src/test/groovy/com/agorapulse/micronaut/aws/sns/SimpleNotificationServiceSpec.groovy +++ b/subprojects/micronaut-aws-sdk-sns/src/test/groovy/com/agorapulse/micronaut/aws/sns/SimpleNotificationServiceSpec.groovy @@ -19,7 +19,6 @@ package com.agorapulse.micronaut.aws.sns import io.micronaut.context.annotation.Property import io.micronaut.test.extensions.spock.annotation.MicronautTest -import spock.lang.PendingFeature import spock.lang.Shared import spock.lang.Specification import spock.lang.Stepwise @@ -89,9 +88,6 @@ class SimpleNotificationServiceSpec extends Specification { endpointArn == newEndpointArn } - @PendingFeature( - reason = 'GCM moved int Firebase Console, latest Localstack fails with "Invalid parameter: Attributes Reason: Platform credentials are invalid"' - ) void 'publish direct'() { when: String messageId = service.sendAndroidAppNotification(endpointArn, [message: 'Hello'], 'key') From c1e53238d9463bd50c51447de1a777f3687b867c Mon Sep 17 00:00:00 2001 From: musketyr Date: Tue, 26 Mar 2024 15:32:11 +0100 Subject: [PATCH 5/6] fixed tests --- .../amazon/awssdk/s3/SimpleStorageServiceSpec.groovy | 8 -------- 1 file changed, 8 deletions(-) diff --git a/subprojects/micronaut-amazon-awssdk-s3/src/test/groovy/com/agorapulse/micronaut/amazon/awssdk/s3/SimpleStorageServiceSpec.groovy b/subprojects/micronaut-amazon-awssdk-s3/src/test/groovy/com/agorapulse/micronaut/amazon/awssdk/s3/SimpleStorageServiceSpec.groovy index c36f48168..c4cab61f8 100644 --- a/subprojects/micronaut-amazon-awssdk-s3/src/test/groovy/com/agorapulse/micronaut/amazon/awssdk/s3/SimpleStorageServiceSpec.groovy +++ b/subprojects/micronaut-amazon-awssdk-s3/src/test/groovy/com/agorapulse/micronaut/amazon/awssdk/s3/SimpleStorageServiceSpec.groovy @@ -35,7 +35,6 @@ import spock.lang.TempDir import spock.lang.Unroll import jakarta.inject.Inject -import spock.util.concurrent.PollingConditions /** * Tests for SimpleStorageService based on Testcontainers. @@ -148,8 +147,6 @@ class SimpleStorageServiceSpec extends Specification { @Unroll void 'move object created with canned acl #desiredAcl'() { - given: - PollingConditions conditions = new PollingConditions(timeout: 30) when: String newKey = 'mix/moved-' + desiredAcl String oldKey = 'mix/to-be-moved-' + desiredAcl @@ -175,11 +172,6 @@ class SimpleStorageServiceSpec extends Specification { amazonS3.getObjectTagging { it.bucket(MY_BUCKET).key(newKey) } .tagSet() .any { it.key() == 'foo' && it.value() == 'bar' } - - conditions.eventually { - !service.exists(oldKey) - } - when: GetObjectAclResponse newAcls = amazonS3.getObjectAcl { it.bucket(MY_BUCKET).key(newKey) } then: From d37dcc89f5235b5dccaf7347575da3aa9b4050aa Mon Sep 17 00:00:00 2001 From: musketyr Date: Tue, 2 Apr 2024 10:53:21 +0200 Subject: [PATCH 6/6] simplified moving S3 objects, fixed tests --- .../s3/DefaultSimpleStorageService.java | 20 ------------------- .../awssdk/s3/SimpleStorageServiceSpec.groovy | 2 +- .../micronaut-amazon-awssdk-sns.gradle | 1 + .../sns/SimpleNotificationServiceSpec.groovy | 9 ++++++++- .../src/test/resources/application-test.yml | 2 +- 5 files changed, 11 insertions(+), 23 deletions(-) diff --git a/subprojects/micronaut-amazon-awssdk-s3/src/main/java/com/agorapulse/micronaut/amazon/awssdk/s3/DefaultSimpleStorageService.java b/subprojects/micronaut-amazon-awssdk-s3/src/main/java/com/agorapulse/micronaut/amazon/awssdk/s3/DefaultSimpleStorageService.java index 7e2eb9bbf..e0a10b0f2 100644 --- a/subprojects/micronaut-amazon-awssdk-s3/src/main/java/com/agorapulse/micronaut/amazon/awssdk/s3/DefaultSimpleStorageService.java +++ b/subprojects/micronaut-amazon-awssdk-s3/src/main/java/com/agorapulse/micronaut/amazon/awssdk/s3/DefaultSimpleStorageService.java @@ -42,17 +42,12 @@ import java.time.Instant; import java.util.Date; import java.util.List; -import java.util.Optional; import java.util.function.Consumer; import java.util.stream.Collectors; public class DefaultSimpleStorageService implements SimpleStorageService { private static final Logger LOGGER = LoggerFactory.getLogger(DefaultSimpleStorageService.class); - private static final String ALL_USERS = "http://acs.amazonaws.com/groups/global/AllUsers"; - private static final String AUTHENTICATED_USERS = "http://acs.amazonaws.com/groups/global/AuthenticatedUsers"; - private static final String READ = "READ"; - private final S3Client s3; private final S3Presigner presigner; private final String defaultBucketName; @@ -218,7 +213,6 @@ public String moveObject(String sourceBucketName, String sourceKey, String desti GetObjectAclResponse acl = s3.getObjectAcl(b -> b.bucket(sourceBucketName).key(sourceKey)); s3.putObjectAcl(b -> { b.bucket(destinationBucketName).key(destinationKey).accessControlPolicy(p -> p.owner(acl.owner()).grants(acl.grants())); - extractCannedAcl(acl.grants()).ifPresent(b::acl); }); s3.deleteObject(b -> b.bucket(sourceBucketName).key(sourceKey)); @@ -230,18 +224,4 @@ public String moveObject(String sourceBucketName, String sourceKey, String desti } } - private static Optional extractCannedAcl(List grants) { - for (Grant grant : grants) { - if (READ.equals(grant.permissionAsString())) { - if (ALL_USERS.equals(grant.grantee().uri())) { - return Optional.of(ObjectCannedACL.PUBLIC_READ); - } - if (AUTHENTICATED_USERS.equals(grant.grantee().uri())) { - return Optional.of(ObjectCannedACL.AUTHENTICATED_READ); - } - } - } - return Optional.empty(); - } - } diff --git a/subprojects/micronaut-amazon-awssdk-s3/src/test/groovy/com/agorapulse/micronaut/amazon/awssdk/s3/SimpleStorageServiceSpec.groovy b/subprojects/micronaut-amazon-awssdk-s3/src/test/groovy/com/agorapulse/micronaut/amazon/awssdk/s3/SimpleStorageServiceSpec.groovy index c4cab61f8..544338a87 100644 --- a/subprojects/micronaut-amazon-awssdk-s3/src/test/groovy/com/agorapulse/micronaut/amazon/awssdk/s3/SimpleStorageServiceSpec.groovy +++ b/subprojects/micronaut-amazon-awssdk-s3/src/test/groovy/com/agorapulse/micronaut/amazon/awssdk/s3/SimpleStorageServiceSpec.groovy @@ -146,7 +146,7 @@ class SimpleStorageServiceSpec extends Specification { } @Unroll - void 'move object created with canned acl #desiredAcl'() { + void 'move object created with canned acl #desiredAcl'(ObjectCannedACL desiredAcl) { when: String newKey = 'mix/moved-' + desiredAcl String oldKey = 'mix/to-be-moved-' + desiredAcl diff --git a/subprojects/micronaut-amazon-awssdk-sns/micronaut-amazon-awssdk-sns.gradle b/subprojects/micronaut-amazon-awssdk-sns/micronaut-amazon-awssdk-sns.gradle index d2a2c421f..ff7e28f5b 100644 --- a/subprojects/micronaut-amazon-awssdk-sns/micronaut-amazon-awssdk-sns.gradle +++ b/subprojects/micronaut-amazon-awssdk-sns/micronaut-amazon-awssdk-sns.gradle @@ -24,4 +24,5 @@ dependencies { implementation 'io.micronaut:micronaut-jackson-databind' testImplementation project(':micronaut-amazon-awssdk-integration-testing') + testImplementation project(':micronaut-amazon-awssdk-sqs') } diff --git a/subprojects/micronaut-amazon-awssdk-sns/src/test/groovy/com/agorapulse/micronaut/amazon/awssdk/sns/SimpleNotificationServiceSpec.groovy b/subprojects/micronaut-amazon-awssdk-sns/src/test/groovy/com/agorapulse/micronaut/amazon/awssdk/sns/SimpleNotificationServiceSpec.groovy index 7d908ab84..9de9d98c2 100644 --- a/subprojects/micronaut-amazon-awssdk-sns/src/test/groovy/com/agorapulse/micronaut/amazon/awssdk/sns/SimpleNotificationServiceSpec.groovy +++ b/subprojects/micronaut-amazon-awssdk-sns/src/test/groovy/com/agorapulse/micronaut/amazon/awssdk/sns/SimpleNotificationServiceSpec.groovy @@ -17,6 +17,7 @@ */ package com.agorapulse.micronaut.amazon.awssdk.sns +import com.agorapulse.micronaut.amazon.awssdk.sqs.SimpleQueueService import io.micronaut.context.annotation.Property import io.micronaut.test.extensions.spock.annotation.MicronautTest import spock.lang.Shared @@ -36,11 +37,13 @@ import jakarta.inject.Inject @Property(name = 'aws.sns.ios-sandbox.arn', value = IOS_SANDBOX_APP_ARN) @Property(name = 'aws.sns.android.arn', value = ANDROID_APP_ARN) @Property(name = 'aws.sns.amazon.arn', value = AMAZON_APP_ARN) +@Property(name = 'aws.sqs.queue', value = TEST_QUEUE) class SimpleNotificationServiceSpec extends Specification { // end::header[] private static final String TEST_TOPIC = 'TestTopic' + private static final String TEST_QUEUE = 'TestQueue' private static final String IOS_APP_ARN = 'arn:aws:sns:us-east-1:000000000000:app/APNS/IOS-APP' private static final String IOS_SANDBOX_APP_ARN = 'arn:aws:sns:us-east-1:000000000000:app/APNS_SANDBOX/IOS-APP' private static final String ANDROID_APP_ARN = 'arn:aws:sns:us-east-1:000000000000:app/GCM/ANDROID-APP' @@ -51,6 +54,8 @@ class SimpleNotificationServiceSpec extends Specification { // end::setup[] @Inject SimpleNotificationServiceConfiguration configuration + @Inject SimpleQueueService simpleQueueService + @Shared String androidEndpointArn @Shared String amazonEndpointArn @Shared String iosEndpointArn @@ -63,6 +68,8 @@ class SimpleNotificationServiceSpec extends Specification { service.createIosApplication('IOS-APP', 'API-KEY', 'fake-cert', true) service.createAndroidApplication('ANDROID-APP', 'API-KEY') service.createAmazonApplication('AMAZON-APP', 'API-KEY', 'API-SECRET') + + simpleQueueService.createQueue(TEST_QUEUE) } void 'new topic'() { @@ -206,7 +213,7 @@ class SimpleNotificationServiceSpec extends Specification { void 'subscribe to the queue'() { when: - subscriptionArn = service.subscribeTopicWithQueue(topicArn, 'fake-queue-arn') + subscriptionArn = service.subscribeTopicWithQueue(topicArn, simpleQueueService.getQueueArn(TEST_QUEUE)) then: subscriptionArn } diff --git a/subprojects/micronaut-amazon-awssdk-sns/src/test/resources/application-test.yml b/subprojects/micronaut-amazon-awssdk-sns/src/test/resources/application-test.yml index d56444221..59cc78b4d 100644 --- a/subprojects/micronaut-amazon-awssdk-sns/src/test/resources/application-test.yml +++ b/subprojects/micronaut-amazon-awssdk-sns/src/test/resources/application-test.yml @@ -1,3 +1,3 @@ localstack: - services: sns + services: sns,sqs shared: true