From 7818d933066d6618ab806989dac4da7d62c60254 Mon Sep 17 00:00:00 2001 From: Vladimir Orany Date: Tue, 2 Apr 2024 13:25:14 +0200 Subject: [PATCH] [2.2.x] upgrade aws libraries (#224) * Upgraded to AWS SDK version used by MariaDB (#219) * Upgraded to the latest AWS Libraries * reverted to older version of SDK used by MariaDB drivers * using dynamic SQS URLs * fixed tests * fixed tests * simplified moving S3 objects, fixed tests (cherry picked from commit a9610e16c759489ce26551942409d59ab6f19ff5) * Upgraded to the latest AWS libraries (#223) * Upgraded to the latest AWS libraries * fixed version properties (cherry picked from commit 40b816fd669674dd595082468d48fbad7f6f41ae) * java 8 compatibility * backported fixed tests --- gradle.properties | 5 +++-- .../LocalstackContainerConfiguration.java | 15 +++++++++++--- .../awssdk/lambda/AbstractClientSpec.groovy | 9 +++++++-- .../src/test/resources/application-test.yml | 2 +- .../s3/DefaultSimpleStorageService.java | 20 ------------------- .../awssdk/s3/SimpleStorageServiceSpec.groovy | 5 +---- .../micronaut-amazon-awssdk-sns.gradle | 1 + .../sns/SimpleNotificationServiceSpec.groovy | 11 +++++++--- .../src/test/resources/application-test.yml | 2 +- .../micronaut-aws-sdk-dependencies.gradle | 5 +++-- .../aws/lambda/AbstractClientSpec.groovy | 9 +++++++-- .../src/test/resources/application-test.yml | 2 +- .../sns/SimpleNotificationServiceSpec.groovy | 4 ---- .../src/test/resources/application-test.yml | 3 ++- 14 files changed, 47 insertions(+), 46 deletions(-) diff --git a/gradle.properties b/gradle.properties index 9c7436afe..70fdf3ce9 100644 --- a/gradle.properties +++ b/gradle.properties @@ -22,9 +22,10 @@ micronautVersion = 3.2.1 gruVersion = 0.9.1 druVersion = 0.8.1 spockVersion = 2.0-groovy-3.0 -awsSdkVersion = 1.12.299 -awsSdk2Version = 2.18.40 +awsSdkVersion = 1.12.692 +awsSdk2Version = 2.25.22 awsLambdaRuntimeVersion=2.5.0 +awsLambdaEventsVersion=3.11.4 testcontainersVersion = 1.17.3 kordampVersion=0.45.0 closureSupportVersion=0.6.3 diff --git a/subprojects/micronaut-amazon-awssdk-integration-testing/src/main/java/com/agorapulse/micronaut/amazon/awssdk/itest/localstack/LocalstackContainerConfiguration.java b/subprojects/micronaut-amazon-awssdk-integration-testing/src/main/java/com/agorapulse/micronaut/amazon/awssdk/itest/localstack/LocalstackContainerConfiguration.java index c3ea87bda..c3a87b24e 100644 --- a/subprojects/micronaut-amazon-awssdk-integration-testing/src/main/java/com/agorapulse/micronaut/amazon/awssdk/itest/localstack/LocalstackContainerConfiguration.java +++ b/subprojects/micronaut-amazon-awssdk-integration-testing/src/main/java/com/agorapulse/micronaut/amazon/awssdk/itest/localstack/LocalstackContainerConfiguration.java @@ -19,14 +19,17 @@ import io.micronaut.context.annotation.ConfigurationProperties; -import java.util.ArrayList; -import java.util.List; +import java.util.*; @ConfigurationProperties("localstack") public class LocalstackContainerConfiguration extends AbstractContainerConfiguration { + public static final Map ALWAYS_PRESENT_ENV = Collections.singletonMap( + "SQS_ENDPOINT_STRATEGY", "dynamic" + ); + public static final String DEFAULT_IMAGE = "localstack/localstack"; - public static final String DEFAULT_TAG = "1.3.0"; + public static final String DEFAULT_TAG = "3.2.0"; public LocalstackContainerConfiguration() { setImage(DEFAULT_IMAGE); @@ -43,4 +46,10 @@ public void setServices(List services) { this.services = services; } + @Override + public Map getEnv() { + Map allEnv = new HashMap<>(super.getEnv()); + ALWAYS_PRESENT_ENV.forEach(allEnv::putIfAbsent); + return allEnv; + } } diff --git a/subprojects/micronaut-amazon-awssdk-lambda/src/test/groovy/com/agorapulse/micronaut/amazon/awssdk/lambda/AbstractClientSpec.groovy b/subprojects/micronaut-amazon-awssdk-lambda/src/test/groovy/com/agorapulse/micronaut/amazon/awssdk/lambda/AbstractClientSpec.groovy index f52b76eb1..a716a3d2a 100644 --- a/subprojects/micronaut-amazon-awssdk-lambda/src/test/groovy/com/agorapulse/micronaut/amazon/awssdk/lambda/AbstractClientSpec.groovy +++ b/subprojects/micronaut-amazon-awssdk-lambda/src/test/groovy/com/agorapulse/micronaut/amazon/awssdk/lambda/AbstractClientSpec.groovy @@ -22,6 +22,7 @@ import org.zeroturnaround.zip.ZipUtil import software.amazon.awssdk.core.SdkBytes import software.amazon.awssdk.services.lambda.LambdaClient import software.amazon.awssdk.services.lambda.model.Runtime +import software.amazon.awssdk.services.lambda.model.State import spock.lang.Shared import spock.lang.Specification import spock.lang.TempDir @@ -41,7 +42,7 @@ abstract class AbstractClientSpec extends Specification { prepareHelloFunction() } - @SuppressWarnings('ImplicitClosureParameter') + @SuppressWarnings(['ImplicitClosureParameter', 'BusyWait']) private void prepareHelloFunction() { boolean alreadyExists = lambda.listFunctions().functions().any { it.functionName() == 'HelloFunction' @@ -62,12 +63,16 @@ abstract class AbstractClientSpec extends Specification { lambda.createFunction { it.functionName('HelloFunction') .runtime(Runtime.NODEJS16_X) - .role('HelloRole') + .role('arn:aws:iam::123456789012:role/HelloRole') .handler('index.handler') .code { it.zipFile(SdkBytes.fromByteArray(functionArchive.bytes)) } .environment { it.variables(MICRNOAUT_ENVIRONMENT: 'itest') } .build() } + + while (lambda.getFunction { it.functionName('HelloFunction') }.configuration().state() != State.ACTIVE) { + Thread.sleep(100) + } } } diff --git a/subprojects/micronaut-amazon-awssdk-lambda/src/test/resources/application-test.yml b/subprojects/micronaut-amazon-awssdk-lambda/src/test/resources/application-test.yml index ddaddf6ab..051f8fdd7 100644 --- a/subprojects/micronaut-amazon-awssdk-lambda/src/test/resources/application-test.yml +++ b/subprojects/micronaut-amazon-awssdk-lambda/src/test/resources/application-test.yml @@ -1,5 +1,5 @@ localstack: - services: lambda + services: lambda,cloudwatchlogs shared: true env: DEBUG: 1 diff --git a/subprojects/micronaut-amazon-awssdk-s3/src/main/java/com/agorapulse/micronaut/amazon/awssdk/s3/DefaultSimpleStorageService.java b/subprojects/micronaut-amazon-awssdk-s3/src/main/java/com/agorapulse/micronaut/amazon/awssdk/s3/DefaultSimpleStorageService.java index 7e2eb9bbf..e0a10b0f2 100644 --- a/subprojects/micronaut-amazon-awssdk-s3/src/main/java/com/agorapulse/micronaut/amazon/awssdk/s3/DefaultSimpleStorageService.java +++ b/subprojects/micronaut-amazon-awssdk-s3/src/main/java/com/agorapulse/micronaut/amazon/awssdk/s3/DefaultSimpleStorageService.java @@ -42,17 +42,12 @@ import java.time.Instant; import java.util.Date; import java.util.List; -import java.util.Optional; import java.util.function.Consumer; import java.util.stream.Collectors; public class DefaultSimpleStorageService implements SimpleStorageService { private static final Logger LOGGER = LoggerFactory.getLogger(DefaultSimpleStorageService.class); - private static final String ALL_USERS = "http://acs.amazonaws.com/groups/global/AllUsers"; - private static final String AUTHENTICATED_USERS = "http://acs.amazonaws.com/groups/global/AuthenticatedUsers"; - private static final String READ = "READ"; - private final S3Client s3; private final S3Presigner presigner; private final String defaultBucketName; @@ -218,7 +213,6 @@ public String moveObject(String sourceBucketName, String sourceKey, String desti GetObjectAclResponse acl = s3.getObjectAcl(b -> b.bucket(sourceBucketName).key(sourceKey)); s3.putObjectAcl(b -> { b.bucket(destinationBucketName).key(destinationKey).accessControlPolicy(p -> p.owner(acl.owner()).grants(acl.grants())); - extractCannedAcl(acl.grants()).ifPresent(b::acl); }); s3.deleteObject(b -> b.bucket(sourceBucketName).key(sourceKey)); @@ -230,18 +224,4 @@ public String moveObject(String sourceBucketName, String sourceKey, String desti } } - private static Optional extractCannedAcl(List grants) { - for (Grant grant : grants) { - if (READ.equals(grant.permissionAsString())) { - if (ALL_USERS.equals(grant.grantee().uri())) { - return Optional.of(ObjectCannedACL.PUBLIC_READ); - } - if (AUTHENTICATED_USERS.equals(grant.grantee().uri())) { - return Optional.of(ObjectCannedACL.AUTHENTICATED_READ); - } - } - } - return Optional.empty(); - } - } diff --git a/subprojects/micronaut-amazon-awssdk-s3/src/test/groovy/com/agorapulse/micronaut/amazon/awssdk/s3/SimpleStorageServiceSpec.groovy b/subprojects/micronaut-amazon-awssdk-s3/src/test/groovy/com/agorapulse/micronaut/amazon/awssdk/s3/SimpleStorageServiceSpec.groovy index fcb1ff9a1..f27f0fa57 100644 --- a/subprojects/micronaut-amazon-awssdk-s3/src/test/groovy/com/agorapulse/micronaut/amazon/awssdk/s3/SimpleStorageServiceSpec.groovy +++ b/subprojects/micronaut-amazon-awssdk-s3/src/test/groovy/com/agorapulse/micronaut/amazon/awssdk/s3/SimpleStorageServiceSpec.groovy @@ -146,7 +146,7 @@ class SimpleStorageServiceSpec extends Specification { } @Unroll - void 'move object created with canned acl #desiredAcl'() { + void 'move object created with canned acl #desiredAcl'(ObjectCannedACL desiredAcl) { when: String newKey = 'mix/moved-' + desiredAcl String oldKey = 'mix/to-be-moved-' + desiredAcl @@ -172,9 +172,6 @@ class SimpleStorageServiceSpec extends Specification { amazonS3.getObjectTagging { it.bucket(MY_BUCKET).key(newKey) } .tagSet() .any { it.key() == 'foo' && it.value() == 'bar' } - - !service.exists(oldKey) - when: GetObjectAclResponse newAcls = amazonS3.getObjectAcl { it.bucket(MY_BUCKET).key(newKey) } then: diff --git a/subprojects/micronaut-amazon-awssdk-sns/micronaut-amazon-awssdk-sns.gradle b/subprojects/micronaut-amazon-awssdk-sns/micronaut-amazon-awssdk-sns.gradle index 68c1ddf35..8d8cf2b01 100644 --- a/subprojects/micronaut-amazon-awssdk-sns/micronaut-amazon-awssdk-sns.gradle +++ b/subprojects/micronaut-amazon-awssdk-sns/micronaut-amazon-awssdk-sns.gradle @@ -23,4 +23,5 @@ dependencies { implementation "io.projectreactor:reactor-core:$projectReactorVersion" testImplementation project(':micronaut-amazon-awssdk-integration-testing') + testImplementation project(':micronaut-amazon-awssdk-sqs') } diff --git a/subprojects/micronaut-amazon-awssdk-sns/src/test/groovy/com/agorapulse/micronaut/amazon/awssdk/sns/SimpleNotificationServiceSpec.groovy b/subprojects/micronaut-amazon-awssdk-sns/src/test/groovy/com/agorapulse/micronaut/amazon/awssdk/sns/SimpleNotificationServiceSpec.groovy index 56852a0c3..9de9d98c2 100644 --- a/subprojects/micronaut-amazon-awssdk-sns/src/test/groovy/com/agorapulse/micronaut/amazon/awssdk/sns/SimpleNotificationServiceSpec.groovy +++ b/subprojects/micronaut-amazon-awssdk-sns/src/test/groovy/com/agorapulse/micronaut/amazon/awssdk/sns/SimpleNotificationServiceSpec.groovy @@ -17,9 +17,9 @@ */ package com.agorapulse.micronaut.amazon.awssdk.sns +import com.agorapulse.micronaut.amazon.awssdk.sqs.SimpleQueueService import io.micronaut.context.annotation.Property import io.micronaut.test.extensions.spock.annotation.MicronautTest -import spock.lang.PendingFeature import spock.lang.Shared import spock.lang.Specification import spock.lang.Stepwise @@ -37,11 +37,13 @@ import jakarta.inject.Inject @Property(name = 'aws.sns.ios-sandbox.arn', value = IOS_SANDBOX_APP_ARN) @Property(name = 'aws.sns.android.arn', value = ANDROID_APP_ARN) @Property(name = 'aws.sns.amazon.arn', value = AMAZON_APP_ARN) +@Property(name = 'aws.sqs.queue', value = TEST_QUEUE) class SimpleNotificationServiceSpec extends Specification { // end::header[] private static final String TEST_TOPIC = 'TestTopic' + private static final String TEST_QUEUE = 'TestQueue' private static final String IOS_APP_ARN = 'arn:aws:sns:us-east-1:000000000000:app/APNS/IOS-APP' private static final String IOS_SANDBOX_APP_ARN = 'arn:aws:sns:us-east-1:000000000000:app/APNS_SANDBOX/IOS-APP' private static final String ANDROID_APP_ARN = 'arn:aws:sns:us-east-1:000000000000:app/GCM/ANDROID-APP' @@ -52,6 +54,8 @@ class SimpleNotificationServiceSpec extends Specification { // end::setup[] @Inject SimpleNotificationServiceConfiguration configuration + @Inject SimpleQueueService simpleQueueService + @Shared String androidEndpointArn @Shared String amazonEndpointArn @Shared String iosEndpointArn @@ -64,6 +68,8 @@ class SimpleNotificationServiceSpec extends Specification { service.createIosApplication('IOS-APP', 'API-KEY', 'fake-cert', true) service.createAndroidApplication('ANDROID-APP', 'API-KEY') service.createAmazonApplication('AMAZON-APP', 'API-KEY', 'API-SECRET') + + simpleQueueService.createQueue(TEST_QUEUE) } void 'new topic'() { @@ -125,7 +131,6 @@ class SimpleNotificationServiceSpec extends Specification { service.registerIosSandboxDevice('ANOTHER-IOS-SANDBOX-TOKEN') } - @PendingFeature (reason = 'Needs to be tested with real SNS') void 'register iOS sandbox device with the same token'() { expect: iosSandboxEndpointArn == service.createPlatformEndpoint(configuration.iosSandbox.arn, 'IOS-SANDBOX-TOKEN') @@ -208,7 +213,7 @@ class SimpleNotificationServiceSpec extends Specification { void 'subscribe to the queue'() { when: - subscriptionArn = service.subscribeTopicWithQueue(topicArn, 'fake-queue-arn') + subscriptionArn = service.subscribeTopicWithQueue(topicArn, simpleQueueService.getQueueArn(TEST_QUEUE)) then: subscriptionArn } diff --git a/subprojects/micronaut-amazon-awssdk-sns/src/test/resources/application-test.yml b/subprojects/micronaut-amazon-awssdk-sns/src/test/resources/application-test.yml index d56444221..59cc78b4d 100644 --- a/subprojects/micronaut-amazon-awssdk-sns/src/test/resources/application-test.yml +++ b/subprojects/micronaut-amazon-awssdk-sns/src/test/resources/application-test.yml @@ -1,3 +1,3 @@ localstack: - services: sns + services: sns,sqs shared: true diff --git a/subprojects/micronaut-aws-sdk-dependencies/micronaut-aws-sdk-dependencies.gradle b/subprojects/micronaut-aws-sdk-dependencies/micronaut-aws-sdk-dependencies.gradle index 830fa4491..907f77060 100644 --- a/subprojects/micronaut-aws-sdk-dependencies/micronaut-aws-sdk-dependencies.gradle +++ b/subprojects/micronaut-aws-sdk-dependencies/micronaut-aws-sdk-dependencies.gradle @@ -55,9 +55,10 @@ dependencies { api "software.amazon.awssdk:sns:$project.awsSdk2Version" api "software.amazon.awssdk:sqs:$project.awsSdk2Version" api "software.amazon.awssdk:sts:$project.awsSdk2Version" + api "software.amazon.awssdk:aws-query-protocol:$project.awsSdk2Version" - api group: 'com.amazonaws', name: 'amazon-dax-client', version: '1.0.205044.0' - api group: 'com.amazonaws', name: 'aws-lambda-java-events', version: '2.2.7' + api group: 'com.amazonaws', name: 'amazon-dax-client', version: '1.0.230341.0' + api group: 'com.amazonaws', name: 'aws-lambda-java-events', version: awsLambdaEventsVersion api group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-cbor', version: project['jackson.datatype.version'] } } diff --git a/subprojects/micronaut-aws-sdk-lambda/src/test/groovy/com/agorapulse/micronaut/aws/lambda/AbstractClientSpec.groovy b/subprojects/micronaut-aws-sdk-lambda/src/test/groovy/com/agorapulse/micronaut/aws/lambda/AbstractClientSpec.groovy index d266725a5..7c69d7e2d 100644 --- a/subprojects/micronaut-aws-sdk-lambda/src/test/groovy/com/agorapulse/micronaut/aws/lambda/AbstractClientSpec.groovy +++ b/subprojects/micronaut-aws-sdk-lambda/src/test/groovy/com/agorapulse/micronaut/aws/lambda/AbstractClientSpec.groovy @@ -22,6 +22,7 @@ import com.amazonaws.services.lambda.AWSLambda import com.amazonaws.services.lambda.model.CreateFunctionRequest import com.amazonaws.services.lambda.model.Environment import com.amazonaws.services.lambda.model.FunctionCode +import com.amazonaws.services.lambda.model.GetFunctionRequest import com.amazonaws.services.lambda.model.Runtime import groovy.transform.CompileStatic import org.zeroturnaround.zip.ZipUtil @@ -46,7 +47,7 @@ abstract class AbstractClientSpec extends Specification { } @CompileStatic - @SuppressWarnings('ImplicitClosureParameter') + @SuppressWarnings(['ImplicitClosureParameter', 'BusyWait']) private void prepareHelloFunction() { boolean alreadyExists = lambda.listFunctions().functions.any { it.functionName == 'HelloFunction' @@ -67,12 +68,16 @@ abstract class AbstractClientSpec extends Specification { CreateFunctionRequest request = new CreateFunctionRequest() .withFunctionName('HelloFunction') .withRuntime(Runtime.Nodejs16X) - .withRole('HelloRole') + .withRole('arn:aws:iam::123456789012:role/HelloRole') .withHandler('index.handler') .withCode(new FunctionCode().withZipFile(ByteBuffer.wrap(functionArchive.bytes))) .withEnvironment(new Environment(variables: [MICRONAUT_ENVIRONMENTS: 'itest'])) lambda.createFunction(request) + + while (lambda.getFunction(new GetFunctionRequest().withFunctionName('HelloFunction')).configuration.state != 'Active') { + Thread.sleep(100) + } } } diff --git a/subprojects/micronaut-aws-sdk-lambda/src/test/resources/application-test.yml b/subprojects/micronaut-aws-sdk-lambda/src/test/resources/application-test.yml index fdac57bd8..d988b40a0 100644 --- a/subprojects/micronaut-aws-sdk-lambda/src/test/resources/application-test.yml +++ b/subprojects/micronaut-aws-sdk-lambda/src/test/resources/application-test.yml @@ -1,5 +1,5 @@ localstack: - services: lambda + services: lambda,cloudwatchlogs shared: true aws: diff --git a/subprojects/micronaut-aws-sdk-sns/src/test/groovy/com/agorapulse/micronaut/aws/sns/SimpleNotificationServiceSpec.groovy b/subprojects/micronaut-aws-sdk-sns/src/test/groovy/com/agorapulse/micronaut/aws/sns/SimpleNotificationServiceSpec.groovy index d4a293e06..072216e14 100644 --- a/subprojects/micronaut-aws-sdk-sns/src/test/groovy/com/agorapulse/micronaut/aws/sns/SimpleNotificationServiceSpec.groovy +++ b/subprojects/micronaut-aws-sdk-sns/src/test/groovy/com/agorapulse/micronaut/aws/sns/SimpleNotificationServiceSpec.groovy @@ -19,7 +19,6 @@ package com.agorapulse.micronaut.aws.sns import io.micronaut.context.annotation.Property import io.micronaut.test.extensions.spock.annotation.MicronautTest -import spock.lang.PendingFeature import spock.lang.Shared import spock.lang.Specification import spock.lang.Stepwise @@ -89,9 +88,6 @@ class SimpleNotificationServiceSpec extends Specification { endpointArn == newEndpointArn } - @PendingFeature( - reason = 'GCM moved int Firebase Console, latest Localstack fails with "Invalid parameter: Attributes Reason: Platform credentials are invalid"' - ) void 'publish direct'() { when: String messageId = service.sendAndroidAppNotification(endpointArn, [message: 'Hello'], 'key') diff --git a/subprojects/micronaut-aws-sdk-sqs/src/test/resources/application-test.yml b/subprojects/micronaut-aws-sdk-sqs/src/test/resources/application-test.yml index cf2f87005..ff9073e80 100644 --- a/subprojects/micronaut-aws-sdk-sqs/src/test/resources/application-test.yml +++ b/subprojects/micronaut-aws-sdk-sqs/src/test/resources/application-test.yml @@ -1,7 +1,8 @@ localstack: services: sqs shared: true - + env: + DEBUG: 1 aws: sqs: tags: