[2.2.x] upgrade aws libraries (#224)

* Upgraded to AWS SDK version used by MariaDB (#219) * Upgraded to the latest AWS Libraries * reverted to older version of SDK used by MariaDB drivers * using dynamic SQS URLs * fixed tests * fixed tests * simplified moving S3 objects, fixed tests (cherry picked from commit a9610e1) * Upgraded to the latest AWS libraries (#223) * Upgraded to the latest AWS libraries * fixed version properties (cherry picked from commit 40b816f) * java 8 compatibility * backported fixed tests
agorapulse · Apr 2, 2024 · 7818d93 · 7818d93
1 parent 647c68b
commit 7818d93
Show file tree

Hide file tree

Showing 14 changed files with 47 additions and 46 deletions.
diff --git a/gradle.properties b/gradle.properties
@@ -22,9 +22,10 @@ micronautVersion = 3.2.1
 gruVersion = 0.9.1
 druVersion = 0.8.1
 spockVersion = 2.0-groovy-3.0
-awsSdkVersion = 1.12.299
-awsSdk2Version = 2.18.40
+awsSdkVersion = 1.12.692
+awsSdk2Version = 2.25.22
 awsLambdaRuntimeVersion=2.5.0
+awsLambdaEventsVersion=3.11.4
 testcontainersVersion = 1.17.3
 kordampVersion=0.45.0
 closureSupportVersion=0.6.3

diff --git a/...agorapulse/micronaut/amazon/awssdk/itest/localstack/LocalstackContainerConfiguration.java b/...agorapulse/micronaut/amazon/awssdk/itest/localstack/LocalstackContainerConfiguration.java
@@ -19,14 +19,17 @@
 
 import io.micronaut.context.annotation.ConfigurationProperties;
 
-import java.util.ArrayList;
-import java.util.List;
+import java.util.*;
 
 @ConfigurationProperties("localstack")
 public class LocalstackContainerConfiguration extends AbstractContainerConfiguration {
 
+    public static final Map<String, String> ALWAYS_PRESENT_ENV = Collections.singletonMap(
+        "SQS_ENDPOINT_STRATEGY", "dynamic"
+    );
+
     public static final String DEFAULT_IMAGE = "localstack/localstack";
-    public static final String DEFAULT_TAG = "1.3.0";
+    public static final String DEFAULT_TAG = "3.2.0";
 
     public LocalstackContainerConfiguration() {
         setImage(DEFAULT_IMAGE);
@@ -43,4 +46,10 @@ public void setServices(List<String> services) {
         this.services = services;
     }
 
+    @Override
+    public Map<String, String> getEnv() {
+        Map<String, String> allEnv = new HashMap<>(super.getEnv());
+        ALWAYS_PRESENT_ENV.forEach(allEnv::putIfAbsent);
+        return allEnv;
+    }
 }
diff --git a/...a/src/test/groovy/com/agorapulse/micronaut/amazon/awssdk/lambda/AbstractClientSpec.groovy b/...a/src/test/groovy/com/agorapulse/micronaut/amazon/awssdk/lambda/AbstractClientSpec.groovy
@@ -22,6 +22,7 @@ import org.zeroturnaround.zip.ZipUtil
 import software.amazon.awssdk.core.SdkBytes
 import software.amazon.awssdk.services.lambda.LambdaClient
 import software.amazon.awssdk.services.lambda.model.Runtime
+import software.amazon.awssdk.services.lambda.model.State
 import spock.lang.Shared
 import spock.lang.Specification
 import spock.lang.TempDir
@@ -41,7 +42,7 @@ abstract class AbstractClientSpec extends Specification {
         prepareHelloFunction()
     }
 
-    @SuppressWarnings('ImplicitClosureParameter')
+    @SuppressWarnings(['ImplicitClosureParameter', 'BusyWait'])
     private void prepareHelloFunction() {
         boolean alreadyExists = lambda.listFunctions().functions().any {
             it.functionName() == 'HelloFunction'
@@ -62,12 +63,16 @@ abstract class AbstractClientSpec extends Specification {
         lambda.createFunction {
             it.functionName('HelloFunction')
               .runtime(Runtime.NODEJS16_X)
-              .role('HelloRole')
+              .role('arn:aws:iam::123456789012:role/HelloRole')
               .handler('index.handler')
               .code { it.zipFile(SdkBytes.fromByteArray(functionArchive.bytes)) }
               .environment { it.variables(MICRNOAUT_ENVIRONMENT: 'itest') }
               .build()
         }
+
+        while (lambda.getFunction { it.functionName('HelloFunction') }.configuration().state() != State.ACTIVE) {
+            Thread.sleep(100)
+        }
     }
 
 }
diff --git a/subprojects/micronaut-amazon-awssdk-lambda/src/test/resources/application-test.yml b/subprojects/micronaut-amazon-awssdk-lambda/src/test/resources/application-test.yml
@@ -1,5 +1,5 @@
 localstack:
-  services: lambda
+  services: lambda,cloudwatchlogs
   shared: true
   env:
     DEBUG: 1

diff --git a/.../src/main/java/com/agorapulse/micronaut/amazon/awssdk/s3/DefaultSimpleStorageService.java b/.../src/main/java/com/agorapulse/micronaut/amazon/awssdk/s3/DefaultSimpleStorageService.java
@@ -42,17 +42,12 @@
 import java.time.Instant;
 import java.util.Date;
 import java.util.List;
-import java.util.Optional;
 import java.util.function.Consumer;
 import java.util.stream.Collectors;
 
 public class DefaultSimpleStorageService implements SimpleStorageService {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(DefaultSimpleStorageService.class);
-    private static final String ALL_USERS = "http://acs.amazonaws.com/groups/global/AllUsers";
-    private static final String AUTHENTICATED_USERS = "http://acs.amazonaws.com/groups/global/AuthenticatedUsers";
-    private static final String READ = "READ";
-
     private final S3Client s3;
     private final S3Presigner presigner;
     private final String defaultBucketName;
@@ -218,7 +213,6 @@ public String moveObject(String sourceBucketName, String sourceKey, String desti
             GetObjectAclResponse acl = s3.getObjectAcl(b -> b.bucket(sourceBucketName).key(sourceKey));
             s3.putObjectAcl(b -> {
                 b.bucket(destinationBucketName).key(destinationKey).accessControlPolicy(p -> p.owner(acl.owner()).grants(acl.grants()));
-                extractCannedAcl(acl.grants()).ifPresent(b::acl);
             });
 
             s3.deleteObject(b -> b.bucket(sourceBucketName).key(sourceKey));
@@ -230,18 +224,4 @@ public String moveObject(String sourceBucketName, String sourceKey, String desti
         }
     }
 
-    private static Optional<ObjectCannedACL> extractCannedAcl(List<Grant> grants) {
-        for (Grant grant : grants) {
-            if (READ.equals(grant.permissionAsString())) {
-                if (ALL_USERS.equals(grant.grantee().uri())) {
-                    return Optional.of(ObjectCannedACL.PUBLIC_READ);
-                }
-                if (AUTHENTICATED_USERS.equals(grant.grantee().uri())) {
-                    return Optional.of(ObjectCannedACL.AUTHENTICATED_READ);
-                }
-            }
-        }
-        return Optional.empty();
-    }
-
 }
diff --git a/...src/test/groovy/com/agorapulse/micronaut/amazon/awssdk/s3/SimpleStorageServiceSpec.groovy b/...src/test/groovy/com/agorapulse/micronaut/amazon/awssdk/s3/SimpleStorageServiceSpec.groovy
@@ -146,7 +146,7 @@ class SimpleStorageServiceSpec extends Specification {
     }
 
     @Unroll
-    void 'move object created with canned acl #desiredAcl'() {
+    void 'move object created with canned acl #desiredAcl'(ObjectCannedACL desiredAcl) {
         when:
             String newKey = 'mix/moved-' + desiredAcl
             String oldKey = 'mix/to-be-moved-' + desiredAcl
@@ -172,9 +172,6 @@ class SimpleStorageServiceSpec extends Specification {
             amazonS3.getObjectTagging { it.bucket(MY_BUCKET).key(newKey) }
                 .tagSet()
                 .any { it.key() == 'foo' && it.value() == 'bar' }
-
-            !service.exists(oldKey)
-
         when:
             GetObjectAclResponse newAcls = amazonS3.getObjectAcl { it.bucket(MY_BUCKET).key(newKey) }
         then:

diff --git a/subprojects/micronaut-amazon-awssdk-sns/micronaut-amazon-awssdk-sns.gradle b/subprojects/micronaut-amazon-awssdk-sns/micronaut-amazon-awssdk-sns.gradle
@@ -23,4 +23,5 @@ dependencies {
     implementation "io.projectreactor:reactor-core:$projectReactorVersion"
 
     testImplementation project(':micronaut-amazon-awssdk-integration-testing')
+    testImplementation project(':micronaut-amazon-awssdk-sqs')
 }
diff --git a/...st/groovy/com/agorapulse/micronaut/amazon/awssdk/sns/SimpleNotificationServiceSpec.groovy b/...st/groovy/com/agorapulse/micronaut/amazon/awssdk/sns/SimpleNotificationServiceSpec.groovy
@@ -17,9 +17,9 @@
  */
 package com.agorapulse.micronaut.amazon.awssdk.sns
 
+import com.agorapulse.micronaut.amazon.awssdk.sqs.SimpleQueueService
 import io.micronaut.context.annotation.Property
 import io.micronaut.test.extensions.spock.annotation.MicronautTest
-import spock.lang.PendingFeature
 import spock.lang.Shared
 import spock.lang.Specification
 import spock.lang.Stepwise
@@ -37,11 +37,13 @@ import jakarta.inject.Inject
 @Property(name = 'aws.sns.ios-sandbox.arn', value = IOS_SANDBOX_APP_ARN)
 @Property(name = 'aws.sns.android.arn', value = ANDROID_APP_ARN)
 @Property(name = 'aws.sns.amazon.arn', value = AMAZON_APP_ARN)
+@Property(name = 'aws.sqs.queue', value = TEST_QUEUE)
 class SimpleNotificationServiceSpec extends Specification {
 
 // end::header[]
 
     private static final String TEST_TOPIC = 'TestTopic'
+    private static final String TEST_QUEUE = 'TestQueue'
     private static final String IOS_APP_ARN = 'arn:aws:sns:us-east-1:000000000000:app/APNS/IOS-APP'
     private static final String IOS_SANDBOX_APP_ARN = 'arn:aws:sns:us-east-1:000000000000:app/APNS_SANDBOX/IOS-APP'
     private static final String ANDROID_APP_ARN = 'arn:aws:sns:us-east-1:000000000000:app/GCM/ANDROID-APP'
@@ -52,6 +54,8 @@ class SimpleNotificationServiceSpec extends Specification {
     // end::setup[]
     @Inject SimpleNotificationServiceConfiguration configuration
 
+    @Inject SimpleQueueService simpleQueueService
+
     @Shared String androidEndpointArn
     @Shared String amazonEndpointArn
     @Shared String iosEndpointArn
@@ -64,6 +68,8 @@ class SimpleNotificationServiceSpec extends Specification {
         service.createIosApplication('IOS-APP', 'API-KEY', 'fake-cert', true)
         service.createAndroidApplication('ANDROID-APP', 'API-KEY')
         service.createAmazonApplication('AMAZON-APP', 'API-KEY', 'API-SECRET')
+
+        simpleQueueService.createQueue(TEST_QUEUE)
     }
 
     void 'new topic'() {
@@ -125,7 +131,6 @@ class SimpleNotificationServiceSpec extends Specification {
             service.registerIosSandboxDevice('ANOTHER-IOS-SANDBOX-TOKEN')
     }
 
-    @PendingFeature (reason = 'Needs to be tested with real SNS')
     void 'register iOS sandbox device with the same token'() {
         expect:
             iosSandboxEndpointArn == service.createPlatformEndpoint(configuration.iosSandbox.arn, 'IOS-SANDBOX-TOKEN')
@@ -208,7 +213,7 @@ class SimpleNotificationServiceSpec extends Specification {
 
     void 'subscribe to the queue'() {
         when:
-            subscriptionArn = service.subscribeTopicWithQueue(topicArn, 'fake-queue-arn')
+            subscriptionArn = service.subscribeTopicWithQueue(topicArn, simpleQueueService.getQueueArn(TEST_QUEUE))
         then:
             subscriptionArn
     }

diff --git a/subprojects/micronaut-amazon-awssdk-sns/src/test/resources/application-test.yml b/subprojects/micronaut-amazon-awssdk-sns/src/test/resources/application-test.yml
@@ -1,3 +1,3 @@
 localstack:
-  services: sns
+  services: sns,sqs
   shared: true
diff --git a/subprojects/micronaut-aws-sdk-dependencies/micronaut-aws-sdk-dependencies.gradle b/subprojects/micronaut-aws-sdk-dependencies/micronaut-aws-sdk-dependencies.gradle
@@ -55,9 +55,10 @@ dependencies {
         api "software.amazon.awssdk:sns:$project.awsSdk2Version"
         api "software.amazon.awssdk:sqs:$project.awsSdk2Version"
         api "software.amazon.awssdk:sts:$project.awsSdk2Version"
+        api "software.amazon.awssdk:aws-query-protocol:$project.awsSdk2Version"
 
-        api group: 'com.amazonaws', name: 'amazon-dax-client', version: '1.0.205044.0'
-        api group: 'com.amazonaws', name: 'aws-lambda-java-events', version: '2.2.7'
+        api group: 'com.amazonaws', name: 'amazon-dax-client', version: '1.0.230341.0'
+        api group: 'com.amazonaws', name: 'aws-lambda-java-events', version: awsLambdaEventsVersion
         api group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-cbor', version: project['jackson.datatype.version']
     }
 }

diff --git a/...-sdk-lambda/src/test/groovy/com/agorapulse/micronaut/aws/lambda/AbstractClientSpec.groovy b/...-sdk-lambda/src/test/groovy/com/agorapulse/micronaut/aws/lambda/AbstractClientSpec.groovy
@@ -22,6 +22,7 @@ import com.amazonaws.services.lambda.AWSLambda
 import com.amazonaws.services.lambda.model.CreateFunctionRequest
 import com.amazonaws.services.lambda.model.Environment
 import com.amazonaws.services.lambda.model.FunctionCode
+import com.amazonaws.services.lambda.model.GetFunctionRequest
 import com.amazonaws.services.lambda.model.Runtime
 import groovy.transform.CompileStatic
 import org.zeroturnaround.zip.ZipUtil
@@ -46,7 +47,7 @@ abstract class AbstractClientSpec extends Specification {
     }
 
     @CompileStatic
-    @SuppressWarnings('ImplicitClosureParameter')
+    @SuppressWarnings(['ImplicitClosureParameter', 'BusyWait'])
     private void prepareHelloFunction() {
         boolean alreadyExists = lambda.listFunctions().functions.any {
             it.functionName == 'HelloFunction'
@@ -67,12 +68,16 @@ abstract class AbstractClientSpec extends Specification {
         CreateFunctionRequest request = new CreateFunctionRequest()
             .withFunctionName('HelloFunction')
             .withRuntime(Runtime.Nodejs16X)
-            .withRole('HelloRole')
+            .withRole('arn:aws:iam::123456789012:role/HelloRole')
             .withHandler('index.handler')
             .withCode(new FunctionCode().withZipFile(ByteBuffer.wrap(functionArchive.bytes)))
             .withEnvironment(new Environment(variables: [MICRONAUT_ENVIRONMENTS: 'itest']))
 
         lambda.createFunction(request)
+
+        while (lambda.getFunction(new GetFunctionRequest().withFunctionName('HelloFunction')).configuration.state != 'Active') {
+            Thread.sleep(100)
+        }
     }
 
 }
diff --git a/subprojects/micronaut-aws-sdk-lambda/src/test/resources/application-test.yml b/subprojects/micronaut-aws-sdk-lambda/src/test/resources/application-test.yml
@@ -1,5 +1,5 @@
 localstack:
-  services: lambda
+  services: lambda,cloudwatchlogs
   shared: true
 
 aws:

diff --git a/...sns/src/test/groovy/com/agorapulse/micronaut/aws/sns/SimpleNotificationServiceSpec.groovy b/...sns/src/test/groovy/com/agorapulse/micronaut/aws/sns/SimpleNotificationServiceSpec.groovy
@@ -19,7 +19,6 @@ package com.agorapulse.micronaut.aws.sns
 
 import io.micronaut.context.annotation.Property
 import io.micronaut.test.extensions.spock.annotation.MicronautTest
-import spock.lang.PendingFeature
 import spock.lang.Shared
 import spock.lang.Specification
 import spock.lang.Stepwise
@@ -89,9 +88,6 @@ class SimpleNotificationServiceSpec extends Specification {
             endpointArn == newEndpointArn
     }
 
-    @PendingFeature(
-        reason = 'GCM moved int Firebase Console, latest Localstack fails with "Invalid parameter: Attributes Reason: Platform credentials are invalid"'
-    )
     void 'publish direct'() {
         when:
             String messageId = service.sendAndroidAppNotification(endpointArn, [message: 'Hello'], 'key')

diff --git a/subprojects/micronaut-aws-sdk-sqs/src/test/resources/application-test.yml b/subprojects/micronaut-aws-sdk-sqs/src/test/resources/application-test.yml
@@ -1,7 +1,8 @@
 localstack:
   services: sqs
   shared: true
-
+  env:
+    DEBUG: 1
 aws:
   sqs:
     tags: