Add S3 Client (#230)

* Checkpoint

* FInish client

---------

Co-authored-by: Dean Hiller <[email protected]>

aws/s3-client/build.gradle

@@ -0,0 +1,30 @@
+plugins {
+    id 'java-library'
+    id 'checkstyle'
+    id 'jacoco' //code coverage
+    id 'eclipse'
+    id 'idea'
+    id 'signing'
+    id 'maven-publish'
+}
+
+group = 'org.webpieces.aws'
+
+apply from: '../../config/global.gradle'
+
+repositories {
+    mavenCentral()
+    maven { url uri('/tmp/myRepo/') } // For testing locally
+}
+
+dependencies {
+
+    implementation libs.aws.s3
+    implementation libs.google.guice
+
+    //implementation libs.jakarta.inject.api
+    //implementation libs.slf4j.api
+
+    api libs.webpieces.core.util
+
+}

aws/s3-client/settings.gradle

@@ -0,0 +1,11 @@
+dependencyResolutionManagement {
+    versionCatalogs {
+        libs {
+            from(files("../../config/libs.versions.toml"))
+        }
+    }
+}
+
+includeBuild '../../core/core-util'
+includeBuild '../../core/core-metrics'
+includeBuild '../../core/core-logging'

aws/s3-client/src/main/java/org/webpieces/aws/storage/api/AWSBlob.java

@@ -0,0 +1,13 @@
+package org.webpieces.aws.storage.api;
+
+public interface AWSBlob {
+
+    String getBucket();
+
+    String getKey();
+
+    String getContentType();
+
+    long getSize();
+
+}

aws/s3-client/src/main/java/org/webpieces/aws/storage/api/AWSRawStorage.java

@@ -0,0 +1,42 @@
+package org.webpieces.aws.storage.api;
+
+import com.google.inject.ImplementedBy;
+import software.amazon.awssdk.services.s3.model.Bucket;
+
+import org.webpieces.aws.storage.impl.raw.AWSRawStorageImpl;
+
+import java.nio.channels.ReadableByteChannel;
+import java.nio.channels.WritableByteChannel;
+import java.util.stream.Stream;
+
+/**
+ * Create as you need GCPRawStorage methods - create in
+ * GCPRawStorage, GCPStorage, GCPRawStorageImpl, LocalStorage for each
+ * method as we go.
+ *
+ * With the switch to the underlying google.Storage interface that is mockable, the implementation
+ * of GCPStorage here is ONE TO ONE since we cannot test ANY code behind this interface as we
+ * swap it out with a mock object to test our systems.
+ *
+ * THIS IS WHAT YOU SHOULD MOCK!!!  It is the lowest level AND 1 to 1 to Google Storage so
+ * we do not have to test anything as Google will test it for us.
+ */
+
+//GCPRawStorageImpl for production, LocalStorage for local dev
+@ImplementedBy(AWSRawStorageImpl.class)
+public interface AWSRawStorage {
+
+    AWSBlob get(String bucket, String blob);
+
+    Stream<AWSBlob> list(String bucket);
+
+    boolean delete(String bucket, String key);
+
+    byte[] readAllBytes(String bucket, String key);
+
+    ReadableByteChannel reader(String bucket, String key);
+
+    WritableByteChannel writer(String bucket, String key);
+
+    boolean copy(String sourceBucket, String sourceKey, String destBucket, String destKey);
+}

aws/s3-client/src/main/java/org/webpieces/aws/storage/api/AWSStorage.java

@@ -0,0 +1,13 @@
+package org.webpieces.aws.storage.api;
+
+import com.google.inject.ImplementedBy;
+import org.webpieces.aws.storage.impl.AWSStorageImpl;
+
+/**
+ * MOCK GCPRawStorage, NOT this class so you are mocking the LOWEST level and testing
+ * your client assertions at test time!!!
+ */
+@ImplementedBy(AWSStorageImpl.class)
+public interface AWSStorage extends AWSRawStorage {
+
+}

aws/s3-client/src/main/java/org/webpieces/aws/storage/impl/AWSStorageImpl.java

@@ -0,0 +1,74 @@
+package org.webpieces.aws.storage.impl;
+
+import org.webpieces.aws.storage.api.AWSBlob;
+import org.webpieces.aws.storage.api.AWSRawStorage;
+import org.webpieces.aws.storage.api.AWSStorage;
+import org.webpieces.util.context.ClientAssertions;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.channels.WritableByteChannel;
+import java.util.stream.Stream;
+
+/**
+ * Since tests mock rawStorage, changes to this class get included in testing.
+ * THIS IS A GOOD THING ^^^^.   Do not break people
+ */
+@Singleton
+public class AWSStorageImpl implements AWSStorage {
+
+    private AWSRawStorage rawStorage;
+    private ClientAssertions clientAssertions;
+    private ChannelWrapper channelWrapper;
+
+    @Inject
+    public AWSStorageImpl(AWSRawStorage rawStorage, ClientAssertions clientAssertions, ChannelWrapper channelWrapper) {
+        this.rawStorage = rawStorage;
+        this.clientAssertions = clientAssertions;
+        this.channelWrapper = channelWrapper;
+    }
+
+    @Override
+    public AWSBlob get(String bucket, String key) {
+        clientAssertions.throwIfCannotGoRemote();
+        return rawStorage.get(bucket, key);
+    }
+
+    @Override
+    public Stream<AWSBlob> list(String bucket) {
+        clientAssertions.throwIfCannotGoRemote();
+        return rawStorage.list(bucket);
+    }
+
+    @Override
+    public boolean delete(String bucket, String key) {
+        clientAssertions.throwIfCannotGoRemote();
+        return rawStorage.delete(bucket, key);
+    }
+
+    @Override
+    public byte[] readAllBytes(String bucket, String key) {
+        clientAssertions.throwIfCannotGoRemote();
+        return rawStorage.readAllBytes(bucket, key);
+    }
+
+    @Override
+    public ReadableByteChannel reader(String bucket, String key) {
+        clientAssertions.throwIfCannotGoRemote();
+        return channelWrapper.newChannelProxy(ReadableByteChannel.class, rawStorage.reader(bucket, key));
+    }
+
+    @Override
+    public WritableByteChannel writer(String bucket, String key) {
+        clientAssertions.throwIfCannotGoRemote();
+        return channelWrapper.newChannelProxy(WritableByteChannel.class, rawStorage.writer(bucket, key));
+    }
+
+    @Override
+    public boolean copy(String sourceBucket, String sourceKey, String destBucket, String destKey) {
+        clientAssertions.throwIfCannotGoRemote();
+        //return null;
+        return rawStorage.copy(sourceBucket, sourceKey, destBucket, destKey);
+    }
+}

aws/s3-client/src/main/java/org/webpieces/aws/storage/impl/ChannelInvocationHandler.java

@@ -0,0 +1,33 @@
+package org.webpieces.aws.storage.impl;
+
+import org.webpieces.util.context.ClientAssertions;
+
+import javax.inject.Inject;
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.Method;
+import java.nio.channels.Channel;
+
+public class ChannelInvocationHandler implements InvocationHandler {
+
+    private ClientAssertions clientAssertions;
+    private Channel channel;
+
+    @Inject
+    public ChannelInvocationHandler(ClientAssertions clientAssertions) {
+        this.clientAssertions = clientAssertions;
+        this.channel = channel;
+    }
+
+    @Override
+    public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
+
+        clientAssertions.throwIfCannotGoRemote();
+
+        return method.invoke(channel, args);
+
+    }
+
+    public void setChannel(Channel channel) {
+        this.channel = channel;
+    }
+}

aws/s3-client/src/main/java/org/webpieces/aws/storage/impl/ChannelWrapper.java

@@ -0,0 +1,28 @@
+package org.webpieces.aws.storage.impl;
+
+import java.lang.reflect.Proxy;
+import java.nio.channels.Channel;
+import javax.inject.Inject;
+import javax.inject.Provider;
+
+public class ChannelWrapper {
+
+    private Provider<ChannelInvocationHandler> invocHandlerProvider;
+
+    @Inject
+    public ChannelWrapper(Provider<ChannelInvocationHandler> invocHandlerProvider) {
+        this.invocHandlerProvider = invocHandlerProvider;
+    }
+
+    public <T extends Channel> T newChannelProxy(Class<T> intf, T channel) {
+        ChannelInvocationHandler invocHandler = invocHandlerProvider.get();
+        invocHandler.setChannel(channel);
+
+        return (T) Proxy.newProxyInstance(channel.getClass().getClassLoader(),
+                new Class[] {intf, Channel.class},
+                invocHandler);
+    }
+
+
+
+}

aws/s3-client/src/main/java/org/webpieces/aws/storage/impl/S3WritableByteChannel.java

@@ -0,0 +1,98 @@
+package org.webpieces.aws.storage.impl;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.WritableByteChannel;
+
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.UploadPartRequest;
+
+public class S3WritableByteChannel implements WritableByteChannel {
+
+    private final S3Client client;
+    private final String bucket;
+    private final String key;
+    private final ByteBuffer bb;
+
+    private boolean closed = false;
+    private int part = 1;
+
+    public S3WritableByteChannel(final S3Client client, final String bucket, final String key) {
+
+        this.client = client;
+        this.bucket = bucket;
+        this.key = key;
+
+        this.bb = ByteBuffer.allocate(1024 * 1024 * 8);
+
+    }
+
+    @Override
+    public int write(ByteBuffer src) throws IOException {
+
+        int bytesWritten = 0;
+
+        while(src.remaining() > 0) {
+
+            if ((src.remaining()) <= (bb.remaining())) {
+                bytesWritten += (src.remaining());
+                bb.put(src);
+            } else {
+
+                ByteBuffer tmp = src.duplicate();
+                tmp.limit(bb.remaining());
+
+                bytesWritten += tmp.remaining();
+                bb.put(tmp);
+
+            }
+
+            if (!bb.hasRemaining()) {
+                bb.flip();
+                uploadPart();
+            }
+
+        }
+
+        return bytesWritten;
+
+    }
+
+    @Override
+    public boolean isOpen() {
+        return closed;
+    }
+
+    @Override
+    public void close() throws IOException {
+
+        if(closed) {
+            return;
+        }
+
+        if(bb.position() > 0) {
+            bb.flip();
+            uploadPart();
+        }
+
+        closed = true;
+
+    }
+
+    private void uploadPart() {
+
+        UploadPartRequest request = UploadPartRequest.builder()
+                .bucket(bucket)
+                .key(key)
+                .partNumber(part)
+                .build();
+
+        client.uploadPart(request, RequestBody.fromRemainingByteBuffer(bb));
+
+        part++;
+        bb.clear();
+
+    }
+
+}

aws/s3-client/src/main/java/org/webpieces/aws/storage/impl/StorageSupplier.java

@@ -0,0 +1,16 @@
+package org.webpieces.aws.storage.impl;
+
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+
+import java.io.Serializable;
+import java.util.function.Supplier;
+
+public class StorageSupplier implements Supplier<S3Client>, Serializable {
+
+    @Override
+    public S3Client get() {
+        return S3Client.builder().region(Region.US_WEST_2).build();
+    }
+
+}