From f58bef69498acac3656cec64df1d85c8f7f0a3be Mon Sep 17 00:00:00 2001 From: Zachary Reed Date: Tue, 15 Aug 2023 07:05:53 -0400 Subject: [PATCH] Add S3 Client (#230) * Checkpoint * FInish client --------- Co-authored-by: Dean Hiller --- aws/s3-client/build.gradle | 30 +++ aws/s3-client/settings.gradle | 11 + .../webpieces/aws/storage/api/AWSBlob.java | 13 + .../aws/storage/api/AWSRawStorage.java | 42 ++++ .../webpieces/aws/storage/api/AWSStorage.java | 13 + .../aws/storage/impl/AWSStorageImpl.java | 74 ++++++ .../impl/ChannelInvocationHandler.java | 33 +++ .../aws/storage/impl/ChannelWrapper.java | 28 +++ .../storage/impl/S3WritableByteChannel.java | 98 ++++++++ .../aws/storage/impl/StorageSupplier.java | 16 ++ .../storage/impl/local/LocalAWSBlobImpl.java | 37 +++ .../aws/storage/impl/local/LocalStorage.java | 186 ++++++++++++++ .../aws/storage/impl/raw/AWSBlobImpl.java | 38 +++ .../storage/impl/raw/AWSRawStorageImpl.java | 90 +++++++ .../org/webpieces/aws/storage/LocalTest.java | 232 ++++++++++++++++++ .../webpieces/aws/storage/ProductionTest.java | 29 +++ .../aws/storage/TestClientAssertions.java | 18 ++ .../aws/storage/TestLocalModule.java | 16 ++ .../webpieces/aws/storage/TestProdModule.java | 14 ++ .../src/test/resources/testbucket/mytest.txt | 1 + config/libs.versions.toml | 4 +- settings.gradle | 1 + 22 files changed, 1023 insertions(+), 1 deletion(-) create mode 100644 aws/s3-client/build.gradle create mode 100644 aws/s3-client/settings.gradle create mode 100644 aws/s3-client/src/main/java/org/webpieces/aws/storage/api/AWSBlob.java create mode 100644 aws/s3-client/src/main/java/org/webpieces/aws/storage/api/AWSRawStorage.java create mode 100644 aws/s3-client/src/main/java/org/webpieces/aws/storage/api/AWSStorage.java create mode 100644 aws/s3-client/src/main/java/org/webpieces/aws/storage/impl/AWSStorageImpl.java create mode 100644 aws/s3-client/src/main/java/org/webpieces/aws/storage/impl/ChannelInvocationHandler.java create mode 100644 aws/s3-client/src/main/java/org/webpieces/aws/storage/impl/ChannelWrapper.java create mode 100644 aws/s3-client/src/main/java/org/webpieces/aws/storage/impl/S3WritableByteChannel.java create mode 100644 aws/s3-client/src/main/java/org/webpieces/aws/storage/impl/StorageSupplier.java create mode 100644 aws/s3-client/src/main/java/org/webpieces/aws/storage/impl/local/LocalAWSBlobImpl.java create mode 100644 aws/s3-client/src/main/java/org/webpieces/aws/storage/impl/local/LocalStorage.java create mode 100644 aws/s3-client/src/main/java/org/webpieces/aws/storage/impl/raw/AWSBlobImpl.java create mode 100644 aws/s3-client/src/main/java/org/webpieces/aws/storage/impl/raw/AWSRawStorageImpl.java create mode 100644 aws/s3-client/src/test/java/org/webpieces/aws/storage/LocalTest.java create mode 100644 aws/s3-client/src/test/java/org/webpieces/aws/storage/ProductionTest.java create mode 100644 aws/s3-client/src/test/java/org/webpieces/aws/storage/TestClientAssertions.java create mode 100644 aws/s3-client/src/test/java/org/webpieces/aws/storage/TestLocalModule.java create mode 100644 aws/s3-client/src/test/java/org/webpieces/aws/storage/TestProdModule.java create mode 100644 aws/s3-client/src/test/resources/testbucket/mytest.txt diff --git a/aws/s3-client/build.gradle b/aws/s3-client/build.gradle new file mode 100644 index 000000000..11f482c2c --- /dev/null +++ b/aws/s3-client/build.gradle @@ -0,0 +1,30 @@ +plugins { + id 'java-library' + id 'checkstyle' + id 'jacoco' //code coverage + id 'eclipse' + id 'idea' + id 'signing' + id 'maven-publish' +} + +group = 'org.webpieces.aws' + +apply from: '../../config/global.gradle' + +repositories { + mavenCentral() + maven { url uri('/tmp/myRepo/') } // For testing locally +} + +dependencies { + + implementation libs.aws.s3 + implementation libs.google.guice + + //implementation libs.jakarta.inject.api + //implementation libs.slf4j.api + + api libs.webpieces.core.util + +} diff --git a/aws/s3-client/settings.gradle b/aws/s3-client/settings.gradle new file mode 100644 index 000000000..2c49acd52 --- /dev/null +++ b/aws/s3-client/settings.gradle @@ -0,0 +1,11 @@ +dependencyResolutionManagement { + versionCatalogs { + libs { + from(files("../../config/libs.versions.toml")) + } + } +} + +includeBuild '../../core/core-util' +includeBuild '../../core/core-metrics' +includeBuild '../../core/core-logging' diff --git a/aws/s3-client/src/main/java/org/webpieces/aws/storage/api/AWSBlob.java b/aws/s3-client/src/main/java/org/webpieces/aws/storage/api/AWSBlob.java new file mode 100644 index 000000000..269eae4e3 --- /dev/null +++ b/aws/s3-client/src/main/java/org/webpieces/aws/storage/api/AWSBlob.java @@ -0,0 +1,13 @@ +package org.webpieces.aws.storage.api; + +public interface AWSBlob { + + String getBucket(); + + String getKey(); + + String getContentType(); + + long getSize(); + +} \ No newline at end of file diff --git a/aws/s3-client/src/main/java/org/webpieces/aws/storage/api/AWSRawStorage.java b/aws/s3-client/src/main/java/org/webpieces/aws/storage/api/AWSRawStorage.java new file mode 100644 index 000000000..e8d31ff6b --- /dev/null +++ b/aws/s3-client/src/main/java/org/webpieces/aws/storage/api/AWSRawStorage.java @@ -0,0 +1,42 @@ +package org.webpieces.aws.storage.api; + +import com.google.inject.ImplementedBy; +import software.amazon.awssdk.services.s3.model.Bucket; + +import org.webpieces.aws.storage.impl.raw.AWSRawStorageImpl; + +import java.nio.channels.ReadableByteChannel; +import java.nio.channels.WritableByteChannel; +import java.util.stream.Stream; + +/** + * Create as you need GCPRawStorage methods - create in + * GCPRawStorage, GCPStorage, GCPRawStorageImpl, LocalStorage for each + * method as we go. + * + * With the switch to the underlying google.Storage interface that is mockable, the implementation + * of GCPStorage here is ONE TO ONE since we cannot test ANY code behind this interface as we + * swap it out with a mock object to test our systems. + * + * THIS IS WHAT YOU SHOULD MOCK!!! It is the lowest level AND 1 to 1 to Google Storage so + * we do not have to test anything as Google will test it for us. + */ + +//GCPRawStorageImpl for production, LocalStorage for local dev +@ImplementedBy(AWSRawStorageImpl.class) +public interface AWSRawStorage { + + AWSBlob get(String bucket, String blob); + + Stream list(String bucket); + + boolean delete(String bucket, String key); + + byte[] readAllBytes(String bucket, String key); + + ReadableByteChannel reader(String bucket, String key); + + WritableByteChannel writer(String bucket, String key); + + boolean copy(String sourceBucket, String sourceKey, String destBucket, String destKey); +} diff --git a/aws/s3-client/src/main/java/org/webpieces/aws/storage/api/AWSStorage.java b/aws/s3-client/src/main/java/org/webpieces/aws/storage/api/AWSStorage.java new file mode 100644 index 000000000..e2ca5ecdf --- /dev/null +++ b/aws/s3-client/src/main/java/org/webpieces/aws/storage/api/AWSStorage.java @@ -0,0 +1,13 @@ +package org.webpieces.aws.storage.api; + +import com.google.inject.ImplementedBy; +import org.webpieces.aws.storage.impl.AWSStorageImpl; + +/** + * MOCK GCPRawStorage, NOT this class so you are mocking the LOWEST level and testing + * your client assertions at test time!!! + */ +@ImplementedBy(AWSStorageImpl.class) +public interface AWSStorage extends AWSRawStorage { + +} diff --git a/aws/s3-client/src/main/java/org/webpieces/aws/storage/impl/AWSStorageImpl.java b/aws/s3-client/src/main/java/org/webpieces/aws/storage/impl/AWSStorageImpl.java new file mode 100644 index 000000000..a955f9d54 --- /dev/null +++ b/aws/s3-client/src/main/java/org/webpieces/aws/storage/impl/AWSStorageImpl.java @@ -0,0 +1,74 @@ +package org.webpieces.aws.storage.impl; + +import org.webpieces.aws.storage.api.AWSBlob; +import org.webpieces.aws.storage.api.AWSRawStorage; +import org.webpieces.aws.storage.api.AWSStorage; +import org.webpieces.util.context.ClientAssertions; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.nio.channels.ReadableByteChannel; +import java.nio.channels.WritableByteChannel; +import java.util.stream.Stream; + +/** + * Since tests mock rawStorage, changes to this class get included in testing. + * THIS IS A GOOD THING ^^^^. Do not break people + */ +@Singleton +public class AWSStorageImpl implements AWSStorage { + + private AWSRawStorage rawStorage; + private ClientAssertions clientAssertions; + private ChannelWrapper channelWrapper; + + @Inject + public AWSStorageImpl(AWSRawStorage rawStorage, ClientAssertions clientAssertions, ChannelWrapper channelWrapper) { + this.rawStorage = rawStorage; + this.clientAssertions = clientAssertions; + this.channelWrapper = channelWrapper; + } + + @Override + public AWSBlob get(String bucket, String key) { + clientAssertions.throwIfCannotGoRemote(); + return rawStorage.get(bucket, key); + } + + @Override + public Stream list(String bucket) { + clientAssertions.throwIfCannotGoRemote(); + return rawStorage.list(bucket); + } + + @Override + public boolean delete(String bucket, String key) { + clientAssertions.throwIfCannotGoRemote(); + return rawStorage.delete(bucket, key); + } + + @Override + public byte[] readAllBytes(String bucket, String key) { + clientAssertions.throwIfCannotGoRemote(); + return rawStorage.readAllBytes(bucket, key); + } + + @Override + public ReadableByteChannel reader(String bucket, String key) { + clientAssertions.throwIfCannotGoRemote(); + return channelWrapper.newChannelProxy(ReadableByteChannel.class, rawStorage.reader(bucket, key)); + } + + @Override + public WritableByteChannel writer(String bucket, String key) { + clientAssertions.throwIfCannotGoRemote(); + return channelWrapper.newChannelProxy(WritableByteChannel.class, rawStorage.writer(bucket, key)); + } + + @Override + public boolean copy(String sourceBucket, String sourceKey, String destBucket, String destKey) { + clientAssertions.throwIfCannotGoRemote(); + //return null; + return rawStorage.copy(sourceBucket, sourceKey, destBucket, destKey); + } +} diff --git a/aws/s3-client/src/main/java/org/webpieces/aws/storage/impl/ChannelInvocationHandler.java b/aws/s3-client/src/main/java/org/webpieces/aws/storage/impl/ChannelInvocationHandler.java new file mode 100644 index 000000000..59c404145 --- /dev/null +++ b/aws/s3-client/src/main/java/org/webpieces/aws/storage/impl/ChannelInvocationHandler.java @@ -0,0 +1,33 @@ +package org.webpieces.aws.storage.impl; + +import org.webpieces.util.context.ClientAssertions; + +import javax.inject.Inject; +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.Method; +import java.nio.channels.Channel; + +public class ChannelInvocationHandler implements InvocationHandler { + + private ClientAssertions clientAssertions; + private Channel channel; + + @Inject + public ChannelInvocationHandler(ClientAssertions clientAssertions) { + this.clientAssertions = clientAssertions; + this.channel = channel; + } + + @Override + public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { + + clientAssertions.throwIfCannotGoRemote(); + + return method.invoke(channel, args); + + } + + public void setChannel(Channel channel) { + this.channel = channel; + } +} diff --git a/aws/s3-client/src/main/java/org/webpieces/aws/storage/impl/ChannelWrapper.java b/aws/s3-client/src/main/java/org/webpieces/aws/storage/impl/ChannelWrapper.java new file mode 100644 index 000000000..0cde81d67 --- /dev/null +++ b/aws/s3-client/src/main/java/org/webpieces/aws/storage/impl/ChannelWrapper.java @@ -0,0 +1,28 @@ +package org.webpieces.aws.storage.impl; + +import java.lang.reflect.Proxy; +import java.nio.channels.Channel; +import javax.inject.Inject; +import javax.inject.Provider; + +public class ChannelWrapper { + + private Provider invocHandlerProvider; + + @Inject + public ChannelWrapper(Provider invocHandlerProvider) { + this.invocHandlerProvider = invocHandlerProvider; + } + + public T newChannelProxy(Class intf, T channel) { + ChannelInvocationHandler invocHandler = invocHandlerProvider.get(); + invocHandler.setChannel(channel); + + return (T) Proxy.newProxyInstance(channel.getClass().getClassLoader(), + new Class[] {intf, Channel.class}, + invocHandler); + } + + + +} diff --git a/aws/s3-client/src/main/java/org/webpieces/aws/storage/impl/S3WritableByteChannel.java b/aws/s3-client/src/main/java/org/webpieces/aws/storage/impl/S3WritableByteChannel.java new file mode 100644 index 000000000..578481ddc --- /dev/null +++ b/aws/s3-client/src/main/java/org/webpieces/aws/storage/impl/S3WritableByteChannel.java @@ -0,0 +1,98 @@ +package org.webpieces.aws.storage.impl; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.WritableByteChannel; + +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.UploadPartRequest; + +public class S3WritableByteChannel implements WritableByteChannel { + + private final S3Client client; + private final String bucket; + private final String key; + private final ByteBuffer bb; + + private boolean closed = false; + private int part = 1; + + public S3WritableByteChannel(final S3Client client, final String bucket, final String key) { + + this.client = client; + this.bucket = bucket; + this.key = key; + + this.bb = ByteBuffer.allocate(1024 * 1024 * 8); + + } + + @Override + public int write(ByteBuffer src) throws IOException { + + int bytesWritten = 0; + + while(src.remaining() > 0) { + + if ((src.remaining()) <= (bb.remaining())) { + bytesWritten += (src.remaining()); + bb.put(src); + } else { + + ByteBuffer tmp = src.duplicate(); + tmp.limit(bb.remaining()); + + bytesWritten += tmp.remaining(); + bb.put(tmp); + + } + + if (!bb.hasRemaining()) { + bb.flip(); + uploadPart(); + } + + } + + return bytesWritten; + + } + + @Override + public boolean isOpen() { + return closed; + } + + @Override + public void close() throws IOException { + + if(closed) { + return; + } + + if(bb.position() > 0) { + bb.flip(); + uploadPart(); + } + + closed = true; + + } + + private void uploadPart() { + + UploadPartRequest request = UploadPartRequest.builder() + .bucket(bucket) + .key(key) + .partNumber(part) + .build(); + + client.uploadPart(request, RequestBody.fromRemainingByteBuffer(bb)); + + part++; + bb.clear(); + + } + +} diff --git a/aws/s3-client/src/main/java/org/webpieces/aws/storage/impl/StorageSupplier.java b/aws/s3-client/src/main/java/org/webpieces/aws/storage/impl/StorageSupplier.java new file mode 100644 index 000000000..76f4c4878 --- /dev/null +++ b/aws/s3-client/src/main/java/org/webpieces/aws/storage/impl/StorageSupplier.java @@ -0,0 +1,16 @@ +package org.webpieces.aws.storage.impl; + +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; + +import java.io.Serializable; +import java.util.function.Supplier; + +public class StorageSupplier implements Supplier, Serializable { + + @Override + public S3Client get() { + return S3Client.builder().region(Region.US_WEST_2).build(); + } + +} diff --git a/aws/s3-client/src/main/java/org/webpieces/aws/storage/impl/local/LocalAWSBlobImpl.java b/aws/s3-client/src/main/java/org/webpieces/aws/storage/impl/local/LocalAWSBlobImpl.java new file mode 100644 index 000000000..b78414571 --- /dev/null +++ b/aws/s3-client/src/main/java/org/webpieces/aws/storage/impl/local/LocalAWSBlobImpl.java @@ -0,0 +1,37 @@ +package org.webpieces.aws.storage.impl.local; + +import org.webpieces.aws.storage.api.AWSBlob; + +public class LocalAWSBlobImpl implements AWSBlob { + String bucket; + String key; + String contentType; + long size; + + public LocalAWSBlobImpl(String bucket, String key, String contentType, long size) { + this.bucket = bucket; + this.key = key; + this.contentType = contentType; + this.size = size; + } + + @Override + public String getBucket() { + return bucket; + } + + @Override + public String getKey() { + return key; + } + + @Override + public String getContentType() { + return contentType; + } + + @Override + public long getSize() { + return size; + } +} diff --git a/aws/s3-client/src/main/java/org/webpieces/aws/storage/impl/local/LocalStorage.java b/aws/s3-client/src/main/java/org/webpieces/aws/storage/impl/local/LocalStorage.java new file mode 100644 index 000000000..225123b06 --- /dev/null +++ b/aws/s3-client/src/main/java/org/webpieces/aws/storage/impl/local/LocalStorage.java @@ -0,0 +1,186 @@ +package org.webpieces.aws.storage.impl.local; + + +import org.webpieces.aws.storage.api.AWSBlob; +import org.webpieces.aws.storage.api.AWSRawStorage; +import org.webpieces.aws.storage.impl.raw.AWSBlobImpl; +import org.webpieces.util.SneakyThrow; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.io.*; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; +import java.nio.channels.ReadableByteChannel; +import java.nio.channels.WritableByteChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.stream.Stream; + +@Singleton +public class LocalStorage implements AWSRawStorage { + public static final String LOCAL_BUILD_DIR = "build/local-s3storage/"; + + @Inject + public LocalStorage() { + } + + @Override + public AWSBlob get(String bucket, String key) { + + InputStream in = getInputStream(bucket, key); + if(in != null) { + return new LocalAWSBlobImpl(bucket, key, null, -1); + } + + Path path = getFilePath(bucket, key); + + if(Files.exists(path)) { + long size = -1; + try { + size = Files.size(path); + } catch(IOException ex) { + throw SneakyThrow.sneak(ex); + } + return new LocalAWSBlobImpl(bucket, key, null, size); + } + + return null; + } + + @Override + public Stream list(String bucket) { + + Path bucketPath = getBucketPath(bucket); + + try { + + if(!Files.exists(bucketPath)) { + Files.createDirectories(bucketPath); + } + + return Files.find(bucketPath, Integer.MAX_VALUE, (p, attr) -> attr.isRegularFile()).map(p -> { + long size = -1; + try { + size = Files.size(p); + } catch (IOException ex) { + throw SneakyThrow.sneak(ex); + } + return new AWSBlobImpl(bucket, bucketPath.relativize(p).toString(), null, size); + }); + + } catch(IOException ex) { + throw SneakyThrow.sneak(ex); + } + + } + + @Override + public boolean delete(String bucket, String key) { + + Path file = getFilePath(bucket, key); + + try { + return Files.deleteIfExists(file); + } catch(IOException ex) { + throw SneakyThrow.sneak(ex); + } + + } + + @Override + public byte[] readAllBytes(String bucket, String key) { + + Path file = getFilePath(bucket, key); + + if(!Files.exists(file)) { + return null; + } + + try { + return Files.readAllBytes(file); + } catch(IOException ex) { + throw SneakyThrow.sneak(ex); + } + + } + + @Override + public ReadableByteChannel reader(String bucket, String key) { + + InputStream in = getInputStream(bucket, key); + if(in != null) { + ReadableByteChannel channel = Channels.newChannel(in); + return channel; + } + + Path path = getFilePath(bucket, key); + + if(!Files.exists(path)) { + return null; + } + + try { + return FileChannel.open(path, StandardOpenOption.READ); + } + catch(IOException ex) { + throw SneakyThrow.sneak(ex); + } + + } + + @Override + public WritableByteChannel writer(String bucket, String key) { + + Path path = getFilePath(bucket, key); + + try { + Files.createDirectories(path.getParent()); + } catch(IOException ex) { + throw SneakyThrow.sneak(ex); + } + + try { + return FileChannel.open(path, StandardOpenOption.CREATE, StandardOpenOption.WRITE); + } catch(IOException ex) { + throw SneakyThrow.sneak(ex); + } + + } + + @Override + public boolean copy(String sourceBucket, String sourceKey, String destBucket, String destKey) { + + ReadableByteChannel source = reader(sourceBucket, sourceKey); + Path dest = getFilePath(destBucket, destKey); + + try { + Files.createDirectories(dest.getParent()); + } catch(IOException ex) { + throw SneakyThrow.sneak(ex); + } + + try(FileChannel fc = FileChannel.open(dest, StandardOpenOption.CREATE, StandardOpenOption.WRITE)) { + fc.transferFrom(source, 0, Long.MAX_VALUE); + } catch(IOException ex) { + throw SneakyThrow.sneak(ex); + } + + return true; + + } + + private Path getBucketPath(String bucket) { + return Path.of(LOCAL_BUILD_DIR, bucket); + } + + private Path getFilePath(String bucket, String key) { + return getBucketPath(bucket).resolve(key); + } + + private InputStream getInputStream(String bucket, String key) { + return this.getClass().getClassLoader().getResourceAsStream(bucket + "/" + key); + } + +} diff --git a/aws/s3-client/src/main/java/org/webpieces/aws/storage/impl/raw/AWSBlobImpl.java b/aws/s3-client/src/main/java/org/webpieces/aws/storage/impl/raw/AWSBlobImpl.java new file mode 100644 index 000000000..253426ce4 --- /dev/null +++ b/aws/s3-client/src/main/java/org/webpieces/aws/storage/impl/raw/AWSBlobImpl.java @@ -0,0 +1,38 @@ +package org.webpieces.aws.storage.impl.raw; + +import org.webpieces.aws.storage.api.AWSBlob; + +public class AWSBlobImpl implements AWSBlob { + + private String bucket; + private String key; + private String contentType; + private long size; + + public AWSBlobImpl(String bucket, String key, String contentType, long size) { + this.bucket = bucket; + this.key = key; + this.contentType = contentType; + this.size = size; + } + + @Override + public String getBucket() { + return bucket; + } + + @Override + public String getKey() { + return key; + } + + @Override + public String getContentType() { + return contentType; + } + + @Override + public long getSize() { + return size; + } +} diff --git a/aws/s3-client/src/main/java/org/webpieces/aws/storage/impl/raw/AWSRawStorageImpl.java b/aws/s3-client/src/main/java/org/webpieces/aws/storage/impl/raw/AWSRawStorageImpl.java new file mode 100644 index 000000000..c6ae43302 --- /dev/null +++ b/aws/s3-client/src/main/java/org/webpieces/aws/storage/impl/raw/AWSRawStorageImpl.java @@ -0,0 +1,90 @@ +package org.webpieces.aws.storage.impl.raw; + +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; +import java.nio.channels.WritableByteChannel; +import java.util.stream.Stream; +import javax.inject.Inject; +import javax.inject.Singleton; + +import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.*; + +import org.webpieces.aws.storage.api.AWSBlob; +import org.webpieces.aws.storage.api.AWSRawStorage; +import org.webpieces.aws.storage.impl.S3WritableByteChannel; +import org.webpieces.aws.storage.impl.StorageSupplier; +import org.webpieces.util.SingletonSupplier; + +/** + * ADD NO CODE to this class as it is not tested until integration time. If it is 1 to 1, + * there is no testing to do and we rely on google's testing of Storage.java they have + */ +@Singleton +public class AWSRawStorageImpl implements AWSRawStorage { //implements Storage { + + private SingletonSupplier storage; + + @Inject + public AWSRawStorageImpl(StorageSupplier storage) { + this.storage = new SingletonSupplier<>(storage); + } + + @Override + public AWSBlob get(String bucket, String key) { + + try { + + HeadObjectResponse response = storage.get().headObject(HeadObjectRequest.builder().bucket(bucket).key(key).build()); + String contentType = response.contentType(); + long size = response.contentLength(); + + return new AWSBlobImpl(bucket, key, contentType, size); + } catch(NoSuchKeyException ex) { + return null; + } + + } + + @Override + public Stream list(String bucket) { + return storage.get().listObjectsV2Paginator(ListObjectsV2Request.builder().bucket(bucket).build()).contents().stream().map(obj -> new AWSBlobImpl(bucket, obj.key(), null, obj.size())); + } + + @Override + public boolean delete(String bucket, String key) { + try { + storage.get().deleteObject(DeleteObjectRequest.builder().bucket(bucket).key(key).build()); + return true; + } catch(AwsServiceException ex) { + return false; + } + } + + @Override + public byte[] readAllBytes(String bucket, String key) { + return storage.get().getObjectAsBytes(GetObjectRequest.builder().bucket(bucket).key(key).build()).asByteArray(); + } + + @Override + public ReadableByteChannel reader(String bucket, String key) { + return Channels.newChannel(storage.get().getObject(GetObjectRequest.builder().bucket(bucket).key(key).build())); + } + + @Override + public WritableByteChannel writer(String bucket, String key) { + return new S3WritableByteChannel(storage.get(), bucket, key); + } + + @Override + public boolean copy(String sourceBucket, String sourceKey, String destBucket, String destKey) { + try { + storage.get().copyObject(CopyObjectRequest.builder().sourceBucket(sourceBucket).sourceKey(sourceKey).destinationBucket(destBucket).destinationKey(destKey).build()); + return true; + } catch(AwsServiceException ex) { + return false; + } + } + +} diff --git a/aws/s3-client/src/test/java/org/webpieces/aws/storage/LocalTest.java b/aws/s3-client/src/test/java/org/webpieces/aws/storage/LocalTest.java new file mode 100644 index 000000000..61e13a8c6 --- /dev/null +++ b/aws/s3-client/src/test/java/org/webpieces/aws/storage/LocalTest.java @@ -0,0 +1,232 @@ +package org.webpieces.aws.storage; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.Module; +import com.google.inject.util.Modules; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import org.webpieces.aws.storage.api.AWSBlob; +import org.webpieces.aws.storage.api.AWSStorage; +import org.webpieces.util.context.Context; + +import java.io.*; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; +import java.nio.channels.WritableByteChannel; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class LocalTest { + + private AWSStorage instance; + + @Before + public void setup() { + + Module testModule = Modules.override(new TestProdModule()).with(new TestLocalModule()); + Injector injector = Guice.createInjector(testModule); + instance = injector.getInstance(AWSStorage.class); + + } + + @After + public void tearDown() { + List buckets = new ArrayList<>(); + buckets.add("testbucket"); + + for(String bucket : buckets) { + deleteFilesInBucket(bucket); + } + } + + private void deleteFilesInBucket(String bucket) { + Stream list = instance.list(bucket); + list.forEach(blob -> instance.delete(blob.getBucket(), blob.getKey())); + } + + @Test + public void testReadFromClasspath() { + ReadableByteChannel channel = instance.reader("testbucket", "mytest.txt"); + InputStream i = Channels.newInputStream(channel); + + String text = new BufferedReader( + new InputStreamReader(i, StandardCharsets.UTF_8)) + .lines() + .collect(Collectors.joining("\n")); + + Assert.assertEquals("Some Test", text); + + } + + @Test + public void testWriteThenReadFromBuildDir() throws IOException { + + writeFile("testbucket", "fileShit.txt"); + + + ReadableByteChannel channel = instance.reader("testbucket", "fileShit.txt"); + + InputStream i = Channels.newInputStream(channel); + + String text = new BufferedReader( + new InputStreamReader(i, StandardCharsets.UTF_8)) + .lines() + .collect(Collectors.joining("\n")); + Assert.assertEquals("testing a bitch", text); + } + + private void writeFile(String bucket, String key) throws IOException { + + WritableByteChannel writer = instance.writer(bucket, key); + OutputStream o = Channels.newOutputStream(writer); + String fkingString = "testing a bitch"; + byte[] bytes = fkingString.getBytes(StandardCharsets.UTF_8); + o.write(bytes); + o.flush(); + o.close(); + } + + @Test + public void testListFilesFromBothResourcesDirAndBuildDir() throws IOException { + //finish this test out + writeFile("listbucket", "fileSystemFile1.txt"); + writeFile("listbucket", "fileSystemFile2.txt"); + + Stream testbucket = instance.list("listbucket"); + + List keys = testbucket.map(blob -> blob.getKey()).collect(Collectors.toList()); + + Collections.sort(keys); + + Assert.assertEquals("fileSystemFile1.txt", keys.get(0)); + Assert.assertEquals("fileSystemFile2.txt", keys.get(1)); + + } + + @Test + public void validateFileNotFoundReturnsNullBlob() { + AWSBlob basket = instance.get("backet", "non-existent"); + Assert.assertNull(basket); + } + + @Test + public void testGetBlobClassPath() { + AWSBlob testbucket = instance.get("testbucket", "mytest.txt"); + Assert.assertEquals("mytest.txt",testbucket.getKey()); + } + + @Test + public void testGetBlobFileSystem() throws IOException { + //create a file + writeFile("testbucket", "fileSystemFile.txt"); + + AWSBlob bucket = instance.get("testbucket", "fileSystemFile.txt"); + Assert.assertEquals("fileSystemFile.txt",bucket.getKey()); + + } + + @Test + public void addFileToBucketAndThenListFiles() throws IOException { + instance.list("ListFilebucket"); + writeFile("ListFilebucket", "mytest1.txt"); + AWSBlob bucket = instance.get("ListFilebucket", "mytest1.txt"); + Assert.assertEquals("mytest1.txt",bucket.getKey());//passed. + + Stream listfilebucket = instance.list("ListFilebucket"); + List list = listfilebucket.map(blob -> blob.getKey()).collect(Collectors.toList()); + + Assert.assertEquals(1,list.size()); + + } + + @Test + public void addFileThenReadThenDeleteThenListFiles() throws IOException { + writeFile("AddReadDeleteListbucket", "mytest1.txt"); + AWSBlob bucket = instance.get("AddReadDeleteListbucket", "mytest1.txt"); + Assert.assertEquals("mytest1.txt",bucket.getKey());//passed. + + ReadableByteChannel readFile = instance.reader("AddReadDeleteListbucket", "mytest1.txt"); + + InputStream i = Channels.newInputStream(readFile); + + String text = new BufferedReader( + new InputStreamReader(i, StandardCharsets.UTF_8)) + .lines() + .collect(Collectors.joining("\n")); + Assert.assertEquals("testing a bitch", text);// Passed. + + boolean success = instance.delete("AddReadDeleteListbucket", "mytest1.txt"); + Assert.assertEquals(true,success);//passed. + + Stream testbucket = instance.list("AddReadDeleteListbucket"); + List list = testbucket.map(blob -> blob.getKey()).collect(Collectors.toList()); + + //length of the blob should be original length. + Assert.assertEquals(0,list.size());// testing on testbucket directory + // with 2 files already existed. + } + + @Test + public void testCopyFromClassPath() throws IOException { + + String bucketName = "testbucket"; + String blobName = "mytest.txt"; + String copyBlobName = "mytest_copy.txt"; + + boolean success = instance.copy(bucketName, blobName, "copybucket", copyBlobName); + Assert.assertEquals(true, success); + + ReadableByteChannel readFile = instance.reader("copybucket", "mytest_copy.txt"); + + InputStream i = Channels.newInputStream(readFile); + + String text = new BufferedReader( + new InputStreamReader(i, StandardCharsets.UTF_8)) + .lines() + .collect(Collectors.joining("\n")); + Assert.assertEquals("Some Test", text);// Passed. + } + + @Test + public void testGetBucket() { + } + + @Test + public void testAllCallsFailInTransaction() { + Context.put("tests",1); + try { + instance.get("testbucket", "fileSystemFile"); + Assert.fail("Was expecting an exception. Should not get here"); + } + catch(IllegalStateException e){ + + }finally{ + Context.clear(); + } + } + + @Test + public void testNoReadingWhileInTransaction() throws IOException{ + ReadableByteChannel reader = instance.reader("testbucket","mytest.txt");//what file should we read? + Context.put("tests",1); + try { + int read = reader.read(ByteBuffer.allocateDirect(2048));//how to read using readableByteChannel. + Assert.fail("Was expecting an exception. Should not get here"); + } + catch(IllegalStateException e){ + + } finally { + Context.clear(); + } + } + +} diff --git a/aws/s3-client/src/test/java/org/webpieces/aws/storage/ProductionTest.java b/aws/s3-client/src/test/java/org/webpieces/aws/storage/ProductionTest.java new file mode 100644 index 000000000..fc0efd7f2 --- /dev/null +++ b/aws/s3-client/src/test/java/org/webpieces/aws/storage/ProductionTest.java @@ -0,0 +1,29 @@ +package org.webpieces.aws.storage; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.Module; +import org.junit.Before; +import org.webpieces.aws.storage.api.AWSStorage; + +/** + * This is for when we have to talk to google to find out what google returns so we can + * simulate google better + */ +public class ProductionTest { + + private AWSStorage instance; + + @Before + public void setup() { + Module module = new TestProdModule(); + Injector injector = Guice.createInjector(module); + instance = injector.getInstance(AWSStorage.class); + } + + //@Test + public void talkToGoogle() { + //write simulation code here + } + +} diff --git a/aws/s3-client/src/test/java/org/webpieces/aws/storage/TestClientAssertions.java b/aws/s3-client/src/test/java/org/webpieces/aws/storage/TestClientAssertions.java new file mode 100644 index 000000000..38c4e9c03 --- /dev/null +++ b/aws/s3-client/src/test/java/org/webpieces/aws/storage/TestClientAssertions.java @@ -0,0 +1,18 @@ +package org.webpieces.aws.storage; + +import org.webpieces.util.context.ClientAssertions; +import org.webpieces.util.context.Context; + +public class TestClientAssertions implements ClientAssertions { + + @Override + public void throwIfCannotGoRemote() { + + Object tests = Context.get("tests"); + if(tests != null){ + throw new IllegalStateException("For testing."); + } + + } + +} diff --git a/aws/s3-client/src/test/java/org/webpieces/aws/storage/TestLocalModule.java b/aws/s3-client/src/test/java/org/webpieces/aws/storage/TestLocalModule.java new file mode 100644 index 000000000..21d59c1a5 --- /dev/null +++ b/aws/s3-client/src/test/java/org/webpieces/aws/storage/TestLocalModule.java @@ -0,0 +1,16 @@ +package org.webpieces.aws.storage; + +import com.google.inject.Binder; +import com.google.inject.Module; + +import org.webpieces.aws.storage.api.AWSRawStorage; +import org.webpieces.aws.storage.impl.local.LocalStorage; + +public class TestLocalModule implements Module { + + @Override + public void configure(Binder binder) { + binder.bind(AWSRawStorage.class).to(LocalStorage.class).asEagerSingleton(); + } + +} diff --git a/aws/s3-client/src/test/java/org/webpieces/aws/storage/TestProdModule.java b/aws/s3-client/src/test/java/org/webpieces/aws/storage/TestProdModule.java new file mode 100644 index 000000000..803d1d64f --- /dev/null +++ b/aws/s3-client/src/test/java/org/webpieces/aws/storage/TestProdModule.java @@ -0,0 +1,14 @@ +package org.webpieces.aws.storage; + +import com.google.inject.Binder; +import com.google.inject.Module; +import org.webpieces.util.context.ClientAssertions; + +public class TestProdModule implements Module { + + @Override + public void configure(Binder binder) { + binder.bind(ClientAssertions.class).toInstance(new TestClientAssertions()); + } + +} diff --git a/aws/s3-client/src/test/resources/testbucket/mytest.txt b/aws/s3-client/src/test/resources/testbucket/mytest.txt new file mode 100644 index 000000000..5166f1208 --- /dev/null +++ b/aws/s3-client/src/test/resources/testbucket/mytest.txt @@ -0,0 +1 @@ +Some Test \ No newline at end of file diff --git a/config/libs.versions.toml b/config/libs.versions.toml index cc38e2cad..efb973d4b 100644 --- a/config/libs.versions.toml +++ b/config/libs.versions.toml @@ -1,5 +1,6 @@ [versions] acme4j = "2.11" +aws = "2.20.110" grpc = "1.37.0" guava = "31.1-jre" jackson = "2.13.3" @@ -25,7 +26,8 @@ apache-commons-beanutils = { module = "commons-beanutils:commons-beanutil apache-commons-io = { module = "commons-io:commons-io", version = "2.8.0" } apache-commons-lang3 = { module = "org.apache.commons:commons-lang3", version = "3.12.0" } apache-commons-text = { module = "org.apache.commons:commons-text", version = "1.9" } -aws-secretsmanager = { module = "software.amazon.awssdk:secretsmanager", version = "2.18.6"} +aws-s3 = { module = "software.amazon.awssdk:s3", version.ref = "aws" } +aws-secretsmanager = { module = "software.amazon.awssdk:secretsmanager", version.ref = "aws"} digitalforge-log4jdbc = { module = "org.digitalforge:log4jdbc", version = "1.0.2" } disruptor = { module = "com.conversantmedia:disruptor", version = "1.2.21" } elasticsearch-client = { module = "org.elasticsearch.client:elasticsearch-rest-client", version = "7.6.1" } diff --git a/settings.gradle b/settings.gradle index d37487adb..c59beaef4 100644 --- a/settings.gradle +++ b/settings.gradle @@ -62,6 +62,7 @@ includeBuild 'core/core-statemachine' includeBuild 'core/core-util' includeBuild 'core/core-persistence' includeBuild 'core/runtimecompile' +includeBuild 'aws/s3-client' includeBuild 'googlecloud/cloudstorage-client' includeBuild 'googlecloud/cloud-tasks' includeBuild 'libraries/lib-elasticsearch'