From 35fea54d63e0752c1df0aefba3c1363084c78926 Mon Sep 17 00:00:00 2001 From: Timur Alperovich Date: Mon, 3 May 2021 00:44:45 -0700 Subject: [PATCH] Alias middleware to remap backend buckets Alias middleware implements a way to remap backend buckets to a configurable front-end name. The mappings are configured as: s3proxy.alias-blobstore. = A single bucket cannot be mapped to multiple names. --- .../java/org/gaul/s3proxy/AliasBlobStore.java | 290 ++++++++++++++++++ src/main/java/org/gaul/s3proxy/Main.java | 8 + .../org/gaul/s3proxy/S3ProxyConstants.java | 3 + .../org/gaul/s3proxy/AliasBlobStoreTest.java | 186 +++++++++++ 4 files changed, 487 insertions(+) create mode 100644 src/main/java/org/gaul/s3proxy/AliasBlobStore.java create mode 100644 src/test/java/org/gaul/s3proxy/AliasBlobStoreTest.java diff --git a/src/main/java/org/gaul/s3proxy/AliasBlobStore.java b/src/main/java/org/gaul/s3proxy/AliasBlobStore.java new file mode 100644 index 00000000..b1d5797a --- /dev/null +++ b/src/main/java/org/gaul/s3proxy/AliasBlobStore.java @@ -0,0 +1,290 @@ +/* + * Copyright 2014-2021 Andrew Gaul + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gaul.s3proxy; + +import static java.util.Objects.requireNonNull; + +import static com.google.common.base.Preconditions.checkArgument; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import com.google.common.collect.BiMap; +import com.google.common.collect.ImmutableBiMap; +import com.google.common.collect.ImmutableList; + +import org.jclouds.blobstore.BlobStore; +import org.jclouds.blobstore.domain.Blob; +import org.jclouds.blobstore.domain.BlobMetadata; +import org.jclouds.blobstore.domain.ContainerAccess; +import org.jclouds.blobstore.domain.MultipartPart; +import org.jclouds.blobstore.domain.MultipartUpload; +import org.jclouds.blobstore.domain.MutableStorageMetadata; +import org.jclouds.blobstore.domain.PageSet; +import org.jclouds.blobstore.domain.StorageMetadata; +import org.jclouds.blobstore.domain.internal.MutableStorageMetadataImpl; +import org.jclouds.blobstore.domain.internal.PageSetImpl; +import org.jclouds.blobstore.options.CopyOptions; +import org.jclouds.blobstore.options.CreateContainerOptions; +import org.jclouds.blobstore.options.GetOptions; +import org.jclouds.blobstore.options.ListContainerOptions; +import org.jclouds.blobstore.options.PutOptions; +import org.jclouds.blobstore.util.ForwardingBlobStore; +import org.jclouds.domain.Location; +import org.jclouds.io.Payload; + +/** + * This class implements a middleware to alias buckets to a different name. + * The aliases are configured as: + * s3proxy.alias-blobstore.<alias name> = <backend bucket> + * + * The aliases appear in bucket listings if the configured + * backend buckets are present. Requests for all other buckets are unaffected. + */ +public final class AliasBlobStore extends ForwardingBlobStore { + private final BiMap aliases; + + private AliasBlobStore(BlobStore delegate, + BiMap aliases) { + super(delegate); + this.aliases = requireNonNull(aliases); + } + + static BlobStore newAliasBlobStore(BlobStore delegate, + BiMap aliases) { + return new AliasBlobStore(delegate, aliases); + } + + private MultipartUpload getDelegateMpu(MultipartUpload mpu) { + return MultipartUpload.create( + getContainer(mpu.containerName()), + mpu.blobName(), + mpu.id(), + mpu.blobMetadata(), + mpu.putOptions()); + } + + public static ImmutableBiMap parseAliases( + Properties properties) { + Map backendBuckets = new HashMap<>(); + for (String key : properties.stringPropertyNames()) { + if (key.startsWith(S3ProxyConstants.PROPERTY_ALIAS_BLOBSTORE)) { + String virtualBucket = key.substring( + S3ProxyConstants.PROPERTY_ALIAS_BLOBSTORE.length() + 1); + String backendBucket = properties.getProperty(key); + checkArgument( + !backendBuckets.containsKey(backendBucket), + "Backend bucket %s is aliased twice", + backendBucket); + backendBuckets.put(backendBucket, virtualBucket); + } + } + return ImmutableBiMap.copyOf(backendBuckets).inverse(); + } + + private String getContainer(String container) { + return this.aliases.getOrDefault(container, container); + } + + @Override + public boolean createContainerInLocation(Location location, + String container) { + return this.delegate().createContainerInLocation(location, + getContainer(container)); + } + + @Override + public boolean createContainerInLocation( + Location location, String container, + CreateContainerOptions options) { + return delegate().createContainerInLocation( + location, getContainer(container), options); + } + + @Override + public boolean containerExists(String container) { + return delegate().containerExists(getContainer(container)); + } + + @Override + public ContainerAccess getContainerAccess(String container) { + return delegate().getContainerAccess(getContainer(container)); + } + + @Override + public void setContainerAccess(String container, + ContainerAccess containerAccess) { + delegate().setContainerAccess(getContainer(container), containerAccess); + } + + @Override + public PageSet list() { + PageSet upstream = this.delegate().list(); + ImmutableList.Builder results = + new ImmutableList.Builder<>(); + for (StorageMetadata sm : upstream) { + if (aliases.containsValue(sm.getName())) { + MutableStorageMetadata bucketAlias = + new MutableStorageMetadataImpl(); + bucketAlias.setName(aliases.inverse().get(sm.getName())); + bucketAlias.setCreationDate(sm.getCreationDate()); + bucketAlias.setETag(sm.getETag()); + bucketAlias.setId(sm.getProviderId()); + bucketAlias.setLastModified(sm.getLastModified()); + bucketAlias.setLocation(sm.getLocation()); + bucketAlias.setSize(sm.getSize()); + bucketAlias.setTier(sm.getTier()); + bucketAlias.setType(sm.getType()); + // TODO: the URI should be rewritten to use the alias + bucketAlias.setUri(sm.getUri()); + bucketAlias.setUserMetadata(sm.getUserMetadata()); + results.add(bucketAlias); + } else { + results.add(sm); + } + } + return new PageSetImpl<>(results.build(), upstream.getNextMarker()); + } + + @Override + public PageSet list(String container) { + return delegate().list(getContainer(container)); + } + + @Override + public PageSet list( + String container, ListContainerOptions options) { + return delegate().list(getContainer(container), options); + } + + @Override + public void clearContainer(String container) { + delegate().clearContainer(getContainer(container)); + } + + @Override + public void clearContainer(String container, ListContainerOptions options) { + delegate().clearContainer(getContainer(container), options); + } + + @Override + public void deleteContainer(String container) { + delegate().deleteContainer(getContainer(container)); + } + + @Override + public boolean deleteContainerIfEmpty(String container) { + return delegate().deleteContainerIfEmpty(getContainer(container)); + } + + @Override + public boolean directoryExists(String container, String directory) { + return delegate().directoryExists(getContainer(container), directory); + } + + @Override + public void createDirectory(String container, String directory) { + delegate().createDirectory(getContainer(container), directory); + } + + @Override + public void deleteDirectory(String container, String directory) { + delegate().deleteDirectory(getContainer(container), directory); + } + + @Override + public boolean blobExists(String container, String name) { + return delegate().blobExists(getContainer(container), name); + } + + @Override + public BlobMetadata blobMetadata(String container, String name) { + return delegate().blobMetadata(getContainer(container), name); + } + + @Override + public Blob getBlob(String containerName, String blobName) { + return delegate().getBlob(getContainer(containerName), blobName); + } + + @Override + public Blob getBlob(String containerName, String blobName, + GetOptions getOptions) { + return delegate().getBlob(getContainer(containerName), blobName, + getOptions); + } + + @Override + public String putBlob(String containerName, Blob blob) { + return delegate().putBlob(getContainer(containerName), blob); + } + + @Override + public String putBlob(final String containerName, Blob blob, + final PutOptions options) { + return delegate().putBlob(getContainer(containerName), blob, + options); + } + + @Override + public void removeBlob(final String containerName, final String blobName) { + delegate().removeBlob(getContainer(containerName), blobName); + } + + @Override + public void removeBlobs(final String containerName, + final Iterable blobNames) { + delegate().removeBlobs(getContainer(containerName), blobNames); + } + + @Override + public String copyBlob(final String fromContainer, final String fromName, + final String toContainer, final String toName, + final CopyOptions options) { + return delegate().copyBlob(getContainer(fromContainer), fromName, + getContainer(toContainer), toName, options); + } + + @Override + public MultipartUpload initiateMultipartUpload( + String container, BlobMetadata blobMetadata, PutOptions options) { + MultipartUpload mpu = delegate().initiateMultipartUpload( + getContainer(container), blobMetadata, options); + return MultipartUpload.create(container, blobMetadata.getName(), + mpu.id(), mpu.blobMetadata(), mpu.putOptions()); + } + + @Override + public void abortMultipartUpload(MultipartUpload mpu) { + delegate().abortMultipartUpload(getDelegateMpu(mpu)); + } + + @Override + public String completeMultipartUpload(final MultipartUpload mpu, + final List parts) { + return delegate().completeMultipartUpload(getDelegateMpu(mpu), parts); + } + + @Override + public MultipartPart uploadMultipartPart(MultipartUpload mpu, + int partNumber, Payload payload) { + return delegate().uploadMultipartPart(getDelegateMpu(mpu), partNumber, + payload); + } +} diff --git a/src/main/java/org/gaul/s3proxy/Main.java b/src/main/java/org/gaul/s3proxy/Main.java index 8d86f3f0..7f084300 100644 --- a/src/main/java/org/gaul/s3proxy/Main.java +++ b/src/main/java/org/gaul/s3proxy/Main.java @@ -33,6 +33,7 @@ import java.util.concurrent.TimeUnit; import com.google.common.base.Strings; +import com.google.common.collect.ImmutableBiMap; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; @@ -224,6 +225,13 @@ private static BlobStore parseMiddlewareProperties(BlobStore blobStore, blobStore = ReadOnlyBlobStore.newReadOnlyBlobStore(blobStore); } + ImmutableBiMap aliases = AliasBlobStore.parseAliases( + properties); + if (!aliases.isEmpty()) { + System.err.println("Using alias backend"); + blobStore = AliasBlobStore.newAliasBlobStore(blobStore, aliases); + } + ImmutableMap shards = ShardedBlobStore.parseBucketShards(properties); ImmutableMap prefixes = diff --git a/src/main/java/org/gaul/s3proxy/S3ProxyConstants.java b/src/main/java/org/gaul/s3proxy/S3ProxyConstants.java index 3cb294c8..510a515b 100644 --- a/src/main/java/org/gaul/s3proxy/S3ProxyConstants.java +++ b/src/main/java/org/gaul/s3proxy/S3ProxyConstants.java @@ -78,6 +78,9 @@ public final class S3ProxyConstants { /** Probability of eventual consistency, between 0.0 and 1.0. */ public static final String PROPERTY_EVENTUAL_CONSISTENCY_PROBABILITY = "s3proxy.eventual-consistency.probability"; + /** Alias a backend bucket to an alternate name. */ + public static final String PROPERTY_ALIAS_BLOBSTORE = + "s3proxy.alias-blobstore"; /** Discard object data. */ public static final String PROPERTY_NULL_BLOBSTORE = "s3proxy.null-blobstore"; diff --git a/src/test/java/org/gaul/s3proxy/AliasBlobStoreTest.java b/src/test/java/org/gaul/s3proxy/AliasBlobStoreTest.java new file mode 100644 index 00000000..7e817ad8 --- /dev/null +++ b/src/test/java/org/gaul/s3proxy/AliasBlobStoreTest.java @@ -0,0 +1,186 @@ +/* + * Copyright 2014-2021 Andrew Gaul + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gaul.s3proxy; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +import com.google.common.collect.ImmutableBiMap; +import com.google.common.collect.ImmutableList; +import com.google.common.hash.HashCode; +import com.google.common.hash.Hashing; +import com.google.common.io.ByteSource; +import com.google.inject.Module; + +import org.assertj.core.api.Assertions; +import org.jclouds.ContextBuilder; +import org.jclouds.blobstore.BlobStore; +import org.jclouds.blobstore.BlobStoreContext; +import org.jclouds.blobstore.domain.Blob; +import org.jclouds.blobstore.domain.BlobMetadata; +import org.jclouds.blobstore.domain.MultipartPart; +import org.jclouds.blobstore.domain.MultipartUpload; +import org.jclouds.blobstore.domain.PageSet; +import org.jclouds.blobstore.domain.StorageMetadata; +import org.jclouds.blobstore.options.PutOptions; +import org.jclouds.io.Payloads; +import org.jclouds.logging.slf4j.config.SLF4JLoggingModule; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public final class AliasBlobStoreTest { + private String containerName; + private String aliasContainerName; + private BlobStoreContext context; + private BlobStore blobStore; + private BlobStore aliasBlobStore; + private List createdContainers; + + @Before + public void setUp() { + containerName = TestUtils.createRandomContainerName(); + aliasContainerName = String.format("alias-%s", containerName); + context = ContextBuilder + .newBuilder("transient") + .credentials("identity", "credential") + .modules(ImmutableList.of(new SLF4JLoggingModule())) + .build(BlobStoreContext.class); + blobStore = context.getBlobStore(); + ImmutableBiMap.Builder aliasesBuilder = + new ImmutableBiMap.Builder<>(); + aliasesBuilder.put(aliasContainerName, containerName); + aliasBlobStore = AliasBlobStore.newAliasBlobStore( + blobStore, aliasesBuilder.build()); + createdContainers = new ArrayList<>(); + } + + @After + public void tearDown() { + if (this.context != null) { + for (String container : this.createdContainers) { + blobStore.deleteContainer(container); + } + context.close(); + } + } + + private void createContainer(String container) { + assertThat(aliasBlobStore.createContainerInLocation( + null, container)).isTrue(); + if (container.equals(aliasContainerName)) { + createdContainers.add(containerName); + } else { + createdContainers.add(container); + } + } + + @Test + public void testListNoAliasContainers() { + String regularContainer = TestUtils.createRandomContainerName(); + createContainer(regularContainer); + PageSet listing = aliasBlobStore.list(); + assertThat(listing.size()).isEqualTo(1); + assertThat(listing.iterator().next().getName()).isEqualTo( + regularContainer); + } + + @Test + public void testListAliasContainer() { + createContainer(aliasContainerName); + PageSet listing = aliasBlobStore.list(); + assertThat(listing.size()).isEqualTo(1); + assertThat(listing.iterator().next().getName()).isEqualTo( + aliasContainerName); + listing = blobStore.list(); + assertThat(listing.size()).isEqualTo(1); + assertThat(listing.iterator().next().getName()).isEqualTo( + containerName); + } + + @Test + public void testAliasBlob() throws IOException { + createContainer(aliasContainerName); + String blobName = TestUtils.createRandomBlobName(); + ByteSource content = TestUtils.randomByteSource().slice(0, 1024); + String contentMD5 = Hashing.md5().hashBytes(content.read()).toString(); + Blob blob = aliasBlobStore.blobBuilder(blobName).payload(content) + .build(); + String eTag = aliasBlobStore.putBlob(aliasContainerName, blob); + assertThat(eTag).isEqualTo(contentMD5); + BlobMetadata blobMetadata = aliasBlobStore.blobMetadata( + aliasContainerName, blobName); + assertThat(blobMetadata.getETag()).isEqualTo(contentMD5); + blob = aliasBlobStore.getBlob(aliasContainerName, blobName); + try (InputStream actual = blob.getPayload().openStream(); + InputStream expected = content.openStream()) { + assertThat(actual).hasContentEqualTo(expected); + } + } + + @Test + public void testAliasMultipartUpload() throws IOException { + createContainer(aliasContainerName); + String blobName = TestUtils.createRandomBlobName(); + ByteSource content = TestUtils.randomByteSource().slice(0, 1024); + HashCode contentHash = Hashing.md5().hashBytes(content.read()); + Blob blob = aliasBlobStore.blobBuilder(blobName).build(); + MultipartUpload mpu = aliasBlobStore.initiateMultipartUpload( + aliasContainerName, blob.getMetadata(), PutOptions.NONE); + assertThat(mpu.containerName()).isEqualTo(aliasContainerName); + MultipartPart part = aliasBlobStore.uploadMultipartPart( + mpu, 1, Payloads.newPayload(content)); + assertThat(part.partETag()).isEqualTo(contentHash.toString()); + ImmutableList.Builder parts = + new ImmutableList.Builder<>(); + parts.add(part); + String mpuETag = aliasBlobStore.completeMultipartUpload(mpu, + parts.build()); + assertThat(mpuETag).isEqualTo( + String.format("\"%s-1\"", + Hashing.md5().hashBytes(contentHash.asBytes()))); + blob = aliasBlobStore.getBlob(aliasContainerName, blobName); + try (InputStream actual = blob.getPayload().openStream(); + InputStream expected = content.openStream()) { + assertThat(actual).hasContentEqualTo(expected); + } + } + + @Test + public void testParseDuplicateAliases() { + Properties properties = new Properties(); + properties.setProperty(String.format("%s.alias", + S3ProxyConstants.PROPERTY_ALIAS_BLOBSTORE), "bucket"); + properties.setProperty(String.format("%s.other-alias", + S3ProxyConstants.PROPERTY_ALIAS_BLOBSTORE), "bucket"); + + try { + AliasBlobStore.parseAliases(properties); + Assertions.failBecauseExceptionWasNotThrown( + IllegalArgumentException.class); + } catch (IllegalArgumentException exc) { + assertThat(exc.getMessage()).isEqualTo( + "Backend bucket bucket is aliased twice"); + } + } +}