From 9be15e1967645a42cbd7007b4f53134929650656 Mon Sep 17 00:00:00 2001 From: Timur Alperovich Date: Tue, 31 Aug 2021 00:34:14 -0700 Subject: [PATCH] Allow differentiating blobstores by buckets Allows differentiating the backend blobstores by configured buckets. The commit allows using the same s3proxy credentials across the different backends and relies on the bucket names to disambiguate them. The "default" blobstore is the one that occurs first. Fixes: #371 --- src/main/java/org/gaul/s3proxy/Main.java | 61 ++++++++++++++----- .../org/gaul/s3proxy/S3ProxyConstants.java | 12 ++++ 2 files changed, 58 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/gaul/s3proxy/Main.java b/src/main/java/org/gaul/s3proxy/Main.java index 7f084300..fd56bb68 100644 --- a/src/main/java/org/gaul/s3proxy/Main.java +++ b/src/main/java/org/gaul/s3proxy/Main.java @@ -24,9 +24,11 @@ import java.io.PrintStream; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ThreadFactory; @@ -101,8 +103,10 @@ public static void main(String[] args) throws Exception { .build(); ExecutorService executorService = DynamicExecutors.newScalingThreadPool( 1, 20, 60 * 1000, factory); - ImmutableMap.Builder> locators = + ImmutableMap.Builder, + Map.Entry> locators = ImmutableMap.builder(); + Set> parsedLocations = new HashSet<>(); for (File propertiesFile : options.propertiesFiles) { Properties properties = new Properties(); try (InputStream is = new FileInputStream(propertiesFile)) { @@ -117,14 +121,42 @@ public static void main(String[] args) throws Exception { String s3ProxyAuthorizationString = properties.getProperty( S3ProxyConstants.PROPERTY_AUTHORIZATION); + ImmutableList.Builder locatorBuckets = + new ImmutableList.Builder<>(); + for (String key : properties.stringPropertyNames()) { + if (key.startsWith(S3ProxyConstants.PROPERTY_BUCKET_LOCATOR)) { + locatorBuckets.add(properties.getProperty(key)); + } + } + ImmutableList buckets = locatorBuckets.build(); + String localIdentity = null; + String localCredential = null; if (AuthenticationType.fromString(s3ProxyAuthorizationString) != AuthenticationType.NONE) { - String localIdentity = properties.getProperty( + localIdentity = properties.getProperty( S3ProxyConstants.PROPERTY_IDENTITY); - String localCredential = properties.getProperty( + localCredential = properties.getProperty( S3ProxyConstants.PROPERTY_CREDENTIAL); - locators.put(localIdentity, Maps.immutableEntry( - localCredential, blobStore)); + Map.Entry key = Maps.immutableEntry( + localIdentity, null); + if (parsedLocations.add(key)) { + locators.put(key, Maps.immutableEntry( + localCredential, blobStore)); + } + } + if (!buckets.isEmpty()) { + for (String bucket : buckets) { + Map.Entry key = Maps.immutableEntry( + localIdentity, bucket); + if (!parsedLocations.add(key)) { + System.err.printf("The same bucket locator cannot be" + + " used in two properties files: %s\n", + bucket); + System.exit(1); + } + locators.put(key, + Maps.immutableEntry(localCredential, blobStore)); + } } S3Proxy.Builder s3ProxyBuilder2 = S3Proxy.Builder @@ -149,23 +181,22 @@ public static void main(String[] args) throws Exception { throw e; } - final Map> locator = - locators.build(); + final Map, Map.Entry> + locator = locators.build(); if (!locator.isEmpty()) { s3Proxy.setBlobStoreLocator(new BlobStoreLocator() { @Override public Map.Entry locateBlobStore( String identity, String container, String blob) { + Map.Entry entry = locator.get( + Maps.immutableEntry(identity, container)); + if (entry != null) { + return entry; + } if (identity == null) { - if (locator.size() == 1) { - return locator.entrySet().iterator().next() - .getValue(); - } - throw new IllegalArgumentException( - "cannot use anonymous access with multiple" + - " backends"); + return locator.entrySet().iterator().next().getValue(); } - return locator.get(identity); + return locator.get(Maps.immutableEntry(identity, null)); } }); } diff --git a/src/main/java/org/gaul/s3proxy/S3ProxyConstants.java b/src/main/java/org/gaul/s3proxy/S3ProxyConstants.java index 510a515b..eec35bdd 100644 --- a/src/main/java/org/gaul/s3proxy/S3ProxyConstants.java +++ b/src/main/java/org/gaul/s3proxy/S3ProxyConstants.java @@ -66,6 +66,18 @@ public final class S3ProxyConstants { "s3proxy.max-single-part-object-size"; public static final String PROPERTY_V4_MAX_NON_CHUNKED_REQUEST_SIZE = "s3proxy.v4-max-non-chunked-request-size"; + /** Used to locate blobstores by specified bucket names. Each property + * file should contain a list of buckets associated with it, e.g. + * s3proxy.bucket-locator.1 = data + * s3proxy.bucket-locator.2 = metadata + * s3proxy.bucket-locator.3 = other + * When a request is made for the specified bucket, the backend defined + * in that properties file is used. This allows using the same + * credentials in multiple properties file and select the backend based + * on the bucket names. + */ + public static final String PROPERTY_BUCKET_LOCATOR = + "s3proxy.bucket-locator"; /** When true, model eventual consistency using two storage backends. */ public static final String PROPERTY_EVENTUAL_CONSISTENCY = "s3proxy.eventual-consistency";