diff --git a/src/main/java/org/gaul/s3proxy/Main.java b/src/main/java/org/gaul/s3proxy/Main.java index 7f084300..fd56bb68 100644 --- a/src/main/java/org/gaul/s3proxy/Main.java +++ b/src/main/java/org/gaul/s3proxy/Main.java @@ -24,9 +24,11 @@ import java.io.PrintStream; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ThreadFactory; @@ -101,8 +103,10 @@ public static void main(String[] args) throws Exception { .build(); ExecutorService executorService = DynamicExecutors.newScalingThreadPool( 1, 20, 60 * 1000, factory); - ImmutableMap.Builder> locators = + ImmutableMap.Builder, + Map.Entry> locators = ImmutableMap.builder(); + Set> parsedLocations = new HashSet<>(); for (File propertiesFile : options.propertiesFiles) { Properties properties = new Properties(); try (InputStream is = new FileInputStream(propertiesFile)) { @@ -117,14 +121,42 @@ public static void main(String[] args) throws Exception { String s3ProxyAuthorizationString = properties.getProperty( S3ProxyConstants.PROPERTY_AUTHORIZATION); + ImmutableList.Builder locatorBuckets = + new ImmutableList.Builder<>(); + for (String key : properties.stringPropertyNames()) { + if (key.startsWith(S3ProxyConstants.PROPERTY_BUCKET_LOCATOR)) { + locatorBuckets.add(properties.getProperty(key)); + } + } + ImmutableList buckets = locatorBuckets.build(); + String localIdentity = null; + String localCredential = null; if (AuthenticationType.fromString(s3ProxyAuthorizationString) != AuthenticationType.NONE) { - String localIdentity = properties.getProperty( + localIdentity = properties.getProperty( S3ProxyConstants.PROPERTY_IDENTITY); - String localCredential = properties.getProperty( + localCredential = properties.getProperty( S3ProxyConstants.PROPERTY_CREDENTIAL); - locators.put(localIdentity, Maps.immutableEntry( - localCredential, blobStore)); + Map.Entry key = Maps.immutableEntry( + localIdentity, null); + if (parsedLocations.add(key)) { + locators.put(key, Maps.immutableEntry( + localCredential, blobStore)); + } + } + if (!buckets.isEmpty()) { + for (String bucket : buckets) { + Map.Entry key = Maps.immutableEntry( + localIdentity, bucket); + if (!parsedLocations.add(key)) { + System.err.printf("The same bucket locator cannot be" + + " used in two properties files: %s\n", + bucket); + System.exit(1); + } + locators.put(key, + Maps.immutableEntry(localCredential, blobStore)); + } } S3Proxy.Builder s3ProxyBuilder2 = S3Proxy.Builder @@ -149,23 +181,22 @@ public static void main(String[] args) throws Exception { throw e; } - final Map> locator = - locators.build(); + final Map, Map.Entry> + locator = locators.build(); if (!locator.isEmpty()) { s3Proxy.setBlobStoreLocator(new BlobStoreLocator() { @Override public Map.Entry locateBlobStore( String identity, String container, String blob) { + Map.Entry entry = locator.get( + Maps.immutableEntry(identity, container)); + if (entry != null) { + return entry; + } if (identity == null) { - if (locator.size() == 1) { - return locator.entrySet().iterator().next() - .getValue(); - } - throw new IllegalArgumentException( - "cannot use anonymous access with multiple" + - " backends"); + return locator.entrySet().iterator().next().getValue(); } - return locator.get(identity); + return locator.get(Maps.immutableEntry(identity, null)); } }); } diff --git a/src/main/java/org/gaul/s3proxy/S3ProxyConstants.java b/src/main/java/org/gaul/s3proxy/S3ProxyConstants.java index 510a515b..eec35bdd 100644 --- a/src/main/java/org/gaul/s3proxy/S3ProxyConstants.java +++ b/src/main/java/org/gaul/s3proxy/S3ProxyConstants.java @@ -66,6 +66,18 @@ public final class S3ProxyConstants { "s3proxy.max-single-part-object-size"; public static final String PROPERTY_V4_MAX_NON_CHUNKED_REQUEST_SIZE = "s3proxy.v4-max-non-chunked-request-size"; + /** Used to locate blobstores by specified bucket names. Each property + * file should contain a list of buckets associated with it, e.g. + * s3proxy.bucket-locator.1 = data + * s3proxy.bucket-locator.2 = metadata + * s3proxy.bucket-locator.3 = other + * When a request is made for the specified bucket, the backend defined + * in that properties file is used. This allows using the same + * credentials in multiple properties file and select the backend based + * on the bucket names. + */ + public static final String PROPERTY_BUCKET_LOCATOR = + "s3proxy.bucket-locator"; /** When true, model eventual consistency using two storage backends. */ public static final String PROPERTY_EVENTUAL_CONSISTENCY = "s3proxy.eventual-consistency";