From eb4f39f3df31357229c6a670c1df34c91c278c7a Mon Sep 17 00:00:00 2001 From: Philip Sampaio Date: Thu, 17 Aug 2023 14:18:02 -0300 Subject: [PATCH] Parse config from bucket URL --- lib/fss/s3.ex | 86 ++++++++++++++++++++++++++++++++++++++++++++ test/fss/s3_test.exs | 85 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 171 insertions(+) diff --git a/lib/fss/s3.ex b/lib/fss/s3.ex index 6e2daf3..f525973 100644 --- a/lib/fss/s3.ex +++ b/lib/fss/s3.ex @@ -146,4 +146,90 @@ defmodule FSS.S3 do token: System.get_env("AWS_SESSION_TOKEN") } end + + @doc """ + Parses configuration from a bucket URL. + + It expects an URL in the following formats: + + - `https://s3.[region].amazonaws.com/[bucket]` + - `https://[bucket].s3.[region].amazonaws.com` + - `https://my-custom-endpoint.com/[bucket]` + + If the URL is not in one of these formats, then an error + is returned. For custom endpoints (services that are compatible + with AWS S3), it's possible to pass the port and use HTTP. + + All the additional required configuration must be passed as + options to the `:config` option. + + ## Options + + * `:config` - It expects a `Config.t()` or a `Keyword.t()` with the keys + representing the attributes of the `Config.t()`. By default it is `nil`, + which means that we are going to try to fetch the credentials and configuration + from the system's environment variables. + + The following env vars are read: + + - `AWS_ACCESS_KEY_ID` + - `AWS_SECRET_ACCESS_KEY` + - `AWS_REGION` or `AWS_DEFAULT_REGION` + - `AWS_SESSION_TOKEN` + + Although you can pass the `:bucket` and `:region` as options, + they are always override by the values parsed from the URL. + """ + @spec parse_config_from_bucket_url(String.t()) :: {:ok, Config.t()} | {:error, Exception.t()} + def parse_config_from_bucket_url(bucket_url, opts \\ []) do + opts = Keyword.validate!(opts, config: nil) + + uri = URI.parse(bucket_url) + + case uri do + %{host: host, path: path} when is_binary(host) and (is_nil(path) or path == "/") -> + base_config = + opts + |> Keyword.fetch!(:config) + |> normalize_config!() + + case String.split(host, ".") do + [bucket, "s3", region, "amazonaws", "com"] -> + {:ok, %Config{base_config | bucket: bucket, region: region}} + + _other -> + {:error, + ArgumentError.exception( + "cannot extract bucket name from URL. Expected URL in the format " <> + "https://s3.[region].amazonaws.com/[bucket], got: " <> + URI.to_string(uri) + )} + end + + %{host: host, path: "/" <> bucket} when is_binary(host) -> + base_config = + opts + |> Keyword.fetch!(:config) + |> normalize_config!() + |> then(fn config -> %Config{config | bucket: bucket} end) + + case String.split(host, ".") do + ["s3", region, "amazonaws", "com"] -> + {:ok, %Config{base_config | region: region}} + + _other -> + endpoint_uri = %URI{uri | path: nil} + + {:ok, %Config{base_config | endpoint: URI.to_string(endpoint_uri)}} + end + + _ -> + {:error, + ArgumentError.exception( + "expected URL in the format " <> + "https://s3.[region].amazonaws.com/[bucket], got: " <> + URI.to_string(uri) + )} + end + end end diff --git a/test/fss/s3_test.exs b/test/fss/s3_test.exs index 1470884..89a4761 100644 --- a/test/fss/s3_test.exs +++ b/test/fss/s3_test.exs @@ -115,4 +115,89 @@ defmodule FSS.S3Test do end end end + + describe "parse_config_from_bucket_url/2" do + setup do + default_config = [ + secret_access_key: "my-secret", + access_key_id: "my-access" + ] + + [default_config: default_config] + end + + test "parses a path-style AWS S3 url", %{default_config: default_config} do + assert {:ok, config} = + S3.parse_config_from_bucket_url("https://s3.us-west-2.amazonaws.com/my-bucket", + config: default_config + ) + + assert %Config{} = config + assert config.region == "us-west-2" + assert config.bucket == "my-bucket" + assert is_nil(config.endpoint) + end + + test "parses a host-style AWS S3 url", %{default_config: default_config} do + assert {:ok, config} = + S3.parse_config_from_bucket_url("https://my-bucket-1.s3.us-west-2.amazonaws.com/", + config: default_config + ) + + assert %Config{} = config + assert config.region == "us-west-2" + assert config.bucket == "my-bucket-1" + assert is_nil(config.endpoint) + end + + test "parses a path-style S3 compatible url", %{default_config: default_config} do + assert {:ok, config} = + S3.parse_config_from_bucket_url("https://storage.googleapis.com/my-bucket-on-gcp", + config: default_config + ) + + assert %Config{} = config + assert is_nil(config.region) + assert config.bucket == "my-bucket-on-gcp" + assert config.endpoint == "https://storage.googleapis.com" + end + + test "parses a path-style S3 compatible url with a port", %{default_config: default_config} do + assert {:ok, config} = + S3.parse_config_from_bucket_url("http://localhost:4852/my-bucket-on-lh", + config: default_config + ) + + assert %Config{} = config + assert is_nil(config.region) + assert config.bucket == "my-bucket-on-lh" + assert config.endpoint == "http://localhost:4852" + end + + test "cannot extract bucket from host-style S3 url", %{default_config: default_config} do + assert {:error, error} = + S3.parse_config_from_bucket_url("https://my-bucket-on-gcp.storage.googleapis.com", + config: default_config + ) + + message = + "cannot extract bucket name from URL. Expected URL in the format " <> + "https://s3.[region].amazonaws.com/[bucket], got: " <> + "https://my-bucket-on-gcp.storage.googleapis.com" + + assert error == ArgumentError.exception(message) + end + + test "cannot parse url without host", %{default_config: default_config} do + assert {:error, error} = + S3.parse_config_from_bucket_url("/my-path", + config: default_config + ) + + message = + "expected URL in the format https://s3.[region].amazonaws.com/[bucket], got: /my-path" + + assert error == ArgumentError.exception(message) + end + end end