From 027126076cbf8d4c59924668552304d4dbf1fe78 Mon Sep 17 00:00:00 2001 From: Sylvain Brunato <61419125+sbrunato@users.noreply.github.com> Date: Thu, 29 Feb 2024 10:01:39 +0100 Subject: [PATCH 1/2] fix: update rio_env for s3 assets (#50) --- eodag_cube/api/product/_product.py | 15 +++++++++++--- tests/context.py | 3 +++ tests/units/test_eoproduct.py | 32 ++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 3 deletions(-) diff --git a/eodag_cube/api/product/_product.py b/eodag_cube/api/product/_product.py index 86aa767..24359d6 100644 --- a/eodag_cube/api/product/_product.py +++ b/eodag_cube/api/product/_product.py @@ -223,16 +223,25 @@ def _get_rio_env(self, dataset_address: str) -> Dict[str, Any]: """ product_location_scheme = dataset_address.split("://")[0] if product_location_scheme == "s3" and hasattr( - self.downloader, "get_bucket_name_and_prefix" + self.downloader, "get_product_bucket_name_and_prefix" ): - bucket_name, prefix = self.downloader.get_bucket_name_and_prefix( + bucket_name, prefix = self.downloader.get_product_bucket_name_and_prefix( self, dataset_address ) auth_dict = self.downloader_auth.authenticate() - return { + rio_env_dict = { "session": rasterio.session.AWSSession( **self.downloader.get_rio_env(bucket_name, prefix, auth_dict) ) } + endpoint_url = getattr(self.downloader.config, "base_uri", None) + if endpoint_url: + aws_s3_endpoint = endpoint_url.split("://")[-1] + rio_env_dict.update( + AWS_S3_ENDPOINT=aws_s3_endpoint, + AWS_HTTPS="YES", + AWS_VIRTUAL_HOSTING="FALSE", + ) + return rio_env_dict else: return {} diff --git a/tests/context.py b/tests/context.py index dcf0779..c47c529 100644 --- a/tests/context.py +++ b/tests/context.py @@ -29,13 +29,16 @@ from eodag.api.product import EOProduct from eodag.api.product.drivers import DRIVERS from eodag.api.product.drivers.base import NoDriver +from eodag.config import PluginConfig from eodag_cube.api.product.drivers.generic import GenericDriver from eodag_cube.api.product.drivers.sentinel2_l1c import Sentinel2L1C from eodag_cube.api.product.drivers.stac_assets import StacAssets from eodag.api.search_result import SearchResult from eodag.cli import download, eodag, list_pt, search_crunch from eodag.plugins.authentication.base import Authentication +from eodag.plugins.authentication.aws_auth import AwsAuth from eodag.plugins.download.base import Download +from eodag.plugins.download.aws import AwsDownload from eodag.plugins.search.base import Search from eodag.rest import server as eodag_http_server from eodag.rest.utils import eodag_api, get_date diff --git a/tests/units/test_eoproduct.py b/tests/units/test_eoproduct.py index cea2681..f17544e 100644 --- a/tests/units/test_eoproduct.py +++ b/tests/units/test_eoproduct.py @@ -21,15 +21,19 @@ import numpy as np import xarray as xr +from rasterio.session import AWSSession from tests import EODagTestCase from tests.context import ( DEFAULT_PROJ, Authentication, + AwsAuth, + AwsDownload, Download, DownloadError, EOProduct, NoDriver, + PluginConfig, Sentinel2L1C, UnsupportedDatasetAddressScheme, config, @@ -187,3 +191,31 @@ def execute_get_data( ) return tuple(itertools.chain.from_iterable(((data,), returned_params))) return data + + def test_get_rio_env(self): + """RIO env should be adapted to the provider config""" + product = EOProduct( + self.provider, self.eoproduct_props, productType=self.product_type + ) + + # http + self.assertDictEqual(product._get_rio_env("https://path/to/asset"), {}) + + # aws s3 + product.register_downloader( + AwsDownload("foo", PluginConfig()), AwsAuth("foo", PluginConfig()) + ) + product.downloader._get_authenticated_objects_unsigned = mock.MagicMock() + product.downloader._get_authenticated_objects_unsigned.__name__ = "mocked" + rio_env = product._get_rio_env("s3://path/to/asset") + self.assertEqual(len(rio_env), 1) + self.assertIsInstance(rio_env["session"], AWSSession) + + # aws s3 with custom endpoint + product.downloader.config.base_uri = "https://some.where" + rio_env = product._get_rio_env("s3://path/to/asset") + self.assertEqual(len(rio_env), 4) + self.assertIsInstance(rio_env["session"], AWSSession) + self.assertEqual(rio_env["AWS_HTTPS"], "YES") + self.assertEqual(rio_env["AWS_S3_ENDPOINT"], "some.where") + self.assertEqual(rio_env["AWS_VIRTUAL_HOSTING"], "FALSE") From 039ae0e2bdb8a639309fc05c56f366da9b8ea60d Mon Sep 17 00:00:00 2001 From: Sylvain Brunato <61419125+sbrunato@users.noreply.github.com> Date: Thu, 29 Feb 2024 10:37:54 +0100 Subject: [PATCH 2/2] build: bump version (#51) --- CHANGES.rst | 5 +++++ eodag_cube/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 7f9760f..c4ab6fb 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,11 @@ Release history --------------- +0.4.1 (2024-02-29) +++++++++++++++++++ + +- Fixes s3 authentication using GDAL/rasterio environment variables (#50) + 0.4.0 (2024-02-19) ++++++++++++++++++ diff --git a/eodag_cube/__init__.py b/eodag_cube/__init__.py index d06df94..3de38db 100644 --- a/eodag_cube/__init__.py +++ b/eodag_cube/__init__.py @@ -19,7 +19,7 @@ __title__ = "eodag-cube" __description__ = "Data access for EODAG" -__version__ = "0.4.0" +__version__ = "0.4.1" __author__ = "CS GROUP - France (CSSI)" __author_email__ = "eodag@csgroup.space" __url__ = "https://github.com/CS-SI/eodag-cube"