From 2f8f863d5c437c138ead6fd044d08142bcaae820 Mon Sep 17 00:00:00 2001 From: Augustin Nesson Date: Fri, 24 May 2024 18:53:33 +0200 Subject: [PATCH 1/4] feat: remove mandatory and fixed parameters from queryables --- eodag/config.py | 2 +- eodag/plugins/search/build_search_result.py | 17 +++++++++-- eodag/plugins/search/qssearch.py | 14 +++++++-- eodag/resources/providers.yml | 20 +++++++++++++ tests/units/test_search_plugins.py | 33 +++++++++++++++++++++ 5 files changed, 80 insertions(+), 6 deletions(-) diff --git a/eodag/config.py b/eodag/config.py index 173e32dce..d5d14b586 100644 --- a/eodag/config.py +++ b/eodag/config.py @@ -334,7 +334,7 @@ class OrderStatus(TypedDict): timeout: float # StaticStacSearch s3_bucket: str # CreodiasS3Search end_date_excluded: bool # BuildSearchResult - remove_from_query: List[str] # BuildSearchResult + remove_from_query: Dict[str, List[Any]] # BuildSearchResult ssl_verify: bool # download ------------------------------------------------------------------------- diff --git a/eodag/plugins/search/build_search_result.py b/eodag/plugins/search/build_search_result.py index 855915a1e..5a25340c4 100644 --- a/eodag/plugins/search/build_search_result.py +++ b/eodag/plugins/search/build_search_result.py @@ -466,8 +466,6 @@ def discover_queryables( # defaults default_queryables = self._get_defaults_as_queryables(product_type) - # remove dataset from queryables - default_queryables.pop("dataset", None) non_empty_kwargs = {k: v for k, v in kwargs.items() if v} @@ -535,4 +533,17 @@ def discover_queryables( field_definitions[param] = get_args(annotated_def) python_queryables = create_model("m", **field_definitions).model_fields - return {**default_queryables, **model_fields_to_annotated(python_queryables)} + queryables = { + **default_queryables, + **model_fields_to_annotated(python_queryables), + } + # remove fixed params from queryables + for param in getattr(self.config, "remove_from_queryables", {}).get( + "shared_queryables", [] + ): + queryables.pop(param) + for param in getattr(self.config, "remove_from_queryables", {}).get( + product_type, [] + ): + queryables.pop(param) + return queryables diff --git a/eodag/plugins/search/qssearch.py b/eodag/plugins/search/qssearch.py index 669f996dc..832288ff6 100644 --- a/eodag/plugins/search/qssearch.py +++ b/eodag/plugins/search/qssearch.py @@ -1699,8 +1699,18 @@ def discover_queryables( ) or json_param ) - - default = kwargs.get(param, None) + # prevent fixed params from being in queryables python model + if param in getattr(self.config, "remove_from_queryables", {}).get( + "shared_queryables", [] + ): + continue + if param in getattr(self.config, "remove_from_queryables", {}).get( + product_type, [] + ): + continue + default = kwargs.get(param, None) or self.config.products.get( + product_type, {} + ).get(param, None) annotated_def = json_field_definition_to_python( json_mtd, default_value=default ) diff --git a/eodag/resources/providers.yml b/eodag/resources/providers.yml index 313ff353a..a32115202 100644 --- a/eodag/resources/providers.yml +++ b/eodag/resources/providers.yml @@ -2283,6 +2283,9 @@ fetch_url: null product_type_fetch_url: null constraints_file_url: "https://datastore.copernicus-climate.eu/cams/published-forms/camsprod/{dataset}/constraints.json" + remove_from_queryables: + shared_queryables: + - dataset metadata_mapping: productType: '$.productType' title: '$.id' @@ -2857,6 +2860,9 @@ fetch_url: null product_type_fetch_url: null constraints_file_url: http://datastore.copernicus-climate.eu/c3s/published-forms/c3sprod/{dataset}/constraints.json + remove_from_queryables: + shared_queryables: + - dataset metadata_mapping: productType: $.productType title: $.id @@ -6527,6 +6533,20 @@ fetch_url: null product_type_fetch_url: null constraints_file_url: eodag/resources/constraints/{dataset}.json + remove_from_queryables: + shared_queryables: + - dataset + - class + - expver + - type + DT_EXTREMES: + - time + DT_CLIMATE_ADAPTATION: + - generation + - realization + - stream + - activity + - experiment metadata_mapping: productType: destination-earth storageStatus: OFFLINE diff --git a/tests/units/test_search_plugins.py b/tests/units/test_search_plugins.py index bad66140b..ba498604c 100644 --- a/tests/units/test_search_plugins.py +++ b/tests/units/test_search_plugins.py @@ -612,6 +612,7 @@ def test_plugins_search_querystringsearch_discover_queryables( timeout=5, ) + self.assertEqual(10, len(queryables)) # queryables from provider constraints file are added (here the ones of ERA5_SL_MONTHLY for wekeo) for provider_queryable in provider_queryables_from_constraints_file: provider_queryable = ( @@ -2053,6 +2054,7 @@ def test_plugins_search_buildsearchresult_discover_queryables( timeout=5, ) + self.assertEqual(11, len(queryables)) # queryables from provider constraints file are added (here the ones of CAMS_EU_AIR_QUALITY_RE for cop_ads) for provider_queryable in provider_queryables_from_constraints_file: provider_queryable = ( @@ -2093,6 +2095,16 @@ def test_plugins_search_buildsearchresult_discover_queryables( set(variable_constraints), set(queryable.__origin__.__args__) ) + # check that fixed params have been removed from queryables if necessary + for param in getattr( + self.search_plugin.config, "remove_from_queryables", {} + ).get("shared_queryables", []): + self.assertNotIn(param, queryables) + for param in getattr( + self.search_plugin.config, "remove_from_queryables", {} + ).get("CAMS_EU_AIR_QUALITY_RE", []): + self.assertNotIn(param, queryables) + # reset mock mock_requests_session_constraints.reset_mock() @@ -2119,6 +2131,16 @@ def test_plugins_search_buildsearchresult_discover_queryables( self.assertEqual("a", queryable.__metadata__[0].get_default()) self.assertFalse(queryable.__metadata__[0].is_required()) + # check that fixed params have been removed from queryables if necessary + for param in getattr( + self.search_plugin.config, "remove_from_queryables", {} + ).get("shared_queryables", []): + self.assertNotIn(param, queryables) + for param in getattr( + self.search_plugin.config, "remove_from_queryables", {} + ).get("CAMS_EU_AIR_QUALITY_RE", []): + self.assertNotIn(param, queryables) + def test_plugins_search_buildsearchresult_discover_queryables_with_local_constraints_file( self, ): @@ -2144,6 +2166,7 @@ def test_plugins_search_buildsearchresult_discover_queryables_with_local_constra ) self.assertIsNotNone(queryables) + self.assertEqual(9, len(queryables)) # queryables from provider constraints file are added (here the ones of CAMS_EU_AIR_QUALITY_RE for cop_ads) for provider_queryable in provider_queryables_from_constraints_file: provider_queryable = ( @@ -2198,6 +2221,16 @@ def test_plugins_search_buildsearchresult_discover_queryables_with_local_constra self.assertEqual("a", queryable.__metadata__[0].get_default()) self.assertFalse(queryable.__metadata__[0].is_required()) + # check that fixed params have been removed from queryables if necessary + for param in getattr( + self.search_plugin.config, "remove_from_queryables", {} + ).get("shared_queryables", []): + self.assertNotIn(param, queryables) + for param in getattr( + self.search_plugin.config, "remove_from_queryables", {} + ).get("CAMS_EU_AIR_QUALITY_RE", []): + self.assertNotIn(param, queryables) + # restore configuration self.search_plugin.config.constraints_file_url = tmp_search_constraints_file_url From 4689c1b96e78ca2b67d8fb4049a063a6db53a4d0 Mon Sep 17 00:00:00 2001 From: Augustin Nesson Date: Fri, 24 May 2024 19:56:35 +0200 Subject: [PATCH 2/4] fix: set correctly 'remove_from_queryables' and 'remove_from_query' in eodag configuration --- eodag/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eodag/config.py b/eodag/config.py index d5d14b586..ff1d00dbf 100644 --- a/eodag/config.py +++ b/eodag/config.py @@ -321,7 +321,7 @@ class OrderStatus(TypedDict): metadata_mapping: Dict[str, Union[str, List[str]]] free_params: Dict[Any, Any] constraints_file_url: str - remove_from_queryables: List[str] + remove_from_queryables: Dict[str, List[Any]] free_text_search_operations: Dict[str, Any] # ODataV4Search metadata_pre_mapping: Dict[str, Any] # ODataV4Search data_request_url: str # DataRequestSearch @@ -334,7 +334,7 @@ class OrderStatus(TypedDict): timeout: float # StaticStacSearch s3_bucket: str # CreodiasS3Search end_date_excluded: bool # BuildSearchResult - remove_from_query: Dict[str, List[Any]] # BuildSearchResult + remove_from_query: List[Any] # BuildSearchResult ssl_verify: bool # download ------------------------------------------------------------------------- From e55405404a7fc6ad0afa0e15ed43d87137ca69fe Mon Sep 17 00:00:00 2001 From: Augustin Nesson Date: Fri, 24 May 2024 19:59:04 +0200 Subject: [PATCH 3/4] feat: remove mandatory and fixed parameters from queryables for providers with qssearch search plugin --- eodag/plugins/search/qssearch.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/eodag/plugins/search/qssearch.py b/eodag/plugins/search/qssearch.py index 832288ff6..a04a9603c 100644 --- a/eodag/plugins/search/qssearch.py +++ b/eodag/plugins/search/qssearch.py @@ -645,7 +645,20 @@ def discover_queryables( field_definitions[param] = get_args(annotated_def) python_queryables = create_model("m", **field_definitions).model_fields - return dict(default_queryables, **model_fields_to_annotated(python_queryables)) + queryables = { + **default_queryables, + **model_fields_to_annotated(python_queryables), + } + # remove fixed params from queryables + for param in getattr(self.config, "remove_from_queryables", {}).get( + "shared_queryables", [] + ): + queryables.pop(param) + for param in getattr(self.config, "remove_from_queryables", {}).get( + product_type, [] + ): + queryables.pop(param) + return queryables def query( self, From 2cb5ff053010896fec781e6b9839571cb1faa123 Mon Sep 17 00:00:00 2001 From: Augustin Nesson Date: Fri, 24 May 2024 20:03:02 +0200 Subject: [PATCH 4/4] test: fix one assertion --- tests/units/test_search_plugins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/units/test_search_plugins.py b/tests/units/test_search_plugins.py index ba498604c..5a3188d99 100644 --- a/tests/units/test_search_plugins.py +++ b/tests/units/test_search_plugins.py @@ -2054,7 +2054,7 @@ def test_plugins_search_buildsearchresult_discover_queryables( timeout=5, ) - self.assertEqual(11, len(queryables)) + self.assertEqual(9, len(queryables)) # queryables from provider constraints file are added (here the ones of CAMS_EU_AIR_QUALITY_RE for cop_ads) for provider_queryable in provider_queryables_from_constraints_file: provider_queryable = (