Update MITAardvark 'rights' field method to support access filter

Why these changes are being introduced: * Enable filtering TIMDEX records based on "Access to files" by providing a static, aggregatable string value indicating the category for a given record. How this addresses that need: * Update MITAardvark.get_rights field method * Accept MITAardvark.source as an argument to determine ownership * Use "dct_accessRights_s" to determine access category Side effects of this change: * Need to determine whether OpenSearch can limit aggregation to a field that is filtered by a subfield (i.e., Rights.kind) and make necessary updates to timdex-ui. Relevant ticket(s): * https://mitlibraries.atlassian.net/browse/GDT-138
MITLibraries · Jan 24, 2024 · 5c6a2d5 · 5c6a2d5
1 parent 4f83410
commit 5c6a2d5
Show file tree

Hide file tree

Showing 3 changed files with 74 additions and 5 deletions.
diff --git a/tests/fixtures/aardvark/aardvark_record_all_fields.jsonl b/tests/fixtures/aardvark/aardvark_record_all_fields.jsonl
@@ -1 +1,4 @@
-{"id": "mit:123", "dcat_bbox": "ENVELOPE(-111.1, -104.0, 45.0, 40.9)", "dcat_keyword_sm": ["Country"], "dcat_theme_sm": ["Political boundaries"], "dct_accessRights_s": "Access note", "dct_alternative_sm": ["Alternate title"], "dct_creator_sm": ["Smith, Jane", "Smith, John"], "dct_description_sm": ["A description"], "dct_format_s": "Shapefile", "dct_identifier_sm": ["abc123"], "dct_issued_s": "2003-10-23", "dct_language_sm": ["eng"], "dct_license_sm": ["http://license.license", "http://another_license.another_license"], "dct_publisher_sm": ["ML InfoMap (Firm)"], "dct_references_s": "{\"https://schema.org/downloadUrl\": [{\"label\": \"Source Metadata\", \"protocol\": \"Download\", \"url\": \"https://example.com/GISPORTAL_GISOWNER01_BOSTONWATER95.source.fgdc.xml\"}, {\"label\": \"Normalized Metadata\", \"protocol\": \"Download\", \"url\": \"https://example.com/GISPORTAL_GISOWNER01_BOSTONWATER95.normalized.aardvark.json\"}, {\"label\": \"Data Zipfile\", \"protocol\": \"Download\", \"url\": \"https://example.com/GISPORTAL_GISOWNER01_BOSTONWATER95.zip\"}]}", "dct_rights_sm": ["Some person has the rights"], "dct_rightsHolder_sm": ["The person with the rights", "Another person with the rights"], "dct_spatial_sm": ["Some city, Some country"], "dct_subject_sm": ["Geography", "Earth"], "dct_temporal_sm": ["1943", "1979"], "dct_title_s": "Test title 1", "gbl_dateRange_drsim": ["[1943 TO 1946]"], "gbl_displayNote_sm": ["Danger: This text will be displayed in a red box","Info: This text will be displayed in a blue box","Tip: This text will be displayed in a green box","Warning: This text will be displayed in a yellow box","This is text without a tag and it will be assigned default 'note' style"], "gbl_indexYear_im": [1943,1944,1945,1946], "gbl_resourceClass_sm": ["Dataset"], "gbl_resourceType_sm": ["Vector data"], "gbl_suppressed_b": false, "locn_geometry": "ENVELOPE(-111.1, -104.0, 45.0, 40.9)", "schema_provider_s": "MIT"}
+{"id": "mit:123", "dcat_bbox": "ENVELOPE(-111.1, -104.0, 45.0, 40.9)", "dcat_keyword_sm": ["Country"], "dcat_theme_sm": ["Political boundaries"], "dct_accessRights_s": "Access note", "dct_alternative_sm": ["Alternate title"], "dct_creator_sm": ["Smith, Jane", "Smith, John"], "dct_description_sm": ["A description"], "dct_format_s": "Shapefile", "dct_identifier_sm": ["abc123"], "dct_issued_s": "2003-10-23", "dct_language_sm": ["eng"], "dct_license_sm": ["http://license.license", "http://another_license.another_license"], "dct_publisher_sm": ["ML InfoMap (Firm)"], "dct_references_s": "{\"https://schema.org/downloadUrl\": [{\"label\": \"Source Metadata\", \"protocol\": \"Download\", \"url\": \"https://example.com/GISPORTAL_GISOWNER01_BOSTONWATER95.source.fgdc.xml\"}, {\"label\": \"Normalized Metadata\", \"protocol\": \"Download\", \"url\": \"https://example.com/GISPORTAL_GISOWNER01_BOSTONWATER95.normalized.aardvark.json\"}, {\"label\": \"Data Zipfile\", \"protocol\": \"Download\", \"url\": \"https://example.com/GISPORTAL_GISOWNER01_BOSTONWATER95.zip\"}]}", "dct_rights_sm": ["Some person has the rights"], "dct_rightsHolder_sm": ["The person with the rights", "Another person with the rights"], "dct_spatial_sm": ["Some city, Some country"], "dct_subject_sm": ["Geography", "Earth"], "dct_temporal_sm": ["1943", "1979"], "dct_title_s": "Test title 1", "gbl_dateRange_drsim": ["[1943 TO 1946]"], "gbl_displayNote_sm": ["Danger: This text will be displayed in a red box","Info: This text will be displayed in a blue box","Tip: This text will be displayed in a green box","Warning: This text will be displayed in a yellow box","This is text without a tag and it will be assigned default 'note' style"], "gbl_indexYear_im": [1943,1944,1945,1946], "gbl_resourceClass_sm": ["Dataset"], "gbl_resourceType_sm": ["Vector data"], "gbl_suppressed_b": false, "locn_geometry": "ENVELOPE(-111.1, -104.0, 45.0, 40.9)", "schema_provider_s": "MIT"}
+{"id": "mit:124", "dcat_bbox": "ENVELOPE(-111.2, -104.0, 45.0, 40.9)", "dcat_keyword_sm": ["Country"], "dcat_theme_sm": ["Political boundaries"], "dct_accessRights_s": "Restricted", "dct_alternative_sm": ["Alternate title"], "dct_creator_sm": ["Smith, Jane", "Smith, John"], "dct_description_sm": ["A description"], "dct_format_s": "Shapefile", "dct_identifier_sm": ["abc123"], "dct_issued_s": "2003-10-23", "dct_language_sm": ["eng"], "dct_license_sm": ["http://license.license", "http://another_license.another_license"], "dct_publisher_sm": ["ML InfoMap (Firm)"], "dct_references_s": "{\"https://schema.org/downloadUrl\": [{\"label\": \"Source Metadata\", \"protocol\": \"Download\", \"url\": \"https://example.com/GISPORTAL_GISOWNER01_BOSTONWATER95.source.fgdc.xml\"}, {\"label\": \"Normalized Metadata\", \"protocol\": \"Download\", \"url\": \"https://example.com/GISPORTAL_GISOWNER01_BOSTONWATER95.normalized.aardvark.json\"}, {\"label\": \"Data Zipfile\", \"protocol\": \"Download\", \"url\": \"https://example.com/GISPORTAL_GISOWNER01_BOSTONWATER95.zip\"}]}", "dct_rights_sm": ["Some person has the rights"], "dct_rightsHolder_sm": ["The person with the rights", "Another person with the rights"], "dct_spatial_sm": ["Some city, Some country"], "dct_subject_sm": ["Geography", "Earth"], "dct_temporal_sm": ["1943", "1979"], "dct_title_s": "Test title 1", "gbl_dateRange_drsim": ["[1943 TO 1946]"], "gbl_displayNote_sm": ["Danger: This text will be displayed in a red box","Info: This text will be displayed in a blue box","Tip: This text will be displayed in a green box","Warning: This text will be displayed in a yellow box","This is text without a tag and it will be assigned default 'note' style"], "gbl_indexYear_im": [1943,1944,1945,1946], "gbl_resourceClass_sm": ["Dataset"], "gbl_resourceType_sm": ["Vector data"], "gbl_suppressed_b": false, "locn_geometry": "ENVELOPE(-111.1, -104.0, 45.0, 40.9)", "schema_provider_s": "MIT"}
+{"id": "mit:125", "dcat_bbox": "ENVELOPE(-111.3, -104.0, 45.0, 40.9)", "dcat_keyword_sm": ["Country"], "dcat_theme_sm": ["Political boundaries"], "dct_accessRights_s": "Public", "dct_alternative_sm": ["Alternate title"], "dct_creator_sm": ["Smith, Jane", "Smith, John"], "dct_description_sm": ["A description"], "dct_format_s": "Shapefile", "dct_identifier_sm": ["abc123"], "dct_issued_s": "2003-10-23", "dct_language_sm": ["eng"], "dct_license_sm": ["http://license.license", "http://another_license.another_license"], "dct_publisher_sm": ["ML InfoMap (Firm)"], "dct_references_s": "{\"https://schema.org/downloadUrl\": [{\"label\": \"Source Metadata\", \"protocol\": \"Download\", \"url\": \"https://example.com/GISPORTAL_GISOWNER01_BOSTONWATER95.source.fgdc.xml\"}, {\"label\": \"Normalized Metadata\", \"protocol\": \"Download\", \"url\": \"https://example.com/GISPORTAL_GISOWNER01_BOSTONWATER95.normalized.aardvark.json\"}, {\"label\": \"Data Zipfile\", \"protocol\": \"Download\", \"url\": \"https://example.com/GISPORTAL_GISOWNER01_BOSTONWATER95.zip\"}]}", "dct_rights_sm": ["Some person has the rights"], "dct_rightsHolder_sm": ["The person with the rights", "Another person with the rights"], "dct_spatial_sm": ["Some city, Some country"], "dct_subject_sm": ["Geography", "Earth"], "dct_temporal_sm": ["1943", "1979"], "dct_title_s": "Test title 1", "gbl_dateRange_drsim": ["[1943 TO 1946]"], "gbl_displayNote_sm": ["Danger: This text will be displayed in a red box","Info: This text will be displayed in a blue box","Tip: This text will be displayed in a green box","Warning: This text will be displayed in a yellow box","This is text without a tag and it will be assigned default 'note' style"], "gbl_indexYear_im": [1943,1944,1945,1946], "gbl_resourceClass_sm": ["Dataset"], "gbl_resourceType_sm": ["Vector data"], "gbl_suppressed_b": false, "locn_geometry": "ENVELOPE(-111.1, -104.0, 45.0, 40.9)", "schema_provider_s": "MIT"}
+{"id": "ogm:123", "dcat_bbox": "ENVELOPE(-111.4, -104.0, 45.0, 40.9)", "dcat_keyword_sm": ["Country"], "dcat_theme_sm": ["Political boundaries"], "dct_accessRights_s": "Restricted", "dct_alternative_sm": ["Alternate title"], "dct_creator_sm": ["Smith, Jane", "Smith, John"], "dct_description_sm": ["A description"], "dct_format_s": "Shapefile", "dct_identifier_sm": ["abc123"], "dct_issued_s": "2003-10-23", "dct_language_sm": ["eng"], "dct_license_sm": ["http://license.license", "http://another_license.another_license"], "dct_publisher_sm": ["ML InfoMap (Firm)"], "dct_references_s": "{\"https://schema.org/downloadUrl\": [{\"label\": \"Source Metadata\", \"protocol\": \"Download\", \"url\": \"https://example.com/GISPORTAL_GISOWNER01_BOSTONWATER95.source.fgdc.xml\"}, {\"label\": \"Normalized Metadata\", \"protocol\": \"Download\", \"url\": \"https://example.com/GISPORTAL_GISOWNER01_BOSTONWATER95.normalized.aardvark.json\"}, {\"label\": \"Data Zipfile\", \"protocol\": \"Download\", \"url\": \"https://example.com/GISPORTAL_GISOWNER01_BOSTONWATER95.zip\"}]}", "dct_rights_sm": ["Some person has the rights"], "dct_rightsHolder_sm": ["The person with the rights", "Another person with the rights"], "dct_spatial_sm": ["Some city, Some country"], "dct_subject_sm": ["Geography", "Earth"], "dct_temporal_sm": ["1943", "1979"], "dct_title_s": "Test title 1", "gbl_dateRange_drsim": ["[1943 TO 1946]"], "gbl_displayNote_sm": ["Danger: This text will be displayed in a red box","Info: This text will be displayed in a blue box","Tip: This text will be displayed in a green box","Warning: This text will be displayed in a yellow box","This is text without a tag and it will be assigned default 'note' style"], "gbl_indexYear_im": [1943,1944,1945,1946], "gbl_resourceClass_sm": ["Dataset"], "gbl_resourceType_sm": ["Vector data"], "gbl_suppressed_b": false, "locn_geometry": "ENVELOPE(-111.1, -104.0, 45.0, 40.9)", "schema_provider_s": "MIT"}
diff --git a/tests/sources/json/test_aardvark.py b/tests/sources/json/test_aardvark.py
@@ -246,7 +246,7 @@ def test_aardvark_get_publication_information_success(aardvark_record_all_fields
 
 
 def test_aardvark_get_rights_success(aardvark_record_all_fields):
-    assert MITAardvark.get_rights(next(aardvark_record_all_fields)) == [
+    assert MITAardvark.get_rights("source", next(aardvark_record_all_fields)) == [
         timdex.Rights(description="Access note", kind="Access"),
         timdex.Rights(uri="http://license.license"),
         timdex.Rights(uri="http://another_license.another_license"),
@@ -256,6 +256,48 @@ def test_aardvark_get_rights_success(aardvark_record_all_fields):
         ),
     ]
 
+    aardvark_record_all_fields_mit_restricted = next(aardvark_record_all_fields)
+    assert MITAardvark.get_rights(
+        "gismit", aardvark_record_all_fields_mit_restricted
+    ) == [
+        timdex.Rights(description="Restricted", kind="Access"),
+        timdex.Rights(
+            description="Requires MIT authentication", kind="Access to files"
+        ),
+        timdex.Rights(uri="http://license.license"),
+        timdex.Rights(uri="http://another_license.another_license"),
+        timdex.Rights(description="Some person has the rights"),
+        timdex.Rights(
+            description="The person with the rights. Another person with the rights"
+        ),
+    ]
+
+    aardvark_record_all_fields_mit_public = next(aardvark_record_all_fields)
+    assert MITAardvark.get_rights("gismit", aardvark_record_all_fields_mit_public) == [
+        timdex.Rights(description="Public", kind="Access"),
+        timdex.Rights(description="Free/open-access", kind="Access to files"),
+        timdex.Rights(uri="http://license.license"),
+        timdex.Rights(uri="http://another_license.another_license"),
+        timdex.Rights(description="Some person has the rights"),
+        timdex.Rights(
+            description="The person with the rights. Another person with the rights"
+        ),
+    ]
+
+    aardvark_record_all_fields_ogm_restricted = next(aardvark_record_all_fields)
+    assert MITAardvark.get_rights(
+        "gisogm", aardvark_record_all_fields_ogm_restricted
+    ) == [
+        timdex.Rights(description="Restricted", kind="Access"),
+        timdex.Rights(description="Not owned by MIT", kind="Access to files"),
+        timdex.Rights(uri="http://license.license"),
+        timdex.Rights(uri="http://another_license.another_license"),
+        timdex.Rights(description="Some person has the rights"),
+        timdex.Rights(
+            description="The person with the rights. Another person with the rights"
+        ),
+    ]
+
 
 def test_aardvark_get_subjects_success(aardvark_record_all_fields):
     assert MITAardvark.get_subjects(next(aardvark_record_all_fields)) == [

diff --git a/transmogrifier/sources/json/aardvark.py b/transmogrifier/sources/json/aardvark.py
@@ -156,7 +156,7 @@ def get_optional_fields(self, source_record: dict) -> dict | None:
         # related_items not used in MITAardvark
 
         # rights
-        fields["rights"] = self.get_rights(source_record) or None
+        fields["rights"] = self.get_rights(self.source, source_record) or None
 
         # subjects
         fields["subjects"] = self.get_subjects(source_record) or None
@@ -349,17 +349,41 @@ def get_publication_information(source_record: dict) -> list[str]:
         return publication_information
 
     @staticmethod
-    def get_rights(source_record: dict) -> list[timdex.Rights]:
+    def get_rights(source: str, source_record: dict) -> list[timdex.Rights]:
         """Get values from source record for TIMDEX rights field."""
         rights = []
 
         if "dct_accessRights_s" in source_record:
+            kind_access_rights = "Access"
+            kind_access_to_files = "Access to files"
             rights.append(
                 timdex.Rights(
-                    description=source_record["dct_accessRights_s"], kind="Access"
+                    description=source_record["dct_accessRights_s"],
+                    kind=kind_access_rights,
                 )
             )
 
+            if source == "gisogm":
+                rights.append(
+                    timdex.Rights(
+                        description="Not owned by MIT", kind=kind_access_to_files
+                    )
+                )
+            elif source == "gismit":
+                if source_record["dct_accessRights_s"] == "Restricted":
+                    rights.append(
+                        timdex.Rights(
+                            description="Requires MIT authentication",
+                            kind=kind_access_to_files,
+                        )
+                    )
+                else:
+                    rights.append(
+                        timdex.Rights(
+                            description="Free/open-access", kind=kind_access_to_files
+                        )
+                    )
+
         rights.extend(
             [
                 timdex.Rights(uri=rights_uri_value)