Adding instr attrs and updating product attrs (#71)

* adding instr attrs and updating product attrs

* pre-commit fixes

* registering a.Instr instead of a.soar.Instr

* adding checks whether soar can query the provided attrs

* adding functionality to only query SOAR

* fixing can_handle_query to check for instr

* fixing cdf descriptors and adding inst to one file

* removing file

* updating values

* adding extra tests

* adding comment

* check for soar provider

* adding test for provider

* adding test for provider

* comments from review

* updating readme

* updating readme

README.rst

@@ -86,6 +86,14 @@ Contributions for new features and bug fixes are welcome.
 Changelog
 =========
 
+1.6
+---
+- Registered a list of instruments available from the SOAR, with the ``a.Instrument`` attribute.
+- Registered the SOAR in the ``a.Provider`` attribute, meaning that a user can specifiy to the Fido search to only query the SOAR by
+use of ``a.Provider.soar``.
+- The ``_can_handle_query`` function within the SOARClient now checks to make sure if the SOAR supplies the queried data which
+fixes a bug which searched the SOAR for any data (e.g. AIA data).
+
 1.5
 ---
 - Registered a list of valid data product identifiers with the ``a.soar.Product`` attribute. To see these use ``print(a.soar.Product)``.

sunpy_soar/attrs.py

@@ -118,3 +118,8 @@ def _(wlk, attr, params):
 @walker.add_applier(Product, Identifier)
 def _(wlk, attr, params):
     params.append(f"descriptor='{attr.value}'")
+
+
+@walker.add_applier(a.Provider)
+def _(wlk, attr, params):
+    params.append(f"provider='{attr.value}'")

sunpy_soar/client.py

@@ -26,6 +26,8 @@ def search(self, *query, **kwargs):
 
         results = []
         for query_parameters in queries:
+            if "provider='SOAR'" in query_parameters:
+                query_parameters.remove("provider='SOAR'")
             results.append(self._do_search(query_parameters))
         table = astropy.table.vstack(results)
         qrt = QueryResponseTable(table, client=self)
@@ -105,7 +107,7 @@ def _do_search(query):
                                      'End time': info['end_time'],
                                      'Data item ID': info['data_item_id'],
                                      'Filename': info['filename'],
-                                     'Filesize': info['filesize']
+                                     'Filesize': info['filesize'],
                                      })
 
     def fetch(self, query_results, *, path, downloader, **kwargs):
@@ -144,15 +146,26 @@ def fetch(self, query_results, *, path, downloader, **kwargs):
     def _can_handle_query(cls, *query):
         """
         Check if this client can handle a given Fido query.
+        Checks to see if a SOAR instrument or product is provided in the query.
 
         Returns
         -------
         bool
             True if this client can handle the given query.
         """
         required = {a.Time}
-        optional = {a.Instrument, a.Level, Product, Identifier}
-        return cls.check_attr_types_in_query(query, required, optional)
+        optional = {a.Instrument, a.Level, a.Provider, Product, Identifier}
+        if not cls.check_attr_types_in_query(query, required, optional):
+            return False
+        # check to make sure the instrument attr passed is one provided by the SOAR.
+        # also check to make sure that the provider passed is the SOAR for which this client can handle.
+        instr = [i[0].lower() for i in cls.register_values()[a.Instrument]]
+        for x in query:
+            if isinstance(x, a.Instrument) and str(x.value).lower() not in instr:
+                return False
+            if isinstance(x, a.Provider) and str(x.value).lower() != "soar":
+                return False
+        return True
 
     @classmethod
     def _attrs_module(cls):
@@ -171,4 +184,11 @@ def load_dataset_values():
 
         # Convert from dict to list of tuples
         all_datasets = [(id, desc) for id, desc in all_datasets.items()]
-        return {Product: all_datasets}
+
+        instr_path = pathlib.Path(__file__).parent / 'data' / 'instrument_attrs.json'
+        with open(instr_path, 'r') as instr_attrs_file:
+            all_instr = json.load(instr_attrs_file)
+
+        all_instr = [(id, desc) for id, desc in all_instr.items()]
+
+        return {Product: all_datasets, a.Instrument: all_instr, a.Provider:  [('SOAR', 'Solar Orbiter Archive.')]}

sunpy_soar/data/attrs.json

@@ -66,23 +66,51 @@
   "EUI-HRIEUVNON-IMAGE": "",
   "EUI-HRIEUVZER-IMAGE": "",
   "EUI-HRILYA1216-IMAGE": "",
-  "LFR": "Solar Orbiter Radio/Plasma Wave, LFR L2 magnetic parameters",
-  "MAG": "Solar Orbiter Magnetometer Level 2 Normal Mode Data in VSO coordinates",
-  "PLASMA PLASMA FREQUENCY VALUE FROM THE PLASMA PEAK POSITION ": "Solar Orbiter Radio/Plasma Wave, data from plasma peak tracking L3",
+  "MAG": "Solar Orbiter Level 2 Low Latency Magnetometer Data",
+  "MAG-IBS": "Solar Orbiter Magnetometer L1 Data",
+  "MAG-OBS": "Solar Orbiter Magnetometer L1 Data",
+  "MAG-RTN": "Solar Orbiter Magnetometer L2 Data",
+  "MAG-RTN DERIVED FROM LL DATA": "Solar Orbiter Magnetometer L2 Data derived from LL data",
+  "MAG-RTN-BURST": "Solar Orbiter Magnetometer Level 2 Burst Mode Data in RTN coordinates",
+  "MAG-RTN-NORMAL": "Solar Orbiter Magnetometer Level 2 Normal Mode Data in RTN coordinates",
+  "MAG-SRF": "Solar Orbiter Magnetometer L2 Data",
+  "MAG-SRF DERIVED FROM LL DATA": "Solar Orbiter Magnetometer L2 Data derived from LL data",
+  "MAG-SRF-BURST": "Solar Orbiter Magnetometer Level 2 Burst Mode Data in SRF coordinates",
+  "MAG-SRF-NORMAL": "Solar Orbiter Magnetometer Level 2 Normal Mode Data in SRF coordinates",
+  "MAG-VSO": "Solar Orbiter Magnetometer L2 Data",
+  "MAG-VSO-BURST": "Solar Orbiter Magnetometer Level 2 Burst Mode Data in VSO coordinates",
+  "MAG-VSO-NORMAL": "Solar Orbiter Magnetometer Level 2 Normal Mode Data in VSO coordinates",
+  "METIS-UV-IMAGE": "",
+  "METIS-VL-IMAGE": "",
+  "METIS-VL-PB": "",
+  "METIS-VL-POL-ANGLE": "",
+  "METIS-VL-STOKES": "",
+  "METIS-VL-TB": "",
+  "PHI-HRT-BAZI": "",
+  "PHI-HRT-BINC": "",
+  "PHI-HRT-BLOS": "",
+  "PHI-HRT-BMAG": "",
+  "PHI-HRT-ICNT": "",
+  "PHI-HRT-STOKES": "",
+  "PHI-HRT-VLOS": "",
+  "RPW-BIA-DENSITY-10-SECONDS-CDAG": "Solar Orbiter Radio/Plasma Wave, LFR L3 plasma density derived from the spacecraft potential, downsampled",
+  "RPW-BIA-DENSITY-CDAG": "Solar Orbiter Radio/Plasma Wave, LFR L3 plasma density derived from the spacecraft potential",
+  "RPW-BIA-EFIELD-10-SECONDS-CDAG": "Solar Orbiter Radio/Plasma Wave, LFR L3 electric field vector, downsampled",
+  "RPW-BIA-EFIELD-CDAG": "Solar Orbiter Radio/Plasma Wave, LFR L3 electric field vector",
+  "RPW-BIA-SCPOT-10-SECONDS-CDAG": "Solar Orbiter Radio/Plasma Wave, LFR L3 spacecraft potential, downsampled",
+  "RPW-BIA-SCPOT-CDAG": "Solar Orbiter Radio/Plasma Wave, LFR L3 spacecraft potential",
   "RPW-HFR-SURV": "Solar Orbiter Radio/Plasma Wave, HFR L2 parameters",
-  "RPW-LFR-BIA-DENSITY": "Solar Orbiter Radio/Plasma Wave, LFR L3 plasma density derived from the spacecraft potential",
-  "RPW-LFR-BIA-DENSITY-10-SECONDS": "Solar Orbiter Radio/Plasma Wave, LFR L3 plasma density derived from the spacecraft potential, downsampled",
-  "RPW-LFR-BIA-EFIELD": "Solar Orbiter Radio/Plasma Wave, LFR L3 electric field vector",
-  "RPW-LFR-BIA-EFIELD-10-SECONDS": "Solar Orbiter Radio/Plasma Wave, LFR L3 electric field vector, downsampled",
-  "RPW-LFR-BIA-SCPOT": "Solar Orbiter Radio/Plasma Wave, LFR L3 spacecraft potential",
-  "RPW-LFR-BIA-SCPOT-10-SECONDS": "Solar Orbiter Radio/Plasma Wave, LFR L3 spacecraft potential, downsampled",
   "RPW-LFR-SURV-ASM": "Solar Orbiter Radio/Plasma Wave, LFR L2 parameters",
   "RPW-LFR-SURV-BP1": "Solar Orbiter Radio/Plasma Wave, LFR L2 parameters",
   "RPW-LFR-SURV-BP2": "Solar Orbiter Radio/Plasma Wave, LFR L2 parameters",
   "RPW-LFR-SURV-CWF": "Solar Orbiter Radio/Plasma Wave, LFR L1 parameters",
+  "RPW-LFR-SURV-CWF-B": "Solar Orbiter Radio/Plasma Wave, LFR L2 magnetic parameters",
   "RPW-LFR-SURV-CWF-E": "Solar Orbiter Radio/Plasma Wave, LFR L2 electric parameters",
+  "RPW-LFR-SURV-CWF-E-CDAG": "Solar Orbiter Radio/Plasma Wave, LFR L2 electric parameters",
   "RPW-LFR-SURV-SWF": "Solar Orbiter Radio/Plasma Wave, LFR L1 parameters",
+  "RPW-LFR-SURV-SWF-B": "Solar Orbiter Radio/Plasma Wave, LFR L2 magnetic parameters",
   "RPW-LFR-SURV-SWF-E": "Solar Orbiter Radio/Plasma Wave, LFR L2 electric parameters",
+  "RPW-LFR-SURV-SWF-E-CDAG": "Solar Orbiter Radio/Plasma Wave, LFR L2 electric parameters",
   "RPW-SBM1": "Solar Orbiter Radio/Plasma Wave, LL02 parameters",
   "RPW-SBM2": "Solar Orbiter Radio/Plasma Wave, LL02 parameters",
   "RPW-TDS-SURV-HIST1D": "Solar Orbiter Radio/Plasma Wave, TDS L2 parameters",
@@ -92,15 +120,26 @@
   "RPW-TDS-SURV-RSWF-B": "Solar Orbiter Radio/Plasma Wave, TDS L2 magnetic parameters",
   "RPW-TDS-SURV-RSWF-E": "Solar Orbiter Radio/Plasma Wave, TDS L2 waveform snapshots",
   "RPW-TDS-SURV-STAT": "Solar Orbiter Radio/Plasma Wave, TDS L2R parameters",
-  "RPW-TDS-SURV-TSWF": "Solar Orbiter Radio/Plasma Wave, TDS L2 waveform snapshots",
+  "RPW-TDS-SURV-TSWF": "Solar Orbiter Radio/Plasma Wave, TDS L1 parameters",
   "RPW-TDS-SURV-TSWF-B": "Solar Orbiter Radio/Plasma Wave, TDS L2 magnetic parameters",
+  "RPW-TDS-SURV-TSWF-E": "Solar Orbiter Radio/Plasma Wave, TDS L2 waveform snapshots",
   "RPW-TNR": "Solar Orbiter Radio/Plasma Wave, LL02 parameters",
+  "RPW-TNR-FP": "Solar Orbiter Radio/Plasma Wave, data from plasma peak tracking L3",
   "RPW-TNR-SURV": "Solar Orbiter Radio/Plasma Wave, TNR L2 parameters",
+  "SENSORRATES": "solo_L1_swa-his-sensor_rates",
   "SOLOHI-1FT": "",
+  "SOLOHI-1UT": "",
+  "SOLOHI-1VT": "",
   "SOLOHI-2FT": "",
+  "SOLOHI-2US": "",
+  "SOLOHI-2UT": "",
   "SOLOHI-3FG": "",
+  "SOLOHI-3FT": "",
+  "SOLOHI-3UT": "",
   "SOLOHI-4-001800001780": "",
   "SOLOHI-4FG": "",
+  "SOLOHI-4FT": "",
+  "SOLOHI-4UT": "",
   "SOLOHI-MAP1-3": "",
   "SOLOHI-MAP2-0": "",
   "SOLOHI-MAP3-0": "",
@@ -112,30 +151,35 @@
   "SPICE-W-EXP": "",
   "SPICE-W-RAS": "",
   "SPICE-W-SIT": "",
-  "SWA-EAS-NM3D-PSD": "SWA-EAS1 Nominal Mode 3D psd data",
   "SWA-EAS-PAD-DEF": "SWA-EAS 2D Burst Mode data",
   "SWA-EAS-PAD-DNF": "SWA-EAS 2D Burst Mode data",
   "SWA-EAS-PAD-PSD": "SWA-EAS 2D Burst Mode data",
   "SWA-EAS-PADC": "SWA-EAS Pitch Angle Counts data",
   "SWA-EAS-PARTMOMS": "SWA-EAS Partial Moments data",
   "SWA-EAS-SS": "SWA-EAS Low Latency LL02 Data",
+  "SWA-EAS1-NM3D": "SWA-EAS1 Nominal Mode 3D data",
   "SWA-EAS1-NM3D-DEF": "SWA-EAS1 Nominal Mode 3D def data",
   "SWA-EAS1-NM3D-DNF": "SWA-EAS1 Nominal Mode 3D dnf data",
-  "SWA-EAS1-NMC": "SWA-EAS1 Nominal Mode 3D data",
+  "SWA-EAS1-NM3D-PSD": "SWA-EAS1 Nominal Mode 3D psd data",
   "SWA-EAS1-SS-DEF": "SWA-EAS1 Single Strahl def data",
   "SWA-EAS1-SS-DNF": "SWA-EAS1 Single Strahl dnf data",
   "SWA-EAS1-SS-PSD": "SWA-EAS1 Single Strahl psd data",
   "SWA-EAS1-SSC": "SWA-EAS1 Single Strahl data",
+  "SWA-EAS2-NM3D": "SWA-EAS2 Nominal Mode 3D data",
   "SWA-EAS2-NM3D-DEF": "SWA-EAS2 Nominal Mode 3D def data",
   "SWA-EAS2-NM3D-DNF": "SWA-EAS2 Nominal Mode 3D dnf data",
   "SWA-EAS2-NM3D-PSD": "SWA-EAS2 Nominal Mode 3D psd data",
-  "SWA-EAS2-NMC": "SWA-EAS2 Nominal Mode 3D data",
   "SWA-EAS2-SS-DEF": "SWA-EAS2 Single Strahl def data",
   "SWA-EAS2-SS-DNF": "SWA-EAS2 Single Strahl dnf data",
   "SWA-EAS2-SS-PSD": "SWA-EAS2 Single Strahl psd data",
   "SWA-EAS2-SSC": "SWA-EAS2 Single Strahl data",
-  "SWA-HIS": "solo_L1_swa-his-sensor_rates",
+  "SWA-HIS-COMP-10MIN": "Solar Orbiter, Level 3 Data, Solar Wind Analyser, Heavy Ion Sensor Composition 10 Minute Resolution",
+  "SWA-HIS-PHA": "solo_L1_swa-his-pha",
   "SWA-HIS-RAT": "SWA-HIS LL02 Ratios and Spectra",
-  "SWA-PAS": "Solar Orbiter Proton Analyser Sensor L2 data",
-  "SWA-PAS-MOM": "SWA-PAS Quick Look Moments"
+  "SWA-PAS-3D": "Solar Orbiter Proton Analyser Sensor L1 data",
+  "SWA-PAS-CAL": "Solar Orbiter Proton Analyser Sensor L1 Inflight calibration",
+  "SWA-PAS-EFLUX": "Solar Orbiter Proton Analyser Sensor L2 data",
+  "SWA-PAS-GRND-MOM": "Solar Orbiter Proton Analyser Sensor L2 data",
+  "SWA-PAS-MOM": "Solar Orbiter Proton Analyser Sensor L1 Onboard Moments",
+  "SWA-PAS-VDF": "Solar Orbiter Proton Analyser Sensor L2 data"
 }

sunpy_soar/data/instrument_attrs.json

@@ -0,0 +1,12 @@
+{
+  "EPD": "Energetic Particle Detector",
+  "EUI": "Extreme UV Imager",
+  "MAG": "Magnetometer",
+  "METIS": "Metis: Visible light and ultraviolet coronagraph",
+  "PHI": "Polarimetric and Helioseismic Imager",
+  "RPW": "Radio and Plasma Wave instrument",
+  "SOLOHI": "Solar Orbiter Heliospheric Imager",
+  "SPICE": "SPectral Investigation of the Coronal Environment",
+  "STIX": "Spectrometer Telescope for Imaging X-rays",
+  "SWA": "Solar Wind Analyser"
+}

sunpy_soar/tests/test_sunpy_soar.py

@@ -102,3 +102,41 @@ def test_registered_attrs():
     attr_str = str(a.soar.Product)
     # Check that at least one attr value is registered
     assert 'epd_ept_asun_burst_ele_close' in attr_str
+
+
+def test_registered_instr_attrs():
+    # Check if the Solo instruments are registered in a.Instrument
+    instr_attr = a.Instrument
+    assert "SOAR" in instr_attr._attr_registry[instr_attr].client
+    assert "stix" in instr_attr._attr_registry[instr_attr].name
+
+
+def test_when_soar_provider_passed():
+    # Tests when a.Provider.soar is passed that only SOARClient results are returned
+    id = a.Instrument('EUI')
+    time = a.Time('2022-04-01 00:00', '2022-04-01 01:00')
+    provider = a.Provider.soar
+    res = Fido.search(time & id & provider)
+    assert len(res) == 1
+    assert "soar" in res.keys()
+
+
+def test_when_sdac_provider_passed():
+    # tests that only VSO EUI results are returned when explicitly setting the provider to SDAC
+    id = a.Instrument('EUI')
+    time = a.Time('2022-04-01 00:00', '2022-04-01 01:00')
+    provider = a.Provider.sdac
+    res = Fido.search(time & id & provider)
+    assert len(res) == 1
+    assert "vso" in res.keys()
+
+
+def test_when_wrong_provider_passed():
+    # Tests that no results are returned when a provider is passed which does not provide EUI data.
+    # This is different from the above test because the SDAC and the SOAR both provide EUI data while
+    # NOAA has no overlap with the data provided by the SOAR.
+    id = a.Instrument('EUI')
+    time = a.Time('2022-04-01 00:00', '2022-04-01 01:00')
+    provider = a.Provider.noaa
+    res = Fido.search(time & id & provider)
+    assert len(res) == 0

tools/update_data.py

@@ -13,7 +13,7 @@ def get_cdf_descriptors():
     res = job.get_results()
     descriptors = {}
     for row in res:
-        desc = row['cdf_descriptor'].split('>')[0].upper()
+        desc = row['logical_source'].split('_')[-1].upper()
         descriptors[desc] = row['logical_source_description']
     return descriptors
 
@@ -38,8 +38,30 @@ def get_all_descriptors():
     return desc
 
 
+def get_all_instruments():
+    # Get the unique instrument names
+    SOAR = TapPlus(url="http://soar.esac.esa.int/soar-sl-tap/tap")
+    job = SOAR.launch_job('select * from soar.instrument')
+    res = job.get_results()
+
+    instruments = ['EPD', 'EUI', 'MAG', 'METIS', 'PHI', 'RPW',
+                   'SOLOHI', 'SPICE', 'STIX', 'SWA']
+    instr_desc = {}
+    for r in res:
+        if r["name"] not in instruments:
+            pass
+        else:
+            instr_desc[r["name"]] = r["long_name"]
+    return instr_desc
+
+
 if __name__ == '__main__':
     attr_file = pathlib.Path(__file__).parent.parent / 'sunpy_soar' / 'data' / 'attrs.json'
     descriptors = get_all_descriptors()
     with open(attr_file, 'w') as attrs_file:
         json.dump(dict(sorted(descriptors.items())), attrs_file, indent=2)
+
+    instr_file = pathlib.Path(__file__).parent.parent / 'sunpy_soar' / 'data' / 'instrument_attrs.json'
+    instr_descriptors = get_all_instruments()
+    with open(instr_file, 'w') as instrs_file:
+        json.dump(dict(sorted(instr_descriptors.items())), instrs_file, indent=2)