From a3add22b430402e362caed5877e10261952ade00 Mon Sep 17 00:00:00 2001 From: Theodlz Date: Fri, 1 Sep 2023 17:35:49 -0700 Subject: [PATCH] add fritz spectra import --- readme.md | 4 + ztf-variable-marshal/server.py | 131 ++++++++++++++++++ .../templates/template-search.html | 1 - .../templates/template-source.html | 72 +++++++++- 4 files changed, 202 insertions(+), 6 deletions(-) diff --git a/readme.md b/readme.md index 49ff420..40461d2 100644 --- a/readme.md +++ b/readme.md @@ -79,6 +79,10 @@ Create `secrets.json` with confidential/secret data: "token": null // set to your token } }, + "fritz": { + "url": "https://fritz.science", + "token": null // set to your token, optional + } } } ``` diff --git a/ztf-variable-marshal/server.py b/ztf-variable-marshal/server.py index aa3397a..b30e8ae 100644 --- a/ztf-variable-marshal/server.py +++ b/ztf-variable-marshal/server.py @@ -11,6 +11,7 @@ import aiofiles import aiohttp +from astropy.time import Time import aiohttp_jinja2 import jinja2 import jwt @@ -183,6 +184,30 @@ async def add_master_program(_mongo): print(_err) +# query fritz +def api(endpoint, method="GET", data=None): + import requests + + try: + base_url = config["fritz"]["url"] + token = config["fritz"]["token"] + response = requests.request( + method, + f"{base_url}/api/{endpoint}", + headers={"Authorization": f"token {token}"}, # noqa + json=data, + ) + if response.status_code != 200: + raise Exception(response.text) + else: + q = response.json()["data"] + return q + except Exception: + return None + + +fritz_instruments = api("instrument") + routes = web.RouteTableDef() @@ -2142,6 +2167,14 @@ async def source_get_handler(request): .to_list(length=None) ) + data = api( + f"sources?ra={source['ra']}&dec={source['dec']}&radius={3/3600}&hasSpectrum=true" + ) + if data is not None and len(data.get("sources", [])) > 0: + source["fritz_sources"] = [d["id"] for d in data["sources"]] + else: + source["fritz_sources"] = [] + context = { "logo": config["server"]["logo"], "user": session["user_id"], @@ -3625,6 +3658,104 @@ async def source_post_handler(request): return web.json_response({"message": "success"}, status=200) + elif _r["action"] == "import_fritz_spectra": + # fetch spectra from Fritz for the source in _r.get('source_id', None) + # and ingest them into the source + + fritz_source_id = _r.get("source_id", None) + data = api(f"sources/{fritz_source_id}/spectra", method="GET") + if data is not None and len(data.get("spectra", [])) > 0: + new = False + for fritz_spectrum in data["spectra"]: + try: + telescope = [ + fritz_instrument["telescope"] + for fritz_instrument in fritz_instruments + if fritz_instrument["name"] + == fritz_spectrum["instrument_name"] + ][0]["nickname"] + except Exception: + telescope = "Unknown" + + flux_unit = fritz_spectrum["units"] + if flux_unit in [None, "None", ""]: + flux_unit = fritz_spectrum.get("altdata", {}).get( + "BUNIT", "Unknown" + ) + + spectrum = { + "instrument": fritz_spectrum["instrument_name"], + "telescope": telescope, + "filter": "Unknown", + "wavelength_unit": "A", + "flux_unit": flux_unit, + "data": { + "wavelength": fritz_spectrum["wavelengths"], + "flux": fritz_spectrum["fluxes"], + "fluxerr": fritz_spectrum["errors"], + }, + "mjd": Time( + fritz_spectrum["observed_at"], format="isot" + ).mjd, + } + + # check that there isn't a spectrum with the same mjd, instrument, telescope, filter already + # in the source + existing_spectra = source["spec"] + exists = False + for existing_spectrum in existing_spectra: + if ( + existing_spectrum["mjd"] == spectrum["mjd"] + and existing_spectrum["instrument"] + == spectrum["instrument"] + and existing_spectrum["telescope"] + == spectrum["telescope"] + and existing_spectrum["filter"] == spectrum["filter"] + ): + exists = True + break + + if exists: + continue + + # generate unique _id: + spectrum["_id"] = random_alphanumeric_str(length=24) + + # make history + time_tag = utc_now() + h = { + "note_type": "spec", + "time_tag": time_tag, + "user": user, + "note": f'{spectrum["telescope"]} {spectrum["instrument"]} {spectrum["filter"]}', + } + + await request.app["mongo"].sources.update_one( + {"_id": _id}, + { + "$push": {"spec": spectrum, "history": h}, + "$set": {"last_modified": utc_now()}, + }, + ) + new = True + + if not new: + return web.json_response( + { + "message": f"All spectra from fritz source {fritz_source_id} already imported" + }, + status=200, + ) + + return web.json_response({"message": "success"}, status=200) + else: + return web.json_response( + { + "message": f"failure: no spectra found in fritz for {fritz_source_id}" + }, + status=200, + ) + else: return web.json_response( {"message": "failure: unknown action requested"}, status=200 diff --git a/ztf-variable-marshal/templates/template-search.html b/ztf-variable-marshal/templates/template-search.html index 6998eb1..e5be77f 100644 --- a/ztf-variable-marshal/templates/template-search.html +++ b/ztf-variable-marshal/templates/template-search.html @@ -332,7 +332,6 @@ // display details function detailFormatter(index, row, element) { var html = []; - console.log(row) {#let jdd = new JulianDate();#} {#let date_utc = moment(jdd.julian(parseFloat(row['jd'])).getDate()).utc().format('YYYYMMDD');#} diff --git a/ztf-variable-marshal/templates/template-source.html b/ztf-variable-marshal/templates/template-source.html index dc1452e..853c36e 100644 --- a/ztf-variable-marshal/templates/template-source.html +++ b/ztf-variable-marshal/templates/template-source.html @@ -262,9 +262,6 @@

{% else %} {% set dec_string = source['dec'] | string %} {% endif %} - ZTF Alerts PanSTARRS-1 @@ -327,10 +324,31 @@

... ] } - "wavelen_unit" can be "A", "nm", or "m". + "wavelength_unit" can be "A", "nm", or "m". You can use either mjd or hjd for the time stamp. Other fields above are mandatory, however you can add any custom fields.

+ +
+ Fritz sources (with spectra, within 3 arcsec): + {% if source['fritz_sources'] | length > 0 %} + {% for fritz_source in source['fritz_sources'] %} +
+

+ + {{ fritz_source }} + + + +

+
+ {% endfor %} + {% else %} +

No sources found.

+ {% endif %} +
+

error_y: {type: 'data', array: {{ lc['data'][key]['magerr'] }}, width: 2, - thickness: 0.8, + thickness: 0.4, color: "{{ lc['color'] }}", opacity: 0.5, visible: true}, @@ -1269,6 +1287,50 @@ {% endfor %} }); {% endif %} + + {# import spectra from a fritz source, takes the source id as input #} + function import_fritz_spectra(source_id) { + bootbox.confirm({ + message: "Do you want to import spectra from a fritz source?", + buttons: { + cancel: { + label: ' No' + }, + confirm: { + label: ' Yes' + } + }, + callback: function (result) { + // confirmed? + if (result) { + $.ajax({url: '{{-script_root-}}/sources/{{-source["_id"]-}}', + method: 'POST', + data: JSON.stringify({'action': 'import_fritz_spectra', 'source_id': source_id}), + processData: false, + contentType: 'application/json', + success: function(data) { + if (data['message'] === 'success') { + showFlashingMessage('Info:', 'Successfully imported spectra: ' + data['message'], 'success'); + setTimeout(window.location.href = '{{-script_root-}}/sources/{{ source["_id"] }}', 1000); + } + else if (data['message'] === `All spectra from fritz source ${source_id} already imported`) { + showFlashingMessage('Info:', `All spectra from fritz source ${source_id} already imported`, 'info'); + } + else { + showFlashingMessage('Info:', 'Failed to import spectra: ' + data['message'], 'danger'); + } + }, + error: function(data) { + showFlashingMessage('Info:', 'Failed to import spectra', 'danger'); + } + }); + } + } + }); + } + + {# import light curves from a fritz source, takes the source id as input #} + {% endblock %}