From 3e746050802830dbd299cf7e5199341797889bca Mon Sep 17 00:00:00 2001 From: Alex Leith Date: Thu, 15 Oct 2020 10:24:16 +1100 Subject: [PATCH] Add tests for dc_tools --- ..._088080_2020-05-25_final.odc-metadata.yaml | 138 ++++++++++++++ ...1-0_088080_2020-05-25_final.stac-item.json | 45 +++++ .../tests/data/sentinel-2-nrt_2020_08_21.json | 12 ++ apps/dc_tools/tests/test_sqs_to_dc.py | 169 ++++++++++++++++++ 4 files changed, 364 insertions(+) create mode 100644 apps/dc_tools/tests/data/ga_ls8c_ard_3-1-0_088080_2020-05-25_final.odc-metadata.yaml create mode 100644 apps/dc_tools/tests/data/ga_ls8c_ard_3-1-0_088080_2020-05-25_final.stac-item.json create mode 100644 apps/dc_tools/tests/data/sentinel-2-nrt_2020_08_21.json create mode 100644 apps/dc_tools/tests/test_sqs_to_dc.py diff --git a/apps/dc_tools/tests/data/ga_ls8c_ard_3-1-0_088080_2020-05-25_final.odc-metadata.yaml b/apps/dc_tools/tests/data/ga_ls8c_ard_3-1-0_088080_2020-05-25_final.odc-metadata.yaml new file mode 100644 index 000000000..ca01a287e --- /dev/null +++ b/apps/dc_tools/tests/data/ga_ls8c_ard_3-1-0_088080_2020-05-25_final.odc-metadata.yaml @@ -0,0 +1,138 @@ +--- +# Dataset +$schema: https://schemas.opendatacube.org/dataset +id: 2aa69fcf-aa55-4747-9d95-3652f9fe79b0 + +label: ga_ls8c_ard_3-1-0_088080_2020-05-25_final +product: + name: ga_ls8c_ard_3 + href: https://collections.dea.ga.gov.au/product/ga_ls8c_ard_3 + +crs: epsg:32656 +geometry: + type: Polygon + coordinates: [[[572743.2722121085, -3077151.309749513], [572745.0, -3077145.0], + [572969.4472838908, -3077198.798209872], [573250.996461604, -3077257.9131992455], + [758606.0005942872, -3121687.9141904702], [759791.1380343756, -3121972.947862498], + [759832.5, -3122002.5], [714367.0886436494, -3312160.9887643186], [714322.5, + -3312262.5], [714228.5968569319, -3312239.9605382206], [714225.0, -3312255.0], + [527767.9988114258, -3267524.1716190595], [527167.7239312489, -3267374.1042750045], + [527095.752540109, -3267322.682433342], [527032.5, -3267307.5], [528337.9117568319, + -3261784.0095614507], [541042.9139480147, -3208729.0004160986], [553732.9148397002, + -3155853.9967016927], [572107.9166454426, -3079413.9891924215], [572647.9478624978, + -3077178.8619656246], [572677.5, -3077137.5], [572743.2722121085, -3077151.309749513]]] +grids: + default: + shape: [7841, 7781] + transform: [30.0, 0.0, 526785.0, 0.0, -30.0, -3077085.0, 0.0, 0.0, 1.0] + g15m: + shape: [15681, 15561] + transform: [15.0, 0.0, 526792.5, 0.0, -15.0, -3077092.5, 0.0, 0.0, 1.0] + +properties: + datetime: 2020-05-25 23:35:47.745731Z + dea:dataset_maturity: final + dtr:end_datetime: 2020-05-25 23:36:02.557719Z + dtr:start_datetime: 2020-05-25 23:35:32.815426Z + eo:cloud_cover: 42.42102985910952 + eo:gsd: 15.0 # Ground sample distance (m) + eo:instrument: OLI_TIRS + eo:platform: landsat-8 + eo:sun_azimuth: 34.22994171 + eo:sun_elevation: 31.7895917 + fmask:clear: 1.0427704509544131 + fmask:cloud: 42.42102985910952 + fmask:cloud_shadow: 2.919727915379625 + fmask:snow: 0.00034972190505407335 + fmask:water: 53.61612205265139 + gqa:abs_iterative_mean_x: 0.17 + gqa:abs_iterative_mean_xy: 0.24 + gqa:abs_iterative_mean_y: 0.16 + gqa:abs_x: 0.28 + gqa:abs_xy: 0.4 + gqa:abs_y: 0.28 + gqa:cep90: 0.4 + gqa:iterative_mean_x: 0.03 + gqa:iterative_mean_xy: 0.12 + gqa:iterative_mean_y: 0.12 + gqa:iterative_stddev_x: 0.21 + gqa:iterative_stddev_xy: 0.26 + gqa:iterative_stddev_y: 0.15 + gqa:mean_x: -0.08 + gqa:mean_xy: 0.15 + gqa:mean_y: 0.12 + gqa:stddev_x: 0.42 + gqa:stddev_xy: 0.58 + gqa:stddev_y: 0.4 + landsat:collection_category: T1 + landsat:collection_number: 1 + landsat:landsat_product_id: LC08_L1TP_088080_20200525_20200608_01_T1 + landsat:landsat_scene_id: LC80880802020146LGN00 + landsat:wrs_path: 88 + landsat:wrs_row: 80 + odc:dataset_version: 3.1.0 + odc:file_format: GeoTIFF + odc:processing_datetime: 2020-06-18 07:07:06.303867Z + odc:producer: ga.gov.au + odc:product_family: ard + odc:region_code: '088080' + +measurements: + nbart_blue: + path: ga_ls8c_nbart_3-1-0_088080_2020-05-25_final_band02.tif + nbart_coastal_aerosol: + path: ga_ls8c_nbart_3-1-0_088080_2020-05-25_final_band01.tif + nbart_green: + path: ga_ls8c_nbart_3-1-0_088080_2020-05-25_final_band03.tif + nbart_nir: + path: ga_ls8c_nbart_3-1-0_088080_2020-05-25_final_band05.tif + nbart_panchromatic: + path: ga_ls8c_nbart_3-1-0_088080_2020-05-25_final_band08.tif + grid: g15m + nbart_red: + path: ga_ls8c_nbart_3-1-0_088080_2020-05-25_final_band04.tif + nbart_swir_1: + path: ga_ls8c_nbart_3-1-0_088080_2020-05-25_final_band06.tif + nbart_swir_2: + path: ga_ls8c_nbart_3-1-0_088080_2020-05-25_final_band07.tif + oa_azimuthal_exiting: + path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_azimuthal-exiting.tif + oa_azimuthal_incident: + path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_azimuthal-incident.tif + oa_combined_terrain_shadow: + path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_combined-terrain-shadow.tif + oa_exiting_angle: + path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_exiting-angle.tif + oa_fmask: + path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_fmask.tif + oa_incident_angle: + path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_incident-angle.tif + oa_nbart_contiguity: + path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_nbart-contiguity.tif + oa_relative_azimuth: + path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_relative-azimuth.tif + oa_relative_slope: + path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_relative-slope.tif + oa_satellite_azimuth: + path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_satellite-azimuth.tif + oa_satellite_view: + path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_satellite-view.tif + oa_solar_azimuth: + path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_solar-azimuth.tif + oa_solar_zenith: + path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_solar-zenith.tif + oa_time_delta: + path: ga_ls8c_oa_3-1-0_088080_2020-05-25_final_time-delta.tif + +accessories: + thumbnail:nbart: + path: ga_ls8c_nbart_3-1-0_088080_2020-05-25_final_thumbnail.jpg + checksum:sha1: + path: ga_ls8c_ard_3-1-0_088080_2020-05-25_final.sha1 + metadata:processor: + path: ga_ls8c_ard_3-1-0_088080_2020-05-25_final.proc-info.yaml + +lineage: + level1: + - b5f234fe-bba8-5483-9bc0-250360d429cf +... diff --git a/apps/dc_tools/tests/data/ga_ls8c_ard_3-1-0_088080_2020-05-25_final.stac-item.json b/apps/dc_tools/tests/data/ga_ls8c_ard_3-1-0_088080_2020-05-25_final.stac-item.json new file mode 100644 index 000000000..3ce25ccce --- /dev/null +++ b/apps/dc_tools/tests/data/ga_ls8c_ard_3-1-0_088080_2020-05-25_final.stac-item.json @@ -0,0 +1,45 @@ +{ + "Type": "Notification", + "MessageId": "741ffdef-6b05-5766-b117-aa4182013584", + "TopicArn": "arn:aws:sns:ap-southeast-2:451924316694:landsat-collection-3-dev-topic", + "Message": "{\n \"stac_version\": \"1.0.0-beta.2\",\n \"stac_extensions\": [\n \"eo\",\n \"projection\",\n \"view\"\n ],\n \"type\": \"Feature\",\n \"id\": \"2aa69fcf-aa55-4747-9d95-3652f9fe79b0\",\n \"bbox\": [\n 153.27898977305418,\n -29.92243448617234,\n 155.6469914729725,\n -27.816894407110592\n ],\n \"geometry\": {\n \"type\": \"Polygon\",\n \"coordinates\": [\n [\n [\n 153.7385878898377,\n -27.81701549471511\n ],\n [\n 153.7386050460989,\n -27.81695844414134\n ],\n [\n 153.7408870777895,\n -27.81743186138652\n ],\n [\n 153.74374919575374,\n -27.81795011248702\n ],\n [\n 155.634437974109,\n -28.195665728285302\n ],\n [\n 155.64656390412654,\n -28.198003116605665\n ],\n [\n 155.6469914729725,\n -28.198261451847454\n ],\n [\n 155.2205985724982,\n -29.92151128958127\n ],\n [\n 155.2201572971803,\n -29.92243448617234\n ],\n [\n 155.21918065761497,\n -29.92224759982275\n ],\n [\n 155.21914643092472,\n -29.92238385153146\n ],\n [\n 153.2865859478465,\n -29.537062581029243\n ],\n [\n 153.28038700032272,\n -29.535721417545275\n ],\n [\n 153.2796429434371,\n -29.535258893290152\n ],\n [\n 153.27898977305418,\n -29.535123241623708\n ],\n [\n 153.29231891428003,\n -29.485243884787124\n ],\n [\n 153.4214079550725,\n -29.006049156688324\n ],\n [\n 153.54919382814404,\n -28.52833499498428\n ],\n [\n 153.73227439530447,\n -27.837474564334862\n ],\n [\n 153.7376217743478,\n -27.817269375728678\n ],\n [\n 153.73791928400988,\n -27.816894407110592\n ],\n [\n 153.7385878898377,\n -27.81701549471511\n ]\n ]\n ]\n },\n \"properties\": {\n \"datetime\": \"2020-05-25T23:35:47.745731Z\",\n \"dea:dataset_maturity\": \"final\",\n \"end_datetime\": \"2020-05-25T23:36:02.557719Z\",\n \"start_datetime\": \"2020-05-25T23:35:32.815426Z\",\n \"eo:cloud_cover\": 42.42102985910952,\n \"gsd\": 15.0,\n \"instruments\": [\n \"OLI_TIRS\"\n ],\n \"platform\": \"landsat-8\",\n \"view:sun_azimuth\": 34.22994171,\n \"view:sun_elevation\": 31.7895917,\n \"fmask:clear\": 1.0427704509544131,\n \"fmask:cloud\": 42.42102985910952,\n \"fmask:cloud_shadow\": 2.919727915379625,\n \"fmask:snow\": 0.00034972190505407335,\n \"fmask:water\": 53.61612205265139,\n \"gqa:abs_iterative_mean_x\": 0.17,\n \"gqa:abs_iterative_mean_xy\": 0.24,\n \"gqa:abs_iterative_mean_y\": 0.16,\n \"gqa:abs_x\": 0.28,\n \"gqa:abs_xy\": 0.4,\n \"gqa:abs_y\": 0.28,\n \"gqa:cep90\": 0.4,\n \"gqa:iterative_mean_x\": 0.03,\n \"gqa:iterative_mean_xy\": 0.12,\n \"gqa:iterative_mean_y\": 0.12,\n \"gqa:iterative_stddev_x\": 0.21,\n \"gqa:iterative_stddev_xy\": 0.26,\n \"gqa:iterative_stddev_y\": 0.15,\n \"gqa:mean_x\": -0.08,\n \"gqa:mean_xy\": 0.15,\n \"gqa:mean_y\": 0.12,\n \"gqa:stddev_x\": 0.42,\n \"gqa:stddev_xy\": 0.58,\n \"gqa:stddev_y\": 0.4,\n \"landsat:collection_category\": \"T1\",\n \"landsat:collection_number\": 1,\n \"landsat:landsat_product_id\": \"LC08_L1TP_088080_20200525_20200608_01_T1\",\n \"landsat:landsat_scene_id\": \"LC80880802020146LGN00\",\n \"landsat:wrs_path\": 88,\n \"landsat:wrs_row\": 80,\n \"odc:dataset_version\": \"3.1.0\",\n \"odc:file_format\": \"GeoTIFF\",\n \"odc:processing_datetime\": \"2020-06-18T07:07:06.303867Z\",\n \"odc:producer\": \"ga.gov.au\",\n \"odc:product_family\": \"ard\",\n \"odc:region_code\": \"088080\",\n \"odc:product\": \"ga_ls8c_ard_3\",\n \"proj:epsg\": 32656,\n \"proj:shape\": [\n 7841,\n 7781\n ],\n \"proj:transform\": [\n 30.0,\n 0.0,\n 526785.0,\n 0.0,\n -30.0,\n -3077085.0,\n 0.0,\n 0.0,\n 1.0\n ]\n },\n \"assets\": {\n \"nbart_blue\": {\n \"eo:bands\": [\n {\n \"name\": \"nbart_blue\"\n }\n ],\n \"type\": \"image/tiff; application=geotiff\",\n \"roles\": [\n \"data\"\n ],\n \"href\": \"http://dea-public-data-dev.s3-ap-southeast-2.amazonaws.com/analysis-ready-data/ga_ls8c_ard_3/088/080/2020/05/25/ga_ls8c_nbart_3-1-0_088080_2020-05-25_final_band02.tif\",\n \"proj:shape\": [\n 7841,\n 7781\n ],\n \"proj:transform\": [\n 30.0,\n 0.0,\n 526785.0,\n 0.0,\n -30.0,\n -3077085.0,\n 0.0,\n 0.0,\n 1.0\n ]\n },\n \"nbart_coastal_aerosol\": {\n \"eo:bands\": [\n {\n \"name\": \"nbart_coastal_aerosol\"\n }\n ],\n \"type\": \"image/tiff; application=geotiff\",\n \"roles\": [\n \"data\"\n ],\n \"href\": \"http://dea-public-data-dev.s3-ap-southeast-2.amazonaws.com/analysis-ready-data/ga_ls8c_ard_3/088/080/2020/05/25/ga_ls8c_nbart_3-1-0_088080_2020-05-25_final_band01.tif\",\n \"proj:shape\": [\n 7841,\n 7781\n ],\n \"proj:transform\": [\n 30.0,\n 0.0,\n 526785.0,\n 0.0,\n -30.0,\n -3077085.0,\n 0.0,\n 0.0,\n 1.0\n ]\n },\n \"nbart_green\": {\n \"eo:bands\": [\n {\n \"name\": \"nbart_green\"\n }\n ],\n \"type\": \"image/tiff; application=geotiff\",\n \"roles\": [\n \"data\"\n ],\n \"href\": \"http://dea-public-data-dev.s3-ap-southeast-2.amazonaws.com/analysis-ready-data/ga_ls8c_ard_3/088/080/2020/05/25/ga_ls8c_nbart_3-1-0_088080_2020-05-25_final_band03.tif\",\n \"proj:shape\": [\n 7841,\n 7781\n ],\n \"proj:transform\": [\n 30.0,\n 0.0,\n 526785.0,\n 0.0,\n -30.0,\n -3077085.0,\n 0.0,\n 0.0,\n 1.0\n ]\n },\n \"nbart_nir\": {\n \"eo:bands\": [\n {\n \"name\": \"nbart_nir\"\n }\n ],\n \"type\": \"image/tiff; application=geotiff\",\n \"roles\": [\n \"data\"\n ],\n \"href\": \"http://dea-public-data-dev.s3-ap-southeast-2.amazonaws.com/analysis-ready-data/ga_ls8c_ard_3/088/080/2020/05/25/ga_ls8c_nbart_3-1-0_088080_2020-05-25_final_band05.tif\",\n \"proj:shape\": [\n 7841,\n 7781\n ],\n \"proj:transform\": [\n 30.0,\n 0.0,\n 526785.0,\n 0.0,\n -30.0,\n -3077085.0,\n 0.0,\n 0.0,\n 1.0\n ]\n },\n \"nbart_panchromatic\": {\n \"eo:bands\": [\n {\n \"name\": \"nbart_panchromatic\"\n }\n ],\n \"type\": \"image/tiff; application=geotiff\",\n \"roles\": [\n \"data\"\n ],\n \"href\": \"http://dea-public-data-dev.s3-ap-southeast-2.amazonaws.com/analysis-ready-data/ga_ls8c_ard_3/088/080/2020/05/25/ga_ls8c_nbart_3-1-0_088080_2020-05-25_final_band08.tif\",\n \"proj:shape\": [\n 15681,\n 15561\n ],\n \"proj:transform\": [\n 15.0,\n 0.0,\n 526792.5,\n 0.0,\n -15.0,\n -3077092.5,\n 0.0,\n 0.0,\n 1.0\n ]\n },\n \"nbart_red\": {\n \"eo:bands\": [\n {\n \"name\": \"nbart_red\"\n }\n ],\n \"type\": \"image/tiff; application=geotiff\",\n \"roles\": [\n \"data\"\n ],\n \"href\": \"http://dea-public-data-dev.s3-ap-southeast-2.amazonaws.com/analysis-ready-data/ga_ls8c_ard_3/088/080/2020/05/25/ga_ls8c_nbart_3-1-0_088080_2020-05-25_final_band04.tif\",\n \"proj:shape\": [\n 7841,\n 7781\n ],\n \"proj:transform\": [\n 30.0,\n 0.0,\n 526785.0,\n 0.0,\n -30.0,\n -3077085.0,\n 0.0,\n 0.0,\n 1.0\n ]\n },\n \"nbart_swir_1\": {\n \"eo:bands\": [\n {\n \"name\": \"nbart_swir_1\"\n }\n ],\n \"type\": \"image/tiff; application=geotiff\",\n \"roles\": [\n \"data\"\n ],\n \"href\": \"http://dea-public-data-dev.s3-ap-southeast-2.amazonaws.com/analysis-ready-data/ga_ls8c_ard_3/088/080/2020/05/25/ga_ls8c_nbart_3-1-0_088080_2020-05-25_final_band06.tif\",\n \"proj:shape\": [\n 7841,\n 7781\n ],\n \"proj:transform\": [\n 30.0,\n 0.0,\n 526785.0,\n 0.0,\n -30.0,\n -3077085.0,\n 0.0,\n 0.0,\n 1.0\n ]\n },\n \"nbart_swir_2\": {\n \"eo:bands\": [\n {\n \"name\": \"nbart_swir_2\"\n }\n ],\n \"type\": \"image/tiff; application=geotiff\",\n \"roles\": [\n \"data\"\n ],\n \"href\": \"http://dea-public-data-dev.s3-ap-southeast-2.amazonaws.com/analysis-ready-data/ga_ls8c_ard_3/088/080/2020/05/25/ga_ls8c_nbart_3-1-0_088080_2020-05-25_final_band07.tif\",\n \"proj:shape\": [\n 7841,\n 7781\n ],\n \"proj:transform\": [\n 30.0,\n 0.0,\n 526785.0,\n 0.0,\n -30.0,\n -3077085.0,\n 0.0,\n 0.0,\n 1.0\n ]\n },\n \"oa_azimuthal_exiting\": {\n \"eo:bands\": [\n {\n \"name\": \"oa_azimuthal_exiting\"\n }\n ],\n \"type\": \"image/tiff; application=geotiff\",\n \"roles\": [\n \"data\"\n ],\n \"href\": \"http://dea-public-data-dev.s3-ap-southeast-2.amazonaws.com/analysis-ready-data/ga_ls8c_ard_3/088/080/2020/05/25/ga_ls8c_oa_3-1-0_088080_2020-05-25_final_azimuthal-exiting.tif\",\n \"proj:shape\": [\n 7841,\n 7781\n ],\n \"proj:transform\": [\n 30.0,\n 0.0,\n 526785.0,\n 0.0,\n -30.0,\n -3077085.0,\n 0.0,\n 0.0,\n 1.0\n ]\n },\n \"oa_azimuthal_incident\": {\n \"eo:bands\": [\n {\n \"name\": \"oa_azimuthal_incident\"\n }\n ],\n \"type\": \"image/tiff; application=geotiff\",\n \"roles\": [\n \"data\"\n ],\n \"href\": \"http://dea-public-data-dev.s3-ap-southeast-2.amazonaws.com/analysis-ready-data/ga_ls8c_ard_3/088/080/2020/05/25/ga_ls8c_oa_3-1-0_088080_2020-05-25_final_azimuthal-incident.tif\",\n \"proj:shape\": [\n 7841,\n 7781\n ],\n \"proj:transform\": [\n 30.0,\n 0.0,\n 526785.0,\n 0.0,\n -30.0,\n -3077085.0,\n 0.0,\n 0.0,\n 1.0\n ]\n },\n \"oa_combined_terrain_shadow\": {\n \"eo:bands\": [\n {\n \"name\": \"oa_combined_terrain_shadow\"\n }\n ],\n \"type\": \"image/tiff; application=geotiff\",\n \"roles\": [\n \"data\"\n ],\n \"href\": \"http://dea-public-data-dev.s3-ap-southeast-2.amazonaws.com/analysis-ready-data/ga_ls8c_ard_3/088/080/2020/05/25/ga_ls8c_oa_3-1-0_088080_2020-05-25_final_combined-terrain-shadow.tif\",\n \"proj:shape\": [\n 7841,\n 7781\n ],\n \"proj:transform\": [\n 30.0,\n 0.0,\n 526785.0,\n 0.0,\n -30.0,\n -3077085.0,\n 0.0,\n 0.0,\n 1.0\n ]\n },\n \"oa_exiting_angle\": {\n \"eo:bands\": [\n {\n \"name\": \"oa_exiting_angle\"\n }\n ],\n \"type\": \"image/tiff; application=geotiff\",\n \"roles\": [\n \"data\"\n ],\n \"href\": \"http://dea-public-data-dev.s3-ap-southeast-2.amazonaws.com/analysis-ready-data/ga_ls8c_ard_3/088/080/2020/05/25/ga_ls8c_oa_3-1-0_088080_2020-05-25_final_exiting-angle.tif\",\n \"proj:shape\": [\n 7841,\n 7781\n ],\n \"proj:transform\": [\n 30.0,\n 0.0,\n 526785.0,\n 0.0,\n -30.0,\n -3077085.0,\n 0.0,\n 0.0,\n 1.0\n ]\n },\n \"oa_fmask\": {\n \"eo:bands\": [\n {\n \"name\": \"oa_fmask\"\n }\n ],\n \"type\": \"image/tiff; application=geotiff\",\n \"roles\": [\n \"data\"\n ],\n \"href\": \"http://dea-public-data-dev.s3-ap-southeast-2.amazonaws.com/analysis-ready-data/ga_ls8c_ard_3/088/080/2020/05/25/ga_ls8c_oa_3-1-0_088080_2020-05-25_final_fmask.tif\",\n \"proj:shape\": [\n 7841,\n 7781\n ],\n \"proj:transform\": [\n 30.0,\n 0.0,\n 526785.0,\n 0.0,\n -30.0,\n -3077085.0,\n 0.0,\n 0.0,\n 1.0\n ]\n },\n \"oa_incident_angle\": {\n \"eo:bands\": [\n {\n \"name\": \"oa_incident_angle\"\n }\n ],\n \"type\": \"image/tiff; application=geotiff\",\n \"roles\": [\n \"data\"\n ],\n \"href\": \"http://dea-public-data-dev.s3-ap-southeast-2.amazonaws.com/analysis-ready-data/ga_ls8c_ard_3/088/080/2020/05/25/ga_ls8c_oa_3-1-0_088080_2020-05-25_final_incident-angle.tif\",\n \"proj:shape\": [\n 7841,\n 7781\n ],\n \"proj:transform\": [\n 30.0,\n 0.0,\n 526785.0,\n 0.0,\n -30.0,\n -3077085.0,\n 0.0,\n 0.0,\n 1.0\n ]\n },\n \"oa_nbart_contiguity\": {\n \"eo:bands\": [\n {\n \"name\": \"oa_nbart_contiguity\"\n }\n ],\n \"type\": \"image/tiff; application=geotiff\",\n \"roles\": [\n \"data\"\n ],\n \"href\": \"http://dea-public-data-dev.s3-ap-southeast-2.amazonaws.com/analysis-ready-data/ga_ls8c_ard_3/088/080/2020/05/25/ga_ls8c_oa_3-1-0_088080_2020-05-25_final_nbart-contiguity.tif\",\n \"proj:shape\": [\n 7841,\n 7781\n ],\n \"proj:transform\": [\n 30.0,\n 0.0,\n 526785.0,\n 0.0,\n -30.0,\n -3077085.0,\n 0.0,\n 0.0,\n 1.0\n ]\n },\n \"oa_relative_azimuth\": {\n \"eo:bands\": [\n {\n \"name\": \"oa_relative_azimuth\"\n }\n ],\n \"type\": \"image/tiff; application=geotiff\",\n \"roles\": [\n \"data\"\n ],\n \"href\": \"http://dea-public-data-dev.s3-ap-southeast-2.amazonaws.com/analysis-ready-data/ga_ls8c_ard_3/088/080/2020/05/25/ga_ls8c_oa_3-1-0_088080_2020-05-25_final_relative-azimuth.tif\",\n \"proj:shape\": [\n 7841,\n 7781\n ],\n \"proj:transform\": [\n 30.0,\n 0.0,\n 526785.0,\n 0.0,\n -30.0,\n -3077085.0,\n 0.0,\n 0.0,\n 1.0\n ]\n },\n \"oa_relative_slope\": {\n \"eo:bands\": [\n {\n \"name\": \"oa_relative_slope\"\n }\n ],\n \"type\": \"image/tiff; application=geotiff\",\n \"roles\": [\n \"data\"\n ],\n \"href\": \"http://dea-public-data-dev.s3-ap-southeast-2.amazonaws.com/analysis-ready-data/ga_ls8c_ard_3/088/080/2020/05/25/ga_ls8c_oa_3-1-0_088080_2020-05-25_final_relative-slope.tif\",\n \"proj:shape\": [\n 7841,\n 7781\n ],\n \"proj:transform\": [\n 30.0,\n 0.0,\n 526785.0,\n 0.0,\n -30.0,\n -3077085.0,\n 0.0,\n 0.0,\n 1.0\n ]\n },\n \"oa_satellite_azimuth\": {\n \"eo:bands\": [\n {\n \"name\": \"oa_satellite_azimuth\"\n }\n ],\n \"type\": \"image/tiff; application=geotiff\",\n \"roles\": [\n \"data\"\n ],\n \"href\": \"http://dea-public-data-dev.s3-ap-southeast-2.amazonaws.com/analysis-ready-data/ga_ls8c_ard_3/088/080/2020/05/25/ga_ls8c_oa_3-1-0_088080_2020-05-25_final_satellite-azimuth.tif\",\n \"proj:shape\": [\n 7841,\n 7781\n ],\n \"proj:transform\": [\n 30.0,\n 0.0,\n 526785.0,\n 0.0,\n -30.0,\n -3077085.0,\n 0.0,\n 0.0,\n 1.0\n ]\n },\n \"oa_satellite_view\": {\n \"eo:bands\": [\n {\n \"name\": \"oa_satellite_view\"\n }\n ],\n \"type\": \"image/tiff; application=geotiff\",\n \"roles\": [\n \"data\"\n ],\n \"href\": \"http://dea-public-data-dev.s3-ap-southeast-2.amazonaws.com/analysis-ready-data/ga_ls8c_ard_3/088/080/2020/05/25/ga_ls8c_oa_3-1-0_088080_2020-05-25_final_satellite-view.tif\",\n \"proj:shape\": [\n 7841,\n 7781\n ],\n \"proj:transform\": [\n 30.0,\n 0.0,\n 526785.0,\n 0.0,\n -30.0,\n -3077085.0,\n 0.0,\n 0.0,\n 1.0\n ]\n },\n \"oa_solar_azimuth\": {\n \"eo:bands\": [\n {\n \"name\": \"oa_solar_azimuth\"\n }\n ],\n \"type\": \"image/tiff; application=geotiff\",\n \"roles\": [\n \"data\"\n ],\n \"href\": \"http://dea-public-data-dev.s3-ap-southeast-2.amazonaws.com/analysis-ready-data/ga_ls8c_ard_3/088/080/2020/05/25/ga_ls8c_oa_3-1-0_088080_2020-05-25_final_solar-azimuth.tif\",\n \"proj:shape\": [\n 7841,\n 7781\n ],\n \"proj:transform\": [\n 30.0,\n 0.0,\n 526785.0,\n 0.0,\n -30.0,\n -3077085.0,\n 0.0,\n 0.0,\n 1.0\n ]\n },\n \"oa_solar_zenith\": {\n \"eo:bands\": [\n {\n \"name\": \"oa_solar_zenith\"\n }\n ],\n \"type\": \"image/tiff; application=geotiff\",\n \"roles\": [\n \"data\"\n ],\n \"href\": \"http://dea-public-data-dev.s3-ap-southeast-2.amazonaws.com/analysis-ready-data/ga_ls8c_ard_3/088/080/2020/05/25/ga_ls8c_oa_3-1-0_088080_2020-05-25_final_solar-zenith.tif\",\n \"proj:shape\": [\n 7841,\n 7781\n ],\n \"proj:transform\": [\n 30.0,\n 0.0,\n 526785.0,\n 0.0,\n -30.0,\n -3077085.0,\n 0.0,\n 0.0,\n 1.0\n ]\n },\n \"oa_time_delta\": {\n \"eo:bands\": [\n {\n \"name\": \"oa_time_delta\"\n }\n ],\n \"type\": \"image/tiff; application=geotiff\",\n \"roles\": [\n \"data\"\n ],\n \"href\": \"http://dea-public-data-dev.s3-ap-southeast-2.amazonaws.com/analysis-ready-data/ga_ls8c_ard_3/088/080/2020/05/25/ga_ls8c_oa_3-1-0_088080_2020-05-25_final_time-delta.tif\",\n \"proj:shape\": [\n 7841,\n 7781\n ],\n \"proj:transform\": [\n 30.0,\n 0.0,\n 526785.0,\n 0.0,\n -30.0,\n -3077085.0,\n 0.0,\n 0.0,\n 1.0\n ]\n },\n \"thumbnail:nbart\": {\n \"title\": \"Thumbnail image\",\n \"type\": \"image/jpeg\",\n \"roles\": [\n \"thumbnail\"\n ],\n \"href\": \"http://dea-public-data-dev.s3-ap-southeast-2.amazonaws.com/analysis-ready-data/ga_ls8c_ard_3/088/080/2020/05/25/ga_ls8c_nbart_3-1-0_088080_2020-05-25_final_thumbnail.jpg\"\n },\n \"checksum:sha1\": {\n \"type\": \"text/plain\",\n \"href\": \"http://dea-public-data-dev.s3-ap-southeast-2.amazonaws.com/analysis-ready-data/ga_ls8c_ard_3/088/080/2020/05/25/ga_ls8c_ard_3-1-0_088080_2020-05-25_final.sha1\"\n },\n \"metadata:processor\": {\n \"type\": \"text/yaml\",\n \"roles\": [\n \"metadata\"\n ],\n \"href\": \"http://dea-public-data-dev.s3-ap-southeast-2.amazonaws.com/analysis-ready-data/ga_ls8c_ard_3/088/080/2020/05/25/ga_ls8c_ard_3-1-0_088080_2020-05-25_final.proc-info.yaml\"\n }\n },\n \"links\": [\n {\n \"rel\": \"self\",\n \"type\": \"application/json\",\n \"href\": \"http://dea-public-data-dev.s3-ap-southeast-2.amazonaws.com/analysis-ready-data/ga_ls8c_ard_3/088/080/2020/05/25/ga_ls8c_ard_3-1-0_088080_2020-05-25_final.stac-item.json\"\n },\n {\n \"title\": \"Source Dataset YAML\",\n \"rel\": \"odc_yaml\",\n \"type\": \"text/yaml\",\n \"href\": \"http://dea-public-data-dev.s3-ap-southeast-2.amazonaws.com/analysis-ready-data/ga_ls8c_ard_3/088/080/2020/05/25/ga_ls8c_ard_3-1-0_088080_2020-05-25_final.odc-metadata.yaml\"\n },\n {\n \"title\": \"Open Data Cube Product Overview\",\n \"rel\": \"product_overview\",\n \"type\": \"text/html\",\n \"href\": \"https://explorer.dev.dea.ga.gov.au/product/ga_ls8c_ard_3\"\n },\n {\n \"title\": \"Open Data Cube Explorer\",\n \"rel\": \"alternative\",\n \"type\": \"text/html\",\n \"href\": \"https://explorer.dev.dea.ga.gov.au/dataset/2aa69fcf-aa55-4747-9d95-3652f9fe79b0\"\n }\n ]\n}", + "Timestamp": "2020-08-13T07:07:48.691Z", + "SignatureVersion": "1", + "Signature": "Ldhhjz/TBHgXNv+A1rkxlxCV9IXnlKgm/XiPx0qeBILteWYuMJ0vNFPhYBfJ8kfTcVeEiZoFNYvP7scVs01IOu2jqGdtbeXoiC1Ng5mI3QylC1jeTfdF2u79MOHuiCvtGzjm+eYEcGwPEqSmavD3/VLeprKnwOOuJ6b49i9C8KnWVeoUPK14G2FpP5vtMMVN/+wyE1UK/NzGmQKKiW2aGhERGeWmqNtRMhl93EWpSdKaaAMYfcYStFt2G6nAPfIrTCAa6+pVDgJnYsba3eyvTjrPdNR9dq6nmjbdxXFffV/ojmLgLaee8lAZKTrVMxKFTMp4P0QBqJze3DP0yiPOPg==", + "SigningCertURL": "https://sns.ap-southeast-2.amazonaws.com/SimpleNotificationService-a86cb10b4e1f29c941702d737128f7b6.pem", + "UnsubscribeURL": "https://sns.ap-southeast-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:ap-southeast-2:451924316694:landsat-collection-3-dev-topic:e55a3989-e0ab-4fe8-9795-0e9232e5056e", + "MessageAttributes": { + "product": { + "Type": "String", + "Value": "ga_ls8c_ard_3" + }, + "datetime": { + "Type": "String", + "Value": "2020-05-25T23:35:47.745731Z" + }, + "bbox.ur_lon": { + "Type": "Number", + "Value": "155.6469914729725" + }, + "bbox.ur_lat": { + "Type": "Number", + "Value": "-27.816894407110592" + }, + "action": { + "Type": "String", + "Value": "ADDED" + }, + "bbox.ll_lon": { + "Type": "Number", + "Value": "153.27898977305418" + }, + "bbox.ll_lat": { + "Type": "Number", + "Value": "-29.92243448617234" + }, + "cloudcover": { + "Type": "Number", + "Value": "42.42102985910952" + } + } +} \ No newline at end of file diff --git a/apps/dc_tools/tests/data/sentinel-2-nrt_2020_08_21.json b/apps/dc_tools/tests/data/sentinel-2-nrt_2020_08_21.json new file mode 100644 index 000000000..b127a4bc2 --- /dev/null +++ b/apps/dc_tools/tests/data/sentinel-2-nrt_2020_08_21.json @@ -0,0 +1,12 @@ +{ + "Type" : "Notification", + "MessageId" : "f2520be8-e2b9-57bb-ad6b-5c516544b979", + "TopicArn" : "arn:aws:sns:ap-southeast-2:538673716275:DEANewData", + "Subject" : "Amazon S3 Notification", + "Message" : "{\r\n \"Records\":\r\n [\r\n {\r\n \"eventVersion\":\"2.1\",\r\n \"eventSource\":\"aws:s3\",\r\n \"awsRegion\":\"ap-southeast-2\",\r\n \"eventTime\":\"2020-08-21T08:28:43.602Z\",\r\n \"eventName\":\"ObjectCreated:Put\",\r\n \"userIdentity\":{\"principalId\":\"AWS:AROAIWSUDPHXXIE56RWBK:wagl_nrt\"},\r\n \"requestParameters\":{\"sourceIPAddress\":\"10.0.11.24\"},\r\n \"responseElements\":{\r\n \"x-amz-request-id\":\"745FE6454A00DB21\",\r\n \"x-amz-id-2\":\"NCqXKGRW6W3Ao1L5HoZJ\/yLy\/5x+MmApOGUzZhZFctvhZY1aDV1kwb9kVfjLau2fJpajPWzH83MROcetXEAEzxRaoL9qBxfY\"\r\n },\r\n \"s3\":{\r\n \"s3SchemaVersion\":\"1.0\",\r\n \"configurationId\":\"DEASentinel2NRT\",\r\n \"bucket\":{\r\n \"name\":\"dea-public-data\",\r\n \"ownerIdentity\":{\"principalId\":\"A2I385BZ60F0QY\"},\r\n \"arn\":\"arn:aws:s3:::dea-public-data\"\r\n },\r\n \"object\":{\r\n \"key\":\"L2\/sentinel-2-nrt\/S2MSIARD\/2020-08-21\/S2B_OPER_MSI_ARD_TL_VGS1_20200821T014801_A018060_T54HVH_N02.09\/ARD-METADATA.yaml\",\r\n \"size\":28324,\r\n \"eTag\":\"439190a173d10147b0ba9c2aa1ca3041\",\r\n \"versionId\":\"s84jtWiCSLLXlQuJ8eteL0HwgK3l0sUc\",\r\n \"sequencer\":\"005F3F85BD1738B0FD\"}\r\n }\r\n }\r\n ]\r\n }", + "Timestamp" : "2020-08-21T08:28:45.921Z", + "SignatureVersion" : "1", + "Signature" : "AGQ4OymLiq1DPUXepUsRv27YnIzhY9VWGt57N4AFrHcuw6XT7C71pI4lXrZkeCUwBWAck31E0nna1ntM1HhcUpd6ZfrrUGJvd82B5HE/ev2Q/FYsCoj41CIX6ghJDvJ5o4tgGrsLYE9FJMDbOo/4hXhn8g23Xgoa/woQHDXfH2bcuJkSfIGznVFnc5hZcwNuV5l4J5KcV0T6Q+LT1LHJD6UMeUQRFrzxanZbF29D5/AbEzH10/nJqWV2GLgLCPN7sQnVC7svA6C5eTMTJzo1vftcM3NHoscAnxGta0AxzKGqUFGxF31/5QLfZTYSXbcPjz/5sf4BqvvJ9Jkct2G6bg==", + "SigningCertURL" : "https://sns.ap-southeast-2.amazonaws.com/SimpleNotificationService-a86cb10b4e1f29c941702d737128f7b6.pem", + "UnsubscribeURL" : "https://sns.ap-southeast-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:ap-southeast-2:538673716275:DEANewData:3484d14d-d66e-493c-a61c-2adcb9fef394" +} \ No newline at end of file diff --git a/apps/dc_tools/tests/test_sqs_to_dc.py b/apps/dc_tools/tests/test_sqs_to_dc.py new file mode 100644 index 000000000..fe40b270a --- /dev/null +++ b/apps/dc_tools/tests/test_sqs_to_dc.py @@ -0,0 +1,169 @@ +""" +Test for SQS to DC tool +""" +import json +from functools import partial +from pprint import pformat + +import pytest + +from pathlib import Path + +from datacube.utils import documents +from deepdiff import DeepDiff +from datetime import date +from odc.index.stac import stac_transform +from odc.apps.dc_tools.sqs_to_dc import ( + get_metadata_uri, + get_metadata_from_s3_record, + get_s3_url, +) + + +TEST_DATA_FOLDER: Path = Path(__file__).parent.joinpath("data") +LANDSAT_C3_SQS_MESSAGE: str = "ga_ls8c_ard_3-1-0_088080_2020-05-25_final.stac-item.json" +LANDSAT_C3_ODC_YAML: str = "ga_ls8c_ard_3-1-0_088080_2020-05-25_final.odc-metadata.yaml" +SENTINEL_2_NRT_MESSAGE: str = "sentinel-2-nrt_2020_08_21.json" +SENTINEL_2_NRT_RECORD_PATH = ("L2/sentinel-2-nrt/S2MSIARD/*/*/ARD-METADATA.yaml",) + +deep_diff = partial( + DeepDiff, significant_digits=6, ignore_type_in_groups=[(tuple, list)] +) + +@pytest.mark.skip(reason="Test is failing due to requiring credentials") +@pytest.mark.skipif( + date.today() > date(2020, 11, 10), reason="dataset has been rotated out" +) +def test_get_metadata_s3_object(sentinel_2_nrt_message, sentinel_2_nrt_record_path): + data, uri = get_metadata_from_s3_record( + sentinel_2_nrt_message, sentinel_2_nrt_record_path + ) + + assert type(data) is dict + assert uri == get_s3_url( + bucket_name="dea-public-data", + obj_key="L2/sentinel-2-nrt/S2MSIARD/2020-08-21/S2B_OPER_MSI_ARD_TL_VGS1_20200821T014801_A018060_T54HVH_N02.09/ARD-METADATA.yaml", + ) + + +def test_get_metadata_uri(ga_ls8c_ard_3_message, ga_ls8c_ard_3_yaml): + actual_doc, uri = get_metadata_uri( + ga_ls8c_ard_3_message, None, "STAC-LINKS-REL:odc_yaml" + ) + + assert type(actual_doc) is dict + assert ga_ls8c_ard_3_yaml["id"] == actual_doc["id"] + assert ga_ls8c_ard_3_yaml["crs"] == actual_doc["crs"] + assert ga_ls8c_ard_3_yaml["product"]["name"] == actual_doc["product"]["name"] + assert ga_ls8c_ard_3_yaml["label"] == actual_doc["label"] + + # Test geometry field + doc_diff = deep_diff(ga_ls8c_ard_3_yaml["geometry"], actual_doc["geometry"]) + assert doc_diff == {}, pformat(doc_diff) + + # Test grids field + doc_diff = deep_diff( + ga_ls8c_ard_3_yaml["grids"]["default"], actual_doc["grids"]["default"] + ) + assert doc_diff == {}, pformat(doc_diff) + + # Test measurements field + doc_diff = deep_diff( + ga_ls8c_ard_3_yaml["measurements"], + actual_doc["measurements"], + exclude_paths=["root['nbart_panchromatic']"], + ) + assert doc_diff == {}, pformat(doc_diff) + + +def test_odc_metadata_link(ga_ls8c_ard_3_message): + actual_doc, uri = get_metadata_uri( + ga_ls8c_ard_3_message, None, "STAC-LINKS-REL:odc_yaml" + ) + assert ( + uri == "http://dea-public-data-dev.s3-ap-southeast-2.amazonaws.com/" + "analysis-ready-data/ga_ls8c_ard_3/088/080/2020/05/25/" + "ga_ls8c_ard_3-1-0_088080_2020-05-25_final.odc-metadata.yaml" + ) + + +def test_stac_link(ga_ls8c_ard_3_message): + metadata, uri = get_metadata_uri(ga_ls8c_ard_3_message, stac_transform, None) + assert ( + uri != "http://dea-public-data-dev.s3-ap-southeast-2.amazonaws.com/" + "analysis-ready-data/ga_ls8c_ard_3/088/080/2020/05/25/" + "ga_ls8c_ard_3-1-0_088080_2020-05-25_final.odc-metadata.yaml" + ) + assert ( + uri == "http://dea-public-data-dev.s3-ap-southeast-2.amazonaws.com/" + "analysis-ready-data/ga_ls8c_ard_3/088/080/2020/05/25/" + "ga_ls8c_ard_3-1-0_088080_2020-05-25_final.stac-item.json" + ) + + +@pytest.mark.skip(reason="Skipping due to issues with coordinate rounding") +def test_transform(ga_ls8c_ard_3_message, ga_ls8c_ard_3_yaml): + actual_doc, uri = get_metadata_uri(ga_ls8c_ard_3_message, stac_transform, None) + + assert ga_ls8c_ard_3_yaml["id"] == actual_doc["id"] + assert ga_ls8c_ard_3_yaml["crs"] == actual_doc["crs"] + assert ga_ls8c_ard_3_yaml["product"]["name"] == actual_doc["product"]["name"] + assert ga_ls8c_ard_3_yaml["label"] == actual_doc["label"] + + # Test geometry field + doc_diff = deep_diff(ga_ls8c_ard_3_yaml["geometry"], actual_doc["geometry"]) + assert doc_diff == {}, pformat(doc_diff) + + # Test grids field + doc_diff = deep_diff(ga_ls8c_ard_3_yaml["grids"], actual_doc["grids"]) + assert doc_diff == {}, pformat(doc_diff) + + # Test measurements field + doc_diff = deep_diff(ga_ls8c_ard_3_yaml["measurements"], actual_doc["measurements"]) + assert doc_diff == {}, pformat(doc_diff) + + # Test properties field + doc_diff = deep_diff( + ga_ls8c_ard_3_yaml["properties"], + actual_doc["properties"], + exclude_paths=[ + "root['odc:product']", + "root['datetime']", + "root['dtr:start_datetime']", + "root['dtr:end_datetime']", + "root['odc:processing_datetime']", + "root['proj:epsg']", + "root['proj:shape']", + "root['proj:transform']", + ], + ) + assert doc_diff == {}, pformat(doc_diff) + + +@pytest.fixture +def ga_ls8c_ard_3_message(): + with TEST_DATA_FOLDER.joinpath(LANDSAT_C3_SQS_MESSAGE).open("r") as f: + body = json.load(f) + metadata = json.loads(body["Message"]) + return metadata + + +@pytest.fixture +def sentinel_2_nrt_record_path(): + return SENTINEL_2_NRT_RECORD_PATH + + +@pytest.fixture +def sentinel_2_nrt_message(): + with TEST_DATA_FOLDER.joinpath(SENTINEL_2_NRT_MESSAGE).open("r") as f: + body = json.load(f) + metadata = json.loads(body["Message"]) + return metadata + + +@pytest.fixture +def ga_ls8c_ard_3_yaml(): + metadata = yield from documents.load_documents( + TEST_DATA_FOLDER.joinpath(LANDSAT_C3_ODC_YAML) + ) + return metadata