Merge pull request #130 from ASFHyP3/develop

Release v0.7.0

CHANGELOG.md

@@ -7,6 +7,12 @@ and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/)
 and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 
+## [0.7.0]
+### Added
+* InSAR stripmap workflow to support AVO's ALOS-1 processing efforts. This workflow is specific to AVO currently, and may not work for others.
+### Changed
+* The naming convention for the burst products has been updated.
+
 ## [0.6.2]
 ### Changed
 * The geocoding DEM is now resampled to ~20m in the case of 5x1 looks.

environment.yml

@@ -14,6 +14,7 @@ dependencies:
   - asf_search>=6.4.0
   - geopandas
   - gdal
+  - opencv
   # For packaging, and testing
   - flake8
   - flake8-import-order

src/hyp3_isce2/burst.py

@@ -6,6 +6,7 @@
 from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass
 from pathlib import Path
+from secrets import token_hex
 from typing import Iterator, List, Optional, Tuple, Union
 
 import asf_search
@@ -327,19 +328,42 @@ def download_bursts(param_list: Iterator[BurstParams]) -> List[BurstMetadata]:
 def get_product_name(
     reference_scene: str,
     secondary_scene: str,
+    pixel_spacing: int
 ) -> str:
     """Get the name of the interferogram product.
 
     Args:
         reference_scene: The reference burst name.
         secondary_scene: The secondary burst name.
+        pixel_spacing: The spacing of the pixels in the output image.
 
     Returns:
         The name of the interferogram product.
     """
-    # If this changes, we will also need to update the burst product README template,
-    # which documents this naming convention.
-    return f'{reference_scene}x{secondary_scene}'
+
+    reference_split = reference_scene.split('_')
+    secondary_split = secondary_scene.split('_')
+
+    platform = reference_split[0]
+    burst_id = reference_split[1]
+    image_plus_swath = reference_split[2]
+    reference_date = reference_split[3][0:8]
+    secondary_date = secondary_split[3][0:8]
+    polarization = reference_split[4]
+    product_type = 'INT'
+    pixel_spacing = str(int(pixel_spacing))
+    product_id = token_hex(2).upper()
+
+    return '_'.join([
+        platform,
+        burst_id,
+        image_plus_swath,
+        reference_date,
+        secondary_date,
+        polarization,
+        product_type + pixel_spacing,
+        product_id
+    ])
 
 
 def get_burst_params(scene_name: str) -> BurstParams:

src/hyp3_isce2/insar_stripmap.py

@@ -3,18 +3,24 @@
 """
 
 import argparse
+import glob
 import logging
 import os
+import shutil
 import site
 import sys
+import zipfile
 from pathlib import Path
+from shutil import make_archive
 
+import asf_search
 from hyp3lib.aws import upload_file_to_s3
-from hyp3lib.image import create_thumbnail
+from shapely.geometry.polygon import Polygon
 
+from hyp3_isce2 import stripmapapp_alos as stripmapapp
+from hyp3_isce2.dem import download_dem_for_isce2
 from hyp3_isce2.logging import configure_root_logger
 
-
 log = logging.getLogger(__name__)
 
 # ISCE needs its applications to be on the system path.
@@ -24,31 +30,89 @@
     os.environ['PATH'] = str(ISCE_APPLICATIONS) + os.pathsep + os.environ['PATH']
 
 
-def insar_stripmap(reference_scene: str, secondary_scene: str) -> Path:
-    """Create an interferogram
-
-    This is a placeholder function. It will be replaced with your actual scientific workflow.
+def insar_stripmap(user: str, password: str, reference_scene: str, secondary_scene: str) -> Path:
+    """Create a Stripmap interferogram
 
     Args:
+        user: Earthdata username
+        password: Earthdata password
         reference_scene: Reference scene name
         secondary_scene: Secondary scene name
 
     Returns:
         Path to the output files
     """
+    session = asf_search.ASFSession().auth_with_creds(user, password)
+
+    reference_product, secondary_product = asf_search.search(
+        granule_list=[reference_scene, secondary_scene],
+        processingLevel=asf_search.L1_0,
+    )
+    assert reference_product.properties['sceneName'] == reference_scene
+    assert secondary_product.properties['sceneName'] == secondary_scene
+    products = (reference_product, secondary_product)
+
+    polygons = [Polygon(product.geometry['coordinates'][0]) for product in products]
+    insar_roi = polygons[0].intersection(polygons[1]).bounds
+
+    dem_path = download_dem_for_isce2(insar_roi, dem_name='glo_30', dem_dir=Path('dem'), buffer=0)
 
-    raise NotImplementedError('This is a placeholder function. Replace it with your actual scientific workflow.')
+    urls = [product.properties['url'] for product in products]
+    asf_search.download_urls(urls=urls, path=os.getcwd(), session=session, processes=2)
+
+    zip_paths = [product.properties['fileName'] for product in products]
+    for zip_path in zip_paths:
+        with zipfile.ZipFile(zip_path, 'r') as zip_file:
+            zip_file.extractall()
+        os.remove(zip_path)
+
+    reference_image = get_product_file(reference_product, 'IMG-')
+    reference_leader = get_product_file(reference_product, 'LED-')
+
+    secondary_image = get_product_file(secondary_product, 'IMG-')
+    secondary_leader = get_product_file(secondary_product, 'LED-')
+
+    config = stripmapapp.StripmapappConfig(
+        reference_image=reference_image,
+        reference_leader=reference_leader,
+        secondary_image=secondary_image,
+        secondary_leader=secondary_leader,
+        roi=insar_roi,
+        dem_filename=str(dem_path),
+    )
+    config_path = config.write_template('stripmapApp.xml')
 
-    product_file = Path("product_file_name.zip")
-    return product_file
+    stripmapapp.run_stripmapapp(start='startup', end='geocode', config_xml=config_path)
+
+    product_dir = Path(f'{reference_scene}x{secondary_scene}')
+    (product_dir / 'interferogram').mkdir(parents=True)
+
+    for filename in os.listdir('interferogram'):
+        path = Path('interferogram') / filename
+        if os.path.isfile(path):
+            shutil.move(path, product_dir / path)
+
+    shutil.move('geometry', product_dir)
+    shutil.move('ionosphere', product_dir)
+
+    return product_dir
+
+
+def get_product_file(product: asf_search.ASFProduct, file_prefix: str) -> str:
+    paths = glob.glob(str(Path(product.properties['fileID']) / f'{file_prefix}*'))
+    assert len(paths) == 1
+    return paths[0]
 
 
 def main():
     """ Entrypoint for the stripmap workflow"""
+
     parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 
-    parser.add_argument('--bucket', type=str, default='', help='AWS S3 bucket HyP3 for upload the final product(s)')
-    parser.add_argument('--bucket-prefix', type=str, default='', help='Add a bucket prefix to product(s)')
+    parser.add_argument('--bucket', help='AWS S3 bucket HyP3 for upload the final product(s)')
+    parser.add_argument('--bucket-prefix', default='', help='Add a bucket prefix to product(s)')
+    parser.add_argument('--username', type=str, required=True)
+    parser.add_argument('--password', type=str, required=True)
     parser.add_argument('--reference-scene', type=str, required=True)
     parser.add_argument('--secondary-scene', type=str, required=True)
 
@@ -59,17 +123,20 @@ def main():
 
     log.info('Begin InSAR Stripmap run')
 
-    product_file = insar_stripmap(
+    product_dir = insar_stripmap(
+        user=args.username,
+        password=args.password,
         reference_scene=args.reference_scene,
         secondary_scene=args.secondary_scene,
     )
 
     log.info('InSAR Stripmap run completed successfully')
 
+    output_zip = make_archive(base_name=product_dir.name, format='zip', base_dir=product_dir)
+
     if args.bucket:
-        upload_file_to_s3(product_file, args.bucket, args.bucket_prefix)
-        browse_images = product_file.with_suffix('.png')
-        for browse in browse_images:
-            thumbnail = create_thumbnail(browse)
-            upload_file_to_s3(browse, args.bucket, args.bucket_prefix)
-            upload_file_to_s3(thumbnail, args.bucket, args.bucket_prefix)
+        # TODO do we want browse images?
+
+        upload_file_to_s3(Path(output_zip), args.bucket, args.bucket_prefix)
+
+        # TODO upload individual files to S3?

src/hyp3_isce2/insar_tops_burst.py

@@ -152,6 +152,7 @@ def make_readme(
         'processor_version': isce.__version__,
         'projection': hyp3_isce2.metadata.util.get_projection(info['coordinateSystem']['wkt']),
         'pixel_spacing': info['geoTransform'][1],
+        'product_name': product_name,
         'reference_burst_name': reference_scene,
         'secondary_burst_name': secondary_scene,
         'range_looks': range_looks,
@@ -427,12 +428,12 @@ def main():
     )
 
     log.info('ISCE2 TopsApp run completed successfully')
-    product_name = get_product_name(reference_scene, secondary_scene)
+    pixel_size = get_pixel_size(args.looks)
+    product_name = get_product_name(reference_scene, secondary_scene, pixel_spacing=int(pixel_size))
 
     product_dir = Path(product_name)
     product_dir.mkdir(parents=True, exist_ok=True)
 
-    pixel_size = get_pixel_size(args.looks)
     translate_outputs(isce_output_dir, product_name, pixel_size=pixel_size)
 
     unwrapped_phase = f'{product_name}/{product_name}_unw_phase.tif'

src/hyp3_isce2/metadata/templates/insar_burst/readme.md.txt.j2

@@ -19,13 +19,11 @@ The source bursts for this InSAR product are:
 
 Processing Date/Time: {{ processing_date.isoformat(timespec='seconds') }}
 
-The product directory is named <reference_burst_name>x<secondary_burst_name>
+The directory name for this product is: {{ product_name }}
 
-The directory name for this product is: {{ reference_burst_name }}x{{ secondary_burst_name }}.
+The output directory uses the following naming convention:
 
-The reference_burst_name and secondary_burst_name use the following naming conventions:
-
-S1_bbbbbb_xxn_yyyymmddThhmmss_pp_cccc-BURST
+S1_bbbbbb_xxn_yyyymmdd_yyyymmdd_pp_INTzz_cccc
 
 bbbbbb: Relative burst ID values assigned by ESA. Each value identifies a consistent burst footprint; relative burst
         ID values differ from one subswath to the next.
@@ -35,7 +33,7 @@ xxn: Three character combination that represents the acquisition mode and subswa
         (1-3 for IW, 1-5 for EW). IW mode indicates Interferometric Wideswath, which acquires a 250 km swath composed
         of three subswaths. EW mode indicates Extra-Wide Swath, which acquires a 400 km swath composed of 5 subswaths.
 
-yyyymmddThhmmss: ISO date and time of acquisition.
+yyyymmdd: Date of acquisition of the reference and secondary images, respectively.
 
 pp: Two character combination that represents the mode of radar orientation (polarization) for both signal
         transmission and reception. The first position represents the transmit orientation mode and the second
@@ -46,7 +44,11 @@ pp: Two character combination that represents the mode of radar orientation (pol
     VH: Vertical Transmit - Horizontal Receive
     VV: Vertical Transmit - Vertical Receive
 
-cccc: 4-character "Product Unique Identifier" from the SLC filename.
+INT: The product type (always INT for InSAR).
+
+zz: The pixel spacing of the output image.
+
+cccc: 4-character unique product identifier.
 
 Files contained in the product directory are named using the directory name followed by a tag indicating the file type.