Merge pull request #242 from ASFHyP3/simplify

Switch back to adding multiburst functionality to insar_tops_burst

CHANGELOG.md

@@ -8,7 +8,10 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [2.1.0]
 ### Added
-- `insar_tops_multi_burst` workflow for processing multiple bursts as one SLC.
+- The ability for the `insar_tops_burst` workflow to support processing multiple bursts as one SLC.
+
+### Changed
+- The interface for `insar_tops_burst` so that it takes `--reference` and `--secondary` granule lists. The positional `granules` argument is now optional and deprecated.
 
 ## [2.0.0]
 ### Changed

README.md

@@ -10,24 +10,23 @@ The HyP3-ISCE2 plugin provides a set of workflows (accessible directly in Python
 
 - `insar_stripmap`: A workflow for creating ALOS-1 geocoded unwrapped interferogram using ISCE2's Stripmap processing workflow
 - `insar_tops`: A workflow for creating full-SLC Sentinel-1 geocoded unwrapped interferogram using ISCE2's TOPS processing workflow
-- `insar_tops_burst`: A workflow for creating single-burst Sentinel-1 geocoded unwrapped interferogram using ISCE2's TOPS processing workflow
-- `insar_tops_multi_burst`: A workflow for creating multi-burst Sentinel-1 geocoded unwrapped interferogram using ISCE2's TOPS processing workflow
+- `insar_tops_burst`: A workflow for creating burst-based Sentinel-1 geocoded unwrapped interferogram using ISCE2's TOPS processing workflow
 ---
 
 To run a workflow, simply run `python -m hyp3_isce2 ++process [WORKFLOW_NAME] [WORKFLOW_ARGS]`. For example, to run the `insar_tops_burst` workflow:
 
 ```
 python -m hyp3_isce2 ++process insar_tops_burst \
-  S1_136231_IW2_20200604T022312_VV_7C85-BURST \
-  S1_136231_IW2_20200616T022313_VV_5D11-BURST \
+  --reference S1_136231_IW2_20200604T022312_VV_7C85-BURST \
+  --secondary S1_136231_IW2_20200616T022313_VV_5D11-BURST \
   --looks 20x4 \
   --apply-water-mask True
 ```
 
 and, for multiple burst pairs:
 
 ```
-python -m hyp3_isce2 ++process insar_tops_multi_burst \
+python -m hyp3_isce2 ++process insar_tops_burst \
   --reference S1_136231_IW2_20200604T022312_VV_7C85-BURST S1_136232_IW2_20200604T022315_VV_7C85-BURST \
   --secondary S1_136231_IW2_20200616T022313_VV_5D11-BURST S1_136232_IW2_20200616T022316_VV_5D11-BURST \
   --looks 20x4 \

pyproject.toml

@@ -53,14 +53,12 @@ Documentation = "https://hyp3-docs.asf.alaska.edu"
 
 [project.scripts]
 insar_tops_burst = "hyp3_isce2.insar_tops_burst:main"
-insar_tops_multi_burst = "hyp3_isce2.insar_tops_multi_burst:main"
 insar_tops = "hyp3_isce2.insar_tops:main"
 insar_stripmap = "hyp3_isce2.insar_stripmap:main"
 merge_tops_bursts = "hyp3_isce2.merge_tops_bursts:main"
 
 [project.entry-points.hyp3]
 insar_tops_burst = "hyp3_isce2.insar_tops_burst:main"
-insar_tops_multi_burst = "hyp3_isce2.insar_tops_multi_burst:main"
 insar_tops = "hyp3_isce2.insar_tops:main"
 insar_stripmap = "hyp3_isce2.insar_stripmap:main"
 merge_tops_bursts = "hyp3_isce2.merge_tops_bursts:main"

src/hyp3_isce2/__main__.py

@@ -19,7 +19,7 @@ def main():
     parser = argparse.ArgumentParser(prefix_chars='+', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     parser.add_argument(
         '++process',
-        choices=['insar_tops_burst', 'insar_tops', 'insar_stripmap', 'merge_tops_bursts', 'insar_tops_multi_burst'],
+        choices=['insar_tops_burst', 'insar_tops', 'insar_stripmap', 'merge_tops_bursts'],
         default='insar_tops_burst',
         help='Select the HyP3 entrypoint to use',  # HyP3 entrypoints are specified in `pyproject.toml`
     )

src/hyp3_isce2/burst.py

@@ -7,7 +7,7 @@
 from dataclasses import dataclass
 from datetime import datetime, timedelta
 from pathlib import Path
-from typing import Iterator, List, Optional, Tuple, Union
+from typing import Iterable, Iterator, List, Optional, Tuple, Union
 
 import asf_search
 import numpy as np
@@ -360,35 +360,47 @@ def get_burst_params(scene_name: str) -> BurstParams:
     )
 
 
-def validate_bursts(reference_scene: str, secondary_scene: str) -> None:
+def validate_bursts(reference: Union[str, Iterable[str]], secondary: Union[str, Iterable[str]]) -> None:
     """Check whether the reference and secondary bursts are valid.
 
     Args:
-        reference_scene: The reference burst name.
-        secondary_scene: The secondary burst name.
-
-    Returns:
-        None
+        reference: Reference granule(s)
+        secondary: Secondary granule(s)
     """
-    ref_split = reference_scene.split('_')
-    sec_split = secondary_scene.split('_')
+    if isinstance(reference, str):
+        reference = [reference]
+    if isinstance(secondary, str):
+        secondary = [secondary]
+
+    if len(reference) < 1 or len(secondary) < 1:
+        raise ValueError('Must include at least 1 reference and 1 secondary burst')
+    if len(reference) != len(secondary):
+        raise ValueError('Must have the same number of reference and secondary bursts')
+
+    ref_num_swath_pol = sorted(g.split('_')[1] + '_' + g.split('_')[2] + '_' + g.split('_')[4] for g in reference)
+    sec_num_swath_pol = sorted(g.split('_')[1] + '_' + g.split('_')[2] + '_' + g.split('_')[4] for g in secondary)
+    if ref_num_swath_pol != sec_num_swath_pol:
+        msg = 'The reference and secondary burst ID sets do not match.\n'
+        msg += f'    Reference IDs: {ref_num_swath_pol}\n'
+        msg += f'    Secondary IDs: {sec_num_swath_pol}'
+        raise ValueError(msg)
+
+    pols = list(set(g.split('_')[4] for g in reference + secondary))
 
-    ref_burst_id = ref_split[1]
-    sec_burst_id = sec_split[1]
+    if len(pols) > 1:
+        raise ValueError(f'All bursts must have a single polarization. Polarizations present: {" ".join(pols)}')
 
-    ref_polarization = ref_split[4]
-    sec_polarization = sec_split[4]
+    if pols[0] not in ['VV', 'HH']:
+        raise ValueError(f'{pols[0]} polarization is not currently supported, only VV and HH.')
 
-    if ref_burst_id != sec_burst_id:
-        raise ValueError(f'The reference and secondary burst IDs are not the same: {ref_burst_id} and {sec_burst_id}.')
+    ref_dates = list(set(g.split('_')[3][:8] for g in reference))
+    sec_dates = list(set(g.split('_')[3][:8] for g in secondary))
 
-    if ref_polarization != sec_polarization:
-        raise ValueError(
-            f'The reference and secondary polarizations are not the same: {ref_polarization} and {sec_polarization}.'
-        )
+    if len(ref_dates) > 1 or len(sec_dates) > 1:
+        raise ValueError('Reference granules must be from one date and secondary granules must be another.')
 
-    if ref_polarization != 'VV' and ref_polarization != 'HH':
-        raise ValueError(f'{ref_polarization} polarization is not currently supported, only VV and HH.')
+    if ref_dates[0] >= sec_dates[0]:
+        raise ValueError('Reference granules must be older than secondary granules.')
 
 
 def load_burst_position(swath_xml_path: str, burst_number: int) -> BurstPosition:
@@ -532,7 +544,7 @@ def safely_multilook(
     if subset_to_valid:
         last_line = position.first_valid_line + position.n_valid_lines
         last_sample = position.first_valid_sample + position.n_valid_samples
-        mask[position.first_valid_line: last_line, position.first_valid_sample: last_sample] = identity_value
+        mask[position.first_valid_line:last_line, position.first_valid_sample:last_sample] = identity_value
     else:
         mask[:, :] = identity_value
 

src/hyp3_isce2/insar_tops_burst.py

@@ -1,13 +1,15 @@
-"""Create a single-burst Sentinel-1 geocoded unwrapped interferogram using ISCE2's TOPS processing workflow"""
+"""Create a Sentinel-1 geocoded unwrapped interferogram using ISCE2's TOPS processing workflow from a set of bursts"""
 
 import argparse
 import logging
 import sys
+import warnings
 from pathlib import Path
 from shutil import copyfile, make_archive
-from typing import Optional
+from typing import Iterable, Optional
 
 import isce  # noqa
+from burst2safe.burst2safe import burst2safe
 from hyp3lib.util import string_is_true
 from isceobj.TopsProc.runMergeBursts import multilook
 from osgeo import gdal
@@ -23,15 +25,10 @@
     validate_bursts,
 )
 from hyp3_isce2.dem import download_dem_for_isce2
+from hyp3_isce2.insar_tops import insar_tops_packaged
 from hyp3_isce2.logger import configure_root_logger
 from hyp3_isce2.s1_auxcal import download_aux_cal
-from hyp3_isce2.utils import (
-    image_math,
-    isce2_copy,
-    make_browse_image,
-    oldest_granule_first,
-    resample_to_radar_io,
-)
+from hyp3_isce2.utils import image_math, isce2_copy, make_browse_image, resample_to_radar_io
 from hyp3_isce2.water_mask import create_water_mask
 
 
@@ -143,7 +140,6 @@ def insar_tops_single_burst(
     bucket: Optional[str] = None,
     bucket_prefix: str = '',
 ):
-    reference, secondary = oldest_granule_first(reference, secondary)
     validate_bursts(reference, secondary)
     swath_number = int(reference[12])
     range_looks, azimuth_looks = [int(value) for value in looks.split('x')]
@@ -201,44 +197,100 @@ def insar_tops_single_burst(
         packaging.upload_product_to_s3(product_dir, output_zip, bucket, bucket_prefix)
 
 
+def insar_tops_multi_burst(
+    reference: Iterable[str],
+    secondary: Iterable[str],
+    swaths: list = [1, 2, 3],
+    looks: str = '20x4',
+    apply_water_mask=False,
+    bucket: Optional[str] = None,
+    bucket_prefix: str = '',
+):
+    validate_bursts(reference, secondary)
+    reference_safe_path = burst2safe(reference)
+    reference_safe = reference_safe_path.name.split('.')[0]
+    secondary_safe_path = burst2safe(secondary)
+    secondary_safe = secondary_safe_path.name.split('.')[0]
+
+    range_looks, azimuth_looks = [int(value) for value in looks.split('x')]
+    swaths = list(set(int(granule.split('_')[2][2]) for granule in reference))
+    polarization = reference[0].split('_')[4]
+
+    log.info('Begin ISCE2 TopsApp run')
+    insar_tops_packaged(
+        reference=reference_safe,
+        secondary=secondary_safe,
+        swaths=swaths,
+        polarization=polarization,
+        azimuth_looks=azimuth_looks,
+        range_looks=range_looks,
+        apply_water_mask=apply_water_mask,
+        bucket=bucket,
+        bucket_prefix=bucket_prefix,
+    )
+    log.info('ISCE2 TopsApp run completed successfully')
+
+
+def oldest_granule_first(g1, g2):
+    if g1[14:29] <= g2[14:29]:
+        return [g1], [g2]
+    return [g2], [g1]
+
+
 def main():
     """HyP3 entrypoint for the burst TOPS workflow"""
     parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-
-    parser.add_argument('--bucket', help='AWS S3 bucket HyP3 for upload the final product(s)')
-    parser.add_argument('--bucket-prefix', default='', help='Add a bucket prefix to product(s)')
+    parser.add_argument('granules', type=str.split, nargs='*', help='Reference and secondary scene names')
+    parser.add_argument('--reference', type=str.split, nargs='+', help='List of reference scenes"')
+    parser.add_argument('--secondary', type=str.split, nargs='+', help='List of secondary scenes"')
     parser.add_argument(
         '--looks', choices=['20x4', '10x2', '5x1'], default='20x4', help='Number of looks to take in range and azimuth'
     )
     parser.add_argument(
-        '--apply-water-mask',
-        type=string_is_true,
-        default=False,
-        help='Apply a water body mask before unwrapping.',
-    )
-    parser.add_argument(
-        'granules',
-        type=str.split,
-        nargs='+',
-        help='Reference and secondary scene names'
+        '--apply-water-mask', type=string_is_true, default=False, help='Apply a water body mask before unwrapping.'
     )
+    parser.add_argument('--bucket', help='AWS S3 bucket HyP3 for upload the final product(s)')
+    parser.add_argument('--bucket-prefix', default='', help='Add a bucket prefix to product(s)')
 
     args = parser.parse_args()
 
-    granules = [item for sublist in args.granules for item in sublist]
-    if len(granules) != 2:
-        parser.error('No more than two granules may be provided.')
-    if len(granules[0]) != len(granules[1]) != 1:
-        parser.error('Must include 1 reference and 1 secondary.')
+    has_granules = args.granules is not None and len(args.granules) > 0
+    has_ref_sec = args.reference is not None and args.secondary is not None
+    if has_granules and has_ref_sec:
+        parser.error('Provide either --reference and --secondary, or the positional granules argument, not both.')
+    elif not has_granules and not has_ref_sec:
+        parser.error('Either --reference and --secondary, or the positional granules argument, must be provided.')
+    elif has_granules:
+        warnings.warn(
+            'The positional argument for granules is deprecated. Please use --reference and --secondary.',
+            DeprecationWarning,
+        )
+        granules = [item for sublist in args.granules for item in sublist]
+        if len(granules) != 2:
+            parser.error('No more than two granules may be provided.')
+        reference, secondary = oldest_granule_first(granules[0], granules[1])
+    else:
+        reference = [item for sublist in args.reference for item in sublist]
+        secondary = [item for sublist in args.secondary for item in sublist]
 
     configure_root_logger()
     log.debug(' '.join(sys.argv))
 
-    insar_tops_single_burst(
-        reference=granules[0],
-        secondary=granules[1],
-        looks=args.looks,
-        apply_water_mask=args.apply_water_mask,
-        bucket=args.bucket,
-        bucket_prefix=args.bucket_prefix,
-    )
+    if len(reference) == 1:
+        insar_tops_single_burst(
+            reference=reference[0],
+            secondary=secondary[0],
+            looks=args.looks,
+            apply_water_mask=args.apply_water_mask,
+            bucket=args.bucket,
+            bucket_prefix=args.bucket_prefix,
+        )
+    else:
+        insar_tops_multi_burst(
+            reference=reference,
+            secondary=secondary,
+            looks=args.looks,
+            apply_water_mask=args.apply_water_mask,
+            bucket=args.bucket,
+            bucket_prefix=args.bucket_prefix,
+        )