From 7050206277098f9d64a2a0b8ceaa062ceb2ef59e Mon Sep 17 00:00:00 2001 From: mfangaritav Date: Tue, 2 May 2023 00:33:43 +0000 Subject: [PATCH 01/45] Adding stripmapapp function --- environment.yml | 1 + src/hyp3_isce2/insar_stripmap.py | 71 +++++++++++++- src/hyp3_isce2/stripmapapp_alos.py | 145 +++++++++++++++++++++++++++++ 3 files changed, 212 insertions(+), 5 deletions(-) create mode 100644 src/hyp3_isce2/stripmapapp_alos.py diff --git a/environment.yml b/environment.yml index c8e7990a..fddd13b6 100644 --- a/environment.yml +++ b/environment.yml @@ -11,6 +11,7 @@ dependencies: - rasterio - shapely - jinja2 + - asf_search # For packaging, and testing - flake8 - flake8-import-order diff --git a/src/hyp3_isce2/insar_stripmap.py b/src/hyp3_isce2/insar_stripmap.py index 5e91aa29..a1a7b25f 100644 --- a/src/hyp3_isce2/insar_stripmap.py +++ b/src/hyp3_isce2/insar_stripmap.py @@ -9,8 +9,17 @@ import sys from pathlib import Path +from hyp3_isce2 import stripmapapp_alos as stripmapapp + from hyp3lib.aws import upload_file_to_s3 from hyp3lib.image import create_thumbnail +from hyp3_isce2.dem import download_dem_for_isce2 +import zipfile +import glob +import os + +import asf_search as asf +from shapely.geometry.polygon import Polygon log = logging.getLogger(__name__) @@ -22,7 +31,7 @@ os.environ['PATH'] = str(ISCE_APPLICATIONS) + os.pathsep + os.environ['PATH'] -def insar_stripmap(reference_scene: str, secondary_scene: str) -> Path: +def insar_stripmap(user: str, password: str, reference_scene: str, secondary_scene: str) -> Path: """Create an interferogram This is a placeholder function. It will be replaced with your actual scientific workflow. @@ -34,11 +43,58 @@ def insar_stripmap(reference_scene: str, secondary_scene: str) -> Path: Returns: Path to the output files """ + session = asf.ASFSession().auth_with_creds(user, password) + + results = asf.granule_search([reference_scene,secondary_scene]) + + polys=[] + durls=[] + for result in results: + if 'L1.0' in result.properties['url']: + polys.append(Polygon(results[0].geometry['coordinates'][0])) + durls.append(result.properties['url']) + + for i in range(len(polys)): + if i==0: + intersection=polys[i].intersection(polys[i+1]) + else: + intersection=polys[i].intersection(intersection) + + dem_dir = Path('dem') + dem_path = download_dem_for_isce2(intersection.bounds, dem_name='glo_30', dem_dir=dem_dir, buffer=0) + + insar_roi=intersection.bounds + asf.download_urls(urls=durls, path='./', session=session,processes=2) + + zips=glob.glob('*.zip') + for i,zipf in enumerate(sorted(zips[0:2])): + with zipfile.ZipFile(zipf, 'r') as zip_ref: + zip_ref.extractall('./') + + if i==0: + reference_image=glob.glob('./'+zipf.split('.zip')[0]+'/IMG-*')[0] + reference_leader=glob.glob('./'+zipf.split('.zip')[0]+'/LED-*')[0] + else: + secondary_image=glob.glob('./'+zipf.split('.zip')[0]+'/IMG-*')[0] + secondary_leader=glob.glob('./'+zipf.split('.zip')[0]+'/LED-*')[0] + + os.remove(zipf) + + config = stripmapapp.StripmapappConfig( + reference_image=reference_image, + reference_leader=reference_leader, + secondary_image=secondary_image, + secondary_leader=secondary_leader, + roi=insar_roi, + dem_filename=str(dem_path), + ) + config_path = config.write_template('stripmapApp.xml') + + stripmapapp.run_stripmapapp(start='startup', end='geocode', config_xml=config_path) + #raise NotImplementedError('This is a placeholder function. Replace it with your actual scientific workflow.') - raise NotImplementedError('This is a placeholder function. Replace it with your actual scientific workflow.') - - product_file = Path("product_file_name.zip") - return product_file + #product_file = Path("product_file_name.zip") + return Path('interferogram') def main(): @@ -47,6 +103,8 @@ def main(): parser.add_argument('--bucket', type=str, default='', help='AWS S3 bucket HyP3 for upload the final product(s)') parser.add_argument('--bucket-prefix', type=str, default='', help='Add a bucket prefix to product(s)') + parser.add_argument('--username', type=str, required=True) + parser.add_argument('--password', type=str, required=True) parser.add_argument('--reference-scene', type=str, required=True) parser.add_argument('--secondary-scene', type=str, required=True) @@ -56,6 +114,8 @@ def main(): log.debug(' '.join(sys.argv)) product_file = insar_stripmap( + user=args.username, + password=args.password, reference_scene=args.reference_scene, secondary_scene=args.secondary_scene, ) @@ -63,6 +123,7 @@ def main(): log.info('InSAR Stripmap run completed successfully') if args.bucket: + base_name = f'{reference_scene}x{secondary_scene}' upload_file_to_s3(product_file, args.bucket, args.bucket_prefix) browse_images = product_file.with_suffix('.png') for browse in browse_images: diff --git a/src/hyp3_isce2/stripmapapp_alos.py b/src/hyp3_isce2/stripmapapp_alos.py new file mode 100644 index 00000000..7fd9f1da --- /dev/null +++ b/src/hyp3_isce2/stripmapapp_alos.py @@ -0,0 +1,145 @@ +from pathlib import Path +from typing import Iterable, Union + +from isce.applications.stripmapApp import Insar +from jinja2 import Template +from osgeo import gdal + +TEMPLATE_DIR = Path(__file__).parent / 'templates' +STRIPMAPAPP_STEPS = [ + 'startup', + 'preprocess', + 'cropraw', + 'formslc', + 'cropslc', + 'verifyDEM', + 'topo', + 'geo2rdr', + 'coarse_resample', + 'misregistration', + 'refined_resample', + 'dense_offsets', + 'rubber_sheet_range', + 'rubber_sheet_azimuth', + 'fine_resample', + 'split_range_spectrum', + 'sub_band_resample', + 'interferogram', + 'sub_band_interferogram', + 'filter', + 'filter_low_band', + 'filter_high_band', + 'unwrap', + 'unwrap_low_band', + 'unwrap_high_band', + 'ionosphere', + 'geocode', + 'geocodeoffsets', + 'endup' +] +STRIPMAPAPP_GEOCODE_LIST = [ + 'interferogram/phsig.cor', + 'interferogram/filt_topophase.unw', + 'interferogram/los.rdr', + 'interferogram/topophase.flat', + 'interferogram/filt_topophase.flat', + 'interferogram/topophase.cor', + 'interferogram/filt_topophase.unw.conncomp', +] + + +class StripmapappConfig: + """Configuration for a topsApp.py run""" + + def __init__( + self, + reference_image: str, + reference_leader: str, + secondary_image: str, + secondary_leader: str, + roi: Iterable[float], + dem_filename: str, + azimuth_looks: int = 14, + range_looks: int = 4, + do_unwrap: bool = True, + ): + self.reference_image = reference_image + self.reference_leader = reference_leader + self.secondary_image = secondary_image + self.secondary_leader = secondary_leader + self.roi = [roi[1], roi[3], roi[0], roi[2]] + self.dem_filename = dem_filename + self.geocode_dem_filename = dem_filename + self.azimuth_looks = azimuth_looks + self.range_looks = range_looks + + # hardcoded params for topsapp burst processing + self.filter_strength = 0.5 + self.filter_coherence = 0.6 + self.do_unwrap = True + self.use_virtual_files = True + self.geocode_list = STRIPMAPAPP_GEOCODE_LIST + + def generate_template(self) -> str: + """Generate the topsApp.py jinja2 template + + Returns: + The rendered template + """ + with open(TEMPLATE_DIR / 'stripmapapp_alos.xml', 'r') as file: + template = Template(file.read()) + return template.render(self.__dict__) + + def write_template(self, filename: Union[str, Path] = 'stripmapApp.xml') -> Path: + """Write the topsApp.py jinja2 template to a file + + Args: + filename: Filename to write the template to + Returns: + The path of the written template + """ + if not isinstance(filename, Path): + filename = Path(filename) + + with open(filename, 'w') as file: + file.write(self.generate_template()) + + return filename + +def run_stripmapapp(dostep: str = '', start: str = '', end: str = '', config_xml: Path = Path('stripmapApp.xml')): + """Run topsApp.py for a burst pair with the desired steps and config file + + Args: + dostep: The step to run + start: The step to start at + stop: The step to stop at + config_xml: The config file to use + + Raises: + ValueError: If dostep is specified, start and stop cannot be used + IOError: If the config file does not exist + ValueError: If the step is not a valid step (see TOPSAPP_STEPS) + """ + if not config_xml.exists(): + raise IOError(f'The config file {config_xml} does not exist!') + + if dostep and (start or end): + raise ValueError('If dostep is specified, start and stop cannot be used') + + step_args = [] + options = { + 'dostep': dostep, + 'start': start, + 'end': end, + } + for key, value in options.items(): + if not value: + continue + if value not in STRIPMAPAPP_STEPS: + raise ValueError(f'{value} is not a valid step') + step_args.append(f'--{key}={value}') + + cmd_line = [str(config_xml)] + step_args + insar = Insar(name='stripmapApp', cmdline=cmd_line) + insar.configure() + insar.run() From dbf0fb899b8fbfb5bd5b8cc182ca27addf25403d Mon Sep 17 00:00:00 2001 From: mfangaritav Date: Thu, 4 May 2023 23:49:07 +0000 Subject: [PATCH 02/45] Adding xml template for stripmapapp --- src/hyp3_isce2/templates/stripmapapp_alos.xml | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 src/hyp3_isce2/templates/stripmapapp_alos.xml diff --git a/src/hyp3_isce2/templates/stripmapapp_alos.xml b/src/hyp3_isce2/templates/stripmapapp_alos.xml new file mode 100644 index 00000000..9b8f6868 --- /dev/null +++ b/src/hyp3_isce2/templates/stripmapapp_alos.xml @@ -0,0 +1,43 @@ + + + + ALOS + + {{ reference_image }} + {{ reference_leader }} + reference + + + {{ secondary_image }} + {{ secondary_leader }} + secondary + + + {{ dem_filename }} + + + {{ filter_strength }} + {{ filter_coherence }} + {{ roi }} + {{ range_looks }} + {{ azimuth_looks }} + True + True + {{ do_unwrap }} + snaphu + True + + + \ No newline at end of file From 51e7ad695e1d795b69be572c9469cfd4887df437 Mon Sep 17 00:00:00 2001 From: mfangaritav Date: Fri, 12 May 2023 00:59:57 +0000 Subject: [PATCH 03/45] Changes in the stripmap function --- environment.yml | 1 + src/hyp3_isce2/insar_stripmap.py | 12 +++++++++--- src/hyp3_isce2/stripmapapp_alos.py | 3 +++ src/hyp3_isce2/templates/stripmapapp_alos.xml | 10 +++++----- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/environment.yml b/environment.yml index fddd13b6..cb6b2124 100644 --- a/environment.yml +++ b/environment.yml @@ -12,6 +12,7 @@ dependencies: - shapely - jinja2 - asf_search + - opencv # For packaging, and testing - flake8 - flake8-import-order diff --git a/src/hyp3_isce2/insar_stripmap.py b/src/hyp3_isce2/insar_stripmap.py index a1a7b25f..e221bcc8 100644 --- a/src/hyp3_isce2/insar_stripmap.py +++ b/src/hyp3_isce2/insar_stripmap.py @@ -91,6 +91,8 @@ def insar_stripmap(user: str, password: str, reference_scene: str, secondary_sce config_path = config.write_template('stripmapApp.xml') stripmapapp.run_stripmapapp(start='startup', end='geocode', config_xml=config_path) + #stripmapapp.run_stripmapapp(start='rubber_sheet_range', end='geocode', config_xml=config_path) + #stripmapapp.run_stripmapapp(start='startup', end='geocode', config_xml=config_path) #raise NotImplementedError('This is a placeholder function. Replace it with your actual scientific workflow.') #product_file = Path("product_file_name.zip") @@ -98,11 +100,13 @@ def insar_stripmap(user: str, password: str, reference_scene: str, secondary_sce def main(): + """ Entrypoint for the stripmap workflow""" + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument('--bucket', type=str, default='', help='AWS S3 bucket HyP3 for upload the final product(s)') - parser.add_argument('--bucket-prefix', type=str, default='', help='Add a bucket prefix to product(s)') + parser.add_argument('--bucket', help='AWS S3 bucket HyP3 for upload the final product(s)') + parser.add_argument('--bucket-prefix', default='', help='Add a bucket prefix to product(s)') parser.add_argument('--username', type=str, required=True) parser.add_argument('--password', type=str, required=True) parser.add_argument('--reference-scene', type=str, required=True) @@ -113,13 +117,14 @@ def main(): logging.basicConfig(stream=sys.stdout, format='%(asctime)s - %(levelname)s - %(message)s', level=logging.INFO) log.debug(' '.join(sys.argv)) + product_file = insar_stripmap( user=args.username, password=args.password, reference_scene=args.reference_scene, secondary_scene=args.secondary_scene, ) - + log.info('InSAR Stripmap run completed successfully') if args.bucket: @@ -130,3 +135,4 @@ def main(): thumbnail = create_thumbnail(browse) upload_file_to_s3(browse, args.bucket, args.bucket_prefix) upload_file_to_s3(thumbnail, args.bucket, args.bucket_prefix) + \ No newline at end of file diff --git a/src/hyp3_isce2/stripmapapp_alos.py b/src/hyp3_isce2/stripmapapp_alos.py index 7fd9f1da..b35571e1 100644 --- a/src/hyp3_isce2/stripmapapp_alos.py +++ b/src/hyp3_isce2/stripmapapp_alos.py @@ -4,6 +4,9 @@ from isce.applications.stripmapApp import Insar from jinja2 import Template from osgeo import gdal +import os + +os.environ["PATH"] += os.pathsep+':'+os.environ['ISCE_HOME']+'/applications' TEMPLATE_DIR = Path(__file__).parent / 'templates' STRIPMAPAPP_STEPS = [ diff --git a/src/hyp3_isce2/templates/stripmapapp_alos.xml b/src/hyp3_isce2/templates/stripmapapp_alos.xml index 9b8f6868..8f87840b 100644 --- a/src/hyp3_isce2/templates/stripmapapp_alos.xml +++ b/src/hyp3_isce2/templates/stripmapapp_alos.xml @@ -15,16 +15,16 @@ {{ dem_filename }} - + {{ filter_strength }} {{ filter_coherence }} {{ roi }} {{ range_looks }} {{ azimuth_looks }} - True + False True {{ do_unwrap }} snaphu @@ -40,4 +40,4 @@ USER_INPUT --> - \ No newline at end of file + From 5a7f7582d00e9b029a7a8c5fcd0d3fb97c51eeb1 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Mon, 26 Jun 2023 15:58:32 -0800 Subject: [PATCH 04/45] remove unused import, fix some whitespace issues --- src/hyp3_isce2/insar_stripmap.py | 57 +++++++++++++++----------------- 1 file changed, 27 insertions(+), 30 deletions(-) diff --git a/src/hyp3_isce2/insar_stripmap.py b/src/hyp3_isce2/insar_stripmap.py index 83a1d5fc..9d066a04 100644 --- a/src/hyp3_isce2/insar_stripmap.py +++ b/src/hyp3_isce2/insar_stripmap.py @@ -4,7 +4,6 @@ import argparse import logging -import os import site import sys from pathlib import Path @@ -46,42 +45,42 @@ def insar_stripmap(user: str, password: str, reference_scene: str, secondary_sce Path to the output files """ session = asf.ASFSession().auth_with_creds(user, password) - - results = asf.granule_search([reference_scene,secondary_scene]) - - polys=[] - durls=[] + + results = asf.granule_search([reference_scene, secondary_scene]) + + polys = [] + durls = [] for result in results: if 'L1.0' in result.properties['url']: polys.append(Polygon(results[0].geometry['coordinates'][0])) durls.append(result.properties['url']) - + for i in range(len(polys)): - if i==0: - intersection=polys[i].intersection(polys[i+1]) + if i == 0: + intersection = polys[i].intersection(polys[i+1]) else: - intersection=polys[i].intersection(intersection) - + intersection = polys[i].intersection(intersection) + dem_dir = Path('dem') dem_path = download_dem_for_isce2(intersection.bounds, dem_name='glo_30', dem_dir=dem_dir, buffer=0) - - insar_roi=intersection.bounds - asf.download_urls(urls=durls, path='./', session=session,processes=2) - - zips=glob.glob('*.zip') - for i,zipf in enumerate(sorted(zips[0:2])): + + insar_roi = intersection.bounds + asf.download_urls(urls=durls, path='./', session=session, processes=2) + + zips = glob.glob('*.zip') + for i, zipf in enumerate(sorted(zips[0:2])): with zipfile.ZipFile(zipf, 'r') as zip_ref: zip_ref.extractall('./') - - if i==0: - reference_image=glob.glob('./'+zipf.split('.zip')[0]+'/IMG-*')[0] - reference_leader=glob.glob('./'+zipf.split('.zip')[0]+'/LED-*')[0] + + if i == 0: + reference_image = glob.glob('./' + zipf.split('.zip')[0] + '/IMG-*')[0] + reference_leader = glob.glob('./' + zipf.split('.zip')[0] + '/LED-*')[0] else: - secondary_image=glob.glob('./'+zipf.split('.zip')[0]+'/IMG-*')[0] - secondary_leader=glob.glob('./'+zipf.split('.zip')[0]+'/LED-*')[0] - + secondary_image = glob.glob('./' + zipf.split('.zip')[0] + '/IMG-*')[0] + secondary_leader = glob.glob('./' + zipf.split('.zip')[0] + '/LED-*')[0] + os.remove(zipf) - + config = stripmapapp.StripmapappConfig( reference_image=reference_image, reference_leader=reference_leader, @@ -91,7 +90,7 @@ def insar_stripmap(user: str, password: str, reference_scene: str, secondary_sce dem_filename=str(dem_path), ) config_path = config.write_template('stripmapApp.xml') - + stripmapapp.run_stripmapapp(start='startup', end='geocode', config_xml=config_path) #stripmapapp.run_stripmapapp(start='rubber_sheet_range', end='geocode', config_xml=config_path) #stripmapapp.run_stripmapapp(start='startup', end='geocode', config_xml=config_path) @@ -102,9 +101,8 @@ def insar_stripmap(user: str, password: str, reference_scene: str, secondary_sce def main(): - """ Entrypoint for the stripmap workflow""" - + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--bucket', help='AWS S3 bucket HyP3 for upload the final product(s)') @@ -127,7 +125,7 @@ def main(): reference_scene=args.reference_scene, secondary_scene=args.secondary_scene, ) - + log.info('InSAR Stripmap run completed successfully') if args.bucket: @@ -138,4 +136,3 @@ def main(): thumbnail = create_thumbnail(browse) upload_file_to_s3(browse, args.bucket, args.bucket_prefix) upload_file_to_s3(thumbnail, args.bucket, args.bucket_prefix) - \ No newline at end of file From 1ef925a9e04e55a55ff2ba17ccf133dfc7a33943 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Mon, 26 Jun 2023 16:10:14 -0800 Subject: [PATCH 05/45] fix miscellaneous issues --- src/hyp3_isce2/insar_stripmap.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/hyp3_isce2/insar_stripmap.py b/src/hyp3_isce2/insar_stripmap.py index 9d066a04..07d25379 100644 --- a/src/hyp3_isce2/insar_stripmap.py +++ b/src/hyp3_isce2/insar_stripmap.py @@ -38,6 +38,8 @@ def insar_stripmap(user: str, password: str, reference_scene: str, secondary_sce This is a placeholder function. It will be replaced with your actual scientific workflow. Args: + user: Earthdata username + password: Earthdata password reference_scene: Reference scene name secondary_scene: Secondary scene name @@ -55,11 +57,9 @@ def insar_stripmap(user: str, password: str, reference_scene: str, secondary_sce polys.append(Polygon(results[0].geometry['coordinates'][0])) durls.append(result.properties['url']) - for i in range(len(polys)): - if i == 0: - intersection = polys[i].intersection(polys[i+1]) - else: - intersection = polys[i].intersection(intersection) + intersection = polys[0].intersection(polys[1]) + for i in range(1, len(polys)): + intersection = polys[i].intersection(intersection) dem_dir = Path('dem') dem_path = download_dem_for_isce2(intersection.bounds, dem_name='glo_30', dem_dir=dem_dir, buffer=0) @@ -129,7 +129,7 @@ def main(): log.info('InSAR Stripmap run completed successfully') if args.bucket: - base_name = f'{reference_scene}x{secondary_scene}' + base_name = f'{args.reference_scene}x{args.secondary_scene}' upload_file_to_s3(product_file, args.bucket, args.bucket_prefix) browse_images = product_file.with_suffix('.png') for browse in browse_images: From edeb17f41aa1c1511ef9518271a51323044bd112 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Tue, 27 Jun 2023 10:51:38 -0800 Subject: [PATCH 06/45] misc cleanup for stripmapapp_alos.py --- src/hyp3_isce2/stripmapapp_alos.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/hyp3_isce2/stripmapapp_alos.py b/src/hyp3_isce2/stripmapapp_alos.py index b35571e1..b859e248 100644 --- a/src/hyp3_isce2/stripmapapp_alos.py +++ b/src/hyp3_isce2/stripmapapp_alos.py @@ -1,14 +1,14 @@ from pathlib import Path -from typing import Iterable, Union +from typing import Union from isce.applications.stripmapApp import Insar from jinja2 import Template -from osgeo import gdal import os -os.environ["PATH"] += os.pathsep+':'+os.environ['ISCE_HOME']+'/applications' +os.environ['PATH'] += os.pathsep + str(Path(os.environ['ISCE_HOME']) / 'applications') TEMPLATE_DIR = Path(__file__).parent / 'templates' + STRIPMAPAPP_STEPS = [ 'startup', 'preprocess', @@ -40,6 +40,7 @@ 'geocodeoffsets', 'endup' ] + STRIPMAPAPP_GEOCODE_LIST = [ 'interferogram/phsig.cor', 'interferogram/filt_topophase.unw', @@ -60,11 +61,10 @@ def __init__( reference_leader: str, secondary_image: str, secondary_leader: str, - roi: Iterable[float], + roi: list[float], dem_filename: str, azimuth_looks: int = 14, range_looks: int = 4, - do_unwrap: bool = True, ): self.reference_image = reference_image self.reference_leader = reference_leader @@ -109,13 +109,14 @@ def write_template(self, filename: Union[str, Path] = 'stripmapApp.xml') -> Path return filename + def run_stripmapapp(dostep: str = '', start: str = '', end: str = '', config_xml: Path = Path('stripmapApp.xml')): """Run topsApp.py for a burst pair with the desired steps and config file Args: dostep: The step to run start: The step to start at - stop: The step to stop at + end: The step to stop at config_xml: The config file to use Raises: From 788d8fe74eca3179ddc5cafe95a94ea4b23ac8d7 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Tue, 27 Jun 2023 10:55:36 -0800 Subject: [PATCH 07/45] reformat xml file --- src/hyp3_isce2/templates/stripmapapp_alos.xml | 78 +++++++++---------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/src/hyp3_isce2/templates/stripmapapp_alos.xml b/src/hyp3_isce2/templates/stripmapapp_alos.xml index 8f87840b..8ecddf1a 100644 --- a/src/hyp3_isce2/templates/stripmapapp_alos.xml +++ b/src/hyp3_isce2/templates/stripmapapp_alos.xml @@ -1,43 +1,43 @@ - - ALOS - - {{ reference_image }} - {{ reference_leader }} - reference + + ALOS + + {{ reference_image }} + {{ reference_leader }} + reference + + + {{ secondary_image }} + {{ secondary_leader }} + secondary + + + {{ dem_filename }} + + + False + False + + {{ filter_strength }} + {{ filter_coherence }} + {{ roi }} + {{ range_looks }} + {{ azimuth_looks }} + False + True + {{ do_unwrap }} + snaphu + True + - - {{ secondary_image }} - {{ secondary_leader }} - secondary - - - {{ dem_filename }} - - - False - False - - {{ filter_strength }} - {{ filter_coherence }} - {{ roi }} - {{ range_looks }} - {{ azimuth_looks }} - False - True - {{ do_unwrap }} - snaphu - True - - From 9fab8c5e4f87db6026959ea7ab9ad5d433729971 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Wed, 28 Jun 2023 11:12:03 -0800 Subject: [PATCH 08/45] revert previous refactor --- src/hyp3_isce2/insar_stripmap.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/hyp3_isce2/insar_stripmap.py b/src/hyp3_isce2/insar_stripmap.py index 07d25379..779f1ffd 100644 --- a/src/hyp3_isce2/insar_stripmap.py +++ b/src/hyp3_isce2/insar_stripmap.py @@ -57,9 +57,11 @@ def insar_stripmap(user: str, password: str, reference_scene: str, secondary_sce polys.append(Polygon(results[0].geometry['coordinates'][0])) durls.append(result.properties['url']) - intersection = polys[0].intersection(polys[1]) - for i in range(1, len(polys)): - intersection = polys[i].intersection(intersection) + for i in range(len(polys)): + if i == 0: + intersection = polys[i].intersection(polys[i + 1]) + else: + intersection = polys[i].intersection(intersection) dem_dir = Path('dem') dem_path = download_dem_for_isce2(intersection.bounds, dem_name='glo_30', dem_dir=dem_dir, buffer=0) From a9f23b2dfbc6d9cde0998ddbbaf40f6876246867 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Wed, 28 Jun 2023 11:16:23 -0800 Subject: [PATCH 09/45] add a TODO --- src/hyp3_isce2/insar_stripmap.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hyp3_isce2/insar_stripmap.py b/src/hyp3_isce2/insar_stripmap.py index 779f1ffd..ac32eaa5 100644 --- a/src/hyp3_isce2/insar_stripmap.py +++ b/src/hyp3_isce2/insar_stripmap.py @@ -32,6 +32,7 @@ os.environ['PATH'] = str(ISCE_APPLICATIONS) + os.pathsep + os.environ['PATH'] +# TODO update docstring description def insar_stripmap(user: str, password: str, reference_scene: str, secondary_scene: str) -> Path: """Create an interferogram From b9e6048097bc5d779f802efbb39d494e3f9d5304 Mon Sep 17 00:00:00 2001 From: mfangaritav Date: Thu, 6 Jul 2023 13:52:50 -0800 Subject: [PATCH 10/45] Update src/hyp3_isce2/insar_stripmap.py Co-authored-by: Jake Herrmann --- src/hyp3_isce2/insar_stripmap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hyp3_isce2/insar_stripmap.py b/src/hyp3_isce2/insar_stripmap.py index ac32eaa5..d753ef21 100644 --- a/src/hyp3_isce2/insar_stripmap.py +++ b/src/hyp3_isce2/insar_stripmap.py @@ -17,7 +17,7 @@ import glob import os -import asf_search as asf +import asf_search from shapely.geometry.polygon import Polygon from hyp3_isce2.logging import configure_root_logger From 938991b1430f5e442423d3bc86574c4357582970 Mon Sep 17 00:00:00 2001 From: mfangaritav Date: Thu, 6 Jul 2023 13:53:04 -0800 Subject: [PATCH 11/45] Update src/hyp3_isce2/insar_stripmap.py Co-authored-by: Jake Herrmann --- src/hyp3_isce2/insar_stripmap.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/hyp3_isce2/insar_stripmap.py b/src/hyp3_isce2/insar_stripmap.py index d753ef21..b03b1878 100644 --- a/src/hyp3_isce2/insar_stripmap.py +++ b/src/hyp3_isce2/insar_stripmap.py @@ -49,7 +49,11 @@ def insar_stripmap(user: str, password: str, reference_scene: str, secondary_sce """ session = asf.ASFSession().auth_with_creds(user, password) - results = asf.granule_search([reference_scene, secondary_scene]) + results = asf_search.search( + granule_list=[reference_scene, secondary_scene], + processingLevel=asf_search.L1_0, + ) + assert len(results) == 2 polys = [] durls = [] From 0a80b216fae530444dab767b26c221ee3edf1551 Mon Sep 17 00:00:00 2001 From: mfangaritav Date: Thu, 6 Jul 2023 13:53:11 -0800 Subject: [PATCH 12/45] Update src/hyp3_isce2/insar_stripmap.py Co-authored-by: Jake Herrmann --- src/hyp3_isce2/insar_stripmap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hyp3_isce2/insar_stripmap.py b/src/hyp3_isce2/insar_stripmap.py index b03b1878..3f25fd22 100644 --- a/src/hyp3_isce2/insar_stripmap.py +++ b/src/hyp3_isce2/insar_stripmap.py @@ -47,7 +47,7 @@ def insar_stripmap(user: str, password: str, reference_scene: str, secondary_sce Returns: Path to the output files """ - session = asf.ASFSession().auth_with_creds(user, password) + session = asf_search.ASFSession().auth_with_creds(user, password) results = asf_search.search( granule_list=[reference_scene, secondary_scene], From a227cde5d7ec4fb5ec2b6d37f4953282c989a9f9 Mon Sep 17 00:00:00 2001 From: mfangaritav Date: Thu, 6 Jul 2023 13:53:21 -0800 Subject: [PATCH 13/45] Update src/hyp3_isce2/insar_stripmap.py Co-authored-by: Jake Herrmann --- src/hyp3_isce2/insar_stripmap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hyp3_isce2/insar_stripmap.py b/src/hyp3_isce2/insar_stripmap.py index 3f25fd22..80adb764 100644 --- a/src/hyp3_isce2/insar_stripmap.py +++ b/src/hyp3_isce2/insar_stripmap.py @@ -72,7 +72,7 @@ def insar_stripmap(user: str, password: str, reference_scene: str, secondary_sce dem_path = download_dem_for_isce2(intersection.bounds, dem_name='glo_30', dem_dir=dem_dir, buffer=0) insar_roi = intersection.bounds - asf.download_urls(urls=durls, path='./', session=session, processes=2) + asf_search.download_urls(urls=durls, path='./', session=session, processes=2) zips = glob.glob('*.zip') for i, zipf in enumerate(sorted(zips[0:2])): From 2b733a652e327962139b09b5c349e590f6652e3f Mon Sep 17 00:00:00 2001 From: mfangaritav Date: Thu, 6 Jul 2023 13:55:06 -0800 Subject: [PATCH 14/45] Update src/hyp3_isce2/insar_stripmap.py Co-authored-by: Jake Herrmann --- src/hyp3_isce2/insar_stripmap.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/hyp3_isce2/insar_stripmap.py b/src/hyp3_isce2/insar_stripmap.py index 80adb764..a98e4864 100644 --- a/src/hyp3_isce2/insar_stripmap.py +++ b/src/hyp3_isce2/insar_stripmap.py @@ -58,9 +58,9 @@ def insar_stripmap(user: str, password: str, reference_scene: str, secondary_sce polys = [] durls = [] for result in results: - if 'L1.0' in result.properties['url']: - polys.append(Polygon(results[0].geometry['coordinates'][0])) - durls.append(result.properties['url']) + # TODO should this reference `result` rather than `results[0]`? + polys.append(Polygon(results[0].geometry['coordinates'][0])) + durls.append(result.properties['url']) for i in range(len(polys)): if i == 0: From 7604798a900fd551641bf9bee06c96aea40cd41f Mon Sep 17 00:00:00 2001 From: mfangaritav Date: Thu, 6 Jul 2023 13:57:07 -0800 Subject: [PATCH 15/45] Update insar_stripmap.py --- src/hyp3_isce2/insar_stripmap.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/hyp3_isce2/insar_stripmap.py b/src/hyp3_isce2/insar_stripmap.py index a98e4864..bdb9a859 100644 --- a/src/hyp3_isce2/insar_stripmap.py +++ b/src/hyp3_isce2/insar_stripmap.py @@ -58,9 +58,8 @@ def insar_stripmap(user: str, password: str, reference_scene: str, secondary_sce polys = [] durls = [] for result in results: - # TODO should this reference `result` rather than `results[0]`? - polys.append(Polygon(results[0].geometry['coordinates'][0])) - durls.append(result.properties['url']) + polys.append(Polygon(result.geometry['coordinates'][0])) + durls.append(result.properties['url']) for i in range(len(polys)): if i == 0: From c101f27a346887c930c8e04b618b653c72969f0e Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Fri, 7 Jul 2023 14:01:20 -0800 Subject: [PATCH 16/45] Finish refactoring `insar_stripmap` function --- src/hyp3_isce2/insar_stripmap.py | 60 +++++++++++++++----------------- 1 file changed, 28 insertions(+), 32 deletions(-) diff --git a/src/hyp3_isce2/insar_stripmap.py b/src/hyp3_isce2/insar_stripmap.py index bdb9a859..fc9739e3 100644 --- a/src/hyp3_isce2/insar_stripmap.py +++ b/src/hyp3_isce2/insar_stripmap.py @@ -49,43 +49,33 @@ def insar_stripmap(user: str, password: str, reference_scene: str, secondary_sce """ session = asf_search.ASFSession().auth_with_creds(user, password) - results = asf_search.search( + reference_product, secondary_product = asf_search.search( granule_list=[reference_scene, secondary_scene], processingLevel=asf_search.L1_0, ) - assert len(results) == 2 + assert reference_product.properties['sceneName'] == reference_scene + assert secondary_product.properties['sceneName'] == secondary_scene + products = (reference_product, secondary_product) - polys = [] - durls = [] - for result in results: - polys.append(Polygon(result.geometry['coordinates'][0])) - durls.append(result.properties['url']) + polygons = [Polygon(product.geometry['coordinates'][0]) for product in products] + insar_roi = polygons[0].intersection(polygons[1]).bounds - for i in range(len(polys)): - if i == 0: - intersection = polys[i].intersection(polys[i + 1]) - else: - intersection = polys[i].intersection(intersection) + dem_path = download_dem_for_isce2(insar_roi, dem_name='glo_30', dem_dir=Path('dem'), buffer=0) - dem_dir = Path('dem') - dem_path = download_dem_for_isce2(intersection.bounds, dem_name='glo_30', dem_dir=dem_dir, buffer=0) + urls = [product.properties['url'] for product in products] + asf_search.download_urls(urls=urls, path=os.getcwd(), session=session, processes=2) - insar_roi = intersection.bounds - asf_search.download_urls(urls=durls, path='./', session=session, processes=2) + zip_paths = [product.properties['fileName'] for product in products] + for zip_path in zip_paths: + with zipfile.ZipFile(zip_path, 'r') as zip_file: + zip_file.extractall() + os.remove(zip_path) - zips = glob.glob('*.zip') - for i, zipf in enumerate(sorted(zips[0:2])): - with zipfile.ZipFile(zipf, 'r') as zip_ref: - zip_ref.extractall('./') + reference_image = get_product_file(reference_product, 'IMG-') + reference_leader = get_product_file(reference_product, 'LED-') - if i == 0: - reference_image = glob.glob('./' + zipf.split('.zip')[0] + '/IMG-*')[0] - reference_leader = glob.glob('./' + zipf.split('.zip')[0] + '/LED-*')[0] - else: - secondary_image = glob.glob('./' + zipf.split('.zip')[0] + '/IMG-*')[0] - secondary_leader = glob.glob('./' + zipf.split('.zip')[0] + '/LED-*')[0] - - os.remove(zipf) + secondary_image = get_product_file(secondary_product, 'IMG-') + secondary_leader = get_product_file(secondary_product, 'LED-') config = stripmapapp.StripmapappConfig( reference_image=reference_image, @@ -98,14 +88,19 @@ def insar_stripmap(user: str, password: str, reference_scene: str, secondary_sce config_path = config.write_template('stripmapApp.xml') stripmapapp.run_stripmapapp(start='startup', end='geocode', config_xml=config_path) + + # TODO is this still needed? #stripmapapp.run_stripmapapp(start='rubber_sheet_range', end='geocode', config_xml=config_path) - #stripmapapp.run_stripmapapp(start='startup', end='geocode', config_xml=config_path) - #raise NotImplementedError('This is a placeholder function. Replace it with your actual scientific workflow.') - #product_file = Path("product_file_name.zip") return Path('interferogram') +def get_product_file(product: asf_search.ASFProduct, file_prefix: str) -> str: + paths = glob.glob(str(Path(product.properties['fileID']) / f'{file_prefix}*')) + assert len(paths) == 1 + return paths[0] + + def main(): """ Entrypoint for the stripmap workflow""" @@ -135,8 +130,9 @@ def main(): log.info('InSAR Stripmap run completed successfully') if args.bucket: - base_name = f'{args.reference_scene}x{args.secondary_scene}' upload_file_to_s3(product_file, args.bucket, args.bucket_prefix) + + # FIXME browse_images is not a list browse_images = product_file.with_suffix('.png') for browse in browse_images: thumbnail = create_thumbnail(browse) From 16f4a172e8e545f8978b2edf77a500c34f45e570 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Fri, 7 Jul 2023 14:20:53 -0800 Subject: [PATCH 17/45] fix uploading --- src/hyp3_isce2/insar_stripmap.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/hyp3_isce2/insar_stripmap.py b/src/hyp3_isce2/insar_stripmap.py index fc9739e3..81ae455e 100644 --- a/src/hyp3_isce2/insar_stripmap.py +++ b/src/hyp3_isce2/insar_stripmap.py @@ -7,11 +7,11 @@ import site import sys from pathlib import Path +from shutil import make_archive from hyp3_isce2 import stripmapapp_alos as stripmapapp from hyp3lib.aws import upload_file_to_s3 -from hyp3lib.image import create_thumbnail from hyp3_isce2.dem import download_dem_for_isce2 import zipfile import glob @@ -120,7 +120,7 @@ def main(): log.info('Begin InSAR Stripmap run') - product_file = insar_stripmap( + product_dir = insar_stripmap( user=args.username, password=args.password, reference_scene=args.reference_scene, @@ -129,12 +129,13 @@ def main(): log.info('InSAR Stripmap run completed successfully') + # TODO is this our desired product name? + product_name = f'{args.reference_scene}x{args.secondary_scene}' + output_zip = make_archive(base_name=product_name, format='zip', base_dir=product_dir) + if args.bucket: - upload_file_to_s3(product_file, args.bucket, args.bucket_prefix) - - # FIXME browse_images is not a list - browse_images = product_file.with_suffix('.png') - for browse in browse_images: - thumbnail = create_thumbnail(browse) - upload_file_to_s3(browse, args.bucket, args.bucket_prefix) - upload_file_to_s3(thumbnail, args.bucket, args.bucket_prefix) + # TODO do we want browse images? + + upload_file_to_s3(Path(output_zip), args.bucket, args.bucket_prefix) + + # TODO upload individual files to S3? From 7c07fd3b190c4fa3fe77a7338c2ef196d8bb141d Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Fri, 7 Jul 2023 14:23:47 -0800 Subject: [PATCH 18/45] remove a todo --- src/hyp3_isce2/insar_stripmap.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/hyp3_isce2/insar_stripmap.py b/src/hyp3_isce2/insar_stripmap.py index 81ae455e..9a335ea6 100644 --- a/src/hyp3_isce2/insar_stripmap.py +++ b/src/hyp3_isce2/insar_stripmap.py @@ -129,7 +129,6 @@ def main(): log.info('InSAR Stripmap run completed successfully') - # TODO is this our desired product name? product_name = f'{args.reference_scene}x{args.secondary_scene}' output_zip = make_archive(base_name=product_name, format='zip', base_dir=product_dir) From 111c1463eda788533f8ccc446416f81df241a4a9 Mon Sep 17 00:00:00 2001 From: mfangaritav Date: Fri, 7 Jul 2023 18:14:56 -0800 Subject: [PATCH 19/45] Update src/hyp3_isce2/insar_stripmap.py Co-authored-by: Jake Herrmann --- src/hyp3_isce2/insar_stripmap.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/hyp3_isce2/insar_stripmap.py b/src/hyp3_isce2/insar_stripmap.py index 9a335ea6..8aec4f04 100644 --- a/src/hyp3_isce2/insar_stripmap.py +++ b/src/hyp3_isce2/insar_stripmap.py @@ -89,9 +89,6 @@ def insar_stripmap(user: str, password: str, reference_scene: str, secondary_sce stripmapapp.run_stripmapapp(start='startup', end='geocode', config_xml=config_path) - # TODO is this still needed? - #stripmapapp.run_stripmapapp(start='rubber_sheet_range', end='geocode', config_xml=config_path) - return Path('interferogram') From 61ec31ae1cca498c616f331106c8b31d58d65100 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Tue, 11 Jul 2023 14:26:55 -0800 Subject: [PATCH 20/45] Include only the important files in output zip --- src/hyp3_isce2/insar_stripmap.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/hyp3_isce2/insar_stripmap.py b/src/hyp3_isce2/insar_stripmap.py index 8aec4f04..65e6b9c8 100644 --- a/src/hyp3_isce2/insar_stripmap.py +++ b/src/hyp3_isce2/insar_stripmap.py @@ -4,6 +4,7 @@ import argparse import logging +import shutil import site import sys from pathlib import Path @@ -89,7 +90,18 @@ def insar_stripmap(user: str, password: str, reference_scene: str, secondary_sce stripmapapp.run_stripmapapp(start='startup', end='geocode', config_xml=config_path) - return Path('interferogram') + product_dir = Path(f'{reference_scene}x{secondary_scene}') + (product_dir / 'interferogram').mkdir(parents=True) + + for filename in os.listdir('interferogram'): + path = Path('interferogram') / filename + if os.path.isfile(path): + shutil.move(path, product_dir / path) + + shutil.move('geometry', product_dir) + shutil.move('ionosphere', product_dir) + + return product_dir def get_product_file(product: asf_search.ASFProduct, file_prefix: str) -> str: @@ -126,8 +138,7 @@ def main(): log.info('InSAR Stripmap run completed successfully') - product_name = f'{args.reference_scene}x{args.secondary_scene}' - output_zip = make_archive(base_name=product_name, format='zip', base_dir=product_dir) + output_zip = make_archive(base_name=product_dir.name, format='zip', base_dir=product_dir) if args.bucket: # TODO do we want browse images? From 929f007f0d48d81b743bc3624f44e28b964e39e9 Mon Sep 17 00:00:00 2001 From: mfangaritav Date: Wed, 12 Jul 2023 11:04:17 -0800 Subject: [PATCH 21/45] Update src/hyp3_isce2/insar_stripmap.py Co-authored-by: Jake Herrmann --- src/hyp3_isce2/insar_stripmap.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/hyp3_isce2/insar_stripmap.py b/src/hyp3_isce2/insar_stripmap.py index 65e6b9c8..786d8a4b 100644 --- a/src/hyp3_isce2/insar_stripmap.py +++ b/src/hyp3_isce2/insar_stripmap.py @@ -33,11 +33,8 @@ os.environ['PATH'] = str(ISCE_APPLICATIONS) + os.pathsep + os.environ['PATH'] -# TODO update docstring description def insar_stripmap(user: str, password: str, reference_scene: str, secondary_scene: str) -> Path: - """Create an interferogram - - This is a placeholder function. It will be replaced with your actual scientific workflow. + """Create a Stripmap interferogram Args: user: Earthdata username From e7d4cab79ad58d99ed154aba313bb0a4f24e9adf Mon Sep 17 00:00:00 2001 From: mfangaritav Date: Wed, 12 Jul 2023 11:04:31 -0800 Subject: [PATCH 22/45] Update src/hyp3_isce2/stripmapapp_alos.py Co-authored-by: Jake Herrmann --- src/hyp3_isce2/stripmapapp_alos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hyp3_isce2/stripmapapp_alos.py b/src/hyp3_isce2/stripmapapp_alos.py index b859e248..012a05c5 100644 --- a/src/hyp3_isce2/stripmapapp_alos.py +++ b/src/hyp3_isce2/stripmapapp_alos.py @@ -76,7 +76,7 @@ def __init__( self.azimuth_looks = azimuth_looks self.range_looks = range_looks - # hardcoded params for topsapp burst processing + # hardcoded params for topsapp stripmap processing self.filter_strength = 0.5 self.filter_coherence = 0.6 self.do_unwrap = True From a4e91da275167710246442de03b56a2a5d63c9f8 Mon Sep 17 00:00:00 2001 From: mfangaritav Date: Wed, 12 Jul 2023 11:08:25 -0800 Subject: [PATCH 23/45] Update src/hyp3_isce2/templates/stripmapapp_alos.xml Co-authored-by: Jake Herrmann --- src/hyp3_isce2/templates/stripmapapp_alos.xml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/hyp3_isce2/templates/stripmapapp_alos.xml b/src/hyp3_isce2/templates/stripmapapp_alos.xml index 8ecddf1a..ed5b02aa 100644 --- a/src/hyp3_isce2/templates/stripmapapp_alos.xml +++ b/src/hyp3_isce2/templates/stripmapapp_alos.xml @@ -29,15 +29,5 @@ {{ do_unwrap }} snaphu True - From 35955065ba4c483f614958cac1cbba556bdbb130 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Fri, 14 Jul 2023 12:35:00 -0800 Subject: [PATCH 24/45] Update environment.yml --- environment.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/environment.yml b/environment.yml index cb7d42cf..496fac0d 100644 --- a/environment.yml +++ b/environment.yml @@ -12,6 +12,8 @@ dependencies: - shapely - jinja2 - asf_search>=6.4.0 + - geopandas + - gdal - opencv # For packaging, and testing - flake8 From 66d7d66c9063a8882b96dbaa798af38a403a5267 Mon Sep 17 00:00:00 2001 From: mfangaritav Date: Fri, 28 Jul 2023 09:18:33 -0800 Subject: [PATCH 25/45] Tests for the stripmap process --- tests/test_stripmapapp.py | 50 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 tests/test_stripmapapp.py diff --git a/tests/test_stripmapapp.py b/tests/test_stripmapapp.py new file mode 100644 index 00000000..6f70109e --- /dev/null +++ b/tests/test_stripmapapp.py @@ -0,0 +1,50 @@ +import pytest + +from hyp3_isce2.stripmapapp_alos import StripmapappConfig, run_stripmapapp + + +def test_stripmap_config(tmp_path): + config = StripmapappConfig( + reference_image='ALPSRP156121200-L1.0/IMG-HH-ALPSRP156121200-H1.0__A', + reference_leader='ALPSRP156121200-L1.0/LED-ALPSRP156121200-H1.0__A', + secondary_image='ALPSRP162831200-L1.0/IMG-HH-ALPSRP162831200-H1.0__A', + secondary_leader='ALPSRP162831200-L1.0/LED-ALPSRP162831200-H1.0__A', + roi=[-153.212, 59.96148524563291, -151.871, 60.56159446867566], + dem_filename='dem/full_res.dem.wgs84', + ) + + template_path = tmp_path / 'stripmapApp.xml' + config.write_template(template_path) + assert template_path.exists() + + with open(template_path, 'r') as template_file: + template = template_file.read() + assert 'ALPSRP156121200-L1.0/IMG-HH-ALPSRP156121200-H1.0__A' in template + assert 'ALPSRP156121200-L1.0/LED-ALPSRP156121200-H1.0__A' in template + assert 'ALPSRP162831200-L1.0/IMG-HH-ALPSRP162831200-H1.0__A' in template + assert 'ALPSRP162831200-L1.0/LED-ALPSRP162831200-H1.0__A' in template + assert 'dem/full_res.dem.wgs84' in template + assert '[59.96148524563291, 60.56159446867566, -153.212, -151.871]' in template + +def test_run_stripmapapp(tmp_path): + with pytest.raises(IOError): + run_stripmapapp('stripmapApp.xml') + + config = StripmapappConfig( + reference_image='', + reference_leader='', + secondary_image='', + secondary_leader='', + roi=[1,2,3,4], + dem_filename='', + ) + template_path = config.write_template(tmp_path / 'stripmapApp.xml') + + with pytest.raises(ValueError, match=r'.*not a valid step.*'): + run_stripmapapp('notastep', config_xml=template_path) + + with pytest.raises(ValueError, match=r'^If dostep is specified, start and stop cannot be used$'): + run_stripmapapp('preprocess', 'startup', config_xml=template_path) + + #monkeypatch.chdir(tmp_path) + #run_stripmapapp('preprocess', config_xml=template_path) \ No newline at end of file From e3d7215d9b8d161fcde4224a29f97a8243a22c27 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Tue, 15 Aug 2023 13:42:19 -0500 Subject: [PATCH 26/45] updated naming convention --- src/hyp3_isce2/burst.py | 37 ++++++++++++++++++++++++++++-- src/hyp3_isce2/insar_tops_burst.py | 4 ++-- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/src/hyp3_isce2/burst.py b/src/hyp3_isce2/burst.py index 1f328298..320976f8 100644 --- a/src/hyp3_isce2/burst.py +++ b/src/hyp3_isce2/burst.py @@ -327,19 +327,52 @@ def download_bursts(param_list: Iterator[BurstParams]) -> List[BurstMetadata]: def get_product_name( reference_scene: str, secondary_scene: str, + pixel_spacing: int ) -> str: """Get the name of the interferogram product. + Format is S1_tttttt_iiw_aaaaaaaa_gggggggg_pp_INTzz_ssss + t: burst id + i: image mode (IW or EW) + w: swath number + a: reference burst date + g: secondary burst date + p: polariztion + z: pixel spacing + s: ASF product id + Args: reference_scene: The reference burst name. secondary_scene: The secondary burst name. - + pixel_spacing: The spacing of the pixels in the output image. + Returns: The name of the interferogram product. """ # If this changes, we will also need to update the burst product README template, # which documents this naming convention. - return f'{reference_scene}x{secondary_scene}' + reference_split = reference_scene.split('_') + secondary_split = secondary_scene.split('_') + + platform = reference_split[0] + burst_id = reference_split[1] + image_plus_swath = reference_split[2] + reference_date = reference_split[3] + secondary_date = secondary_split[3] + polarization = reference_split[4] + pixel_spacing = "INT" + str(pixel_spacing) + product_id = reference_split[5][0:4] + + return '_'.join([ + platform, + burst_id, + image_plus_swath, + reference_date, + secondary_date, + polarization, + pixel_spacing, + product_id + ]) def get_burst_params(scene_name: str) -> BurstParams: diff --git a/src/hyp3_isce2/insar_tops_burst.py b/src/hyp3_isce2/insar_tops_burst.py index 5fe392ee..fab44c61 100644 --- a/src/hyp3_isce2/insar_tops_burst.py +++ b/src/hyp3_isce2/insar_tops_burst.py @@ -427,12 +427,12 @@ def main(): ) log.info('ISCE2 TopsApp run completed successfully') - product_name = get_product_name(reference_scene, secondary_scene) + pixel_size = get_pixel_size(args.looks) + product_name = get_product_name(reference_scene, secondary_scene, pixel_spacing=int(pixel_size)) product_dir = Path(product_name) product_dir.mkdir(parents=True, exist_ok=True) - pixel_size = get_pixel_size(args.looks) translate_outputs(isce_output_dir, product_name, pixel_size=pixel_size) unwrapped_phase = f'{product_name}/{product_name}_unw_phase.tif' From a4f20f9f7511f3fc20d4b37bcea000aeae580eb4 Mon Sep 17 00:00:00 2001 From: mfangaritav Date: Tue, 15 Aug 2023 18:23:03 -0800 Subject: [PATCH 27/45] Update tests/test_stripmapapp.py Co-authored-by: Forrest Williams <31411324+forrestfwilliams@users.noreply.github.com> --- tests/test_stripmapapp.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/test_stripmapapp.py b/tests/test_stripmapapp.py index 6f70109e..9dcdef79 100644 --- a/tests/test_stripmapapp.py +++ b/tests/test_stripmapapp.py @@ -44,7 +44,4 @@ def test_run_stripmapapp(tmp_path): run_stripmapapp('notastep', config_xml=template_path) with pytest.raises(ValueError, match=r'^If dostep is specified, start and stop cannot be used$'): - run_stripmapapp('preprocess', 'startup', config_xml=template_path) - - #monkeypatch.chdir(tmp_path) - #run_stripmapapp('preprocess', config_xml=template_path) \ No newline at end of file + run_stripmapapp('preprocess', 'startup', config_xml=template_path) \ No newline at end of file From 8a4e89aa6d9449ab1c36917424a4501464074e00 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Wed, 16 Aug 2023 13:46:32 -0500 Subject: [PATCH 28/45] use token_hex for product_id --- src/hyp3_isce2/burst.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/hyp3_isce2/burst.py b/src/hyp3_isce2/burst.py index 320976f8..e2222b71 100644 --- a/src/hyp3_isce2/burst.py +++ b/src/hyp3_isce2/burst.py @@ -6,6 +6,7 @@ from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from pathlib import Path +from secrets import token_hex from typing import Iterator, List, Optional, Tuple, Union import asf_search @@ -361,7 +362,7 @@ def get_product_name( secondary_date = secondary_split[3] polarization = reference_split[4] pixel_spacing = "INT" + str(pixel_spacing) - product_id = reference_split[5][0:4] + product_id = token_hex(2).upper() return '_'.join([ platform, From 86027abd9c016e65a7194f9bcec80f065cb67ecd Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Wed, 16 Aug 2023 14:18:41 -0500 Subject: [PATCH 29/45] updated readme template --- src/hyp3_isce2/insar_tops_burst.py | 1 + .../metadata/templates/insar_burst/readme.md.txt.j2 | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/hyp3_isce2/insar_tops_burst.py b/src/hyp3_isce2/insar_tops_burst.py index fab44c61..5bf85a01 100644 --- a/src/hyp3_isce2/insar_tops_burst.py +++ b/src/hyp3_isce2/insar_tops_burst.py @@ -152,6 +152,7 @@ def make_readme( 'processor_version': isce.__version__, 'projection': hyp3_isce2.metadata.util.get_projection(info['coordinateSystem']['wkt']), 'pixel_spacing': info['geoTransform'][1], + 'product_name': product_name, 'reference_burst_name': reference_scene, 'secondary_burst_name': secondary_scene, 'range_looks': range_looks, diff --git a/src/hyp3_isce2/metadata/templates/insar_burst/readme.md.txt.j2 b/src/hyp3_isce2/metadata/templates/insar_burst/readme.md.txt.j2 index ece6d137..419523b3 100644 --- a/src/hyp3_isce2/metadata/templates/insar_burst/readme.md.txt.j2 +++ b/src/hyp3_isce2/metadata/templates/insar_burst/readme.md.txt.j2 @@ -19,13 +19,11 @@ The source bursts for this InSAR product are: Processing Date/Time: {{ processing_date.isoformat(timespec='seconds') }} -The product directory is named x +The directory name for this product is: {{ product_name }} -The directory name for this product is: {{ reference_burst_name }}x{{ secondary_burst_name }}. +The output directory uses the following naming convention: -The reference_burst_name and secondary_burst_name use the following naming conventions: - -S1_bbbbbb_xxn_yyyymmddThhmmss_pp_cccc-BURST +S1_bbbbbb_xxn_yyyymmddThhmmss_yyyymmddThhmmss_pp_INTzz_cccc bbbbbb: Relative burst ID values assigned by ESA. Each value identifies a consistent burst footprint; relative burst ID values differ from one subswath to the next. @@ -35,7 +33,7 @@ xxn: Three character combination that represents the acquisition mode and subswa (1-3 for IW, 1-5 for EW). IW mode indicates Interferometric Wideswath, which acquires a 250 km swath composed of three subswaths. EW mode indicates Extra-Wide Swath, which acquires a 400 km swath composed of 5 subswaths. -yyyymmddThhmmss: ISO date and time of acquisition. +yyyymmddThhmmss: ISO date and time of acquisition of the reference and secondary images, respectively. pp: Two character combination that represents the mode of radar orientation (polarization) for both signal transmission and reception. The first position represents the transmit orientation mode and the second @@ -46,6 +44,8 @@ pp: Two character combination that represents the mode of radar orientation (pol VH: Vertical Transmit - Horizontal Receive VV: Vertical Transmit - Vertical Receive +zz: The pixel spacing of the output image. + cccc: 4-character "Product Unique Identifier" from the SLC filename. Files contained in the product directory are named using the directory name followed by a tag indicating the file type. From 871bbbfd0502ce96166337601d6861b0d5ca8e1d Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Wed, 16 Aug 2023 14:27:17 -0500 Subject: [PATCH 30/45] Fixes for Flake8 --- src/hyp3_isce2/burst.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/src/hyp3_isce2/burst.py b/src/hyp3_isce2/burst.py index e2222b71..c2dc420e 100644 --- a/src/hyp3_isce2/burst.py +++ b/src/hyp3_isce2/burst.py @@ -332,26 +332,15 @@ def get_product_name( ) -> str: """Get the name of the interferogram product. - Format is S1_tttttt_iiw_aaaaaaaa_gggggggg_pp_INTzz_ssss - t: burst id - i: image mode (IW or EW) - w: swath number - a: reference burst date - g: secondary burst date - p: polariztion - z: pixel spacing - s: ASF product id - Args: reference_scene: The reference burst name. secondary_scene: The secondary burst name. pixel_spacing: The spacing of the pixels in the output image. - + Returns: The name of the interferogram product. """ - # If this changes, we will also need to update the burst product README template, - # which documents this naming convention. + reference_split = reference_scene.split('_') secondary_split = secondary_scene.split('_') From dc86499135f4c616df0e7ef142f2efb347f9d9ac Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Wed, 16 Aug 2023 14:45:33 -0500 Subject: [PATCH 31/45] insure that the pixel-spacing is an integer --- src/hyp3_isce2/burst.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hyp3_isce2/burst.py b/src/hyp3_isce2/burst.py index c2dc420e..7a8b909c 100644 --- a/src/hyp3_isce2/burst.py +++ b/src/hyp3_isce2/burst.py @@ -350,7 +350,7 @@ def get_product_name( reference_date = reference_split[3] secondary_date = secondary_split[3] polarization = reference_split[4] - pixel_spacing = "INT" + str(pixel_spacing) + pixel_spacing = "INT" + str(int(pixel_spacing)) product_id = token_hex(2).upper() return '_'.join([ From d002f345f1c5ec4b80a9289ccdadeab6bea41d8b Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Wed, 16 Aug 2023 14:46:35 -0500 Subject: [PATCH 32/45] updated unit tests for get_product_name --- tests/test_burst.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/test_burst.py b/tests/test_burst.py index f53910d5..3feb3f2f 100644 --- a/tests/test_burst.py +++ b/tests/test_burst.py @@ -1,4 +1,5 @@ from pathlib import Path +from re import match from unittest.mock import patch import asf_search @@ -89,7 +90,18 @@ def test_get_region_of_interest(tmp_path, orbit): def test_get_product_name(): - assert burst.get_product_name('A', 'B') == 'AxB' + + reference_name = "S1_136231_IW2_20200604T022312_VV_7C85-BURST" + secondary_name = "S1_136231_IW2_20200616T022313_VV_5D11-BURST" + + name_20m = burst.get_product_name(reference_name, secondary_name, pixel_spacing = 20.0) + name_80m = burst.get_product_name(reference_name, secondary_name, pixel_spacing = 80) + + assert match("[A-F0-9]{4}", name_20m[-4:]) != None + assert match("[A-F0-9]{4}", name_80m[-4:]) != None + + assert name_20m.startswith('S1_136231_IW2_20200604T022312_20200616T022313_VV_INT20') + assert name_80m.startswith('S1_136231_IW2_20200604T022312_20200616T022313_VV_INT80') def mock_asf_search_results( From 725450040f93dd9c0e9497a0508f1872a195a019 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Wed, 16 Aug 2023 14:48:34 -0500 Subject: [PATCH 33/45] fixes for flake8 --- tests/test_burst.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_burst.py b/tests/test_burst.py index 3feb3f2f..a41bee1e 100644 --- a/tests/test_burst.py +++ b/tests/test_burst.py @@ -94,11 +94,11 @@ def test_get_product_name(): reference_name = "S1_136231_IW2_20200604T022312_VV_7C85-BURST" secondary_name = "S1_136231_IW2_20200616T022313_VV_5D11-BURST" - name_20m = burst.get_product_name(reference_name, secondary_name, pixel_spacing = 20.0) - name_80m = burst.get_product_name(reference_name, secondary_name, pixel_spacing = 80) + name_20m = burst.get_product_name(reference_name, secondary_name, pixel_spacing=20.0) + name_80m = burst.get_product_name(reference_name, secondary_name, pixel_spacing=80) - assert match("[A-F0-9]{4}", name_20m[-4:]) != None - assert match("[A-F0-9]{4}", name_80m[-4:]) != None + assert match("[A-F0-9]{4}", name_20m[-4:]) is not None + assert match("[A-F0-9]{4}", name_80m[-4:]) is not None assert name_20m.startswith('S1_136231_IW2_20200604T022312_20200616T022313_VV_INT20') assert name_80m.startswith('S1_136231_IW2_20200604T022312_20200616T022313_VV_INT80') From 076497db2cbf7fed43c4ff1542c6bbbbebde93e2 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Wed, 16 Aug 2023 14:51:29 -0500 Subject: [PATCH 34/45] Updated Changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a674f579..32187322 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.7.0] +### Changed +* The naming convention for the burst products has been updated. + ## [0.6.2] ### Changed * The geocoding DEM is now resampled to ~20m in the case of 5x1 looks. From 328a2cac06d274d615c247d57a0b3181e3ddab0c Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 17 Aug 2023 13:19:57 -0500 Subject: [PATCH 35/45] product identifier no longer from SLC --- src/hyp3_isce2/metadata/templates/insar_burst/readme.md.txt.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hyp3_isce2/metadata/templates/insar_burst/readme.md.txt.j2 b/src/hyp3_isce2/metadata/templates/insar_burst/readme.md.txt.j2 index 419523b3..92fcfe9d 100644 --- a/src/hyp3_isce2/metadata/templates/insar_burst/readme.md.txt.j2 +++ b/src/hyp3_isce2/metadata/templates/insar_burst/readme.md.txt.j2 @@ -46,7 +46,7 @@ pp: Two character combination that represents the mode of radar orientation (pol zz: The pixel spacing of the output image. -cccc: 4-character "Product Unique Identifier" from the SLC filename. +cccc: 4-character unique product identifier. Files contained in the product directory are named using the directory name followed by a tag indicating the file type. From 6a6146da372865466ea4d27883bd8727ad42a0c8 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 17 Aug 2023 13:28:31 -0500 Subject: [PATCH 36/45] seperate type and spacing, remove time --- src/hyp3_isce2/burst.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/hyp3_isce2/burst.py b/src/hyp3_isce2/burst.py index 7a8b909c..d8a4336b 100644 --- a/src/hyp3_isce2/burst.py +++ b/src/hyp3_isce2/burst.py @@ -347,10 +347,11 @@ def get_product_name( platform = reference_split[0] burst_id = reference_split[1] image_plus_swath = reference_split[2] - reference_date = reference_split[3] - secondary_date = secondary_split[3] + reference_date = reference_split[3][0:8] + secondary_date = secondary_split[3][0:8] polarization = reference_split[4] - pixel_spacing = "INT" + str(int(pixel_spacing)) + product_type = "INT" + pixel_spacing = str(int(pixel_spacing)) product_id = token_hex(2).upper() return '_'.join([ @@ -360,7 +361,7 @@ def get_product_name( reference_date, secondary_date, polarization, - pixel_spacing, + product_type + pixel_spacing, product_id ]) From 74946a9437d90978ffc520360945bd2f8ebc1ae6 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 17 Aug 2023 13:28:44 -0500 Subject: [PATCH 37/45] updated tests for removed time --- tests/test_burst.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_burst.py b/tests/test_burst.py index a41bee1e..76da92c3 100644 --- a/tests/test_burst.py +++ b/tests/test_burst.py @@ -100,8 +100,8 @@ def test_get_product_name(): assert match("[A-F0-9]{4}", name_20m[-4:]) is not None assert match("[A-F0-9]{4}", name_80m[-4:]) is not None - assert name_20m.startswith('S1_136231_IW2_20200604T022312_20200616T022313_VV_INT20') - assert name_80m.startswith('S1_136231_IW2_20200604T022312_20200616T022313_VV_INT80') + assert name_20m.startswith('S1_136231_IW2_20200604_20200616_VV_INT20') + assert name_80m.startswith('S1_136231_IW2_20200604_20200616_VV_INT80') def mock_asf_search_results( From 7baedb1f9b67ac54dfb2d2f8ec54fdf841383c51 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 17 Aug 2023 13:28:51 -0500 Subject: [PATCH 38/45] removed time --- .../metadata/templates/insar_burst/readme.md.txt.j2 | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/hyp3_isce2/metadata/templates/insar_burst/readme.md.txt.j2 b/src/hyp3_isce2/metadata/templates/insar_burst/readme.md.txt.j2 index 92fcfe9d..29821617 100644 --- a/src/hyp3_isce2/metadata/templates/insar_burst/readme.md.txt.j2 +++ b/src/hyp3_isce2/metadata/templates/insar_burst/readme.md.txt.j2 @@ -23,7 +23,7 @@ The directory name for this product is: {{ product_name }} The output directory uses the following naming convention: -S1_bbbbbb_xxn_yyyymmddThhmmss_yyyymmddThhmmss_pp_INTzz_cccc +S1_bbbbbb_xxn_yyyymmdd_yyyymmdd_pp_INTzz_cccc bbbbbb: Relative burst ID values assigned by ESA. Each value identifies a consistent burst footprint; relative burst ID values differ from one subswath to the next. @@ -33,7 +33,7 @@ xxn: Three character combination that represents the acquisition mode and subswa (1-3 for IW, 1-5 for EW). IW mode indicates Interferometric Wideswath, which acquires a 250 km swath composed of three subswaths. EW mode indicates Extra-Wide Swath, which acquires a 400 km swath composed of 5 subswaths. -yyyymmddThhmmss: ISO date and time of acquisition of the reference and secondary images, respectively. +yyyymmdd: Date of acquisition of the reference and secondary images, respectively. pp: Two character combination that represents the mode of radar orientation (polarization) for both signal transmission and reception. The first position represents the transmit orientation mode and the second @@ -44,6 +44,8 @@ pp: Two character combination that represents the mode of radar orientation (pol VH: Vertical Transmit - Horizontal Receive VV: Vertical Transmit - Vertical Receive +INT: The product type (always INT for InSAR). + zz: The pixel spacing of the output image. cccc: 4-character unique product identifier. From 36453261065a4bdebe22107fbfa66a984a310091 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Thu, 17 Aug 2023 13:29:35 -0500 Subject: [PATCH 39/45] changed to single quotes --- src/hyp3_isce2/burst.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hyp3_isce2/burst.py b/src/hyp3_isce2/burst.py index d8a4336b..56806749 100644 --- a/src/hyp3_isce2/burst.py +++ b/src/hyp3_isce2/burst.py @@ -350,7 +350,7 @@ def get_product_name( reference_date = reference_split[3][0:8] secondary_date = secondary_split[3][0:8] polarization = reference_split[4] - product_type = "INT" + product_type = 'INT' pixel_spacing = str(int(pixel_spacing)) product_id = token_hex(2).upper() From a9acc81a2bcdb75a35a9ce710ecbf00ac3c24bbf Mon Sep 17 00:00:00 2001 From: mfangaritav Date: Thu, 17 Aug 2023 11:28:03 -0800 Subject: [PATCH 40/45] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32187322..66688622 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [0.7.0] ### Changed * The naming convention for the burst products has been updated. +* InSAR stripmap workflow to support AVO's ALOS-1 processing efforts. This workflow is specific to AVO currently, and may not work for others. ## [0.6.2] ### Changed From 8cb56cf81a9ee8a8e1b1a4826c6756f8f785f2ef Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Fri, 18 Aug 2023 12:37:15 -0500 Subject: [PATCH 41/45] flake8 fixes --- src/hyp3_isce2/insar_stripmap.py | 13 ++++++------- src/hyp3_isce2/stripmapapp_alos.py | 3 ++- tests/test_stripmapapp.py | 7 ++++--- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/hyp3_isce2/insar_stripmap.py b/src/hyp3_isce2/insar_stripmap.py index 786d8a4b..2d7efc6b 100644 --- a/src/hyp3_isce2/insar_stripmap.py +++ b/src/hyp3_isce2/insar_stripmap.py @@ -10,19 +10,18 @@ from pathlib import Path from shutil import make_archive -from hyp3_isce2 import stripmapapp_alos as stripmapapp - -from hyp3lib.aws import upload_file_to_s3 -from hyp3_isce2.dem import download_dem_for_isce2 -import zipfile +import asf_search import glob import os +import zipfile -import asf_search from shapely.geometry.polygon import Polygon -from hyp3_isce2.logging import configure_root_logger +from hyp3lib.aws import upload_file_to_s3 +from hyp3_isce2 import stripmapapp_alos as stripmapapp +from hyp3_isce2.dem import download_dem_for_isce2 +from hyp3_isce2.logging import configure_root_logger log = logging.getLogger(__name__) diff --git a/src/hyp3_isce2/stripmapapp_alos.py b/src/hyp3_isce2/stripmapapp_alos.py index 012a05c5..6fbace4e 100644 --- a/src/hyp3_isce2/stripmapapp_alos.py +++ b/src/hyp3_isce2/stripmapapp_alos.py @@ -1,9 +1,10 @@ +import os + from pathlib import Path from typing import Union from isce.applications.stripmapApp import Insar from jinja2 import Template -import os os.environ['PATH'] += os.pathsep + str(Path(os.environ['ISCE_HOME']) / 'applications') diff --git a/tests/test_stripmapapp.py b/tests/test_stripmapapp.py index 9dcdef79..a7021015 100644 --- a/tests/test_stripmapapp.py +++ b/tests/test_stripmapapp.py @@ -25,7 +25,8 @@ def test_stripmap_config(tmp_path): assert 'ALPSRP162831200-L1.0/LED-ALPSRP162831200-H1.0__A' in template assert 'dem/full_res.dem.wgs84' in template assert '[59.96148524563291, 60.56159446867566, -153.212, -151.871]' in template - + + def test_run_stripmapapp(tmp_path): with pytest.raises(IOError): run_stripmapapp('stripmapApp.xml') @@ -35,7 +36,7 @@ def test_run_stripmapapp(tmp_path): reference_leader='', secondary_image='', secondary_leader='', - roi=[1,2,3,4], + roi=[1, 2, 3, 4], dem_filename='', ) template_path = config.write_template(tmp_path / 'stripmapApp.xml') @@ -44,4 +45,4 @@ def test_run_stripmapapp(tmp_path): run_stripmapapp('notastep', config_xml=template_path) with pytest.raises(ValueError, match=r'^If dostep is specified, start and stop cannot be used$'): - run_stripmapapp('preprocess', 'startup', config_xml=template_path) \ No newline at end of file + run_stripmapapp('preprocess', 'startup', config_xml=template_path) From 70fd036a054aa9dd5d6519ea46a011e34443f493 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Fri, 18 Aug 2023 12:37:23 -0500 Subject: [PATCH 42/45] updated changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66688622..8403df5b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,9 +8,10 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [0.7.0] +### Added +* InSAR stripmap workflow to support AVO's ALOS-1 processing efforts. This workflow is specific to AVO currently, and may not work for others. ### Changed * The naming convention for the burst products has been updated. -* InSAR stripmap workflow to support AVO's ALOS-1 processing efforts. This workflow is specific to AVO currently, and may not work for others. ## [0.6.2] ### Changed From 6eca46b92fece82eb60a569d11f8737415330067 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Fri, 18 Aug 2023 12:40:18 -0500 Subject: [PATCH 43/45] flake8 fixes --- src/hyp3_isce2/insar_stripmap.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/hyp3_isce2/insar_stripmap.py b/src/hyp3_isce2/insar_stripmap.py index 2d7efc6b..7dd10956 100644 --- a/src/hyp3_isce2/insar_stripmap.py +++ b/src/hyp3_isce2/insar_stripmap.py @@ -3,22 +3,22 @@ """ import argparse +import glob import logging +import os import shutil import site import sys + from pathlib import Path from shutil import make_archive import asf_search -import glob -import os import zipfile from shapely.geometry.polygon import Polygon from hyp3lib.aws import upload_file_to_s3 - from hyp3_isce2 import stripmapapp_alos as stripmapapp from hyp3_isce2.dem import download_dem_for_isce2 from hyp3_isce2.logging import configure_root_logger From 883d0565b4dc2ef8f866346f515e1f2f175bd442 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Fri, 18 Aug 2023 12:42:36 -0500 Subject: [PATCH 44/45] import order fixes --- src/hyp3_isce2/insar_stripmap.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/hyp3_isce2/insar_stripmap.py b/src/hyp3_isce2/insar_stripmap.py index 7dd10956..79066ba1 100644 --- a/src/hyp3_isce2/insar_stripmap.py +++ b/src/hyp3_isce2/insar_stripmap.py @@ -9,16 +9,14 @@ import shutil import site import sys - +import zipfile from pathlib import Path from shutil import make_archive import asf_search -import zipfile - +from hyp3lib.aws import upload_file_to_s3 from shapely.geometry.polygon import Polygon -from hyp3lib.aws import upload_file_to_s3 from hyp3_isce2 import stripmapapp_alos as stripmapapp from hyp3_isce2.dem import download_dem_for_isce2 from hyp3_isce2.logging import configure_root_logger From 4a7690225be1403d3e85ffe3f1d7a676a7bff393 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Fri, 18 Aug 2023 12:45:02 -0500 Subject: [PATCH 45/45] import order fixes for flake8 --- src/hyp3_isce2/stripmapapp_alos.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/hyp3_isce2/stripmapapp_alos.py b/src/hyp3_isce2/stripmapapp_alos.py index 6fbace4e..5aea3c36 100644 --- a/src/hyp3_isce2/stripmapapp_alos.py +++ b/src/hyp3_isce2/stripmapapp_alos.py @@ -1,5 +1,4 @@ import os - from pathlib import Path from typing import Union