Release v8.1.3

.github/dependabot.yml

@@ -1,8 +1,26 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
 version: 2
 updates:
-  - package-ecosystem: "github-actions"
-    directory: "/"
+  - package-ecosystem: pip
+    directory: /
     schedule:
-      interval: "daily"
+      interval: weekly
     labels:
-      - "bumpless"
+      - bumpless
+    ignore:
+      - dependency-name: gdal[numpy]
+      - dependency-name: matplotlib
+      - dependency-name: numpy
+      - dependency-name: packaging
+      - dependency-name: scipy
+      - dependency-name: shapely
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: weekly
+    labels:
+      - bumpless

.github/workflows/changelog.yml

@@ -13,6 +13,4 @@ on:
 
 jobs:
   call-changelog-check-workflow:
-    uses: ASFHyP3/actions/.github/workflows/[email protected]
-    secrets:
-      USER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    uses: ASFHyP3/actions/.github/workflows/[email protected]

.github/workflows/create-jira-issue.yml

@@ -6,7 +6,7 @@ on:
 
 jobs:
   call-create-jira-issue-workflow:
-    uses: ASFHyP3/actions/.github/workflows/[email protected].0
+    uses: ASFHyP3/actions/.github/workflows/[email protected].1
     secrets:
       JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }}
       JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }}

.github/workflows/labeled-pr.yml

@@ -12,4 +12,4 @@ on:
 
 jobs:
   call-labeled-pr-check-workflow:
-    uses: ASFHyP3/actions/.github/workflows/[email protected].0
+    uses: ASFHyP3/actions/.github/workflows/[email protected].1

.github/workflows/release-template-comment.yml

@@ -7,6 +7,6 @@ on:
 
 jobs:
   call-release-checklist-workflow:
-    uses: ASFHyP3/actions/.github/workflows/[email protected].0
+    uses: ASFHyP3/actions/.github/workflows/[email protected].1
     secrets:
       USER_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/release.yml

@@ -7,7 +7,7 @@ on:
 
 jobs:
   call-release-workflow:
-    uses: ASFHyP3/actions/.github/workflows/[email protected].0
+    uses: ASFHyP3/actions/.github/workflows/[email protected].1
     with:
       release_prefix: HyP3 GAMMA
     secrets:

.github/workflows/static-analysis.yml

@@ -4,9 +4,9 @@ on: push
 
 jobs:
   call-flake8-workflow:
-    uses: ASFHyP3/actions/.github/workflows/[email protected].0
+    uses: ASFHyP3/actions/.github/workflows/[email protected].1
     with:
       local_package_names: hyp3_gamma
 
   call-secrets-analysis-workflow:
-    uses: ASFHyP3/actions/.github/workflows/[email protected].0
+    uses: ASFHyP3/actions/.github/workflows/[email protected].1

.github/workflows/tag-version.yml

@@ -7,6 +7,6 @@ on:
 
 jobs:
   call-bump-version-workflow:
-    uses: ASFHyP3/actions/.github/workflows/[email protected].0
+    uses: ASFHyP3/actions/.github/workflows/[email protected].1
     secrets:
       USER_TOKEN: ${{ secrets.TOOLS_BOT_PAK }}

.github/workflows/test-and-build.yml

@@ -18,14 +18,14 @@ env:
 
 jobs:
   call-pytest-workflow:
-    uses: ASFHyP3/actions/.github/workflows/[email protected].0
+    uses: ASFHyP3/actions/.github/workflows/[email protected].1
     with:
       local_package_name: hyp3_gamma
       python_versions: >-
         ["3.10"]
 
   call-version-info-workflow:
-    uses: ASFHyP3/actions/.github/workflows/[email protected].0
+    uses: ASFHyP3/actions/.github/workflows/[email protected].1
     with:
       python_version: '3.10'
 
@@ -62,7 +62,7 @@ jobs:
           tar -zxvf GAMMA_SOFTWARE-20230712_ISP_DIFF_LAT.linux64_ubuntu2204.tar.gz
 
       - name: Build, tag, and push image to Amazon ECR
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: .
           no-cache: true
@@ -76,14 +76,14 @@ jobs:
 
       - name: Add test tag
         if: github.ref == 'refs/heads/develop'
-        uses: akhilerm/tag-push-action@v2.1.0
+        uses: akhilerm/tag-push-action@v2.2.0
         with:
           src: ${{ env.HYP3_REGISTRY }}/${{ github.event.repository.name }}:${{ env.SDIST_VERSION }}
           dst: ${{ env.HYP3_REGISTRY }}/${{ github.event.repository.name }}:test
 
       - name: Add latest tag
         if: github.ref == 'refs/heads/main'
-        uses: akhilerm/tag-push-action@v2.1.0
+        uses: akhilerm/tag-push-action@v2.2.0
         with:
           src: ${{ env.HYP3_REGISTRY }}/${{ github.event.repository.name }}:${{ env.SDIST_VERSION }}
           dst: ${{ env.HYP3_REGISTRY }}/${{ github.event.repository.name }}:latest

CHANGELOG.md

@@ -7,6 +7,11 @@ and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/)
 and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 
+## [8.1.3]
+### Changed
+* Input SLCs for insar jobs are now downloaded in parallel in `__main__.insar()`
+* Consolidated python dependencies to minimize differences between local, docker, and github environments
+
 ## [8.1.2]
 ### Changed
 * Enabled LZW compression for intermediate GeoTIFFs generated by `hyp3_gamma.water_mask` to reduce required disk space

README.md

@@ -14,7 +14,6 @@ Ubuntu 20.04 is recommended for GAMMA support.
    cd hyp3-gamma
    conda env create -f environment.yml
    conda activate hyp3-gamma
-   python -m pip install -e .[develop]
    ```
 1. Check hyp3_gamma is installed
    ```

environment.yml

@@ -5,24 +5,7 @@ channels:
 dependencies:
   - python=3.10
   - pip
-  # For packaging, and testing
-  - flake8
-  - flake8-import-order
-  - flake8-blind-except
-  - flake8-builtins
-  - setuptools
-  - setuptools_scm
-  - wheel
-  - pytest
-  - pytest-console-scripts
-  - pytest-cov
-  # For running
-  - gdal>=3.4,<3.5
-  - geopandas
-  - hyp3lib>=3,<4
-  - jinja2
-  - lxml
-  - numpy>=1.21,<1.22
-  - pillow
-  - python-dateutil
-  - rtree
+  - gdal=3.4.1
+  - numpy=1.21.5
+  - pip:
+      - --editable .[develop]

hyp3_gamma/__main__.py

@@ -1,6 +1,7 @@
 """
 rtc_gamma and insar_gamma processing for HyP3
 """
+import concurrent.futures
 import logging
 import os
 import sys
@@ -176,9 +177,9 @@ def insar():
 
     write_credentials_to_netrc_file(username, password)
 
-    g1, g2 = util.earlier_granule_first(args.granules[0], args.granules[1])
-    reference_granule = util.get_granule(g1)
-    secondary_granule = util.get_granule(g2)
+    with concurrent.futures.ProcessPoolExecutor() as executor:
+        downloaded_granules = [granule for granule in executor.map(util.get_granule, args.granules)]
+    reference_granule, secondary_granule = util.earlier_granule_first(*downloaded_granules)
 
     rlooks, alooks = (20, 4) if args.looks == '20x4' else (10, 2)
 

hyp3_gamma/metadata/templates/insar/readme.md.txt.j2

@@ -226,9 +226,9 @@ The DEM used for this product is {{ dem_name }}, which has a native pixel spacin
 ----------------
 ## 11. Water Mask
 
-The water mask identifies coastal waters and major inland waterbodies. These areas are assigned a pixel value of 0. All remaining pixels (land, islands in large lakes, small inland waterbodies, and landfast Antarctic ice) have a value of 1. The water mask is stored as an 8-bit unsigned integer GeoTIFF file and is always included in the product package.
+The water mask identifies coastal waters and most inland waterbodies. These areas are assigned a pixel value of 0. All remaining pixels (land, islands, very small inland waterbodies, and landfast Antarctic ice) have a value of 1. All areas between 85 and 90 degrees north latitude are assigned a value of 0, and all areas between 85 and 90 degrees south latitude are assigned a value of 1. The water mask is stored as an 8-bit unsigned integer GeoTIFF file and is always included in the product package.
 
-The water mask is generated by ASF using data from OpenStreetMap(https://www.openstreetmap.org/about) and/or ESA WorldCover(https://esa-worldcover.org/en/about/about) depending on location. Areas within Canada, Alaska, and Russia are primarily covered by ESA WorldCover data, while the rest of the world is covered by OpenStreetMaps data. Refer to our InSAR Water Masking Tutorial for more information: https://arcg.is/19aKHj
+The water mask is generated by ASF using data from OpenStreetMap (https://www.openstreetmap.org/about) and/or ESA WorldCover (https://esa-worldcover.org/en/about/about), depending on location. Areas within Canada, Alaska, and Russia are primarily covered by ESA WorldCover data, while the rest of the world is covered by OpenStreetMaps data. Refer to our InSAR Water Masking Tutorial for more information: https://arcg.is/19aKHj
 
 Note that applying the mask after phase unwrapping does not prevent unwrapping errors caused by the inclusion of water pixels as valid data during the phase unwrapping process. When submitting InSAR jobs for processing, there is an option to "Apply Water Mask". Selecting this option uses the water mask to adjust the validity mask used for phase unwrapping. This prevents phase unwrapping from occurring over large expanses of water, which can lead to unexpected deformation signals or phase jumps in the unwrapped outputs. This mask {{ "was" if water_mask_applied else "was not" }} applied during phase unwrapping. Refer to the Unwrapped Interferogram description above (item 2) for more information.
 

pyproject.toml

@@ -1,2 +1,53 @@
 [build-system]
-requires = ["setuptools>=42", "wheel", "setuptools_scm[toml]>=3.4"]
+requires = ["setuptools>=61.0", "setuptools_scm[toml]>=6.2"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "hyp3_gamma"
+requires-python = "~=3.10"
+authors = [
+    {name="ASF Tools Team", email="[email protected]"},
+]
+description = "HyP3 plugin for SAR processing with GAMMA"
+license = {text = "BSD-3-Clause"}
+classifiers=[
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: BSD License",
+    "Natural Language :: English",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+]
+
+dynamic = ["version", "readme", "dependencies", "optional-dependencies"]
+
+[project.urls]
+Homepage = "https://github.com/ASFHyP3/hyp3-gamma"
+Documentation = "https://hyp3-docs.asf.alaska.edu"
+
+[project.scripts]
+hyp3_gamma = "hyp3_gamma.__main__:main"
+rtc = "hyp3_gamma.__main__:rtc"
+"rtc_sentinel.py" = "hyp3_gamma.rtc:rtc_sentinel.main"
+insar = "hyp3_gamma.__main__:insar"
+"ifm_sentinel.py" = "hyp3_gamma.insar.ifm_sentinel:main"
+"interf_pwr_s1_lt_tops_proc.py" = "hyp3_gamma.insar.interf_pwr_s1_lt_tops_proc:main"
+"unwrapping_geocoding.py" = "hyp3_gamma.insar.unwrapping_geocoding:main"
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+script_launch_mode = "subprocess"
+
+[tool.setuptools]
+include-package-data = true
+zip-safe = false
+
+[tool.setuptools.dynamic]
+readme = {file = ["README.md"], content-type = "text/markdown"}
+dependencies = {file = ["requirements.txt"]}
+optional-dependencies = {develop = {file = ["requirements-develop.txt"]}}
+
+[tool.setuptools.packages.find]
+where = ["."]
+
+[tool.setuptools_scm]

requirements-develop.txt

@@ -0,0 +1,8 @@
+flake8
+flake8-import-order
+flake8-blind-except
+flake8-builtins
+pytest
+pytest-cov
+pytest-console-scripts
+setuptools_scm

requirements.txt

@@ -0,0 +1,16 @@
+# required by GAMMA and pinned to versions available via apt; ignored in dependabot.yml
+gdal[numpy]==3.4.1
+matplotlib==3.5.1
+numpy==1.21.5
+packaging==21.3
+scipy==1.8.0
+shapely==1.8.0
+# required by hyp3_gamma
+geopandas==0.14.3
+hyp3lib==3.0.0
+jinja2==3.1.4
+lxml==4.8.0
+pillow==9.0.1
+pyproj==3.6.1
+python-dateutil==2.9.0.post0
+rtree==1.2.0

tests/test_water_mask.py

@@ -55,7 +55,7 @@ def test_get_tiles(tmp_path, test_data_dir):
 
 def test_create_water_mask(tmp_path, test_data_dir):
     input_image = str(test_data_dir / 'water_mask_input.tif')
-    output_image = str(test_data_dir / 'water_mask_output.wgs84')
+    output_image = str(tmp_path / 'water_mask_output.wgs84')
     validation_text = str(test_data_dir / 'water_mask_output_info.txt')
     water_mask.create_water_mask(input_image, output_image, gdal_format='ISCE', tmp_path=tmp_path)
     info_from_img = gdal.Info(output_image)