diff --git a/.github/dependabot.yml b/.github/dependabot.yml index f2494b1b..e88e5ee0 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,8 +1,26 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file + version: 2 updates: - - package-ecosystem: "github-actions" - directory: "/" + - package-ecosystem: pip + directory: / schedule: - interval: "daily" + interval: weekly labels: - - "bumpless" + - bumpless + ignore: + - dependency-name: gdal[numpy] + - dependency-name: matplotlib + - dependency-name: numpy + - dependency-name: packaging + - dependency-name: scipy + - dependency-name: shapely + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + labels: + - bumpless diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 3ce6f429..6047ddb1 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -13,6 +13,4 @@ on: jobs: call-changelog-check-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.11.0 - secrets: - USER_TOKEN: ${{ secrets.GITHUB_TOKEN }} + uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.11.1 diff --git a/.github/workflows/create-jira-issue.yml b/.github/workflows/create-jira-issue.yml index 0b69efec..129ff5f1 100644 --- a/.github/workflows/create-jira-issue.yml +++ b/.github/workflows/create-jira-issue.yml @@ -6,7 +6,7 @@ on: jobs: call-create-jira-issue-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-create-jira-issue.yml@v0.11.0 + uses: ASFHyP3/actions/.github/workflows/reusable-create-jira-issue.yml@v0.11.1 secrets: JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }} JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }} diff --git a/.github/workflows/labeled-pr.yml b/.github/workflows/labeled-pr.yml index 66ba502e..3ba21604 100644 --- a/.github/workflows/labeled-pr.yml +++ b/.github/workflows/labeled-pr.yml @@ -12,4 +12,4 @@ on: jobs: call-labeled-pr-check-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.11.0 + uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.11.1 diff --git a/.github/workflows/release-template-comment.yml b/.github/workflows/release-template-comment.yml index cae89e6f..01d20589 100644 --- a/.github/workflows/release-template-comment.yml +++ b/.github/workflows/release-template-comment.yml @@ -7,6 +7,6 @@ on: jobs: call-release-checklist-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-release-checklist-comment.yml@v0.11.0 + uses: ASFHyP3/actions/.github/workflows/reusable-release-checklist-comment.yml@v0.11.1 secrets: USER_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 54a3499e..49dd3c92 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,7 +7,7 @@ on: jobs: call-release-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.11.0 + uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.11.1 with: release_prefix: HyP3 GAMMA secrets: diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index 0679b824..38841ac8 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -4,9 +4,9 @@ on: push jobs: call-flake8-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-flake8.yml@v0.11.0 + uses: ASFHyP3/actions/.github/workflows/reusable-flake8.yml@v0.11.1 with: local_package_names: hyp3_gamma call-secrets-analysis-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-secrets-analysis.yml@v0.11.0 + uses: ASFHyP3/actions/.github/workflows/reusable-secrets-analysis.yml@v0.11.1 diff --git a/.github/workflows/tag-version.yml b/.github/workflows/tag-version.yml index 4714c1f1..7bc79246 100644 --- a/.github/workflows/tag-version.yml +++ b/.github/workflows/tag-version.yml @@ -7,6 +7,6 @@ on: jobs: call-bump-version-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.11.0 + uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.11.1 secrets: USER_TOKEN: ${{ secrets.TOOLS_BOT_PAK }} diff --git a/.github/workflows/test-and-build.yml b/.github/workflows/test-and-build.yml index 41859f7b..e2d7a2b8 100644 --- a/.github/workflows/test-and-build.yml +++ b/.github/workflows/test-and-build.yml @@ -18,14 +18,14 @@ env: jobs: call-pytest-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-pytest.yml@v0.11.0 + uses: ASFHyP3/actions/.github/workflows/reusable-pytest.yml@v0.11.1 with: local_package_name: hyp3_gamma python_versions: >- ["3.10"] call-version-info-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-version-info.yml@v0.11.0 + uses: ASFHyP3/actions/.github/workflows/reusable-version-info.yml@v0.11.1 with: python_version: '3.10' @@ -62,7 +62,7 @@ jobs: tar -zxvf GAMMA_SOFTWARE-20230712_ISP_DIFF_LAT.linux64_ubuntu2204.tar.gz - name: Build, tag, and push image to Amazon ECR - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: . no-cache: true @@ -76,14 +76,14 @@ jobs: - name: Add test tag if: github.ref == 'refs/heads/develop' - uses: akhilerm/tag-push-action@v2.1.0 + uses: akhilerm/tag-push-action@v2.2.0 with: src: ${{ env.HYP3_REGISTRY }}/${{ github.event.repository.name }}:${{ env.SDIST_VERSION }} dst: ${{ env.HYP3_REGISTRY }}/${{ github.event.repository.name }}:test - name: Add latest tag if: github.ref == 'refs/heads/main' - uses: akhilerm/tag-push-action@v2.1.0 + uses: akhilerm/tag-push-action@v2.2.0 with: src: ${{ env.HYP3_REGISTRY }}/${{ github.event.repository.name }}:${{ env.SDIST_VERSION }} dst: ${{ env.HYP3_REGISTRY }}/${{ github.event.repository.name }}:latest diff --git a/CHANGELOG.md b/CHANGELOG.md index 9258f76f..bb23e717 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [8.1.3] +### Changed +* Input SLCs for insar jobs are now downloaded in parallel in `__main__.insar()` +* Consolidated python dependencies to minimize differences between local, docker, and github environments + ## [8.1.2] ### Changed * Enabled LZW compression for intermediate GeoTIFFs generated by `hyp3_gamma.water_mask` to reduce required disk space diff --git a/README.md b/README.md index cbc0537b..48669544 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,6 @@ Ubuntu 20.04 is recommended for GAMMA support. cd hyp3-gamma conda env create -f environment.yml conda activate hyp3-gamma - python -m pip install -e .[develop] ``` 1. Check hyp3_gamma is installed ``` diff --git a/environment.yml b/environment.yml index b8c92b3e..5773f6b5 100644 --- a/environment.yml +++ b/environment.yml @@ -5,24 +5,7 @@ channels: dependencies: - python=3.10 - pip - # For packaging, and testing - - flake8 - - flake8-import-order - - flake8-blind-except - - flake8-builtins - - setuptools - - setuptools_scm - - wheel - - pytest - - pytest-console-scripts - - pytest-cov - # For running - - gdal>=3.4,<3.5 - - geopandas - - hyp3lib>=3,<4 - - jinja2 - - lxml - - numpy>=1.21,<1.22 - - pillow - - python-dateutil - - rtree + - gdal=3.4.1 + - numpy=1.21.5 + - pip: + - --editable .[develop] diff --git a/hyp3_gamma/__main__.py b/hyp3_gamma/__main__.py index 53869f11..e5817378 100644 --- a/hyp3_gamma/__main__.py +++ b/hyp3_gamma/__main__.py @@ -1,6 +1,7 @@ """ rtc_gamma and insar_gamma processing for HyP3 """ +import concurrent.futures import logging import os import sys @@ -176,9 +177,9 @@ def insar(): write_credentials_to_netrc_file(username, password) - g1, g2 = util.earlier_granule_first(args.granules[0], args.granules[1]) - reference_granule = util.get_granule(g1) - secondary_granule = util.get_granule(g2) + with concurrent.futures.ProcessPoolExecutor() as executor: + downloaded_granules = [granule for granule in executor.map(util.get_granule, args.granules)] + reference_granule, secondary_granule = util.earlier_granule_first(*downloaded_granules) rlooks, alooks = (20, 4) if args.looks == '20x4' else (10, 2) diff --git a/hyp3_gamma/metadata/templates/insar/readme.md.txt.j2 b/hyp3_gamma/metadata/templates/insar/readme.md.txt.j2 index e5636c00..46526cb0 100644 --- a/hyp3_gamma/metadata/templates/insar/readme.md.txt.j2 +++ b/hyp3_gamma/metadata/templates/insar/readme.md.txt.j2 @@ -226,9 +226,9 @@ The DEM used for this product is {{ dem_name }}, which has a native pixel spacin ---------------- ## 11. Water Mask -The water mask identifies coastal waters and major inland waterbodies. These areas are assigned a pixel value of 0. All remaining pixels (land, islands in large lakes, small inland waterbodies, and landfast Antarctic ice) have a value of 1. The water mask is stored as an 8-bit unsigned integer GeoTIFF file and is always included in the product package. +The water mask identifies coastal waters and most inland waterbodies. These areas are assigned a pixel value of 0. All remaining pixels (land, islands, very small inland waterbodies, and landfast Antarctic ice) have a value of 1. All areas between 85 and 90 degrees north latitude are assigned a value of 0, and all areas between 85 and 90 degrees south latitude are assigned a value of 1. The water mask is stored as an 8-bit unsigned integer GeoTIFF file and is always included in the product package. -The water mask is generated by ASF using data from OpenStreetMap(https://www.openstreetmap.org/about) and/or ESA WorldCover(https://esa-worldcover.org/en/about/about) depending on location. Areas within Canada, Alaska, and Russia are primarily covered by ESA WorldCover data, while the rest of the world is covered by OpenStreetMaps data. Refer to our InSAR Water Masking Tutorial for more information: https://arcg.is/19aKHj +The water mask is generated by ASF using data from OpenStreetMap (https://www.openstreetmap.org/about) and/or ESA WorldCover (https://esa-worldcover.org/en/about/about), depending on location. Areas within Canada, Alaska, and Russia are primarily covered by ESA WorldCover data, while the rest of the world is covered by OpenStreetMaps data. Refer to our InSAR Water Masking Tutorial for more information: https://arcg.is/19aKHj Note that applying the mask after phase unwrapping does not prevent unwrapping errors caused by the inclusion of water pixels as valid data during the phase unwrapping process. When submitting InSAR jobs for processing, there is an option to "Apply Water Mask". Selecting this option uses the water mask to adjust the validity mask used for phase unwrapping. This prevents phase unwrapping from occurring over large expanses of water, which can lead to unexpected deformation signals or phase jumps in the unwrapped outputs. This mask {{ "was" if water_mask_applied else "was not" }} applied during phase unwrapping. Refer to the Unwrapped Interferogram description above (item 2) for more information. diff --git a/pyproject.toml b/pyproject.toml index 51000e45..26a74463 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,2 +1,53 @@ [build-system] -requires = ["setuptools>=42", "wheel", "setuptools_scm[toml]>=3.4"] +requires = ["setuptools>=61.0", "setuptools_scm[toml]>=6.2"] +build-backend = "setuptools.build_meta" + +[project] +name = "hyp3_gamma" +requires-python = "~=3.10" +authors = [ + {name="ASF Tools Team", email="UAF-asf-apd@alaska.edu"}, +] +description = "HyP3 plugin for SAR processing with GAMMA" +license = {text = "BSD-3-Clause"} +classifiers=[ + "Intended Audience :: Science/Research", + "License :: OSI Approved :: BSD License", + "Natural Language :: English", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", +] + +dynamic = ["version", "readme", "dependencies", "optional-dependencies"] + +[project.urls] +Homepage = "https://github.com/ASFHyP3/hyp3-gamma" +Documentation = "https://hyp3-docs.asf.alaska.edu" + +[project.scripts] +hyp3_gamma = "hyp3_gamma.__main__:main" +rtc = "hyp3_gamma.__main__:rtc" +"rtc_sentinel.py" = "hyp3_gamma.rtc:rtc_sentinel.main" +insar = "hyp3_gamma.__main__:insar" +"ifm_sentinel.py" = "hyp3_gamma.insar.ifm_sentinel:main" +"interf_pwr_s1_lt_tops_proc.py" = "hyp3_gamma.insar.interf_pwr_s1_lt_tops_proc:main" +"unwrapping_geocoding.py" = "hyp3_gamma.insar.unwrapping_geocoding:main" + +[tool.pytest.ini_options] +testpaths = ["tests"] +script_launch_mode = "subprocess" + +[tool.setuptools] +include-package-data = true +zip-safe = false + +[tool.setuptools.dynamic] +readme = {file = ["README.md"], content-type = "text/markdown"} +dependencies = {file = ["requirements.txt"]} +optional-dependencies = {develop = {file = ["requirements-develop.txt"]}} + +[tool.setuptools.packages.find] +where = ["."] + +[tool.setuptools_scm] diff --git a/requirements-develop.txt b/requirements-develop.txt new file mode 100644 index 00000000..574794fb --- /dev/null +++ b/requirements-develop.txt @@ -0,0 +1,8 @@ +flake8 +flake8-import-order +flake8-blind-except +flake8-builtins +pytest +pytest-cov +pytest-console-scripts +setuptools_scm diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..2fc3546b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,16 @@ +# required by GAMMA and pinned to versions available via apt; ignored in dependabot.yml +gdal[numpy]==3.4.1 +matplotlib==3.5.1 +numpy==1.21.5 +packaging==21.3 +scipy==1.8.0 +shapely==1.8.0 +# required by hyp3_gamma +geopandas==0.14.3 +hyp3lib==3.0.0 +jinja2==3.1.4 +lxml==4.8.0 +pillow==9.0.1 +pyproj==3.6.1 +python-dateutil==2.9.0.post0 +rtree==1.2.0 diff --git a/setup.py b/setup.py deleted file mode 100644 index 3337de0a..00000000 --- a/setup.py +++ /dev/null @@ -1,73 +0,0 @@ -from pathlib import Path - -from setuptools import find_packages, setup - -readme = Path(__file__).parent / 'README.md' - -setup( - name='hyp3_gamma', - use_scm_version=True, - description='HyP3 plugin for SAR processing with GAMMA', - long_description=readme.read_text(), - long_description_content_type='text/markdown', - - url='https://github.com/ASFHyP3/hyp3-gamma', - project_urls={ - 'Documentation': 'https://hyp3-docs.asf.alaska.edu', - }, - - author='ASF APD/Tools Team', - author_email='uaf-asf-apd@alaska.edu', - - license='BSD', - include_package_data=True, - - classifiers=[ - 'Intended Audience :: Science/Research', - 'License :: OSI Approved :: BSD License', - 'Natural Language :: English', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.10', - ], - - python_requires='~=3.10', - - install_requires=[ - 'gdal>=3.4,<3.5', - 'geopandas', - 'hyp3lib>=3,<4', - 'jinja2', - 'lxml', - 'numpy>=1.21,<1.22', - 'pillow', - 'python-dateutil', - 'rtree' - ], - - extras_require={ - 'develop': [ - 'flake8', - 'flake8-import-order', - 'flake8-blind-except', - 'flake8-builtins', - 'pytest', - 'pytest-cov', - 'pytest-console-scripts', - ] - }, - - packages=find_packages(), - - entry_points={'console_scripts': [ - 'hyp3_gamma = hyp3_gamma.__main__:main', - 'rtc = hyp3_gamma.__main__:rtc', - 'rtc_sentinel.py = hyp3_gamma.rtc.rtc_sentinel:main', - 'insar = hyp3_gamma.__main__:insar', - 'ifm_sentinel.py = hyp3_gamma.insar.ifm_sentinel:main', - 'interf_pwr_s1_lt_tops_proc.py = hyp3_gamma.insar.interf_pwr_s1_lt_tops_proc:main', - 'unwrapping_geocoding.py = hyp3_gamma.insar.unwrapping_geocoding:main', - ] - }, - - zip_safe=False, -) diff --git a/tests/test_water_mask.py b/tests/test_water_mask.py index bf29fbf9..f493ea19 100644 --- a/tests/test_water_mask.py +++ b/tests/test_water_mask.py @@ -55,7 +55,7 @@ def test_get_tiles(tmp_path, test_data_dir): def test_create_water_mask(tmp_path, test_data_dir): input_image = str(test_data_dir / 'water_mask_input.tif') - output_image = str(test_data_dir / 'water_mask_output.wgs84') + output_image = str(tmp_path / 'water_mask_output.wgs84') validation_text = str(test_data_dir / 'water_mask_output_info.txt') water_mask.create_water_mask(input_image, output_image, gdal_format='ISCE', tmp_path=tmp_path) info_from_img = gdal.Info(output_image)