From ad2b3694625f39400e72149b29a0c86ff807f8e7 Mon Sep 17 00:00:00 2001 From: Scott Henderson Date: Fri, 12 Jan 2024 08:47:48 -0800 Subject: [PATCH 01/25] add docker storage docs --- README.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/README.md b/README.md index ef1f8154..ffbdff12 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,37 @@ docker run -it --rm \ **NOTE** Each workflow can also be accessed via an alternative CLI with the format (`[WORKFLOW_NAME] [WORKFLOW_ARGS]`) +#### Docker outputs + +To retain outputs running via docker there are two recommended approaches: + +1. Use a volume mount + +Add the `-w /tmp -v [localdir]:/tmp` flags after docker run. `-w` changes the working directory of the container to `/tmp` and -v will mount whichever local directory you choose so that such that hyp3_isce3 outputs are written locally. + +1. Copy outputs to remote object storage + +Append the `--bucket` and `--bucket-prefix` to [WORKFLOW_ARGS]. This also requires AWS credentials to write to the bucket are available to the running container, for example to write outputs to a hypothetical bucket `s3://hyp3-isce2-test-bucket/test-run/` + +``` +docker run -it --rm \ + -e AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE \ + -e AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY \ + -e AWS_SESSION_TOKEN=averylongTokenString \ + -e EARTHDATA_USERNAME=[YOUR_USERNAME_HERE] \ + -e EARTHDATA_PASSWORD=[YOUR_PASSWORD_HERE] \ + -e ESA_USERNAME=[YOUR_USERNAME_HERE] \ + -e ESA_PASSWORD=[YOUR_PASSWORD_HERE] \ + ghcr.io/asfhyp3/hyp3-isce2:latest \ + ++process [WORKFLOW_NAME] \ + [WORKFLOW_ARGS] \ + --bucket "hyp3-isce2-test-bucket" + --bucket-prefix "test-run" +``` + +Note: you might want to use [Docker `--env-file`](https://docs.docker.com/engine/reference/commandline/run/#env) to capture all the necessary environment variabes + + ## Developer Setup 1. Ensure that conda is installed on your system (we recommend using [mambaforge](https://github.com/conda-forge/miniforge#mambaforge) to reduce setup times). 2. Download a local version of the `hyp3-isce2` repository (`git clone https://github.com/ASFHyP3/hyp3-isce2.git`) From 705c6d7134cc0b73215107ad0a55f4d280d6225b Mon Sep 17 00:00:00 2001 From: Scott Henderson Date: Fri, 12 Jan 2024 17:54:15 -0800 Subject: [PATCH 02/25] docs --- README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index ffbdff12..f7c45acf 100644 --- a/README.md +++ b/README.md @@ -75,23 +75,23 @@ docker run -it --rm \ **NOTE** Each workflow can also be accessed via an alternative CLI with the format (`[WORKFLOW_NAME] [WORKFLOW_ARGS]`) -#### Docker outputs +#### Docker Outputs -To retain outputs running via docker there are two recommended approaches: +To retain hyp3_isce2 output files running via Docker there are two recommended approaches: 1. Use a volume mount -Add the `-w /tmp -v [localdir]:/tmp` flags after docker run. `-w` changes the working directory of the container to `/tmp` and -v will mount whichever local directory you choose so that such that hyp3_isce3 outputs are written locally. +Add the `-w /tmp -v [localdir]:/tmp` flags after docker run. `-w` changes the working directory of the container to `/tmp` and `-v` will mount whichever local directory you choose so that such that hyp3_isce3 outputs are preserved locally. 1. Copy outputs to remote object storage -Append the `--bucket` and `--bucket-prefix` to [WORKFLOW_ARGS]. This also requires AWS credentials to write to the bucket are available to the running container, for example to write outputs to a hypothetical bucket `s3://hyp3-isce2-test-bucket/test-run/` +Append the `--bucket` and `--bucket-prefix` to [WORKFLOW_ARGS]. *Only the final output files and zipped archive of those files is uploaded.* This also requires that AWS credentials to write to the bucket are available to the running container. For example, to write outputs to a hypothetical bucket `s3://hypothetical-bucket/test-run/`: ``` docker run -it --rm \ - -e AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE \ - -e AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY \ - -e AWS_SESSION_TOKEN=averylongTokenString \ + -e AWS_ACCESS_KEY_ID=[YOUR_KEY] \ + -e AWS_SECRET_ACCESS_KEY=[YOUR_SECRET] \ + -e AWS_SESSION_TOKEN=[YOUR_TOKEN] \ -e EARTHDATA_USERNAME=[YOUR_USERNAME_HERE] \ -e EARTHDATA_PASSWORD=[YOUR_PASSWORD_HERE] \ -e ESA_USERNAME=[YOUR_USERNAME_HERE] \ @@ -99,11 +99,11 @@ docker run -it --rm \ ghcr.io/asfhyp3/hyp3-isce2:latest \ ++process [WORKFLOW_NAME] \ [WORKFLOW_ARGS] \ - --bucket "hyp3-isce2-test-bucket" + --bucket "hypothetical-bucket" \ --bucket-prefix "test-run" ``` -Note: you might want to use [Docker `--env-file`](https://docs.docker.com/engine/reference/commandline/run/#env) to capture all the necessary environment variabes +Tip: you might can use [Docker `--env-file`](https://docs.docker.com/engine/reference/commandline/run/#env) to capture all the necessary environment variabes in a single file. ## Developer Setup From bd5eb5829e1ba2277b76b8956bbb781ebca84f7b Mon Sep 17 00:00:00 2001 From: Scott Henderson Date: Tue, 2 Apr 2024 09:49:43 -0700 Subject: [PATCH 03/25] Update README.md Co-authored-by: Joseph H Kennedy --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f7c45acf..b2262964 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ docker run -it --rm \ --bucket-prefix "test-run" ``` -Tip: you might can use [Docker `--env-file`](https://docs.docker.com/engine/reference/commandline/run/#env) to capture all the necessary environment variabes in a single file. +Tip: you can use [`docker run --env-file`](https://docs.docker.com/reference/cli/docker/container/run/#env) to capture all the necessary environment variables in a single file. ## Developer Setup From df1a4cb4119859efd5de578cc8d3089d4572f1ba Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Jun 2024 20:25:42 +0000 Subject: [PATCH 04/25] Bump ASFHyP3/actions from 0.11.0 to 0.11.1 Bumps [ASFHyP3/actions](https://github.com/asfhyp3/actions) from 0.11.0 to 0.11.1. - [Release notes](https://github.com/asfhyp3/actions/releases) - [Changelog](https://github.com/ASFHyP3/actions/blob/develop/CHANGELOG.md) - [Commits](https://github.com/asfhyp3/actions/compare/v0.11.0...v0.11.1) --- updated-dependencies: - dependency-name: ASFHyP3/actions dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/changelog.yml | 2 +- .github/workflows/create-jira-issue.yml | 2 +- .github/workflows/labeled-pr.yml | 2 +- .github/workflows/release-checklist-comment.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/static-analysis.yml | 4 ++-- .github/workflows/tag-version.yml | 2 +- .github/workflows/test-and-build.yml | 6 +++--- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 3a01155a..616667e8 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -14,6 +14,6 @@ on: jobs: call-changelog-check-workflow: # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.11.0 + uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.11.1 secrets: USER_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/create-jira-issue.yml b/.github/workflows/create-jira-issue.yml index 0b69efec..129ff5f1 100644 --- a/.github/workflows/create-jira-issue.yml +++ b/.github/workflows/create-jira-issue.yml @@ -6,7 +6,7 @@ on: jobs: call-create-jira-issue-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-create-jira-issue.yml@v0.11.0 + uses: ASFHyP3/actions/.github/workflows/reusable-create-jira-issue.yml@v0.11.1 secrets: JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }} JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }} diff --git a/.github/workflows/labeled-pr.yml b/.github/workflows/labeled-pr.yml index 63517428..592ee6d5 100644 --- a/.github/workflows/labeled-pr.yml +++ b/.github/workflows/labeled-pr.yml @@ -13,4 +13,4 @@ on: jobs: call-labeled-pr-check-workflow: # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.11.0 + uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.11.1 diff --git a/.github/workflows/release-checklist-comment.yml b/.github/workflows/release-checklist-comment.yml index 48f0e865..8826453e 100644 --- a/.github/workflows/release-checklist-comment.yml +++ b/.github/workflows/release-checklist-comment.yml @@ -10,7 +10,7 @@ on: jobs: call-release-workflow: # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-release-checklist-comment.yml@v0.11.0 + uses: ASFHyP3/actions/.github/workflows/reusable-release-checklist-comment.yml@v0.11.1 permissions: pull-requests: write secrets: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ab4dea5a..89f8f5a5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,7 +8,7 @@ on: jobs: call-release-workflow: # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.11.0 + uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.11.1 with: release_prefix: HyP3 ISCE2 secrets: diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index 8af2d4ba..eb3ebb64 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -5,10 +5,10 @@ on: push jobs: call-secrets-analysis-workflow: # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-secrets-analysis.yml@v0.11.0 + uses: ASFHyP3/actions/.github/workflows/reusable-secrets-analysis.yml@v0.11.1 call-flake8-workflow: # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-flake8.yml@v0.11.0 + uses: ASFHyP3/actions/.github/workflows/reusable-flake8.yml@v0.11.1 with: local_package_names: hyp3_isce2 diff --git a/.github/workflows/tag-version.yml b/.github/workflows/tag-version.yml index 51bbccaf..dba5905a 100644 --- a/.github/workflows/tag-version.yml +++ b/.github/workflows/tag-version.yml @@ -8,7 +8,7 @@ on: jobs: call-bump-version-workflow: # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.11.0 + uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.11.1 with: user: tools-bot email: UAF-asf-apd@alaska.edu diff --git a/.github/workflows/test-and-build.yml b/.github/workflows/test-and-build.yml index 6bab5887..600729ff 100644 --- a/.github/workflows/test-and-build.yml +++ b/.github/workflows/test-and-build.yml @@ -13,7 +13,7 @@ on: jobs: call-pytest-workflow: # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-pytest.yml@v0.11.0 + uses: ASFHyP3/actions/.github/workflows/reusable-pytest.yml@v0.11.1 with: local_package_name: hyp3_isce2 python_versions: >- @@ -21,12 +21,12 @@ jobs: call-version-info-workflow: # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-version-info.yml@v0.11.0 + uses: ASFHyP3/actions/.github/workflows/reusable-version-info.yml@v0.11.1 call-docker-ghcr-workflow: needs: call-version-info-workflow # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-docker-ghcr.yml@v0.11.0 + uses: ASFHyP3/actions/.github/workflows/reusable-docker-ghcr.yml@v0.11.1 with: version_tag: ${{ needs.call-version-info-workflow.outputs.version_tag }} release_branch: main From 739072b536c8758c4cc138391130b16ca36e541a Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Mon, 17 Jun 2024 14:16:14 -0800 Subject: [PATCH 05/25] Update .github/workflows/changelog.yml --- .github/workflows/changelog.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 616667e8..2e5415ca 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -15,5 +15,3 @@ jobs: call-changelog-check-workflow: # Docs: https://github.com/ASFHyP3/actions uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.11.1 - secrets: - USER_TOKEN: ${{ secrets.GITHUB_TOKEN }} From e094d3dc6dab7a82d1018fd02d0c95e29a9759d5 Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Wed, 26 Jun 2024 14:12:01 -0500 Subject: [PATCH 06/25] Drop Support for Python 3.8 and 3.9 --- CHANGELOG.md | 3 +++ environment.yml | 2 +- pyproject.toml | 4 +--- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc9d5d78..1b188ace 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.1.0] +### Removed +* Support for Python 3.8 and 3.9 has been dropped, 3.10 is now the minimum version. ## [1.0.1] ### Changed diff --git a/environment.yml b/environment.yml index ece430b5..d112f464 100644 --- a/environment.yml +++ b/environment.yml @@ -3,7 +3,7 @@ channels: - conda-forge - nodefaults dependencies: - - python>=3.9 + - python>=3.10 - numpy - pip - isce2>=2.6.3 diff --git a/pyproject.toml b/pyproject.toml index dbf2fbb4..260771fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "hyp3_isce2" -requires-python = ">=3.8" +requires-python = ">=3.10" authors = [ {name="tools-bot", email="UAF-asf-apd@alaska.edu"}, ] @@ -16,8 +16,6 @@ classifiers=[ "Natural Language :: English", "Operating System :: OS Independent", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", ] From ee24f24d44c4a300cd49b454ed8c466dd805804c Mon Sep 17 00:00:00 2001 From: Andrew Player Date: Wed, 26 Jun 2024 14:22:09 -0500 Subject: [PATCH 07/25] pytest workflow to use 3.10 as min version --- .github/workflows/test-and-build.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-and-build.yml b/.github/workflows/test-and-build.yml index 600729ff..1b5bdbb4 100644 --- a/.github/workflows/test-and-build.yml +++ b/.github/workflows/test-and-build.yml @@ -17,11 +17,13 @@ jobs: with: local_package_name: hyp3_isce2 python_versions: >- - ["3.9", "3.10"] + ["3.10", "3.11"] call-version-info-workflow: # Docs: https://github.com/ASFHyP3/actions uses: ASFHyP3/actions/.github/workflows/reusable-version-info.yml@v0.11.1 + with: + python_version: "3.10" call-docker-ghcr-workflow: needs: call-version-info-workflow From 66c434937fa1f8ac2376af11f51175fb719755e5 Mon Sep 17 00:00:00 2001 From: jacquelynsmale Date: Fri, 28 Jun 2024 09:40:49 -0800 Subject: [PATCH 08/25] update the parsing --- src/hyp3_isce2/merge_tops_bursts.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/hyp3_isce2/merge_tops_bursts.py b/src/hyp3_isce2/merge_tops_bursts.py index ae71429c..6201661b 100644 --- a/src/hyp3_isce2/merge_tops_bursts.py +++ b/src/hyp3_isce2/merge_tops_bursts.py @@ -128,9 +128,8 @@ def get_burst_metadata(product_paths: Iterable[Path]) -> Iterable[BurstProduct]: burst_indexes = [result.properties['burst']['burstIndex'] for result in results] polarization = [result.properties['polarization'] for result in results] start_utc = [ - datetime.datetime.strptime(result.properties['startTime'], '%Y-%m-%dT%H:%M:%S.%fZ') for result in results + datetime.datetime.strptime(result.properties['startTime'], '%Y-%m-%dT%H:%M:%SZ') for result in results ] - relative_orbits = [result.properties['pathNumber'] for result in results] n_lines = [int(meta['Radarnlines']) for meta in metas] n_samples = [int(meta['Radarnsamples']) for meta in metas] range_looks = [int(meta['Rangelooks']) for meta in metas] From 1d8a3420198851580f110efd11d470ee0054ccd6 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Fri, 28 Jun 2024 15:56:25 -0800 Subject: [PATCH 09/25] Update environment.yml --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index d112f464..596a7e61 100644 --- a/environment.yml +++ b/environment.yml @@ -7,7 +7,7 @@ dependencies: - numpy - pip - isce2>=2.6.3 - - dem_stitcher>=2.4.0 + - dem_stitcher>=2.5.6 - rasterio - shapely - jinja2 From 41f0c7ef092364c0bc58ac4dfae2d729fc38f6c7 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Fri, 28 Jun 2024 15:57:26 -0800 Subject: [PATCH 10/25] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 260771fa..040f5952 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ classifiers=[ ] dependencies = [ "numpy", - "dem_stitcher>=2.4.0", + "dem_stitcher>=2.5.6", "rasterio", "shapely", "jinja2", From 7794d36a47be3ca0de797f46662b62fca8894a57 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Fri, 28 Jun 2024 16:01:57 -0800 Subject: [PATCH 11/25] Update CHANGELOG.md --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b188ace..94fc09e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.1.1] +### Fixed +* Upgraded to `dem_sticher>=2.5.6` to resolve geoid download errors. Fixes [#215](https://github.com/ASFHyP3/hyp3-isce2/issues/215) + ## [1.1.0] ### Removed * Support for Python 3.8 and 3.9 has been dropped, 3.10 is now the minimum version. From 3b12806eac9f57304d01fc248f00a66c2e351c21 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Fri, 28 Jun 2024 16:32:47 -0800 Subject: [PATCH 12/25] Create .flake8 --- .flake8 | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..463e03a9 --- /dev/null +++ b/.flake8 @@ -0,0 +1,2 @@ +per-file-ignores = + src/hyp3_isce2/logging.py:A005 From e27bf3c216c650eebef75b88fd60b20fbc0d89c6 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Fri, 28 Jun 2024 16:37:21 -0800 Subject: [PATCH 13/25] Update .flake8 --- .flake8 | 1 + 1 file changed, 1 insertion(+) diff --git a/.flake8 b/.flake8 index 463e03a9..ecb2fd9f 100644 --- a/.flake8 +++ b/.flake8 @@ -1,2 +1,3 @@ +[flake8] per-file-ignores = src/hyp3_isce2/logging.py:A005 From a5d28c412fbda7d59805992a02359870669bd28b Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Mon, 1 Jul 2024 13:40:02 -0500 Subject: [PATCH 14/25] switch name of logging module to logger to fix built-in shadowing --- src/hyp3_isce2/{logging.py => logger.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/hyp3_isce2/{logging.py => logger.py} (100%) diff --git a/src/hyp3_isce2/logging.py b/src/hyp3_isce2/logger.py similarity index 100% rename from src/hyp3_isce2/logging.py rename to src/hyp3_isce2/logger.py From ca0afe334bde76d81a0e43e0cc25c89219894db3 Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Mon, 1 Jul 2024 13:43:07 -0500 Subject: [PATCH 15/25] fix references to hyp3_isc2.logging --- src/hyp3_isce2/insar_stripmap.py | 2 +- src/hyp3_isce2/insar_tops.py | 2 +- src/hyp3_isce2/insar_tops_burst.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/hyp3_isce2/insar_stripmap.py b/src/hyp3_isce2/insar_stripmap.py index 2353b749..b0cc8412 100644 --- a/src/hyp3_isce2/insar_stripmap.py +++ b/src/hyp3_isce2/insar_stripmap.py @@ -18,7 +18,7 @@ from hyp3_isce2 import stripmapapp_alos as stripmapapp from hyp3_isce2.dem import download_dem_for_isce2 -from hyp3_isce2.logging import configure_root_logger +from hyp3_isce2.logger import configure_root_logger log = logging.getLogger(__name__) diff --git a/src/hyp3_isce2/insar_tops.py b/src/hyp3_isce2/insar_tops.py index 639ffd80..669894a9 100644 --- a/src/hyp3_isce2/insar_tops.py +++ b/src/hyp3_isce2/insar_tops.py @@ -13,7 +13,7 @@ from hyp3_isce2 import slc from hyp3_isce2 import topsapp from hyp3_isce2.dem import download_dem_for_isce2 -from hyp3_isce2.logging import configure_root_logger +from hyp3_isce2.logger import configure_root_logger from hyp3_isce2.s1_auxcal import download_aux_cal from hyp3_isce2.utils import get_esa_credentials diff --git a/src/hyp3_isce2/insar_tops_burst.py b/src/hyp3_isce2/insar_tops_burst.py index 64d420d1..d2047016 100644 --- a/src/hyp3_isce2/insar_tops_burst.py +++ b/src/hyp3_isce2/insar_tops_burst.py @@ -35,7 +35,7 @@ validate_bursts, ) from hyp3_isce2.dem import download_dem_for_isce2 -from hyp3_isce2.logging import configure_root_logger +from hyp3_isce2.logger import configure_root_logger from hyp3_isce2.s1_auxcal import download_aux_cal from hyp3_isce2.utils import ( ParameterFile, From b1899a7c0c7621ec9014ff6346479fd5e8c7932e Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Mon, 1 Jul 2024 13:45:00 -0500 Subject: [PATCH 16/25] update .flak8 to remove ignore --- .flake8 | 3 --- 1 file changed, 3 deletions(-) diff --git a/.flake8 b/.flake8 index ecb2fd9f..e69de29b 100644 --- a/.flake8 +++ b/.flake8 @@ -1,3 +0,0 @@ -[flake8] -per-file-ignores = - src/hyp3_isce2/logging.py:A005 From 1491872eb2f4c749c124833647fcf619c787672e Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Mon, 1 Jul 2024 13:46:10 -0500 Subject: [PATCH 17/25] update changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94fc09e7..4142bf5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.1.2] +### Fixed +* Renamed `hyp3_isce2.logging` to `hyp3_isce2.logger` to avoid shadowing built-in. + ## [1.1.1] ### Fixed * Upgraded to `dem_sticher>=2.5.6` to resolve geoid download errors. Fixes [#215](https://github.com/ASFHyP3/hyp3-isce2/issues/215) From 6dd9d0585a214ae85707a4198fe610269de61de1 Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Mon, 1 Jul 2024 13:59:08 -0500 Subject: [PATCH 18/25] remove flake8 config --- .flake8 | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .flake8 diff --git a/.flake8 b/.flake8 deleted file mode 100644 index e69de29b..00000000 From c37dceba08824c6010855e612066569c1b61701c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2024 20:21:28 +0000 Subject: [PATCH 19/25] Bump ASFHyP3/actions from 0.11.1 to 0.11.2 Bumps [ASFHyP3/actions](https://github.com/asfhyp3/actions) from 0.11.1 to 0.11.2. - [Release notes](https://github.com/asfhyp3/actions/releases) - [Changelog](https://github.com/ASFHyP3/actions/blob/develop/CHANGELOG.md) - [Commits](https://github.com/asfhyp3/actions/compare/v0.11.1...v0.11.2) --- updated-dependencies: - dependency-name: ASFHyP3/actions dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/changelog.yml | 2 +- .github/workflows/create-jira-issue.yml | 2 +- .github/workflows/labeled-pr.yml | 2 +- .github/workflows/release-checklist-comment.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/static-analysis.yml | 4 ++-- .github/workflows/tag-version.yml | 2 +- .github/workflows/test-and-build.yml | 6 +++--- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 2e5415ca..1116782e 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -14,4 +14,4 @@ on: jobs: call-changelog-check-workflow: # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.11.1 + uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.11.2 diff --git a/.github/workflows/create-jira-issue.yml b/.github/workflows/create-jira-issue.yml index 129ff5f1..99489d50 100644 --- a/.github/workflows/create-jira-issue.yml +++ b/.github/workflows/create-jira-issue.yml @@ -6,7 +6,7 @@ on: jobs: call-create-jira-issue-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-create-jira-issue.yml@v0.11.1 + uses: ASFHyP3/actions/.github/workflows/reusable-create-jira-issue.yml@v0.11.2 secrets: JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }} JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }} diff --git a/.github/workflows/labeled-pr.yml b/.github/workflows/labeled-pr.yml index 592ee6d5..50e66b7f 100644 --- a/.github/workflows/labeled-pr.yml +++ b/.github/workflows/labeled-pr.yml @@ -13,4 +13,4 @@ on: jobs: call-labeled-pr-check-workflow: # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.11.1 + uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.11.2 diff --git a/.github/workflows/release-checklist-comment.yml b/.github/workflows/release-checklist-comment.yml index 8826453e..26d10fdf 100644 --- a/.github/workflows/release-checklist-comment.yml +++ b/.github/workflows/release-checklist-comment.yml @@ -10,7 +10,7 @@ on: jobs: call-release-workflow: # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-release-checklist-comment.yml@v0.11.1 + uses: ASFHyP3/actions/.github/workflows/reusable-release-checklist-comment.yml@v0.11.2 permissions: pull-requests: write secrets: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 89f8f5a5..88ddf931 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,7 +8,7 @@ on: jobs: call-release-workflow: # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.11.1 + uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.11.2 with: release_prefix: HyP3 ISCE2 secrets: diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index eb3ebb64..867b6851 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -5,10 +5,10 @@ on: push jobs: call-secrets-analysis-workflow: # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-secrets-analysis.yml@v0.11.1 + uses: ASFHyP3/actions/.github/workflows/reusable-secrets-analysis.yml@v0.11.2 call-flake8-workflow: # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-flake8.yml@v0.11.1 + uses: ASFHyP3/actions/.github/workflows/reusable-flake8.yml@v0.11.2 with: local_package_names: hyp3_isce2 diff --git a/.github/workflows/tag-version.yml b/.github/workflows/tag-version.yml index dba5905a..da2a91c3 100644 --- a/.github/workflows/tag-version.yml +++ b/.github/workflows/tag-version.yml @@ -8,7 +8,7 @@ on: jobs: call-bump-version-workflow: # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.11.1 + uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.11.2 with: user: tools-bot email: UAF-asf-apd@alaska.edu diff --git a/.github/workflows/test-and-build.yml b/.github/workflows/test-and-build.yml index 1b5bdbb4..5991aee4 100644 --- a/.github/workflows/test-and-build.yml +++ b/.github/workflows/test-and-build.yml @@ -13,7 +13,7 @@ on: jobs: call-pytest-workflow: # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-pytest.yml@v0.11.1 + uses: ASFHyP3/actions/.github/workflows/reusable-pytest.yml@v0.11.2 with: local_package_name: hyp3_isce2 python_versions: >- @@ -21,14 +21,14 @@ jobs: call-version-info-workflow: # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-version-info.yml@v0.11.1 + uses: ASFHyP3/actions/.github/workflows/reusable-version-info.yml@v0.11.2 with: python_version: "3.10" call-docker-ghcr-workflow: needs: call-version-info-workflow # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-docker-ghcr.yml@v0.11.1 + uses: ASFHyP3/actions/.github/workflows/reusable-docker-ghcr.yml@v0.11.2 with: version_tag: ${{ needs.call-version-info-workflow.outputs.version_tag }} release_branch: main From dae0c46462d917cf117c8973d2a52014b2eedfa9 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2024 20:57:45 +0000 Subject: [PATCH 20/25] update coverage image --- images/coverage.svg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/images/coverage.svg b/images/coverage.svg index 6c15cace..1618ede7 100644 --- a/images/coverage.svg +++ b/images/coverage.svg @@ -9,13 +9,13 @@ - + coverage coverage - 75% - 75% + 74% + 74% From 1b2cce85c40d016c4779758f06a650a258bfba7b Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Mon, 1 Jul 2024 16:00:50 -0500 Subject: [PATCH 21/25] update changelog --- CHANGELOG.md | 1 + src/hyp3_isce2/merge_tops_bursts.py | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4142bf5c..471a071c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [1.1.2] ### Fixed * Renamed `hyp3_isce2.logging` to `hyp3_isce2.logger` to avoid shadowing built-in. +* Parsing of product start times in `merge_tops_bursts`. ## [1.1.1] ### Fixed diff --git a/src/hyp3_isce2/merge_tops_bursts.py b/src/hyp3_isce2/merge_tops_bursts.py index 6201661b..38fa1623 100644 --- a/src/hyp3_isce2/merge_tops_bursts.py +++ b/src/hyp3_isce2/merge_tops_bursts.py @@ -130,6 +130,7 @@ def get_burst_metadata(product_paths: Iterable[Path]) -> Iterable[BurstProduct]: start_utc = [ datetime.datetime.strptime(result.properties['startTime'], '%Y-%m-%dT%H:%M:%SZ') for result in results ] + relative_orbits = [result.properties['pathNumber'] for result in results] n_lines = [int(meta['Radarnlines']) for meta in metas] n_samples = [int(meta['Radarnsamples']) for meta in metas] range_looks = [int(meta['Rangelooks']) for meta in metas] From c4499565786a855671b92de95c3bdb32ccceaaaf Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Tue, 2 Jul 2024 08:00:29 -0500 Subject: [PATCH 22/25] switch to using umm start time to get exact start time --- CHANGELOG.md | 2 +- src/hyp3_isce2/merge_tops_bursts.py | 6 +++--- tests/test_merge_tops_bursts.py | 7 +++++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 471a071c..62cc6c79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [1.1.2] ### Fixed * Renamed `hyp3_isce2.logging` to `hyp3_isce2.logger` to avoid shadowing built-in. -* Parsing of product start times in `merge_tops_bursts`. +* Source of product start times in `merge_tops_bursts` to use the `asf_search` umm record. ## [1.1.1] ### Fixed diff --git a/src/hyp3_isce2/merge_tops_bursts.py b/src/hyp3_isce2/merge_tops_bursts.py index 38fa1623..f3ef8457 100644 --- a/src/hyp3_isce2/merge_tops_bursts.py +++ b/src/hyp3_isce2/merge_tops_bursts.py @@ -127,9 +127,9 @@ def get_burst_metadata(product_paths: Iterable[Path]) -> Iterable[BurstProduct]: burst_ids = [result.properties['burst']['relativeBurstID'] for result in results] burst_indexes = [result.properties['burst']['burstIndex'] for result in results] polarization = [result.properties['polarization'] for result in results] - start_utc = [ - datetime.datetime.strptime(result.properties['startTime'], '%Y-%m-%dT%H:%M:%SZ') for result in results - ] + + start_utc_strs = [result.umm['TemporalExtent']['RangeDateTime']['BeginningDateTime'] for result in results] + start_utc = [datetime.datetime.strptime(utc, '%Y-%m-%dT%H:%M:%S.%fZ') for utc in start_utc_strs] relative_orbits = [result.properties['pathNumber'] for result in results] n_lines = [int(meta['Radarnlines']) for meta in metas] n_samples = [int(meta['Radarnsamples']) for meta in metas] diff --git a/tests/test_merge_tops_bursts.py b/tests/test_merge_tops_bursts.py index 54ff6bb0..29b8fa75 100644 --- a/tests/test_merge_tops_bursts.py +++ b/tests/test_merge_tops_bursts.py @@ -27,13 +27,16 @@ def mock_asf_search_results( path_number: int, ) -> asf_search.ASFSearchResults: product = asf_search.ASFProduct() - product.umm = {'InputGranules': [slc_name]} + product.umm = { + 'InputGranules': [slc_name], + 'TemporalExtent': {'RangeDateTime': {'BeginningDateTime': '2020-06-04T02:23:13.963847Z'}}, + } product.properties.update( { 'burst': {'subswath': subswath, 'burstIndex': burst_index, 'relativeBurstID': burst_id}, 'polarization': polarization, 'url': f'https://foo.com/{slc_name}/baz.zip', - 'startTime': '2020-06-04T02:23:13.963847Z', + # 'startTime': '2020-06-04T02:23:13.963847Z', 'pathNumber': path_number, } ) From 894c93b1881b7dc4443fc59d3a337c2a48043ae5 Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Tue, 2 Jul 2024 08:00:51 -0500 Subject: [PATCH 23/25] add --overwrite to watermask call to simplify local testing --- src/hyp3_isce2/water_mask.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/hyp3_isce2/water_mask.py b/src/hyp3_isce2/water_mask.py index 489869e6..06681287 100644 --- a/src/hyp3_isce2/water_mask.py +++ b/src/hyp3_isce2/water_mask.py @@ -125,6 +125,7 @@ def create_water_mask(input_image: str, output_image: str, gdal_format='ISCE', t merged_warped_path, f'--outfile={output_image}', '--calc="numpy.abs((A.astype(numpy.int16) + 1) - 2)"', # Change 1's to 0's and 0's to 1's. - f'--format={gdal_format}' + f'--format={gdal_format}', + '--overwrite', ] subprocess.run(flip_values_command, check=True) From 838d47ec655fdc95b68ea5ad630f24ffa642a759 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 2 Jul 2024 13:02:51 +0000 Subject: [PATCH 24/25] update coverage image --- images/coverage.svg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/images/coverage.svg b/images/coverage.svg index 1618ede7..6c15cace 100644 --- a/images/coverage.svg +++ b/images/coverage.svg @@ -9,13 +9,13 @@ - + coverage coverage - 74% - 74% + 75% + 75% From 2925b394532400c469284ae9a6a3ec32fec49659 Mon Sep 17 00:00:00 2001 From: Forrest Williams Date: Tue, 2 Jul 2024 08:06:53 -0500 Subject: [PATCH 25/25] remove obsolete code --- tests/test_merge_tops_bursts.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_merge_tops_bursts.py b/tests/test_merge_tops_bursts.py index 29b8fa75..c2b8bfec 100644 --- a/tests/test_merge_tops_bursts.py +++ b/tests/test_merge_tops_bursts.py @@ -36,7 +36,6 @@ def mock_asf_search_results( 'burst': {'subswath': subswath, 'burstIndex': burst_index, 'relativeBurstID': burst_id}, 'polarization': polarization, 'url': f'https://foo.com/{slc_name}/baz.zip', - # 'startTime': '2020-06-04T02:23:13.963847Z', 'pathNumber': path_number, } )