diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 3a01155a..1116782e 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -14,6 +14,4 @@ on: jobs: call-changelog-check-workflow: # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.11.0 - secrets: - USER_TOKEN: ${{ secrets.GITHUB_TOKEN }} + uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.11.2 diff --git a/.github/workflows/create-jira-issue.yml b/.github/workflows/create-jira-issue.yml index 0b69efec..99489d50 100644 --- a/.github/workflows/create-jira-issue.yml +++ b/.github/workflows/create-jira-issue.yml @@ -6,7 +6,7 @@ on: jobs: call-create-jira-issue-workflow: - uses: ASFHyP3/actions/.github/workflows/reusable-create-jira-issue.yml@v0.11.0 + uses: ASFHyP3/actions/.github/workflows/reusable-create-jira-issue.yml@v0.11.2 secrets: JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }} JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }} diff --git a/.github/workflows/labeled-pr.yml b/.github/workflows/labeled-pr.yml index 63517428..50e66b7f 100644 --- a/.github/workflows/labeled-pr.yml +++ b/.github/workflows/labeled-pr.yml @@ -13,4 +13,4 @@ on: jobs: call-labeled-pr-check-workflow: # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.11.0 + uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.11.2 diff --git a/.github/workflows/release-checklist-comment.yml b/.github/workflows/release-checklist-comment.yml index 48f0e865..26d10fdf 100644 --- a/.github/workflows/release-checklist-comment.yml +++ b/.github/workflows/release-checklist-comment.yml @@ -10,7 +10,7 @@ on: jobs: call-release-workflow: # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-release-checklist-comment.yml@v0.11.0 + uses: ASFHyP3/actions/.github/workflows/reusable-release-checklist-comment.yml@v0.11.2 permissions: pull-requests: write secrets: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ab4dea5a..88ddf931 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,7 +8,7 @@ on: jobs: call-release-workflow: # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.11.0 + uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.11.2 with: release_prefix: HyP3 ISCE2 secrets: diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index 8af2d4ba..867b6851 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -5,10 +5,10 @@ on: push jobs: call-secrets-analysis-workflow: # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-secrets-analysis.yml@v0.11.0 + uses: ASFHyP3/actions/.github/workflows/reusable-secrets-analysis.yml@v0.11.2 call-flake8-workflow: # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-flake8.yml@v0.11.0 + uses: ASFHyP3/actions/.github/workflows/reusable-flake8.yml@v0.11.2 with: local_package_names: hyp3_isce2 diff --git a/.github/workflows/tag-version.yml b/.github/workflows/tag-version.yml index 51bbccaf..da2a91c3 100644 --- a/.github/workflows/tag-version.yml +++ b/.github/workflows/tag-version.yml @@ -8,7 +8,7 @@ on: jobs: call-bump-version-workflow: # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.11.0 + uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.11.2 with: user: tools-bot email: UAF-asf-apd@alaska.edu diff --git a/.github/workflows/test-and-build.yml b/.github/workflows/test-and-build.yml index 6bab5887..5991aee4 100644 --- a/.github/workflows/test-and-build.yml +++ b/.github/workflows/test-and-build.yml @@ -13,20 +13,22 @@ on: jobs: call-pytest-workflow: # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-pytest.yml@v0.11.0 + uses: ASFHyP3/actions/.github/workflows/reusable-pytest.yml@v0.11.2 with: local_package_name: hyp3_isce2 python_versions: >- - ["3.9", "3.10"] + ["3.10", "3.11"] call-version-info-workflow: # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-version-info.yml@v0.11.0 + uses: ASFHyP3/actions/.github/workflows/reusable-version-info.yml@v0.11.2 + with: + python_version: "3.10" call-docker-ghcr-workflow: needs: call-version-info-workflow # Docs: https://github.com/ASFHyP3/actions - uses: ASFHyP3/actions/.github/workflows/reusable-docker-ghcr.yml@v0.11.0 + uses: ASFHyP3/actions/.github/workflows/reusable-docker-ghcr.yml@v0.11.2 with: version_tag: ${{ needs.call-version-info-workflow.outputs.version_tag }} release_branch: main diff --git a/CHANGELOG.md b/CHANGELOG.md index cc9d5d78..62cc6c79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.1.2] +### Fixed +* Renamed `hyp3_isce2.logging` to `hyp3_isce2.logger` to avoid shadowing built-in. +* Source of product start times in `merge_tops_bursts` to use the `asf_search` umm record. + +## [1.1.1] +### Fixed +* Upgraded to `dem_sticher>=2.5.6` to resolve geoid download errors. Fixes [#215](https://github.com/ASFHyP3/hyp3-isce2/issues/215) + +## [1.1.0] +### Removed +* Support for Python 3.8 and 3.9 has been dropped, 3.10 is now the minimum version. ## [1.0.1] ### Changed diff --git a/README.md b/README.md index c64afdab..bf1822eb 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,37 @@ docker run -it --rm \ **NOTE** Each workflow can also be accessed via an alternative CLI with the format (`[WORKFLOW_NAME] [WORKFLOW_ARGS]`) +#### Docker Outputs + +To retain hyp3_isce2 output files running via Docker there are two recommended approaches: + +1. Use a volume mount + +Add the `-w /tmp -v [localdir]:/tmp` flags after docker run. `-w` changes the working directory of the container to `/tmp` and `-v` will mount whichever local directory you choose so that such that hyp3_isce3 outputs are preserved locally. + +1. Copy outputs to remote object storage + +Append the `--bucket` and `--bucket-prefix` to [WORKFLOW_ARGS]. *Only the final output files and zipped archive of those files is uploaded.* This also requires that AWS credentials to write to the bucket are available to the running container. For example, to write outputs to a hypothetical bucket `s3://hypothetical-bucket/test-run/`: + +``` +docker run -it --rm \ + -e AWS_ACCESS_KEY_ID=[YOUR_KEY] \ + -e AWS_SECRET_ACCESS_KEY=[YOUR_SECRET] \ + -e AWS_SESSION_TOKEN=[YOUR_TOKEN] \ + -e EARTHDATA_USERNAME=[YOUR_USERNAME_HERE] \ + -e EARTHDATA_PASSWORD=[YOUR_PASSWORD_HERE] \ + -e ESA_USERNAME=[YOUR_USERNAME_HERE] \ + -e ESA_PASSWORD=[YOUR_PASSWORD_HERE] \ + ghcr.io/asfhyp3/hyp3-isce2:latest \ + ++process [WORKFLOW_NAME] \ + [WORKFLOW_ARGS] \ + --bucket "hypothetical-bucket" \ + --bucket-prefix "test-run" +``` + +Tip: you can use [`docker run --env-file`](https://docs.docker.com/reference/cli/docker/container/run/#env) to capture all the necessary environment variables in a single file. + + ## Developer Setup 1. Ensure that conda is installed on your system (we recommend using [mambaforge](https://github.com/conda-forge/miniforge#mambaforge) to reduce setup times). 2. Download a local version of the `hyp3-isce2` repository (`git clone https://github.com/ASFHyP3/hyp3-isce2.git`) diff --git a/environment.yml b/environment.yml index ece430b5..596a7e61 100644 --- a/environment.yml +++ b/environment.yml @@ -3,11 +3,11 @@ channels: - conda-forge - nodefaults dependencies: - - python>=3.9 + - python>=3.10 - numpy - pip - isce2>=2.6.3 - - dem_stitcher>=2.4.0 + - dem_stitcher>=2.5.6 - rasterio - shapely - jinja2 diff --git a/pyproject.toml b/pyproject.toml index dbf2fbb4..040f5952 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "hyp3_isce2" -requires-python = ">=3.8" +requires-python = ">=3.10" authors = [ {name="tools-bot", email="UAF-asf-apd@alaska.edu"}, ] @@ -16,14 +16,12 @@ classifiers=[ "Natural Language :: English", "Operating System :: OS Independent", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", ] dependencies = [ "numpy", - "dem_stitcher>=2.4.0", + "dem_stitcher>=2.5.6", "rasterio", "shapely", "jinja2", diff --git a/src/hyp3_isce2/insar_stripmap.py b/src/hyp3_isce2/insar_stripmap.py index 2353b749..b0cc8412 100644 --- a/src/hyp3_isce2/insar_stripmap.py +++ b/src/hyp3_isce2/insar_stripmap.py @@ -18,7 +18,7 @@ from hyp3_isce2 import stripmapapp_alos as stripmapapp from hyp3_isce2.dem import download_dem_for_isce2 -from hyp3_isce2.logging import configure_root_logger +from hyp3_isce2.logger import configure_root_logger log = logging.getLogger(__name__) diff --git a/src/hyp3_isce2/insar_tops.py b/src/hyp3_isce2/insar_tops.py index 639ffd80..669894a9 100644 --- a/src/hyp3_isce2/insar_tops.py +++ b/src/hyp3_isce2/insar_tops.py @@ -13,7 +13,7 @@ from hyp3_isce2 import slc from hyp3_isce2 import topsapp from hyp3_isce2.dem import download_dem_for_isce2 -from hyp3_isce2.logging import configure_root_logger +from hyp3_isce2.logger import configure_root_logger from hyp3_isce2.s1_auxcal import download_aux_cal from hyp3_isce2.utils import get_esa_credentials diff --git a/src/hyp3_isce2/insar_tops_burst.py b/src/hyp3_isce2/insar_tops_burst.py index 64d420d1..d2047016 100644 --- a/src/hyp3_isce2/insar_tops_burst.py +++ b/src/hyp3_isce2/insar_tops_burst.py @@ -35,7 +35,7 @@ validate_bursts, ) from hyp3_isce2.dem import download_dem_for_isce2 -from hyp3_isce2.logging import configure_root_logger +from hyp3_isce2.logger import configure_root_logger from hyp3_isce2.s1_auxcal import download_aux_cal from hyp3_isce2.utils import ( ParameterFile, diff --git a/src/hyp3_isce2/logging.py b/src/hyp3_isce2/logger.py similarity index 100% rename from src/hyp3_isce2/logging.py rename to src/hyp3_isce2/logger.py diff --git a/src/hyp3_isce2/merge_tops_bursts.py b/src/hyp3_isce2/merge_tops_bursts.py index ae71429c..f3ef8457 100644 --- a/src/hyp3_isce2/merge_tops_bursts.py +++ b/src/hyp3_isce2/merge_tops_bursts.py @@ -127,9 +127,9 @@ def get_burst_metadata(product_paths: Iterable[Path]) -> Iterable[BurstProduct]: burst_ids = [result.properties['burst']['relativeBurstID'] for result in results] burst_indexes = [result.properties['burst']['burstIndex'] for result in results] polarization = [result.properties['polarization'] for result in results] - start_utc = [ - datetime.datetime.strptime(result.properties['startTime'], '%Y-%m-%dT%H:%M:%S.%fZ') for result in results - ] + + start_utc_strs = [result.umm['TemporalExtent']['RangeDateTime']['BeginningDateTime'] for result in results] + start_utc = [datetime.datetime.strptime(utc, '%Y-%m-%dT%H:%M:%S.%fZ') for utc in start_utc_strs] relative_orbits = [result.properties['pathNumber'] for result in results] n_lines = [int(meta['Radarnlines']) for meta in metas] n_samples = [int(meta['Radarnsamples']) for meta in metas] diff --git a/src/hyp3_isce2/water_mask.py b/src/hyp3_isce2/water_mask.py index 489869e6..06681287 100644 --- a/src/hyp3_isce2/water_mask.py +++ b/src/hyp3_isce2/water_mask.py @@ -125,6 +125,7 @@ def create_water_mask(input_image: str, output_image: str, gdal_format='ISCE', t merged_warped_path, f'--outfile={output_image}', '--calc="numpy.abs((A.astype(numpy.int16) + 1) - 2)"', # Change 1's to 0's and 0's to 1's. - f'--format={gdal_format}' + f'--format={gdal_format}', + '--overwrite', ] subprocess.run(flip_values_command, check=True) diff --git a/tests/test_merge_tops_bursts.py b/tests/test_merge_tops_bursts.py index 54ff6bb0..c2b8bfec 100644 --- a/tests/test_merge_tops_bursts.py +++ b/tests/test_merge_tops_bursts.py @@ -27,13 +27,15 @@ def mock_asf_search_results( path_number: int, ) -> asf_search.ASFSearchResults: product = asf_search.ASFProduct() - product.umm = {'InputGranules': [slc_name]} + product.umm = { + 'InputGranules': [slc_name], + 'TemporalExtent': {'RangeDateTime': {'BeginningDateTime': '2020-06-04T02:23:13.963847Z'}}, + } product.properties.update( { 'burst': {'subswath': subswath, 'burstIndex': burst_index, 'relativeBurstID': burst_id}, 'polarization': polarization, 'url': f'https://foo.com/{slc_name}/baz.zip', - 'startTime': '2020-06-04T02:23:13.963847Z', 'pathNumber': path_number, } )