Merge pull request #154 from ASFHyP3/develop

Release v0.9.0

CHANGELOG.md

@@ -6,6 +6,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/)
 and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.9.0]
+
+### Changed
+* Upgraded `hyp3lib` dependency to version `2.x.x`.
+* As of [HyP3-lib v2.0.0](https://github.com/ASFHyP3/hyp3-lib/releases/tag/v2.0.0), the [Copernicus Data Space Ecosystem (CDSE)](https://dataspace.copernicus.eu/) will now be used for downloading Sentinel-1 orbit files from ESA.
+* CDSE credentials must be provided via the `--esa-username` and `--esa-password` command-line options, the `ESA_USERNAME` and `ESA_PASSWORD` environment variables, or a `.netrc` file.
+* All the special ISCE2 environment variable, python path, and system path handling has been moved to `hyp3_isce2.__init__.py` to ensure it's always done before using any object in this package.
+* All [subprocess](https://docs.python.org/3/library/subprocess.html#using-the-subprocess-module) calls use `subprocess.run`, as recommended.
+
 ## [0.8.1]
 ### Fixed
 * Fixed a typo in the call to `imageMath.py`.

README.md

@@ -50,12 +50,13 @@ Note that ISCE2 currently only supports masking *after* phase unwrapping. As suc
 phase unwrapping errors that may occur over water, but simply removes distracting signals afterwards to improve 
 the visualization of the interferogram.
 
-### Earthdata Login
+### Earthdata Login and ESA Credentials
 
-For all workflows the user will need to provide their Earthdata login credentials to download input data. 
-If you do not already have an account, you can sign up [here](https://urs.earthdata.nasa.gov/home). 
-Your credentials can either be passed to the workflows via environment variables 
-(`EARTHDATA_USERNAME` and `EARTHDATA_PASSWORD`), or via your `.netrc` file. If you haven't set up a `.netrc` file 
+For all workflows, the user must provide their Earthdata Login credentials and ESA Copernicus Data Space Ecosystem (CDSE) credentials in order to download input data.
+If you do not already have an Earthdata account, you can sign up [here](https://urs.earthdata.nasa.gov/home). 
+If you do not already have a CDSE account, you can sign up [here](https://dataspace.copernicus.eu). 
+Your credentials can be passed to the workflows via command-line options (`--esa-username` and  `--esa-password`), environment variables 
+(`EARTHDATA_USERNAME`, `EARTHDATA_PASSWORD`, `ESA_USERNAME`, and `ESA_PASSWORD`), or via your `.netrc` file. If you haven't set up a `.netrc` file 
 before, check out this [guide](https://harmony.earthdata.nasa.gov/docs#getting-started) to get started.
 
 ### Docker Container
@@ -65,6 +66,8 @@ deployment. To run the current version of the project's container, use this comm
 docker run -it --rm \
     -e EARTHDATA_USERNAME=[YOUR_USERNAME_HERE] \
     -e EARTHDATA_PASSWORD=[YOUR_PASSWORD_HERE] \
+    -e ESA_USERNAME=[YOUR_USERNAME_HERE] \
+    -e ESA_PASSWORD=[YOUR_PASSWORD_HERE] \
     ghcr.io/asfhyp3/hyp3-isce2:latest \
     ++process [WORKFLOW_NAME] \
     [WORKFLOW_ARGS]

environment.yml

@@ -17,6 +17,7 @@ dependencies:
   - opencv
   - s3fs
   - pyarrow
+  - hyp3lib>=2,<3
   # For packaging, and testing
   - flake8
   - flake8-import-order
@@ -29,5 +30,3 @@ dependencies:
   - pytest-console-scripts
   - pytest-cov
   - pytest-mock
-  # For running
-  - hyp3lib>=1.6,<2

pyproject.toml

@@ -22,7 +22,20 @@ classifiers=[
     "Programming Language :: Python :: 3.11",
 ]
 dependencies = [
-    # insert python dependencies as list here
+  "numpy",
+  "dem_stitcher>=2.4.0",
+  "rasterio",
+  "shapely",
+  "jinja2",
+  "asf_search>=6.4.0",
+  "geopandas",
+  "gdal",
+  "s3fs",
+  "pyarrow",
+  "hyp3lib>=2,<3",
+  # Conda-forge only dependencies are listed below
+  # "opencv",
+  # "isce2>=2.6.3",
 ]
 dynamic = ["version", "readme"]
 

src/hyp3_isce2/__init__.py

@@ -1,6 +1,17 @@
 """HyP3 plugin for ISCE2 processing"""
-
+import os
 from importlib.metadata import version
+from pathlib import Path
+
+# Ensures all ISCE2 paths and environment variables are set when using this module, see:
+# https://github.com/isce-framework/isce2/blob/main/__init__.py#L41-L50
+import isce  # noqa: F401
+# ISCE2 also needs its applications to be on the system path, even though they say it's only "for convenience", see:
+# https://github.com/isce-framework/isce2#setup-your-environment
+ISCE_APPLICATIONS = str(Path(os.environ['ISCE_HOME']) / 'applications')
+if ISCE_APPLICATIONS not in (PATH := os.environ['PATH'].split(os.pathsep)):
+    os.environ['PATH'] = os.pathsep.join([ISCE_APPLICATIONS] + PATH)
+
 
 __version__ = version(__name__)
 

src/hyp3_isce2/burst.py

@@ -10,7 +10,6 @@
 from typing import Iterator, List, Optional, Tuple, Union
 
 import asf_search
-import isce  # noqa: F401
 import requests
 from isceobj.Sensor.TOPS.Sentinel1 import Sentinel1
 from lxml import etree

src/hyp3_isce2/dem.py

@@ -13,7 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import site
 import subprocess
 from pathlib import Path
 
@@ -40,9 +39,8 @@ def tag_dem_xml_as_ellipsoidal(dem_path: Path) -> str:
 
 
 def fix_image_xml(xml_path: str) -> None:
-    fix_image_path = Path(site.getsitepackages()[0]) / 'isce' / 'applications' / 'fixImageXml.py'
-    fix_cmd = ' '.join([str(fix_image_path), '-i', xml_path, '--full'])
-    subprocess.check_call(fix_cmd, shell=True)
+    cmd = ['fixImageXml.py', '-i', xml_path, '--full']
+    subprocess.run(cmd, check=True)
 
 
 def buffer_extent(extent: list, buffer: float) -> list:

src/hyp3_isce2/insar_stripmap.py

@@ -7,7 +7,6 @@
 import logging
 import os
 import shutil
-import site
 import sys
 import zipfile
 from pathlib import Path
@@ -23,12 +22,6 @@
 
 log = logging.getLogger(__name__)
 
-# ISCE needs its applications to be on the system path.
-# See https://github.com/isce-framework/isce2#setup-your-environment
-ISCE_APPLICATIONS = Path(site.getsitepackages()[0]) / 'isce' / 'applications'
-if str(ISCE_APPLICATIONS) not in os.environ['PATH'].split(os.pathsep):
-    os.environ['PATH'] = str(ISCE_APPLICATIONS) + os.pathsep + os.environ['PATH']
-
 
 def insar_stripmap(user: str, password: str, reference_scene: str, secondary_scene: str) -> Path:
     """Create a Stripmap interferogram

src/hyp3_isce2/insar_tops.py

@@ -2,11 +2,10 @@
 
 import argparse
 import logging
-import os
-import site
 import sys
 from pathlib import Path
 from shutil import copyfile, make_archive
+from typing import Optional
 
 from hyp3lib.aws import upload_file_to_s3
 from hyp3lib.get_orb import downloadSentinelOrbitFile
@@ -16,16 +15,11 @@
 from hyp3_isce2.dem import download_dem_for_isce2
 from hyp3_isce2.logging import configure_root_logger
 from hyp3_isce2.s1_auxcal import download_aux_cal
+from hyp3_isce2.utils import get_esa_credentials
 
 
 log = logging.getLogger(__name__)
 
-# ISCE needs its applications to be on the system path.
-# See https://github.com/isce-framework/isce2#setup-your-environment
-ISCE_APPLICATIONS = Path(site.getsitepackages()[0]) / 'isce' / 'applications'
-if str(ISCE_APPLICATIONS) not in os.environ['PATH'].split(os.pathsep):
-    os.environ['PATH'] = str(ISCE_APPLICATIONS) + os.pathsep + os.environ['PATH']
-
 
 def insar_tops(
     reference_scene: str,
@@ -34,6 +28,8 @@ def insar_tops(
     polarization: str = 'VV',
     azimuth_looks: int = 4,
     range_looks: int = 20,
+    esa_username: Optional[str] = None,
+    esa_password: Optional[str] = None,
 ) -> Path:
     """Create a full-SLC interferogram
 
@@ -44,10 +40,15 @@ def insar_tops(
         polarization: Polarization to use
         azimuth_looks: Number of azimuth looks
         range_looks: Number of range looks
+        esa_username: Username for ESA's Copernicus Data Space Ecosystem
+        esa_password: Password for ESA's Copernicus Data Space Ecosystem
 
     Returns:
         Path to the output files
     """
+    if (esa_username is None) or (esa_password is None):
+        esa_username, esa_password = get_esa_credentials()
+
     orbit_dir = Path('orbits')
     aux_cal_dir = Path('aux_cal')
     dem_dir = Path('dem')
@@ -62,7 +63,7 @@ def insar_tops(
 
     orbit_dir.mkdir(exist_ok=True, parents=True)
     for granule in (reference_scene, secondary_scene):
-        downloadSentinelOrbitFile(granule, str(orbit_dir))
+        downloadSentinelOrbitFile(granule, str(orbit_dir), esa_credentials=(esa_username, esa_password))
 
     config = topsapp.TopsappBurstConfig(
         reference_safe=f'{reference_scene}.SAFE',
@@ -90,6 +91,8 @@ def main():
 
     parser.add_argument('--bucket', help='AWS S3 bucket HyP3 for upload the final product(s)')
     parser.add_argument('--bucket-prefix', default='', help='Add a bucket prefix to product(s)')
+    parser.add_argument('--esa-username', default=None, help="Username for ESA\'s Copernicus Data Space Ecosystem")
+    parser.add_argument('--esa-password', default=None, help="Password for ESA\'s Copernicus Data Space Ecosystem")
     parser.add_argument('--reference-scene', type=str, required=True)
     parser.add_argument('--secondary-scene', type=str, required=True)
     parser.add_argument('--polarization', type=str, default='VV')
@@ -114,6 +117,8 @@ def main():
         polarization=args.polarization,
         azimuth_looks=azimuth_looks,
         range_looks=range_looks,
+        esa_username=args.esa_username,
+        esa_password=args.esa_password,
     )
 
     log.info('ISCE2 TopsApp run completed successfully')