Merge pull request #4 from ASFHyP3/develop

Use `s1_orbits` (ASFHyP3#228)

CHANGELOG.md

@@ -6,6 +6,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/)
 and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [2.0.0]
+### Changed
+- Orbit files are now retrieved using the [s1-orbits](https://github.com/ASFHyP3/sentinel1-orbits-py) library.
+### Removed
+- Removed support for the `--esa-username` and `--esa-password` command-line options and the `ESA_USERNAME` and `ESA_PASSWORD` environment variables.
+
 ## [1.1.3]
 ### Fixed
 * Upgrade to `dem_stitcher>=2.5.8` to resolve geoid download errors. See [dem-stitcher #96](https://github.com/ACCESS-Cloud-Based-InSAR/dem-stitcher/issues/96).

README.md

@@ -73,13 +73,12 @@ those with 5x1 looks have a pixel spacing of 20 m.
 There is always a water mask geotiff file included in the product package, but setting the **apply-water-mask** 
 (`--apply-water-mask`) option to True will apply the mask to the wrapped interferogram prior to phase unwrapping.
 
-### Earthdata Login and ESA Credentials
+### Earthdata Login Credentials
 
-For all workflows, the user must provide their Earthdata Login credentials and ESA Copernicus Data Space Ecosystem (CDSE) credentials in order to download input data.
+For all workflows, the user must provide their Earthdata Login credentials in order to download input data.
 If you do not already have an Earthdata account, you can sign up [here](https://urs.earthdata.nasa.gov/home). 
-If you do not already have a CDSE account, you can sign up [here](https://dataspace.copernicus.eu). 
-Your credentials can be passed to the workflows via command-line options (`--esa-username` and  `--esa-password`), environment variables 
-(`EARTHDATA_USERNAME`, `EARTHDATA_PASSWORD`, `ESA_USERNAME`, and `ESA_PASSWORD`), or via your `.netrc` file. If you haven't set up a `.netrc` file 
+Your credentials can be passed to the workflows via environment variables
+(`EARTHDATA_USERNAME`, `EARTHDATA_PASSWORD`) or via your `.netrc` file. If you haven't set up a `.netrc` file
 before, check out this [guide](https://harmony.earthdata.nasa.gov/docs#getting-started) to get started.
 
 ### Docker Container
@@ -89,8 +88,6 @@ deployment. To run the current version of the project's container, use this comm
 docker run -it --rm \
     -e EARTHDATA_USERNAME=[YOUR_USERNAME_HERE] \
     -e EARTHDATA_PASSWORD=[YOUR_PASSWORD_HERE] \
-    -e ESA_USERNAME=[YOUR_USERNAME_HERE] \
-    -e ESA_PASSWORD=[YOUR_PASSWORD_HERE] \
     ghcr.io/asfhyp3/hyp3-isce2:latest \
     ++process [WORKFLOW_NAME] \
     [WORKFLOW_ARGS]
@@ -117,8 +114,6 @@ docker run -it --rm \
     -e AWS_SESSION_TOKEN=[YOUR_TOKEN] \ 
     -e EARTHDATA_USERNAME=[YOUR_USERNAME_HERE] \
     -e EARTHDATA_PASSWORD=[YOUR_PASSWORD_HERE] \
-    -e ESA_USERNAME=[YOUR_USERNAME_HERE] \
-    -e ESA_PASSWORD=[YOUR_PASSWORD_HERE] \
     ghcr.io/asfhyp3/hyp3-isce2:latest \
       ++process [WORKFLOW_NAME] \
       [WORKFLOW_ARGS] \

environment.yml

@@ -28,3 +28,4 @@ dependencies:
   - pytest-mock
   # For running
   - hyp3lib>=3,<4
+  - s1_orbits

pyproject.toml

@@ -28,6 +28,7 @@ dependencies = [
   "asf_search>=6.4.0",
   "gdal",
   "hyp3lib>=3,<4",
+  "s1_orbits",
   # Conda-forge only dependencies are listed below
   # "opencv",
   # "isce2>=2.6.3",

src/hyp3_isce2/insar_tops.py

@@ -5,17 +5,15 @@
 import sys
 from pathlib import Path
 from shutil import copyfile, make_archive
-from typing import Optional
 
 from hyp3lib.aws import upload_file_to_s3
-from hyp3lib.get_orb import downloadSentinelOrbitFile
+from s1_orbits import fetch_for_scene
 
 from hyp3_isce2 import slc
 from hyp3_isce2 import topsapp
 from hyp3_isce2.dem import download_dem_for_isce2
 from hyp3_isce2.logger import configure_root_logger
 from hyp3_isce2.s1_auxcal import download_aux_cal
-from hyp3_isce2.utils import get_esa_credentials
 
 
 log = logging.getLogger(__name__)
@@ -28,8 +26,6 @@ def insar_tops(
     polarization: str = 'VV',
     azimuth_looks: int = 4,
     range_looks: int = 20,
-    esa_username: Optional[str] = None,
-    esa_password: Optional[str] = None,
 ) -> Path:
     """Create a full-SLC interferogram
 
@@ -40,15 +36,10 @@ def insar_tops(
         polarization: Polarization to use
         azimuth_looks: Number of azimuth looks
         range_looks: Number of range looks
-        esa_username: Username for ESA's Copernicus Data Space Ecosystem
-        esa_password: Password for ESA's Copernicus Data Space Ecosystem
 
     Returns:
         Path to the output files
     """
-    if (esa_username is None) or (esa_password is None):
-        esa_username, esa_password = get_esa_credentials()
-
     orbit_dir = Path('orbits')
     aux_cal_dir = Path('aux_cal')
     dem_dir = Path('dem')
@@ -63,7 +54,9 @@ def insar_tops(
 
     orbit_dir.mkdir(exist_ok=True, parents=True)
     for granule in (reference_scene, secondary_scene):
-        downloadSentinelOrbitFile(granule, str(orbit_dir), esa_credentials=(esa_username, esa_password))
+        log.info(f'Downloading orbit file for {granule}')
+        orbit_file = fetch_for_scene(granule, dir=orbit_dir)
+        log.info(f'Got orbit file {orbit_file} from s1_orbits')
 
     config = topsapp.TopsappBurstConfig(
         reference_safe=f'{reference_scene}.SAFE',
@@ -93,8 +86,6 @@ def main():
 
     parser.add_argument('--bucket', help='AWS S3 bucket HyP3 for upload the final product(s)')
     parser.add_argument('--bucket-prefix', default='', help='Add a bucket prefix to product(s)')
-    parser.add_argument('--esa-username', default=None, help="Username for ESA\'s Copernicus Data Space Ecosystem")
-    parser.add_argument('--esa-password', default=None, help="Password for ESA\'s Copernicus Data Space Ecosystem")
     parser.add_argument('--reference-scene', type=str, required=True)
     parser.add_argument('--secondary-scene', type=str, required=True)
     parser.add_argument('--polarization', type=str, choices=['VV', 'HH'], default='VV')
@@ -119,8 +110,6 @@ def main():
         polarization=args.polarization,
         azimuth_looks=azimuth_looks,
         range_looks=range_looks,
-        esa_username=args.esa_username,
-        esa_password=args.esa_password,
     )
 
     log.info('ISCE2 TopsApp run completed successfully')

src/hyp3_isce2/insar_tops_burst.py

@@ -13,13 +13,13 @@
 
 import isce
 from hyp3lib.aws import upload_file_to_s3
-from hyp3lib.get_orb import downloadSentinelOrbitFile
 from hyp3lib.image import create_thumbnail
 from hyp3lib.util import string_is_true
 from isceobj.TopsProc.runMergeBursts import multilook
 from lxml import etree
 from osgeo import gdal, gdalconst
 from pyproj import CRS
+from s1_orbits import fetch_for_scene
 
 import hyp3_isce2
 import hyp3_isce2.metadata.util
@@ -39,7 +39,6 @@
 from hyp3_isce2.s1_auxcal import download_aux_cal
 from hyp3_isce2.utils import (
     ParameterFile,
-    get_esa_credentials,
     image_math,
     isce2_copy,
     make_browse_image,
@@ -70,8 +69,6 @@ def insar_tops_burst(
     azimuth_looks: int = 4,
     range_looks: int = 20,
     apply_water_mask: bool = False,
-    esa_username: Optional[str] = None,
-    esa_password: Optional[str] = None,
 ) -> Path:
     """Create a burst interferogram
 
@@ -82,15 +79,10 @@ def insar_tops_burst(
         azimuth_looks: Number of azimuth looks
         range_looks: Number of range looks
         apply_water_mask: Whether to apply a pre-unwrap water mask
-        esa_username: Username for ESA's Copernicus Data Space Ecosystem
-        esa_password: Password for ESA's Copernicus Data Space Ecosystem
 
     Returns:
         Path to results directory
     """
-    if (esa_username is None) or (esa_password is None):
-        esa_username, esa_password = get_esa_credentials()
-
     orbit_dir = Path('orbits')
     aux_cal_dir = Path('aux_cal')
     dem_dir = Path('dem')
@@ -125,7 +117,9 @@ def insar_tops_burst(
 
     orbit_dir.mkdir(exist_ok=True, parents=True)
     for granule in (ref_params.granule, sec_params.granule):
-        downloadSentinelOrbitFile(granule, str(orbit_dir), esa_credentials=(esa_username, esa_password))
+        log.info(f'Downloading orbit file for {granule}')
+        orbit_file = fetch_for_scene(granule, dir=orbit_dir)
+        log.info(f'Got orbit file {orbit_file} from s1_orbits')
 
     config = topsapp.TopsappBurstConfig(
         reference_safe=f'{ref_params.granule}.SAFE',
@@ -494,8 +488,6 @@ def main():
 
     parser.add_argument('--bucket', help='AWS S3 bucket HyP3 for upload the final product(s)')
     parser.add_argument('--bucket-prefix', default='', help='Add a bucket prefix to product(s)')
-    parser.add_argument('--esa-username', default=None, help="Username for ESA's Copernicus Data Space Ecosystem")
-    parser.add_argument('--esa-password', default=None, help="Password for ESA's Copernicus Data Space Ecosystem")
     parser.add_argument(
         '--looks', choices=['20x4', '10x2', '5x1'], default='20x4', help='Number of looks to take in range and azimuth'
     )
@@ -534,8 +526,6 @@ def main():
         range_looks=range_looks,
         swath_number=swath_number,
         apply_water_mask=apply_water_mask,
-        esa_username=args.esa_username,
-        esa_password=args.esa_password,
     )
 
     log.info('ISCE2 TopsApp run completed successfully')

src/hyp3_isce2/utils.py

@@ -1,12 +1,9 @@
-import netrc
-import os
 import shutil
 import subprocess
 from dataclasses import dataclass
 from datetime import datetime
 from pathlib import Path
-from platform import system
-from typing import Optional, Tuple
+from typing import Optional
 
 import isceobj
 import numpy as np
@@ -17,8 +14,6 @@
 
 gdal.UseExceptions()
 
-ESA_HOST = 'dataspace.copernicus.eu'
-
 
 class GDALConfigManager:
     """Context manager for setting GDAL config options temporarily"""
@@ -133,28 +128,6 @@ def write(self, out_path: Path):
         out_path.write_text(self.__str__())
 
 
-def get_esa_credentials() -> Tuple[str, str]:
-    netrc_name = '_netrc' if system().lower() == 'windows' else '.netrc'
-    netrc_file = Path.home() / netrc_name
-
-    if 'ESA_USERNAME' in os.environ and 'ESA_PASSWORD' in os.environ:
-        username = os.environ['ESA_USERNAME']
-        password = os.environ['ESA_PASSWORD']
-        return username, password
-
-    if netrc_file.exists():
-        netrc_credentials = netrc.netrc(netrc_file)
-        if ESA_HOST in netrc_credentials.hosts:
-            username = netrc_credentials.hosts[ESA_HOST][0]
-            password = netrc_credentials.hosts[ESA_HOST][2]
-            return username, password
-
-    raise ValueError(
-        'Please provide Copernicus Data Space Ecosystem (CDSE) credentials via the '
-        'ESA_USERNAME and ESA_PASSWORD environment variables, or your netrc file.'
-    )
-
-
 def utm_from_lon_lat(lon: float, lat: float) -> int:
     """Get the UTM zone EPSG code from a longitude and latitude.
     See https://en.wikipedia.org/wiki/Universal_Transverse_Mercator_coordinate_system

tests/test_utils.py

@@ -5,7 +5,6 @@
 from unittest.mock import patch
 
 import numpy as np
-import pytest
 from osgeo import gdal
 
 import hyp3_isce2.utils as utils
@@ -152,50 +151,6 @@ def test_resample_to_radar_io(tmp_path, test_merge_dir):
     assert np.all(outputarray == array)
 
 
-def test_get_esa_credentials_env(tmp_path, monkeypatch):
-    with monkeypatch.context() as m:
-        m.setenv('ESA_USERNAME', 'foo')
-        m.setenv('ESA_PASSWORD', 'bar')
-        m.setenv('HOME', str(tmp_path))
-        (tmp_path / '.netrc').write_text(f'machine {utils.ESA_HOST} login netrc_username password netrc_password')
-
-        username, password = utils.get_esa_credentials()
-        assert username == 'foo'
-        assert password == 'bar'
-
-
-def test_get_esa_credentials_netrc(tmp_path, monkeypatch):
-    with monkeypatch.context() as m:
-        m.delenv('ESA_USERNAME', raising=False)
-        m.delenv('ESA_PASSWORD', raising=False)
-        m.setenv('HOME', str(tmp_path))
-        (tmp_path / '.netrc').write_text(f'machine {utils.ESA_HOST} login foo password bar')
-
-        username, password = utils.get_esa_credentials()
-        assert username == 'foo'
-        assert password == 'bar'
-
-
-def test_get_esa_credentials_missing(tmp_path, monkeypatch):
-    with monkeypatch.context() as m:
-        m.delenv('ESA_USERNAME', raising=False)
-        m.setenv('ESA_PASSWORD', 'env_password')
-        m.setenv('HOME', str(tmp_path))
-        (tmp_path / '.netrc').write_text('')
-        msg = 'Please provide.*'
-        with pytest.raises(ValueError, match=msg):
-            utils.get_esa_credentials()
-
-    with monkeypatch.context() as m:
-        m.setenv('ESA_USERNAME', 'env_username')
-        m.delenv('ESA_PASSWORD', raising=False)
-        m.setenv('HOME', str(tmp_path))
-        (tmp_path / '.netrc').write_text('')
-        msg = 'Please provide.*'
-        with pytest.raises(ValueError, match=msg):
-            utils.get_esa_credentials()
-
-
 def test_create_image(tmp_path):
     def _check_create_image(path: str, image_subtype: str = 'default'):
         # create an isce image (includes binary, .vrt, and .xml files)