Merge pull request #25 from ASFHyP3/develop

Release v0.4.0

CHANGELOG.md

@@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/)
 and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.4.0]
+
+### Added
+* Support for GPU accelerated processing using CUDA.
 
 ## [0.3.0]
 

Dockerfile

@@ -18,6 +18,8 @@ LABEL org.opencontainers.image.documentation="https://hyp3-docs.asf.alaska.edu"
 
 ARG DEBIAN_FRONTEND=noninteractive
 
+ENV USEGPU="false"
+
 ENV PYTHONDONTWRITEBYTECODE=true
 ENV PROC_HOME=/home/conda/back-projection
 ENV MYHOME=/home/conda
@@ -36,9 +38,9 @@ RUN groupadd -g "${CONDA_GID}" --system conda && \
     echo ". /opt/conda/etc/profile.d/conda.sh" >> /home/conda/.profile && \
     echo "conda activate base" >> /home/conda/.profile
 
+SHELL ["/bin/bash", "-l", "-c"]
 
 USER ${CONDA_UID}
-SHELL ["/bin/bash", "-l", "-c"]
 WORKDIR /home/conda/
 
 RUN curl -sL https://github.com/ASFHyP3/back-projection/archive/refs/tags/v${BACK_PROJECTION_TAG}.tar.gz > ./back-projection.tar.gz && \
@@ -47,10 +49,10 @@ RUN curl -sL https://github.com/ASFHyP3/back-projection/archive/refs/tags/v${BAC
     rm ./back-projection.tar.gz && \
     rm -rf ./back-projection/fft
 
-COPY --chown=${CONDA_UID}:${CONDA_GID} ./scripts/build_proc_cpu.sh ./back-projection
+COPY --chown=${CONDA_UID}:${CONDA_GID} ./scripts/build_proc.sh ./back-projection
 RUN cd /home/conda/back-projection && \
-    chmod +x ./build_proc_cpu.sh && \
-    ./build_proc_cpu.sh && \
+    chmod +x ./build_proc.sh && \
+    ./build_proc.sh && \
     find $PROC_HOME -type f -name "*.py" -exec chmod +x {} + && \
     cd /home/conda/
 

Dockerfile.gpu

@@ -0,0 +1,86 @@
+FROM nvidia/cuda:12.4.1-devel-ubuntu20.04
+
+# For opencontainers label definitions, see:
+#    https://github.com/opencontainers/image-spec/blob/master/annotations.md
+LABEL org.opencontainers.image.title="HyP3 back-projection"
+LABEL org.opencontainers.image.description="HyP3 plugin for back-projection processing"
+LABEL org.opencontainers.image.vendor="Alaska Satellite Facility"
+LABEL org.opencontainers.image.authors="ASF Tools Team <[email protected]>"
+LABEL org.opencontainers.image.licenses="BSD-3-Clause"
+LABEL org.opencontainers.image.url="https://github.com/ASFHyP3/hyp3-back-projection"
+LABEL org.opencontainers.image.source="https://github.com/ASFHyP3/hyp3-back-projection"
+LABEL org.opencontainers.image.documentation="https://hyp3-docs.asf.alaska.edu"
+
+ARG DEBIAN_FRONTEND=noninteractive
+ARG CONDA_UID=1000
+ARG CONDA_GID=1000
+ARG BACK_PROJECTION_TAG=0.2.0
+ARG FFTW_TAG=3.3.9
+ARG MINIFORGE_NAME=Miniforge3
+ARG MINIFORGE_VERSION=24.3.0-0
+
+# USEGPU environment variable used by build_proc.sh
+ENV USEGPU="true" 
+ENV CONDA_DIR=/opt/conda
+ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
+ENV PATH=${CONDA_DIR}/bin:${PATH}
+ENV PYTHONDONTWRITEBYTECODE=true
+ENV PROC_HOME=/home/conda/back-projection
+ENV MYHOME=/home/conda
+
+# Conda setup
+RUN apt-get update > /dev/null && \
+    apt-get install --no-install-recommends --yes \
+        wget bzip2 ca-certificates \
+        git \
+        tini \
+        > /dev/null && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* && \
+    wget --no-hsts --quiet https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/${MINIFORGE_NAME}-${MINIFORGE_VERSION}-Linux-$(uname -m).sh -O /tmp/miniforge.sh && \
+    /bin/bash /tmp/miniforge.sh -b -p ${CONDA_DIR} && \
+    rm /tmp/miniforge.sh && \
+    conda clean --tarballs --index-cache --packages --yes && \
+    find ${CONDA_DIR} -follow -type f -name '*.a' -delete && \
+    find ${CONDA_DIR} -follow -type f -name '*.pyc' -delete && \
+    conda clean --force-pkgs-dirs --all --yes  && \
+    echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> /etc/skel/.bashrc && \
+    echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> ~/.bashrc
+
+RUN apt-get update && apt-get install -y --no-install-recommends unzip vim curl build-essential gfortran libfftw3-dev && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+RUN groupadd -g "${CONDA_GID}" --system conda && \
+    useradd -l -u "${CONDA_UID}" -g "${CONDA_GID}" --system -d /home/conda -m  -s /bin/bash conda && \
+    chown -R conda:conda /opt && \
+    echo ". /opt/conda/etc/profile.d/conda.sh" >> /home/conda/.profile && \
+    echo "conda activate base" >> /home/conda/.profile
+
+SHELL ["/bin/bash", "-l", "-c"]
+
+USER ${CONDA_UID}
+WORKDIR /home/conda/
+
+RUN curl -sL https://github.com/ASFHyP3/back-projection/archive/refs/tags/v${BACK_PROJECTION_TAG}.tar.gz > ./back-projection.tar.gz && \
+    mkdir -p ./back-projection && \
+    tar -xvf ./back-projection.tar.gz -C ./back-projection/ --strip=1 && \
+    rm ./back-projection.tar.gz && \
+    rm -rf ./back-projection/fft
+
+COPY --chown=${CONDA_UID}:${CONDA_GID} ./scripts/build_proc.sh ./back-projection
+RUN cd /home/conda/back-projection && \
+    chmod +x ./build_proc.sh && \
+    ./build_proc.sh && \
+    find $PROC_HOME -type f -name "*.py" -exec chmod +x {} + && \
+    cd /home/conda/
+
+COPY --chown=${CONDA_UID}:${CONDA_GID} . /hyp3-back-projection/
+
+RUN mamba env create -f /hyp3-back-projection/environment.yml && \
+    conda clean -afy && \
+    conda activate hyp3-back-projection && \
+    sed -i 's/conda activate base/conda activate hyp3-back-projection/g' /home/conda/.profile && \
+    python -m pip install --no-cache-dir /hyp3-back-projection
+
+ENTRYPOINT ["/hyp3-back-projection/src/hyp3_back_projection/etc/entrypoint.sh"]
+CMD ["-h"]

README.md

@@ -51,3 +51,36 @@ Your credentials can be passed to the workflows via command-line options (`--esa
 
 If you haven't set up a `.netrc` file
 before, check out this [guide](https://harmony.earthdata.nasa.gov/docs#getting-started) to get started.
+
+## GPU Setup:
+In order for Docker to be able to use the host's GPU, the host must have the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/index.html) installed and configured. 
+The process is different for different OS's and Linux distros. The setup process for the most common distros, including Ubuntu, 
+can be found [here](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html#configuration). Make sure to follow the [Docker configuration steps](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html#configuration) after installing the package.
+
+### EC2 Setup
+When running on an EC2 instance, the following setup is recommended:
+1. Create a [P3-family EC2 instance](https://aws.amazon.com/ec2/instance-types/p3/) with the [Amazon Linux 2 AMI with NVIDIA TESLA GPU Driver](https://aws.amazon.com/marketplace/pp/prodview-64e4rx3h733ru?sr=0-4&ref_=beagle&applicationId=AWSMPContessa)
+2. Install Docker and the nvidia-container-toolkit on the EC2 instance:
+```bash
+sudo yum-config-manager --disable amzn2-graphics
+curl -s -L https://nvidia.github.io/libnvidia-container/stable/rpm/nvidia-container-toolkit.repo | sudo tee /etc/yum.repos.d/nvidia-container-toolkit.repo
+sudo yum install docker -y
+sudo yum install nvidia-container-toolkit -y
+sudo yum-config-manager --enable amzn2-graphics
+```
+3. Optionally, set up Docker to not require `sudo` and to start when the EC2 instance starts
+```bash
+sudo systemctl start docker && \
+sudo usermod -a -G docker ec2-user && \
+sudo systemctl enable docker
+```
+4. Exit the EC2 instance and re-enter
+5. To test the GPU setup, run the base NVIDIA container:
+```bash
+docker run -it --gpus all nvidia/cuda:12.4.1-devel-ubuntu20.04 nvidia-smi
+```
+6. Build the actual container and run it:
+```bash
+docker build -t back-projection:gpu -f Dockerfile.gpu .
+docker run --gpus=all --rm -it back-projection:gpu ++process back_projection --help
+```

scripts/build_proc_cpu.sh → scripts/build_proc.sh

@@ -3,8 +3,10 @@
 MULTIARCH_DIR=/usr/lib/$(gcc -print-multiarch)
 FFTW_LIB=$MULTIARCH_DIR/libfftw3f.a
 echo 'using FFTW library:' $FFTW_LIB
+if [[ "$USEGPU" == "true" ]]; then
+    echo 'building with GPU support'
+fi
 
-# Works
 cd DEM
 gfortran -o mosaicDEM mosaicDEM.f90
 gfortran -o createspecialdem createspecialdem.f90
@@ -19,9 +21,10 @@ gfortran -c processsubcpu.f90 backprojectcpusub.f90 bounds.f90 orbitrangetime.f9
 gcc -o sentinel_raw_process_cpu sentinel_raw_process_cpu.o decode_line_memory.o processsubcpu.o backprojectcpusub.o azimuth_compress_cpu.o bounds.o orbitrangetime.o latlon.o intp_orbit.o radar_to_xyz.o unitvec.o tcnbasis.o curvature.o cross.o orbithermite.o filelen.o io.o sentineltimingsub.o getburststatevectors.o $FFTW_LIB -lgfortran -lgomp -lm -lrt -lpthread
 echo 'built sentinel_raw_process_cpu'
 
-# nvcc -o howmanygpus howmanygpus.cu
-#
-# echo 'built howmanygpus'
+if [[ "$USEGPU" == "true" ]]; then
+    nvcc -o howmanygpus howmanygpus.cu
+    echo 'built howmanygpus'
+fi
 
 cd geo2rdr
 gfortran -o estimatebaseline estimatebaseline.f90 intp_orbit.f90 latlon.f90 orbithermite.f -ffixed-line-length-none
@@ -80,20 +83,20 @@ cd ..
 
 echo 'built snaphu'
 
-# nvcc -o gpu_arch gpu_arch.cu
-# echo 'built gpu architecture probe'
-#
-# ./gpu_arch | cat > GPU_ARCH; source ./GPU_ARCH; rm GPU_ARCH
-
 cd sentinel
 
 gcc -c filelen.c io.c sentinel_raw_process.c decode_line_memory.c -lm -fopenmp
 
 echo 'built raw_process components in sentinel'
 
-# nvcc -gencode arch=compute_$GPU_ARCH,code=sm_$GPU_ARCH -c azimuth_compress.cu -Wno-deprecated-gpu-targets
+if [[ "$USEGPU" == "true" ]]; then
+    nvcc -gencode arch=compute_89,code=sm_89 -c azimuth_compress.cu -Wno-deprecated-gpu-targets
+fi
 
 gfortran -c processsub.f90 backprojectgpusub.f90 bounds.f90 orbitrangetime.f90 latlon.f90 intp_orbit.f90 radar_to_xyz.f90 unitvec.f90 tcnbasis.f90 curvature.f90 cross.f90 orbithermite.f sentineltimingsub.f90 getburststatevectors.f90 -ffixed-line-length-none -fopenmp
 
-# nvcc -o sentinel_raw_process sentinel_raw_process.o decode_line_memory.o processsub.o backprojectgpusub.o azimuth_compress.o bounds.o orbitrangetime.o latlon.o intp_orbit.o radar_to_xyz.o unitvec.o tcnbasis.o curvature.o cross.o orbithermite.o filelen.o io.o sentineltimingsub.o getburststatevectors.o $FFTW_LIB -lstdc++ -lgfortran -lgomp
+if [[ "$USEGPU" == "true" ]]; then
+    nvcc -o sentinel_raw_process sentinel_raw_process.o decode_line_memory.o processsub.o backprojectgpusub.o azimuth_compress.o bounds.o orbitrangetime.o latlon.o intp_orbit.o radar_to_xyz.o unitvec.o tcnbasis.o curvature.o cross.o orbithermite.o filelen.o io.o sentineltimingsub.o getburststatevectors.o $FFTW_LIB -lstdc++ -lgfortran -lgomp
+fi
+
 cd ..

src/hyp3_back_projection/back_projection.py

@@ -4,6 +4,7 @@
 
 import argparse
 import logging
+import os
 import zipfile
 from pathlib import Path
 from typing import Iterable, Optional
@@ -30,26 +31,40 @@ def create_param_file(dem_path: Path, dem_rsc_path: Path, output_dir: Path):
         f.write('\n'.join(lines))
 
 
-def back_project_single_granule(granule_path: Path, orbit_path: Path, work_dir: Path) -> None:
-    """Back-project a single Sentinel-1 level-0 granule.
-
-    Args:
-        granule_path: Path to the granule to back-project
-        orbit_path: Path to the orbit file for the granule
-    """
-    required_files = ['elevation.dem', 'elevation.dem.rsc', 'params']
+def check_required_files(required_files: Iterable, work_dir: Path) -> None:
     for file in required_files:
         if not (work_dir / file).exists():
             raise FileNotFoundError(f'Missing required file: {file}')
 
-    args = [str(granule_path.with_suffix('')), str(orbit_path)]
-    utils.call_stanford_module('sentinel/sentinel_scene_cpu.py', args, work_dir=work_dir)
+
+def clean_up_after_back_projection(work_dir: Path) -> None:
     patterns = ['*hgt*', 'dem*', 'DEM*', 'q*', '*positionburst*']
     for pattern in patterns:
         [f.unlink() for f in work_dir.glob(pattern)]
 
 
-def create_product(work_dir):
+def back_project_granules(granule_orbit_pairs: Iterable, work_dir: Path, gpu: bool = False) -> None:
+    """Back-project a set of Sentinel-1 level-0 granules using the CPU-based workflow.
+
+    Args:
+        granule_orbit_pairs: List of tuples of granule and orbit file paths
+        work_dir: Working directory for processing
+    """
+    check_required_files(['elevation.dem', 'elevation.dem.rsc', 'params'], work_dir)
+
+    if gpu:
+        os.environ['CUDA_DEVICE_ORDER'] = 'PCI_BUS_ID'
+        os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+    cmd = 'sentinel/sentinel_scene_multigpu.py' if gpu else 'sentinel/sentinel_scene_cpu.py'
+    for granule_path, orbit_path in granule_orbit_pairs:
+        args = [str(granule_path.with_suffix('')), str(orbit_path)]
+        utils.call_stanford_module(cmd, args, work_dir=work_dir)
+
+    clean_up_after_back_projection(work_dir)
+
+
+def create_product(work_dir) -> Path:
     """Create a product zip file.
     Includes files needed for further processing (gslc, orbit, and parameter file).
 
@@ -88,7 +103,8 @@ def back_project(
     bucket: str = None,
     bucket_prefix: str = '',
     work_dir: Optional[Path] = None,
-) -> Path:
+    gpu: bool = False,
+):
     """Back-project a set of Sentinel-1 level-0 granules.
 
     Args:
@@ -100,6 +116,7 @@ def back_project(
         bucket: AWS S3 bucket for uploading the final product(s)
         bucket_prefix: Add a bucket prefix to the product(s)
         work_dir: Working directory for processing
+        gpu: Use the GPU-based version of the workflow
     """
     utils.set_creds('EARTHDATA', earthdata_username, earthdata_password)
     utils.set_creds('ESA', esa_username, esa_password)
@@ -108,27 +125,26 @@ def back_project(
 
     print('Downloading data...')
     bboxs = []
-    back_project_args = []
+    granule_orbit_pairs = []
     for granule in granules:
-        granule_path, granule_bbox = utils.download_raw_granule(granule, work_dir)
+        granule_path, granule_bbox = utils.download_raw_granule(granule, work_dir, unzip=True)
         orbit_path = utils.download_orbit(granule, work_dir)
         bboxs.append(granule_bbox)
-        back_project_args.append((granule_path, orbit_path))
+        granule_orbit_pairs.append((granule_path, orbit_path))
 
     full_bbox = unary_union(bboxs).buffer(0.1)
     dem_path = dem.download_dem_for_back_projection(full_bbox, work_dir)
     create_param_file(dem_path, dem_path.with_suffix('.dem.rsc'), work_dir)
 
-    for granule_path, orbit_path in back_project_args:
-        back_project_single_granule(granule_path, orbit_path, work_dir=work_dir)
+    back_project_granules(granule_orbit_pairs, work_dir=work_dir, gpu=gpu)
 
     utils.call_stanford_module('util/merge_slcs.py', work_dir=work_dir)
 
     if bucket:
         zip_path = create_product(work_dir)
         upload_file_to_s3(zip_path, bucket, bucket_prefix)
 
-    print(f'Finish back-projection for {list(work_dir.glob("S1*.geo"))[0].with_suffix("").name}!')
+    print(f'Finished back-projection for {list(work_dir.glob("S1*.geo"))[0].with_suffix("").name}!')
 
 
 def main():
@@ -146,6 +162,7 @@ def main():
     parser.add_argument('--esa-password', default=None, help="Password for ESA's Copernicus Data Space Ecosystem")
     parser.add_argument('--bucket', help='AWS S3 bucket HyP3 for upload the final product(s)')
     parser.add_argument('--bucket-prefix', default='', help='Add a bucket prefix to product(s)')
+    parser.add_argument('--gpu', default=False, action='store_true', help='Use the GPU-based version of the workflow.')
     parser.add_argument('granules', nargs='+', help='Level-0 S1 granule to back-project.')
     args = parser.parse_args()