From a0f7a4b96da0f336e1fa562f040e5bb75fbb6539 Mon Sep 17 00:00:00 2001 From: Ritwika Dasgupta Date: Mon, 13 May 2024 01:37:29 +0900 Subject: [PATCH 1/2] parallel downloads --- src/mintpy/tropo_pyaps3.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/mintpy/tropo_pyaps3.py b/src/mintpy/tropo_pyaps3.py index 64939eca1..f854af757 100644 --- a/src/mintpy/tropo_pyaps3.py +++ b/src/mintpy/tropo_pyaps3.py @@ -466,7 +466,21 @@ def check_pyaps_account_config(tropo_model): ############################################################### +from concurrent.futures import ThreadPoolExecutor + def dload_grib_files(grib_files, tropo_model='ERA5', snwe=None): + # import pdb; pdb.set_trace() + with ThreadPoolExecutor(max_workers=64) as worker: + futures = [] + for grib_file in grib_files: + future = worker.submit(dload_grib_files_worker, [grib_file], tropo_model, snwe) + futures.append(future) + for future in futures: + future.result() + + return dload_grib_files_worker(grib_files, tropo_model, snwe) + +def dload_grib_files_worker(grib_files, tropo_model='ERA5', snwe=None): """Download weather re-analysis grib files using PyAPS Parameters: grib_files : list of string of grib files Returns: grib_files : list of string From a9b9109ba60adc638586869986da3650950edf55 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 29 May 2024 03:25:18 +0000 Subject: [PATCH 2/2] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/mintpy/tropo_pyaps3.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mintpy/tropo_pyaps3.py b/src/mintpy/tropo_pyaps3.py index f854af757..b590c19e0 100644 --- a/src/mintpy/tropo_pyaps3.py +++ b/src/mintpy/tropo_pyaps3.py @@ -468,6 +468,7 @@ def check_pyaps_account_config(tropo_model): ############################################################### from concurrent.futures import ThreadPoolExecutor + def dload_grib_files(grib_files, tropo_model='ERA5', snwe=None): # import pdb; pdb.set_trace() with ThreadPoolExecutor(max_workers=64) as worker: