Better handling of aborts in parallel runs

seismic/bulk_station_orientations.py

@@ -34,6 +34,7 @@
 from matplotlib.backends.backend_pdf import PdfPages
 import matplotlib.pyplot as plt
 from shutil import rmtree
+from seismic.misc_p import parallel_abort
 
 logging.basicConfig()
 
@@ -212,7 +213,11 @@ def main(src_h5_event_file, network, output_basename, station_list, dump_swp_dat
         proc_hdfkeys = list(proc_hdfkeys)
 
         # trim stations to be processed based on the user-provided network- and station-list
-        proc_hdfkeys = rf_util.trim_hdf_keys(proc_hdfkeys, network, station_list)
+        try:
+            proc_hdfkeys = rf_util.trim_hdf_keys(proc_hdfkeys, network, station_list)
+        except Exception as e:
+            parallel_abort(str(e), logger)
+        # end try
 
         # split work-load over all procs
         proc_hdfkeys = split_list(proc_hdfkeys, nproc)

seismic/misc_p.py

@@ -14,6 +14,7 @@
 
 import os
 from mpi4py import MPI
+from logging import Logger
 
 class ProgressTracker:
     def __init__(self, output_folder, restart_mode=False):
@@ -51,3 +52,11 @@ def increment(self):
     # end func
 # end class
 
+def parallel_abort(msg: str, logger:Logger=None):
+    comm = MPI.COMM_WORLD
+    nproc = comm.Get_size()
+    rank = comm.Get_rank()
+
+    if(logger is not None): logger.error('Aborting job from rank {}: {}'.format(rank, msg))
+    comm.Abort()
+# end func