Merge branch 'master' into osx-ci

.flake8

@@ -8,7 +8,7 @@
 # W504: line break after binary operator
 #       (Raised by flake8 even when it is followed)
 ignore = E126, E402, E129, W504
-max-line-length = 155
+max-line-length = 146
 exclude = test_import_fail.py,
   parsl/executors/workqueue/parsl_coprocess.py
 # E741 disallows ambiguous single letter names which look like numbers

Makefile

@@ -122,4 +122,4 @@ coverage: ## show the coverage report
 
 .PHONY: clean
 clean: ## clean up the environment by deleting the .venv, dist, eggs, mypy caches, coverage info, etc
-	rm -rf .venv $(DEPS) dist *.egg-info .mypy_cache build .pytest_cache .coverage runinfo_* $(WORKQUEUE_INSTALL)
+	rm -rf .venv $(DEPS) dist *.egg-info .mypy_cache build .pytest_cache .coverage runinfo $(WORKQUEUE_INSTALL)

README.rst

@@ -59,6 +59,10 @@ then explore the `parallel computing patterns <https://parsl.readthedocs.io/en/s
 .. |NSF-1550528| image:: https://img.shields.io/badge/NSF-1550528-blue.svg
    :target: https://nsf.gov/awardsearch/showAward?AWD_ID=1550528
    :alt: NSF award info
+.. |NSF-1550475| image:: https://img.shields.io/badge/NSF-1550475-blue.svg
+   :target: https://nsf.gov/awardsearch/showAward?AWD_ID=1550475
+   :alt: NSF award info
+
 
 Quickstart
 ==========
@@ -93,6 +97,7 @@ For Developers
 
 2. Build and Test::
 
+    $ cd parsl # navigate to the root directory of the project
     $ make   # show all available makefile targets
     $ make virtualenv # create a virtual environment
     $ source .venv/bin/activate # activate the virtual environment
@@ -115,7 +120,7 @@ Parsl is supported in Python 3.8+. Requirements can be found `here <requirements
 Code of Conduct
 ===============
 
-Parsl seeks to foster an open and welcoming environment - Please see the `Parsl Code of Conduct <https://github.com/Parsl/parsl/blob/master/CoC.md>`_ for more details.
+Parsl seeks to foster an open and welcoming environment - Please see the `Parsl Code of Conduct <https://github.com/Parsl/parsl/blob/master/CODE_OF_CONDUCT.md>`_ for more details.
 
 Contributing
 ============

docs/README.rst

@@ -21,7 +21,15 @@ Local builds
 
 To build the documentation locally, use::
 
-    $ make html
+    $ make clean html
+
+To view the freshly rebuilt docs, use::
+
+    $ cd _build/html
+    $ python3 -m http.server 8080
+
+Once the python http server is launched, point your browser to `http://localhost:8080 <http://localhost:8080>`_
+
 
 Regenerate module stubs
 --------------------------

docs/devguide/index.rst

@@ -5,8 +5,6 @@ Developer documentation
    :maxdepth: 3
 
    contributing
-   changelog
-   design
    roadmap
    packaging
    ../README

docs/historical/index.rst

@@ -0,0 +1,9 @@
+Historical Documents
+====================
+
+.. toctree::
+   :maxdepth: 2
+
+   changelog
+   design
+   performance

docs/index.rst

@@ -108,6 +108,7 @@ Table of Contents
    faq
    reference
    devguide/index
+   historical/index
 
 
 Indices and tables

docs/quickstart.rst

@@ -93,8 +93,6 @@ with hello world Python and Bash apps.
     import parsl
     from parsl import python_app, bash_app
 
-    parsl.load()
-
     @python_app
     def hello_python (message):
         return 'Hello %s' % message
@@ -103,11 +101,13 @@ with hello world Python and Bash apps.
     def hello_bash(message, stdout='hello-stdout'):
         return 'echo "Hello %s"' % message
 
-    # invoke the Python app and print the result
-    print(hello_python('World (Python)').result())
+    
+    with parsl.load():
+        # invoke the Python app and print the result
+        print(hello_python('World (Python)').result())
 
-    # invoke the Bash app and read the result from a file
-    hello_bash('World (Bash)').result()
+        # invoke the Bash app and read the result from a file
+        hello_bash('World (Bash)').result()
 
     with open('hello-stdout', 'r') as f:
         print(f.read())

docs/reference.rst

@@ -113,7 +113,6 @@ Providers
     parsl.providers.GridEngineProvider
     parsl.providers.LocalProvider
     parsl.providers.LSFProvider
-    parsl.providers.GridEngineProvider
     parsl.providers.SlurmProvider
     parsl.providers.TorqueProvider
     parsl.providers.KubernetesProvider

docs/userguide/configuring.rst

@@ -60,9 +60,11 @@ All Parsl applications start by creating or importing a configuration then calli
     from parsl.configs.htex_local import config
     import parsl
 
-    parsl.load(config)
+    with parsl.load(config):
 
 The ``load`` statement can happen after Apps are defined but must occur before tasks are started.
+Loading the Config object within context manager like ``with`` is recommended
+for implicit cleaning of DFK on exiting the context manager  
 
 The :class:`~parsl.config.Config` object may not be used again after loaded.
 Consider a configuration function if the application will shut down and re-launch the DFK.
@@ -75,9 +77,11 @@ Consider a configuration function if the application will shut down and re-launc
     def make_config() -> Config:
         return Config(...)
 
-    parsl.load(make_config())
+    with parsl.load(make_config()):
+        # Your workflow here
     parsl.clear()  # Stops Parsl
-    parsl.load(make_config())  # Re-launches with a fresh configuration
+    with parsl.load(make_config()):  # Re-launches with a fresh configuration
+        # Your workflow here
 
 
 How to Configure
@@ -446,16 +450,7 @@ The following snippet shows an example configuration for accessing NSCC's **ASPI
 .. literalinclude:: ../../parsl/configs/ASPIRE1.py
 
 
-Blue Waters (NCSA)
-------------------
-
-.. image:: https://www.cray.com/sites/default/files/images/Solutions_Images/bluewaters.png
-
-The following snippet shows an example configuration for executing remotely on Blue Waters, a flagship machine at the National Center for Supercomputing Applications.
-The configuration assumes the user is running on a login node and uses the `parsl.providers.TorqueProvider` to interface
-with the scheduler, and uses the `parsl.launchers.AprunLauncher` to launch workers.
 
-.. literalinclude:: ../../parsl/configs/bluewaters.py
 
 Illinois Campus Cluster (UIUC)
 ------------------------------
@@ -493,12 +488,12 @@ This system uses Grid Engine which Parsl interfaces with using the `parsl.provid
 .. literalinclude:: ../../parsl/configs/cc_in2p3.py
 
 
-CCL (Notre Dame, with Work Queue)
----------------------------------
+CCL (Notre Dame, TaskVine)
+--------------------------
 
-.. image:: http://ccl.cse.nd.edu/software/workqueue/WorkQueueLogoSmall.png
+.. image:: https://ccl.cse.nd.edu/software/taskvine/taskvine-logo.png
 
-To utilize Work Queue with Parsl, please install the full CCTools software package within an appropriate Anaconda or Miniconda environment
+To utilize TaskVine with Parsl, please install the full CCTools software package within an appropriate Anaconda or Miniconda environment
 (instructions for installing Miniconda can be found `in the Conda install guide <https://docs.conda.io/projects/conda/en/latest/user-guide/install/>`_):
 
 .. code-block:: bash
@@ -507,17 +502,17 @@ To utilize Work Queue with Parsl, please install the full CCTools software packa
    $ conda activate <environment>
    $ conda install -y -c conda-forge ndcctools parsl
 
-This creates a Conda environment on your machine with all the necessary tools and setup needed to utilize Work Queue with the Parsl library.
+This creates a Conda environment on your machine with all the necessary tools and setup needed to utilize TaskVine with the Parsl library.
+
+The following snippet shows an example configuration for using the Parsl/TaskVine executor to run applications on the local machine.
+This examples uses the `parsl.executors.taskvine.TaskVineExecutor` to schedule tasks, and a local worker will be started automatically. 
+For more information on using TaskVine, including configurations for remote execution, visit the 
+`TaskVine/Parsl documentation online <https://cctools.readthedocs.io/en/latest/taskvine/#parsl>`_.
 
-The following snippet shows an example configuration for using the Work Queue distributed framework to run applications on remote machines at large.
-This examples uses the `parsl.executors.WorkQueueExecutor` to schedule tasks locally,
-and assumes that Work Queue workers have been externally connected to the master using the
-`work_queue_factory <https://cctools.readthedocs.io/en/latest/man_pages/work_queue_factory/>`_ or
-`condor_submit_workers <https://cctools.readthedocs.io/en/latest/man_pages/condor_submit_workers/>`_ command line utilities from CCTools.
-For more information on using Work Queue or to get help with running applications using CCTools,
-visit the `CCTools documentation online <https://cctools.readthedocs.io/en/latest/help/>`_.
+.. literalinclude::  ../../parsl/configs/vineex_local.py
 
-.. literalinclude::  ../../parsl/configs/wqex_local.py
+TaskVine's predecessor, WorkQueue, may continue to be used with Parsl.
+For more information on using WorkQueue visit the `CCTools documentation online <https://cctools.readthedocs.io/en/latest/help/>`_.
 
 Expanse (SDSC)
 --------------

docs/userguide/execution.rst

@@ -39,7 +39,7 @@ parameters include access keys, instance type, and spot bid price
 Parsl currently supports the following providers:
 
 1. `parsl.providers.LocalProvider`: The provider allows you to run locally on your laptop or workstation.
-2. `parsl.providers.CobaltProvider`: This provider allows you to schedule resources via the Cobalt scheduler.
+2. `parsl.providers.CobaltProvider`: This provider allows you to schedule resources via the Cobalt scheduler. **This provider is deprecated and will be removed by 2024.04**.
 3. `parsl.providers.SlurmProvider`: This provider allows you to schedule resources via the Slurm scheduler.
 4. `parsl.providers.CondorProvider`: This provider allows you to schedule resources via the Condor scheduler.
 5. `parsl.providers.GridEngineProvider`: This provider allows you to schedule resources via the GridEngine scheduler.
@@ -48,7 +48,8 @@ Parsl currently supports the following providers:
 8. `parsl.providers.GoogleCloudProvider`: This provider allows you to provision and manage cloud nodes from Google Cloud.
 9. `parsl.providers.KubernetesProvider`: This provider allows you to provision and manage containers on a Kubernetes cluster.
 10. `parsl.providers.AdHocProvider`: This provider allows you manage execution over a collection of nodes to form an ad-hoc cluster.
-11. `parsl.providers.LSFProvider`: This provider allows you to schedule resources via IBM's LSF scheduler
+11. `parsl.providers.LSFProvider`: This provider allows you to schedule resources via IBM's LSF scheduler.
+
 
 
 Executors

docs/userguide/index.rst

@@ -21,4 +21,3 @@ User guide
    usage_tracking
    plugins
    parsl_perf
-   performance

docs/userguide/mpi_apps.rst

@@ -1,14 +1,6 @@
 MPI Apps
 ========
 
-.. note::
-
-    Parsl's support for MPI Apps described below is pending release.
-    Please use the ``mpi_experimental_3`` branch to use the functionality
-    described in this document. To install directly from github:
-
-    >>  pip install git+https://github.com/Parsl/parsl.git@mpi_experimental_3
-
 MPI applications run multiple copies of a program that complete a single task by
 coordinating using messages passed within or across nodes.
 Starting MPI application requires invoking a "launcher" code (e.g., ``mpiexec``) from one node
@@ -146,7 +138,7 @@ Writing MPI-Compatible Apps
 ++++++++++++++++++++++++++++
 
 In MPI mode, the :class:`~parsl.executors.high_throughput.executor.HighThroughputExecutor` can execute both Python or Bash Apps which invokes the MPI application.
-However, it is important to not that Python Apps that directly use ``mpi4py`` is not supported.
+However, it is important to note that Python Apps that directly use ``mpi4py`` is not supported.
 
 For multi-node MPI applications, especially when running multiple applications within a single batch job,
 it is important to specify the resource requirements for the app so that the Parsl worker can provision

parsl/addresses.py

@@ -81,7 +81,9 @@ def address_by_hostname() -> str:
 def address_by_interface(ifname: str) -> str:
     """Returns the IP address of the given interface name, e.g. 'eth0'
 
-    This is taken from a Stack Overflow answer: https://stackoverflow.com/questions/24196932/how-can-i-get-the-ip-address-of-eth0-in-python#24196955
+    This is taken from a Stack Overflow answer:
+    https://stackoverflow.com/questions/24196932/how-can-i-get-the-ip-address-of-eth0-in-python#24196955
+
 
     Parameters
     ----------

parsl/config.py

@@ -55,6 +55,8 @@ class Config(RepresentationMixin):
         or `None`.
         If 'none' or `None`, dynamic scaling will be disabled. Default is 'simple'. The literal value `None` is
         deprecated.
+    strategy_period : float or int, optional
+        How often the scaling strategy should be executed. Default is 5 seconds.
     max_idletime : float, optional
         The maximum idle time allowed for an executor before strategy could shut down unused blocks. Default is 120.0 seconds.
     usage_tracking : bool, optional
@@ -88,6 +90,7 @@ def __init__(self,
                  retry_handler: Optional[Callable[[Exception, TaskRecord], float]] = None,
                  run_dir: str = 'runinfo',
                  strategy: Optional[str] = 'simple',
+                 strategy_period: Union[float, int] = 5,
                  max_idletime: float = 120.0,
                  monitoring: Optional[MonitoringHub] = None,
                  usage_tracking: bool = False,
@@ -121,6 +124,7 @@ def __init__(self,
         self.retry_handler = retry_handler
         self.run_dir = run_dir
         self.strategy = strategy
+        self.strategy_period = strategy_period
         self.max_idletime = max_idletime
         self.usage_tracking = usage_tracking
         self.initialize_logging = initialize_logging