From 34c23b7f7134389df82ff51769ea77f55701d3e5 Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Fri, 17 Sep 2021 18:18:21 -0700
Subject: [PATCH 01/73] Add script for updating the schema in the model form
 repo and add a job to the GHA workflow so that the schema is updated when new
 tags are pushed

---
 .github/workflows/test-install.yml | 29 +++++++++++++++++++
 requirements_development.txt       |  1 +
 utils/test-install-base.yml        | 22 ++++++++++++++
 utils/update_form_schema.py        | 46 ++++++++++++++++++++++++++++++
 4 files changed, 98 insertions(+)
 create mode 100644 utils/update_form_schema.py

diff --git a/.github/workflows/test-install.yml b/.github/workflows/test-install.yml
index f369e1603..dcb383d92 100644
--- a/.github/workflows/test-install.yml
+++ b/.github/workflows/test-install.yml
@@ -1672,3 +1672,32 @@ jobs:
       run: python utils/build_docker.py --push
     - name: Build and push executable docker images for the new release
       run: python utils/build_docker.py --push executable
+  schema:
+    name: Update the schema used by the model submission form
+    needs:
+    - test_pip
+    - test_conda
+    - test_rmq_pip
+    - test_rmq_conda
+    - test_stripped
+    - docs
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: Install dependencies
+      run: 'python -m pip install PyGithub
+
+        python utils/setup_test_env.py install pip
+
+        '
+    - name: Verify installation
+      run: python utils/setup_test_env.py verify
+    - name: Update the form
+      run: python utils/update_form_schema.py
diff --git a/requirements_development.txt b/requirements_development.txt
index 3a2166050..f66d8af4a 100644
--- a/requirements_development.txt
+++ b/requirements_development.txt
@@ -1,2 +1,3 @@
 black
 git  # [conda]
+PyGithub
diff --git a/utils/test-install-base.yml b/utils/test-install-base.yml
index 8ee644c21..99fb52edd 100644
--- a/utils/test-install-base.yml
+++ b/utils/test-install-base.yml
@@ -543,3 +543,25 @@ jobs:
       run: python utils/build_docker.py --push
     - name: Build and push executable docker images for the new release
       run: python utils/build_docker.py --push executable
+
+  schema:
+    name: Update the schema used by the model submission form
+    needs: [test_pip, test_conda, test_rmq_pip, test_rmq_conda, test_stripped, docs]
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: Install dependencies
+      run: |
+        python -m pip install PyGithub
+        python utils/setup_test_env.py install pip
+    - name: Verify installation
+      run: python utils/setup_test_env.py verify
+    - name: Update the form
+      run: python utils/update_form_schema.py
diff --git a/utils/update_form_schema.py b/utils/update_form_schema.py
new file mode 100644
index 000000000..0ecf57b7b
--- /dev/null
+++ b/utils/update_form_schema.py
@@ -0,0 +1,46 @@
+import os
+import json
+import argparse
+from github import Github
+from yggdrasil import __version__
+from yggdrasil.schema import get_model_form_schema
+
+
+def update_schema(token):
+    r"""Update the model form schema via pull request to the
+    cropsinsilico/model_submission_form repository.
+
+    Args:
+        token (str): Github authentication token that should be used.
+
+    """
+    contents = json.dumps(get_model_form_schema(), indent='    ')
+    ver = __version__
+    repo_name = "cropsinsilico/model_submission_form"
+    msg = f"Update model form schema to version for yggdrasil {ver}"
+    branch = f"schema-update-{ver}"
+    schema = "static/model.json"
+    gh = Github(token)
+    repo = gh.get_repo(repo_name)
+    main = repo.get_branch('main')
+    schema_sha = repo.get_contents(schema).sha
+    repo.create_git_ref(f"refs/heads/{branch}", main.commit.sha)
+    repo.update_file(schema, msg, contents, schema_sha, branch=branch)
+    repo.create_pull(title=f"Update schema to {ver}",
+                     body=msg, head=branch, base="main")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        "Update the model form schema via pull request to the"
+        "cropsinsilico/model_submission_form repository.")
+    parser.add_argument(
+        '--token',
+        help=("Github authentication token that should be used to open the "
+              "pull request. The token must have write access to the "
+              "cropsinsilico/model_submission_form repository."))
+    args = parser.parse_args()
+    if args.token is None:
+        args.token = os.environ.get('YGGDRASIL_UPDATE_TOKEN', None)
+    assert(args.token)
+    update_schema = update_schema(args.token)

From 9507a1e9c7b0eec74aea945557b76b185baf4ce2 Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Fri, 17 Sep 2021 18:36:23 -0700
Subject: [PATCH 02/73] Replace 'any' type in form schemas

---
 yggdrasil/schema.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/yggdrasil/schema.py b/yggdrasil/schema.py
index eac223122..4cf5daf16 100644
--- a/yggdrasil/schema.py
+++ b/yggdrasil/schema.py
@@ -141,7 +141,9 @@ def convert_extended2base(s):
                 'function': 'string', 'class': 'string',
                 'instance': 'string', '1darray': 'array',
                 'ndarray': 'array', 'obj': 'object',
-                'ply': 'object'}
+                'ply': 'object',
+                'any': ["number", "string", "boolean", "object", "array",
+                        "null"]}
     if isinstance(s, (list, tuple)):
         s = [convert_extended2base(x) for x in s]
     elif isinstance(s, (dict, OrderedDict)):

From b19dc0bb46c82351a2ac6c3f9058f3abebc1ab8a Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Fri, 17 Sep 2021 18:56:56 -0700
Subject: [PATCH 03/73] Add repository_commit parameter to model schema and
 require it for model submissions

---
 yggdrasil/.ygg_schema.yml            | 11 ++++++++---
 yggdrasil/drivers/ModelDriver.py     |  6 ++++++
 yggdrasil/schema.py                  | 12 ++++++------
 yggdrasil/tests/test_yamlfile.py     |  1 +
 yggdrasil/tests/yamls/FakePlant.yaml |  1 +
 yggdrasil/yamlfile.py                | 10 ++++++++--
 6 files changed, 30 insertions(+), 11 deletions(-)

diff --git a/yggdrasil/.ygg_schema.yml b/yggdrasil/.ygg_schema.yml
index 315a5ea20..30cc228bf 100644
--- a/yggdrasil/.ygg_schema.yml
+++ b/yggdrasil/.ygg_schema.yml
@@ -181,8 +181,8 @@ definitions:
           type: array
       required:
       - datatype
-      - name
       - commtype
+      - name
       title: comm_base
       type: object
     - anyOf:
@@ -370,8 +370,8 @@ definitions:
             - $ref: '#/definitions/transform'
           type: array
       required:
-      - inputs
       - outputs
+      - inputs
       title: connection_base
       type: object
     - anyOf:
@@ -1561,6 +1561,11 @@ definitions:
           items:
             type: string
           type: array
+        repository_commit:
+          description: Commit that should be checked out in the model repository specified
+            by repository_url. If not provided, the most recent commit on the default
+            branch will be used.
+          type: string
         repository_url:
           description: URL for the git repository containing the model source code.
             If provided, relative paths in the model YAML definition will be considered
@@ -1802,9 +1807,9 @@ definitions:
           type: string
       required:
       - args
+      - language
       - name
       - working_dir
-      - language
       title: model_base
       type: object
     - anyOf:
diff --git a/yggdrasil/drivers/ModelDriver.py b/yggdrasil/drivers/ModelDriver.py
index 32b53607b..f43c24ed0 100755
--- a/yggdrasil/drivers/ModelDriver.py
+++ b/yggdrasil/drivers/ModelDriver.py
@@ -173,6 +173,9 @@ class ModelDriver(Driver):
             the model source code. If provided, relative paths in the model
             YAML definition will be considered relative to the repository root
             directory.
+        repository_commit (str, optional): Commit that should be checked out
+            in the model repository specified by repository_url. If not
+            provided, the most recent commit on the default branch will be used.
         description (str, optional): Description of the model. This parameter
             is only used in the model repository or when providing the model
             as a service.
@@ -287,6 +290,8 @@ class ModelDriver(Driver):
             source code. If provided, relative paths in the model YAML
             definition will be considered relative to the repository root
             directory.
+        repository_commit (str): Commit that should be checked out in the
+            model repository specified by repository_url.
         description (str): Description of the model. This parameter is only
             used in the model repository or when providing the model as a
             service.
@@ -393,6 +398,7 @@ class ModelDriver(Driver):
         'allow_threading': {'type': 'boolean'},
         'copies': {'type': 'integer', 'default': 1, 'minimum': 1},
         'repository_url': {'type': 'string'},
+        'repository_commit': {'type': 'string'},
         'description': {'type': 'string'},
         'contact_email': {'type': 'string'}}
     _schema_excluded_from_class = ['name', 'language', 'args', 'working_dir']
diff --git a/yggdrasil/schema.py b/yggdrasil/schema.py
index 4cf5daf16..66908f92e 100644
--- a/yggdrasil/schema.py
+++ b/yggdrasil/schema.py
@@ -901,7 +901,7 @@ def model_form_schema_props(self):
                         "items": {"$ref": "#/definitions/transform"}}}},
             'required': {
                 'model': ['args', 'inputs', 'outputs', 'description',
-                          'repository_url']},
+                          'repository_url', 'repository_commit']},
             'remove': {
                 'comm': ['is_default', 'length_map', 'serializer',
                          'address', 'dont_copy', 'for_service',
@@ -922,9 +922,9 @@ def model_form_schema_props(self):
                           'aggregation', 'interpolation', 'synonyms',
                           'driver']},
             'order': {
-                'model': ['name', 'repository_url', 'contact_email',
-                          'language', 'description', 'args', 'inputs',
-                          'outputs']},
+                'model': ['name', 'repository_url', 'repository_commit',
+                          'contact_email', 'language', 'description',
+                          'args', 'inputs', 'outputs']},
         }
         return prop
         
@@ -977,10 +977,10 @@ def model_form_schema(self):
                 out['definitions'][x]['properties'][k].update(
                     {"$ref": "#/definitions/serializer"})
         prop = self.model_form_schema_props
-        for k in ['inputs', 'outputs']:
+        for k in ['inputs', 'outputs', 'repository_commit']:
             out['definitions']['model']['properties'][k].pop('default', None)
             desc = out['definitions']['model']['properties'][k]['description'].split(
-                ' A full description')[0]
+                '.')[0]
             out['definitions']['model']['properties'][k]['description'] = desc
         out['definitions']['model']['properties']['args']['minItems'] = 1
         for k, rlist in prop['remove'].items():
diff --git a/yggdrasil/tests/test_yamlfile.py b/yggdrasil/tests/test_yamlfile.py
index da04369fe..0560cba2f 100644
--- a/yggdrasil/tests/test_yamlfile.py
+++ b/yggdrasil/tests/test_yamlfile.py
@@ -655,6 +655,7 @@ class TestYamlModelSubmission(YamlTestBase):
                   '  - name: FakeModel',
                   '    repository_url: https://github.com/cropsinsilico/'
                   'example-fakemodel',
+                  '    repository_commit: e4bc7932c3c0c68fb3852cfb864777ca64cba448',
                   '    language: python',
                   '    args:',
                   '      - ./src/fakemodel.py',
diff --git a/yggdrasil/tests/yamls/FakePlant.yaml b/yggdrasil/tests/yamls/FakePlant.yaml
index 7ee6ea41b..79aa85681 100644
--- a/yggdrasil/tests/yamls/FakePlant.yaml
+++ b/yggdrasil/tests/yamls/FakePlant.yaml
@@ -21,4 +21,5 @@
       name: ./Output/output.txt
       filetype: table
   repository_url: https://github.com/cropsinsilico/example-fakemodel
+  repository_commit: e4bc7932c3c0c68fb3852cfb864777ca64cba448
   description: Example model submission
\ No newline at end of file
diff --git a/yggdrasil/yamlfile.py b/yggdrasil/yamlfile.py
index 565616369..6660b5e8a 100644
--- a/yggdrasil/yamlfile.py
+++ b/yggdrasil/yamlfile.py
@@ -36,13 +36,15 @@ def no_duplicates_constructor(loader, node, deep=False):
                         no_duplicates_constructor)
 
 
-def clone_github_repo(fname):
+def clone_github_repo(fname, commit=None):
     r"""Clone a GitHub repository, returning the path to the local copy of the
     file pointed to by the URL if there is one.
 
     Args:
         fname (str): URL to a GitHub repository or a file in a GitHub
             repository that should be cloned.
+        commit (str, optional): Commit that should be checked out. Defaults
+            to None and the HEAD of the default branch is used.
 
 
     Returns:
@@ -79,6 +81,8 @@ def clone_github_repo(fname):
             reponame
         # clone the repo into the appropriate directory
         repo = git.Repo.clone_from(cloneurl, os.path.join(owner, reponame))
+        if commit is not None:
+            repo.git.checkout(commit)
         repo.close()
         # now that it is cloned, just pass the yaml file (and path) onwards
     return os.path.realpath(fname)
@@ -195,7 +199,9 @@ def prep_yaml(files):
             for x in yml[k]:
                 if isinstance(x, dict):
                     if (k == 'models') and ('repository_url' in x):
-                        repo_dir = clone_github_repo(x['repository_url'])
+                        repo_dir = clone_github_repo(
+                            x['repository_url'],
+                            commit=x.get('repository_commit', None))
                         x.setdefault('working_dir', repo_dir)
                     else:
                         x.setdefault('working_dir', yml['working_dir'])

From e3e7069e489772c426c490568e3c7a81b4f13e8e Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Fri, 17 Sep 2021 19:15:46 -0700
Subject: [PATCH 04/73] Allow directory of YAMLs to be passed to
 validate_model_submission and add test that validates model repository

---
 yggdrasil/services.py            |  9 ++++++++-
 yggdrasil/tests/test_services.py | 16 ++++++++++++++++
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/yggdrasil/services.py b/yggdrasil/services.py
index 85e5e2ceb..1dd88c6e7 100644
--- a/yggdrasil/services.py
+++ b/yggdrasil/services.py
@@ -1119,11 +1119,18 @@ def validate_model_submission(fname):
     the yggdrasil model repository.
 
     Args:
-        fname (str): YAML file to validate.
+        fname (str): YAML file to validate or directory in which to check
+            each of the YAML files.
 
     """
     import glob
     from yggdrasil import yamlfile, runner
+    if os.path.isdir(fname):
+        files = sorted(glob.glob(os.path.join(fname, '*.yml'))
+                       + glob.glob(os.path.join(fname, '*.yaml')))
+        for x in files:
+            validate_model_submission(x)
+        return
     # 1-2. YAML syntax and schema
     yml = yamlfile.parse_yaml(fname, model_submission=True)
     # 3a. LICENSE
diff --git a/yggdrasil/tests/test_services.py b/yggdrasil/tests/test_services.py
index ca447061f..d64875661 100644
--- a/yggdrasil/tests/test_services.py
+++ b/yggdrasil/tests/test_services.py
@@ -347,3 +347,19 @@ def test_validate_model_submission():
     finally:
         if os.path.isfile('cropsinsilico/example-fakemodel/fakemodel.yml'):
             git.rmtree("cropsinsilico")
+
+
+def test_validate_model_repo():
+    r"""Test validation of YAMLs in the model repository."""
+    import git
+    import tempfile
+    dest = os.path.join(tempfile.gettempdir(), "model_repo")
+    url = "https://github.com/cropsinsilico/yggdrasil_models"
+    for x in [url, url + "_test"]:
+        try:
+            repo = git.Repo.clone_from(x, dest)
+            validate_model_submission(dest)
+            repo.close()
+        finally:
+            if os.path.isdir(dest):
+                git.rmtree(dest)

From b3092024c35ab60720d4eb74accdc27cd12746ec Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Fri, 17 Sep 2021 22:51:31 -0700
Subject: [PATCH 05/73] Require that arguments in form schema be non-zero
 length

---
 yggdrasil/schema.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/yggdrasil/schema.py b/yggdrasil/schema.py
index 66908f92e..b8b2e37af 100644
--- a/yggdrasil/schema.py
+++ b/yggdrasil/schema.py
@@ -983,6 +983,7 @@ def model_form_schema(self):
                 '.')[0]
             out['definitions']['model']['properties'][k]['description'] = desc
         out['definitions']['model']['properties']['args']['minItems'] = 1
+        out['definitions']['model']['properties']['args']['items']['minLength'] = 1
         for k, rlist in prop['remove'].items():
             for p in rlist:
                 out['definitions'][k]['properties'].pop(p, None)

From 092207d43fece1cc4e470071a0718fb704157cab Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Fri, 17 Sep 2021 23:17:55 -0700
Subject: [PATCH 06/73] Require default_file or default_value in the model form
 and update descriptions Refactor schema generation for the model submission
 form

---
 yggdrasil/.ygg_schema.yml           |  10 ++-
 yggdrasil/communication/FileComm.py |   2 +-
 yggdrasil/drivers/ModelDriver.py    |   4 +-
 yggdrasil/schema.py                 | 114 ++++++++++++++++++----------
 yggdrasil/yamlfile.py               |   4 +-
 5 files changed, 83 insertions(+), 51 deletions(-)

diff --git a/yggdrasil/.ygg_schema.yml b/yggdrasil/.ygg_schema.yml
index 30cc228bf..6606b1690 100644
--- a/yggdrasil/.ygg_schema.yml
+++ b/yggdrasil/.ygg_schema.yml
@@ -181,8 +181,8 @@ definitions:
           type: array
       required:
       - datatype
-      - commtype
       - name
+      - commtype
       title: comm_base
       type: object
     - anyOf:
@@ -704,9 +704,9 @@ definitions:
             is used.
           type: string
       required:
-      - filetype
       - name
       - working_dir
+      - filetype
       title: file_base
       type: object
     - anyOf:
@@ -1280,7 +1280,9 @@ definitions:
             that should be passed as input to the model program or language executable
             (e.g. source code or configuration file for a domain specific language).
           items:
+            minLength: 1
             type: string
+          minItems: 1
           type: array
         builddir:
           type: string
@@ -1806,10 +1808,10 @@ definitions:
             is used.
           type: string
       required:
-      - args
-      - language
       - name
       - working_dir
+      - args
+      - language
       title: model_base
       type: object
     - anyOf:
diff --git a/yggdrasil/communication/FileComm.py b/yggdrasil/communication/FileComm.py
index 8d44f7b62..995a5186b 100755
--- a/yggdrasil/communication/FileComm.py
+++ b/yggdrasil/communication/FileComm.py
@@ -80,7 +80,7 @@ class FileComm(CommBase.CommBase):
                                   'class': SerializeBase}],
                        'default': {'seritype': 'direct'}}}
     _schema_excluded_from_inherit = (
-        ['commtype', 'datatype', 'read_meth', 'serializer', 'default_value']
+        ['commtype', 'datatype', 'read_meth', 'serializer']
         + CommBase.CommBase._model_schema_prop)
     _schema_excluded_from_class_validation = ['serializer']
     _schema_base_class = None
diff --git a/yggdrasil/drivers/ModelDriver.py b/yggdrasil/drivers/ModelDriver.py
index f43c24ed0..a4f3bbc27 100755
--- a/yggdrasil/drivers/ModelDriver.py
+++ b/yggdrasil/drivers/ModelDriver.py
@@ -315,8 +315,8 @@ class ModelDriver(Driver):
                          'is written in. A list of available '
                          'languages can be found :ref:`here <'
                          'schema_table_model_subtype_rst>`.')},
-        'args': {'type': 'array',
-                 'items': {'type': 'string'}},
+        'args': {'type': 'array', 'minItems': 1,
+                 'items': {'type': 'string', 'minLength': 1}},
         'inputs': {'type': 'array', 'default': [],
                    'items': {'$ref': '#/definitions/comm'},
                    'description': (
diff --git a/yggdrasil/schema.py b/yggdrasil/schema.py
index b8b2e37af..eb0f1fa9d 100644
--- a/yggdrasil/schema.py
+++ b/yggdrasil/schema.py
@@ -898,7 +898,12 @@ def model_form_schema_props(self):
                 'comm': {
                     'transform': {
                         "type": "array",
-                        "items": {"$ref": "#/definitions/transform"}}}},
+                        "items": {"$ref": "#/definitions/transform"}},
+                    'default_file': {
+                        '$ref': '#/definitions/file'}},
+                'file': {
+                    'serializer': {
+                        '$ref': '#/definitions/serializer'}}},
             'required': {
                 'model': ['args', 'inputs', 'outputs', 'description',
                           'repository_url', 'repository_commit']},
@@ -906,7 +911,8 @@ def model_form_schema_props(self):
                 'comm': ['is_default', 'length_map', 'serializer',
                          'address', 'dont_copy', 'for_service',
                          'send_converter', 'recv_converter', 'client_id',
-                         'cookies', 'host', 'params', 'port'],
+                         'cookies', 'host', 'params', 'port', 'commtype'],
+                'ocomm': ['default_value'],
                 'file': ['is_default', 'length_map',
                          'wait_for_creation', 'working_dir',
                          'read_meth', 'in_temp',
@@ -924,7 +930,25 @@ def model_form_schema_props(self):
             'order': {
                 'model': ['name', 'repository_url', 'repository_commit',
                           'contact_email', 'language', 'description',
-                          'args', 'inputs', 'outputs']},
+                          'args', 'inputs', 'outputs'],
+                'comm': ['name', 'datatype']},
+            'update': {
+                'model': {
+                    'inputs': {
+                        'description': ('Zero or more channels carrying '
+                                        'input to the model'),
+                        'items': {'$ref': '#/definitions/icomm'}},
+                    'outputs': {
+                        'description': ('Zero or more channels carrying '
+                                        'output from the model'),
+                        'items': {'$ref': '#/definitions/ocomm'}},
+                    'repository_commit': {
+                        'description': ('Commit that should be checked out '
+                                        'from the model repository.')}},
+                'file': {
+                    'name': {
+                        'description': ('Path to a file in the model '
+                                        'repository')}}},
         }
         return prop
         
@@ -956,6 +980,8 @@ def model_form_schema(self):
             if types:
                 out['definitions']['schema']['properties'][k]['options'] = {
                     'dependencies': {'type': types}}
+        for k in ['comm', 'file', 'model']:
+            out['definitions'][k].pop('description', '')
         for k in out['definitions'].keys():
             if k in ['schema', 'simpleTypes']:
                 continue
@@ -966,54 +992,60 @@ def model_form_schema(self):
             for p, v in list(out['definitions'][k]['properties'].items()):
                 if v.get('description', '').startswith('[DEPRECATED]'):
                     out['definitions'][k]['properties'].pop(p)
-        for x in ['comm', 'file']:
-            for k in ['send_converter', 'recv_converter']:
-                out['definitions'][x]['properties'][k].pop('oneOf', None)
-                out['definitions'][x]['properties'][k].update(
-                    type='array', items={"$ref": "#/definitions/transform"})
-        for x in ['file']:
-            for k in ['serializer']:
-                out['definitions'][x]['properties'][k].pop('oneOf', None)
-                out['definitions'][x]['properties'][k].update(
-                    {"$ref": "#/definitions/serializer"})
+        # Process based on model_form_schema_props
         prop = self.model_form_schema_props
-        for k in ['inputs', 'outputs', 'repository_commit']:
-            out['definitions']['model']['properties'][k].pop('default', None)
-            desc = out['definitions']['model']['properties'][k]['description'].split(
-                '.')[0]
-            out['definitions']['model']['properties'][k]['description'] = desc
-        out['definitions']['model']['properties']['args']['minItems'] = 1
-        out['definitions']['model']['properties']['args']['items']['minLength'] = 1
-        for k, rlist in prop['remove'].items():
-            for p in rlist:
+
+        def adjust_definitions(k):
+            # Remove
+            for p in prop['remove'].get(k, []):
                 out['definitions'][k]['properties'].pop(p, None)
-        for k, rdict in prop['replace'].items():
-            for r in rdict.keys():
-                if 'description' in out['definitions'][k]['properties'][r]:
-                    rdict[r]['description'] = (
+                if p in out['definitions'][k].get('required', []):
+                    out['definitions'][k]['required'].remove(p)
+            # Replace
+            for r, v in prop['replace'].get(k, {}).items():
+                if 'description' in out['definitions'][k]['properties'].get(r, {}):
+                    v['description'] = (
                         out['definitions'][k]['properties'][r]['description'])
-            out['definitions'][k]['properties'].update(rdict)
-        for k, rlist in prop['required'].items():
+                out['definitions'][k]['properties'][r] = v
+            # Required
             out['definitions'][k].setdefault('required', [])
-            for p in rlist:
+            for p in prop['required'].get(k, []):
                 if p not in out['definitions'][k]['required']:
                     out['definitions'][k]['required'].append(p)
-        # for k, adict in prop['add'].items():
-        #     out['definitions'][k]['properties'].update(adict)
-        for k, rlist in prop['order'].items():
-            for i, p in enumerate(rlist):
+            # Update
+            for p, new in prop['update'].get(k, {}).items():
+                out['definitions'][k]['properties'][p].update(new)
+            # Add
+            # out['definitions'][k]['properties'].update(
+            #     prop['add'].get(k, {}))
+            # Order
+            for i, p in enumerate(prop['order'].get(k, [])):
                 out['definitions'][k]['properties'][p]['propertyOrder'] = i
-        out.update(out['definitions'].pop('model'))
-        out['definitions'].pop('connection')
+
+        # Update definitions
+        for k in ['model', 'comm', 'file']:
+            adjust_definitions(k)
+        for k in ['icomm', 'ocomm']:
+            out['definitions'][k] = copy.deepcopy(out['definitions']['comm'])
+            adjust_definitions(k)
+        out['definitions']['icomm']['oneOf'] = [
+            {'title': 'default file',
+             'required': ['default_file'],
+             'not': {'required': ['default_value']}},
+            {'title': 'default value',
+             'required': ['default_value'],
+             'not': {'required': ['default_file']}}]
+        # Adjust formating
         for x in [out] + list(out['definitions'].values()):
             for p, v in x.get('properties', {}).items():
                 if v.get("type", None) == "boolean":
                     v.setdefault("format", "checkbox")
+        # Isolate model
+        out.update(out['definitions'].pop('model'))
+        out['definitions'].pop('connection')
         out.update(
             title='Model YAML Schema',
             description='Schema for yggdrasil model YAML input files.')
-        out['definitions']['comm']['properties']['default_file'] = {
-            '$ref': '#/definitions/file'}
         out = convert_extended2base(out)
         return out
 
@@ -1100,12 +1132,12 @@ def validate_model_submission(self, obj, **kwargs):
 
         Args:
             obj (object): Object to validate.
-            **kwargs: Additional keyword arguments are passed to
-                yggdrasil.metaschema.validate_instance.
+            **kwargs: Additional keyword arguments are ignored.
 
         """
-        return metaschema.validate_instance(obj, self.model_form_schema,
-                                            **kwargs)
+        import jsonschema
+        jsonschema.validate(obj, self.model_form_schema)
+        return obj
 
     def validate_component(self, comp_name, obj, **kwargs):
         r"""Validate an object against a specific component.
diff --git a/yggdrasil/yamlfile.py b/yggdrasil/yamlfile.py
index 6660b5e8a..9a7e2c7f4 100644
--- a/yggdrasil/yamlfile.py
+++ b/yggdrasil/yamlfile.py
@@ -253,9 +253,7 @@ def parse_yaml(files, complete_partial=False, partial_commtype=None,
         models = []
         for yml in yml_prep['models']:
             wd = yml.pop('working_dir', None)
-            x = s.validate_model_submission(yml, normalize=True,
-                                            no_defaults=True,
-                                            required_defaults=True)
+            x = s.validate_model_submission(yml)
             if wd:
                 x['working_dir'] = wd
             models.append(x)

From 3647136cbedb97ad1ae8dbc6ce47cb5389cdd7f7 Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Wed, 22 Sep 2021 11:16:54 -0700
Subject: [PATCH 07/73] Only require args in the model form schema (timesync
 models in integrations do not require them)

---
 yggdrasil/.ygg_schema.yml        | 7 +++----
 yggdrasil/drivers/ModelDriver.py | 2 +-
 yggdrasil/schema.py              | 3 ++-
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/yggdrasil/.ygg_schema.yml b/yggdrasil/.ygg_schema.yml
index 6606b1690..c243729be 100644
--- a/yggdrasil/.ygg_schema.yml
+++ b/yggdrasil/.ygg_schema.yml
@@ -180,8 +180,8 @@ definitions:
             type: string
           type: array
       required:
-      - datatype
       - name
+      - datatype
       - commtype
       title: comm_base
       type: object
@@ -705,8 +705,8 @@ definitions:
           type: string
       required:
       - name
-      - working_dir
       - filetype
+      - working_dir
       title: file_base
       type: object
     - anyOf:
@@ -1282,7 +1282,6 @@ definitions:
           items:
             minLength: 1
             type: string
-          minItems: 1
           type: array
         builddir:
           type: string
@@ -1809,8 +1808,8 @@ definitions:
           type: string
       required:
       - name
-      - working_dir
       - args
+      - working_dir
       - language
       title: model_base
       type: object
diff --git a/yggdrasil/drivers/ModelDriver.py b/yggdrasil/drivers/ModelDriver.py
index a4f3bbc27..545aa5e26 100755
--- a/yggdrasil/drivers/ModelDriver.py
+++ b/yggdrasil/drivers/ModelDriver.py
@@ -315,7 +315,7 @@ class ModelDriver(Driver):
                          'is written in. A list of available '
                          'languages can be found :ref:`here <'
                          'schema_table_model_subtype_rst>`.')},
-        'args': {'type': 'array', 'minItems': 1,
+        'args': {'type': 'array',
                  'items': {'type': 'string', 'minLength': 1}},
         'inputs': {'type': 'array', 'default': [],
                    'items': {'$ref': '#/definitions/comm'},
diff --git a/yggdrasil/schema.py b/yggdrasil/schema.py
index eb0f1fa9d..8ba42b300 100644
--- a/yggdrasil/schema.py
+++ b/yggdrasil/schema.py
@@ -944,7 +944,8 @@ def model_form_schema_props(self):
                         'items': {'$ref': '#/definitions/ocomm'}},
                     'repository_commit': {
                         'description': ('Commit that should be checked out '
-                                        'from the model repository.')}},
+                                        'from the model repository.')},
+                    'args': {'minItems': 1}},
                 'file': {
                     'name': {
                         'description': ('Path to a file in the model '

From 5a36518c5da498b0f8ca6efa43543609c7d3d29f Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Wed, 22 Sep 2021 12:04:19 -0700
Subject: [PATCH 08/73] Update validation CLI to do full validation.

---
 yggdrasil/command_line.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/yggdrasil/command_line.py b/yggdrasil/command_line.py
index 67c3b1cfd..418ffd894 100755
--- a/yggdrasil/command_line.py
+++ b/yggdrasil/command_line.py
@@ -832,14 +832,18 @@ class validate_yaml(SubCommand):
                    'ensuring that it is part of a complete integration.')}),
         (('--model-submission', ),
          {'action': 'store_true',
-          'help': ('Validate a YAML against the schema for submissions to '
-                   'the yggdrasil model repository.')})]
+          'help': ('Validate a YAML against the requirements for '
+                   'submissions to the yggdrasil model repository.')})]
 
     @classmethod
     def func(cls, args):
-        from yggdrasil import yamlfile
-        yamlfile.parse_yaml(args.yamlfile, model_only=args.model_only,
-                            model_submission=args.model_submission)
+        if args.model_submission:
+            from yggdrasil.services import validate_model_submission
+            validate_model_submission(args.yamlfile)
+        else:
+            from yggdrasil import yamlfile
+            yamlfile.parse_yaml(args.yamlfile, model_only=args.model_only,
+                                model_submission=args.model_submission)
         logger.info("Validation succesful.")
 
 

From 0e35e8a90999257a16bcdcf99e298cc2ceb02c52 Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Wed, 22 Sep 2021 13:16:58 -0700
Subject: [PATCH 09/73] Fix tests to conform to new model form schema

---
 yggdrasil/tests/test_yamlfile.py     | 1 +
 yggdrasil/tests/yamls/FakePlant.yaml | 2 --
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/yggdrasil/tests/test_yamlfile.py b/yggdrasil/tests/test_yamlfile.py
index 0560cba2f..01808ba28 100644
--- a/yggdrasil/tests/test_yamlfile.py
+++ b/yggdrasil/tests/test_yamlfile.py
@@ -656,6 +656,7 @@ class TestYamlModelSubmission(YamlTestBase):
                   '    repository_url: https://github.com/cropsinsilico/'
                   'example-fakemodel',
                   '    repository_commit: e4bc7932c3c0c68fb3852cfb864777ca64cba448',
+                  '    description: Example model submission',
                   '    language: python',
                   '    args:',
                   '      - ./src/fakemodel.py',
diff --git a/yggdrasil/tests/yamls/FakePlant.yaml b/yggdrasil/tests/yamls/FakePlant.yaml
index 79aa85681..0c8dc4a12 100644
--- a/yggdrasil/tests/yamls/FakePlant.yaml
+++ b/yggdrasil/tests/yamls/FakePlant.yaml
@@ -6,7 +6,6 @@
   language: python
   inputs:
   - name: photosynthesis_rate
-    commtype: default
     datatype:
       type: bytes
     default_file:
@@ -14,7 +13,6 @@
       filetype: table
   outputs:
   - name: growth_rate
-    commtype: default
     datatype:
       type: bytes
     default_file:

From 47d076b21b43af5f165c6273ae560c10cff627e0 Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Wed, 22 Sep 2021 13:52:09 -0700
Subject: [PATCH 10/73] Added --model_repository option to the
 integration-service-manager CLI Preload models from the model repository into
 the service manager registry Added test for calling one of the preloaded
 models

---
 HISTORY.rst                      |  6 ++++++
 yggdrasil/command_line.py        | 11 +++++++++--
 yggdrasil/services.py            | 33 +++++++++++++++++++++++++++++---
 yggdrasil/tests/test_services.py |  8 ++++++--
 yggdrasil/yamlfile.py            | 14 +++++++++-----
 5 files changed, 60 insertions(+), 12 deletions(-)

diff --git a/HISTORY.rst b/HISTORY.rst
index bb60acf0f..c3cfacd54 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -2,6 +2,12 @@
 History
 =======
 
+1.8.1 (2021-09-XX) Minor updates to support model submission form development
+------------------
+
+* Added --model_repository option to the integration-service-manager CLI
+* Preload models from the model repository into the service manager registry
+
 1.8.0 (2021-09-15) Support for REST API based communicators, running integrations as services, and connecting to remote integration services
 ------------------
 
diff --git a/yggdrasil/command_line.py b/yggdrasil/command_line.py
index 418ffd894..2730f9358 100755
--- a/yggdrasil/command_line.py
+++ b/yggdrasil/command_line.py
@@ -328,7 +328,12 @@ class integration_service_manager(SubCommand):
                                    'registered integration.')}),
                         (('--with-coverage', ),
                          {'action': 'store_true',
-                          'help': ('Enable coverage cleanup for testing.')})]),
+                          'help': ('Enable coverage cleanup for testing.')}),
+                        (('--model-repository', ),
+                         {'type': str,
+                          'help': ('URL for a directory in a Git repository '
+                                   'containing models that should be loaded '
+                                   'into the service manager registry.')})]),
                 ArgumentParser(
                     name='stop',
                     help=('Stop an integration service manager or '
@@ -405,7 +410,9 @@ def func(cls, args):
                 if not x.is_running:
                     x.start_server(
                         remote_url=getattr(args, 'remote_url', None),
-                        with_coverage=getattr(args, 'with_coverage', False))
+                        with_coverage=getattr(args, 'with_coverage', False),
+                        model_repository=getattr(args, 'model_repository',
+                                                 None))
             else:
                 x.send_request(integration_name,
                                yamls=integration_yamls,
diff --git a/yggdrasil/services.py b/yggdrasil/services.py
index 1dd88c6e7..97c52e039 100644
--- a/yggdrasil/services.py
+++ b/yggdrasil/services.py
@@ -5,6 +5,7 @@
 import json
 import traceback
 import yaml
+import glob
 import pprint
 import functools
 import threading
@@ -127,7 +128,7 @@ def set_log_level(self, log_level):
         logging.basicConfig(level=log_level)
 
     def start_server(self, remote_url=None, with_coverage=False,
-                     log_level=None):
+                     log_level=None, model_repository=None):
         r"""Start the server.
 
         Args:
@@ -140,12 +141,17 @@ def start_server(self, remote_url=None, with_coverage=False,
                 with coverage. Defaults to False.
             log_level (int, optional): Level of log messages that should be
                 printed. Defaults to None and is ignored.
+            model_repository (str, optional): URL of directory in a Git
+                repository containing YAMLs that should be added to the model
+                registry. Defaults to None and is ignored.
 
         """
         if remote_url is None:
             remote_url = os.environ.get(_service_host_env, None)
         if remote_url is None:
             remote_url = self.address
+        if model_repository is not None:
+            self.registry.add_from_repository(model_repository)
         os.environ.setdefault(_service_host_env, remote_url)
         if log_level is not None:
             self.set_log_level(log_level)
@@ -1078,6 +1084,28 @@ def remove(self, name):
             registry.pop(k)
         self.save(registry)
 
+    def add_from_repository(self, model_repository, directory=None):
+        r"""Add integration services to the registry from a repository of
+        model YAMLs.
+
+        Args:
+            model_repository (str): URL of directory in a Git repository
+                containing YAMLs that should be added to the model registry.
+            directory (str, optional): Directory where services from the
+                model_repository should be cloned. Defaults to
+                '~/.yggdrasil_service'.
+
+        """
+        from yggdrasil.yamlfile import clone_github_repo
+        if directory is None:
+            directory = os.path.join('~', '.yggdrasil_services')
+        yaml_dir = clone_github_repo(model_repository,
+                                     local_directory=directory)
+        yaml_files = (glob.glob(os.path.join(yaml_dir, '*.yaml'))
+                      + glob.glob(os.path.join(yaml_dir, '*.yml')))
+        for x in yaml_files:
+            self.add(os.path.splitext(os.path.basename(x))[0], x)
+
     def add(self, name, yamls=None, **kwargs):
         r"""Add an integration service to the registry.
 
@@ -1102,7 +1130,7 @@ def add(self, name, yamls=None, **kwargs):
             assert(yamls)
             collection = {name: dict(kwargs, name=name, yamls=yamls)}
         for k, v in collection.items():
-            if k in registry:
+            if (k in registry) and (registry[k] != v):
                 old = pprint.pformat(registry[k])
                 new = pprint.pformat(v)
                 raise ValueError(f"There is an registry integration "
@@ -1123,7 +1151,6 @@ def validate_model_submission(fname):
             each of the YAML files.
 
     """
-    import glob
     from yggdrasil import yamlfile, runner
     if os.path.isdir(fname):
         files = sorted(glob.glob(os.path.join(fname, '*.yml'))
diff --git a/yggdrasil/tests/test_services.py b/yggdrasil/tests/test_services.py
index d64875661..1437ac521 100644
--- a/yggdrasil/tests/test_services.py
+++ b/yggdrasil/tests/test_services.py
@@ -72,11 +72,13 @@ def check_settings(service_type, partial_commtype=None):
 def running_service(service_type, partial_commtype=None, with_coverage=False):
     r"""Context manager to run and clean-up an integration service."""
     check_settings(service_type, partial_commtype)
+    model_repo = "https://github.com/cropsinsilico/yggdrasil_models_test/models"
     log_level = logging.ERROR
     args = [sys.executable, "-m", "yggdrasil", "integration-service-manager",
             f"--service-type={service_type}"]
     if partial_commtype is not None:
         args.append(f"--commtype={partial_commtype}")
+    args += ["start", f"--model-repository={model_repo}"]
     package_dir = None
     process_kws = {}
     if with_coverage:
@@ -97,7 +99,8 @@ def running_service(service_type, partial_commtype=None, with_coverage=False):
         lines[-1] += ')'
         lines += ['assert(not srv.is_running)',
                   f'srv.start_server(with_coverage={with_coverage},',
-                  f'                 log_level={log_level})']
+                  f'                 log_level={log_level},'
+                  f'                 model_repository=\'{model_repo}\')']
         with open(script_path, 'w') as fd:
             fd.write('\n'.join(lines))
         args = [sys.executable, script_path]
@@ -255,6 +258,7 @@ def test_registered_service(self, running_service):
         test_yml = ex_yamls['fakeplant']['python']
         assert_raises(KeyError, cli.registry.remove, 'test')
         assert_raises(ServerError, cli.send_request, 'test')
+        print(cli.send_request('FakePlant'))
         cli.registry.add('test', test_yml, namespace='remote')
         print(cli.send_request('test'))
         assert_raises(ValueError, cli.registry.add, 'test', [test_yml])
@@ -272,7 +276,7 @@ def test_registered_service(self, running_service):
             cli.registry.add(reg_coll)
             print(cli.send_request('photosynthesis', namespace='phot'))
             assert_raises(ValueError, cli.registry.add,
-                          'photosynthesis', [test_yml])
+                          'photosynthesis', [test_yml], invalid=1)
             cli.send_request('photosynthesis', action='stop')
             cli.registry.remove(reg_coll)
             assert_raises(KeyError, cli.registry.remove, 'photosynthesis')
diff --git a/yggdrasil/yamlfile.py b/yggdrasil/yamlfile.py
index 9a7e2c7f4..91c566633 100644
--- a/yggdrasil/yamlfile.py
+++ b/yggdrasil/yamlfile.py
@@ -36,7 +36,7 @@ def no_duplicates_constructor(loader, node, deep=False):
                         no_duplicates_constructor)
 
 
-def clone_github_repo(fname, commit=None):
+def clone_github_repo(fname, commit=None, local_directory=None):
     r"""Clone a GitHub repository, returning the path to the local copy of the
     file pointed to by the URL if there is one.
 
@@ -45,15 +45,18 @@ def clone_github_repo(fname, commit=None):
             repository that should be cloned.
         commit (str, optional): Commit that should be checked out. Defaults
             to None and the HEAD of the default branch is used.
-
+        local_directory (str, optional): Local directory that the file should
+            be cloned into. Defaults to None and the current working directory
+            will be used.
 
     Returns:
         str: Path to the local copy of the repository or file in the
             repository.
 
     """
-    
     from yggdrasil.services import _service_host_env
+    if local_directory is None:
+        local_directory = os.getcwd()
     # make sure we start with a full url
     if 'http' not in fname:
         url = 'http://github.com/' + fname
@@ -69,7 +72,7 @@ def clone_github_repo(fname, commit=None):
     reponame = splitpath[2]
     # the full path is the file name and location
     # turn the file path into an os based format
-    fname = os.path.join(*splitpath)
+    fname = os.path.join(local_directory, *splitpath)
     # check to see if the file already exists, and clone if it does not
     if not os.path.exists(fname):
         if os.environ.get(_service_host_env, False):
@@ -80,7 +83,8 @@ def clone_github_repo(fname, commit=None):
         cloneurl = parsed.scheme + '://' + parsed.netloc + '/' + owner + '/' +\
             reponame
         # clone the repo into the appropriate directory
-        repo = git.Repo.clone_from(cloneurl, os.path.join(owner, reponame))
+        repo = git.Repo.clone_from(cloneurl, os.path.join(local_directory,
+                                                          owner, reponame))
         if commit is not None:
             repo.git.checkout(commit)
         repo.close()

From 8316bc469bbc9ac13abf7fc4e5b4a951d78817f4 Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Wed, 22 Sep 2021 14:22:32 -0700
Subject: [PATCH 11/73] Allow validate_model_submission to receive a list so it
 works with the new CLI

---
 yggdrasil/services.py            | 6 +++++-
 yggdrasil/tests/test_services.py | 2 +-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/yggdrasil/services.py b/yggdrasil/services.py
index 97c52e039..bd23789a8 100644
--- a/yggdrasil/services.py
+++ b/yggdrasil/services.py
@@ -1152,7 +1152,11 @@ def validate_model_submission(fname):
 
     """
     from yggdrasil import yamlfile, runner
-    if os.path.isdir(fname):
+    if isinstance(fname, list):
+        for x in fname:
+            validate_model_submission(x)
+        return
+    elif os.path.isdir(fname):
         files = sorted(glob.glob(os.path.join(fname, '*.yml'))
                        + glob.glob(os.path.join(fname, '*.yaml')))
         for x in files:
diff --git a/yggdrasil/tests/test_services.py b/yggdrasil/tests/test_services.py
index 1437ac521..6502f92c1 100644
--- a/yggdrasil/tests/test_services.py
+++ b/yggdrasil/tests/test_services.py
@@ -344,7 +344,7 @@ def test_validate_model_submission():
     try:
         fname = os.path.join(os.path.dirname(__file__), 'yamls',
                              'FakePlant.yaml')
-        validate_model_submission(fname)
+        validate_model_submission([fname])
         os.remove(os.path.join('cropsinsilico', 'example-fakemodel',
                                'LICENSE'))
         assert_raises(RuntimeError, validate_model_submission, fname)

From 92c1fdbf4f9f5d41773e78efc078c7664cf67fab Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Wed, 22 Sep 2021 14:43:04 -0700
Subject: [PATCH 12/73] Preload YAMLs in the model repository to avoid hold on
 git cloning on the service manager Add example READMEs to the manifest Minor
 doc updates Push latest tag by default in build_docker script

---
 MANIFEST.in            |  1 +
 docs/source/docker.rst |  2 +-
 utils/build_docker.py  | 15 ++++++++++++++-
 yggdrasil/services.py  |  7 ++++++-
 4 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/MANIFEST.in b/MANIFEST.in
index f80b5c655..4c79a4992 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -20,6 +20,7 @@ recursive-include yggdrasil *.json
 recursive-include yggdrasil *.f90
 recursive-include yggdrasil *.xml
 recursive-include yggdrasil/examples *.py
+recursive-include yggdrasil/examples *.rst
 recursive-include yggdrasil/demos *.py
 recursive-include yggdrasil/tests/scripts *.py
 recursive-include yggdrasil/examples Makefile_linux
diff --git a/docs/source/docker.rst b/docs/source/docker.rst
index 34b25ea75..c57519a36 100644
--- a/docs/source/docker.rst
+++ b/docs/source/docker.rst
@@ -83,7 +83,7 @@ Then the Dockerfile that will be able to run those services is
 
 .. code-block:: docker
 
-   FROM cropsinsilico/yggdrasil-service:1.7.0
+   FROM cropsinsilico/yggdrasil-service:v1.8.0
    COPY services.yml .
    COPY foo ./foo
    COPY bar ./bar
diff --git a/utils/build_docker.py b/utils/build_docker.py
index 0e3427ce0..29aa8cdef 100644
--- a/utils/build_docker.py
+++ b/utils/build_docker.py
@@ -7,7 +7,7 @@
 
 
 def build(dockerfile, tag, flags=[], repo='cropsinsilico/yggdrasil',
-          context=_utils_dir):
+          context=_utils_dir, disable_latest=False):
     r"""Build a docker image.
 
     Args:
@@ -20,11 +20,17 @@ def build(dockerfile, tag, flags=[], repo='cropsinsilico/yggdrasil',
         context (str, optional): Directory that should be provided as the
             context for the image. Defaults to the directory containing this
             script.
+        disable_latest (bool, optional): If True, the new image will not
+            be tagged 'latest' in addition to the provided tag value. Defaults
+            to False.
 
     """
     args = ['docker', 'build', '-t', f'{repo}:{tag}', '-f', dockerfile] + flags
     args.append(context)
     subprocess.call(args)
+    if not disable_latest:
+        args = ['docker', 'tag', f'{repo}:{tag}', f"{repo}:latest"]
+        subprocess.call(args)
 
 
 def push_image(tag, repo='cropsinsilico/yggdrasil'):
@@ -140,6 +146,10 @@ def params_service(params):
     parser.add_argument(
         "--push", action="store_true",
         help="After successfully building the image, push it to DockerHub.")
+    parser.add_argument(
+        "--disable-latest", action="store_true",
+        help=("Don't tag the new image as 'latest' in addition to the "
+              "version/commit specific tag."))
     subparsers = parser.add_subparsers(
         dest="type",
         help=("Type of docker image that should be built "
@@ -163,8 +173,11 @@ def params_service(params):
         params = params_executable(params)
     elif args.type == 'service':
         params = params_service(params)
+    params.setdefault('disable_latest', args.disable_latest)
     dockerfile = params.pop('dockerfile')
     tag = params.pop('tag')
     build(dockerfile, tag, **params)
     if args.push:
         push_image(tag, repo=params['repo'])
+        if not params['disable_latest']:
+            push_image('latest', repo=params['repo'])
diff --git a/yggdrasil/services.py b/yggdrasil/services.py
index bd23789a8..ffa4a374d 100644
--- a/yggdrasil/services.py
+++ b/yggdrasil/services.py
@@ -1096,7 +1096,7 @@ def add_from_repository(self, model_repository, directory=None):
                 '~/.yggdrasil_service'.
 
         """
-        from yggdrasil.yamlfile import clone_github_repo
+        from yggdrasil.yamlfile import clone_github_repo, prep_yaml
         if directory is None:
             directory = os.path.join('~', '.yggdrasil_services')
         yaml_dir = clone_github_repo(model_repository,
@@ -1104,6 +1104,11 @@ def add_from_repository(self, model_repository, directory=None):
         yaml_files = (glob.glob(os.path.join(yaml_dir, '*.yaml'))
                       + glob.glob(os.path.join(yaml_dir, '*.yml')))
         for x in yaml_files:
+            # Calling prep_yaml allows the model repositories to be cloned
+            # in advance to circumvent th hold place on git cloning on the
+            # service manager (these models are assumed to be vetted so
+            # they do not pose a security risk).
+            prep_yaml(x)
             self.add(os.path.splitext(os.path.basename(x))[0], x)
 
     def add(self, name, yamls=None, **kwargs):

From a8223b2ce6e243a6c46ed07f2ef3162bfb6306ce Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Wed, 22 Sep 2021 16:46:58 -0700
Subject: [PATCH 13/73] Allow mustache replacement parameters to be passed to
 runner and via service request

---
 yggdrasil/runner.py              | 19 +++++++++++--------
 yggdrasil/tests/test_services.py | 23 +++++++++++++++--------
 yggdrasil/yamlfile.py            | 25 ++++++++++++++++++-------
 3 files changed, 44 insertions(+), 23 deletions(-)

diff --git a/yggdrasil/runner.py b/yggdrasil/runner.py
index a9a76c96c..022cc2ec7 100755
--- a/yggdrasil/runner.py
+++ b/yggdrasil/runner.py
@@ -238,12 +238,12 @@ class YggRunner(YggClass):
         namespace (str, optional): Name that should be used to uniquely
             identify any RMQ exchange. Defaults to the value in the config
             file.
-        host (str, optional): Name of the host that the models will be launched
-            from. Defaults to None.
+        host (str, optional): Name of the host that the models will be
+            launched from. Defaults to None.
         rank (int, optional): Rank of this set of models if run in parallel.
             Defaults to 0.
-        ygg_debug_level (str, optional): Level for Ygg debug messages. Defaults
-            to environment variable 'YGG_DEBUG'.
+        ygg_debug_level (str, optional): Level for Ygg debug messages.
+            Defaults to environment variable 'YGG_DEBUG'.
         rmq_debug_level (str, optional): Level for RabbitMQ debug messages.
             Defaults to environment variable 'RMQ_DEBUG'.
         ygg_debug_prefix (str, optional): Prefix for Ygg debug messages.
@@ -257,10 +257,13 @@ class YggRunner(YggClass):
         partial_commtype (dict, optional): Communicator kwargs that should be
             be used for the connections to the unpaired channels when
             complete_partial is True. Defaults to None and will be ignored.
+        yaml_param (dict, optional): Parameters that should be used in
+            mustache formatting of YAML files. Defaults to None and is
+            ignored.
 
     Attributes:
-        namespace (str): Name that should be used to uniquely identify any RMQ
-            exchange.
+        namespace (str): Name that should be used to uniquely identify any
+            RMQ exchange.
         host (str): Name of the host that the models will be launched from.
         rank (int): Rank of this set of models if run in parallel.
         modeldrivers (dict): Model drivers associated with this run.
@@ -276,7 +279,7 @@ def __init__(self, modelYmls, namespace=None, host=None, rank=0,
                  ygg_debug_prefix=None, connection_task_method='thread',
                  as_service=False, complete_partial=False,
                  partial_commtype=None, production_run=False,
-                 mpi_tag_start=None):
+                 mpi_tag_start=None, yaml_param=None):
         self.mpi_comm = None
         name = 'runner'
         if MPI is not None:
@@ -317,7 +320,7 @@ def __init__(self, modelYmls, namespace=None, host=None, rank=0,
         else:
             self.drivers = yamlfile.parse_yaml(
                 modelYmls, complete_partial=complete_partial,
-                partial_commtype=partial_commtype)
+                partial_commtype=partial_commtype, yaml_param=yaml_param)
             self.connectiondrivers = self.drivers['connection']
             self.modeldrivers = self.drivers['model']
             for x in self.modeldrivers.values():
diff --git a/yggdrasil/tests/test_services.py b/yggdrasil/tests/test_services.py
index 6502f92c1..c4f24055f 100644
--- a/yggdrasil/tests/test_services.py
+++ b/yggdrasil/tests/test_services.py
@@ -189,7 +189,7 @@ def running_service(self, request):
             self.cli = None
 
     def call_integration_service(self, cli, yamls, test_yml, copy_yml=None,
-                                 name='test'):
+                                 name='test', yaml_param=None):
         r"""Call an integration that includes a service."""
         remote_yml = '_remote'.join(os.path.splitext(test_yml))
         yamls = copy.copy(yamls)
@@ -205,12 +205,17 @@ def call_integration_service(self, cli, yamls, test_yml, copy_yml=None,
                 remote_code = 'a'
             else:
                 remote_code = 'w'
+            lines = ['service:',
+                     f'    name: {name}',
+                     f'    yamls: [{test_yml}]',
+                     f'    type: {service_type}',
+                     f'    address: {address}']
+            if yaml_param:
+                lines.append('    yaml_param:')
+                for k, v in yaml_param.items():
+                    lines.append(f'        {k}: "{v}"')
             with open(remote_yml, remote_code) as fd:
-                fd.write('\n'.join(['service:',
-                                    f'    name: {name}',
-                                    f'    yamls: [{test_yml}]',
-                                    f'    type: {service_type}',
-                                    f'    address: {address}']))
+                fd.write('\n'.join(lines))
             r = runner.get_runner(yamls)
             r.run()
             assert(not r.error_flag)
@@ -301,8 +306,9 @@ def test_calling_server_as_service(self, running_service):
         if (((running_service.commtype != 'rest')
              or (running_service.service_type != 'flask'))):
             pytest.skip("redundent test")  # pragma: testing
-        os.environ.update(FIB_ITERATIONS='3',
+        yaml_param = dict(FIB_ITERATIONS='3',
                           FIB_SERVER_SLEEP_SECONDS='0.01')
+        os.environ.update(yaml_param)
         yamls = ex_yamls['rpcFib']['all_nomatlab']
         service = None
         for x in yamls:
@@ -310,7 +316,8 @@ def test_calling_server_as_service(self, running_service):
                 service = x
                 break
         self.call_integration_service(running_service, yamls, service,
-                                      name='rpcFibSrv')
+                                      name='rpcFibSrv',
+                                      yaml_param=yaml_param)
 
     def test_calling_service_as_function(self, running_service):
         r"""Test calling an integrations as a service in an integration."""
diff --git a/yggdrasil/yamlfile.py b/yggdrasil/yamlfile.py
index 91c566633..189d0fe04 100644
--- a/yggdrasil/yamlfile.py
+++ b/yggdrasil/yamlfile.py
@@ -92,7 +92,7 @@ def clone_github_repo(fname, commit=None, local_directory=None):
     return os.path.realpath(fname)
 
 
-def load_yaml(fname):
+def load_yaml(fname, yaml_param=None):
     r"""Parse a yaml file defining a run.
 
     Args:
@@ -104,6 +104,9 @@ def load_yaml(fname):
             YAML file (the server is assumed to be github.com if not given)
             (foo/bar/yam/interesting.yaml will be interpreted as
             http://github.com/foo/bar/yam/interesting.yml).
+        yaml_param (dict, optional): Parameters that should be used in
+            mustache formatting of YAML files. Defaults to None and is
+            ignored.
 
     Returns:
         dict: Contents of yaml file.
@@ -130,9 +133,11 @@ def load_yaml(fname):
         else:
             fname = os.path.join(os.getcwd(), 'stream')
     # Mustache replace vars
+    if yaml_param is None:
+        yaml_param = {}
     yamlparsed = fd.read()
     yamlparsed = chevron.render(
-        sio.StringIO(yamlparsed).getvalue(), dict(os.environ))
+        sio.StringIO(yamlparsed).getvalue(), dict(os.environ, **yaml_param))
     if fname.endswith('.json'):
         yamlparsed = json.loads(yamlparsed)
     else:
@@ -145,7 +150,7 @@ def load_yaml(fname):
     return yamlparsed
 
 
-def prep_yaml(files):
+def prep_yaml(files, yaml_param=None):
     r"""Prepare yaml to be parsed by jsonschema including covering backwards
     compatible options.
 
@@ -153,6 +158,9 @@ def prep_yaml(files):
         files (str, list): Either the path to a single yaml file or a list of
             yaml files. Entries can also be opened file descriptors for files
             containing YAML documents or pre-loaded YAML documents.
+        yaml_param (dict, optional): Parameters that should be used in
+            mustache formatting of YAML files. Defaults to None and is
+            ignored.
 
     Returns:
         dict: YAML ready to be parsed using schema.
@@ -162,7 +170,7 @@ def prep_yaml(files):
     # Load each file
     if not isinstance(files, list):
         files = [files]
-    yamls = [load_yaml(f) for f in files]
+    yamls = [load_yaml(f, yaml_param=yaml_param) for f in files]
     # Load files pointed to
     for y in yamls:
         if 'include' in y:
@@ -184,7 +192,7 @@ def prep_yaml(files):
                 y['models'].append(y.pop('model'))
         for x in services:
             request = {'action': 'start'}
-            for k in ['name', 'yamls']:
+            for k in ['name', 'yamls', 'yaml_param']:
                 if k in x:
                     request[k] = x.pop(k)
             if 'type' in x:
@@ -219,7 +227,7 @@ def prep_yaml(files):
 
 
 def parse_yaml(files, complete_partial=False, partial_commtype=None,
-               model_only=False, model_submission=False):
+               model_only=False, model_submission=False, yaml_param=None):
     r"""Parse list of yaml files.
 
     Args:
@@ -237,6 +245,9 @@ def parse_yaml(files, complete_partial=False, partial_commtype=None,
         model_submission (bool, optional): If True, the YAML will be evaluated
             as a submission to the yggdrasil model repository and model_only
             will be set to True. Defaults to False.
+        yaml_param (dict, optional): Parameters that should be used in
+            mustache formatting of YAML files. Defaults to None and is
+            ignored.
 
     Raises:
         ValueError: If the yml dictionary is missing a required keyword or
@@ -250,7 +261,7 @@ def parse_yaml(files, complete_partial=False, partial_commtype=None,
     """
     s = get_schema()
     # Parse files using schema
-    yml_prep = prep_yaml(files)
+    yml_prep = prep_yaml(files, yaml_param=yaml_param)
     # print('prepped')
     # pprint.pprint(yml_prep)
     if model_submission:

From 4eedc41e7fee5e699c1d2961175873f2c6eab4e0 Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Thu, 23 Sep 2021 10:54:51 -0700
Subject: [PATCH 14/73] Ensure that ~ is expanded into the user home directory
 in service manager Allow directory for git clones in YAML to be specified via
 keyword/env variable and use that to place service clones in the same
 directory Fix bug relying on non-exitance of a YAML file now loaded by the
 service manager at start

---
 yggdrasil/services.py            | 13 ++++++++++---
 yggdrasil/tests/test_services.py |  4 ++--
 yggdrasil/yamlfile.py            | 33 +++++++++++++++++++++++---------
 3 files changed, 36 insertions(+), 14 deletions(-)

diff --git a/yggdrasil/services.py b/yggdrasil/services.py
index ffa4a374d..6d31b32bb 100644
--- a/yggdrasil/services.py
+++ b/yggdrasil/services.py
@@ -18,6 +18,7 @@
 
 
 _service_host_env = 'YGGDRASIL_SERVICE_HOST_URL'
+_service_repo_dir = 'YGGDRASIL_SERVICE_REPO_DIR'
 _default_service_type = ygg_cfg.get('services', 'default_type', 'flask')
 _default_commtype = ygg_cfg.get('services', 'default_comm', None)
 _default_address = ygg_cfg.get('services', 'address', None)
@@ -151,7 +152,8 @@ def start_server(self, remote_url=None, with_coverage=False,
         if remote_url is None:
             remote_url = self.address
         if model_repository is not None:
-            self.registry.add_from_repository(model_repository)
+            repo_dir = self.registry.add_from_repository(model_repository)
+            os.environ.setdefault(_service_repo_dir, repo_dir)
         os.environ.setdefault(_service_host_env, remote_url)
         if log_level is not None:
             self.set_log_level(log_level)
@@ -1095,10 +1097,14 @@ def add_from_repository(self, model_repository, directory=None):
                 model_repository should be cloned. Defaults to
                 '~/.yggdrasil_service'.
 
+        Returns:
+            str: The directory where the repositories were cloned.
+
         """
         from yggdrasil.yamlfile import clone_github_repo, prep_yaml
         if directory is None:
-            directory = os.path.join('~', '.yggdrasil_services')
+            directory = os.path.expanduser(
+                os.path.join('~', '.yggdrasil_services'))
         yaml_dir = clone_github_repo(model_repository,
                                      local_directory=directory)
         yaml_files = (glob.glob(os.path.join(yaml_dir, '*.yaml'))
@@ -1108,8 +1114,9 @@ def add_from_repository(self, model_repository, directory=None):
             # in advance to circumvent th hold place on git cloning on the
             # service manager (these models are assumed to be vetted so
             # they do not pose a security risk).
-            prep_yaml(x)
+            prep_yaml(x, directory_for_clones=directory)
             self.add(os.path.splitext(os.path.basename(x))[0], x)
+        return directory
 
     def add(self, name, yamls=None, **kwargs):
         r"""Add an integration service to the registry.
diff --git a/yggdrasil/tests/test_services.py b/yggdrasil/tests/test_services.py
index c4f24055f..910ece193 100644
--- a/yggdrasil/tests/test_services.py
+++ b/yggdrasil/tests/test_services.py
@@ -227,9 +227,9 @@ def test_git_fails(self, running_service):
         r"""Test that sending a request for a git YAML fails."""
         cli = running_service
         test_yml = ("git:https://github.com/cropsinsilico/example-fakemodel/"
-                    "fakemodel.yml")
+                    "fakemodel3.yml")
         assert(not os.path.isfile(
-            "cropsinsilico/example-fakemodel/fakemodel.yml"))
+            "cropsinsilico/example-fakemodel/fakemodel3.yml"))
         assert_raises(ServerError, cli.send_request, yamls=test_yml,
                       action='start')
 
diff --git a/yggdrasil/yamlfile.py b/yggdrasil/yamlfile.py
index 189d0fe04..c6b3b6048 100644
--- a/yggdrasil/yamlfile.py
+++ b/yggdrasil/yamlfile.py
@@ -54,9 +54,9 @@ def clone_github_repo(fname, commit=None, local_directory=None):
             repository.
 
     """
-    from yggdrasil.services import _service_host_env
+    from yggdrasil.services import _service_host_env, _service_repo_dir
     if local_directory is None:
-        local_directory = os.getcwd()
+        local_directory = os.environ.get(_service_repo_dir, os.getcwd())
     # make sure we start with a full url
     if 'http' not in fname:
         url = 'http://github.com/' + fname
@@ -92,7 +92,7 @@ def clone_github_repo(fname, commit=None, local_directory=None):
     return os.path.realpath(fname)
 
 
-def load_yaml(fname, yaml_param=None):
+def load_yaml(fname, yaml_param=None, directory_for_clones=None):
     r"""Parse a yaml file defining a run.
 
     Args:
@@ -107,6 +107,9 @@ def load_yaml(fname, yaml_param=None):
         yaml_param (dict, optional): Parameters that should be used in
             mustache formatting of YAML files. Defaults to None and is
             ignored.
+        directory_for_clones (str, optional): Directory that git repositories
+            should be cloned into. Defaults to None and the current working
+            directory will be used.
 
     Returns:
         dict: Contents of yaml file.
@@ -120,7 +123,8 @@ def load_yaml(fname, yaml_param=None):
     elif isinstance(fname, str):
         # pull foreign file
         if fname.startswith('git:'):
-            fname = clone_github_repo(fname[4:])
+            fname = clone_github_repo(fname[4:],
+                                      local_directory=directory_for_clones)
         fname = os.path.realpath(fname)
         if not os.path.isfile(fname):
             raise IOError("Unable locate yaml file %s" % fname)
@@ -150,7 +154,7 @@ def load_yaml(fname, yaml_param=None):
     return yamlparsed
 
 
-def prep_yaml(files, yaml_param=None):
+def prep_yaml(files, yaml_param=None, directory_for_clones=None):
     r"""Prepare yaml to be parsed by jsonschema including covering backwards
     compatible options.
 
@@ -161,6 +165,9 @@ def prep_yaml(files, yaml_param=None):
         yaml_param (dict, optional): Parameters that should be used in
             mustache formatting of YAML files. Defaults to None and is
             ignored.
+        directory_for_clones (str, optional): Directory that git repositories
+            should be cloned into. Defaults to None and the current working
+            directory will be used.
 
     Returns:
         dict: YAML ready to be parsed using schema.
@@ -170,7 +177,9 @@ def prep_yaml(files, yaml_param=None):
     # Load each file
     if not isinstance(files, list):
         files = [files]
-    yamls = [load_yaml(f, yaml_param=yaml_param) for f in files]
+    yamls = [load_yaml(f, yaml_param=yaml_param,
+                       directory_for_clones=directory_for_clones)
+             for f in files]
     # Load files pointed to
     for y in yamls:
         if 'include' in y:
@@ -213,7 +222,8 @@ def prep_yaml(files, yaml_param=None):
                     if (k == 'models') and ('repository_url' in x):
                         repo_dir = clone_github_repo(
                             x['repository_url'],
-                            commit=x.get('repository_commit', None))
+                            commit=x.get('repository_commit', None),
+                            local_directory=directory_for_clones)
                         x.setdefault('working_dir', repo_dir)
                     else:
                         x.setdefault('working_dir', yml['working_dir'])
@@ -227,7 +237,8 @@ def prep_yaml(files, yaml_param=None):
 
 
 def parse_yaml(files, complete_partial=False, partial_commtype=None,
-               model_only=False, model_submission=False, yaml_param=None):
+               model_only=False, model_submission=False, yaml_param=None,
+               directory_for_clones=None):
     r"""Parse list of yaml files.
 
     Args:
@@ -248,6 +259,9 @@ def parse_yaml(files, complete_partial=False, partial_commtype=None,
         yaml_param (dict, optional): Parameters that should be used in
             mustache formatting of YAML files. Defaults to None and is
             ignored.
+        directory_for_clones (str, optional): Directory that git repositories
+            should be cloned into. Defaults to None and the current working
+            directory will be used.
 
     Raises:
         ValueError: If the yml dictionary is missing a required keyword or
@@ -261,7 +275,8 @@ def parse_yaml(files, complete_partial=False, partial_commtype=None,
     """
     s = get_schema()
     # Parse files using schema
-    yml_prep = prep_yaml(files, yaml_param=yaml_param)
+    yml_prep = prep_yaml(files, yaml_param=yaml_param,
+                         directory_for_clones=directory_for_clones)
     # print('prepped')
     # pprint.pprint(yml_prep)
     if model_submission:

From dd77d0c4c32f448694fe9baa242d086e7d71b616 Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Thu, 23 Sep 2021 13:26:20 -0700
Subject: [PATCH 15/73] Fix path to models in model repo in service test

---
 yggdrasil/tests/test_services.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/yggdrasil/tests/test_services.py b/yggdrasil/tests/test_services.py
index 910ece193..c88beac07 100644
--- a/yggdrasil/tests/test_services.py
+++ b/yggdrasil/tests/test_services.py
@@ -369,7 +369,11 @@ def test_validate_model_repo():
     for x in [url, url + "_test"]:
         try:
             repo = git.Repo.clone_from(x, dest)
-            validate_model_submission(dest)
+            model_dir = os.path.join(dest, "models")
+            if os.path.isdir(model_dir):
+                # This condition can be removed once there are models in the
+                # non-dev repository
+                validate_model_submission(model_dir)
             repo.close()
         finally:
             if os.path.isdir(dest):

From ff46a84b0027a94ae1bf49ff593f269d89f7c3e0 Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Fri, 24 Sep 2021 12:24:25 -0700
Subject: [PATCH 16/73] Added validation_command option to model schema that
 can be used to validate a run on completion via the 'validate' runner option
 Added dependencies and additional_dependencies options to model schema that
 can be used to specify packages that should be installed for a model

---
 HISTORY.rst                                   |   4 +-
 yggdrasil/.ygg_schema.yml                     |  57 ++++++-
 yggdrasil/command_line.py                     |   9 +-
 yggdrasil/drivers/ModelDriver.py              | 152 +++++++++++++++++-
 yggdrasil/drivers/PythonModelDriver.py        |  31 ++++
 yggdrasil/drivers/RModelDriver.py             |  28 ++++
 yggdrasil/drivers/tests/test_ModelDriver.py   |  10 +-
 .../drivers/tests/test_PythonModelDriver.py   |   9 +-
 yggdrasil/drivers/tests/test_RModelDriver.py  |   7 +
 yggdrasil/runner.py                           |   9 +-
 yggdrasil/services.py                         |   4 +-
 yggdrasil/tests/yamls/FakePlant.yaml          |   9 +-
 12 files changed, 313 insertions(+), 16 deletions(-)

diff --git a/HISTORY.rst b/HISTORY.rst
index c3cfacd54..3f6a1759e 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -2,11 +2,13 @@
 History
 =======
 
-1.8.1 (2021-09-XX) Minor updates to support model submission form development
+1.8.1 (2021-09-25) Minor updates to support model submission form development
 ------------------
 
 * Added --model_repository option to the integration-service-manager CLI
 * Preload models from the model repository into the service manager registry
+* Added validation_command option to model schema that can be used to validate a run on completion via the 'validate' runner option
+* Added dependencies and additional_dependencies options to model schema that can be used to specify packages that should be installed for a model
 
 1.8.0 (2021-09-15) Support for REST API based communicators, running integrations as services, and connecting to remote integration services
 ------------------
diff --git a/yggdrasil/.ygg_schema.yml b/yggdrasil/.ygg_schema.yml
index c243729be..f384508f9 100644
--- a/yggdrasil/.ygg_schema.yml
+++ b/yggdrasil/.ygg_schema.yml
@@ -180,9 +180,9 @@ definitions:
             type: string
           type: array
       required:
-      - name
       - datatype
       - commtype
+      - name
       title: comm_base
       type: object
     - anyOf:
@@ -704,9 +704,9 @@ definitions:
             is used.
           type: string
       required:
+      - working_dir
       - name
       - filetype
-      - working_dir
       title: file_base
       type: object
     - anyOf:
@@ -1253,6 +1253,26 @@ definitions:
         - args
       description: Base schema for all subtypes of model components.
       properties:
+        additional_dependencies:
+          additionalProperties:
+            items:
+              oneOf:
+              - type: string
+              - additionalProperties: false
+                properties:
+                  arguments:
+                    type: string
+                  package:
+                    type: string
+                  package_manager:
+                    type: string
+                required:
+                - package
+                type: object
+            type: array
+          description: A mapping between languages and lists of packages in those
+            languages that are required by the model.
+          type: object
         additional_variables:
           additionalProperties:
             items:
@@ -1337,6 +1357,28 @@ definitions:
             use any of the files located there since OSR always assumes the included
             file paths are relative. Defaults to False.
           type: boolean
+        dependencies:
+          description: A list of packages required by the model that are written in
+            the same language as the model. If the package requires dependencies outside
+            the language of the model. use the additional_dependencies parameter to
+            provide them. If you need a version of the package from a specific package
+            manager, a mapping with 'package' and 'package_manager' fields can be
+            provided instead of just the name of the package.
+          items:
+            oneOf:
+            - type: string
+            - additionalProperties: false
+              properties:
+                arguments:
+                  type: string
+                package:
+                  type: string
+                package_manager:
+                  type: string
+              required:
+              - package
+              type: object
+          type: array
         description:
           description: Description of the model. This parameter is only used in the
             model repository or when providing the model as a service.
@@ -1793,6 +1835,11 @@ definitions:
           items:
             type: string
           type: array
+        validation_command:
+          description: Path to a validation command that can be used to verify that
+            the model ran as expected. A non-zero return code is taken to indicate
+            failure.
+          type: string
         with_strace:
           default: false
           description: If True, the command is run with strace (on Linux) or dtrace
@@ -1807,10 +1854,10 @@ definitions:
             is used.
           type: string
       required:
-      - name
-      - args
-      - working_dir
       - language
+      - working_dir
+      - args
+      - name
       title: model_base
       type: object
     - anyOf:
diff --git a/yggdrasil/command_line.py b/yggdrasil/command_line.py
index 2730f9358..37a62f809 100755
--- a/yggdrasil/command_line.py
+++ b/yggdrasil/command_line.py
@@ -225,7 +225,11 @@ class yggrun(SubCommand):
           'help': 'Number of MPI processes to run on.'}),
         (('--mpi-tag-start', ),
          {'type': int, 'default': 0,
-          'help': 'Tag that MPI communications should start at.'})]
+          'help': 'Tag that MPI communications should start at.'}),
+        (('--validate', ),
+         {'action': 'store_true',
+          'help': ('Validate the run via model validation commands on '
+                   'completion.')})]
 
     @classmethod
     def add_arguments(cls, parser, **kwargs):
@@ -252,7 +256,8 @@ def func(cls, args):
         with config.parser_config(args):
             runner.run(args.yamlfile, ygg_debug_prefix=prog,
                        production_run=args.production_run,
-                       mpi_tag_start=args.mpi_tag_start)
+                       mpi_tag_start=args.mpi_tag_start,
+                       validate=args.validate)
 
 
 class integration_service_manager(SubCommand):
diff --git a/yggdrasil/drivers/ModelDriver.py b/yggdrasil/drivers/ModelDriver.py
index 545aa5e26..f59112372 100755
--- a/yggdrasil/drivers/ModelDriver.py
+++ b/yggdrasil/drivers/ModelDriver.py
@@ -182,6 +182,19 @@ class ModelDriver(Driver):
         contact_email (str, optional): Email address that should be used to
             contact the maintainer of the model. This parameter is only used
             in the model repository.
+        validation_command (str, optional): Path to a validation command that
+            can be used to verify that the model ran as expected. A non-zero
+            return code is taken to indicate failure.
+        dependencies (list, optional): A list of packages required by the
+            model that are written in the same language as the model. If the
+            package requires dependencies outside the language of the model.
+            use the additional_dependencies parameter to provide them. If you
+            need a version of the package from a specific package manager,
+            a mapping with 'package' and 'package_manager' fields can be
+            provided instead of just the name of the package.
+        additional_dependencies (dict, optional): A mapping between languages
+            and lists of packages in those languages that are required by the
+            model.
         **kwargs: Additional keyword arguments are passed to parent class.
 
     Class Attributes:
@@ -298,6 +311,19 @@ class ModelDriver(Driver):
         contact_email (str): Email address that should be used to contact the
             maintainer of the model. This parameter is only used in the model
             repository.
+        validation_command (str): Path to a validation command that can be
+            used to verify that the model ran as expected. A non-zero return
+            code is taken to indicate failure.
+        dependencies (list): A list of packages required by the model that are
+            written in the same language as the model. If the package requires
+            dependencies outside the language of the model, use the
+            additional_dependencies parameter to provide them. If you need a
+            version of the package from a specific package manager, a mapping
+            
+            with 'package' and 'package_manager' fields can be provided
+            instead of just the name of the package.
+        additional_dependencies (dict): A mapping between languages and lists
+            of packages in those languages that are required by the model.
 
     Raises:
         RuntimeError: If both with_strace and with_valgrind are True.
@@ -400,7 +426,32 @@ class ModelDriver(Driver):
         'repository_url': {'type': 'string'},
         'repository_commit': {'type': 'string'},
         'description': {'type': 'string'},
-        'contact_email': {'type': 'string'}}
+        'contact_email': {'type': 'string'},
+        'validation_command': {'type': 'string'},
+        'dependencies': {
+            'type': 'array',
+            'items': {'oneOf': [
+                {'type': 'string'},
+                {'type': 'object',
+                 'required': ['package'],
+                 'properties': {
+                     'package': {'type': 'string'},
+                     'package_manager': {'type': 'string'},
+                     'arguments': {'type': 'string'}},
+                 'additionalProperties': False}]}},
+        'additional_dependencies': {
+            'type': 'object',
+            'additionalProperties': {
+                'type': 'array',
+                'items': {'oneOf': [
+                    {'type': 'string'},
+                    {'type': 'object',
+                     'required': ['package'],
+                     'properties': {
+                         'package': {'type': 'string'},
+                         'package_manager': {'type': 'string'},
+                         'arguments': {'type': 'string'}},
+                     'additionalProperties': False}]}}}}
     _schema_excluded_from_class = ['name', 'language', 'args', 'working_dir']
     _schema_excluded_from_class_validation = ['inputs', 'outputs']
     
@@ -525,6 +576,13 @@ def __init__(self, name, args, model_index=0, copy_index=-1, clients=[],
         if self.function:
             self.wrapper_products.append(args[0])
         self.wrapper_products += self.write_wrappers()
+        # Install dependencies
+        if self.dependencies:
+            self.install_model_dependencies(self.dependencies)
+        if self.additional_dependencies:
+            for language, v in self.additional_dependencies.items():
+                drv = import_component('model', language)
+                drv.install_model_dependencies(v)
 
     @staticmethod
     def before_registration(cls):
@@ -776,6 +834,92 @@ def write_wrappers(self, **kwargs):
 
         """
         return []
+
+    @classmethod
+    def install_model_dependencies(cls, dependencies, always_yes=False):
+        r"""Install any dependencies required by the model.
+
+        Args:
+            dependencies (list): Dependencies that should be installed.
+            always_yes (bool, optional): If True, the package manager will
+                not ask users for input during installation. Defaults to
+                False.
+
+        """
+        packages = {}
+        for x in dependencies:
+            if isinstance(x, str):
+                x = {'package': x}
+            if x.get('arguments', None):
+                cls.install_dependency(always_yes=always_yes, **x)
+            else:
+                packages.setdefault(x.get('package_manager', None), [])
+                packages[x.get('package_manager', None)].append(
+                    x['package'])
+        for k, v in packages.items():
+            cls.install_dependency(v, package_manager=k,
+                                   always_yes=always_yes)
+
+    @classmethod
+    def install_dependency(cls, package, package_manager=None,
+                           arguments=None, command=None, always_yes=False):
+        r"""Install a dependency.
+
+        Args:
+            package (str): Name of the package that should be installed. If
+                the package manager supports it, this can include version
+                requirements.
+            package_manager (str, optional): Package manager that should be
+                used to install the package.
+            arguments (str, optional): Additional arguments that should be
+                passed to the package manager.
+            command (list, optional): Command that should be used to
+                install the package.
+            always_yes (bool, optional): If True, the package manager will
+                not ask users for input during installation. Defaults to
+                False.
+
+        """
+        if isinstance(package, str):
+            package = package.split()
+        if package_manager is None:
+            if tools.get_conda_prefix():
+                package_manager = 'conda'
+            elif platform._is_mac:
+                package_manager = 'brew'
+            elif platform._is_linux:
+                package_manager = 'apt'
+            elif platform._is_win:
+                package_manager = 'choco'
+        yes_cmd = []
+        if command:
+            cmd = copy.copy(command)
+        elif package_manager == 'conda':
+            cmd = ['conda', 'install'] + package
+            yes_cmd = ['-y']
+        elif package_manager == 'brew':
+            cmd = ['brew', 'install'] + package
+        elif package_manager == 'apt':
+            cmd = ['apt-get', 'install'] + package
+            yes_cmd = ['-y']
+        elif package_manager == 'choco':
+            cmd = ['choco', 'install'] + package
+        elif package_manager == 'vcpkg':
+            cmd = ['vcpkg.exe', 'install'] + package
+        else:
+            package_managers = {'pip': 'python',
+                                'cran': 'r'}
+            if package_manager in package_managers:
+                drv = import_component(
+                    'model', package_managers[package_manager])
+                return drv.install_dependency(package_manager, package)
+            raise NotImplementedError(f"Unsupported package manager: "
+                                      f"{package_manager}")
+        if arguments:
+            cmd += arguments.split()
+        if always_yes:
+            cmd += yes_cmd
+        subprocess.check_call(cmd)
         
     def model_command(self):
         r"""Return the command that should be used to run the model.
@@ -883,6 +1027,12 @@ def run_executable(cls, args, return_process=False, debug_flags=None,
         except (subprocess.CalledProcessError, OSError) as e:  # pragma: debug
             raise RuntimeError("Could not call command '%s': %s"
                                % (' '.join(cmd), e))
+
+    def run_validation(self):
+        r"""Run the validation script for the model."""
+        if not self.validation_command:
+            return
+        subprocess.check_call(self.validation_command.split())
         
     def run_model(self, return_process=True, **kwargs):
         r"""Run the model. Unless overridden, the model will be run using
diff --git a/yggdrasil/drivers/PythonModelDriver.py b/yggdrasil/drivers/PythonModelDriver.py
index 608b1f77b..cdf310fa7 100755
--- a/yggdrasil/drivers/PythonModelDriver.py
+++ b/yggdrasil/drivers/PythonModelDriver.py
@@ -225,3 +225,34 @@ def write_finalize_oiter(cls, var):
                  'units.get_units({name}[0]))').format(
                      name=var['name'])]
         return out
+
+    @classmethod
+    def install_dependency(cls, package, package_manager=None, **kwargs):
+        r"""Install a dependency.
+
+        Args:
+            package (str): Name of the package that should be installed. If
+                the package manager supports it, this can include version
+                requirements.
+            package_manager (str, optional): Package manager that should be
+                used to install the package.
+            **kwargs: Additional keyword arguments are passed to the parent
+                class.
+
+        """
+        if package_manager in [None, 'pip']:
+            if isinstance(package, str):
+                package = package.split()
+            kwargs.setdefault(
+                'command',
+                [cls.get_interpreter(), '-m', 'pip', 'install'] + package)
+        return super(PythonModelDriver, cls).install_dependency(
+            package, package_manager=package_manager, **kwargs)
+        
+    def run_validation(self):
+        r"""Run the validation script for the model."""
+        if ((self.validation_script
+             and (self.validation_script.split()[0].endswith('.py')))):
+            self.validation_script = (
+                f"{self.get_interpreter()} {self.validation_script}")
+        return super(PythonModelDriver, self).run_validation()
diff --git a/yggdrasil/drivers/RModelDriver.py b/yggdrasil/drivers/RModelDriver.py
index d873fd679..54e8850ac 100644
--- a/yggdrasil/drivers/RModelDriver.py
+++ b/yggdrasil/drivers/RModelDriver.py
@@ -478,3 +478,31 @@ def write_finalize_oiter(cls, var):
             '}',
         ]
         return out
+
+    @classmethod
+    def install_dependency(cls, package, package_manager=None, **kwargs):
+        r"""Install a dependency.
+
+        Args:
+            package (str): Name of the package that should be installed. If
+                the package manager supports it, this can include version
+                requirements.
+            package_manager (str, optional): Package manager that should be
+                used to install the package.
+            **kwargs: Additional keyword arguments are passed to the parent
+                class.
+
+        """
+        if package_manager in [None, 'cran', 'CRAN']:
+            if isinstance(package, list):
+                package = (
+                    'c(' + ', '.join([f"\"{x}\"" for x in package]) + ')')
+            else:
+                package = f'\"{package}\"'
+            kwargs.setdefault(
+                'command',
+                [cls.get_interpreter(), '-e',
+                 (f'\'install.packages({package}, dep=TRUE, '
+                  f'repos=\"http://cloud.r-project.org\")\'')])
+        return super(RModelDriver, cls).install_dependency(
+            package, package_manager=package_manager, **kwargs)
diff --git a/yggdrasil/drivers/tests/test_ModelDriver.py b/yggdrasil/drivers/tests/test_ModelDriver.py
index 7a8437398..308ed8d6d 100644
--- a/yggdrasil/drivers/tests/test_ModelDriver.py
+++ b/yggdrasil/drivers/tests/test_ModelDriver.py
@@ -470,7 +470,15 @@ def test_split_line(self, vals=None):
         for line, kwargs, splits in vals:
             self.assert_equal(
                 self.import_cls.split_line(line, **kwargs), splits)
-    
+
+    def test_install_model_dependencies(self, deps=None):
+        r"""Test install_model_dependencies."""
+        if (deps is None) and (self.import_cls.language == 'c'):
+            deps = ['doxygen', 'cmake']
+        else:
+            deps = []
+        self.import_cls.install_model_dependencies(deps, always_yes=True)
+
 
 class TestModelDriverNoStart(TestModelParam, parent.TestDriverNoStart):
     r"""Test runner for basic ModelDriver class."""
diff --git a/yggdrasil/drivers/tests/test_PythonModelDriver.py b/yggdrasil/drivers/tests/test_PythonModelDriver.py
index 13b3ba054..cb567974f 100644
--- a/yggdrasil/drivers/tests/test_PythonModelDriver.py
+++ b/yggdrasil/drivers/tests/test_PythonModelDriver.py
@@ -10,7 +10,14 @@ class TestPythonModelParam(parent.TestInterpretedModelParam):
 class TestPythonModelDriverNoInit(TestPythonModelParam,
                                   parent.TestInterpretedModelDriverNoInit):
     r"""Test runner for PythonModelDriver without init."""
-    pass
+
+    def test_install_model_dependencies(self, deps=None):
+        r"""Test install_model_dependencies."""
+        if deps is None:
+            deps = [{'package': 'numpy', 'arguments': '-v'},
+                    'requests', 'pyyaml']
+        super(TestPythonModelDriverNoInit, self).test_install_model_dependencies(
+            deps=deps)
         
 
 class TestPythonModelDriverNoStart(TestPythonModelParam,
diff --git a/yggdrasil/drivers/tests/test_RModelDriver.py b/yggdrasil/drivers/tests/test_RModelDriver.py
index dca7e5b81..415201057 100644
--- a/yggdrasil/drivers/tests/test_RModelDriver.py
+++ b/yggdrasil/drivers/tests/test_RModelDriver.py
@@ -42,6 +42,13 @@ def test_python2language(self):
         for a, b in test_vars:
             self.assert_equal(self.import_cls.python2language(a), b)
 
+    def test_install_model_dependencies(self, deps=None):
+        r"""Test install_model_dependencies."""
+        if deps is None:
+            deps = ['units', 'zeallot']
+        super(TestRModelDriverNoInit, self).test_install_model_dependencies(
+            deps=deps)
+
 
 class TestRModelDriverNoStart(TestRModelParam,
                               parent.TestInterpretedModelDriverNoStart):
diff --git a/yggdrasil/runner.py b/yggdrasil/runner.py
index 022cc2ec7..b107a6ca8 100755
--- a/yggdrasil/runner.py
+++ b/yggdrasil/runner.py
@@ -260,6 +260,9 @@ class YggRunner(YggClass):
         yaml_param (dict, optional): Parameters that should be used in
             mustache formatting of YAML files. Defaults to None and is
             ignored.
+        validate (bool, optional): If True, the validation scripts for each
+            modle (if present), will be run after the integration finishes
+            running. Defaults to False.
 
     Attributes:
         namespace (str): Name that should be used to uniquely identify any
@@ -279,7 +282,7 @@ def __init__(self, modelYmls, namespace=None, host=None, rank=0,
                  ygg_debug_prefix=None, connection_task_method='thread',
                  as_service=False, complete_partial=False,
                  partial_commtype=None, production_run=False,
-                 mpi_tag_start=None, yaml_param=None):
+                 mpi_tag_start=None, yaml_param=None, validate=False):
         self.mpi_comm = None
         name = 'runner'
         if MPI is not None:
@@ -309,6 +312,7 @@ def __init__(self, modelYmls, namespace=None, host=None, rank=0,
         self.error_flag = False
         self.complete_partial = complete_partial
         self.partial_commtype = partial_commtype
+        self.validate = validate
         self.debug("Running in %s with path %s namespace %s rank %d",
                    os.getcwd(), sys.path, namespace, rank)
         # Update environment based on config
@@ -439,6 +443,9 @@ def run(self, signal_handler=None, timer=None, t0=None):
                 tprev = t
             self.info(40 * '=')
             self.info('%20s\t%f', "Total", tprev - t0)
+        if self.validate:
+            for v in self.modeldrivers.values():
+                v.run_validation()
         return times
 
     @property
diff --git a/yggdrasil/services.py b/yggdrasil/services.py
index 6d31b32bb..3c6cf2982 100644
--- a/yggdrasil/services.py
+++ b/yggdrasil/services.py
@@ -1185,5 +1185,5 @@ def validate_model_submission(fname):
             break
     else:
         raise RuntimeError("Model repository does not contain a LICENSE file.")
-    # 4. Run
-    runner.run(fname)
+    # 4. Run & validate
+    runner.run(fname, validate=True)
diff --git a/yggdrasil/tests/yamls/FakePlant.yaml b/yggdrasil/tests/yamls/FakePlant.yaml
index 0c8dc4a12..d408b2af4 100644
--- a/yggdrasil/tests/yamls/FakePlant.yaml
+++ b/yggdrasil/tests/yamls/FakePlant.yaml
@@ -19,5 +19,10 @@
       name: ./Output/output.txt
       filetype: table
   repository_url: https://github.com/cropsinsilico/example-fakemodel
-  repository_commit: e4bc7932c3c0c68fb3852cfb864777ca64cba448
-  description: Example model submission
\ No newline at end of file
+  repository_commit: bb39d0db7e4883dbef2c1cb90bc563fc9d757a19
+  description: Example model submission
+  validation_command: validate.py
+  dependencies:
+  - numpy
+  additional_dependencies:
+    c++: [doxygen]

From f010191901303775e5cadc906c2535d22e835ef4 Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Fri, 24 Sep 2021 22:25:30 -0700
Subject: [PATCH 17/73] Fix bugs in validation and dependencies Refactor status
 message for fork comm Allow log level to be passed via CLI to service manager

---
 yggdrasil/command_line.py                   | 10 +++++--
 yggdrasil/communication/CommBase.py         | 21 ++++++++-------
 yggdrasil/communication/ForkComm.py         | 29 ++++++++++++++-------
 yggdrasil/drivers/ModelDriver.py            | 18 +++++++++++--
 yggdrasil/drivers/PythonModelDriver.py      |  8 +++---
 yggdrasil/drivers/tests/test_ModelDriver.py |  5 +++-
 yggdrasil/runner.py                         | 12 +++------
 yggdrasil/tests/test_services.py            |  3 ++-
 8 files changed, 68 insertions(+), 38 deletions(-)

diff --git a/yggdrasil/command_line.py b/yggdrasil/command_line.py
index 37a62f809..785dc1ff9 100755
--- a/yggdrasil/command_line.py
+++ b/yggdrasil/command_line.py
@@ -338,7 +338,12 @@ class integration_service_manager(SubCommand):
                          {'type': str,
                           'help': ('URL for a directory in a Git repository '
                                    'containing models that should be loaded '
-                                   'into the service manager registry.')})]),
+                                   'into the service manager registry.')}),
+                        (('--log-level', ),
+                         {'type': int,
+                          'help': ('Level of logging that should be '
+                                   'performed for the service manager '
+                                   'application.')})]),
                 ArgumentParser(
                     name='stop',
                     help=('Stop an integration service manager or '
@@ -417,7 +422,8 @@ def func(cls, args):
                         remote_url=getattr(args, 'remote_url', None),
                         with_coverage=getattr(args, 'with_coverage', False),
                         model_repository=getattr(args, 'model_repository',
-                                                 None))
+                                                 None),
+                        log_level=getattr(args, 'log_level', None))
             else:
                 x.send_request(integration_name,
                                yamls=integration_yamls,
diff --git a/yggdrasil/communication/CommBase.py b/yggdrasil/communication/CommBase.py
index 12771c35e..97c958868 100755
--- a/yggdrasil/communication/CommBase.py
+++ b/yggdrasil/communication/CommBase.py
@@ -848,7 +848,11 @@ def get_testing_options(cls, serializer=None, **kwargs):
         else:
             seri_cls = import_component('serializer', serializer)
         out_seri = seri_cls.get_testing_options(**kwargs)
-        out = {'kwargs': out_seri['kwargs'],
+        out = {'attributes': ['name', 'address', 'direction',
+                              'serializer', 'recv_timeout',
+                              'close_on_eof_recv', 'opp_address',
+                              'opp_comms', 'maxMsgSize'],
+               'kwargs': out_seri['kwargs'],
                'send': copy.deepcopy(out_seri['objects']),
                'msg': out_seri['objects'][0],
                'contents': out_seri['contents'],
@@ -876,11 +880,11 @@ def get_status_message(self, nindent=0, extra_lines_before=None,
         Args:
             nindent (int, optional): Number of tabs that should be used to
                 indent each line. Defaults to 0.
-            extra_lines_before (list, optional): Additional lines that should be
-                added to the beginning of the default print message. Defaults to
-                empty list if not provided.
-            extra_lines_after (list, optional): Additional lines that should be
-                added to the end of the default print message. Defaults to
+            extra_lines_before (list, optional): Additional lines that should
+                be added to the beginning of the default print message.
+                Defaults to empty list if not provided.
+            extra_lines_after (list, optional): Additional lines that should
+                be added to the end of the default print message. Defaults to
                 empty list if not provided.
                 
         Returns:
@@ -1532,10 +1536,9 @@ def is_empty(self, msg, emsg):
             bool: True if the object is empty, False otherwise.
 
         """
-        from yggdrasil.tests import assert_equal
         try:
-            assert_equal(msg, emsg, dont_print_diff=True)
-        except AssertionError:
+            assert(msg == emsg)
+        except BaseException:
             return False
         return True
 
diff --git a/yggdrasil/communication/ForkComm.py b/yggdrasil/communication/ForkComm.py
index a8adde20d..1ae6ee9ea 100644
--- a/yggdrasil/communication/ForkComm.py
+++ b/yggdrasil/communication/ForkComm.py
@@ -157,17 +157,26 @@ def disconnect(self):
             x.disconnect()
         super(ForkComm, self).disconnect()
         
-    def printStatus(self, nindent=0, return_str=False, **kwargs):
-        r"""Print status of the communicator."""
-        out = super(ForkComm, self).printStatus(nindent=nindent,
-                                                return_str=return_str,
-                                                **kwargs)
-        for x in self.comm_list:
-            x_out = x.printStatus(nindent=nindent + 1, return_str=return_str)
-            if return_str:
-                out += '\n' + x_out
-        return out
+    def get_status_message(self, **kwargs):
+        r"""Return lines composing a status message.
+        
+        Args:
+            **kwargs: Keyword arguments are passed on to the parent class's
+                method.
+                
+        Returns:
+            tuple(list, prefix): Lines composing the status message and the
+                prefix string used for the last message.
 
+        """
+        nindent = kwargs.get('nindent', 0)
+        extra_lines_after = ['%-15s: %s' % ('pattern', self.pattern)]
+        for x in self.comm_list:
+            extra_lines_after += x.get_status_message(nindent=nindent + 1)[0]
+        extra_lines_after += kwargs.get('extra_lines_after', [])
+        kwargs['extra_lines_after'] = extra_lines_after
+        return super(ForkComm, self).get_status_message(**kwargs)
+        
     def __len__(self):
         return len(self.comm_list)
 
diff --git a/yggdrasil/drivers/ModelDriver.py b/yggdrasil/drivers/ModelDriver.py
index f59112372..7c4248768 100755
--- a/yggdrasil/drivers/ModelDriver.py
+++ b/yggdrasil/drivers/ModelDriver.py
@@ -892,15 +892,26 @@ def install_dependency(cls, package, package_manager=None,
             elif platform._is_win:
                 package_manager = 'choco'
         yes_cmd = []
+        cmd_kwargs = {}
         if command:
             cmd = copy.copy(command)
         elif package_manager == 'conda':
             cmd = ['conda', 'install'] + package
+            if platform._is_win:  # pragma: windows
+                # Conda commands must be run on the shell on windows as it
+                # is implemented as a batch script
+                cmd.insert(0, 'call')
+                cmd_kwargs['shell'] = True
             yes_cmd = ['-y']
         elif package_manager == 'brew':
             cmd = ['brew', 'install'] + package
         elif package_manager == 'apt':
             cmd = ['apt-get', 'install'] + package
+            if bool(os.environ.get('GITHUB_ACTIONS', False)):
+                # Only enable sudo for testing, otherwise allow the user to
+                # decide if they want to run yggdrasil with sudo, or just
+                # install the dependencies themselves
+                cmd.insert(0, 'sudo')
             yes_cmd = ['-y']
         elif package_manager == 'choco':
             cmd = ['choco', 'install'] + package
@@ -919,7 +930,9 @@ def install_dependency(cls, package, package_manager=None,
             cmd += arguments.split()
         if always_yes:
             cmd += yes_cmd
-        subprocess.check_call(cmd)
+        if cmd_kwargs.get('shell', False):
+            cmd = ' '.join(cmd)
+        subprocess.check_call(cmd, **cmd_kwargs)
         
     def model_command(self):
         r"""Return the command that should be used to run the model.
@@ -1032,7 +1045,8 @@ def run_validation(self):
         r"""Run the validation script for the model."""
         if not self.validation_command:
             return
-        subprocess.check_call(self.validation_command.split())
+        subprocess.check_call(self.validation_command.split(),
+                              cwd=self.working_dir)
         
     def run_model(self, return_process=True, **kwargs):
         r"""Run the model. Unless overridden, the model will be run using
diff --git a/yggdrasil/drivers/PythonModelDriver.py b/yggdrasil/drivers/PythonModelDriver.py
index cdf310fa7..d1a3487fd 100755
--- a/yggdrasil/drivers/PythonModelDriver.py
+++ b/yggdrasil/drivers/PythonModelDriver.py
@@ -251,8 +251,8 @@ def install_dependency(cls, package, package_manager=None, **kwargs):
         
     def run_validation(self):
         r"""Run the validation script for the model."""
-        if ((self.validation_script
-             and (self.validation_script.split()[0].endswith('.py')))):
-            self.validation_script = (
-                f"{self.get_interpreter()} {self.validation_script}")
+        if ((self.validation_command
+             and (self.validation_command.split()[0].endswith('.py')))):
+            self.validation_command = (
+                f"{self.get_interpreter()} {self.validation_command}")
         return super(PythonModelDriver, self).run_validation()
diff --git a/yggdrasil/drivers/tests/test_ModelDriver.py b/yggdrasil/drivers/tests/test_ModelDriver.py
index 308ed8d6d..a7d5272d3 100644
--- a/yggdrasil/drivers/tests/test_ModelDriver.py
+++ b/yggdrasil/drivers/tests/test_ModelDriver.py
@@ -474,7 +474,10 @@ def test_split_line(self, vals=None):
     def test_install_model_dependencies(self, deps=None):
         r"""Test install_model_dependencies."""
         if (deps is None) and (self.import_cls.language == 'c'):
-            deps = ['doxygen', 'cmake']
+            if platform._is_win:  # pragma: windows
+                deps = ['cmake']
+            else:
+                deps = ['doxygen', 'cmake']
         else:
             deps = []
         self.import_cls.install_model_dependencies(deps, always_yes=True)
diff --git a/yggdrasil/runner.py b/yggdrasil/runner.py
index b107a6ca8..fc51eff2f 100755
--- a/yggdrasil/runner.py
+++ b/yggdrasil/runner.py
@@ -4,7 +4,6 @@
 import time
 import copy
 import signal
-import traceback
 import atexit
 from pprint import pformat
 from itertools import chain
@@ -445,7 +444,7 @@ def run(self, signal_handler=None, timer=None, t0=None):
             self.info('%20s\t%f', "Total", tprev - t0)
         if self.validate:
             for v in self.modeldrivers.values():
-                v.run_validation()
+                v['instance'].run_validation()
         return times
 
     @property
@@ -1039,10 +1038,5 @@ def get_runner(models, **kwargs):
 
 def run(*args, **kwargs):
     yggRunner = get_runner(*args, **kwargs)
-    try:
-        yggRunner.run()
-        yggRunner.debug("runner returns, exiting")
-    except Exception as ex:  # pragma: debug
-        yggRunner.pprint("yggrun exception: %s" % type(ex))
-        print(traceback.format_exc())
-    print('')
+    yggRunner.run()
+    yggRunner.debug("runner returns, exiting")
diff --git a/yggdrasil/tests/test_services.py b/yggdrasil/tests/test_services.py
index c88beac07..905c52e36 100644
--- a/yggdrasil/tests/test_services.py
+++ b/yggdrasil/tests/test_services.py
@@ -78,7 +78,8 @@ def running_service(service_type, partial_commtype=None, with_coverage=False):
             f"--service-type={service_type}"]
     if partial_commtype is not None:
         args.append(f"--commtype={partial_commtype}")
-    args += ["start", f"--model-repository={model_repo}"]
+    args += ["start", f"--model-repository={model_repo}",
+             f"--log-level={log_level}"]
     package_dir = None
     process_kws = {}
     if with_coverage:

From bf3a7db867ea7e9dcef7d2933cd1490f1671f231 Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Mon, 27 Sep 2021 12:04:26 -0700
Subject: [PATCH 18/73] Revert to using the assert_equal function until testing
 is migrated.

---
 HISTORY.rst                         | 2 +-
 yggdrasil/communication/CommBase.py | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/HISTORY.rst b/HISTORY.rst
index 3f6a1759e..6ce8f65d4 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -2,7 +2,7 @@
 History
 =======
 
-1.8.1 (2021-09-25) Minor updates to support model submission form development
+1.8.1 (2021-09-27) Minor updates to support model submission form development
 ------------------
 
 * Added --model_repository option to the integration-service-manager CLI
diff --git a/yggdrasil/communication/CommBase.py b/yggdrasil/communication/CommBase.py
index 97c958868..e8269ef04 100755
--- a/yggdrasil/communication/CommBase.py
+++ b/yggdrasil/communication/CommBase.py
@@ -1537,8 +1537,9 @@ def is_empty(self, msg, emsg):
 
         """
         try:
-            assert(msg == emsg)
-        except BaseException:
+            from yggdrasil.tests import assert_equal
+            assert_equal(msg, emsg)
+        except AssertionError:
             return False
         return True
 

From 78d26c54886dbd61d18c42b4e9e06b10c5454636 Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Mon, 27 Sep 2021 13:46:56 -0700
Subject: [PATCH 19/73] Change the dependency to one that can be used on all
 platforms in tests.

---
 yggdrasil/tests/yamls/FakePlant.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yggdrasil/tests/yamls/FakePlant.yaml b/yggdrasil/tests/yamls/FakePlant.yaml
index d408b2af4..e9ab859c0 100644
--- a/yggdrasil/tests/yamls/FakePlant.yaml
+++ b/yggdrasil/tests/yamls/FakePlant.yaml
@@ -25,4 +25,4 @@
   dependencies:
   - numpy
   additional_dependencies:
-    c++: [doxygen]
+    c++: [cmake]

From 0a6a7a9574a81552931e39f4991d5f6888dc0434 Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Mon, 27 Sep 2021 14:13:08 -0700
Subject: [PATCH 20/73] Raise an error when an intergration fails rather than
 relying on log to indicate one

---
 yggdrasil/examples/tests/__init__.py | 5 ++++-
 yggdrasil/runner.py                  | 7 +++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/yggdrasil/examples/tests/__init__.py b/yggdrasil/examples/tests/__init__.py
index aa4d6da6c..3b82cba5f 100644
--- a/yggdrasil/examples/tests/__init__.py
+++ b/yggdrasil/examples/tests/__init__.py
@@ -380,7 +380,10 @@ def run_example(self):
             self.runner = runner.get_runner(self.yaml, namespace=self.namespace,
                                             production_run=True,
                                             mpi_tag_start=mpi_tag_start)
-            self.runner.run()
+            try:
+                self.runner.run()
+            except runner.IntegrationError:
+                pass
             self.runner.printStatus()
             self.runner.printStatus(return_str=True)
             if self.mpi_rank != 0:
diff --git a/yggdrasil/runner.py b/yggdrasil/runner.py
index fc51eff2f..c00a3f9e2 100755
--- a/yggdrasil/runner.py
+++ b/yggdrasil/runner.py
@@ -23,6 +23,11 @@
 COLOR_NORMAL = '\033[0m'
 
 
+class IntegrationError(BaseException):
+    r"""Error raised when there is an error in an integration."""
+    pass
+
+
 class YggFunction(YggClass):
     r"""This class wraps function-like behavior around a model.
 
@@ -442,6 +447,8 @@ def run(self, signal_handler=None, timer=None, t0=None):
                 tprev = t
             self.info(40 * '=')
             self.info('%20s\t%f', "Total", tprev - t0)
+        if self.error_flag:
+            raise IntegrationError("Error running the integration.")
         if self.validate:
             for v in self.modeldrivers.values():
                 v['instance'].run_validation()

From 9eacdef859a4fdc9c404e719f045379958cb4cb6 Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Mon, 27 Sep 2021 19:03:27 -0700
Subject: [PATCH 21/73] Fix bug in runner tests with added integration error

---
 yggdrasil/tests/test_runner.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/yggdrasil/tests/test_runner.py b/yggdrasil/tests/test_runner.py
index bf7996d9e..f61abd77c 100644
--- a/yggdrasil/tests/test_runner.py
+++ b/yggdrasil/tests/test_runner.py
@@ -1,3 +1,4 @@
+import pytest
 import os
 import unittest
 import signal
@@ -22,8 +23,9 @@ def test_get_run():
     namespace = "test_run_%s" % str(uuid.uuid4)
     runner.run([ex_yamls['hello']['python']],
                namespace=namespace)
-    runner.run([ex_yamls['model_error']['python']],
-               namespace=namespace)
+    with pytest.raises(runner.IntegrationError):
+        runner.run([ex_yamls['model_error']['python']],
+                   namespace=namespace)
 
 
 def test_run_process_connections():

From 1dd445d71b0b50099cec56f2f1640714344febda Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Tue, 28 Sep 2021 10:50:41 -0700
Subject: [PATCH 22/73] Update the commit in the FakePlant YAML

---
 HISTORY.rst                          | 2 +-
 yggdrasil/tests/yamls/FakePlant.yaml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/HISTORY.rst b/HISTORY.rst
index 6ce8f65d4..2a1c5d618 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -2,7 +2,7 @@
 History
 =======
 
-1.8.1 (2021-09-27) Minor updates to support model submission form development
+1.8.1 (2021-09-28) Minor updates to support model submission form development
 ------------------
 
 * Added --model_repository option to the integration-service-manager CLI
diff --git a/yggdrasil/tests/yamls/FakePlant.yaml b/yggdrasil/tests/yamls/FakePlant.yaml
index e9ab859c0..29c719866 100644
--- a/yggdrasil/tests/yamls/FakePlant.yaml
+++ b/yggdrasil/tests/yamls/FakePlant.yaml
@@ -19,7 +19,7 @@
       name: ./Output/output.txt
       filetype: table
   repository_url: https://github.com/cropsinsilico/example-fakemodel
-  repository_commit: bb39d0db7e4883dbef2c1cb90bc563fc9d757a19
+  repository_commit: 498d04b7c98078fbd8d608f860882dc571476fd8
   description: Example model submission
   validation_command: validate.py
   dependencies:

From e4270ac5f6b746a0553f3464e918824cbfb597bf Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Tue, 28 Sep 2021 21:20:36 -0700
Subject: [PATCH 23/73] Fix test for installing model dependencies

---
 HISTORY.rst                                 |  2 +-
 yggdrasil/drivers/ModelDriver.py            | 10 ++++++---
 yggdrasil/drivers/PythonModelDriver.py      |  2 +-
 yggdrasil/drivers/tests/test_ModelDriver.py | 25 ++++++++++++++++-----
 4 files changed, 28 insertions(+), 11 deletions(-)

diff --git a/HISTORY.rst b/HISTORY.rst
index 2a1c5d618..67fef74b4 100644
--- a/HISTORY.rst
+++ b/HISTORY.rst
@@ -2,7 +2,7 @@
 History
 =======
 
-1.8.1 (2021-09-28) Minor updates to support model submission form development
+1.8.1 (2021-09-29) Minor updates to support model submission form development
 ------------------
 
 * Added --model_repository option to the integration-service-manager CLI
diff --git a/yggdrasil/drivers/ModelDriver.py b/yggdrasil/drivers/ModelDriver.py
index 7c4248768..ebdab84ec 100755
--- a/yggdrasil/drivers/ModelDriver.py
+++ b/yggdrasil/drivers/ModelDriver.py
@@ -861,7 +861,7 @@ def install_model_dependencies(cls, dependencies, always_yes=False):
                                    always_yes=always_yes)
 
     @classmethod
-    def install_dependency(cls, package, package_manager=None,
+    def install_dependency(cls, package=None, package_manager=None,
                            arguments=None, command=None, always_yes=False):
         r"""Install a dependency.
 
@@ -880,6 +880,7 @@ def install_dependency(cls, package, package_manager=None,
                 False.
 
         """
+        assert(package)
         if isinstance(package, str):
             package = package.split()
         if package_manager is None:
@@ -916,14 +917,17 @@ def install_dependency(cls, package, package_manager=None,
         elif package_manager == 'choco':
             cmd = ['choco', 'install'] + package
         elif package_manager == 'vcpkg':
-            cmd = ['vcpkg.exe', 'install'] + package
+            cmd = ['vcpkg.exe', 'install'] + package + ['--triplet',
+                                                        'x64-windows']
         else:
             package_managers = {'pip': 'python',
                                 'cran': 'r'}
             if package_manager in package_managers:
                 drv = import_component(
                     'model', package_managers[package_manager])
-                return drv.install_dependency(package_manager, package)
+                return drv.install_dependency(
+                    package=package, package_manager=package_manager,
+                    arguments=arguments, always_yes=always_yes)
             raise NotImplementedError(f"Unsupported package manager: "
                                       f"{package_manager}")
         if arguments:
diff --git a/yggdrasil/drivers/PythonModelDriver.py b/yggdrasil/drivers/PythonModelDriver.py
index d1a3487fd..12aa83460 100755
--- a/yggdrasil/drivers/PythonModelDriver.py
+++ b/yggdrasil/drivers/PythonModelDriver.py
@@ -227,7 +227,7 @@ def write_finalize_oiter(cls, var):
         return out
 
     @classmethod
-    def install_dependency(cls, package, package_manager=None, **kwargs):
+    def install_dependency(cls, package=None, package_manager=None, **kwargs):
         r"""Install a dependency.
 
         Args:
diff --git a/yggdrasil/drivers/tests/test_ModelDriver.py b/yggdrasil/drivers/tests/test_ModelDriver.py
index a7d5272d3..ee1e25526 100644
--- a/yggdrasil/drivers/tests/test_ModelDriver.py
+++ b/yggdrasil/drivers/tests/test_ModelDriver.py
@@ -1,3 +1,4 @@
+import pytest
 import os
 import copy
 import unittest
@@ -473,14 +474,26 @@ def test_split_line(self, vals=None):
 
     def test_install_model_dependencies(self, deps=None):
         r"""Test install_model_dependencies."""
-        if (deps is None) and (self.import_cls.language == 'c'):
-            if platform._is_win:  # pragma: windows
-                deps = ['cmake']
+        if deps is None:
+            if self.import_cls.language == 'c':
+                deps = [
+                    "cmake",
+                    {"package_manager": "pip", "package": "pyyaml",
+                     "arguments": "-v"},
+                    {"package": "cmake", "arguments": "-v"}]
+                if not platform._is_win:  # pragma: windows
+                    from yggdrasil import tools
+                    if not tools.get_conda_prefix():
+                        deps.append({"package_manager": "vcpkg",
+                                     "package": "zmq"})
+                else:
+                    deps.append('doxygen')
             else:
-                deps = ['doxygen', 'cmake']
-        else:
-            deps = []
+                deps = []
         self.import_cls.install_model_dependencies(deps, always_yes=True)
+        with pytest.raises(NotImplementedError):
+            self.import_cls.install_dependency(
+                'invalid', package_manager='invalid')
 
 
 class TestModelDriverNoStart(TestModelParam, parent.TestDriverNoStart):

From 63250b0530f22dfbc01f0ed6d4e3bf84a5bce838 Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Tue, 28 Sep 2021 21:46:56 -0700
Subject: [PATCH 24/73] Add missing test for R dependency install

---
 yggdrasil/drivers/tests/test_RModelDriver.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/yggdrasil/drivers/tests/test_RModelDriver.py b/yggdrasil/drivers/tests/test_RModelDriver.py
index 415201057..2521a73e9 100644
--- a/yggdrasil/drivers/tests/test_RModelDriver.py
+++ b/yggdrasil/drivers/tests/test_RModelDriver.py
@@ -45,7 +45,8 @@ def test_python2language(self):
     def test_install_model_dependencies(self, deps=None):
         r"""Test install_model_dependencies."""
         if deps is None:
-            deps = ['units', 'zeallot']
+            deps = ['units', 'zeallot',
+                    {'package': 'units', 'arguments': '-v'}]
         super(TestRModelDriverNoInit, self).test_install_model_dependencies(
             deps=deps)
 

From 6d810dcc247e4fa875e28387ba0f39c8f5026e0d Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Tue, 28 Sep 2021 23:38:23 -0700
Subject: [PATCH 25/73] Fix typo in dependency test

---
 yggdrasil/drivers/tests/test_ModelDriver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yggdrasil/drivers/tests/test_ModelDriver.py b/yggdrasil/drivers/tests/test_ModelDriver.py
index ee1e25526..7a01282e4 100644
--- a/yggdrasil/drivers/tests/test_ModelDriver.py
+++ b/yggdrasil/drivers/tests/test_ModelDriver.py
@@ -481,7 +481,7 @@ def test_install_model_dependencies(self, deps=None):
                     {"package_manager": "pip", "package": "pyyaml",
                      "arguments": "-v"},
                     {"package": "cmake", "arguments": "-v"}]
-                if not platform._is_win:  # pragma: windows
+                if platform._is_win:  # pragma: windows
                     from yggdrasil import tools
                     if not tools.get_conda_prefix():
                         deps.append({"package_manager": "vcpkg",

From c4e89778b44382aacd3bcd6890eb3687585e88ef Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Wed, 29 Sep 2021 12:42:05 -0700
Subject: [PATCH 26/73] Try installing czmq from vcpkg instead

---
 yggdrasil/drivers/ModelDriver.py            | 4 ++--
 yggdrasil/drivers/tests/test_ModelDriver.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/yggdrasil/drivers/ModelDriver.py b/yggdrasil/drivers/ModelDriver.py
index ebdab84ec..305ce2584 100755
--- a/yggdrasil/drivers/ModelDriver.py
+++ b/yggdrasil/drivers/ModelDriver.py
@@ -917,8 +917,8 @@ def install_dependency(cls, package=None, package_manager=None,
         elif package_manager == 'choco':
             cmd = ['choco', 'install'] + package
         elif package_manager == 'vcpkg':
-            cmd = ['vcpkg.exe', 'install'] + package + ['--triplet',
-                                                        'x64-windows']
+            cmd = ['vcpkg.exe', 'install', '--triplet', 'x64-windows']
+            cmd += package
         else:
             package_managers = {'pip': 'python',
                                 'cran': 'r'}
diff --git a/yggdrasil/drivers/tests/test_ModelDriver.py b/yggdrasil/drivers/tests/test_ModelDriver.py
index 7a01282e4..e0c4bb44b 100644
--- a/yggdrasil/drivers/tests/test_ModelDriver.py
+++ b/yggdrasil/drivers/tests/test_ModelDriver.py
@@ -485,7 +485,7 @@ def test_install_model_dependencies(self, deps=None):
                     from yggdrasil import tools
                     if not tools.get_conda_prefix():
                         deps.append({"package_manager": "vcpkg",
-                                     "package": "zmq"})
+                                     "package": "czmq"})
                 else:
                     deps.append('doxygen')
             else:

From 9edd19a1eeb9337377cff230f7adfd80723b3b56 Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Wed, 29 Sep 2021 16:51:32 -0700
Subject: [PATCH 27/73] Update tests to use 3.7 for most jobs and 3.6 in a
 special case (prompted by update to GHA mac images that deprecated python
 3.6)

---
 .github/workflows/test-install.yml | 14 +++++++-------
 utils/test-install-base.yml        | 14 +++++++-------
 2 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/test-install.yml b/.github/workflows/test-install.yml
index dcb383d92..61172d8bb 100644
--- a/.github/workflows/test-install.yml
+++ b/.github/workflows/test-install.yml
@@ -44,7 +44,7 @@ jobs:
         - macos-latest
         - windows-latest
         python-version:
-        - 3.6
+        - 3.7
         install-method:
         - pip
         test-flags1:
@@ -61,7 +61,7 @@ jobs:
         - true
         include:
         - os: windows-latest
-          python-version: 3.6
+          python-version: 3.7
           install-method: pip
           test-flags1: --long-running --languages cpp
           test-flags2: --long-running --skip-languages c++
@@ -351,7 +351,7 @@ jobs:
         - macos-latest
         - windows-latest
         python-version:
-        - 3.6
+        - 3.7
         install-method:
         - conda
         test-flags1:
@@ -364,7 +364,7 @@ jobs:
         - true
         include:
         - os: ubuntu-latest
-          python-version: 3.7
+          python-version: 3.6
           install-method: conda
           test-flags1: --long-running
           test-flags2: ''
@@ -679,7 +679,7 @@ jobs:
         os:
         - ubuntu-latest
         python-version:
-        - 3.6
+        - 3.7
         install-method:
         - pip
         test-flags1:
@@ -981,7 +981,7 @@ jobs:
         os:
         - ubuntu-latest
         python-version:
-        - 3.6
+        - 3.7
         install-method:
         - conda
         test-flags1:
@@ -1295,7 +1295,7 @@ jobs:
         - macos-latest
         - windows-latest
         python-version:
-        - 3.6
+        - 3.7
         install-method:
         - pip
         test-flags1:
diff --git a/utils/test-install-base.yml b/utils/test-install-base.yml
index 99fb52edd..f25eeb2eb 100644
--- a/utils/test-install-base.yml
+++ b/utils/test-install-base.yml
@@ -38,7 +38,7 @@ jobs:
       max-parallel: 20
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: [3.6]
+        python-version: [3.7]
         install-method: [pip]
         test-flags1: [--long-running --test-suite=timing --test-suite=demos --test-suite=top, --test-suite=examples --separate-test='--test-suite=mpi --write-script=run_mpi.sh --with-mpi=3', --test-suite=types --skip-languages c c++, --test-suite=types --languages c cpp]
         test-flags2: [""]
@@ -46,7 +46,7 @@ jobs:
         install-mpi: [true]
         include:
           - os: windows-latest
-            python-version: 3.6
+            python-version: 3.7
             install-method: pip
             test-flags1: --long-running --languages cpp
             test-flags2: --long-running --skip-languages c++
@@ -348,7 +348,7 @@ jobs:
       max-parallel: 20
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: [3.6]
+        python-version: [3.7]
         install-method: [conda]
         test-flags1: [--long-running --separate-test='--test-suite=mpi --write-script=run_mpi.sh']
         test-flags2: [--with-examples examples/tests/test_backwards.py]
@@ -356,7 +356,7 @@ jobs:
         install-mpi: [true]
         include:
           - os: ubuntu-latest
-            python-version: 3.7
+            python-version: 3.6
             install-method: conda
             test-flags1: --long-running
             test-flags2: ""
@@ -400,7 +400,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python-version: [3.6]
+        python-version: [3.7]
         install-method: [pip]
         test-flags1: [--long-running --languages python R matlab]
         test-flags2: [""]
@@ -425,7 +425,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python-version: [3.6]
+        python-version: [3.7]
         install-method: [conda]
         test-flags1: [--long-running --languages python R matlab]
         test-flags2: [tests/test_services.py --nocapture]
@@ -455,7 +455,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: [3.6]
+        python-version: [3.7]
         install-method: [pip]
         test-flags1: [--long-running]
         test-flags2: [""]

From fbf04eebfaff9ce5b75d4026e036f721dd2a7ed1 Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Wed, 29 Sep 2021 17:33:25 -0700
Subject: [PATCH 28/73] Try avoiding error on new GHA images by limiting calls
 to import_component at initial import

---
 yggdrasil/communication/CommBase.py | 24 +++---------------------
 yggdrasil/communication/FileComm.py |  8 ++++++--
 2 files changed, 9 insertions(+), 23 deletions(-)

diff --git a/yggdrasil/communication/CommBase.py b/yggdrasil/communication/CommBase.py
index e8269ef04..1c56e2514 100755
--- a/yggdrasil/communication/CommBase.py
+++ b/yggdrasil/communication/CommBase.py
@@ -576,7 +576,6 @@ class CommBase(tools.YggClass):
         'for_service': {'type': 'boolean', 'default': False}}
     _schema_excluded_from_class = ['name']
     _default_serializer = 'default'
-    _default_serializer_class = None
     _schema_excluded_from_class_validation = ['datatype']
     is_file = False
     _maxMsgSize = 0
@@ -751,12 +750,8 @@ def _init_before_open(self, **kwargs):
         if self.serializer is None:
             # Get serializer class
             if seri_cls is None:
-                if (((seri_kws['seritype'] == self._default_serializer)
-                     and (self._default_serializer_class is not None))):
-                    seri_cls = self._default_serializer_class
-                else:
-                    seri_cls = import_component('serializer',
-                                                subtype=seri_kws['seritype'])
+                seri_cls = import_component('serializer',
+                                            subtype=seri_kws['seritype'])
             # Recover keyword arguments for serializer passed to comm class
             for k in seri_cls.seri_kws():
                 if k in kwargs:
@@ -811,15 +806,6 @@ def _init_before_open(self, **kwargs):
                               subtype=filter_schema.identify_subtype(self.filter))
             self.filter = create_component('filter', **filter_kws)
 
-    @staticmethod
-    def before_registration(cls):
-        r"""Operations that should be performed to modify class attributes prior
-        to registration."""
-        tools.YggClass.before_registration(cls)
-        cls._default_serializer_class = import_component('serializer',
-                                                         cls._default_serializer,
-                                                         without_schema=True)
-
     @classmethod
     def get_testing_options(cls, serializer=None, **kwargs):
         r"""Method to return a dictionary of testing options for this class.
@@ -842,11 +828,7 @@ def get_testing_options(cls, serializer=None, **kwargs):
         """
         if serializer is None:
             serializer = cls._default_serializer
-        if (((serializer == cls._default_serializer)
-             and (cls._default_serializer_class is not None))):
-            seri_cls = cls._default_serializer_class
-        else:
-            seri_cls = import_component('serializer', serializer)
+        seri_cls = import_component('serializer', serializer)
         out_seri = seri_cls.get_testing_options(**kwargs)
         out = {'attributes': ['name', 'address', 'direction',
                               'serializer', 'recv_timeout',
diff --git a/yggdrasil/communication/FileComm.py b/yggdrasil/communication/FileComm.py
index 995a5186b..8a34510a6 100755
--- a/yggdrasil/communication/FileComm.py
+++ b/yggdrasil/communication/FileComm.py
@@ -4,6 +4,7 @@
 from yggdrasil import platform, tools
 from yggdrasil.serialize.SerializeBase import SerializeBase
 from yggdrasil.communication import CommBase
+from yggdrasil.components import import_component
 
 
 class FileComm(CommBase.CommBase):
@@ -139,8 +140,10 @@ def before_registration(cls):
         # Add serializer properties to schema
         if cls._filetype != 'binary':
             assert('serializer' not in cls._schema_properties)
+            serializer_class = import_component('serializer',
+                                                cls._default_serializer)
             cls._schema_properties.update(
-                cls._default_serializer_class._schema_properties)
+                serializer_class._schema_properties)
             del cls._schema_properties['seritype']
         cls._commtype = cls._filetype
 
@@ -182,7 +185,8 @@ def get_testing_options(cls, read_meth=None, **kwargs):
                 out['contents'] += comment
                 out['recv_partial'].append([])
         else:
-            seri_cls = cls._default_serializer_class
+            seri_cls = import_component('serializer',
+                                        cls._default_serializer)
             if seri_cls.concats_as_str:
                 out['recv_partial'] = [[x] for x in out['recv']]
                 out['recv'] = seri_cls.concatenate(out['recv'], **out['kwargs'])

From 9e0a67a9c478402b16d383760e2cfd5d1d00ce67 Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Wed, 29 Sep 2021 17:53:25 -0700
Subject: [PATCH 29/73] Add debug message to determine why properties are not
 being imported after the GHA update

---
 yggdrasil/components.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/yggdrasil/components.py b/yggdrasil/components.py
index 7c71c3c67..2b8fdc0b8 100644
--- a/yggdrasil/components.py
+++ b/yggdrasil/components.py
@@ -49,6 +49,7 @@ def __init__(self, *args, import_function=None, **kwargs):
         # self._schema_directory = os.path.join(self._directory, 'schemas')
         self._import_function = import_function
         self._imported = False
+        print('ClassRegistry', self._module, self._directory)
         super(ClassRegistry, self).__init__(*args, **kwargs)
 
     def import_classes(self):
@@ -56,6 +57,8 @@ def import_classes(self):
         if self._imported:
             return
         self._imported = True
+        print('ClassRegistry import_classes', self._module,
+              glob.glob(os.path.join(self._directory, '*.py')))
         for x in glob.glob(os.path.join(self._directory, '*.py')):
             mod = os.path.basename(x)[:-3]
             if not mod.startswith('__'):

From 09a92ec47be287c18573eed4e66e7ac82a35a3d8 Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Wed, 29 Sep 2021 18:27:10 -0700
Subject: [PATCH 30/73] More debug messages

---
 yggdrasil/metaschema/datatypes/__init__.py  | 3 +++
 yggdrasil/metaschema/properties/__init__.py | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/yggdrasil/metaschema/datatypes/__init__.py b/yggdrasil/metaschema/datatypes/__init__.py
index c29cefd52..9b1df1f71 100644
--- a/yggdrasil/metaschema/datatypes/__init__.py
+++ b/yggdrasil/metaschema/datatypes/__init__.py
@@ -86,6 +86,9 @@ def register_type(type_class):
     for p in type_class.properties:
         prop_class = get_metaschema_property(p)
         if prop_class.name != p:
+            from yggdrasil.metaschema.properties import get_registered_properties
+            x = get_registered_properties()
+            print(prop_class.name, x.keys())
             raise ValueError("Type '%s' has unregistered property '%s'."
                              % (type_name, p))
     # Update property class with this type's info
diff --git a/yggdrasil/metaschema/properties/__init__.py b/yggdrasil/metaschema/properties/__init__.py
index 2fe18a36c..c63b44e1b 100644
--- a/yggdrasil/metaschema/properties/__init__.py
+++ b/yggdrasil/metaschema/properties/__init__.py
@@ -42,6 +42,7 @@ def register_metaschema_property(prop_class):
     if _metaschema is not None:
         if prop_name not in _metaschema['properties']:
             raise ValueError("Property '%s' not in pre-loaded metaschema." % prop_name)
+    print(f"Registered {prop_name} {prop_class}")
     _metaschema_properties[prop_name] = prop_class
     return prop_class
 
@@ -64,6 +65,7 @@ def get_registered_properties():
         dict: Registered property/class pairs.
 
     """
+    global _metaschema_properties
     return _metaschema_properties
 
 
@@ -80,6 +82,7 @@ def get_metaschema_property(property_name, skip_generic=False):
 
     """
     from yggdrasil.metaschema.properties import MetaschemaProperty
+    global _metaschema_properties
     out = _metaschema_properties.get(property_name, None)
     if (out is None) and (not skip_generic):
         out = MetaschemaProperty.MetaschemaProperty

From aab95366fedf3aa0b74391712ffd46ffcb38120a Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Wed, 29 Sep 2021 18:50:48 -0700
Subject: [PATCH 31/73] More debug

---
 yggdrasil/components.py                     | 1 +
 yggdrasil/metaschema/properties/__init__.py | 1 +
 2 files changed, 2 insertions(+)

diff --git a/yggdrasil/components.py b/yggdrasil/components.py
index 2b8fdc0b8..a91398cf1 100644
--- a/yggdrasil/components.py
+++ b/yggdrasil/components.py
@@ -62,6 +62,7 @@ def import_classes(self):
         for x in glob.glob(os.path.join(self._directory, '*.py')):
             mod = os.path.basename(x)[:-3]
             if not mod.startswith('__'):
+                print('ClassRegistry importing', mod)
                 importlib.import_module(self._module + '.%s' % mod)
         if self._import_function is not None:
             self._import_function()
diff --git a/yggdrasil/metaschema/properties/__init__.py b/yggdrasil/metaschema/properties/__init__.py
index c63b44e1b..a15e7f52c 100644
--- a/yggdrasil/metaschema/properties/__init__.py
+++ b/yggdrasil/metaschema/properties/__init__.py
@@ -25,6 +25,7 @@ def register_metaschema_property(prop_class):
     from yggdrasil.metaschema import _metaschema, _base_validator
     global _metaschema_properties
     prop_name = prop_class.name
+    print(f"Registering {prop_name} {prop_class}")
     if _metaschema_properties.has_entry(prop_name):
         raise ValueError("Property '%s' already registered." % prop_name)
     if prop_name in _base_validator.VALIDATORS:

From f1dc53a127534de88eabcfad1b19778f9ec24d1c Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Wed, 29 Sep 2021 19:00:30 -0700
Subject: [PATCH 32/73] Try allowing double import

---
 yggdrasil/components.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yggdrasil/components.py b/yggdrasil/components.py
index a91398cf1..924257297 100644
--- a/yggdrasil/components.py
+++ b/yggdrasil/components.py
@@ -56,7 +56,6 @@ def import_classes(self):
         r"""Import all classes in the same directory."""
         if self._imported:
             return
-        self._imported = True
         print('ClassRegistry import_classes', self._module,
               glob.glob(os.path.join(self._directory, '*.py')))
         for x in glob.glob(os.path.join(self._directory, '*.py')):
@@ -64,6 +63,7 @@ def import_classes(self):
             if not mod.startswith('__'):
                 print('ClassRegistry importing', mod)
                 importlib.import_module(self._module + '.%s' % mod)
+        self._imported = True
         if self._import_function is not None:
             self._import_function()
 

From 298a84a00d8c6e760b828ca77c2476732aa84c95 Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Wed, 29 Sep 2021 19:03:51 -0700
Subject: [PATCH 33/73] Sort modules in class registry import to preserve order
 for testing

---
 yggdrasil/components.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/yggdrasil/components.py b/yggdrasil/components.py
index 924257297..e784ec3c5 100644
--- a/yggdrasil/components.py
+++ b/yggdrasil/components.py
@@ -57,8 +57,8 @@ def import_classes(self):
         if self._imported:
             return
         print('ClassRegistry import_classes', self._module,
-              glob.glob(os.path.join(self._directory, '*.py')))
-        for x in glob.glob(os.path.join(self._directory, '*.py')):
+              sorted(glob.glob(os.path.join(self._directory, '*.py'))))
+        for x in sorted(glob.glob(os.path.join(self._directory, '*.py'))):
             mod = os.path.basename(x)[:-3]
             if not mod.startswith('__'):
                 print('ClassRegistry importing', mod)

From eb6d30d8faeb89d5b604c4c13698d8c0c5374ed6 Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Thu, 30 Sep 2021 10:54:38 -0700
Subject: [PATCH 34/73] Revert "Try allowing double import"

This reverts commit f1dc53a127534de88eabcfad1b19778f9ec24d1c.
---
 yggdrasil/components.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yggdrasil/components.py b/yggdrasil/components.py
index e784ec3c5..7b3b419bf 100644
--- a/yggdrasil/components.py
+++ b/yggdrasil/components.py
@@ -56,6 +56,7 @@ def import_classes(self):
         r"""Import all classes in the same directory."""
         if self._imported:
             return
+        self._imported = True
         print('ClassRegistry import_classes', self._module,
               sorted(glob.glob(os.path.join(self._directory, '*.py'))))
         for x in sorted(glob.glob(os.path.join(self._directory, '*.py'))):
@@ -63,7 +64,6 @@ def import_classes(self):
             if not mod.startswith('__'):
                 print('ClassRegistry importing', mod)
                 importlib.import_module(self._module + '.%s' % mod)
-        self._imported = True
         if self._import_function is not None:
             self._import_function()
 

From 9689a96a33d261cc129cbd975ccbe94c15fcbbad Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Thu, 30 Sep 2021 10:54:56 -0700
Subject: [PATCH 35/73] Revert "More debug"

This reverts commit aab95366fedf3aa0b74391712ffd46ffcb38120a.
---
 yggdrasil/components.py                     | 1 -
 yggdrasil/metaschema/properties/__init__.py | 1 -
 2 files changed, 2 deletions(-)

diff --git a/yggdrasil/components.py b/yggdrasil/components.py
index 7b3b419bf..ae0f9d5bd 100644
--- a/yggdrasil/components.py
+++ b/yggdrasil/components.py
@@ -62,7 +62,6 @@ def import_classes(self):
         for x in sorted(glob.glob(os.path.join(self._directory, '*.py'))):
             mod = os.path.basename(x)[:-3]
             if not mod.startswith('__'):
-                print('ClassRegistry importing', mod)
                 importlib.import_module(self._module + '.%s' % mod)
         if self._import_function is not None:
             self._import_function()
diff --git a/yggdrasil/metaschema/properties/__init__.py b/yggdrasil/metaschema/properties/__init__.py
index a15e7f52c..c63b44e1b 100644
--- a/yggdrasil/metaschema/properties/__init__.py
+++ b/yggdrasil/metaschema/properties/__init__.py
@@ -25,7 +25,6 @@ def register_metaschema_property(prop_class):
     from yggdrasil.metaschema import _metaschema, _base_validator
     global _metaschema_properties
     prop_name = prop_class.name
-    print(f"Registering {prop_name} {prop_class}")
     if _metaschema_properties.has_entry(prop_name):
         raise ValueError("Property '%s' already registered." % prop_name)
     if prop_name in _base_validator.VALIDATORS:

From 1b1ca21297f32a12141b3ac18d64b7b011eb0d74 Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Thu, 30 Sep 2021 10:55:06 -0700
Subject: [PATCH 36/73] Revert "More debug messages"

This reverts commit 09a92ec47be287c18573eed4e66e7ac82a35a3d8.
---
 yggdrasil/metaschema/datatypes/__init__.py  | 3 ---
 yggdrasil/metaschema/properties/__init__.py | 3 ---
 2 files changed, 6 deletions(-)

diff --git a/yggdrasil/metaschema/datatypes/__init__.py b/yggdrasil/metaschema/datatypes/__init__.py
index 9b1df1f71..c29cefd52 100644
--- a/yggdrasil/metaschema/datatypes/__init__.py
+++ b/yggdrasil/metaschema/datatypes/__init__.py
@@ -86,9 +86,6 @@ def register_type(type_class):
     for p in type_class.properties:
         prop_class = get_metaschema_property(p)
         if prop_class.name != p:
-            from yggdrasil.metaschema.properties import get_registered_properties
-            x = get_registered_properties()
-            print(prop_class.name, x.keys())
             raise ValueError("Type '%s' has unregistered property '%s'."
                              % (type_name, p))
     # Update property class with this type's info
diff --git a/yggdrasil/metaschema/properties/__init__.py b/yggdrasil/metaschema/properties/__init__.py
index c63b44e1b..2fe18a36c 100644
--- a/yggdrasil/metaschema/properties/__init__.py
+++ b/yggdrasil/metaschema/properties/__init__.py
@@ -42,7 +42,6 @@ def register_metaschema_property(prop_class):
     if _metaschema is not None:
         if prop_name not in _metaschema['properties']:
             raise ValueError("Property '%s' not in pre-loaded metaschema." % prop_name)
-    print(f"Registered {prop_name} {prop_class}")
     _metaschema_properties[prop_name] = prop_class
     return prop_class
 
@@ -65,7 +64,6 @@ def get_registered_properties():
         dict: Registered property/class pairs.
 
     """
-    global _metaschema_properties
     return _metaschema_properties
 
 
@@ -82,7 +80,6 @@ def get_metaschema_property(property_name, skip_generic=False):
 
     """
     from yggdrasil.metaschema.properties import MetaschemaProperty
-    global _metaschema_properties
     out = _metaschema_properties.get(property_name, None)
     if (out is None) and (not skip_generic):
         out = MetaschemaProperty.MetaschemaProperty

From 69ef1f73511bca2f109474846151ca0748e2247e Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Thu, 30 Sep 2021 10:58:51 -0700
Subject: [PATCH 37/73] Revert "Add debug message to determine why properties
 are not being imported after the GHA update"

This reverts commit 9e0a67a9c478402b16d383760e2cfd5d1d00ce67.
---
 yggdrasil/components.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/yggdrasil/components.py b/yggdrasil/components.py
index ae0f9d5bd..af24a3aec 100644
--- a/yggdrasil/components.py
+++ b/yggdrasil/components.py
@@ -49,7 +49,6 @@ def __init__(self, *args, import_function=None, **kwargs):
         # self._schema_directory = os.path.join(self._directory, 'schemas')
         self._import_function = import_function
         self._imported = False
-        print('ClassRegistry', self._module, self._directory)
         super(ClassRegistry, self).__init__(*args, **kwargs)
 
     def import_classes(self):
@@ -57,8 +56,6 @@ def import_classes(self):
         if self._imported:
             return
         self._imported = True
-        print('ClassRegistry import_classes', self._module,
-              sorted(glob.glob(os.path.join(self._directory, '*.py'))))
         for x in sorted(glob.glob(os.path.join(self._directory, '*.py'))):
             mod = os.path.basename(x)[:-3]
             if not mod.startswith('__'):

From 7eef31bab1339c9f1b44ac2b1b4a5e6d17e78f23 Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Thu, 30 Sep 2021 10:59:07 -0700
Subject: [PATCH 38/73] Revert "Try avoiding error on new GHA images by
 limiting calls to import_component at initial import"

This reverts commit fbf04eebfaff9ce5b75d4026e036f721dd2a7ed1.
---
 yggdrasil/communication/CommBase.py | 24 +++++++++++++++++++++---
 yggdrasil/communication/FileComm.py |  8 ++------
 2 files changed, 23 insertions(+), 9 deletions(-)

diff --git a/yggdrasil/communication/CommBase.py b/yggdrasil/communication/CommBase.py
index 1c56e2514..e8269ef04 100755
--- a/yggdrasil/communication/CommBase.py
+++ b/yggdrasil/communication/CommBase.py
@@ -576,6 +576,7 @@ class CommBase(tools.YggClass):
         'for_service': {'type': 'boolean', 'default': False}}
     _schema_excluded_from_class = ['name']
     _default_serializer = 'default'
+    _default_serializer_class = None
     _schema_excluded_from_class_validation = ['datatype']
     is_file = False
     _maxMsgSize = 0
@@ -750,8 +751,12 @@ def _init_before_open(self, **kwargs):
         if self.serializer is None:
             # Get serializer class
             if seri_cls is None:
-                seri_cls = import_component('serializer',
-                                            subtype=seri_kws['seritype'])
+                if (((seri_kws['seritype'] == self._default_serializer)
+                     and (self._default_serializer_class is not None))):
+                    seri_cls = self._default_serializer_class
+                else:
+                    seri_cls = import_component('serializer',
+                                                subtype=seri_kws['seritype'])
             # Recover keyword arguments for serializer passed to comm class
             for k in seri_cls.seri_kws():
                 if k in kwargs:
@@ -806,6 +811,15 @@ def _init_before_open(self, **kwargs):
                               subtype=filter_schema.identify_subtype(self.filter))
             self.filter = create_component('filter', **filter_kws)
 
+    @staticmethod
+    def before_registration(cls):
+        r"""Operations that should be performed to modify class attributes prior
+        to registration."""
+        tools.YggClass.before_registration(cls)
+        cls._default_serializer_class = import_component('serializer',
+                                                         cls._default_serializer,
+                                                         without_schema=True)
+
     @classmethod
     def get_testing_options(cls, serializer=None, **kwargs):
         r"""Method to return a dictionary of testing options for this class.
@@ -828,7 +842,11 @@ def get_testing_options(cls, serializer=None, **kwargs):
         """
         if serializer is None:
             serializer = cls._default_serializer
-        seri_cls = import_component('serializer', serializer)
+        if (((serializer == cls._default_serializer)
+             and (cls._default_serializer_class is not None))):
+            seri_cls = cls._default_serializer_class
+        else:
+            seri_cls = import_component('serializer', serializer)
         out_seri = seri_cls.get_testing_options(**kwargs)
         out = {'attributes': ['name', 'address', 'direction',
                               'serializer', 'recv_timeout',
diff --git a/yggdrasil/communication/FileComm.py b/yggdrasil/communication/FileComm.py
index 8a34510a6..995a5186b 100755
--- a/yggdrasil/communication/FileComm.py
+++ b/yggdrasil/communication/FileComm.py
@@ -4,7 +4,6 @@
 from yggdrasil import platform, tools
 from yggdrasil.serialize.SerializeBase import SerializeBase
 from yggdrasil.communication import CommBase
-from yggdrasil.components import import_component
 
 
 class FileComm(CommBase.CommBase):
@@ -140,10 +139,8 @@ def before_registration(cls):
         # Add serializer properties to schema
         if cls._filetype != 'binary':
             assert('serializer' not in cls._schema_properties)
-            serializer_class = import_component('serializer',
-                                                cls._default_serializer)
             cls._schema_properties.update(
-                serializer_class._schema_properties)
+                cls._default_serializer_class._schema_properties)
             del cls._schema_properties['seritype']
         cls._commtype = cls._filetype
 
@@ -185,8 +182,7 @@ def get_testing_options(cls, read_meth=None, **kwargs):
                 out['contents'] += comment
                 out['recv_partial'].append([])
         else:
-            seri_cls = import_component('serializer',
-                                        cls._default_serializer)
+            seri_cls = cls._default_serializer_class
             if seri_cls.concats_as_str:
                 out['recv_partial'] = [[x] for x in out['recv']]
                 out['recv'] = seri_cls.concatenate(out['recv'], **out['kwargs'])

From bd96ca1a46fd742bad8905a5bb3f85605880caae Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Thu, 30 Sep 2021 15:23:52 -0700
Subject: [PATCH 39/73] Prevent ArrayMetaschemaType from being imported when
 loading the schema

---
 yggdrasil/metaschema/encoder.py      | 2 +-
 yggdrasil/serialize/SerializeBase.py | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/yggdrasil/metaschema/encoder.py b/yggdrasil/metaschema/encoder.py
index f1d01b30b..7dc1af050 100644
--- a/yggdrasil/metaschema/encoder.py
+++ b/yggdrasil/metaschema/encoder.py
@@ -36,7 +36,7 @@ def string2import(s):
 
     """
     pkg_mod = s.split(u':')
-    if len(pkg_mod) == 2:
+    if (len(pkg_mod) == 2) and (not s.startswith('http')) and (' ' not in s):
         try:
             mod = importlib.import_module(pkg_mod[0])
             s = getattr(mod, pkg_mod[1])
diff --git a/yggdrasil/serialize/SerializeBase.py b/yggdrasil/serialize/SerializeBase.py
index 27bb76346..ae6120a95 100644
--- a/yggdrasil/serialize/SerializeBase.py
+++ b/yggdrasil/serialize/SerializeBase.py
@@ -7,11 +7,7 @@
 from yggdrasil.metaschema.datatypes import (
     guess_type_from_obj, get_type_from_def, get_type_class, compare_schema,
     type2numpy)
-from yggdrasil.metaschema.properties.ScalarMetaschemaProperties import (
-    _flexible_types)
 from yggdrasil.metaschema.datatypes.MetaschemaType import MetaschemaType
-from yggdrasil.metaschema.datatypes.ArrayMetaschemaType import (
-    OneDArrayMetaschemaType)
 
 
 class SerializeBase(tools.YggClass):
@@ -570,6 +566,10 @@ def update_typedef_from_oldstyle(self, typedef):
                 continue
             # Key specific changes to type
             if k == 'format_str':
+                from yggdrasil.metaschema.datatypes.ArrayMetaschemaType import (
+                    OneDArrayMetaschemaType)
+                from yggdrasil.metaschema.properties.ScalarMetaschemaProperties import (
+                    _flexible_types)
                 v = tools.bytes2str(v)
                 fmts = serialize.extract_formats(v)
                 if 'type' in typedef:

From 69ec9e198fc9be8cb94c57316b459a4a9217f8a1 Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Thu, 30 Sep 2021 19:14:54 -0700
Subject: [PATCH 40/73] Update component registration to create registry in
 constants

---
 yggdrasil/.ygg_schema.yml          | 1448 ++++++++++++++--------------
 yggdrasil/command_line.py          |    3 +-
 yggdrasil/components.py            |  330 +++----
 yggdrasil/constants.py             |  134 +++
 yggdrasil/drivers/ModelDriver.py   |    4 +-
 yggdrasil/schema.py                |  117 ++-
 yggdrasil/tests/test_components.py |    5 +-
 yggdrasil/tools.py                 |   48 +
 8 files changed, 1122 insertions(+), 967 deletions(-)

diff --git a/yggdrasil/.ygg_schema.yml b/yggdrasil/.ygg_schema.yml
index f384508f9..6e4a4d057 100644
--- a/yggdrasil/.ygg_schema.yml
+++ b/yggdrasil/.ygg_schema.yml
@@ -180,44 +180,55 @@ definitions:
             type: string
           type: array
       required:
+      - name
       - datatype
       - commtype
-      - name
       title: comm_base
       type: object
     - anyOf:
       - additionalProperties: true
-        description: Schema for comm component ['mpi'] subtype.
+        description: Schema for comm component ['buffer'] subtype.
         properties:
           commtype:
             default: default
-            description: MPI communicator.
+            description: Communication mechanism that should be used.
             enum:
-            - mpi
+            - buffer
             type: string
-        title: MPIComm
+        title: yggdrasil.communication.BufferComm.BufferComm
         type: object
       - additionalProperties: true
-        description: Schema for comm component ['rmq'] subtype.
+        description: Schema for comm component ['default'] subtype.
         properties:
           commtype:
             default: default
-            description: RabbitMQ connection.
+            description: Communication mechanism selected based on the current platform.
             enum:
-            - rmq
+            - default
             type: string
-        title: RMQComm
+        title: yggdrasil.communication.DefaultComm.DefaultComm
         type: object
       - additionalProperties: true
-        description: Schema for comm component ['buffer'] subtype.
+        description: Schema for comm component ['ipc'] subtype.
         properties:
           commtype:
             default: default
-            description: Communication mechanism that should be used.
+            description: Interprocess communication (IPC) queue.
             enum:
-            - buffer
+            - ipc
+            type: string
+        title: yggdrasil.communication.IPCComm.IPCComm
+        type: object
+      - additionalProperties: true
+        description: Schema for comm component ['mpi'] subtype.
+        properties:
+          commtype:
+            default: default
+            description: MPI communicator.
+            enum:
+            - mpi
             type: string
-        title: BufferComm
+        title: yggdrasil.communication.MPIComm.MPIComm
         type: object
       - additionalProperties: true
         description: Schema for comm component ['rest'] subtype.
@@ -242,21 +253,18 @@ definitions:
             type: object
           port:
             type: int
-        title: RESTComm
+        title: yggdrasil.communication.RESTComm.RESTComm
         type: object
       - additionalProperties: true
-        description: Schema for comm component ['value'] subtype.
+        description: Schema for comm component ['rmq'] subtype.
         properties:
           commtype:
             default: default
-            description: Constant value.
+            description: RabbitMQ connection.
             enum:
-            - value
+            - rmq
             type: string
-          count:
-            default: 1
-            type: integer
-        title: ValueComm
+        title: yggdrasil.communication.RMQComm.RMQComm
         type: object
       - additionalProperties: true
         description: Schema for comm component ['rmq_async'] subtype.
@@ -267,29 +275,21 @@ definitions:
             enum:
             - rmq_async
             type: string
-        title: RMQAsyncComm
-        type: object
-      - additionalProperties: true
-        description: Schema for comm component ['ipc'] subtype.
-        properties:
-          commtype:
-            default: default
-            description: Interprocess communication (IPC) queue.
-            enum:
-            - ipc
-            type: string
-        title: IPCComm
+        title: yggdrasil.communication.RMQAsyncComm.RMQAsyncComm
         type: object
       - additionalProperties: true
-        description: Schema for comm component ['default'] subtype.
+        description: Schema for comm component ['value'] subtype.
         properties:
           commtype:
             default: default
-            description: Communication mechanism selected based on the current platform.
+            description: Constant value.
             enum:
-            - default
+            - value
             type: string
-        title: DefaultComm
+          count:
+            default: 1
+            type: integer
+        title: yggdrasil.communication.ValueComm.ValueComm
         type: object
       - additionalProperties: true
         description: Schema for comm component ['zmq'] subtype.
@@ -300,7 +300,7 @@ definitions:
             enum:
             - zmq
             type: string
-        title: ZMQComm
+        title: yggdrasil.communication.ZMQComm.ZMQComm
         type: object
     description: Schema for comm components.
     title: comm
@@ -370,8 +370,8 @@ definitions:
             - $ref: '#/definitions/transform'
           type: array
       required:
-      - outputs
       - inputs
+      - outputs
       title: connection_base
       type: object
     - anyOf:
@@ -384,29 +384,27 @@ definitions:
             enum:
             - connection
             type: string
-        title: ConnectionDriver
+        title: yggdrasil.drivers.ConnectionDriver.ConnectionDriver
         type: object
       - additionalProperties: true
-        description: Schema for connection component ['rpc_response'] subtype.
+        description: Schema for connection component ['input'] subtype.
         properties:
           connection_type:
-            description: Connection between one or more comms/files and one or more
-              comms/files.
+            description: Connection between one or more comms/files and a model.
             enum:
-            - rpc_response
+            - input
             type: string
-        title: RPCResponseDriver
+        title: yggdrasil.drivers.InputDriver.InputDriver
         type: object
       - additionalProperties: true
-        description: Schema for connection component ['rpc_request'] subtype.
+        description: Schema for connection component ['file_input'] subtype.
         properties:
           connection_type:
-            description: Connection between one or more comms/files and one or more
-              comms/files.
+            description: Connection between a file and a model.
             enum:
-            - rpc_request
+            - file_input
             type: string
-        title: RPCRequestDriver
+        title: yggdrasil.drivers.FileInputDriver.FileInputDriver
         type: object
       - additionalProperties: true
         description: Schema for connection component ['output'] subtype.
@@ -416,7 +414,7 @@ definitions:
             enum:
             - output
             type: string
-        title: OutputDriver
+        title: yggdrasil.drivers.OutputDriver.OutputDriver
         type: object
       - additionalProperties: true
         description: Schema for connection component ['file_output'] subtype.
@@ -426,27 +424,29 @@ definitions:
             enum:
             - file_output
             type: string
-        title: FileOutputDriver
+        title: yggdrasil.drivers.FileOutputDriver.FileOutputDriver
         type: object
       - additionalProperties: true
-        description: Schema for connection component ['input'] subtype.
+        description: Schema for connection component ['rpc_response'] subtype.
         properties:
           connection_type:
-            description: Connection between one or more comms/files and a model.
+            description: Connection between one or more comms/files and one or more
+              comms/files.
             enum:
-            - input
+            - rpc_response
             type: string
-        title: InputDriver
+        title: yggdrasil.drivers.RPCResponseDriver.RPCResponseDriver
         type: object
       - additionalProperties: true
-        description: Schema for connection component ['file_input'] subtype.
+        description: Schema for connection component ['rpc_request'] subtype.
         properties:
           connection_type:
-            description: Connection between a file and a model.
+            description: Connection between one or more comms/files and one or more
+              comms/files.
             enum:
-            - file_input
+            - rpc_request
             type: string
-        title: FileInputDriver
+        title: yggdrasil.drivers.RPCRequestDriver.RPCRequestDriver
         type: object
     description: Schema for connection components.
     title: connection
@@ -704,8 +704,8 @@ definitions:
             is used.
           type: string
       required:
-      - working_dir
       - name
+      - working_dir
       - filetype
       title: file_base
       type: object
@@ -739,7 +739,34 @@ definitions:
               type: instance
         required:
         - serializer
-        title: FileComm
+        title: yggdrasil.communication.FileComm.FileComm
+        type: object
+      - additionalProperties: true
+        description: Schema for file component ['ascii'] subtype.
+        properties:
+          comment:
+            default: '# '
+            description: One or more characters indicating a comment. Defaults to
+              '# '.
+            type: string
+          datatype:
+            description: JSON schema defining the type of object that the serializer
+              will be used to serialize/deserialize. Defaults to default_datatype.
+            type: schema
+          filetype:
+            default: binary
+            description: This file is read/written as encoded text one line at a time.
+            enum:
+            - ascii
+            type: string
+          newline:
+            default: '
+
+              '
+            description: One or more characters indicating a newline. Defaults to
+              '\n'.
+            type: string
+        title: yggdrasil.communication.AsciiFileComm.AsciiFileComm
         type: object
       - additionalProperties: true
         description: Schema for file component ['map'] subtype.
@@ -772,10 +799,10 @@ definitions:
             description: One or more characters indicating a newline. Defaults to
               '\n'.
             type: string
-        title: AsciiMapComm
+        title: yggdrasil.communication.AsciiMapComm.AsciiMapComm
         type: object
       - additionalProperties: true
-        description: Schema for file component ['ply'] subtype.
+        description: Schema for file component ['table'] subtype.
         properties:
           comment:
             default: '# '
@@ -786,12 +813,18 @@ definitions:
             description: JSON schema defining the type of object that the serializer
               will be used to serialize/deserialize. Defaults to default_datatype.
             type: schema
+          delimiter:
+            default: "\t"
+            description: Character(s) that should be used to separate columns. Defaults
+              to '\t'.
+            type: string
           filetype:
             default: binary
-            description: The file is in the `Ply <http://paulbourke.net/dataformats/ply/>`_
-              data format for 3D structures.
+            description: The file is an ASCII table that will be read/written one
+              row at a time. If ``as_array`` is ``True``, the table will be read/written
+              all at once.
             enum:
-            - ply
+            - table
             type: string
           newline:
             default: '
@@ -800,10 +833,15 @@ definitions:
             description: One or more characters indicating a newline. Defaults to
               '\n'.
             type: string
-        title: PlyFileComm
+          use_astropy:
+            default: false
+            description: If True, the astropy package will be used to serialize/deserialize
+              table. Defaults to False.
+            type: boolean
+        title: yggdrasil.communication.AsciiTableComm.AsciiTableComm
         type: object
       - additionalProperties: true
-        description: Schema for file component ['netcdf'] subtype.
+        description: Schema for file component ['json'] subtype.
         properties:
           comment:
             default: '# '
@@ -816,10 +854,17 @@ definitions:
             type: schema
           filetype:
             default: binary
-            description: The file is read/written as netCDF.
+            description: The file contains a JSON serialized object.
             enum:
-            - netcdf
+            - json
             type: string
+          indent:
+            default: "\t"
+            description: String or number of spaces that should be used to indent
+              each level within the seiralized structure. Defaults to '\t'.
+            type:
+            - string
+            - int
           newline:
             default: '
 
@@ -827,29 +872,15 @@ definitions:
             description: One or more characters indicating a newline. Defaults to
               '\n'.
             type: string
-          read_attributes:
-            default: false
-            description: If True, the attributes are read in as well as the variables.
-              Defaults to False.
+          sort_keys:
+            default: true
+            description: If True, the serialization of dictionaries will be in key
+              sorted order. Defaults to True.
             type: boolean
-          variables:
-            description: List of variables to read in. If not provided, all variables
-              will be read.
-            items:
-              type: string
-            type: array
-          version:
-            default: 1
-            description: Version of netCDF format that should be used. Defaults to
-              1. Options are 1 (classic format) and 2 (64-bit offset format).
-            enum:
-            - 1
-            - 2
-            type: integer
-        title: NetCDFFileComm
+        title: yggdrasil.communication.JSONFileComm.JSONFileComm
         type: object
       - additionalProperties: true
-        description: Schema for file component ['wofost'] subtype.
+        description: Schema for file component ['mat'] subtype.
         properties:
           comment:
             default: '# '
@@ -860,16 +891,12 @@ definitions:
             description: JSON schema defining the type of object that the serializer
               will be used to serialize/deserialize. Defaults to default_datatype.
             type: schema
-          delimiter:
-            default: ' = '
-            description: Delimiter that should be used to separate name/value pairs
-              in the map. Defaults to \t.
-            type: string
           filetype:
             default: binary
-            description: The file is a WOFOST parameter file.
+            description: The file is a Matlab .mat file containing one or more serialized
+              Matlab variables.
             enum:
-            - wofost
+            - mat
             type: string
           newline:
             default: '
@@ -878,10 +905,10 @@ definitions:
             description: One or more characters indicating a newline. Defaults to
               '\n'.
             type: string
-        title: WOFOSTParamFileComm
+        title: yggdrasil.communication.MatFileComm.MatFileComm
         type: object
       - additionalProperties: true
-        description: Schema for file component ['table'] subtype.
+        description: Schema for file component ['netcdf'] subtype.
         properties:
           comment:
             default: '# '
@@ -892,18 +919,11 @@ definitions:
             description: JSON schema defining the type of object that the serializer
               will be used to serialize/deserialize. Defaults to default_datatype.
             type: schema
-          delimiter:
-            default: "\t"
-            description: Character(s) that should be used to separate columns. Defaults
-              to '\t'.
-            type: string
           filetype:
             default: binary
-            description: The file is an ASCII table that will be read/written one
-              row at a time. If ``as_array`` is ``True``, the table will be read/written
-              all at once.
+            description: The file is read/written as netCDF.
             enum:
-            - table
+            - netcdf
             type: string
           newline:
             default: '
@@ -912,15 +932,29 @@ definitions:
             description: One or more characters indicating a newline. Defaults to
               '\n'.
             type: string
-          use_astropy:
+          read_attributes:
             default: false
-            description: If True, the astropy package will be used to serialize/deserialize
-              table. Defaults to False.
+            description: If True, the attributes are read in as well as the variables.
+              Defaults to False.
             type: boolean
-        title: AsciiTableComm
+          variables:
+            description: List of variables to read in. If not provided, all variables
+              will be read.
+            items:
+              type: string
+            type: array
+          version:
+            default: 1
+            description: Version of netCDF format that should be used. Defaults to
+              1. Options are 1 (classic format) and 2 (64-bit offset format).
+            enum:
+            - 1
+            - 2
+            type: integer
+        title: yggdrasil.communication.NetCDFFileComm.NetCDFFileComm
         type: object
       - additionalProperties: true
-        description: Schema for file component ['mat'] subtype.
+        description: Schema for file component ['ply'] subtype.
         properties:
           comment:
             default: '# '
@@ -933,10 +967,10 @@ definitions:
             type: schema
           filetype:
             default: binary
-            description: The file is a Matlab .mat file containing one or more serialized
-              Matlab variables.
+            description: The file is in the `Ply <http://paulbourke.net/dataformats/ply/>`_
+              data format for 3D structures.
             enum:
-            - mat
+            - ply
             type: string
           newline:
             default: '
@@ -945,10 +979,10 @@ definitions:
             description: One or more characters indicating a newline. Defaults to
               '\n'.
             type: string
-        title: MatFileComm
+        title: yggdrasil.communication.PlyFileComm.PlyFileComm
         type: object
       - additionalProperties: true
-        description: Schema for file component ['ascii'] subtype.
+        description: Schema for file component ['obj'] subtype.
         properties:
           comment:
             default: '# '
@@ -961,9 +995,10 @@ definitions:
             type: schema
           filetype:
             default: binary
-            description: This file is read/written as encoded text one line at a time.
+            description: The file is in the `Obj <http://paulbourke.net/dataformats/obj/>`_
+              data format for 3D structures.
             enum:
-            - ascii
+            - obj
             type: string
           newline:
             default: '
@@ -972,7 +1007,7 @@ definitions:
             description: One or more characters indicating a newline. Defaults to
               '\n'.
             type: string
-        title: AsciiFileComm
+        title: yggdrasil.communication.ObjFileComm.ObjFileComm
         type: object
       - additionalProperties: true
         description: Schema for file component ['pandas'] subtype.
@@ -1019,10 +1054,10 @@ definitions:
             description: If True, the astropy package will be used to serialize/deserialize
               table. Defaults to False.
             type: boolean
-        title: PandasFileComm
+        title: yggdrasil.communication.PandasFileComm.PandasFileComm
         type: object
       - additionalProperties: true
-        description: Schema for file component ['obj'] subtype.
+        description: Schema for file component ['pickle'] subtype.
         properties:
           comment:
             default: '# '
@@ -1035,10 +1070,9 @@ definitions:
             type: schema
           filetype:
             default: binary
-            description: The file is in the `Obj <http://paulbourke.net/dataformats/obj/>`_
-              data format for 3D structures.
+            description: The file contains one or more pickled Python objects.
             enum:
-            - obj
+            - pickle
             type: string
           newline:
             default: '
@@ -1047,10 +1081,10 @@ definitions:
             description: One or more characters indicating a newline. Defaults to
               '\n'.
             type: string
-        title: ObjFileComm
+        title: yggdrasil.communication.PickleFileComm.PickleFileComm
         type: object
       - additionalProperties: true
-        description: Schema for file component ['yaml'] subtype.
+        description: Schema for file component ['wofost'] subtype.
         properties:
           comment:
             default: '# '
@@ -1061,30 +1095,17 @@ definitions:
             description: JSON schema defining the type of object that the serializer
               will be used to serialize/deserialize. Defaults to default_datatype.
             type: schema
-          default_flow_style:
-            default: false
-            description: If True, nested collections will be serialized in the block
-              style. If False, they will always be serialized in the flow style. See
-              `PyYAML Documentation <https://pyyaml.org/wiki/PyYAMLDocumentation>`_.
-            type: boolean
-          encoding:
-            default: utf-8
-            description: Encoding that should be used to serialize the object. Defaults
-              to 'utf-8'.
+          delimiter:
+            default: ' = '
+            description: Delimiter that should be used to separate name/value pairs
+              in the map. Defaults to \t.
             type: string
           filetype:
             default: binary
-            description: The file contains a YAML serialized object.
+            description: The file is a WOFOST parameter file.
             enum:
-            - yaml
+            - wofost
             type: string
-          indent:
-            default: "\t"
-            description: String or number of spaces that should be used to indent
-              each level within the seiralized structure. Defaults to '\t'.
-            type:
-            - string
-            - int
           newline:
             default: '
 
@@ -1092,10 +1113,10 @@ definitions:
             description: One or more characters indicating a newline. Defaults to
               '\n'.
             type: string
-        title: YAMLFileComm
+        title: yggdrasil.communication.WOFOSTParamFileComm.WOFOSTParamFileComm
         type: object
       - additionalProperties: true
-        description: Schema for file component ['json'] subtype.
+        description: Schema for file component ['yaml'] subtype.
         properties:
           comment:
             default: '# '
@@ -1106,11 +1127,22 @@ definitions:
             description: JSON schema defining the type of object that the serializer
               will be used to serialize/deserialize. Defaults to default_datatype.
             type: schema
+          default_flow_style:
+            default: false
+            description: If True, nested collections will be serialized in the block
+              style. If False, they will always be serialized in the flow style. See
+              `PyYAML Documentation <https://pyyaml.org/wiki/PyYAMLDocumentation>`_.
+            type: boolean
+          encoding:
+            default: utf-8
+            description: Encoding that should be used to serialize the object. Defaults
+              to 'utf-8'.
+            type: string
           filetype:
             default: binary
-            description: The file contains a JSON serialized object.
+            description: The file contains a YAML serialized object.
             enum:
-            - json
+            - yaml
             type: string
           indent:
             default: "\t"
@@ -1126,39 +1158,7 @@ definitions:
             description: One or more characters indicating a newline. Defaults to
               '\n'.
             type: string
-          sort_keys:
-            default: true
-            description: If True, the serialization of dictionaries will be in key
-              sorted order. Defaults to True.
-            type: boolean
-        title: JSONFileComm
-        type: object
-      - additionalProperties: true
-        description: Schema for file component ['pickle'] subtype.
-        properties:
-          comment:
-            default: '# '
-            description: One or more characters indicating a comment. Defaults to
-              '# '.
-            type: string
-          datatype:
-            description: JSON schema defining the type of object that the serializer
-              will be used to serialize/deserialize. Defaults to default_datatype.
-            type: schema
-          filetype:
-            default: binary
-            description: The file contains one or more pickled Python objects.
-            enum:
-            - pickle
-            type: string
-          newline:
-            default: '
-
-              '
-            description: One or more characters indicating a newline. Defaults to
-              '\n'.
-            type: string
-        title: PickleFileComm
+        title: yggdrasil.communication.YAMLFileComm.YAMLFileComm
         type: object
     description: Schema for file components.
     title: file
@@ -1204,25 +1204,7 @@ definitions:
           filtertype:
             enum:
             - direct
-        title: DirectFilter
-        type: object
-      - additionalProperties: true
-        description: Schema for filter component ['statement'] subtype.
-        properties:
-          filtertype:
-            enum:
-            - statement
-          statement:
-            description: Python statement in terms of the message as represented by
-              the string "%x%" that should evaluate to a boolean, True if the message
-              should pass through the filter, False if it should not. The statement
-              should only use a limited set of builtins and the math library (See
-              yggdrasil.tools.safe_eval). If more complex relationships are required,
-              use the FunctionFilter class.
-            type: string
-        required:
-        - statement
-        title: StatementFilter
+        title: yggdrasil.communication.filters.DirectFilter.DirectFilter
         type: object
       - additionalProperties: true
         description: Schema for filter component ['function'] subtype.
@@ -1241,7 +1223,25 @@ definitions:
             type: function
         required:
         - function
-        title: FunctionFilter
+        title: yggdrasil.communication.filters.FunctionFilter.FunctionFilter
+        type: object
+      - additionalProperties: true
+        description: Schema for filter component ['statement'] subtype.
+        properties:
+          filtertype:
+            enum:
+            - statement
+          statement:
+            description: Python statement in terms of the message as represented by
+              the string "%x%" that should evaluate to a boolean, True if the message
+              should pass through the filter, False if it should not. The statement
+              should only use a limited set of builtins and the math library (See
+              yggdrasil.tools.safe_eval). If more complex relationships are required,
+              use the FunctionFilter class.
+            type: string
+        required:
+        - statement
+        title: yggdrasil.communication.filters.StatementFilter.StatementFilter
         type: object
     description: Schema for filter components.
     title: filter
@@ -1304,6 +1304,8 @@ definitions:
             type: string
           type: array
         builddir:
+          description: Directory where the build should be saved. Defaults to <sourcedir>/build.
+            It can be relative to working_dir or absolute.
           type: string
         buildfile:
           type: string
@@ -1855,16 +1857,67 @@ definitions:
           type: string
       required:
       - language
-      - working_dir
       - args
       - name
+      - working_dir
       title: model_base
       type: object
     - anyOf:
       - additionalProperties: true
-        description: Schema for model component ['make'] subtype.
+        description: Schema for model component ['c'] subtype.
+        properties:
+          compiler:
+            description: Command or path to executable that should be used to compile
+              the model. If not provided, the compiler will be determined based on
+              configuration options for the language (if present) and the registered
+              compilers that are available on the current operating system.
+            type: string
+          compiler_flags:
+            default: []
+            description: Flags that should be passed to the compiler during compilation.
+              If nto provided, the compiler flags will be determined based on configuration
+              options for the language (if present), the compiler defaults, and the
+              default_compiler_flags class attribute.
+            items:
+              type: string
+            type: array
+          language:
+            default: executable
+            description: Model is written in C.
+            enum:
+            - c
+            type: string
+          linker:
+            description: Command or path to executable that should be used to link
+              the model. If not provided, the linker will be determined based on configuration
+              options for the language (if present) and the registered linkers that
+              are available on the current operating system
+            type: string
+          linker_flags:
+            default: []
+            description: Flags that should be passed to the linker during compilation.
+              If nto provided, the linker flags will be determined based on configuration
+              options for the language (if present), the linker defaults, and the
+              default_linker_flags class attribute.
+            items:
+              type: string
+            type: array
+          source_files:
+            default: []
+            description: Source files that should be compiled into an executable.
+              Defaults to an empty list and the driver will search for a source file
+              based on the model executable (the first model argument).
+            items:
+              type: string
+            type: array
+        title: yggdrasil.drivers.CModelDriver.CModelDriver
+        type: object
+      - additionalProperties: true
+        description: Schema for model component ['cmake'] subtype.
         properties:
           builddir:
+            description: Directory where the build should be saved. Defaults to <sourcedir>/build.
+              It can be relative to working_dir or absolute.
             type: string
           buildfile:
             type: string
@@ -1883,6 +1936,11 @@ definitions:
             items:
               type: string
             type: array
+          configuration:
+            default: Release
+            description: Build type/configuration that should be built. Defaults to
+              'Release'.
+            type: string
           env_compiler:
             description: Environment variable where the compiler executable should
               be stored for use within the Makefile. If not provided, this will be
@@ -1905,9 +1963,9 @@ definitions:
             type: string
           language:
             default: executable
-            description: Model is written in C/C++ and has a Makefile for compilation.
+            description: Model is written in C/C++ and has a CMake build system.
             enum:
-            - make
+            - cmake
             type: string
           linker:
             description: Command or path to executable that should be used to link
@@ -1924,16 +1982,6 @@ definitions:
             items:
               type: string
             type: array
-          makedir:
-            description: Directory where make should be invoked from if it is not
-              the same as the directory containing the makefile. Defaults to directory
-              containing makefile if provided, otherwise working_dir.
-            type: string
-          makefile:
-            default: Makefile
-            description: Path to make file either absolute, relative to makedir (if
-              provided), or relative to working_dir. Defaults to Makefile.
-            type: string
           source_files:
             default: []
             description: Source files that should be compiled into an executable.
@@ -1942,6 +1990,11 @@ definitions:
             items:
               type: string
             type: array
+          sourcedir:
+            description: Source directory to call cmake on. If not provided it is
+              set to working_dir. This should be the directory containing the CMakeLists.txt
+              file. It can be relative to working_dir or absolute.
+            type: string
           target:
             description: Make target that should be built to create the model executable.
               Defaults to None.
@@ -1971,54 +2024,61 @@ definitions:
             items:
               type: string
             type: array
-        title: MakeModelDriver
-        type: object
-      - additionalProperties: true
-        description: Schema for model component ['mpi'] subtype.
-        properties:
-          language:
-            default: executable
-            description: Model is being run on another MPI process and this driver
-              is used as as stand-in to monitor it on the root process.
-            enum:
-            - mpi
-            type: string
-        title: MPIPartnerModel
+        title: yggdrasil.drivers.CMakeModelDriver.CMakeModelDriver
         type: object
       - additionalProperties: true
-        description: Schema for model component ['python'] subtype.
+        description: Schema for model component ['c++', 'cpp', 'cxx'] subtype.
         properties:
-          interpreter:
-            description: Name or path of interpreter executable that should be used
-              to run the model. If not provided, the interpreter will be determined
-              based on configuration options for the language (if present) and the
-              default_interpreter class attribute.
+          compiler:
+            description: Command or path to executable that should be used to compile
+              the model. If not provided, the compiler will be determined based on
+              configuration options for the language (if present) and the registered
+              compilers that are available on the current operating system.
             type: string
-          interpreter_flags:
+          compiler_flags:
             default: []
-            description: Flags that should be passed to the interpreter when running
-              the model. If not provided, the flags are determined based on configuration
-              options for the language (if present) and the default_interpreter_flags
-              class attribute.
+            description: Flags that should be passed to the compiler during compilation.
+              If nto provided, the compiler flags will be determined based on configuration
+              options for the language (if present), the compiler defaults, and the
+              default_compiler_flags class attribute.
             items:
               type: string
             type: array
           language:
             default: executable
-            description: Model is written in Python.
+            description: Model is written in C++.
             enum:
-            - python
+            - c++
+            - cpp
+            - cxx
             type: string
-          skip_interpreter:
-            default: false
-            description: If True, no interpreter will be added to the arguments. This
-              should only be used for subclasses that will not be invoking the model
-              via the command line. Defaults to False.
-            type: boolean
-        title: PythonModelDriver
+          linker:
+            description: Command or path to executable that should be used to link
+              the model. If not provided, the linker will be determined based on configuration
+              options for the language (if present) and the registered linkers that
+              are available on the current operating system
+            type: string
+          linker_flags:
+            default: []
+            description: Flags that should be passed to the linker during compilation.
+              If nto provided, the linker flags will be determined based on configuration
+              options for the language (if present), the linker defaults, and the
+              default_linker_flags class attribute.
+            items:
+              type: string
+            type: array
+          source_files:
+            default: []
+            description: Source files that should be compiled into an executable.
+              Defaults to an empty list and the driver will search for a source file
+              based on the model executable (the first model argument).
+            items:
+              type: string
+            type: array
+        title: yggdrasil.drivers.CPPModelDriver.CPPModelDriver
         type: object
       - additionalProperties: true
-        description: Schema for model component ['lpy'] subtype.
+        description: Schema for model component ['dummy'] subtype.
         properties:
           interpreter:
             description: Name or path of interpreter executable that should be used
@@ -2037,9 +2097,10 @@ definitions:
             type: array
           language:
             default: executable
-            description: Model is an LPy system.
+            description: The programming language that the model is written in. A
+              list of available languages can be found :ref:`here <schema_table_model_subtype_rst>`.
             enum:
-            - lpy
+            - dummy
             type: string
           skip_interpreter:
             default: false
@@ -2047,7 +2108,7 @@ definitions:
               should only be used for subclasses that will not be invoking the model
               via the command line. Defaults to False.
             type: boolean
-        title: LPyModelDriver
+        title: yggdrasil.drivers.DummyModelDriver.DummyModelDriver
         type: object
       - additionalProperties: true
         description: Schema for model component ['executable'] subtype.
@@ -2058,10 +2119,10 @@ definitions:
             enum:
             - executable
             type: string
-        title: ExecutableModelDriver
+        title: yggdrasil.drivers.ExecutableModelDriver.ExecutableModelDriver
         type: object
       - additionalProperties: true
-        description: Schema for model component ['c'] subtype.
+        description: Schema for model component ['fortran'] subtype.
         properties:
           compiler:
             description: Command or path to executable that should be used to compile
@@ -2080,9 +2141,9 @@ definitions:
             type: array
           language:
             default: executable
-            description: Model is written in C.
+            description: Model is written in Fortran.
             enum:
-            - c
+            - fortran
             type: string
           linker:
             description: Command or path to executable that should be used to link
@@ -2107,11 +2168,98 @@ definitions:
             items:
               type: string
             type: array
-        title: CModelDriver
+          standard:
+            default: f2003
+            description: Fortran standard that should be used. Defaults to 'f2003'.
+            enum:
+            - f2003
+            - f2008
+            type: string
+        title: yggdrasil.drivers.FortranModelDriver.FortranModelDriver
         type: object
       - additionalProperties: true
-        description: Schema for model component ['c++', 'cpp', 'cxx'] subtype.
+        description: Schema for model component ['python'] subtype.
+        properties:
+          interpreter:
+            description: Name or path of interpreter executable that should be used
+              to run the model. If not provided, the interpreter will be determined
+              based on configuration options for the language (if present) and the
+              default_interpreter class attribute.
+            type: string
+          interpreter_flags:
+            default: []
+            description: Flags that should be passed to the interpreter when running
+              the model. If not provided, the flags are determined based on configuration
+              options for the language (if present) and the default_interpreter_flags
+              class attribute.
+            items:
+              type: string
+            type: array
+          language:
+            default: executable
+            description: Model is written in Python.
+            enum:
+            - python
+            type: string
+          skip_interpreter:
+            default: false
+            description: If True, no interpreter will be added to the arguments. This
+              should only be used for subclasses that will not be invoking the model
+              via the command line. Defaults to False.
+            type: boolean
+        title: yggdrasil.drivers.PythonModelDriver.PythonModelDriver
+        type: object
+      - additionalProperties: true
+        description: Schema for model component ['lpy'] subtype.
+        properties:
+          interpreter:
+            description: Name or path of interpreter executable that should be used
+              to run the model. If not provided, the interpreter will be determined
+              based on configuration options for the language (if present) and the
+              default_interpreter class attribute.
+            type: string
+          interpreter_flags:
+            default: []
+            description: Flags that should be passed to the interpreter when running
+              the model. If not provided, the flags are determined based on configuration
+              options for the language (if present) and the default_interpreter_flags
+              class attribute.
+            items:
+              type: string
+            type: array
+          language:
+            default: executable
+            description: Model is an LPy system.
+            enum:
+            - lpy
+            type: string
+          skip_interpreter:
+            default: false
+            description: If True, no interpreter will be added to the arguments. This
+              should only be used for subclasses that will not be invoking the model
+              via the command line. Defaults to False.
+            type: boolean
+        title: yggdrasil.drivers.LPyModelDriver.LPyModelDriver
+        type: object
+      - additionalProperties: true
+        description: Schema for model component ['mpi'] subtype.
         properties:
+          language:
+            default: executable
+            description: Model is being run on another MPI process and this driver
+              is used as as stand-in to monitor it on the root process.
+            enum:
+            - mpi
+            type: string
+        title: yggdrasil.drivers.MPIPartnerModel.MPIPartnerModel
+        type: object
+      - additionalProperties: true
+        description: Schema for model component ['make'] subtype.
+        properties:
+          builddir:
+            type: string
+          buildfile:
+            type: string
           compiler:
             description: Command or path to executable that should be used to compile
               the model. If not provided, the compiler will be determined based on
@@ -2127,13 +2275,31 @@ definitions:
             items:
               type: string
             type: array
+          env_compiler:
+            description: Environment variable where the compiler executable should
+              be stored for use within the Makefile. If not provided, this will be
+              determined by the target language driver.
+            type: string
+          env_compiler_flags:
+            description: Environment variable where the compiler flags should be stored
+              (including those required to compile against the |yggdrasil| interface).
+              If not provided, this will be determined by the target language driver.
+            type: string
+          env_linker:
+            description: Environment variable where the linker executable should be
+              stored for use within the Makefile. If not provided, this will be determined
+              by the target language driver.
+            type: string
+          env_linker_flags:
+            description: Environment variable where the linker flags should be stored
+              (including those required to link against the |yggdrasil| interface).
+              If not provided, this will be determined by the target language driver.
+            type: string
           language:
             default: executable
-            description: Model is written in C++.
+            description: Model is written in C/C++ and has a Makefile for compilation.
             enum:
-            - c++
-            - cpp
-            - cxx
+            - make
             type: string
           linker:
             description: Command or path to executable that should be used to link
@@ -2150,6 +2316,16 @@ definitions:
             items:
               type: string
             type: array
+          makedir:
+            description: Directory where make should be invoked from if it is not
+              the same as the directory containing the makefile. Defaults to directory
+              containing makefile if provided, otherwise working_dir.
+            type: string
+          makefile:
+            default: Makefile
+            description: Path to make file either absolute, relative to makedir (if
+              provided), or relative to working_dir. Defaults to Makefile.
+            type: string
           source_files:
             default: []
             description: Source files that should be compiled into an executable.
@@ -2158,7 +2334,73 @@ definitions:
             items:
               type: string
             type: array
-        title: CPPModelDriver
+          target:
+            description: Make target that should be built to create the model executable.
+              Defaults to None.
+            type: string
+          target_compiler:
+            description: Compilation tool that should be used to compile the target
+              language. Defaults to None and will be set based on the selected language
+              driver.
+            type: string
+          target_compiler_flags:
+            description: Compilation flags that should be passed to the target language
+              compiler. Defaults to [].
+            items:
+              type: string
+            type: array
+          target_language:
+            description: Language that the target is written in. Defaults to None
+              and will be set based on the source files provided.
+            type: string
+          target_linker:
+            description: Compilation tool that should be used to link the target language.
+              Defaults to None and will be set based on the selected language driver.
+            type: string
+          target_linker_flags:
+            description: Linking flags that should be passed to the target language
+              linker. Defaults to [].
+            items:
+              type: string
+            type: array
+        title: yggdrasil.drivers.MakeModelDriver.MakeModelDriver
+        type: object
+      - additionalProperties: true
+        description: Schema for model component ['matlab'] subtype.
+        properties:
+          interpreter:
+            description: Name or path of interpreter executable that should be used
+              to run the model. If not provided, the interpreter will be determined
+              based on configuration options for the language (if present) and the
+              default_interpreter class attribute.
+            type: string
+          interpreter_flags:
+            default: []
+            description: Flags that should be passed to the interpreter when running
+              the model. If not provided, the flags are determined based on configuration
+              options for the language (if present) and the default_interpreter_flags
+              class attribute.
+            items:
+              type: string
+            type: array
+          language:
+            default: executable
+            description: Model is written in Matlab.
+            enum:
+            - matlab
+            type: string
+          skip_interpreter:
+            default: false
+            description: If True, no interpreter will be added to the arguments. This
+              should only be used for subclasses that will not be invoking the model
+              via the command line. Defaults to False.
+            type: boolean
+          use_symunit:
+            default: false
+            description: If True, input/output variables with units will be represented
+              in Matlab using symunit. Defaults to False.
+            type: boolean
+        title: yggdrasil.drivers.MatlabModelDriver.MatlabModelDriver
         type: object
       - additionalProperties: true
         description: Schema for model component ['osr'] subtype.
@@ -2200,10 +2442,10 @@ definitions:
               additional export modules that output at a shorter rate, the existing
               table of values will be extrapolated.
             type: object
-        title: OSRModelDriver
+        title: yggdrasil.drivers.OSRModelDriver.OSRModelDriver
         type: object
       - additionalProperties: true
-        description: Schema for model component ['matlab'] subtype.
+        description: Schema for model component ['R', 'r'] subtype.
         properties:
           interpreter:
             description: Name or path of interpreter executable that should be used
@@ -2222,9 +2464,10 @@ definitions:
             type: array
           language:
             default: executable
-            description: Model is written in Matlab.
+            description: Model is written in R.
             enum:
-            - matlab
+            - R
+            - r
             type: string
           skip_interpreter:
             default: false
@@ -2232,16 +2475,25 @@ definitions:
               should only be used for subclasses that will not be invoking the model
               via the command line. Defaults to False.
             type: boolean
-          use_symunit:
-            default: false
-            description: If True, input/output variables with units will be represented
-              in Matlab using symunit. Defaults to False.
-            type: boolean
-        title: MatlabModelDriver
+        title: yggdrasil.drivers.RModelDriver.RModelDriver
         type: object
       - additionalProperties: true
-        description: Schema for model component ['R', 'r'] subtype.
+        description: Schema for model component ['sbml'] subtype.
         properties:
+          integrator:
+            default: cvode
+            description: Name of integrator that should be used. Valid options include
+              ['cvode', 'gillespie', 'rk4', 'rk45']. Defaults to 'cvode'.
+            enum:
+            - cvode
+            - gillespie
+            - rk4
+            - rk45
+            type: string
+          integrator_settings:
+            default: {}
+            description: Settings for the integrator. Defaults to empty dict.
+            type: object
           interpreter:
             description: Name or path of interpreter executable that should be used
               to run the model. If not provided, the interpreter will be determined
@@ -2259,18 +2511,50 @@ definitions:
             type: array
           language:
             default: executable
-            description: Model is written in R.
+            description: Model is an SBML model.
             enum:
-            - R
-            - r
+            - sbml
             type: string
+          only_output_final_step:
+            default: false
+            description: If True, only the final timestep is output. Defaults to False.
+            type: boolean
+          reset:
+            default: false
+            description: If True, the simulation will be reset to it's initial values
+              before each call (including the start time). Defaults to False.
+            type: boolean
+          selections:
+            default: []
+            description: Variables to include in the output. Defaults to None and
+              the time/floating selections will be returned.
+            items:
+              type: string
+            type: array
           skip_interpreter:
             default: false
             description: If True, no interpreter will be added to the arguments. This
               should only be used for subclasses that will not be invoking the model
               via the command line. Defaults to False.
             type: boolean
-        title: RModelDriver
+          skip_start_time:
+            default: false
+            description: If True, the results for the initial time step will not be
+              output. Defaults to False. This option is ignored if only_output_final_step
+              is True.
+            type: boolean
+          start_time:
+            default: 0.0
+            description: Time that simulation should be started from. If 'reset' is
+              True, the start time will always be the provided value, otherwise, the
+              start time will be the end of the previous call after the first call.
+              Defaults to 0.0.
+            type: number
+          steps:
+            default: 1
+            description: Number of steps that should be output. Defaults to None.
+            type: integer
+        title: yggdrasil.drivers.SBMLModelDriver.SBMLModelDriver
         type: object
       - additionalProperties: true
         description: Schema for model component ['timesync'] subtype.
@@ -2370,289 +2654,7 @@ definitions:
               (implies equivalence with the base variable in everything but name and
               units) or mappings with the keys:'
             type: object
-        title: TimeSyncModelDriver
-        type: object
-      - additionalProperties: true
-        description: Schema for model component ['sbml'] subtype.
-        properties:
-          integrator:
-            default: cvode
-            description: Name of integrator that should be used. Valid options include
-              ['cvode', 'gillespie', 'rk4', 'rk45']. Defaults to 'cvode'.
-            enum:
-            - cvode
-            - gillespie
-            - rk4
-            - rk45
-            type: string
-          integrator_settings:
-            default: {}
-            description: Settings for the integrator. Defaults to empty dict.
-            type: object
-          interpreter:
-            description: Name or path of interpreter executable that should be used
-              to run the model. If not provided, the interpreter will be determined
-              based on configuration options for the language (if present) and the
-              default_interpreter class attribute.
-            type: string
-          interpreter_flags:
-            default: []
-            description: Flags that should be passed to the interpreter when running
-              the model. If not provided, the flags are determined based on configuration
-              options for the language (if present) and the default_interpreter_flags
-              class attribute.
-            items:
-              type: string
-            type: array
-          language:
-            default: executable
-            description: Model is an SBML model.
-            enum:
-            - sbml
-            type: string
-          only_output_final_step:
-            default: false
-            description: If True, only the final timestep is output. Defaults to False.
-            type: boolean
-          reset:
-            default: false
-            description: If True, the simulation will be reset to it's initial values
-              before each call (including the start time). Defaults to False.
-            type: boolean
-          selections:
-            default: []
-            description: Variables to include in the output. Defaults to None and
-              the time/floating selections will be returned.
-            items:
-              type: string
-            type: array
-          skip_interpreter:
-            default: false
-            description: If True, no interpreter will be added to the arguments. This
-              should only be used for subclasses that will not be invoking the model
-              via the command line. Defaults to False.
-            type: boolean
-          skip_start_time:
-            default: false
-            description: If True, the results for the initial time step will not be
-              output. Defaults to False. This option is ignored if only_output_final_step
-              is True.
-            type: boolean
-          start_time:
-            default: 0.0
-            description: Time that simulation should be started from. If 'reset' is
-              True, the start time will always be the provided value, otherwise, the
-              start time will be the end of the previous call after the first call.
-              Defaults to 0.0.
-            type: number
-          steps:
-            default: 1
-            description: Number of steps that should be output. Defaults to None.
-            type: integer
-        title: SBMLModelDriver
-        type: object
-      - additionalProperties: true
-        description: Schema for model component ['cmake'] subtype.
-        properties:
-          builddir:
-            description: Directory where the build should be saved. Defaults to <sourcedir>/build.
-              It can be relative to working_dir or absolute.
-            type: string
-          buildfile:
-            type: string
-          compiler:
-            description: Command or path to executable that should be used to compile
-              the model. If not provided, the compiler will be determined based on
-              configuration options for the language (if present) and the registered
-              compilers that are available on the current operating system.
-            type: string
-          compiler_flags:
-            default: []
-            description: Flags that should be passed to the compiler during compilation.
-              If nto provided, the compiler flags will be determined based on configuration
-              options for the language (if present), the compiler defaults, and the
-              default_compiler_flags class attribute.
-            items:
-              type: string
-            type: array
-          configuration:
-            default: Release
-            description: Build type/configuration that should be built. Defaults to
-              'Release'.
-            type: string
-          env_compiler:
-            description: Environment variable where the compiler executable should
-              be stored for use within the Makefile. If not provided, this will be
-              determined by the target language driver.
-            type: string
-          env_compiler_flags:
-            description: Environment variable where the compiler flags should be stored
-              (including those required to compile against the |yggdrasil| interface).
-              If not provided, this will be determined by the target language driver.
-            type: string
-          env_linker:
-            description: Environment variable where the linker executable should be
-              stored for use within the Makefile. If not provided, this will be determined
-              by the target language driver.
-            type: string
-          env_linker_flags:
-            description: Environment variable where the linker flags should be stored
-              (including those required to link against the |yggdrasil| interface).
-              If not provided, this will be determined by the target language driver.
-            type: string
-          language:
-            default: executable
-            description: Model is written in C/C++ and has a CMake build system.
-            enum:
-            - cmake
-            type: string
-          linker:
-            description: Command or path to executable that should be used to link
-              the model. If not provided, the linker will be determined based on configuration
-              options for the language (if present) and the registered linkers that
-              are available on the current operating system
-            type: string
-          linker_flags:
-            default: []
-            description: Flags that should be passed to the linker during compilation.
-              If nto provided, the linker flags will be determined based on configuration
-              options for the language (if present), the linker defaults, and the
-              default_linker_flags class attribute.
-            items:
-              type: string
-            type: array
-          source_files:
-            default: []
-            description: Source files that should be compiled into an executable.
-              Defaults to an empty list and the driver will search for a source file
-              based on the model executable (the first model argument).
-            items:
-              type: string
-            type: array
-          sourcedir:
-            description: Source directory to call cmake on. If not provided it is
-              set to working_dir. This should be the directory containing the CMakeLists.txt
-              file. It can be relative to working_dir or absolute.
-            type: string
-          target:
-            description: Make target that should be built to create the model executable.
-              Defaults to None.
-            type: string
-          target_compiler:
-            description: Compilation tool that should be used to compile the target
-              language. Defaults to None and will be set based on the selected language
-              driver.
-            type: string
-          target_compiler_flags:
-            description: Compilation flags that should be passed to the target language
-              compiler. Defaults to [].
-            items:
-              type: string
-            type: array
-          target_language:
-            description: Language that the target is written in. Defaults to None
-              and will be set based on the source files provided.
-            type: string
-          target_linker:
-            description: Compilation tool that should be used to link the target language.
-              Defaults to None and will be set based on the selected language driver.
-            type: string
-          target_linker_flags:
-            description: Linking flags that should be passed to the target language
-              linker. Defaults to [].
-            items:
-              type: string
-            type: array
-        title: CMakeModelDriver
-        type: object
-      - additionalProperties: true
-        description: Schema for model component ['fortran'] subtype.
-        properties:
-          compiler:
-            description: Command or path to executable that should be used to compile
-              the model. If not provided, the compiler will be determined based on
-              configuration options for the language (if present) and the registered
-              compilers that are available on the current operating system.
-            type: string
-          compiler_flags:
-            default: []
-            description: Flags that should be passed to the compiler during compilation.
-              If nto provided, the compiler flags will be determined based on configuration
-              options for the language (if present), the compiler defaults, and the
-              default_compiler_flags class attribute.
-            items:
-              type: string
-            type: array
-          language:
-            default: executable
-            description: Model is written in Fortran.
-            enum:
-            - fortran
-            type: string
-          linker:
-            description: Command or path to executable that should be used to link
-              the model. If not provided, the linker will be determined based on configuration
-              options for the language (if present) and the registered linkers that
-              are available on the current operating system
-            type: string
-          linker_flags:
-            default: []
-            description: Flags that should be passed to the linker during compilation.
-              If nto provided, the linker flags will be determined based on configuration
-              options for the language (if present), the linker defaults, and the
-              default_linker_flags class attribute.
-            items:
-              type: string
-            type: array
-          source_files:
-            default: []
-            description: Source files that should be compiled into an executable.
-              Defaults to an empty list and the driver will search for a source file
-              based on the model executable (the first model argument).
-            items:
-              type: string
-            type: array
-          standard:
-            default: f2003
-            description: Fortran standard that should be used. Defaults to 'f2003'.
-            enum:
-            - f2003
-            - f2008
-            type: string
-        title: FortranModelDriver
-        type: object
-      - additionalProperties: true
-        description: Schema for model component ['dummy'] subtype.
-        properties:
-          interpreter:
-            description: Name or path of interpreter executable that should be used
-              to run the model. If not provided, the interpreter will be determined
-              based on configuration options for the language (if present) and the
-              default_interpreter class attribute.
-            type: string
-          interpreter_flags:
-            default: []
-            description: Flags that should be passed to the interpreter when running
-              the model. If not provided, the flags are determined based on configuration
-              options for the language (if present) and the default_interpreter_flags
-              class attribute.
-            items:
-              type: string
-            type: array
-          language:
-            default: executable
-            description: The programming language that the model is written in. A
-              list of available languages can be found :ref:`here <schema_table_model_subtype_rst>`.
-            enum:
-            - dummy
-            type: string
-          skip_interpreter:
-            default: false
-            description: If True, no interpreter will be added to the arguments. This
-              should only be used for subclasses that will not be invoking the model
-              via the command line. Defaults to False.
-            type: boolean
-        title: DummyModelDriver
+        title: yggdrasil.drivers.TimeSyncModelDriver.TimeSyncModelDriver
         type: object
     description: Schema for model components.
     title: model
@@ -2687,8 +2689,8 @@ definitions:
           type: boolean
         delimiter:
           default: "\t"
-          description: Character(s) that should be used to separate columns. Defaults
-            to '\t'.
+          description: Delimiter that should be used to separate name/value pairs
+            in the map. Defaults to \t.
           type: string
         encoded_datatype:
           description: JSON schema describing the type that serialized objects should
@@ -2783,6 +2785,23 @@ definitions:
       title: serializer_base
       type: object
     - anyOf:
+      - additionalProperties: true
+        description: Schema for serializer component ['map'] subtype.
+        properties:
+          delimiter:
+            default: "\t"
+            description: Delimiter that should be used to separate name/value pairs
+              in the map. Defaults to \t.
+            type: string
+          seritype:
+            default: default
+            description: Serialzation of mapping between key/value pairs with one
+              pair per line and using a character delimiter to separate keys and values.
+            enum:
+            - map
+            type: string
+        title: yggdrasil.serialize.AsciiMapSerialize.AsciiMapSerialize
+        type: object
       - additionalProperties: true
         description: Schema for serializer component ['default'] subtype.
         properties:
@@ -2794,7 +2813,7 @@ definitions:
             enum:
             - default
             type: string
-        title: DefaultSerialize
+        title: yggdrasil.serialize.DefaultSerialize.DefaultSerialize
         type: object
       - additionalProperties: true
         description: Schema for serializer component ['table'] subtype.
@@ -2839,56 +2858,18 @@ definitions:
             description: If True, the astropy package will be used to serialize/deserialize
               table. Defaults to False.
             type: boolean
-        title: AsciiTableSerialize
+        title: yggdrasil.serialize.AsciiTableSerialize.AsciiTableSerialize
         type: object
       - additionalProperties: true
-        description: Schema for serializer component ['pandas'] subtype.
+        description: Schema for serializer component ['direct'] subtype.
         properties:
-          delimiter:
-            default: "\t"
-            description: Character(s) that should be used to separate columns. Defaults
-              to '\t'.
-            type: string
-          field_names:
-            description: The names of fields in the format string. If not provided,
-              names are set based on the order of the fields in the format string.
-            items:
-              type: string
-            type: array
-          field_units:
-            description: The units of fields in the format string. If not provided,
-              all fields are assumed to be dimensionless.
-            items:
-              type: string
-            type: array
-          format_str:
-            description: If provided, this string will be used to format messages
-              from a list of arguments and parse messages to get a list of arguments
-              in C printf/scanf style. Defaults to None and messages are assumed to
-              already be bytes.
-            type: string
-          no_header:
-            default: false
-            description: If True, headers will not be read or serialized from/to tables.
-              Defaults to False.
-            type: boolean
           seritype:
             default: default
-            description: Serializes tables using the pandas package.
+            description: Direct serialization of bytes.
             enum:
-            - pandas
+            - direct
             type: string
-          str_as_bytes:
-            default: false
-            description: If True, strings in columns are read as bytes. Defaults to
-              False.
-            type: boolean
-          use_astropy:
-            default: false
-            description: If True, the astropy package will be used to serialize/deserialize
-              table. Defaults to False.
-            type: boolean
-        title: PandasSerialize
+        title: yggdrasil.serialize.DirectSerialize.DirectSerialize
         type: object
       - additionalProperties: true
         description: Schema for serializer component ['functional'] subtype.
@@ -2916,29 +2897,7 @@ definitions:
             enum:
             - functional
             type: string
-        title: FunctionalSerialize
-        type: object
-      - additionalProperties: true
-        description: Schema for serializer component ['mat'] subtype.
-        properties:
-          seritype:
-            default: default
-            description: Serializes objects using the Matlab .mat format.
-            enum:
-            - mat
-            type: string
-        title: MatSerialize
-        type: object
-      - additionalProperties: true
-        description: Schema for serializer component ['direct'] subtype.
-        properties:
-          seritype:
-            default: default
-            description: Direct serialization of bytes.
-            enum:
-            - direct
-            type: string
-        title: DirectSerialize
+        title: yggdrasil.serialize.FunctionalSerialize.FunctionalSerialize
         type: object
       - additionalProperties: true
         description: Schema for serializer component ['json'] subtype.
@@ -2961,35 +2920,29 @@ definitions:
             description: If True, the serialization of dictionaries will be in key
               sorted order. Defaults to True.
             type: boolean
-        title: JSONSerialize
+        title: yggdrasil.serialize.JSONSerialize.JSONSerialize
         type: object
       - additionalProperties: true
-        description: Schema for serializer component ['ply'] subtype.
+        description: Schema for serializer component ['mat'] subtype.
         properties:
           seritype:
             default: default
-            description: Serialize 3D structures using Ply format.
+            description: Serializes objects using the Matlab .mat format.
             enum:
-            - ply
+            - mat
             type: string
-        title: PlySerialize
+        title: yggdrasil.serialize.MatSerialize.MatSerialize
         type: object
       - additionalProperties: true
-        description: Schema for serializer component ['map'] subtype.
+        description: Schema for serializer component ['ply'] subtype.
         properties:
-          delimiter:
-            default: "\t"
-            description: Delimiter that should be used to separate name/value pairs
-              in the map. Defaults to \t.
-            type: string
           seritype:
             default: default
-            description: Serialzation of mapping between key/value pairs with one
-              pair per line and using a character delimiter to separate keys and values.
+            description: Serialize 3D structures using Ply format.
             enum:
-            - map
+            - ply
             type: string
-        title: AsciiMapSerialize
+        title: yggdrasil.serialize.PlySerialize.PlySerialize
         type: object
       - additionalProperties: true
         description: Schema for serializer component ['obj'] subtype.
@@ -3000,7 +2953,56 @@ definitions:
             enum:
             - obj
             type: string
-        title: ObjSerialize
+        title: yggdrasil.serialize.ObjSerialize.ObjSerialize
+        type: object
+      - additionalProperties: true
+        description: Schema for serializer component ['pandas'] subtype.
+        properties:
+          delimiter:
+            default: "\t"
+            description: Character(s) that should be used to separate columns. Defaults
+              to '\t'.
+            type: string
+          field_names:
+            description: The names of fields in the format string. If not provided,
+              names are set based on the order of the fields in the format string.
+            items:
+              type: string
+            type: array
+          field_units:
+            description: The units of fields in the format string. If not provided,
+              all fields are assumed to be dimensionless.
+            items:
+              type: string
+            type: array
+          format_str:
+            description: If provided, this string will be used to format messages
+              from a list of arguments and parse messages to get a list of arguments
+              in C printf/scanf style. Defaults to None and messages are assumed to
+              already be bytes.
+            type: string
+          no_header:
+            default: false
+            description: If True, headers will not be read or serialized from/to tables.
+              Defaults to False.
+            type: boolean
+          seritype:
+            default: default
+            description: Serializes tables using the pandas package.
+            enum:
+            - pandas
+            type: string
+          str_as_bytes:
+            default: false
+            description: If True, strings in columns are read as bytes. Defaults to
+              False.
+            type: boolean
+          use_astropy:
+            default: false
+            description: If True, the astropy package will be used to serialize/deserialize
+              table. Defaults to False.
+            type: boolean
+        title: yggdrasil.serialize.PandasSerialize.PandasSerialize
         type: object
       - additionalProperties: true
         description: Schema for serializer component ['pickle'] subtype.
@@ -3011,7 +3013,24 @@ definitions:
             enum:
             - pickle
             type: string
-        title: PickleSerialize
+        title: yggdrasil.serialize.PickleSerialize.PickleSerialize
+        type: object
+      - additionalProperties: true
+        description: Schema for serializer component ['wofost'] subtype.
+        properties:
+          delimiter:
+            default: ' = '
+            description: Delimiter that should be used to separate name/value pairs
+              in the map. Defaults to \t.
+            type: string
+          seritype:
+            default: default
+            description: Serialization of mapping between keys and scalar or array
+              values as used in the WOFOST parameter files.
+            enum:
+            - wofost
+            type: string
+        title: yggdrasil.serialize.WOFOSTParamSerialize.WOFOSTParamSerialize
         type: object
       - additionalProperties: true
         description: Schema for serializer component ['yaml'] subtype.
@@ -3040,24 +3059,7 @@ definitions:
             enum:
             - yaml
             type: string
-        title: YAMLSerialize
-        type: object
-      - additionalProperties: true
-        description: Schema for serializer component ['wofost'] subtype.
-        properties:
-          delimiter:
-            default: ' = '
-            description: Delimiter that should be used to separate name/value pairs
-              in the map. Defaults to \t.
-            type: string
-          seritype:
-            default: default
-            description: Serialization of mapping between keys and scalar or array
-              values as used in the WOFOST parameter files.
-            enum:
-            - wofost
-            type: string
-        title: WOFOSTParamSerialize
+        title: yggdrasil.serialize.YAMLSerialize.YAMLSerialize
         type: object
     description: Schema for serializer components.
     title: serializer
@@ -3142,7 +3144,7 @@ definitions:
           transformtype:
             enum:
             - array
-        title: ArrayTransform
+        title: yggdrasil.communication.transforms.ArrayTransform.ArrayTransform
         type: object
       - additionalProperties: true
         description: Schema for transform component ['pandas'] subtype.
@@ -3154,41 +3156,27 @@ definitions:
           transformtype:
             enum:
             - pandas
-        title: PandasTransform
+        title: yggdrasil.communication.transforms.PandasTransform.PandasTransform
         type: object
       - additionalProperties: true
-        description: Schema for transform component ['select_fields'] subtype.
+        description: Schema for transform component ['direct'] subtype.
         properties:
-          original_order:
-            description: The original order of fields that should be used for selecting
-              from lists/tuples.
-            items:
-              type: string
-            type: array
-          selected:
-            description: A list of fields that should be selected.
-            items:
-              type: string
-            type: array
-          single_as_scalar:
-            description: If True and only a single field is selected, the transformed
-              messages will be scalars rather than arrays with single elements. Defaults
-              to False.
-            type: boolean
           transformtype:
             enum:
-            - select_fields
-        required:
-        - selected
-        title: SelectFieldsTransform
+            - direct
+        title: yggdrasil.communication.transforms.DirectTransform.DirectTransform
         type: object
       - additionalProperties: true
-        description: Schema for transform component ['iterate'] subtype.
+        description: Schema for transform component ['filter'] subtype.
         properties:
+          filter:
+            $ref: '#/definitions/filter'
           transformtype:
             enum:
-            - iterate
-        title: IterateTransform
+            - filter
+        required:
+        - filter
+        title: yggdrasil.communication.transforms.FilterTransform.FilterTransform
         type: object
       - additionalProperties: true
         description: Schema for transform component ['function'] subtype.
@@ -3206,32 +3194,15 @@ definitions:
             - function
         required:
         - function
-        title: FunctionTransform
-        type: object
-      - additionalProperties: true
-        description: Schema for transform component ['statement'] subtype.
-        properties:
-          statement:
-            description: Python statement in terms of the message as represented by
-              the string "%x%" that should evaluate to the transformed message. The
-              statement should only use a limited set of builtins and the math library
-              (See yggdrasil.tools.safe_eval). If more complex relationships are required,
-              use the FunctionTransform class.
-            type: string
-          transformtype:
-            enum:
-            - statement
-        required:
-        - statement
-        title: StatementTransform
+        title: yggdrasil.communication.transforms.FunctionTransform.FunctionTransform
         type: object
       - additionalProperties: true
-        description: Schema for transform component ['direct'] subtype.
+        description: Schema for transform component ['iterate'] subtype.
         properties:
           transformtype:
             enum:
-            - direct
-        title: DirectTransform
+            - iterate
+        title: yggdrasil.communication.transforms.IterateTransform.IterateTransform
         type: object
       - additionalProperties: true
         description: Schema for transform component ['map_fields'] subtype.
@@ -3246,19 +3217,50 @@ definitions:
             - map_fields
         required:
         - map
-        title: MapFieldsTransform
+        title: yggdrasil.communication.transforms.MapFieldsTransform.MapFieldsTransform
         type: object
       - additionalProperties: true
-        description: Schema for transform component ['filter'] subtype.
+        description: Schema for transform component ['select_fields'] subtype.
         properties:
-          filter:
-            $ref: '#/definitions/filter'
+          original_order:
+            description: The original order of fields that should be used for selecting
+              from lists/tuples.
+            items:
+              type: string
+            type: array
+          selected:
+            description: A list of fields that should be selected.
+            items:
+              type: string
+            type: array
+          single_as_scalar:
+            description: If True and only a single field is selected, the transformed
+              messages will be scalars rather than arrays with single elements. Defaults
+              to False.
+            type: boolean
           transformtype:
             enum:
-            - filter
+            - select_fields
         required:
-        - filter
-        title: FilterTransform
+        - selected
+        title: yggdrasil.communication.transforms.SelectFieldsTransform.SelectFieldsTransform
+        type: object
+      - additionalProperties: true
+        description: Schema for transform component ['statement'] subtype.
+        properties:
+          statement:
+            description: Python statement in terms of the message as represented by
+              the string "%x%" that should evaluate to the transformed message. The
+              statement should only use a limited set of builtins and the math library
+              (See yggdrasil.tools.safe_eval). If more complex relationships are required,
+              use the FunctionTransform class.
+            type: string
+          transformtype:
+            enum:
+            - statement
+        required:
+        - statement
+        title: yggdrasil.communication.transforms.StatementTransform.StatementTransform
         type: object
     description: Schema for transform components.
     title: transform
diff --git a/yggdrasil/command_line.py b/yggdrasil/command_line.py
index 785dc1ff9..3a024008f 100755
--- a/yggdrasil/command_line.py
+++ b/yggdrasil/command_line.py
@@ -1291,7 +1291,8 @@ def func(cls, args):
                 os.remove(schema._schema_fname)
             schema.clear_schema()
             schema.init_schema()
-        schema.update_constants()
+        else:
+            schema.update_constants()
 
 
 class yggmodelform(SubCommand):
diff --git a/yggdrasil/components.py b/yggdrasil/components.py
index af24a3aec..8eecdddd4 100644
--- a/yggdrasil/components.py
+++ b/yggdrasil/components.py
@@ -4,39 +4,19 @@
 import six
 import inspect
 import importlib
-# import warnings
+import contextlib
 import weakref
 from collections import OrderedDict
 from yggdrasil.doctools import docs2args
 
 
 _registry = {}
-_registry_defaults = {}
-_registry_base_classes = {}
-_registry_class2subtype = {}
 _registry_complete = False
-_comptype2key = {'comm': 'commtype',
-                 'file': 'filetype',
-                 'model': 'language',
-                 'connection': 'connection_type',
-                 # 'datatype': None,
-                 'serializer': 'seritype',
-                 'filter': 'filtertype',
-                 'transform': 'transformtype'}
-# 'compiler': 'toolname',
-# 'linker': 'toolname',
-# 'archiver': 'toolname'}
-_comptype2mod = {'serializer': 'serialize',
-                 'comm': 'communication',
-                 'file': 'communication',
-                 'model': 'drivers',
-                 'connection': 'drivers',
-                 'filter': 'communication.filters',
-                 'transform': 'communication.transforms'}
-# 'datatype': ['metaschema', 'datatypes'],
-# 'compiler': 'drivers',
-# 'linker': 'drivers',
-# 'archiver': 'drivers'}
+
+
+class ComponentError(BaseException):
+    r"""Error raised when there is a problem import a component."""
+    pass
 
 
 class ClassRegistry(OrderedDict):
@@ -46,7 +26,6 @@ def __init__(self, *args, import_function=None, **kwargs):
         module = inspect.getmodule(inspect.stack()[1][0])
         self._module = module.__name__
         self._directory = os.path.dirname(module.__file__)
-        # self._schema_directory = os.path.join(self._directory, 'schemas')
         self._import_function = import_function
         self._imported = False
         super(ClassRegistry, self).__init__(*args, **kwargs)
@@ -98,52 +77,68 @@ def has_entry(self, key):
         return super(ClassRegistry, self).__contains__(key)
 
 
-def init_registry():
+def registration_in_progress():
+    r"""Determine if a registration is in progress."""
+    return bool(os.environ.get('YGGDRASIL_REGISTRATION_IN_PROGRESS', None))
+
+
+@contextlib.contextmanager
+def registering(recurse=False):
+    r"""Context for preforming registration."""
+    if not recurse:
+        assert(not registration_in_progress())
+    try:
+        os.environ['YGGDRASIL_REGISTRATION_IN_PROGRESS'] = '1'
+        yield
+    finally:
+        if 'YGGDRASIL_REGISTRATION_IN_PROGRESS' in os.environ:
+            del os.environ['YGGDRASIL_REGISTRATION_IN_PROGRESS']
+
+
+def init_registry(recurse=False):
     r"""Initialize the registries and schema."""
+    from yggdrasil.tools import import_all_modules
     global _registry
     global _registry_complete
-    if not _registry_complete:
-        comp_list = []
-        mod_list = []
-        for k, v in _comptype2mod.items():
-            if v not in mod_list:
-                comp_list.append(k)
-                mod_list.append(v)
-        for k in comp_list:
-            import_all_components(k)
+    with registering(recurse=recurse):
+        import_all_modules(exclude=['yggdrasil.examples',
+                                    'yggdrasil.languages',
+                                    'yggdrasil.interface',
+                                    'yggdrasil.timing'],
+                           do_first=['yggdrasil.serialize'])
         _registry_complete = True
     return _registry
 
 
-# This dosn't work as desried because classes that have already been imported
-# will not call registration on second import
-# def clear_registry():
-#     r"""Reset registries."""
-#     global _registry
-#     global _registry_defaults
-#     global _registry_class2subtype
-#     global _registry_complete
-#     _registry = {}
-#     _registry_defaults = {}
-#     _registry_class2subtype = {}
-#     _registry_complete = False
+def get_registry(comptype=None):
+    r"""Get the registry that should be used for looking up components.
+
+    Args:
+        comptype (str, optional): The name of a component to get the
+            registry for. Defaults to None and the entire registry will be
+            returned.
+
+    """
+    global _registry
+    if registration_in_progress():
+        out = _registry
+    else:
+        from yggdrasil import constants
+        out = constants.COMPONENT_REGISTRY
+    if comptype:
+        if comptype not in out:  # pragma: debug
+            raise Exception(f"Importing a component type that has not yet "
+                            f"been registered: {comptype}")
+        out = out[comptype]
+    return out
+
 
-    
 def suspend_registry():
     r"""Suspend the registry by storing the global registries in a dictionary."""
     global _registry
-    global _registry_defaults
-    global _registry_base_classes
-    global _registry_class2subtype
     global _registry_complete
-    out = {'_registry': _registry, '_registry_defaults': _registry_defaults,
-           '_registry_base_classes': _registry_base_classes,
-           '_registry_class2subtype': _registry_class2subtype,
-           '_registry_complete': _registry_complete}
+    out = {'_registry': _registry, '_registry_complete': _registry_complete}
     _registry = {}
-    _registry_defaults = {}
-    _registry_base_classes = {}
-    _registry_class2subtype = {}
     _registry_complete = False
     return out
 
@@ -151,59 +146,21 @@ def suspend_registry():
 def restore_registry(reg_dict):
     r"""Restore the registry to values in the provided dictionary."""
     global _registry
-    global _registry_defaults
-    global _registry_base_classes
-    global _registry_class2subtype
     global _registry_complete
     _registry = reg_dict['_registry']
-    _registry_defaults = reg_dict['_registry_defaults']
-    _registry_base_classes = reg_dict['_registry_base_classes']
-    _registry_class2subtype = reg_dict['_registry_class2subtype']
     _registry_complete = reg_dict['_registry_complete']
 
 
-def import_all_components(comptype):
-    r"""Dynamically import all component classes for a component type.
-
-    Args:
-        comptype (str): Component type.
-
-    """
-    # Get module and directory
-    mod = copy.deepcopy(_comptype2mod[comptype])
-    moddir = os.path.join(*copy.deepcopy(_comptype2mod[comptype]).split('.'))
-    # The next three lines will be required if there are ever any components
-    # nested in multiple directories (e.g. metaschema/datatypes)
-    # if isinstance(mod, list):
-    #     mod = '.'.join(mod)
-    #     moddir = os.path.join(*moddir)
-    moddir = os.path.join(os.path.dirname(__file__), moddir)
-    modbase = importlib.import_module('yggdrasil.%s' % mod)
-    non_comp = [os.path.splitext(x)[0] for x in
-                getattr(modbase, '_non_component_modules', [])]
-    # Import all files
-    for x in glob.glob(os.path.join(moddir, '*.py')):
-        xbase = os.path.splitext(os.path.basename(x))[0]
-        if (not xbase.startswith('__')) and (xbase not in non_comp):
-            importlib.import_module('yggdrasil.%s.%s'
-                                    % (mod, xbase))
-
-
-def import_component(comptype, subtype=None, without_schema=False,
-                     **kwargs):
+def import_component(comptype, subtype=None, **kwargs):
     r"""Dynamically import a component by name.
 
     Args:
         comptype (str): Component type.
-        subtype (str, optional): Component subtype. If subtype is not one of the
-            registered subtypes for the specified comptype, subtype is treated
-            as the name of class. Defaults to None if not provided and the
-            default subtype defined in the schema for the specified component
-            will be used.
-        without_schema (bool, optional): If True, the schema is not used to
-            import the component and subtype must be the name of a component
-            class. Defaults to False. subtype must be provided if without_schema
-            is True.
+        subtype (str, optional): Component subtype. If subtype is not one of
+            the registered subtypes for the specified comptype, subtype is
+            treated as the name of class. Defaults to None if not provided and
+            the default subtype defined in the schema for the specified
+            component will be used.
         **kwargs: Additional keyword arguments are used to determine the
             subtype if it is None.
 
@@ -211,70 +168,39 @@ def import_component(comptype, subtype=None, without_schema=False,
         class: Component class.
 
     Raises:
-        ValueError: If subtype is not provided, but without_schema is True.
         ValueError: If comptype is not a registered component type.
         ValueError: If subtype is not a registered subtype or the name of a
             registered subtype class for the specified comptype.
             
 
     """
-    # Get module
-    mod = _comptype2mod[comptype]
-    # if isinstance(mod, list):
-    #     mod = '.'.join(mod)
-    # Set direct import shortcuts for unregistered classes
-    if (((subtype is None) and (comptype in _comptype2key)
-         and (_comptype2key[comptype] in kwargs))):
-        subtype = kwargs[_comptype2key[comptype]]
+    registry = get_registry(comptype=comptype)
+    if subtype is None:
+        subtype = kwargs.get(registry["key"], None)
     if (comptype == 'comm') and (subtype is None):
         subtype = 'DefaultComm'
-    if (((comptype in ['comm', 'file']) and (subtype is not None)
-         and ((subtype == 'CommBase') or subtype.endswith('Comm')))):
-        without_schema = True
-    # Set default based on registry to avoid schema if possible
-    if (subtype is None) and (comptype in _registry_defaults):
-        subtype = _registry_defaults.get(comptype, None)
-    # Check registered components to prevent importing multiple times
-    if subtype in _registry.get(comptype, {}):
-        out_cls = _registry[comptype][subtype]
-    elif subtype in _registry_class2subtype.get(comptype, {}):
-        out_cls = _registry[comptype][_registry_class2subtype[comptype][subtype]]
+    if subtype is None:
+        subtype = registry["default"]
+    if subtype in registry["subtypes"]:
+        class_name = registry["subtypes"][subtype]
     else:
-        # Get class name
-        if without_schema:
-            if subtype is None:  # pragma: debug
-                raise ValueError("subtype must be provided if without_schema is True.")
-            class_name = subtype
-        else:
-            from yggdrasil.schema import get_schema
-            s = get_schema().get(comptype, None)
-            if s is None:  # pragma: debug
-                raise ValueError("Unrecognized component type: %s" % comptype)
-            if subtype is None:  # pragma: no cover
-                # This will only be called if the test is run before the component
-                # module is imported
-                subtype = s.default_subtype
-            if subtype in s.class2subtype:
-                class_name = subtype
-            else:
-                class_name = s.subtype2class.get(subtype, None)
-                if class_name is None:
-                    # Attempt file since they are subclass of comm
-                    if (comptype == 'comm'):
-                        try:
-                            return import_component('file', subtype)
-                        except ValueError:
-                            pass
-                    raise ValueError("Unrecognized %s subtype: %s"
-                                     % (comptype, subtype))
+        class_name = subtype
+    # Check registered components to prevent importing multiple times
+    if class_name not in registry.get("classes", {}):
+        registry.setdefault("classes", {})
         try:
-            out_mod = importlib.import_module('yggdrasil.%s.%s' % (mod, class_name))
-        except ImportError:  # pragma: debug
-            import_all_components(comptype)
-            return import_component(comptype, subtype=subtype,
-                                    without_schema=without_schema,
-                                    **kwargs)
-        out_cls = getattr(out_mod, class_name)
+            registry["classes"][class_name] = getattr(
+                importlib.import_module(f"{registry['module']}.{class_name}"),
+                class_name)
+        except ImportError:
+            if comptype == 'comm':
+                try:
+                    return import_component('file', subtype, **kwargs)
+                except ComponentError:
+                    pass
+            raise ComponentError(f"Could not locate a {comptype} component "
+                                 f"{subtype}.")
+    out_cls = registry["classes"][class_name]
     # Check for an aliased class
     if hasattr(out_cls, '_get_alias'):
         out_cls = out_cls._get_alias()
@@ -318,18 +244,13 @@ def create_component(comptype, subtype=None, **kwargs):
     return cls(**kwargs)
 
 
-def get_component_base_class(comptype, subtype=None, without_schema=False,
-                             **kwargs):
+def get_component_base_class(comptype, subtype=None, **kwargs):
     r"""Determine the base class for a component type.
 
     Args:
         comptype (str): The name of a component to test against.
         subtype (str, optional): Subtype to use to determine the component
             base class. Defaults to None.
-        without_schema (bool, optional): If True, the schema is not used to
-            import the component and subtype must be the name of a component
-            class. Defaults to False. subtype must be provided if without_schema
-            is True.
         **kwargs: Additional keyword arguments are used to determine the
             subtype if it is None.
     
@@ -337,33 +258,20 @@ def get_component_base_class(comptype, subtype=None, without_schema=False,
         ComponentBase: Component base class.
 
     """
-    if comptype in _registry_base_classes:
-        base_class_name = _registry_base_classes[comptype]
-    else:  # pragma: no cover
-        # Only called during initial import which is not covered by
-        # the test runner
-        default_class = import_component(comptype, subtype=subtype,
-                                         without_schema=without_schema,
-                                         **kwargs)
-        base_class_name = default_class._schema_base_class
-    base_class = import_component(comptype, subtype=base_class_name,
-                                  without_schema=True)
-    return base_class
-
-
-def isinstance_component(x, comptype, subtype=None, without_schema=False,
-                         **kwargs):
+    registry = get_registry(comptype=comptype)
+    base_class_name = registry['base']
+    return import_component(comptype, subtype=base_class_name, **kwargs)
+
+
+def isinstance_component(x, comptype, subtype=None, **kwargs):
     r"""Determine if an object is an instance of a component type.
 
     Args:
         x (object): Object to test.
-        comptype (str, list): The name of one or more components to test against.
+        comptype (str, list): The name of one or more components to test
+            against.
         subtype (str, optional): Subtype to use to determine the component
             base class. Defaults to None.
-        without_schema (bool, optional): If True, the schema is not used to
-            import the component and subtype must be the name of a component
-            class. Defaults to False. subtype must be provided if without_schema
-            is True.
         **kwargs: Additional keyword arguments are used to determine the
             subtype if it is None.
 
@@ -378,7 +286,6 @@ def isinstance_component(x, comptype, subtype=None, without_schema=False,
         else:
             return False
     base_class = get_component_base_class(comptype, subtype=subtype,
-                                          without_schema=without_schema,
                                           **kwargs)
     return isinstance(x, base_class)
 
@@ -474,12 +381,13 @@ def __new__(meta, name, bases, class_dict):
                 args_dict = docs2args(x.__doc__)
                 for k, v in cls._schema_properties.items():
                     if k in args_dict:
-                        v.setdefault('description', args_dict[k]['description'])
+                        v.setdefault('description',
+                                     args_dict[k]['description'])
             # Determine base class
-            global _registry_base_classes
             if cls._schema_base_class is None:
-                if cls._schema_type in _registry_base_classes:
-                    cls._schema_base_class = _registry_base_classes[cls._schema_type]
+                reg = get_registry()
+                if cls._schema_type in reg:
+                    cls._schema_base_class = reg[cls._schema_type]['base']
                 else:
                     base_comp = cls.__name__
                     for i, x in enumerate(cls.__mro__):
@@ -487,29 +395,30 @@ def __new__(meta, name, bases, class_dict):
                             break
                         base_comp = x.__name__
                     else:  # pragma: debug
-                        raise RuntimeError(("Could not determine base class for %s "
-                                            "from %s.") % (cls, bases))
+                        raise RuntimeError(
+                            f"Could not determine base class for {cls} "
+                            f"from {bases}.")
                     cls._schema_base_class = base_comp
             # Register
             global _registry
-            global _registry_defaults
-            global _registry_class2subtype
             yaml_typ = cls._schema_type
             default_subtype = cls._schema_properties.get(
                 cls._schema_subtype_key, {}).get('default',
                                                  cls._schema_subtype_default)
             if yaml_typ not in _registry:
-                _registry[yaml_typ] = OrderedDict()
-                _registry_defaults[yaml_typ] = default_subtype
-                _registry_base_classes[yaml_typ] = cls._schema_base_class
-                _registry_class2subtype[yaml_typ] = {}
+                _registry[yaml_typ] = OrderedDict([
+                    ("classes", OrderedDict()),
+                    ("module", '.'.join(cls.__module__.split('.')[:-1])),
+                    ("default", default_subtype),
+                    ("base", cls._schema_base_class),
+                    ("key", cls._schema_subtype_key),
+                    ("subtypes", {})])
             elif default_subtype is not None:
-                assert(_registry_defaults[yaml_typ] == default_subtype)
-            if cls.__name__ not in _registry[yaml_typ]:
-                _registry[yaml_typ][cls.__name__] = cls
-                _registry_class2subtype[yaml_typ][subtype] = cls.__name__
-            if not (os.environ.get('YGG_RUNNING_YGGSCHEMA', 'None').lower()
-                    in ['true', '1']):
+                assert(_registry[yaml_typ]["default"] == default_subtype)
+            if cls.__name__ not in _registry[yaml_typ]["classes"]:
+                _registry[yaml_typ]["classes"][cls.__name__] = cls
+                _registry[yaml_typ]["subtypes"][subtype] = cls.__name__
+            if not registration_in_progress():
                 cls.after_registration(cls)
                 cls.finalize_registration(cls)
         return cls
@@ -689,23 +598,30 @@ def __init__(self, skip_component_schema_normalization=None, **kwargs):
             #                   % (k, v, getattr(self, k)))
         self.extra_kwargs = kwargs
 
+    @staticmethod
+    def before_schema(cls):
+        r"""Operations that should be performed before the schema is generated.
+        These actions will only be performed if a new schema is being generated.
+        """
+        pass
+
     @staticmethod
     def before_registration(cls):
         r"""Operations that should be performed to modify class attributes prior
         to registration. These actions will still be performed if the environment
-        variable YGG_RUNNING_YGGSCHEMA is set."""
+        variable YGGDRASIL_REGISTRATION_IN_PROGRESS is set."""
         pass
 
     @staticmethod
     def after_registration(cls):
         r"""Operations that should be preformed to modify class attributes after
         registration. These actions will not be performed if the environment
-        variable YGG_RUNNING_YGGSCHEMA is set."""
+        variable YGGDRASIL_REGISTRATION_IN_PROGRESS is set."""
         pass
 
     @staticmethod
     def finalize_registration(cls):
         r"""Final operations to perform after a class has been fully initialized.
         These actions will not be performed if the environment variable
-        YGG_RUNNING_YGGSCHEMA is set."""
+        YGGDRASIL_REGISTRATION_IN_PROGRESS is set."""
         pass
diff --git a/yggdrasil/constants.py b/yggdrasil/constants.py
index 1a8551cb9..67dab568d 100644
--- a/yggdrasil/constants.py
+++ b/yggdrasil/constants.py
@@ -6,6 +6,140 @@
 # is generated by yggdrasil.schema.update_constants
 # ======================================================
 
+
+# Component registry
+COMPONENT_REGISTRY = {
+    'comm': {
+        'base': 'CommBase',
+        'default': 'default',
+        'key': 'commtype',
+        'module': 'yggdrasil.communication',
+        'subtypes': {
+            'buffer': 'BufferComm',
+            'default': 'DefaultComm',
+            'ipc': 'IPCComm',
+            'mpi': 'MPIComm',
+            'rest': 'RESTComm',
+            'rmq': 'RMQComm',
+            'rmq_async': 'RMQAsyncComm',
+            'value': 'ValueComm',
+            'zmq': 'ZMQComm',
+        },
+    },
+    'connection': {
+        'base': 'ConnectionDriver',
+        'default': None,
+        'key': 'connection_type',
+        'module': 'yggdrasil.drivers',
+        'subtypes': {
+            'connection': 'ConnectionDriver',
+            'file_input': 'FileInputDriver',
+            'file_output': 'FileOutputDriver',
+            'input': 'InputDriver',
+            'output': 'OutputDriver',
+            'rpc_request': 'RPCRequestDriver',
+            'rpc_response': 'RPCResponseDriver',
+        },
+    },
+    'file': {
+        'base': 'FileComm',
+        'default': 'binary',
+        'key': 'filetype',
+        'module': 'yggdrasil.communication',
+        'subtypes': {
+            'ascii': 'AsciiFileComm',
+            'binary': 'FileComm',
+            'json': 'JSONFileComm',
+            'map': 'AsciiMapComm',
+            'mat': 'MatFileComm',
+            'netcdf': 'NetCDFFileComm',
+            'obj': 'ObjFileComm',
+            'pandas': 'PandasFileComm',
+            'pickle': 'PickleFileComm',
+            'ply': 'PlyFileComm',
+            'table': 'AsciiTableComm',
+            'wofost': 'WOFOSTParamFileComm',
+            'yaml': 'YAMLFileComm',
+        },
+    },
+    'filter': {
+        'base': 'FilterBase',
+        'default': None,
+        'key': 'filtertype',
+        'module': 'yggdrasil.communication.filters',
+        'subtypes': {
+            'direct': 'DirectFilter',
+            'function': 'FunctionFilter',
+            'statement': 'StatementFilter',
+        },
+    },
+    'model': {
+        'base': 'ModelDriver',
+        'default': 'executable',
+        'key': 'language',
+        'module': 'yggdrasil.drivers',
+        'subtypes': {
+            'R': 'RModelDriver',
+            'c': 'CModelDriver',
+            'c++': 'CPPModelDriver',
+            'cmake': 'CMakeModelDriver',
+            'cpp': 'CPPModelDriver',
+            'cxx': 'CPPModelDriver',
+            'dummy': 'DummyModelDriver',
+            'executable': 'ExecutableModelDriver',
+            'fortran': 'FortranModelDriver',
+            'lpy': 'LPyModelDriver',
+            'make': 'MakeModelDriver',
+            'matlab': 'MatlabModelDriver',
+            'mpi': 'MPIPartnerModel',
+            'osr': 'OSRModelDriver',
+            'python': 'PythonModelDriver',
+            'r': 'RModelDriver',
+            'sbml': 'SBMLModelDriver',
+            'timesync': 'TimeSyncModelDriver',
+        },
+    },
+    'serializer': {
+        'base': 'SerializeBase',
+        'default': 'default',
+        'key': 'seritype',
+        'module': 'yggdrasil.serialize',
+        'subtypes': {
+            'default': 'DefaultSerialize',
+            'direct': 'DirectSerialize',
+            'functional': 'FunctionalSerialize',
+            'json': 'JSONSerialize',
+            'map': 'AsciiMapSerialize',
+            'mat': 'MatSerialize',
+            'obj': 'ObjSerialize',
+            'pandas': 'PandasSerialize',
+            'pickle': 'PickleSerialize',
+            'ply': 'PlySerialize',
+            'table': 'AsciiTableSerialize',
+            'wofost': 'WOFOSTParamSerialize',
+            'yaml': 'YAMLSerialize',
+        },
+    },
+    'transform': {
+        'base': 'TransformBase',
+        'default': None,
+        'key': 'transformtype',
+        'module': 'yggdrasil.communication.transforms',
+        'subtypes': {
+            'array': 'ArrayTransform',
+            'direct': 'DirectTransform',
+            'filter': 'FilterTransform',
+            'function': 'FunctionTransform',
+            'iterate': 'IterateTransform',
+            'map_fields': 'MapFieldsTransform',
+            'pandas': 'PandasTransform',
+            'select_fields': 'SelectFieldsTransform',
+            'statement': 'StatementTransform',
+        },
+    },
+}
+
+# Language driver constants
 LANG2EXT = {
     'R': '.R',
     'c': '.c',
diff --git a/yggdrasil/drivers/ModelDriver.py b/yggdrasil/drivers/ModelDriver.py
index 305ce2584..9dd29ed7a 100755
--- a/yggdrasil/drivers/ModelDriver.py
+++ b/yggdrasil/drivers/ModelDriver.py
@@ -12,7 +12,7 @@
 from collections import OrderedDict
 from pprint import pformat
 from yggdrasil import platform, tools, languages, multitasking
-from yggdrasil.components import import_component, import_all_components
+from yggdrasil.components import import_component
 from yggdrasil.drivers.Driver import Driver
 from yggdrasil.metaschema.datatypes import is_default_typedef
 from yggdrasil.metaschema.properties.ScalarMetaschemaProperties import (
@@ -45,7 +45,7 @@ def remove_product(product, check_for_source=False, **kwargs):
         RuntimeError: If the product cannot be removed.
 
     """
-    import_all_components('model')
+    tools.import_all_modules('yggdrasil.drivers')
     source_keys = list(_map_language_ext.keys())
     if '.exe' in source_keys:  # pragma: windows
         source_keys.remove('.exe')
diff --git a/yggdrasil/schema.py b/yggdrasil/schema.py
index 8ba42b300..04448b527 100644
--- a/yggdrasil/schema.py
+++ b/yggdrasil/schema.py
@@ -3,6 +3,7 @@
 import pprint
 import yaml
 import json
+import importlib
 from collections import OrderedDict
 from jsonschema.exceptions import ValidationError
 from yggdrasil import metaschema
@@ -70,16 +71,10 @@ def init_schema(fname=None):
 
 def create_schema():
     r"""Create a new schema from the registry."""
-    from yggdrasil.components import init_registry
-    try:
-        old_env = os.environ.get('YGG_RUNNING_YGGSCHEMA', None)
-        os.environ['YGG_RUNNING_YGGSCHEMA'] = '1'
-        x = SchemaRegistry(init_registry())
-    finally:
-        if old_env is None:
-            del os.environ['YGG_RUNNING_YGGSCHEMA']
-        else:  # pragma: no cover
-            os.environ['YGG_RUNNING_YGGSCHEMA'] = old_env
+    from yggdrasil.components import init_registry, registering
+    with registering():
+        x = SchemaRegistry(init_registry(recurse=True))
+        update_constants(x)
     return x
 
 
@@ -222,11 +217,13 @@ def get_model_form_schema(fname_dst=None, **kwargs):
     return out
 
 
-def update_constants():
+def update_constants(schema=None):
     r"""Update constants.py with info from the schema."""
     from yggdrasil.components import import_component
     from yggdrasil.drivers.CompiledModelDriver import (
         get_compilation_tool_registry)
+    if schema is None:
+        schema = get_schema()
 
     def as_lines(x, newline='\n', key_order=None):
         out = ""
@@ -253,8 +250,18 @@ def as_lines(x, newline='\n', key_order=None):
         "# is generated by yggdrasil.schema.update_constants\n"
         "# ======================================================\n")
     filename = os.path.join(os.path.dirname(__file__), 'constants.py')
-    s = get_schema()
-    drivers = {k: import_component('model', k) for k in s['model'].subtypes}
+    # Component information
+    component_registry = {}
+    for k, v in schema.items():
+        component_registry[k] = {
+            'module': v.module,
+            'default': v.default_subtype,
+            'base': v.base_subtype_class_name,
+            'key': v.subtype_key,
+            'subtypes': v.subtype2class}
+    # Language driver information
+    drivers = {k: import_component('model', k)
+               for k in schema['model'].subtypes}
     language_cat = ['compiled', 'interpreted', 'build', 'dsl', 'other']
     typemap = {'compiler': 'compiled', 'interpreter': 'interpreted'}
     lang2ext = {'yaml': '.yml', 'executable': '.exe'}
@@ -295,6 +302,10 @@ def as_lines(x, newline='\n', key_order=None):
     with open(filename, 'r') as fd:
         lines = [fd.read().split(separator)[0], separator[1:]]
     lines += [
+        "", "# Component registry",
+        f"COMPONENT_REGISTRY = {as_lines(component_registry)}"]
+    lines += [
+        "", "# Language driver constants",
         "LANG2EXT = %s" % as_lines(lang2ext),
         "EXT2LANG = {v: k for k, v in LANG2EXT.items()}",
         "LANGUAGES = %s" % as_lines(languages, key_order=language_cat)]
@@ -342,19 +353,18 @@ class ComponentSchema(object):
             associated values of the subtype_key property for this component.
 
     """
-    # _subtype_keys = {'model': 'language', 'comm': 'commtype', 'file': 'filetype',
-    #                  'connection': 'connection_type'}
 
-    def __init__(self, schema_type, subtype_key,
-                 schema_registry=None, schema_subtypes=None):
+    def __init__(self, schema_type, subtype_key, schema_registry=None,
+                 module=None, schema_subtypes=None):
         self._storage = SchemaDict()
         self._base_schema = None
-        self.schema_registry = schema_registry
         self.schema_type = schema_type
         self.subtype_key = subtype_key
+        self.schema_registry = schema_registry
         if schema_subtypes is None:
             schema_subtypes = {}
         self.schema_subtypes = schema_subtypes
+        self.module = module
         super(ComponentSchema, self).__init__()
 
     def identify_subtype(self, doc):
@@ -535,9 +545,15 @@ def from_schema(cls, schema, schema_registry=None):
         out = cls(schema_type, subtype_key, schema_registry=schema_registry)
         out._base_schema = schema['allOf'][0]
         for v in subt_schema:
-            out._storage[v['title']] = v
+            v_class_name = v['title'].split('.')[-1]
+            out._storage[v_class_name] = v
             subtypes = v['properties'][out.subtype_key]['enum']
-            out.schema_subtypes[v['title']] = subtypes
+            out.schema_subtypes[v_class_name] = subtypes
+            v_module = '.'.join(v['title'].split('.')[:-2])
+            if out.module is None:
+                out.module = v_module
+            else:
+                assert(v_module == out.module)
         # Remove subtype specific properties
         for k in subt_props:
             if k != out.subtype_key:
@@ -552,12 +568,12 @@ def from_schema(cls, schema, schema_registry=None):
         return out
 
     @classmethod
-    def from_registry(cls, schema_type, schema_classes, **kwargs):
+    def from_registry(cls, schema_type, registry, **kwargs):
         r"""Construct a ComponentSchema from a registry entry.
 
         Args:
             schema_type (str): Name of component type to build.
-            schema_classes (list): List of classes for the component type.
+            registry (dict): Registry information for the component.
             **kwargs: Additional keyword arguments are passed to the class
                 __init__ method.
 
@@ -565,10 +581,15 @@ def from_registry(cls, schema_type, schema_classes, **kwargs):
             ComponentSchema: Schema with information from classes.
 
         """
-        out = None
-        for x in schema_classes.values():
-            if out is None:
-                out = cls(schema_type, x._schema_subtype_key, **kwargs)
+        schema_subtypes = {}
+        for k, v in registry['subtypes'].items():
+            if v not in schema_subtypes:
+                schema_subtypes[v] = []
+            schema_subtypes[v].append(k)
+        kwargs.update(module=registry['module'],
+                      schema_subtypes=schema_subtypes)
+        out = cls(schema_type, registry['key'], **kwargs)
+        for x in registry['classes'].values():
             out.append(x, verify=True)
         return out
 
@@ -603,14 +624,38 @@ def subtype2class(self):
                 out[iv] = k
         return out
 
+    @property
+    def base_subtype_class_name(self):
+        r"""str: Name of base class for the subtype."""
+        if not getattr(self, '_base_subtype_class_name', None):
+            self.base_subtype_class
+        return self._base_subtype_class_name
+            
     @property
     def base_subtype_class(self):
         r"""ComponentClass: Base class for the subtype."""
         if not getattr(self, '_base_subtype_class', None):
-            from yggdrasil.components import get_component_base_class
-            keys = list(self.subtype2class.values())
-            self._base_subtype_class = get_component_base_class(
-                self.schema_type, subtype=keys[0], without_schema=True)
+            if getattr(self, '_base_subtype_class_name', None):
+                self._base_subtype_class = getattr(
+                    importlib.import_module(
+                        f"{self.module}.{self._base_subtype_class_name}"),
+                    self._base_subtype_class_name)
+            else:
+                default_class = list(self.schema_subtypes.keys())[0]
+                cls = getattr(
+                    importlib.import_module(f"{self.module}.{default_class}"),
+                    default_class)
+                base_class = cls
+                for i, x in enumerate(cls.__mro__):
+                    if x._schema_type != cls._schema_type:
+                        break
+                    base_class = x
+                else:  # pragma: debug
+                    raise RuntimeError(
+                        f"Could not determine a base class for "
+                        f"{self.schema_type} (using class {cls})")
+                self._base_subtype_class = base_class
+                self._base_subtype_class_name = base_class.__name__
         return self._base_subtype_class
 
     @property
@@ -644,15 +689,20 @@ def append(self, comp_cls, verify=False):
         assert(comp_cls._schema_type == self.schema_type)
         assert(comp_cls._schema_subtype_key == self.subtype_key)
         name = comp_cls.__name__
-        # name = '%s.%s' % (comp_cls.__module__, comp_cls.__name__)
+        fullname = f'{comp_cls.__module__}.{comp_cls.__name__}'
+        subtype_module = '.'.join(comp_cls.__module__.split('.')[:-1])
         # Append subtype
         subtype_list = getattr(comp_cls, '_%s' % self.subtype_key, None)
         if not isinstance(subtype_list, list):
             subtype_list = [subtype_list]
         subtype_list += getattr(comp_cls, '_%s_aliases' % self.subtype_key, [])
         self.schema_subtypes[name] = subtype_list
+        if self.module is None:
+            self.module = subtype_module
+        else:
+            assert(subtype_module == self.module)
         # Create new schema for subtype
-        new_schema = {'title': name,
+        new_schema = {'title': fullname,
                       'description': ('Schema for %s component %s subtype.'
                                       % (self.schema_type, subtype_list)),
                       'type': 'object',
@@ -1067,6 +1117,9 @@ def __getitem__(self, k):
     def keys(self):
         return self._storage.keys()
 
+    def items(self):
+        return self._storage.items()
+
     def __eq__(self, other):
         if not hasattr(other, 'schema'):
             return False
diff --git a/yggdrasil/tests/test_components.py b/yggdrasil/tests/test_components.py
index 477186d56..bc4358302 100644
--- a/yggdrasil/tests/test_components.py
+++ b/yggdrasil/tests/test_components.py
@@ -1,4 +1,4 @@
-from yggdrasil.tests import assert_raises
+import pytest
 from yggdrasil import components
 
 
@@ -16,7 +16,8 @@ def test_import_component():
     components.import_component('serializer', 'PandasSerialize')
     # Test access to file through comm (including error)
     components.import_component('comm', 'pickle')
-    assert_raises(ValueError, components.import_component, 'comm', 'invalid')
+    with pytest.raises(components.ComponentError):
+        components.import_component('comm', 'invalid')
     # Tests with registry suspended
     out = components.suspend_registry()
     components.import_component('serializer')
diff --git a/yggdrasil/tools.py b/yggdrasil/tools.py
index eb4ff4f01..356651e21 100644
--- a/yggdrasil/tools.py
+++ b/yggdrasil/tools.py
@@ -6,6 +6,7 @@
 import os
 import re
 import sys
+import glob
 import sysconfig
 try:
     from distutils import sysconfig as distutils_sysconfig
@@ -1253,6 +1254,53 @@ def print_encoded(msg, *args, **kwargs):
             print(str2bytes(msg), *args, **kwargs)
 
 
+def import_all_modules(base=None, exclude=None, do_first=None):
+    r"""Import all yggdrasil modules.
+
+    Args:
+        base (str, optional): Base module to start from. Defaults to
+            'yggdrasil'.
+        exclude (list, optional): Modules that should not be imported.
+            Defaults to empty list.
+        do_first (list, optional): Modules that should be import first.
+            Defaults to empty list.
+
+    """
+    if base is None:
+        base = 'yggdrasil'
+    if exclude is None:
+        exclude = []
+    if do_first is None:
+        do_first = []
+    assert(base.startswith('yggdrasil'))
+    for x in do_first:
+        import_all_modules(x, exclude=exclude)
+    exclude = exclude + do_first
+    directory = os.path.dirname(__file__)
+    parts = base.split('.')[1:]
+    if parts:
+        directory = os.path.join(directory, *parts)
+    if not os.path.isfile(os.path.join(directory, '__init__.py')):
+        return
+    if (base in exclude) or base.endswith('tests'):
+        return
+    importlib.import_module(base)
+    for x in sorted(glob.glob(os.path.join(directory, '*.py'))):
+        x_base = os.path.basename(x)
+        if x_base.startswith('__') and x_base.endswith('__.py'):
+            continue
+        x_mod = f"{base}.{os.path.splitext(os.path.basename(x))[0]}"
+        if x_mod in exclude:
+            continue
+        importlib.import_module(x_mod)
+    for x in sorted(glob.glob(os.path.join(directory, '*', ''))):
+        if x.startswith('__') and x.endswith('__'):
+            continue
+        next_module = os.path.basename(os.path.dirname(x))
+        import_all_modules(f"{base}.{next_module}",
+                           exclude=exclude + do_first)
+
+
 class TimeOut(object):
     r"""Class for checking if a period of time has been elapsed.
 

From 957caab63b7e339ddb8e52d233b7e8ca26cf7d56 Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Fri, 1 Oct 2021 10:16:57 -0700
Subject: [PATCH 41/73] Do not import serializer by default for communicators
 at import

---
 yggdrasil/communication/CommBase.py | 24 +++---------------------
 yggdrasil/communication/FileComm.py | 15 ++++++++++++---
 2 files changed, 15 insertions(+), 24 deletions(-)

diff --git a/yggdrasil/communication/CommBase.py b/yggdrasil/communication/CommBase.py
index e8269ef04..1c56e2514 100755
--- a/yggdrasil/communication/CommBase.py
+++ b/yggdrasil/communication/CommBase.py
@@ -576,7 +576,6 @@ class CommBase(tools.YggClass):
         'for_service': {'type': 'boolean', 'default': False}}
     _schema_excluded_from_class = ['name']
     _default_serializer = 'default'
-    _default_serializer_class = None
     _schema_excluded_from_class_validation = ['datatype']
     is_file = False
     _maxMsgSize = 0
@@ -751,12 +750,8 @@ def _init_before_open(self, **kwargs):
         if self.serializer is None:
             # Get serializer class
             if seri_cls is None:
-                if (((seri_kws['seritype'] == self._default_serializer)
-                     and (self._default_serializer_class is not None))):
-                    seri_cls = self._default_serializer_class
-                else:
-                    seri_cls = import_component('serializer',
-                                                subtype=seri_kws['seritype'])
+                seri_cls = import_component('serializer',
+                                            subtype=seri_kws['seritype'])
             # Recover keyword arguments for serializer passed to comm class
             for k in seri_cls.seri_kws():
                 if k in kwargs:
@@ -811,15 +806,6 @@ def _init_before_open(self, **kwargs):
                               subtype=filter_schema.identify_subtype(self.filter))
             self.filter = create_component('filter', **filter_kws)
 
-    @staticmethod
-    def before_registration(cls):
-        r"""Operations that should be performed to modify class attributes prior
-        to registration."""
-        tools.YggClass.before_registration(cls)
-        cls._default_serializer_class = import_component('serializer',
-                                                         cls._default_serializer,
-                                                         without_schema=True)
-
     @classmethod
     def get_testing_options(cls, serializer=None, **kwargs):
         r"""Method to return a dictionary of testing options for this class.
@@ -842,11 +828,7 @@ def get_testing_options(cls, serializer=None, **kwargs):
         """
         if serializer is None:
             serializer = cls._default_serializer
-        if (((serializer == cls._default_serializer)
-             and (cls._default_serializer_class is not None))):
-            seri_cls = cls._default_serializer_class
-        else:
-            seri_cls = import_component('serializer', serializer)
+        seri_cls = import_component('serializer', serializer)
         out_seri = seri_cls.get_testing_options(**kwargs)
         out = {'attributes': ['name', 'address', 'direction',
                               'serializer', 'recv_timeout',
diff --git a/yggdrasil/communication/FileComm.py b/yggdrasil/communication/FileComm.py
index 995a5186b..d1517872d 100755
--- a/yggdrasil/communication/FileComm.py
+++ b/yggdrasil/communication/FileComm.py
@@ -4,6 +4,7 @@
 from yggdrasil import platform, tools
 from yggdrasil.serialize.SerializeBase import SerializeBase
 from yggdrasil.communication import CommBase
+from yggdrasil.components import import_component, registration_in_progress
 
 
 class FileComm(CommBase.CommBase):
@@ -139,9 +140,17 @@ def before_registration(cls):
         # Add serializer properties to schema
         if cls._filetype != 'binary':
             assert('serializer' not in cls._schema_properties)
-            cls._schema_properties.update(
-                cls._default_serializer_class._schema_properties)
-            del cls._schema_properties['seritype']
+            if registration_in_progress():
+                seri = import_component('serializer', cls._default_serializer)
+                new = seri._schema_properties
+            else:
+                from yggdrasil.schema import get_schema
+                schema = get_schema()
+                new = schema['file'].get_subtype_schema(
+                    cls._default_serializer)['properties']
+            cls._schema_properties.update(new)
+            for k in ['driver', 'args', 'seritype']:
+                cls._schema_properties.pop(k, None)
         cls._commtype = cls._filetype
 
     @classmethod

From 9420943d5b19da6e91ef1eae6fa85203d2ecfa35 Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Fri, 1 Oct 2021 13:02:40 -0700
Subject: [PATCH 42/73] Move data types, serialization, and communication
 constants into constants.py

---
 yggdrasil/communication/CommBase.py           |  18 ++--
 yggdrasil/communication/FileComm.py           |  20 ++--
 yggdrasil/communication/ServerComm.py         |   3 +-
 .../transforms/ArrayTransform.py              |  13 ++-
 yggdrasil/constants.py                        |  78 ++++++++++++++
 yggdrasil/drivers/CModelDriver.py             |   8 +-
 yggdrasil/drivers/FortranModelDriver.py       |   8 +-
 yggdrasil/drivers/ModelDriver.py              |  16 ++-
 yggdrasil/languages/Python/YggInterface.py    |   6 +-
 .../languages/Python/test_YggInterface.py     |  12 +--
 yggdrasil/metaschema/__init__.py              |  94 ++++++++++++++++
 .../datatypes/ArrayMetaschemaType.py          |   5 +-
 .../datatypes/InstanceMetaschemaType.py       |   2 +-
 .../metaschema/datatypes/MetaschemaType.py    |  30 +++---
 .../datatypes/MultiMetaschemaType.py          |   4 +-
 .../datatypes/ScalarMetaschemaType.py         |  39 ++++---
 yggdrasil/metaschema/datatypes/__init__.py    |  48 +--------
 .../metaschema/datatypes/tests/__init__.py    |   4 +-
 .../datatypes/tests/test_MetaschemaType.py    |   6 +-
 .../tests/test_MultiMetaschemaType.py         |   2 +-
 .../tests/test_ScalarMetaschemaType.py        |   6 +-
 yggdrasil/metaschema/encoder.py               |   8 +-
 .../properties/ArgsMetaschemaProperty.py      |   2 +-
 .../properties/ScalarMetaschemaProperties.py  | 101 ++----------------
 .../properties/TypeMetaschemaProperty.py      |   3 +-
 .../tests/test_MetaschemaProperty.py          |   4 +-
 .../tests/test_ScalarMetaschemaProperties.py  |  16 +--
 yggdrasil/serialize/AsciiMapSerialize.py      |   5 +-
 yggdrasil/serialize/AsciiTableSerialize.py    |   9 +-
 yggdrasil/serialize/PlySerialize.py           |   4 +-
 yggdrasil/serialize/SerializeBase.py          |  20 ++--
 yggdrasil/serialize/__init__.py               |  55 ++++------
 .../serialize/tests/test_SerializeBase.py     |   8 +-
 yggdrasil/serialize/tests/test_serialize.py   |   6 +-
 yggdrasil/tools.py                            |   2 -
 yggdrasil/units.py                            |  12 ++-
 36 files changed, 354 insertions(+), 323 deletions(-)

diff --git a/yggdrasil/communication/CommBase.py b/yggdrasil/communication/CommBase.py
index 1c56e2514..0c93081ad 100755
--- a/yggdrasil/communication/CommBase.py
+++ b/yggdrasil/communication/CommBase.py
@@ -7,13 +7,13 @@
 import time
 import collections
 import numpy as np
-from yggdrasil import tools, multitasking
-from yggdrasil.tools import YGG_MSG_EOF
+from yggdrasil import tools, multitasking, constants
 from yggdrasil.communication import (
     new_comm, get_comm, determine_suffix, TemporaryCommunicationError,
     import_comm, check_env_for_address)
-from yggdrasil.components import import_component, create_component
-from yggdrasil.metaschema.datatypes import MetaschemaTypeError, type2numpy
+from yggdrasil.components import (
+    import_component, create_component, ComponentError)
+from yggdrasil.metaschema import MetaschemaTypeError, type2numpy
 from yggdrasil.metaschema.datatypes.MetaschemaType import MetaschemaType
 from yggdrasil.communication.transforms.TransformBase import TransformBase
 from yggdrasil.serialize import consolidate_array
@@ -33,10 +33,6 @@
 FLAG_EMPTY = 6
 
 
-YGG_CLIENT_INI = b'YGG_BEGIN_CLIENT'
-YGG_CLIENT_EOF = b'YGG_END_CLIENT'
-
-
 class NeverMatch(Exception):
     'An exception class that is never raised by any code anywhere'
 
@@ -959,7 +955,7 @@ def is_installed(cls, language=None):
                 try:
                     drv = import_component('model', language)
                     out = drv.is_comm_installed(commtype=cls._commtype)
-                except ValueError:
+                except ComponentError:
                     out = False
         return out
 
@@ -1364,7 +1360,7 @@ def n_msg_send_drain(self):
     @property
     def eof_msg(self):
         r"""str: Message indicating EOF."""
-        return YGG_MSG_EOF
+        return constants.YGG_MSG_EOF
 
     def is_eof(self, msg):
         r"""Determine if a message is an EOF.
@@ -2011,7 +2007,7 @@ def prepare_message(self, *args, header_kwargs=None, skip_serialization=False,
         # Make duplicates
         once_per_partner = ((msg.flag == FLAG_EOF)
                             or (isinstance(msg.args, bytes)
-                                and (msg.args == YGG_CLIENT_EOF)))
+                                and (msg.args == constants.YGG_CLIENT_EOF)))
         if once_per_partner and (self.partner_copies > 1):
             self.debug("Sending %s to %d model(s)", msg.args,
                        self.partner_copies)
diff --git a/yggdrasil/communication/FileComm.py b/yggdrasil/communication/FileComm.py
index d1517872d..369b7f01e 100755
--- a/yggdrasil/communication/FileComm.py
+++ b/yggdrasil/communication/FileComm.py
@@ -4,7 +4,7 @@
 from yggdrasil import platform, tools
 from yggdrasil.serialize.SerializeBase import SerializeBase
 from yggdrasil.communication import CommBase
-from yggdrasil.components import import_component, registration_in_progress
+from yggdrasil.components import import_component
 
 
 class FileComm(CommBase.CommBase):
@@ -140,14 +140,14 @@ def before_registration(cls):
         # Add serializer properties to schema
         if cls._filetype != 'binary':
             assert('serializer' not in cls._schema_properties)
-            if registration_in_progress():
-                seri = import_component('serializer', cls._default_serializer)
-                new = seri._schema_properties
-            else:
-                from yggdrasil.schema import get_schema
-                schema = get_schema()
-                new = schema['file'].get_subtype_schema(
-                    cls._default_serializer)['properties']
+            # if registration_in_progress():
+            seri = import_component('serializer', cls._default_serializer)
+            new = seri._schema_properties
+            # else:
+            #     from yggdrasil.schema import get_schema
+            #     schema = get_schema()
+            #     new = schema['file'].get_subtype_schema(
+            #         cls._default_serializer)['properties']
             cls._schema_properties.update(new)
             for k in ['driver', 'args', 'seritype']:
                 cls._schema_properties.pop(k, None)
@@ -191,7 +191,7 @@ def get_testing_options(cls, read_meth=None, **kwargs):
                 out['contents'] += comment
                 out['recv_partial'].append([])
         else:
-            seri_cls = cls._default_serializer_class
+            seri_cls = import_component('serializer', cls._default_serializer)
             if seri_cls.concats_as_str:
                 out['recv_partial'] = [[x] for x in out['recv']]
                 out['recv'] = seri_cls.concatenate(out['recv'], **out['kwargs'])
diff --git a/yggdrasil/communication/ServerComm.py b/yggdrasil/communication/ServerComm.py
index 585d5b3a8..199e8817b 100644
--- a/yggdrasil/communication/ServerComm.py
+++ b/yggdrasil/communication/ServerComm.py
@@ -1,6 +1,7 @@
 import os
 import uuid
 from collections import OrderedDict
+from yggdrasil import constants
 from yggdrasil.communication import CommBase, get_comm, import_comm
 
 
@@ -362,7 +363,7 @@ def finalize_message(self, msg, **kwargs):
         """
         def check_for_client_info(msg):
             if msg.flag == CommBase.FLAG_SUCCESS:
-                if isinstance(msg.args, bytes) and (msg.args == CommBase.YGG_CLIENT_EOF):
+                if isinstance(msg.args, bytes) and (msg.args == constants.YGG_CLIENT_EOF):
                     self.debug("Client signed off: %s", msg.header['model'])
                     self.closed_clients.append(msg.header['model'])
                     msg.flag = CommBase.FLAG_SKIP
diff --git a/yggdrasil/communication/transforms/ArrayTransform.py b/yggdrasil/communication/transforms/ArrayTransform.py
index 766f31bfd..667ba4d3c 100644
--- a/yggdrasil/communication/transforms/ArrayTransform.py
+++ b/yggdrasil/communication/transforms/ArrayTransform.py
@@ -1,12 +1,11 @@
 import numpy as np
 import copy
 import pandas
+from yggdrasil import constants
 from yggdrasil.communication.transforms.TransformBase import TransformBase
-from yggdrasil.metaschema.datatypes import type2numpy
+from yggdrasil.metaschema import type2numpy
 from yggdrasil.serialize import (
     consolidate_array, pandas2numpy, numpy2pandas, dict2list)
-from yggdrasil.metaschema.properties.ScalarMetaschemaProperties import (
-    _valid_types, _flexible_types)
 
 
 class ArrayTransform(TransformBase):
@@ -63,7 +62,7 @@ def get_summary(cls, x, subtype=False):
                 s = (s,)
             t = '1darray'
         elif ((x['type'] == 'scalar')
-              or (x['type'] in _valid_types)):
+              or (x['type'] in constants.VALID_TYPES)):
             s = (1,)
             t = 'scalar'
         else:
@@ -71,11 +70,11 @@ def get_summary(cls, x, subtype=False):
                                   "to array elements.") % x['type'])
         subt = x.get('subtype', x['type'])
         title = x.get('title', None)
-        assert(subt in _valid_types)
+        assert(subt in constants.VALID_TYPES)
         if subtype:
             out = {'type': t, 'subtype': subt,
                    'shape': s, 'title': title}
-            if subt not in _flexible_types:
+            if subt not in constants.FLEXIBLE_TYPES:
                 out['precision'] = x.get('precision', 0)
         else:
             out = {'type': t, 'shape': s}
@@ -264,7 +263,7 @@ def transform_array_items(cls, items, order=None):
                     subtype=x.get('subtype', x['type']))
                for x in base_types]
         for i, x in enumerate(out):
-            if x['subtype'] in _flexible_types:
+            if x['subtype'] in constants.FLEXIBLE_TYPES:
                 x['precision'] = max(
                     [y['items'][i].get('precision', 0) for y in items])
                 if x['precision'] == 0:
diff --git a/yggdrasil/constants.py b/yggdrasil/constants.py
index 67dab568d..9b249db72 100644
--- a/yggdrasil/constants.py
+++ b/yggdrasil/constants.py
@@ -1,4 +1,82 @@
 """Constants used by yggdrasil."""
+import numpy as np
+from collections import OrderedDict
+# No other yggdrasil modules should be import here
+# TODO: Move platform constants into this module?
+from yggdrasil import platform
+
+
+# Type related constants
+NUMPY_NUMERIC_TYPES = [
+    'int',
+    'uint',
+    'float',
+    'complex',
+]
+NUMPY_STRING_TYPES = [
+    'bytes',
+    'unicode',
+    'str',
+]
+NUMPY_TYPES = NUMPY_NUMERIC_TYPES + NUMPY_STRING_TYPES
+FLEXIBLE_TYPES = [
+    'string',
+    'bytes',
+    'unicode',
+]
+PYTHON_SCALARS = OrderedDict([
+    ('float', [float]),
+    ('int', [int, np.signedinteger]),
+    ('uint', [np.unsignedinteger]),
+    ('complex', [complex]),
+    ('bytes', [bytes]),
+    ('unicode', [str]),
+])
+VALID_TYPES = OrderedDict([(k, k) for k in NUMPY_NUMERIC_TYPES])
+VALID_TYPES.update([
+    ('bytes', 'bytes'),
+    ('unicode', 'str'),
+])
+NUMPY_PRECISIONS = {
+    'float': [16, 32, 64],
+    'int': [8, 16, 32, 64],
+    'uint': [8, 16, 32, 64],
+    'complex': [64, 128],
+}
+if not platform._is_win:
+    # Not available on windows
+    NUMPY_PRECISIONS['float'].append(128)
+    NUMPY_PRECISIONS['complex'].append(256)
+for T, T_NP in VALID_TYPES.items():
+    PYTHON_SCALARS[T].append(np.dtype(T_NP).type)
+    if T in NUMPY_PRECISIONS:
+        PYTHON_SCALARS[T] += [np.dtype(T_NP + str(P)).type
+                              for P in NUMPY_PRECISIONS[T]]
+ALL_PYTHON_SCALARS = []
+for k, v in PYTHON_SCALARS.items():
+    PYTHON_SCALARS[k] = tuple(set(v))
+    ALL_PYTHON_SCALARS += list(v)
+ALL_PYTHON_SCALARS = tuple(ALL_PYTHON_SCALARS)
+ALL_PYTHON_ARRAYS = (np.ndarray,)
+
+
+# Serialization constants
+FMT_CHAR = b'%'
+YGG_MSG_HEAD = b'YGG_MSG_HEAD'
+DEFAULT_COMMENT = b'# '
+DEFAULT_DELIMITER = b'\t'
+DEFAULT_NEWLINE = b'\n'
+FMT_CHAR_STR = FMT_CHAR.decode("utf-8")
+DEFAULT_COMMENT_STR = DEFAULT_COMMENT.decode("utf-8")
+DEFAULT_DELIMITER_STR = DEFAULT_DELIMITER.decode("utf-8")
+DEFAULT_NEWLINE_STR = DEFAULT_NEWLINE.decode("utf-8")
+
+
+# Communication constants
+YGG_MSG_EOF = b'EOF!!!'
+YGG_MSG_BUF = 1024 * 2
+YGG_CLIENT_INI = b'YGG_BEGIN_CLIENT'
+YGG_CLIENT_EOF = b'YGG_END_CLIENT'
 
 
 # ======================================================
diff --git a/yggdrasil/drivers/CModelDriver.py b/yggdrasil/drivers/CModelDriver.py
index ba50b9fb1..14871dfeb 100755
--- a/yggdrasil/drivers/CModelDriver.py
+++ b/yggdrasil/drivers/CModelDriver.py
@@ -7,12 +7,10 @@
 import numpy as np
 import sysconfig
 from collections import OrderedDict
-from yggdrasil import platform, tools
+from yggdrasil import platform, tools, constants
 from yggdrasil.drivers.CompiledModelDriver import (
     CompiledModelDriver, CompilerBase, LinkerBase, ArchiverBase,
     get_compilation_tool)
-from yggdrasil.metaschema.properties.ScalarMetaschemaProperties import (
-    _valid_types)
 from yggdrasil.languages import get_language_dir
 from yggdrasil.config import ygg_cfg
 from numpy import distutils as numpy_distutils
@@ -1052,7 +1050,7 @@ def parse_var_definition(cls, io, value, **kwargs):
                 x['datatype']['shape'] = [
                     int(float(s.strip('[]')))
                     for s in x.pop('shape').split('][')]
-                assert(x['datatype']['subtype'] in _valid_types)
+                assert(x['datatype']['subtype'] in constants.VALID_TYPES)
                 if len(x['datatype']['shape']) == 1:
                     x['datatype']['length'] = x['datatype'].pop(
                         'shape')[0]
@@ -1353,7 +1351,7 @@ def get_json_type(cls, native_type):
             if nptr > 0:
                 out['subtype'] = out['type']
                 out['type'] = '1darray'
-        if out['type'] in _valid_types:
+        if out['type'] in constants.VALID_TYPES:
             out['subtype'] = out['type']
             out['type'] = 'scalar'
         return out
diff --git a/yggdrasil/drivers/FortranModelDriver.py b/yggdrasil/drivers/FortranModelDriver.py
index 9400ff93d..286122a2b 100644
--- a/yggdrasil/drivers/FortranModelDriver.py
+++ b/yggdrasil/drivers/FortranModelDriver.py
@@ -3,14 +3,12 @@
 import copy
 import logging
 from collections import OrderedDict
-from yggdrasil import platform, tools
+from yggdrasil import platform, tools, constants
 from yggdrasil.languages import get_language_dir
 from yggdrasil.drivers import CModelDriver
 from yggdrasil.drivers.CompiledModelDriver import (
     CompilerBase, CompiledModelDriver, get_compilation_tool,
     get_compilation_tool_registry)
-from yggdrasil.metaschema.properties.ScalarMetaschemaProperties import (
-    _valid_types)
 
 
 logger = logging.getLogger(__name__)
@@ -643,7 +641,7 @@ def get_json_type(cls, native_type):
                 out['type'] = 'ndarray'
                 if shape[0] not in '*:':
                     out['shape'] = [int(i) for i in shape]
-        if out['type'] in _valid_types:
+        if out['type'] in constants.VALID_TYPES:
             out['subtype'] = out['type']
             out['type'] = 'scalar'
         return out
@@ -1131,7 +1129,7 @@ def write_type_def(cls, name, datatype, **kwargs):
         if datatype.get('subtype', datatype['type']) in ['bytes', 'unicode']:
             if 'precision' not in datatype:
                 datatype = dict(datatype, precision=0)
-        elif datatype.get('subtype', datatype['type']) in _valid_types:
+        elif datatype.get('subtype', datatype['type']) in constants.VALID_TYPES:
             datatype.setdefault('precision', 32)
         out = super(FortranModelDriver, cls).write_type_def(
             name, datatype, **kwargs)
diff --git a/yggdrasil/drivers/ModelDriver.py b/yggdrasil/drivers/ModelDriver.py
index 9dd29ed7a..13c1cfc00 100755
--- a/yggdrasil/drivers/ModelDriver.py
+++ b/yggdrasil/drivers/ModelDriver.py
@@ -11,12 +11,10 @@
 import asyncio
 from collections import OrderedDict
 from pprint import pformat
-from yggdrasil import platform, tools, languages, multitasking
+from yggdrasil import platform, tools, languages, multitasking, constants
 from yggdrasil.components import import_component
 from yggdrasil.drivers.Driver import Driver
 from yggdrasil.metaschema.datatypes import is_default_typedef
-from yggdrasil.metaschema.properties.ScalarMetaschemaProperties import (
-    _valid_types)
 from queue import Empty
 logger = logging.getLogger(__name__)
 
@@ -2812,11 +2810,11 @@ def write_type_decl(cls, name, datatype, name_base=None,
                          'precision': 64, 'length': len(datatype['shape'])}},
                     definitions=definitions,
                     requires_freeing=requires_freeing)
-        elif datatype['type'] in ['ply', 'obj', '1darray',
-                                  'scalar', 'boolean', 'null',
-                                  'number', 'integer', 'string',
-                                  'class', 'function', 'instance',
-                                  'schema', 'any'] + list(_valid_types.keys()):
+        elif datatype['type'] in (['ply', 'obj', '1darray', 'scalar',
+                                   'boolean', 'null', 'number', 'integer',
+                                   'string', 'class', 'function', 'instance',
+                                   'schema', 'any']
+                                  + list(constants.VALID_TYPES.keys())):
             pass
         else:  # pragma: debug
             raise ValueError(("Cannot create %s version of type "
@@ -2938,7 +2936,7 @@ def write_type_def(cls, name, datatype, name_base=None,
                 keys['ndim'] = 0
                 keys['shape'] = cls.function_param['null']
             keys['units'] = datatype.get('units', '')
-        elif (typename == 'scalar') or (typename in _valid_types):
+        elif (typename == 'scalar') or (typename in constants.VALID_TYPES):
             keys['subtype'] = datatype.get('subtype', datatype['type'])
             keys['units'] = datatype.get('units', '')
             if keys['subtype'] in ['bytes', 'string', 'unicode']:
diff --git a/yggdrasil/languages/Python/YggInterface.py b/yggdrasil/languages/Python/YggInterface.py
index 1d04e4647..6d5b2025e 100755
--- a/yggdrasil/languages/Python/YggInterface.py
+++ b/yggdrasil/languages/Python/YggInterface.py
@@ -1,11 +1,11 @@
 import os
-from yggdrasil import tools
+from yggdrasil import tools, constants
 from yggdrasil.communication.DefaultComm import DefaultComm
 
 
 YGG_MSG_MAX = tools.get_YGG_MSG_MAX()
-YGG_MSG_EOF = tools.YGG_MSG_EOF
-YGG_MSG_BUF = tools.YGG_MSG_BUF
+YGG_MSG_EOF = constants.YGG_MSG_EOF
+YGG_MSG_BUF = constants.YGG_MSG_BUF
 YGG_SERVER_INPUT = os.environ.get('YGG_SERVER_INPUT', False)
 YGG_SERVER_OUTPUT = os.environ.get('YGG_SERVER_OUTPUT', False)
 YGG_MODEL_NAME = os.environ.get('YGG_MODEL_NAME', False)
diff --git a/yggdrasil/languages/Python/test_YggInterface.py b/yggdrasil/languages/Python/test_YggInterface.py
index f0faea996..5e5668e01 100644
--- a/yggdrasil/languages/Python/test_YggInterface.py
+++ b/yggdrasil/languages/Python/test_YggInterface.py
@@ -3,8 +3,8 @@
 import flaky
 from yggdrasil.communication import get_comm
 from yggdrasil.interface import YggInterface
-from yggdrasil.tools import (
-    YGG_MSG_EOF, get_YGG_MSG_MAX, YGG_MSG_BUF, is_lang_installed)
+from yggdrasil import constants
+from yggdrasil.tools import get_YGG_MSG_MAX, is_lang_installed
 from yggdrasil.components import import_component
 from yggdrasil.drivers import ConnectionDriver
 from yggdrasil.tests import YggTestClassInfo, assert_equal, assert_raises
@@ -43,12 +43,12 @@ def test_maxMsgSize():
 
 def test_eof_msg():
     r"""Test eof message signal."""
-    assert_equal(YggInterface.eof_msg(), YGG_MSG_EOF)
+    assert_equal(YggInterface.eof_msg(), constants.YGG_MSG_EOF)
 
 
 def test_bufMsgSize():
     r"""Test buf message size."""
-    assert_equal(YggInterface.bufMsgSize(), YGG_MSG_BUF)
+    assert_equal(YggInterface.bufMsgSize(), constants.YGG_MSG_BUF)
 
 
 def test_init():
@@ -100,7 +100,7 @@ def do_send_recv(language='python', fmt='%f\\n%d', msg=[float(1.0), np.int32(2)]
             o.close(linger=True)
             # Input
             assert_equal(i.recv(), (True, converter(msg)))
-            assert_equal(i.recv(), (False, converter(YGG_MSG_EOF)))
+            assert_equal(i.recv(), (False, converter(constants.YGG_MSG_EOF)))
     finally:
         iodrv.terminate()
 
@@ -122,7 +122,7 @@ def test_YggInit_backwards():
 def test_YggInit_variables():
     r"""Test Matlab interface for variables."""
     assert_equal(YggInterface.YggInit('YGG_MSG_MAX'), YGG_MSG_MAX)
-    assert_equal(YggInterface.YggInit('YGG_MSG_EOF'), YGG_MSG_EOF)
+    assert_equal(YggInterface.YggInit('YGG_MSG_EOF'), constants.YGG_MSG_EOF)
     assert_equal(YggInterface.YggInit('YGG_MSG_EOF'),
                  YggInterface.YggInit('CIS_MSG_EOF'))
     assert_equal(YggInterface.YggInit('YGG_MSG_EOF'),
diff --git a/yggdrasil/metaschema/__init__.py b/yggdrasil/metaschema/__init__.py
index 18f4abe25..31b6d80c7 100644
--- a/yggdrasil/metaschema/__init__.py
+++ b/yggdrasil/metaschema/__init__.py
@@ -2,7 +2,9 @@
 import copy
 import pprint
 import jsonschema
+import numpy as np
 import yggdrasil
+from yggdrasil import constants, units
 from yggdrasil.metaschema.encoder import encode_json, decode_json
 from yggdrasil.metaschema.properties import get_registered_properties
 from yggdrasil.metaschema.datatypes import get_registered_types
@@ -28,6 +30,11 @@
 _base_validator = jsonschema.validators.validator_for(_base_schema)
 
         
+class MetaschemaTypeError(TypeError):
+    r"""Error that should be raised when a class encounters a type it cannot handle."""
+    pass
+
+
 def create_metaschema(overwrite=False):
     r"""Create the meta schema for validating ygg schema.
 
@@ -194,3 +201,90 @@ def normalize_instance(obj, schema, **kwargs):
     cls = get_validator()
     cls.check_schema(schema)
     return cls(schema).normalize(obj, **kwargs)
+
+
+def data2dtype(data):
+    r"""Get numpy data type for an object.
+
+    Args:
+        data (object): Python object.
+
+    Returns:
+        np.dtype: Numpy data type.
+
+    """
+    data_nounits = units.get_data(data)
+    if isinstance(data_nounits, np.ndarray):
+        dtype = data_nounits.dtype
+    elif isinstance(data_nounits, (list, dict, tuple)):
+        raise MetaschemaTypeError
+    elif isinstance(data_nounits,
+                    np.dtype(constants.VALID_TYPES['bytes']).type):
+        dtype = np.array(data_nounits).dtype
+    else:
+        dtype = np.array([data_nounits]).dtype
+    return dtype
+
+
+def definition2dtype(props):
+    r"""Get numpy data type for a type definition.
+
+    Args:
+        props (dict): Type definition properties.
+        
+    Returns:
+        np.dtype: Numpy data type.
+
+    """
+    typename = props.get('subtype', None)
+    if typename is None:
+        typename = props.get('type', None)
+        if typename is None:
+            raise KeyError('Could not find type in dictionary')
+    if ('precision' not in props):
+        if typename in constants.FLEXIBLE_TYPES:
+            out = np.dtype((constants.VALID_TYPES[typename]))
+        else:
+            raise RuntimeError("Precision required for type: '%s'" % typename)
+    elif typename == 'unicode':
+        out = np.dtype((constants.VALID_TYPES[typename],
+                        int(props['precision'] // 32)))
+    elif typename in constants.FLEXIBLE_TYPES:
+        out = np.dtype((constants.VALID_TYPES[typename],
+                        int(props['precision'] // 8)))
+    else:
+        out = np.dtype('%s%d' % (constants.VALID_TYPES[typename],
+                                 int(props['precision'])))
+    return out
+
+
+def type2numpy(typedef):
+    r"""Convert a type definition into a numpy dtype.
+
+    Args:
+        typedef (dict): Type definition.
+
+    Returns:
+        np.dtype: Numpy data type.
+
+    """
+    out = None
+    if ((isinstance(typedef, dict) and ('type' in typedef)
+         and (typedef['type'] == 'array') and ('items' in typedef))):
+        if isinstance(typedef['items'], dict):
+            as_array = (typedef['items']['type'] in ['1darray', 'ndarray'])
+            if as_array:
+                out = definition2dtype(typedef['items'])
+        elif isinstance(typedef['items'], (list, tuple)):
+            as_array = True
+            dtype_list = []
+            field_names = []
+            for i, x in enumerate(typedef['items']):
+                if x['type'] not in ['1darray', 'ndarray']:
+                    as_array = False
+                    break
+                dtype_list.append(definition2dtype(x))
+                field_names.append(x.get('title', 'f%d' % i))
+            if as_array:
+                out = np.dtype(dict(names=field_names, formats=dtype_list))
+    return out
diff --git a/yggdrasil/metaschema/datatypes/ArrayMetaschemaType.py b/yggdrasil/metaschema/datatypes/ArrayMetaschemaType.py
index 7af1097fd..c72ecdd68 100644
--- a/yggdrasil/metaschema/datatypes/ArrayMetaschemaType.py
+++ b/yggdrasil/metaschema/datatypes/ArrayMetaschemaType.py
@@ -1,7 +1,6 @@
 from yggdrasil import units
 from yggdrasil.metaschema.datatypes.ScalarMetaschemaType import (
     ScalarMetaschemaType)
-from yggdrasil.metaschema.properties import ScalarMetaschemaProperties
 
 
 class OneDArrayMetaschemaType(ScalarMetaschemaType):
@@ -11,7 +10,7 @@ class OneDArrayMetaschemaType(ScalarMetaschemaType):
     description = 'A 1D array with or without units.'
     properties = ['length']
     metadata_properties = ['length']
-    python_types = ScalarMetaschemaProperties._all_python_arrays
+    python_types = units.ALL_PYTHON_ARRAYS_WITH_UNITS
 
     @classmethod
     def validate(cls, obj, raise_errors=False):
@@ -43,7 +42,7 @@ class NDArrayMetaschemaType(ScalarMetaschemaType):
     description = 'An ND array with or without units.'
     properties = ['shape']
     metadata_properties = ['shape']
-    python_types = ScalarMetaschemaProperties._all_python_arrays
+    python_types = units.ALL_PYTHON_ARRAYS_WITH_UNITS
 
     @classmethod
     def validate(cls, obj, raise_errors=False):
diff --git a/yggdrasil/metaschema/datatypes/InstanceMetaschemaType.py b/yggdrasil/metaschema/datatypes/InstanceMetaschemaType.py
index 928b28075..f798ff0f9 100644
--- a/yggdrasil/metaschema/datatypes/InstanceMetaschemaType.py
+++ b/yggdrasil/metaschema/datatypes/InstanceMetaschemaType.py
@@ -1,4 +1,4 @@
-from yggdrasil.metaschema.datatypes import MetaschemaTypeError
+from yggdrasil.metaschema import MetaschemaTypeError
 from yggdrasil.metaschema.datatypes.MetaschemaType import MetaschemaType
 from yggdrasil.metaschema.datatypes.JSONArrayMetaschemaType import (
     JSONArrayMetaschemaType)
diff --git a/yggdrasil/metaschema/datatypes/MetaschemaType.py b/yggdrasil/metaschema/datatypes/MetaschemaType.py
index 6b95ff061..a4f22f63b 100644
--- a/yggdrasil/metaschema/datatypes/MetaschemaType.py
+++ b/yggdrasil/metaschema/datatypes/MetaschemaType.py
@@ -3,11 +3,11 @@
 import uuid
 import importlib
 import jsonschema
-from yggdrasil import tools
+from yggdrasil import constants
 from yggdrasil.metaschema import (get_metaschema, get_validator, encoder,
-                                  validate_instance)
+                                  validate_instance, MetaschemaTypeError)
 from yggdrasil.metaschema.datatypes import (
-    MetaschemaTypeError, MetaschemaTypeMeta, compare_schema, YGG_MSG_HEAD,
+    MetaschemaTypeMeta, compare_schema,
     get_type_class, conversions, is_default_typedef)
 from yggdrasil.metaschema.properties import get_metaschema_property
 
@@ -650,7 +650,7 @@ def serialize(self, obj, no_metadata=False, dont_encode=False,
             if k in kwargs:
                 raise RuntimeError("'%s' is a reserved keyword in the metadata." % k)
         if ((isinstance(obj, bytes)
-             and ((obj == tools.YGG_MSG_EOF) or kwargs.get('raw', False)
+             and ((obj == constants.YGG_MSG_EOF) or kwargs.get('raw', False)
                   or dont_encode))):
             metadata = kwargs
             data = obj
@@ -668,7 +668,8 @@ def serialize(self, obj, no_metadata=False, dont_encode=False,
             return data
         metadata['size'] = len(data)
         metadata.setdefault('id', str(uuid.uuid4()))
-        header = YGG_MSG_HEAD + encoder.encode_json(metadata) + YGG_MSG_HEAD
+        header = (constants.YGG_MSG_HEAD + encoder.encode_json(metadata)
+                  + constants.YGG_MSG_HEAD)
         if (max_header_size > 0) and (len(header) > max_header_size):
             metadata_type = metadata
             metadata = {}
@@ -678,10 +679,13 @@ def serialize(self, obj, no_metadata=False, dont_encode=False,
                 if k in metadata_type:
                     metadata[k] = metadata_type.pop(k)
             assert(metadata)
-            data = (encoder.encode_json(metadata_type) + YGG_MSG_HEAD + data)
+            data = (encoder.encode_json(metadata_type)
+                    + constants.YGG_MSG_HEAD + data)
             metadata['size'] = len(data)
             metadata['type_in_data'] = True
-            header = YGG_MSG_HEAD + encoder.encode_json(metadata) + YGG_MSG_HEAD
+            header = (constants.YGG_MSG_HEAD
+                      + encoder.encode_json(metadata)
+                      + constants.YGG_MSG_HEAD)
             if len(header) > max_header_size:  # pragma: debug
                 raise AssertionError(("The header is larger (%d) than the "
                                       "maximum (%d): %.100s...")
@@ -717,17 +721,17 @@ def deserialize(self, msg, no_data=False, metadata=None, dont_decode=False,
         if not isinstance(msg, bytes):
             raise TypeError("Message to be deserialized is not bytes type.")
         # Check for header
-        if msg.startswith(YGG_MSG_HEAD):
+        if msg.startswith(constants.YGG_MSG_HEAD):
             if metadata is not None:
                 raise ValueError("Metadata in header and provided by keyword.")
-            _, metadata, data = msg.split(YGG_MSG_HEAD, 2)
+            _, metadata, data = msg.split(constants.YGG_MSG_HEAD, 2)
             if len(metadata) == 0:
                 metadata = dict(size=len(data))
             else:
                 metadata = encoder.decode_json(metadata)
         elif isinstance(metadata, dict) and metadata.get('type_in_data', False):
-            assert(msg.count(YGG_MSG_HEAD) == 1)
-            typedef, data = msg.split(YGG_MSG_HEAD, 1)
+            assert(msg.count(constants.YGG_MSG_HEAD) == 1)
+            typedef, data = msg.split(constants.YGG_MSG_HEAD, 1)
             if len(typedef) > 0:
                 metadata.update(encoder.decode_json(typedef))
             metadata.pop('type_in_data')
@@ -736,13 +740,13 @@ def deserialize(self, msg, no_data=False, metadata=None, dont_decode=False,
             data = msg
             if metadata is None:
                 metadata = dict(size=len(msg))
-                if (((len(msg) > 0) and (msg != tools.YGG_MSG_EOF)
+                if (((len(msg) > 0) and (msg != constants.YGG_MSG_EOF)
                      and (not is_default_typedef(self._typedef))
                      and (not dont_decode))):
                     raise ValueError("Header marker not in message.")
         # Set flags based on data
         metadata['incomplete'] = (len(data) < metadata['size'])
-        if (data == tools.YGG_MSG_EOF):
+        if (data == constants.YGG_MSG_EOF):
             metadata['raw'] = True
         # Return based on flags
         if no_data:
diff --git a/yggdrasil/metaschema/datatypes/MultiMetaschemaType.py b/yggdrasil/metaschema/datatypes/MultiMetaschemaType.py
index b52129559..60d7a4f92 100644
--- a/yggdrasil/metaschema/datatypes/MultiMetaschemaType.py
+++ b/yggdrasil/metaschema/datatypes/MultiMetaschemaType.py
@@ -1,7 +1,7 @@
 import copy
 from collections import OrderedDict
-from yggdrasil.metaschema.datatypes import (
-    get_type_class, MetaschemaTypeError)
+from yggdrasil.metaschema import MetaschemaTypeError
+from yggdrasil.metaschema.datatypes import get_type_class
 from yggdrasil.metaschema.datatypes.MetaschemaType import MetaschemaType
 from yggdrasil.metaschema.properties.TypeMetaschemaProperty import (
     TypeMetaschemaProperty)
diff --git a/yggdrasil/metaschema/datatypes/ScalarMetaschemaType.py b/yggdrasil/metaschema/datatypes/ScalarMetaschemaType.py
index f48b1d39c..a985026c9 100644
--- a/yggdrasil/metaschema/datatypes/ScalarMetaschemaType.py
+++ b/yggdrasil/metaschema/datatypes/ScalarMetaschemaType.py
@@ -2,12 +2,12 @@
 import copy
 import warnings
 import base64
-from yggdrasil import units
+from yggdrasil import units, constants
+from yggdrasil.metaschema import data2dtype, definition2dtype
 from yggdrasil.metaschema.datatypes.MetaschemaType import MetaschemaType
 from yggdrasil.metaschema.datatypes.FixedMetaschemaType import (
     create_fixed_type_class)
-from yggdrasil.metaschema.properties import (
-    get_metaschema_property, ScalarMetaschemaProperties)
+from yggdrasil.metaschema.properties import get_metaschema_property
 
 
 class ScalarMetaschemaType(MetaschemaType):
@@ -25,7 +25,7 @@ class ScalarMetaschemaType(MetaschemaType):
     definition_properties = ['subtype']
     metadata_properties = ['subtype', 'precision', 'units']
     extract_properties = ['subtype', 'precision', 'units']
-    python_types = ScalarMetaschemaProperties._all_python_scalars
+    python_types = units.ALL_PYTHON_SCALARS_WITH_UNITS
 
     @classmethod
     def validate(cls, obj, raise_errors=False):
@@ -44,13 +44,12 @@ def validate(cls, obj, raise_errors=False):
             obj = obj.reshape((1, ))[0]
         if super(ScalarMetaschemaType, cls).validate(units.get_data(obj),
                                                      raise_errors=raise_errors):
-            dtype = ScalarMetaschemaProperties.data2dtype(obj)
+            dtype = data2dtype(obj)
             if cls.is_fixed and ('subtype' in cls.fixed_properties):
                 type_list = [
-                    ScalarMetaschemaProperties._valid_types[
-                        cls.fixed_properties['subtype']]]
+                    constants.VALID_TYPES[cls.fixed_properties['subtype']]]
             else:
-                type_list = ScalarMetaschemaProperties._valid_numpy_types
+                type_list = constants.NUMPY_TYPES
             if dtype.name.startswith(tuple(type_list)):
                 return True
             else:
@@ -103,7 +102,7 @@ def normalize(cls, obj):
                 else:
                     obj = str(obj)
             else:
-                dtype = ScalarMetaschemaProperties._python_scalars[
+                dtype = units.PYTHON_SCALARS_WITH_UNITS[
                     cls.fixed_properties['subtype']][0]
                 try:
                     obj = dtype(obj)
@@ -183,7 +182,7 @@ def decode_data(cls, obj, typedef):
 
         """
         bytes = base64.decodebytes(obj.encode('ascii'))
-        dtype = ScalarMetaschemaProperties.definition2dtype(typedef)
+        dtype = definition2dtype(typedef)
         arr = np.frombuffer(bytes, dtype=dtype)
         # arr = np.fromstring(bytes, dtype=dtype)
         if 'shape' in typedef:
@@ -210,7 +209,7 @@ def transform_type(cls, obj, typedef=None):
         typedef0 = cls.encode_type(obj)
         typedef1 = copy.deepcopy(typedef0)
         typedef1.update(**typedef)
-        dtype = ScalarMetaschemaProperties.definition2dtype(typedef1)
+        dtype = definition2dtype(typedef1)
         arr = cls.to_array(obj).astype(dtype, casting='same_kind')
         out = cls.from_array(arr, unit_str=typedef0.get('units', None),
                              dtype=dtype, typedef=typedef)
@@ -231,7 +230,7 @@ def to_array(cls, obj):
         if isinstance(obj_nounits, np.ndarray):
             arr = obj_nounits
         else:
-            dtype = ScalarMetaschemaProperties.data2dtype(obj_nounits)
+            dtype = data2dtype(obj_nounits)
             arr = np.array([obj_nounits], dtype=dtype)
         return arr
         
@@ -255,9 +254,9 @@ def from_array(cls, arr, unit_str=None, dtype=None, typedef=None):
 
         """
         # if (typedef is None) and (dtype is not None):
-        #     typedef = ScalarMetaschemaProperties.dtype2definition(dtype)
+        #     typedef = dtype2definition(dtype)
         # elif (dtype is None) and (typedef is not None):
-        #     dtype = ScalarMetaschemaProperties.definition2dtype(typedef)
+        #     dtype = definition2dtype(typedef)
         if (cls.name not in ['1darray', 'ndarray']) and (arr.ndim > 0):
             out = arr[0]
         else:
@@ -267,7 +266,7 @@ def from_array(cls, arr, unit_str=None, dtype=None, typedef=None):
             out = cls.as_python_type(out, typedef)
         if unit_str is not None:
             if dtype is None:
-                dtype = ScalarMetaschemaProperties.data2dtype(out)
+                dtype = data2dtype(out)
             out = units.add_units(out, unit_str, dtype=dtype)
         return out
 
@@ -285,7 +284,7 @@ def get_extract_properties(cls, metadata):
         """
         out = super(ScalarMetaschemaType, cls).get_extract_properties(metadata)
         dtype = metadata.get('subtype', metadata['type'])
-        if (((dtype in ScalarMetaschemaProperties._flexible_types)
+        if (((dtype in constants.FLEXIBLE_TYPES)
              and (metadata['type'] not in ['1darray', 'ndarray'])
              and (not metadata.get('fixed_precision', False)))):
             out.remove('precision')
@@ -308,7 +307,7 @@ def as_python_type(cls, obj, typedef):
         if ((isinstance(typedef, dict)
              and (typedef.get('type', '1darray') not in ['1darray', 'ndarray']))):
             stype = typedef.get('subtype', typedef.get('type', None))
-            py_type = ScalarMetaschemaProperties._python_scalars[stype][0]
+            py_type = units.PYTHON_SCALARS_WITH_UNITS[stype][0]
             if np.dtype(py_type) == type(obj):
                 obj = py_type(obj)
         return obj
@@ -324,7 +323,7 @@ def _generate_data(cls, typedef):
             object: Python object of the specified type.
 
         """
-        dtype = ScalarMetaschemaProperties.definition2dtype(typedef)
+        dtype = definition2dtype(typedef)
         if typedef['type'] == '1darray':
             out = np.zeros(typedef.get('length', 2), dtype)
         elif typedef['type'] == 'ndarray':
@@ -336,7 +335,7 @@ def _generate_data(cls, typedef):
 
 
 # Dynamically create explicity scalar classes for shorthand
-for t in ScalarMetaschemaProperties._valid_types.keys():
+for t in constants.VALID_TYPES.keys():
     short_doc = 'A %s value with or without units.' % t
     long_doc = ('%s\n\n'
                 '    Developer Notes:\n'
@@ -344,6 +343,6 @@ def _generate_data(cls, typedef):
     kwargs = {'target_globals': globals(),
               '__doc__': long_doc,
               '__module__': ScalarMetaschemaType.__module__,
-              'python_types': ScalarMetaschemaProperties._python_scalars[t]}
+              'python_types': units.PYTHON_SCALARS_WITH_UNITS[t]}
     create_fixed_type_class(t, short_doc, ScalarMetaschemaType,
                             {'subtype': t}, **kwargs)
diff --git a/yggdrasil/metaschema/datatypes/__init__.py b/yggdrasil/metaschema/datatypes/__init__.py
index c29cefd52..3e0091b07 100644
--- a/yggdrasil/metaschema/datatypes/__init__.py
+++ b/yggdrasil/metaschema/datatypes/__init__.py
@@ -2,7 +2,7 @@
 import glob
 import jsonschema
 import copy
-import numpy as np
+from yggdrasil import constants
 from yggdrasil.components import ClassRegistry
 from yggdrasil.metaschema.encoder import decode_json
 from yggdrasil.metaschema.properties import get_metaschema_property
@@ -11,7 +11,6 @@
 _schema_dir = os.path.join(os.path.dirname(__file__), 'schemas')
 # _base_validator = jsonschema.validators.validator_for({"$schema": ""})
 _base_validator = jsonschema.validators._LATEST_VERSION
-YGG_MSG_HEAD = b'YGG_MSG_HEAD'
 _property_attributes = ['properties', 'definition_properties',
                         'metadata_properties', 'extract_properties']
 
@@ -39,11 +38,6 @@ def import_schema_types():
                             % new_names)
     
 
-class MetaschemaTypeError(TypeError):
-    r"""Error that should be raised when a class encounters a type it cannot handle."""
-    pass
-
-
 _default_typedef = {'type': 'bytes'}
 _type_registry = ClassRegistry(import_function=import_schema_types)
 
@@ -302,8 +296,8 @@ def guess_type_from_msg(msg):
 
     """
     try:
-        if YGG_MSG_HEAD in msg:
-            _, metadata, data = msg.split(YGG_MSG_HEAD, 2)
+        if constants.YGG_MSG_HEAD in msg:
+            _, metadata, data = msg.split(constants.YGG_MSG_HEAD, 2)
             metadata = decode_json(metadata)
             cls = _type_registry[metadata['datatype']['type']]
         else:
@@ -456,7 +450,7 @@ def decode(msg):
 
     """
     cls = guess_type_from_msg(msg)
-    metadata = decode_json(msg.split(YGG_MSG_HEAD, 2)[1])
+    metadata = decode_json(msg.split(constants.YGG_MSG_HEAD, 2)[1])
     typedef = cls.extract_typedef(metadata.get('datatype', {}))
     cls_inst = cls(**typedef)
     obj = cls_inst.deserialize(msg)[0]
@@ -581,37 +575,3 @@ def generate_data(typedef):
     """
     type_cls = get_type_class(typedef['type'])
     return type_cls.generate_data(typedef)
-
-
-def type2numpy(typedef):
-    r"""Convert a type definition into a numpy dtype.
-
-    Args:
-        typedef (dict): Type definition.
-
-    Returns:
-        np.dtype: Numpy data type.
-
-    """
-    from yggdrasil.metaschema.properties.ScalarMetaschemaProperties import (
-        definition2dtype)
-    out = None
-    if ((isinstance(typedef, dict) and ('type' in typedef)
-         and (typedef['type'] == 'array') and ('items' in typedef))):
-        if isinstance(typedef['items'], dict):
-            as_array = (typedef['items']['type'] in ['1darray', 'ndarray'])
-            if as_array:
-                out = definition2dtype(typedef['items'])
-        elif isinstance(typedef['items'], (list, tuple)):
-            as_array = True
-            dtype_list = []
-            field_names = []
-            for i, x in enumerate(typedef['items']):
-                if x['type'] not in ['1darray', 'ndarray']:
-                    as_array = False
-                    break
-                dtype_list.append(definition2dtype(x))
-                field_names.append(x.get('title', 'f%d' % i))
-            if as_array:
-                out = np.dtype(dict(names=field_names, formats=dtype_list))
-    return out
diff --git a/yggdrasil/metaschema/datatypes/tests/__init__.py b/yggdrasil/metaschema/datatypes/tests/__init__.py
index cfc336602..5101318f0 100644
--- a/yggdrasil/metaschema/datatypes/tests/__init__.py
+++ b/yggdrasil/metaschema/datatypes/tests/__init__.py
@@ -1,5 +1,5 @@
 from yggdrasil.tests import assert_raises, assert_equal
-from yggdrasil.metaschema import datatypes
+from yggdrasil.metaschema import datatypes, MetaschemaTypeError
 from yggdrasil.metaschema.tests import _valid_objects
 from yggdrasil.metaschema.datatypes.ScalarMetaschemaType import (
     ScalarMetaschemaType)
@@ -53,7 +53,7 @@ def test_guess_type_from_obj():
     for t, x in _valid_objects.items():
         assert_equal(datatypes.guess_type_from_obj(x).name, t)
     for x in invalid_objects:
-        assert_raises(datatypes.MetaschemaTypeError,
+        assert_raises(MetaschemaTypeError,
                       datatypes.guess_type_from_obj, x)
 
 
diff --git a/yggdrasil/metaschema/datatypes/tests/test_MetaschemaType.py b/yggdrasil/metaschema/datatypes/tests/test_MetaschemaType.py
index 9ecbc15ac..5cd2a0ff4 100644
--- a/yggdrasil/metaschema/datatypes/tests/test_MetaschemaType.py
+++ b/yggdrasil/metaschema/datatypes/tests/test_MetaschemaType.py
@@ -3,7 +3,8 @@
 import copy
 import pprint
 import jsonschema
-from yggdrasil.metaschema.datatypes import MetaschemaTypeError, YGG_MSG_HEAD
+from yggdrasil import constants
+from yggdrasil.metaschema import MetaschemaTypeError
 from yggdrasil.tests import YggTestClassInfo, assert_equal
 
 
@@ -301,7 +302,8 @@ def test_deserialize_empty(self):
         self.assert_result_equal(out[0], self.instance._empty_msg)
         self.assert_equal(out[1], dict(size=0, incomplete=False))
         # Empty metadata and message
-        out = self.instance.deserialize((2 * YGG_MSG_HEAD) + self._empty_msg)
+        out = self.instance.deserialize((2 * constants.YGG_MSG_HEAD)
+                                        + self._empty_msg)
         self.assert_result_equal(out[0], self.instance._empty_msg)
         self.assert_equal(out[1], dict(size=0, incomplete=False))
 
diff --git a/yggdrasil/metaschema/datatypes/tests/test_MultiMetaschemaType.py b/yggdrasil/metaschema/datatypes/tests/test_MultiMetaschemaType.py
index 583e8f1c5..10bc7454c 100644
--- a/yggdrasil/metaschema/datatypes/tests/test_MultiMetaschemaType.py
+++ b/yggdrasil/metaschema/datatypes/tests/test_MultiMetaschemaType.py
@@ -1,4 +1,4 @@
-from yggdrasil.metaschema.datatypes import MetaschemaTypeError
+from yggdrasil.metaschema import MetaschemaTypeError
 from yggdrasil.metaschema.datatypes.tests import test_MetaschemaType as parent
 from yggdrasil.metaschema.datatypes.MultiMetaschemaType import (
     create_multitype_class)
diff --git a/yggdrasil/metaschema/datatypes/tests/test_ScalarMetaschemaType.py b/yggdrasil/metaschema/datatypes/tests/test_ScalarMetaschemaType.py
index 48314f79d..b997fbfcf 100644
--- a/yggdrasil/metaschema/datatypes/tests/test_ScalarMetaschemaType.py
+++ b/yggdrasil/metaschema/datatypes/tests/test_ScalarMetaschemaType.py
@@ -1,9 +1,7 @@
 import copy
 import numpy as np
-from yggdrasil import units, platform
+from yggdrasil import units, platform, constants
 from yggdrasil.metaschema.datatypes.tests import test_MetaschemaType as parent
-from yggdrasil.metaschema.properties.ScalarMetaschemaProperties import (
-    _valid_types)
 
 
 class TestScalarMetaschemaType(parent.TestMetaschemaType):
@@ -109,7 +107,7 @@ def test_from_array(self):
 
 
 # Dynamically create tests for dynamic and explicitly typed scalars
-for t in _valid_types.keys():
+for t in constants.VALID_TYPES.keys():
     iattr_imp = {'_type': t}
     if t == 'complex':
         iattr_imp['_prec'] = 64
diff --git a/yggdrasil/metaschema/encoder.py b/yggdrasil/metaschema/encoder.py
index f1d01b30b..7f1b5308a 100644
--- a/yggdrasil/metaschema/encoder.py
+++ b/yggdrasil/metaschema/encoder.py
@@ -50,8 +50,8 @@ class JSONReadableEncoder(stdjson.JSONEncoder):
 
     def default(self, o):  # pragma: no cover
         r"""Encoder that allows for expansion types."""
-        from yggdrasil.metaschema.datatypes import (
-            encode_data_readable, MetaschemaTypeError)
+        from yggdrasil.metaschema import MetaschemaTypeError
+        from yggdrasil.metaschema.datatypes import encode_data_readable
         try:
             return encode_data_readable(o)
         except MetaschemaTypeError:
@@ -63,8 +63,8 @@ class JSONEncoder(_json_encoder):
 
     def default(self, o):
         r"""Encoder that allows for expansion types."""
-        from yggdrasil.metaschema.datatypes import (
-            encode_data, MetaschemaTypeError)
+        from yggdrasil.metaschema import MetaschemaTypeError
+        from yggdrasil.metaschema.datatypes import encode_data
         try:
             return encode_data(o)
         except MetaschemaTypeError:
diff --git a/yggdrasil/metaschema/properties/ArgsMetaschemaProperty.py b/yggdrasil/metaschema/properties/ArgsMetaschemaProperty.py
index 64e0084bd..c7e49fa24 100644
--- a/yggdrasil/metaschema/properties/ArgsMetaschemaProperty.py
+++ b/yggdrasil/metaschema/properties/ArgsMetaschemaProperty.py
@@ -1,5 +1,5 @@
 import weakref
-from yggdrasil.metaschema.datatypes import MetaschemaTypeError
+from yggdrasil.metaschema import MetaschemaTypeError
 from yggdrasil.metaschema.properties.MetaschemaProperty import (
     MetaschemaProperty)
 from yggdrasil.metaschema.properties.JSONArrayMetaschemaProperties import (
diff --git a/yggdrasil/metaschema/properties/ScalarMetaschemaProperties.py b/yggdrasil/metaschema/properties/ScalarMetaschemaProperties.py
index b3cda3ac9..f13eb834d 100644
--- a/yggdrasil/metaschema/properties/ScalarMetaschemaProperties.py
+++ b/yggdrasil/metaschema/properties/ScalarMetaschemaProperties.py
@@ -1,96 +1,7 @@
-import numpy as np
-from yggdrasil import units, platform
-from yggdrasil.metaschema.datatypes import MetaschemaTypeError
-from yggdrasil.metaschema.properties.MetaschemaProperty import MetaschemaProperty
-from collections import OrderedDict
-
-
-_valid_numpy_types = ['int', 'uint', 'float', 'complex']
-_valid_types = OrderedDict([(k, k) for k in _valid_numpy_types])
-_flexible_types = ['string', 'bytes', 'unicode']
-_python_scalars = OrderedDict([('float', [float]),
-                               ('int', [int]), ('uint', []),
-                               ('complex', [complex])])
-_valid_numpy_types += ['bytes', 'unicode', 'str']
-_valid_types['bytes'] = 'bytes'
-_valid_types['unicode'] = 'str'
-_python_scalars.update([('bytes', [bytes]), ('unicode', [str])])
-for t, t_np in _valid_types.items():
-    prec_list = []
-    if t in ['float']:
-        prec_list = [16, 32, 64]
-        if not platform._is_win:
-            prec_list.append(128)  # Not available on windows
-    if t in ['int', 'uint']:
-        prec_list = [8, 16, 32, 64]
-    elif t in ['complex']:
-        prec_list = [64, 128]
-        if not platform._is_win:
-            prec_list.append(256)  # Not available on windows
-    _python_scalars[t].append(np.dtype(t_np).type)
-    for p in prec_list:
-        _python_scalars[t].append(np.dtype(t_np + str(p)).type)
-# For some reason windows fails to check types on ints in some cases
-_python_scalars['int'].append(np.signedinteger)
-_python_scalars['uint'].append(np.unsignedinteger)
-_all_python_scalars = [units._unit_quantity]
-for k in _python_scalars.keys():
-    _python_scalars[k].append(units._unit_quantity)
-    _all_python_scalars += list(_python_scalars[k])
-    _python_scalars[k] = tuple(_python_scalars[k])
-_all_python_arrays = tuple(set([np.ndarray, units._unit_array]))
-_all_python_scalars = tuple(set(_all_python_scalars))
-
-
-def data2dtype(data):
-    r"""Get numpy data type for an object.
-
-    Args:
-        data (object): Python object.
-
-    Returns:
-        np.dtype: Numpy data type.
-
-    """
-    data_nounits = units.get_data(data)
-    if isinstance(data_nounits, np.ndarray):
-        dtype = data_nounits.dtype
-    elif isinstance(data_nounits, (list, dict, tuple)):
-        raise MetaschemaTypeError
-    elif isinstance(data_nounits, np.dtype(_valid_types['bytes']).type):
-        dtype = np.array(data_nounits).dtype
-    else:
-        dtype = np.array([data_nounits]).dtype
-    return dtype
-
-
-def definition2dtype(props):
-    r"""Get numpy data type for a type definition.
-
-    Args:
-        props (dict): Type definition properties.
-        
-    Returns:
-        np.dtype: Numpy data type.
-
-    """
-    typename = props.get('subtype', None)
-    if typename is None:
-        typename = props.get('type', None)
-        if typename is None:
-            raise KeyError('Could not find type in dictionary')
-    if ('precision' not in props):
-        if typename in _flexible_types:
-            out = np.dtype((_valid_types[typename]))
-        else:
-            raise RuntimeError("Precision required for type: '%s'" % typename)
-    elif typename == 'unicode':
-        out = np.dtype((_valid_types[typename], int(props['precision'] // 32)))
-    elif typename in _flexible_types:
-        out = np.dtype((_valid_types[typename], int(props['precision'] // 8)))
-    else:
-        out = np.dtype('%s%d' % (_valid_types[typename], int(props['precision'])))
-    return out
+from yggdrasil import units, constants
+from yggdrasil.metaschema import data2dtype, MetaschemaTypeError
+from yggdrasil.metaschema.properties.MetaschemaProperty import (
+    MetaschemaProperty)
 
 
 class SubtypeMetaschemaProperty(MetaschemaProperty):
@@ -99,14 +10,14 @@ class SubtypeMetaschemaProperty(MetaschemaProperty):
     name = 'subtype'
     schema = {'description': 'The base type for each item.',
               'type': 'string',
-              'enum': [k for k in sorted(_valid_types.keys())]}
+              'enum': [k for k in sorted(constants.VALID_TYPES.keys())]}
 
     @classmethod
     def encode(cls, instance, typedef=None):
         r"""Encoder for the 'subtype' scalar property."""
         dtype = data2dtype(instance)
         out = None
-        for k, v in _valid_types.items():
+        for k, v in constants.VALID_TYPES.items():
             if dtype.name.startswith(v):
                 out = k
                 break
diff --git a/yggdrasil/metaschema/properties/TypeMetaschemaProperty.py b/yggdrasil/metaschema/properties/TypeMetaschemaProperty.py
index e27336132..0e595bd0c 100644
--- a/yggdrasil/metaschema/properties/TypeMetaschemaProperty.py
+++ b/yggdrasil/metaschema/properties/TypeMetaschemaProperty.py
@@ -1,5 +1,6 @@
+from yggdrasil.metaschema import MetaschemaTypeError
 from yggdrasil.metaschema.datatypes import (
-    get_registered_types, get_type_class, MetaschemaTypeError)
+    get_registered_types, get_type_class)
 from yggdrasil.metaschema.properties.MetaschemaProperty import MetaschemaProperty
 
 
diff --git a/yggdrasil/metaschema/properties/tests/test_MetaschemaProperty.py b/yggdrasil/metaschema/properties/tests/test_MetaschemaProperty.py
index 0be3e5e25..fe6a762dd 100644
--- a/yggdrasil/metaschema/properties/tests/test_MetaschemaProperty.py
+++ b/yggdrasil/metaschema/properties/tests/test_MetaschemaProperty.py
@@ -1,6 +1,6 @@
 from yggdrasil.tests import YggTestClassInfo, assert_equal
-from yggdrasil.metaschema import get_validator, get_metaschema
-from yggdrasil.metaschema.datatypes import MetaschemaTypeError
+from yggdrasil.metaschema import (
+    get_validator, get_metaschema, MetaschemaTypeError)
 from yggdrasil.metaschema.properties.MetaschemaProperty import (
     create_property)
 
diff --git a/yggdrasil/metaschema/properties/tests/test_ScalarMetaschemaProperties.py b/yggdrasil/metaschema/properties/tests/test_ScalarMetaschemaProperties.py
index 58d9ee25e..8a746224a 100644
--- a/yggdrasil/metaschema/properties/tests/test_ScalarMetaschemaProperties.py
+++ b/yggdrasil/metaschema/properties/tests/test_ScalarMetaschemaProperties.py
@@ -1,24 +1,24 @@
 import numpy as np
-from yggdrasil import units
+from yggdrasil import units, constants
 from yggdrasil.tests import assert_raises, assert_equal
-from yggdrasil.metaschema.properties import ScalarMetaschemaProperties
 from yggdrasil.metaschema.properties.tests import (
     test_MetaschemaProperty as parent)
-from yggdrasil.metaschema.datatypes import MetaschemaTypeError
+from yggdrasil.metaschema import (data2dtype, definition2dtype,
+                                  MetaschemaTypeError)
 
 
 def test_data2dtype_errors():
     r"""Check that error is raised for list, dict, & tuple objects."""
-    assert_raises(MetaschemaTypeError, ScalarMetaschemaProperties.data2dtype, [])
+    assert_raises(MetaschemaTypeError, data2dtype, [])
 
 
 def test_definition2dtype_errors():
     r"""Check that error raised if type not specified."""
-    assert_raises(KeyError, ScalarMetaschemaProperties.definition2dtype, {})
-    assert_raises(RuntimeError, ScalarMetaschemaProperties.definition2dtype,
+    assert_raises(KeyError, definition2dtype, {})
+    assert_raises(RuntimeError, definition2dtype,
                   {'type': 'float'})
-    assert_equal(ScalarMetaschemaProperties.definition2dtype({'type': 'bytes'}),
-                 np.dtype((ScalarMetaschemaProperties._valid_types['bytes'])))
+    assert_equal(definition2dtype({'type': 'bytes'}),
+                 np.dtype((constants.VALID_TYPES['bytes'])))
 
 
 class TestSubtypeMetaschemaProperty(parent.TestMetaschemaProperty):
diff --git a/yggdrasil/serialize/AsciiMapSerialize.py b/yggdrasil/serialize/AsciiMapSerialize.py
index 581d3b6fb..79482eb27 100644
--- a/yggdrasil/serialize/AsciiMapSerialize.py
+++ b/yggdrasil/serialize/AsciiMapSerialize.py
@@ -1,6 +1,5 @@
 import json
-from yggdrasil import tools
-from yggdrasil.serialize import _default_delimiter_str
+from yggdrasil import tools, constants
 from yggdrasil.serialize.SerializeBase import SerializeBase
 from yggdrasil.metaschema.encoder import JSONReadableEncoder
 
@@ -21,7 +20,7 @@ class AsciiMapSerialize(SerializeBase):
                                    'values.')
     _schema_properties = {
         'delimiter': {'type': 'string',
-                      'default': _default_delimiter_str}}
+                      'default': constants.DEFAULT_DELIMITER_STR}}
     _attr_conv = SerializeBase._attr_conv  # + ['delimiter']
     default_datatype = {'type': 'object'}
     concats_as_str = False
diff --git a/yggdrasil/serialize/AsciiTableSerialize.py b/yggdrasil/serialize/AsciiTableSerialize.py
index 0ce723dc9..9070eecfd 100644
--- a/yggdrasil/serialize/AsciiTableSerialize.py
+++ b/yggdrasil/serialize/AsciiTableSerialize.py
@@ -1,8 +1,6 @@
-from yggdrasil import units, serialize, tools
-from yggdrasil.serialize import _default_delimiter_str
+from yggdrasil import units, serialize, tools, constants
 from yggdrasil.serialize.DefaultSerialize import DefaultSerialize
-from yggdrasil.metaschema.properties.ScalarMetaschemaProperties import (
-    definition2dtype, data2dtype)
+from yggdrasil.metaschema import definition2dtype, data2dtype
 
 
 class AsciiTableSerialize(DefaultSerialize):
@@ -50,7 +48,8 @@ class AsciiTableSerialize(DefaultSerialize):
         'field_names': {'type': 'array', 'items': {'type': 'string'}},
         'field_units': {'type': 'array', 'items': {'type': 'string'}},
         'as_array': {'type': 'boolean', 'default': False},
-        'delimiter': {'type': 'string', 'default': _default_delimiter_str},
+        'delimiter': {'type': 'string',
+                      'default': constants.DEFAULT_DELIMITER_STR},
         'use_astropy': {'type': 'boolean', 'default': False}}
     _attr_conv = DefaultSerialize._attr_conv + ['format_str', 'delimiter']
     has_header = True
diff --git a/yggdrasil/serialize/PlySerialize.py b/yggdrasil/serialize/PlySerialize.py
index 24a315e2a..4b3dcc623 100644
--- a/yggdrasil/serialize/PlySerialize.py
+++ b/yggdrasil/serialize/PlySerialize.py
@@ -1,4 +1,4 @@
-from yggdrasil.serialize import _default_newline_str
+from yggdrasil import constants
 from yggdrasil.serialize.SerializeBase import SerializeBase
 from yggdrasil.metaschema.datatypes.PlyMetaschemaType import PlyDict
 
@@ -25,7 +25,7 @@ class PlySerialize(SerializeBase):
     _schema_subtype_description = ('Serialize 3D structures using Ply format.')
     _schema_properties = {
         'newline': {'type': 'string',
-                    'default': _default_newline_str}}
+                    'default': constants.DEFAULT_NEWLINE_STR}}
     default_datatype = {'type': 'ply'}
     concats_as_str = False
 
diff --git a/yggdrasil/serialize/SerializeBase.py b/yggdrasil/serialize/SerializeBase.py
index 27bb76346..0387d9cfe 100644
--- a/yggdrasil/serialize/SerializeBase.py
+++ b/yggdrasil/serialize/SerializeBase.py
@@ -3,15 +3,11 @@
 import pprint
 import numpy as np
 import warnings
-from yggdrasil import tools, units, serialize
+from yggdrasil import tools, units, serialize, constants
 from yggdrasil.metaschema.datatypes import (
-    guess_type_from_obj, get_type_from_def, get_type_class, compare_schema,
-    type2numpy)
-from yggdrasil.metaschema.properties.ScalarMetaschemaProperties import (
-    _flexible_types)
+    guess_type_from_obj, get_type_from_def, get_type_class, compare_schema)
+from yggdrasil.metaschema import type2numpy
 from yggdrasil.metaschema.datatypes.MetaschemaType import MetaschemaType
-from yggdrasil.metaschema.datatypes.ArrayMetaschemaType import (
-    OneDArrayMetaschemaType)
 
 
 class SerializeBase(tools.YggClass):
@@ -59,9 +55,9 @@ class SerializeBase(tools.YggClass):
                      'default': 'default',
                      'description': ('Serializer type.')},
         'newline': {'type': 'string',
-                    'default': serialize._default_newline_str},
+                    'default': constants.DEFAULT_NEWLINE_STR},
         'comment': {'type': 'string',
-                    'default': serialize._default_comment_str},
+                    'default': constants.DEFAULT_COMMENT_STR},
         'datatype': {'type': 'schema'}}
     _oldstyle_kws = ['format_str', 'field_names', 'field_units', 'as_array']
     _attr_conv = ['newline', 'comment']
@@ -570,6 +566,8 @@ def update_typedef_from_oldstyle(self, typedef):
                 continue
             # Key specific changes to type
             if k == 'format_str':
+                from yggdrasil.metaschema.datatypes.ArrayMetaschemaType import (
+                    OneDArrayMetaschemaType)
                 v = tools.bytes2str(v)
                 fmts = serialize.extract_formats(v)
                 if 'type' in typedef:
@@ -593,7 +591,7 @@ def update_typedef_from_oldstyle(self, typedef):
                         itype['type'] = '1darray'
                     else:
                         itype['type'] = itype.pop('subtype')
-                        if (((itype['type'] in _flexible_types)
+                        if (((itype['type'] in constants.FLEXIBLE_TYPES)
                              and ('precision' in itype))):
                             del itype['precision']
                     typedef['items'].append(itype)
@@ -689,7 +687,7 @@ def serialize(self, args, header_kwargs=None, add_serializer_info=False,
         """
         if header_kwargs is None:
             header_kwargs = {}
-        if isinstance(args, bytes) and (args == tools.YGG_MSG_EOF):
+        if isinstance(args, bytes) and (args == constants.YGG_MSG_EOF):
             header_kwargs['raw'] = True
         self.initialize_from_message(args, **header_kwargs)
         metadata = {'no_metadata': no_metadata,
diff --git a/yggdrasil/serialize/__init__.py b/yggdrasil/serialize/__init__.py
index 54aecd109..8b5a22670 100644
--- a/yggdrasil/serialize/__init__.py
+++ b/yggdrasil/serialize/__init__.py
@@ -3,7 +3,7 @@
 import numpy as np
 import pandas
 import io as sio
-from yggdrasil import platform, units, scanf, tools
+from yggdrasil import platform, units, scanf, tools, constants
 try:
     from astropy.io import ascii as apy_ascii
     from astropy.table import Table as apy_Table
@@ -15,16 +15,6 @@
     _use_astropy = False
 
 
-_fmt_char = b'%'
-_default_comment = b'# '
-_default_delimiter = b'\t'
-_default_newline = b'\n'
-_fmt_char_str = _fmt_char.decode("utf-8")
-_default_comment_str = _default_comment.decode("utf-8")
-_default_delimiter_str = _default_delimiter.decode("utf-8")
-_default_newline_str = _default_newline.decode("utf-8")
-
-
 def extract_formats(fmt_str):
     r"""Locate format codes within a format string.
 
@@ -601,7 +591,7 @@ def format2table(fmt_str):
         out['newline'] = fmt_rem
     seps = set(seps)
     if len(seps) == 0:
-        out['delimiter'] = _default_delimiter
+        out['delimiter'] = constants.DEFAULT_DELIMITER
     elif len(seps) == 1:
         out['delimiter'] = list(seps)[0]
     elif len(seps) > 1:
@@ -619,22 +609,22 @@ def table2format(fmts=[], delimiter=None, newline=None, comment=None):
         fmts (list, optional): List of format codes for each column. Defaults to
             [].
         delimiter (bytes, optional): String used to separate columns. Defaults
-            to _default_delimiter.
+            to constants.DEFAULT_DELIMITER.
         newline (bytes, optional): String used to indicate the end of a table
-            line. Defaults to _default_newline.
+            line. Defaults to constants.DEFAULT_NEWLINE.
         comment (bytes, optional): String that should be prepended to the format
-            string to indicate a comment. Defaults to _default_comment.
+            string to indicate a comment. Defaults to constants.DEFAULT_COMMENT.
 
     Returns:
         str, bytes: Table format string.
 
     """
     if delimiter is None:
-        delimiter = _default_delimiter
+        delimiter = constants.DEFAULT_DELIMITER
     if newline is None:
-        newline = _default_newline
+        newline = constants.DEFAULT_NEWLINE
     if comment is None:
-        comment = _default_comment
+        comment = constants.DEFAULT_COMMENT
     if isinstance(fmts, np.dtype):
         fmts = nptype2cformat(fmts)
     bytes_fmts = tools.str2bytes(fmts, recurse=True)
@@ -891,13 +881,13 @@ def format_header(format_str=None, dtype=None,
             specified, the formats will not be part of the header.
         comment (bytes, optional): String that should be used to comment the
             header lines. If not provided and not in format_str, defaults to
-            _default_comment.
+            constants.DEFAULT_COMMENT.
         delimiter (bytes, optional): String that should be used to separate
             columns. If not provided and not in format_str, defaults to
-            _default_delimiter.
+            constants.DEFAULT_DELIMITER.
         newline (bytes, optional): String that should be used to end lines in
             the table. If not provided and not in format_str, defaults to
-            _default_newline.
+            constants.DEFAULT_NEWLINE.
         field_names (list, optional): List of field names that should be
             included in the header. If not provided and dtype is None, names
             will not be included in the header.
@@ -930,11 +920,11 @@ def format_header(format_str=None, dtype=None,
         if field_names is None:
             field_names = [n.encode("utf-8") for n in dtype.names]
     if delimiter is None:
-        delimiter = _default_delimiter
+        delimiter = constants.DEFAULT_DELIMITER
     if comment is None:
-        comment = _default_comment
+        comment = constants.DEFAULT_COMMENT
     if newline is None:
-        newline = _default_newline
+        newline = constants.DEFAULT_NEWLINE
     # Get count of fields
     if fmts is not None:
         nfld = len(fmts)
@@ -955,8 +945,8 @@ def format_header(format_str=None, dtype=None,
     return out
 
 
-def discover_header(fd, serializer, newline=_default_newline,
-                    comment=_default_comment, delimiter=None,
+def discover_header(fd, serializer, newline=constants.DEFAULT_NEWLINE,
+                    comment=constants.DEFAULT_COMMENT, delimiter=None,
                     lineno_format=None, lineno_names=None, lineno_units=None,
                     use_astropy=False):
     r"""Discover ASCII table header info from a file.
@@ -966,13 +956,13 @@ def discover_header(fd, serializer, newline=_default_newline,
         serializer (DefaultSerialize): Serializer that should be updated with
             header information.
         newline (str, optional): Newline character that should be used to split
-            header if it is not already a list. Defaults to _default_newline.
+            header if it is not already a list. Defaults to constants.DEFAULT_NEWLINE.
         comment (bytes, optional): String that should be used to mark the
             header lines. If not provided and not in format_str, defaults to
-            _default_comment.
+            constants.DEFAULT_COMMENT.
         delimiter (bytes, optional): String that should be used to separate
             columns. If not provided and not in format_str, defaults to
-            _default_delimiter.
+            constants.DEFAULT_DELIMITER.
         lineno_format (int, optional): Line number where formats are located.
             If not provided, an attempt will be made to locate one.
         lineno_names (int, optional): Line number where field names are located.
@@ -1039,14 +1029,14 @@ def discover_header(fd, serializer, newline=_default_newline,
     fd.seek(prev_pos + header_size)
 
 
-def parse_header(header, newline=_default_newline, lineno_format=None,
+def parse_header(header, newline=constants.DEFAULT_NEWLINE, lineno_format=None,
                  lineno_names=None, lineno_units=None):
     r"""Parse an ASCII table header to get information about the table.
 
     Args:
         header (list, str): Header lines that should be parsed.
         newline (str, optional): Newline character that should be used to split
-            header if it is not already a list. Defaults to _default_newline.
+            header if it is not already a list. Defaults to constants.DEFAULT_NEWLINE.
         lineno_format (int, optional): Line number where formats are located.
             If not provided, an attempt will be made to locate one.
         lineno_names (int, optional): Line number where field names are located.
@@ -1077,7 +1067,8 @@ def parse_header(header, newline=_default_newline, lineno_format=None,
         out['format_str'] = header[lineno_format].split(info['comment'])[-1]
     else:
         ncol = 0
-        out.update(delimiter=_default_delimiter, comment=_default_comment,
+        out.update(delimiter=constants.DEFAULT_DELIMITER,
+                   comment=constants.DEFAULT_COMMENT,
                    fmts=[])
     out.setdefault('newline', newline)
     # Use explicit lines for names & units
diff --git a/yggdrasil/serialize/tests/test_SerializeBase.py b/yggdrasil/serialize/tests/test_SerializeBase.py
index 000d59e15..58f8f9c75 100644
--- a/yggdrasil/serialize/tests/test_SerializeBase.py
+++ b/yggdrasil/serialize/tests/test_SerializeBase.py
@@ -1,6 +1,6 @@
 import copy
 from yggdrasil.tests import YggTestClassInfo, assert_equal
-from yggdrasil import tools
+from yggdrasil import constants
 from yggdrasil.components import import_component
 from yggdrasil.serialize import SerializeBase
 
@@ -39,7 +39,7 @@ def inst_kwargs(self):
     def empty_head(self, msg):
         r"""dict: Empty header for message only contains the size."""
         out = dict(size=len(msg), incomplete=False)
-        if msg == tools.YGG_MSG_EOF:  # pragma: debug
+        if msg == constants.YGG_MSG_EOF:  # pragma: debug
             out['eof'] = True
         return out
 
@@ -143,7 +143,7 @@ def test_serialize_eof(self):
         r"""Test serialize/deserialize EOF."""
         if (self._cls == 'SerializeBase'):
             return
-        iobj = tools.YGG_MSG_EOF
+        iobj = constants.YGG_MSG_EOF
         msg = self.instance.serialize(iobj)
         iout, ihead = self.instance.deserialize(msg)
         self.assert_equal(iout, iobj)
@@ -153,7 +153,7 @@ def test_serialize_eof_header(self):
         r"""Test serialize/deserialize EOF with header."""
         if (self._cls == 'SerializeBase'):
             return
-        iobj = tools.YGG_MSG_EOF
+        iobj = constants.YGG_MSG_EOF
         msg = self.instance.serialize(iobj, header_kwargs=self._header_info)
         iout, ihead = self.instance.deserialize(msg)
         self.assert_equal(iout, iobj)
diff --git a/yggdrasil/serialize/tests/test_serialize.py b/yggdrasil/serialize/tests/test_serialize.py
index 4e6a70fff..95cbcf1e6 100644
--- a/yggdrasil/serialize/tests/test_serialize.py
+++ b/yggdrasil/serialize/tests/test_serialize.py
@@ -1,5 +1,5 @@
 import numpy as np
-from yggdrasil import serialize, platform
+from yggdrasil import serialize, platform, constants
 from yggdrasil.tests import assert_raises, assert_equal
 
 
@@ -107,10 +107,10 @@ def test_cformat2nptype():
         if isinstance(a, str):
             a = [a]
         for _ia in a:
-            if _ia.startswith(serialize._fmt_char_str):
+            if _ia.startswith(constants.FMT_CHAR_STR):
                 ia = _ia.encode("utf-8")
             else:
-                ia = serialize._fmt_char + _ia.encode("utf-8")
+                ia = constants.FMT_CHAR + _ia.encode("utf-8")
             assert_equal(serialize.cformat2nptype(ia), np.dtype(b))  # .str)
             # assert_equal(serialize.cformat2nptype(ia), np.dtype(b).str)
     assert_raises(TypeError, serialize.cformat2nptype, 0)
diff --git a/yggdrasil/tools.py b/yggdrasil/tools.py
index 356651e21..b87ab4537 100644
--- a/yggdrasil/tools.py
+++ b/yggdrasil/tools.py
@@ -28,8 +28,6 @@
 
 
 logger = logging.getLogger(__name__)
-YGG_MSG_EOF = b'EOF!!!'
-YGG_MSG_BUF = 1024 * 2
 
 
 _stack_in_log = False
diff --git a/yggdrasil/units.py b/yggdrasil/units.py
index 2a67b5c32..b69a7bc5a 100644
--- a/yggdrasil/units.py
+++ b/yggdrasil/units.py
@@ -2,12 +2,22 @@
 import numpy as np
 import pandas as pd
 import unyt
-from yggdrasil import tools
+from collections import OrderedDict
+from yggdrasil import tools, constants
 _unit_quantity = unyt.array.unyt_quantity
 _unit_array = unyt.array.unyt_array
 _ureg_unyt = None
 
 
+PYTHON_SCALARS_WITH_UNITS = OrderedDict([
+    (k, tuple(list(v) + [_unit_quantity]))
+    for k, v in constants.PYTHON_SCALARS.items()])
+ALL_PYTHON_ARRAYS_WITH_UNITS = tuple(
+    list(constants.ALL_PYTHON_ARRAYS) + [_unit_array])
+ALL_PYTHON_SCALARS_WITH_UNITS = tuple(
+    list(constants.ALL_PYTHON_SCALARS) + [_unit_quantity])
+
+
 def get_ureg():
     r"""Get the unit registry."""
     global _ureg_unyt

From 3ecfde398656c1988ee8a61894455eb44443b7dc Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Fri, 1 Oct 2021 14:45:42 -0700
Subject: [PATCH 43/73] Remove use of jsonschema compat module (removed in
 4.0.0) and fix missed uses of old constants

---
 .../metaschema/properties/JSONObjectMetaschemaProperties.py  | 3 +--
 yggdrasil/schema.py                                          | 5 ++---
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/yggdrasil/metaschema/properties/JSONObjectMetaschemaProperties.py b/yggdrasil/metaschema/properties/JSONObjectMetaschemaProperties.py
index 99c797bc8..3e0033fbc 100644
--- a/yggdrasil/metaschema/properties/JSONObjectMetaschemaProperties.py
+++ b/yggdrasil/metaschema/properties/JSONObjectMetaschemaProperties.py
@@ -1,4 +1,3 @@
-from jsonschema.compat import iteritems
 from yggdrasil.metaschema import normalizer as normalizer_mod
 from yggdrasil.metaschema.datatypes import encode_type, compare_schema
 from yggdrasil.metaschema.properties.MetaschemaProperty import MetaschemaProperty
@@ -34,7 +33,7 @@ def normalize(cls, validator, value, instance, schema):
         r"""Normalization method for 'properties' container property."""
         if not isinstance(instance, dict):
             return instance
-        for property, subschema in iteritems(value):
+        for property, subschema in value.items():
             if property not in instance:
                 instance[property] = normalizer_mod.UndefinedProperty()
         return instance
diff --git a/yggdrasil/schema.py b/yggdrasil/schema.py
index 04448b527..2552f3cfe 100644
--- a/yggdrasil/schema.py
+++ b/yggdrasil/schema.py
@@ -1006,10 +1006,9 @@ def model_form_schema_props(self):
     @property
     def model_form_schema(self):
         r"""dict: Schema for generating a model YAML form."""
-        from yggdrasil.metaschema.properties.ScalarMetaschemaProperties import (
-            _valid_types)
+        from yggdrasil import constants
         out = self.get_schema(for_form=True)
-        scalar_types = list(_valid_types.keys())
+        scalar_types = list(constants.VALID_TYPES.keys())
         meta = copy.deepcopy(metaschema._metaschema)
         meta_prop = {
             'subtype': ['1darray', 'ndarray'],

From 075117c7fad8a1c6a160fd9bcf2412b0f3b7d53c Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Fri, 1 Oct 2021 15:12:36 -0700
Subject: [PATCH 44/73] Use format string in search for libraries and
 temporarily search all directories for libraries on mac

---
 yggdrasil/drivers/CompiledModelDriver.py | 34 +++++++++++++-----------
 1 file changed, 18 insertions(+), 16 deletions(-)

diff --git a/yggdrasil/drivers/CompiledModelDriver.py b/yggdrasil/drivers/CompiledModelDriver.py
index 700b855ad..08a4d5c4f 100644
--- a/yggdrasil/drivers/CompiledModelDriver.py
+++ b/yggdrasil/drivers/CompiledModelDriver.py
@@ -903,6 +903,8 @@ def get_search_path(cls, env_only=False, libtype=None, cfg=None):
             base_paths = ['/usr', os.path.join('/usr', 'local')]
         if platform._is_mac:
             base_paths.append('/Library/Developer/CommandLineTools/usr')
+            # REMOVE THIS!
+            paths.append('/')
         if libtype == 'include':
             suffix = 'include'
         else:
@@ -2708,19 +2710,19 @@ def get_dependency_library(cls, dep, default=None, libtype=None,
                 if os.path.isfile(libinfo[libtype]):
                     out = libinfo[libtype]
                 else:  # pragma: no cover
-                    out = cls.cfg.get(dep_lang, '%s_%s' % (dep, libtype), None)
-            elif cls.cfg.has_option(dep_lang, '%s_%s' % (dep, libtype)):
-                out = cls.cfg.get(dep_lang, '%s_%s' % (dep, libtype))
+                    out = cls.cfg.get(dep_lang, f'{dep}_{libtype}', None)
+            elif cls.cfg.has_option(dep_lang, f'{dep}_{libtype}'):
+                out = cls.cfg.get(dep_lang, f'{dep}_{libtype}')
             else:
                 libtype_found = []
                 for k in libtype_list:
-                    if cls.cfg.has_option(dep_lang, '%s_%s' % (dep, k)):
+                    if cls.cfg.has_option(dep_lang, f'{dep}_{k}'):
                         libtype_found.append(k)
                 if len(libtype_found) > 0:
-                    raise ValueError(("A '%s' library could not be located for "
-                                      "dependency '%s', but one or more "
-                                      "libraries of types %s were found.")
-                                     % (libtype, dep, libtype_found))
+                    raise ValueError(f"A '{libtype}' library could not be "
+                                     f"located for dependency '{dep}', but "
+                                     f"one or more libraries of types "
+                                     f"{libtype_found} were found.")
             # TODO: CLEANUP
             if platform._is_win and out and out.endswith('.lib'):  # pragma: windows
                 if tool is None:
@@ -3328,7 +3330,7 @@ def configure(cls, cfg, **kwargs):
             cfg.add_section(cls.language)
         for k, v in kwargs.items():
             if k not in ['compiler', 'linker', 'archiver']:  # pragma: debug
-                raise ValueError("Unexpected configuration option: '%s'" % k)
+                raise ValueError(f"Unexpected configuration option: '{k}'")
             vtool = None
             try:
                 vtool = get_compilation_tool(k, v)
@@ -3339,10 +3341,10 @@ def configure(cls, cfg, **kwargs):
                         vtool = vreg
                         break
             if not vtool:  # pragma: debug
-                raise ValueError("Could not locate a %s tool '%s'." % (k, v))
+                raise ValueError(f"Could not locate a {k} tool '{v}'.")
             cfg.set(cls.language, k, vtool.toolname)
             if os.path.isfile(v):
-                cfg.set(cls.language, '%s_executable' % vtool.toolname, v)
+                cfg.set(cls.language, f'{vtool.toolname}_executable', v)
         # Call __func__ to avoid direct invoking of class which dosn't exist
         # in after_registration where this is called
         return ModelDriver.configure.__func__(cls, cfg)
@@ -3429,16 +3431,16 @@ def configure_library(cls, cfg, k):
         for t in v.keys():
             fname = v[t]
             assert(isinstance(fname, str))
-            opt = '%s_%s' % (k, t)
+            opt = f'{k}_{t}'
             if t in ['libtype', 'language']:
                 continue
             elif t in ['include']:
-                desc_end = '%s headers' % k
+                desc_end = f'{k} headers'
             elif t in ['static', 'shared']:
-                desc_end = '%s %s library' % (k, t)
+                desc_end = f'{k} {t} library'
             else:  # pragma: completion
-                desc_end = '%s %s' % (k, t)
-            desc = 'The full path to the directory containing %s.' % desc_end
+                desc_end = f'{k} {t}'
+            desc = f'The full path to the directory containing {desc_end}.'
             if cfg.has_option(k_lang, opt):
                 continue
             if os.path.isabs(fname):

From bc6220ebcb1b52ed2c01dc904be6460e31fcbc7f Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Fri, 1 Oct 2021 15:47:26 -0700
Subject: [PATCH 45/73] Remove another reference to jsonschema.compat module

---
 yggdrasil/metaschema/datatypes/JSONMetaschemaType.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/yggdrasil/metaschema/datatypes/JSONMetaschemaType.py b/yggdrasil/metaschema/datatypes/JSONMetaschemaType.py
index c44b998f5..ed1fb8533 100644
--- a/yggdrasil/metaschema/datatypes/JSONMetaschemaType.py
+++ b/yggdrasil/metaschema/datatypes/JSONMetaschemaType.py
@@ -1,5 +1,4 @@
 import numbers
-from jsonschema.compat import str_types, int_types
 from yggdrasil.metaschema.datatypes.MetaschemaType import MetaschemaType
 
 
@@ -74,7 +73,7 @@ class JSONIntegerMetaschemaType(JSONMetaschemaTypeBase):
 
     name = 'integer'
     description = 'JSON integer type.'
-    python_types = int_types
+    python_types = (int,)
     # TODO: Find a better way to signify this for creating the table
     cross_language_support = False
     example_data = int(1)
@@ -147,7 +146,7 @@ class JSONStringMetaschemaType(JSONMetaschemaTypeBase):
 
     name = 'string'
     description = 'JSON string type.'
-    python_types = str_types
+    python_types = (str,)
     example_data = 'hello'
 
     @classmethod

From 7a28559859b13f7ab78c750fc0a5522ac9cace37 Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Fri, 1 Oct 2021 16:24:53 -0700
Subject: [PATCH 46/73] Search some different paths on mac

---
 yggdrasil/drivers/CompiledModelDriver.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/yggdrasil/drivers/CompiledModelDriver.py b/yggdrasil/drivers/CompiledModelDriver.py
index 08a4d5c4f..dc387653d 100644
--- a/yggdrasil/drivers/CompiledModelDriver.py
+++ b/yggdrasil/drivers/CompiledModelDriver.py
@@ -904,7 +904,8 @@ def get_search_path(cls, env_only=False, libtype=None, cfg=None):
         if platform._is_mac:
             base_paths.append('/Library/Developer/CommandLineTools/usr')
             # REMOVE THIS!
-            paths.append('/')
+            paths.append('/usr')
+            paths.append('/Library')
         if libtype == 'include':
             suffix = 'include'
         else:

From dbff2fa9bd57100c4f05a1948c8afa4dd1a92352 Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Fri, 1 Oct 2021 16:29:47 -0700
Subject: [PATCH 47/73] Add another search path on mac

---
 yggdrasil/drivers/CompiledModelDriver.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/yggdrasil/drivers/CompiledModelDriver.py b/yggdrasil/drivers/CompiledModelDriver.py
index dc387653d..28b036c92 100644
--- a/yggdrasil/drivers/CompiledModelDriver.py
+++ b/yggdrasil/drivers/CompiledModelDriver.py
@@ -902,10 +902,11 @@ def get_search_path(cls, env_only=False, libtype=None, cfg=None):
         else:
             base_paths = ['/usr', os.path.join('/usr', 'local')]
         if platform._is_mac:
-            base_paths.append('/Library/Developer/CommandLineTools/usr')
-            # REMOVE THIS!
-            paths.append('/usr')
-            paths.append('/Library')
+            base_paths += [
+                '/Library/Developer/CommandLineTools/usr',
+                # XCode >= 12
+                '/Applications/Xcode.app/Contents/Developer/'
+                'Toolchains/XcodeDefault.xctoolchain/usr']
         if libtype == 'include':
             suffix = 'include'
         else:

From c81e7d761d488eb5710f4e319b49b7f62b4b31a4 Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Fri, 1 Oct 2021 17:27:12 -0700
Subject: [PATCH 48/73] Pin jsonschema to 3.2.0 until bugs can be fixed in the
 normalization

---
 recipe/meta.yaml | 2 +-
 requirements.txt | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/recipe/meta.yaml b/recipe/meta.yaml
index 26131759d..034a190ef 100644
--- a/recipe/meta.yaml
+++ b/recipe/meta.yaml
@@ -60,7 +60,7 @@ requirements:
     - git
     - GitPython <=3.1.20  # [py==36]
     - GitPython  # [py>36]
-    - jsonschema
+    - jsonschema==3.2.0
     - matplotlib-base
     - numpy >=1.13.0
     - pandas
diff --git a/requirements.txt b/requirements.txt
index 1ab8d129a..a18ca2512 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
 chevron
 flask
-jsonschema>=3
+jsonschema==3.2.0
 matplotlib
 numpy>=1.13.0
 pandas

From 2c398557bf190fccd00af730788ea748b77503d0 Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Fri, 1 Oct 2021 17:50:07 -0700
Subject: [PATCH 49/73] Search homebrew llvm for libc++ on mac

---
 yggdrasil/drivers/CompiledModelDriver.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/yggdrasil/drivers/CompiledModelDriver.py b/yggdrasil/drivers/CompiledModelDriver.py
index 28b036c92..96a2c3303 100644
--- a/yggdrasil/drivers/CompiledModelDriver.py
+++ b/yggdrasil/drivers/CompiledModelDriver.py
@@ -907,6 +907,8 @@ def get_search_path(cls, env_only=False, libtype=None, cfg=None):
                 # XCode >= 12
                 '/Applications/Xcode.app/Contents/Developer/'
                 'Toolchains/XcodeDefault.xctoolchain/usr']
+            # Check homebrew llvm
+            paths.append('/usr/local/Cellar/llvm/')
         if libtype == 'include':
             suffix = 'include'
         else:

From c3520fc45693a34a9db8c36661d93379cf9edfb1 Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Fri, 1 Oct 2021 18:31:19 -0700
Subject: [PATCH 50/73] Fix change of error on failed component import

---
 yggdrasil/communication/CommBase.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yggdrasil/communication/CommBase.py b/yggdrasil/communication/CommBase.py
index 0c93081ad..3b74d3f94 100755
--- a/yggdrasil/communication/CommBase.py
+++ b/yggdrasil/communication/CommBase.py
@@ -774,7 +774,7 @@ def _init_before_open(self, **kwargs):
                 if isinstance(iv, str):
                     try:
                         iv = create_component('transform', subtype=iv)
-                    except ValueError:
+                    except ComponentError:
                         iv = None
                 elif isinstance(iv, dict):
                     from yggdrasil.schema import get_schema

From e9849ee0f76b1846a0a76f8d2b39d05d8608916d Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Fri, 1 Oct 2021 19:02:53 -0700
Subject: [PATCH 51/73] Check mac SDK in search for libc++

---
 yggdrasil/drivers/CompiledModelDriver.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/yggdrasil/drivers/CompiledModelDriver.py b/yggdrasil/drivers/CompiledModelDriver.py
index 96a2c3303..ebe453555 100644
--- a/yggdrasil/drivers/CompiledModelDriver.py
+++ b/yggdrasil/drivers/CompiledModelDriver.py
@@ -902,13 +902,23 @@ def get_search_path(cls, env_only=False, libtype=None, cfg=None):
         else:
             base_paths = ['/usr', os.path.join('/usr', 'local')]
         if platform._is_mac:
+            macos_sdkroot = cfg.get('c', 'macos_sdkroot', None)
             base_paths += [
                 '/Library/Developer/CommandLineTools/usr',
                 # XCode >= 12
                 '/Applications/Xcode.app/Contents/Developer/'
                 'Toolchains/XcodeDefault.xctoolchain/usr']
+            if macos_sdkroot is not None:
+                base_paths.append(os.path.join(macos_sdkroot, 'usr'))
+                if 'Platforms' in macos_sdkroot:
+                    base_paths.append(
+                        os.path.join(
+                            macos_sdkroot.split('/Platforms')[0],
+                            'Toolchains/XcodeDefault.xctoolchain/usr'))
+            # /Applications/Xcode_12.5.1.app/Contents/Developer/Platforms/
+            # MacOSX.platform/Developer/SDKs/MacOSX.sdk
             # Check homebrew llvm
-            paths.append('/usr/local/Cellar/llvm/')
+            # paths.append('/usr/local/Cellar/llvm/')
         if libtype == 'include':
             suffix = 'include'
         else:

From 6cb32cd129ec9640a61a20b2e2c21ad2588d3480 Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Mon, 4 Oct 2021 10:38:13 -0700
Subject: [PATCH 52/73] Fix bugs in migration to using constants and
 ComponentError

---
 yggdrasil/components.py                     | 10 +++++-----
 yggdrasil/drivers/RPCRequestDriver.py       |  5 +++--
 yggdrasil/examples/tests/test_transforms.py |  5 +++--
 yggdrasil/schema.py                         |  3 +--
 4 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/yggdrasil/components.py b/yggdrasil/components.py
index 8eecdddd4..dd0ed6d6e 100644
--- a/yggdrasil/components.py
+++ b/yggdrasil/components.py
@@ -168,9 +168,9 @@ def import_component(comptype, subtype=None, **kwargs):
         class: Component class.
 
     Raises:
-        ValueError: If comptype is not a registered component type.
-        ValueError: If subtype is not a registered subtype or the name of a
-            registered subtype class for the specified comptype.
+        ComponentError: If comptype is not a registered component type.
+        ComponentError: If subtype is not a registered subtype or the name of
+            a registered subtype class for the specified comptype.
             
 
     """
@@ -229,13 +229,13 @@ def create_component(comptype, subtype=None, **kwargs):
             options.
     
     Raises:
-        ValueError: If comptype is not a registered component type.
+        ComponentError: If comptype is not a registered component type.
 
     """
     from yggdrasil.schema import get_schema
     s = get_schema().get(comptype, None)
     if s is None:  # pragma: debug
-        raise ValueError("Unrecognized component type: %s" % comptype)
+        raise ComponentError("Unrecognized component type: %s" % comptype)
     if s.subtype_key in kwargs:
         subtype = kwargs[s.subtype_key]
     if subtype is None:
diff --git a/yggdrasil/drivers/RPCRequestDriver.py b/yggdrasil/drivers/RPCRequestDriver.py
index 54bef20f7..feb6b68ac 100644
--- a/yggdrasil/drivers/RPCRequestDriver.py
+++ b/yggdrasil/drivers/RPCRequestDriver.py
@@ -1,3 +1,4 @@
+from yggdrasil import constants
 from yggdrasil.drivers.ConnectionDriver import ConnectionDriver, run_remotely
 from yggdrasil.drivers.RPCResponseDriver import RPCResponseDriver
 from yggdrasil.communication import CommBase
@@ -109,7 +110,7 @@ def remove_model(self, direction, name):
             clients = self.clients
             if (direction == "input") and (name in clients) and (len(clients) > 1):
                 super(RPCRequestDriver, self).send_message(
-                    CommBase.CommMessage(args=CommBase.YGG_CLIENT_EOF,
+                    CommBase.CommMessage(args=constants.YGG_CLIENT_EOF,
                                          flag=CommBase.FLAG_SUCCESS),
                     header_kwargs={'raw': True, 'model': name},
                     skip_processing=True)
@@ -172,7 +173,7 @@ def send_message(self, msg, **kwargs):
         if msg.flag != CommBase.FLAG_EOF:
             # Remove client that signed off
             if ((msg.header.get('raw', False)
-                 and (msg.args == CommBase.YGG_CLIENT_EOF))):  # pragma: intermittent
+                 and (msg.args == constants.YGG_CLIENT_EOF))):  # pragma: intermittent
                 self.remove_model('input', msg.header['model'])
                 return True
             with self.lock:
diff --git a/yggdrasil/examples/tests/test_transforms.py b/yggdrasil/examples/tests/test_transforms.py
index 7f6d00748..a2fb019b9 100644
--- a/yggdrasil/examples/tests/test_transforms.py
+++ b/yggdrasil/examples/tests/test_transforms.py
@@ -3,7 +3,8 @@
 import numpy as np
 from yggdrasil import tools, units
 from yggdrasil.tests import assert_equal
-from yggdrasil.components import import_component, create_component
+from yggdrasil.components import (
+    ComponentError, import_component, create_component)
 from yggdrasil.languages import get_language_ext
 from yggdrasil.examples import _example_dir
 from yggdrasil.examples.tests import ExampleTstBase
@@ -78,7 +79,7 @@ def check_received_data(cls, transform, x_recv):
         """
         try:
             t = create_component('transform', subtype=transform)
-        except ValueError:
+        except ComponentError:
             def t(x):
                 return x
         x_sent = t(cls.get_test_data(transform))
diff --git a/yggdrasil/schema.py b/yggdrasil/schema.py
index 2552f3cfe..85c511af6 100644
--- a/yggdrasil/schema.py
+++ b/yggdrasil/schema.py
@@ -459,8 +459,7 @@ def get_subtype_schema(self, subtype, unique=False, relaxed=False,
             else:
                 from yggdrasil.components import import_component
                 comp_cls = import_component(
-                    self.schema_type, subtype=subtype,
-                    without_schema=True)
+                    self.schema_type, subtype=subtype)
             out = {'oneOf': [out, {'type': 'instance',
                                    'class': comp_cls}]}
         return out

From 385cae9ecb3b06c41ad7313c1780c578b68045eb Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Mon, 4 Oct 2021 11:33:53 -0700
Subject: [PATCH 53/73] Explicitly convert integer64 to integer in print
 statement for ascii_io example in R (the sprintf is no longer converting
 integer64 types for an unknown reason)

---
 yggdrasil/examples/ascii_io/src/ascii_io.R | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yggdrasil/examples/ascii_io/src/ascii_io.R b/yggdrasil/examples/ascii_io/src/ascii_io.R
index 039ca0a8b..f03e9e5ce 100644
--- a/yggdrasil/examples/ascii_io/src/ascii_io.R
+++ b/yggdrasil/examples/ascii_io/src/ascii_io.R
@@ -44,7 +44,7 @@ while (ret) {
   if (ret) {
     # If the receive was succesful, send the values to output.
     # Formatting is taken care of on the output driver side.
-    fprintf("Table: %s, %d, %3.1f, %s", line[[1]], line[[2]], line[[3]], line[[4]])
+    fprintf("Table: %s, %d, %3.1f, %s", line[[1]], as.integer(line[[2]]), line[[3]], line[[4]])
     ret <- out_table$send(line[[1]], line[[2]], line[[3]], line[[4]])
     if (!ret) {
       stop("ascii_io(R): ERROR SENDING ROW")

From c5f04e719e0e1d288d904b489c146ab7ee9935c4 Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Mon, 4 Oct 2021 19:22:54 -0700
Subject: [PATCH 54/73] Don't use set on PYTHON_TYPES constants, order implies
 the order types should be tried.

---
 utils/setup_test_env.py                  | 2 +-
 yggdrasil/constants.py                   | 4 ++--
 yggdrasil/drivers/CompiledModelDriver.py | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/utils/setup_test_env.py b/utils/setup_test_env.py
index c16d3e2f2..7a04fdcc5 100644
--- a/utils/setup_test_env.py
+++ b/utils/setup_test_env.py
@@ -329,7 +329,7 @@ def create_env(method, python, name=None, packages=None, init=_on_ci):
 
     """
     cmds = ["echo Creating test environment using %s..." % method]
-    major, minor = [int(x) for x in python.split('.')]
+    major, minor = [int(x) for x in python.split('.')][:2]
     if name is None:
         name = '%s%s' % (method, python.replace('.', ''))
     if packages is None:
diff --git a/yggdrasil/constants.py b/yggdrasil/constants.py
index 9b249db72..9418796e7 100644
--- a/yggdrasil/constants.py
+++ b/yggdrasil/constants.py
@@ -54,9 +54,9 @@
                               for P in NUMPY_PRECISIONS[T]]
 ALL_PYTHON_SCALARS = []
 for k, v in PYTHON_SCALARS.items():
-    PYTHON_SCALARS[k] = tuple(set(v))
+    PYTHON_SCALARS[k] = tuple(v)
     ALL_PYTHON_SCALARS += list(v)
-ALL_PYTHON_SCALARS = tuple(ALL_PYTHON_SCALARS)
+ALL_PYTHON_SCALARS = tuple(set(ALL_PYTHON_SCALARS))
 ALL_PYTHON_ARRAYS = (np.ndarray,)
 
 
diff --git a/yggdrasil/drivers/CompiledModelDriver.py b/yggdrasil/drivers/CompiledModelDriver.py
index ebe453555..0b9fe6a8a 100644
--- a/yggdrasil/drivers/CompiledModelDriver.py
+++ b/yggdrasil/drivers/CompiledModelDriver.py
@@ -913,7 +913,7 @@ def get_search_path(cls, env_only=False, libtype=None, cfg=None):
                 if 'Platforms' in macos_sdkroot:
                     base_paths.append(
                         os.path.join(
-                            macos_sdkroot.split('/Platforms')[0],
+                            macos_sdkroot.split('/Platforms', 1)[0],
                             'Toolchains/XcodeDefault.xctoolchain/usr'))
             # /Applications/Xcode_12.5.1.app/Contents/Developer/Platforms/
             # MacOSX.platform/Developer/SDKs/MacOSX.sdk

From 99060e4548ffdb673d9cdc23f28e95194595f20e Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Mon, 4 Oct 2021 19:25:06 -0700
Subject: [PATCH 55/73] More search paths for libc++ on mac

---
 yggdrasil/drivers/CompiledModelDriver.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/yggdrasil/drivers/CompiledModelDriver.py b/yggdrasil/drivers/CompiledModelDriver.py
index 0b9fe6a8a..d7b2c29ee 100644
--- a/yggdrasil/drivers/CompiledModelDriver.py
+++ b/yggdrasil/drivers/CompiledModelDriver.py
@@ -915,16 +915,18 @@ def get_search_path(cls, env_only=False, libtype=None, cfg=None):
                         os.path.join(
                             macos_sdkroot.split('/Platforms', 1)[0],
                             'Toolchains/XcodeDefault.xctoolchain/usr'))
-            # /Applications/Xcode_12.5.1.app/Contents/Developer/Platforms/
-            # MacOSX.platform/Developer/SDKs/MacOSX.sdk
-            # Check homebrew llvm
-            # paths.append('/usr/local/Cellar/llvm/')
         if libtype == 'include':
             suffix = 'include'
         else:
             suffix = 'lib'
         for base in base_paths:
             paths.append(os.path.join(base, suffix))
+        if platform._is_mac:
+            # Check homebrew llvm
+            # paths.append('/usr/local/Cellar/llvm/')
+            paths += [
+                "/Library", "/Applications",
+                "/usr/local/Cellar/llvm/"]
         out = []
         for x in paths:
             if x and (x not in out) and os.path.isdir(x):

From 430e9a461185851f3d988977e93bd48f63ddee46 Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Mon, 4 Oct 2021 19:26:32 -0700
Subject: [PATCH 56/73] Restore previous version of R ascii_io

---
 yggdrasil/examples/ascii_io/src/ascii_io.R | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yggdrasil/examples/ascii_io/src/ascii_io.R b/yggdrasil/examples/ascii_io/src/ascii_io.R
index f03e9e5ce..039ca0a8b 100644
--- a/yggdrasil/examples/ascii_io/src/ascii_io.R
+++ b/yggdrasil/examples/ascii_io/src/ascii_io.R
@@ -44,7 +44,7 @@ while (ret) {
   if (ret) {
     # If the receive was succesful, send the values to output.
     # Formatting is taken care of on the output driver side.
-    fprintf("Table: %s, %d, %3.1f, %s", line[[1]], as.integer(line[[2]]), line[[3]], line[[4]])
+    fprintf("Table: %s, %d, %3.1f, %s", line[[1]], line[[2]], line[[3]], line[[4]])
     ret <- out_table$send(line[[1]], line[[2]], line[[3]], line[[4]])
     if (!ret) {
       stop("ascii_io(R): ERROR SENDING ROW")

From 15daa6aea670c9cf9d01c527faa759284eddc97e Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Mon, 4 Oct 2021 20:44:19 -0700
Subject: [PATCH 57/73] Add np.signedinteger and np.unsignedinteger to set of
 types last

---
 yggdrasil/constants.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/yggdrasil/constants.py b/yggdrasil/constants.py
index 9418796e7..1e6175810 100644
--- a/yggdrasil/constants.py
+++ b/yggdrasil/constants.py
@@ -26,8 +26,8 @@
 ]
 PYTHON_SCALARS = OrderedDict([
     ('float', [float]),
-    ('int', [int, np.signedinteger]),
-    ('uint', [np.unsignedinteger]),
+    ('int', [int]),
+    ('uint', []),
     ('complex', [complex]),
     ('bytes', [bytes]),
     ('unicode', [str]),
@@ -52,6 +52,8 @@
     if T in NUMPY_PRECISIONS:
         PYTHON_SCALARS[T] += [np.dtype(T_NP + str(P)).type
                               for P in NUMPY_PRECISIONS[T]]
+PYTHON_SCALARS['int'].append(np.signedinteger)
+PYTHON_SCALARS['uint'].append(np.unsignedinteger)
 ALL_PYTHON_SCALARS = []
 for k, v in PYTHON_SCALARS.items():
     PYTHON_SCALARS[k] = tuple(v)

From 9f3f1f6ae804766639f8088a033c7886b96f67bf Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Mon, 4 Oct 2021 20:50:32 -0700
Subject: [PATCH 58/73] Narrow the set of directories searched for libc++ on
 mac

---
 yggdrasil/drivers/CompiledModelDriver.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/yggdrasil/drivers/CompiledModelDriver.py b/yggdrasil/drivers/CompiledModelDriver.py
index d7b2c29ee..3f57b104d 100644
--- a/yggdrasil/drivers/CompiledModelDriver.py
+++ b/yggdrasil/drivers/CompiledModelDriver.py
@@ -924,8 +924,11 @@ def get_search_path(cls, env_only=False, libtype=None, cfg=None):
         if platform._is_mac:
             # Check homebrew llvm
             # paths.append('/usr/local/Cellar/llvm/')
+            for x in glob.glob(os.path.join(
+                    macos_sdkroot.split('/Platforms', 1)[0], 'Platforms',
+                    '*', '')):
+                paths.append(x)
             paths += [
-                "/Library", "/Applications",
                 "/usr/local/Cellar/llvm/"]
         out = []
         for x in paths:

From 03b46aec9b345fd2cc9f3f408490ce0cf592315f Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Tue, 5 Oct 2021 10:09:45 -0700
Subject: [PATCH 59/73] Discard appleTV tools in search for libc++ in xtools

---
 yggdrasil/drivers/CompiledModelDriver.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/yggdrasil/drivers/CompiledModelDriver.py b/yggdrasil/drivers/CompiledModelDriver.py
index 3f57b104d..4b42f49fe 100644
--- a/yggdrasil/drivers/CompiledModelDriver.py
+++ b/yggdrasil/drivers/CompiledModelDriver.py
@@ -927,7 +927,8 @@ def get_search_path(cls, env_only=False, libtype=None, cfg=None):
             for x in glob.glob(os.path.join(
                     macos_sdkroot.split('/Platforms', 1)[0], 'Platforms',
                     '*', '')):
-                paths.append(x)
+                if 'AppleTV' not in x:
+                    paths.append(x)
             paths += [
                 "/usr/local/Cellar/llvm/"]
         out = []

From 6ad16307018bce318711b3178fa102e57c66fb2c Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Tue, 5 Oct 2021 15:37:29 -0700
Subject: [PATCH 60/73] Discard iPhoneOS tools in search for libc++ in xtools

---
 yggdrasil/drivers/CompiledModelDriver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yggdrasil/drivers/CompiledModelDriver.py b/yggdrasil/drivers/CompiledModelDriver.py
index 4b42f49fe..37f4939b0 100644
--- a/yggdrasil/drivers/CompiledModelDriver.py
+++ b/yggdrasil/drivers/CompiledModelDriver.py
@@ -927,7 +927,7 @@ def get_search_path(cls, env_only=False, libtype=None, cfg=None):
             for x in glob.glob(os.path.join(
                     macos_sdkroot.split('/Platforms', 1)[0], 'Platforms',
                     '*', '')):
-                if 'AppleTV' not in x:
+                if ('AppleTV' not in x) and ('iPhoneOS' not in x):
                     paths.append(x)
             paths += [
                 "/usr/local/Cellar/llvm/"]

From 7dfe0737e9170e4b7b4e45f3fcb7999aff74ff73 Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Tue, 5 Oct 2021 17:54:19 -0700
Subject: [PATCH 61/73] Discard WatchOS tools in search for libc++ in xtools

---
 yggdrasil/drivers/CompiledModelDriver.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/yggdrasil/drivers/CompiledModelDriver.py b/yggdrasil/drivers/CompiledModelDriver.py
index 37f4939b0..526b43160 100644
--- a/yggdrasil/drivers/CompiledModelDriver.py
+++ b/yggdrasil/drivers/CompiledModelDriver.py
@@ -927,7 +927,8 @@ def get_search_path(cls, env_only=False, libtype=None, cfg=None):
             for x in glob.glob(os.path.join(
                     macos_sdkroot.split('/Platforms', 1)[0], 'Platforms',
                     '*', '')):
-                if ('AppleTV' not in x) and ('iPhoneOS' not in x):
+                if ((('AppleTV' not in x) and ('iPhoneOS' not in x)
+                     and ('WatchOS' not in x))):
                     paths.append(x)
             paths += [
                 "/usr/local/Cellar/llvm/"]

From ec5ada7415180f1bc9f0bb3401d40c811f25f739 Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Mon, 11 Oct 2021 14:08:36 -0700
Subject: [PATCH 62/73] Add additional sleep time to the file output driver
 tests.

---
 yggdrasil/drivers/tests/test_FileOutputDriver.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/yggdrasil/drivers/tests/test_FileOutputDriver.py b/yggdrasil/drivers/tests/test_FileOutputDriver.py
index e5ad80051..1c40e9cf4 100644
--- a/yggdrasil/drivers/tests/test_FileOutputDriver.py
+++ b/yggdrasil/drivers/tests/test_FileOutputDriver.py
@@ -93,9 +93,9 @@ def setup(self):
         super(TestFileOutputDriver, self).setup()
         self.send_file_contents()
         
-    # def run_before_stop(self):
-    #     r"""Commands to run while the instance is running."""
-    #     self.send_file_contents()
+    def run_before_stop(self):
+        r"""Commands to run while the instance is running."""
+        self.instance.wait(1.0)
 
     @property
     def contents_to_read(self):

From 8db4533aab4138f8f25bdde9bcb05f70b0830aaf Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Tue, 12 Oct 2021 14:22:41 -0500
Subject: [PATCH 63/73] Fix only_python install on linuux

---
 utils/setup_test_env.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/utils/setup_test_env.py b/utils/setup_test_env.py
index 7a04fdcc5..60d6089e7 100644
--- a/utils/setup_test_env.py
+++ b/utils/setup_test_env.py
@@ -782,7 +782,7 @@ def install_deps(method, return_commands=False, verbose=False,
         ]
     if fallback_to_conda:
         cmds.append("%s update --all" % CONDA_CMD)
-    if install_opts['R'] and (not fallback_to_conda):
+    if install_opts['R'] and (not fallback_to_conda) and (not only_python):
         # TODO: Test split installation where r-base is installed from
         # conda and the R dependencies are installed from CRAN?
         if _is_linux:

From ce1a37a2960dc56a12cb5103299228d990f32ab9 Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Tue, 12 Oct 2021 16:08:38 -0500
Subject: [PATCH 64/73] Update SBML to ignore 'init(*)' simulation parameters
 in selection and handle move to NamedArray in libroadrunner >2.1.0

---
 yggdrasil/drivers/SBMLModelDriver.py          |  6 +++++-
 yggdrasil/examples/sbml1/Output/expected.txt  | 12 ++++++------
 yggdrasil/examples/sbml2/Output/expected0.txt | 14 +++++++-------
 yggdrasil/examples/sbml2/Output/expected1.txt | 14 +++++++-------
 4 files changed, 25 insertions(+), 21 deletions(-)

diff --git a/yggdrasil/drivers/SBMLModelDriver.py b/yggdrasil/drivers/SBMLModelDriver.py
index be49f9c51..5debb6e8e 100644
--- a/yggdrasil/drivers/SBMLModelDriver.py
+++ b/yggdrasil/drivers/SBMLModelDriver.py
@@ -107,7 +107,7 @@ def model_wrapper(cls, model_file, start_time, steps,
             integrator=integrator, integrator_settings=integrator_settings,
         )
         if not selections:
-            selections = list(model.keys())
+            selections = [k for k in model.keys() if not k.startswith('init(')]
         if 'time' not in selections:
             selections = ['time'] + selections
         for k, v in output_map.items():
@@ -194,4 +194,8 @@ def call_model(cls, model, curr_time, end_time, steps,
                              steps=int(steps))
         # Unsupported?
         # variableStep=variable_step)
+        try:
+            out = {k: out[k] for k in out.colnames}
+        except IndexError:
+            out = {k: out[:, i] for i, k in enumerate(out.colnames)}
         return end_time, out
diff --git a/yggdrasil/examples/sbml1/Output/expected.txt b/yggdrasil/examples/sbml1/Output/expected.txt
index 66546638f..42c0e0122 100644
--- a/yggdrasil/examples/sbml1/Output/expected.txt
+++ b/yggdrasil/examples/sbml1/Output/expected.txt
@@ -1,6 +1,6 @@
-# time	S1	S2	S3	S4	X0	X1	[S1]	[S2]	[S3]	[S4]	[X0]	[X1]	compartment	J0_VM1	J0_Keq1	J0_h	J4_V4	J4_KS4	J0	J1	J2	J3	J4	init([S1])	init([S2])	init([S3])	init([S4])	init(S1)	init(S2)	init(S3)	init(S4)	S1'	S2'	S3'	S4'	eigen(S1)	eigenReal(S1)	eigenImag(S1)	eigen(S2)	eigenReal(S2)	eigenImag(S2)	eigen(S3)	eigenReal(S3)	eigenImag(S3)	eigen(S4)	eigenReal(S4)	eigenImag(S4)
-# %g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g
-0	0	0	0	0	10	0	0	0	0	0	10	0	1	10	10	10	2.5	0.5	9.09091	0	0	0	0	0	0	0	0	0	0	0	0	9.09091	0	0	0	0	-18.6453	0	0	-13.0021	0	0	-6.96727	0	0	-3.29967	0
-5	1.2678	0.730189	0.707615	1.1401	10	0	1.2678	0.730189	0.707615	1.1401	10	0	1	10	10	10	2.5	0.5	6.17918	3.74171	2.41474	1.68414	1.73785	0	0	0	0	0	0	0	0	2.43747	1.32697	0.730594	-0.0537064	0	0.0124582	2.20188	0	0.0124582	-2.20188	0	-7.03899	1.64292	0	-7.03899	-1.64292
-5	5	5	0.707615	1.1401	10	0	5	5	0.707615	1.1401	10	0	1	10	10	10	2.5	0.5	4.81979	3.63636	7.24323	1.68414	1.73785	0	0	0	0	0	0	0	0	1.18343	-3.60686	5.55908	-0.0537064	0	-5.39048	0	0	-2.1071	0	0	-0.178106	0.740418	0	-0.178106	-0.740418
-10	0.0498036	0.15262	0.582381	1.81007	10	0	0.0498036	0.15262	0.582381	1.81007	10	0	1	10	10	10	2.5	0.5	0.257216	0.160339	0.208324	0.64958	1.95889	0	0	0	0	0	0	0	0	0.0968772	-0.0479853	-0.441256	-1.30931	0	-11.8326	0	0	-0.906626	1.28292	0	-0.906626	-1.28292	0	-7.02511	0
+# time	S1	S2	S3	S4	X0	X1	[S1]	[S2]	[S3]	[S4]	[X0]	[X1]	compartment	J0_VM1	J0_Keq1	J0_h	J4_V4	J4_KS4	J0	J1	J2	J3	J4	S1'	S2'	S3'	S4'	eigen(S1)	eigenReal(S1)	eigenImag(S1)	eigen(S2)	eigenReal(S2)	eigenImag(S2)	eigen(S3)	eigenReal(S3)	eigenImag(S3)	eigen(S4)	eigenReal(S4)	eigenImag(S4)
+# %g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g
+0	0	0	0	0	10	0	0	0	0	0	10	0	1	10	10	10	2.5	0.5	9.09091	0	0	0	0	9.09091	0	0	0	0	-18.6453	0	0	-13.0021	0	0	-6.96727	0	0	-3.29967	0
+5	1.2678	0.730189	0.707615	1.1401	10	0	1.2678	0.730189	0.707615	1.1401	10	0	1	10	10	10	2.5	0.5	6.17918	3.74171	2.41474	1.68414	1.73785	2.43747	1.32697	0.730594	-0.0537064	0	0.0124582	2.20188	0	0.0124582	-2.20188	0	-7.03899	1.64292	0	-7.03899	-1.64292
+5	5	5	0.707615	1.1401	10	0	5	5	0.707615	1.1401	10	0	1	10	10	10	2.5	0.5	4.81979	3.63636	7.24323	1.68414	1.73785	1.18343	-3.60686	5.55908	-0.0537064	0	-5.39048	0	0	-2.1071	0	0	-0.178106	0.740418	0	-0.178106	-0.740418
+10	0.0498036	0.15262	0.582381	1.81007	10	0	0.0498036	0.15262	0.582381	1.81007	10	0	1	10	10	10	2.5	0.5	0.257216	0.160339	0.208324	0.64958	1.95889	0.0968772	-0.0479853	-0.441256	-1.30931	0	-11.8326	0	0	-0.906626	1.28292	0	-0.906626	-1.28292	0	-7.02511	0
diff --git a/yggdrasil/examples/sbml2/Output/expected0.txt b/yggdrasil/examples/sbml2/Output/expected0.txt
index 82f015283..344217fb3 100644
--- a/yggdrasil/examples/sbml2/Output/expected0.txt
+++ b/yggdrasil/examples/sbml2/Output/expected0.txt
@@ -1,7 +1,7 @@
-# time	S1	S2	S3	S4	X0	X1	[S1]	[S2]	[S3]	[S4]	[X0]	[X1]	compartment	J0_VM1	J0_Keq1	J0_h	J4_V4	J4_KS4	J0	J1	J2	J3	J4	init([S1])	init([S2])	init([S3])	init([S4])	init(S1)	init(S2)	init(S3)	init(S4)	S1'	S2'	S3'	S4'	eigen(S1)	eigenReal(S1)	eigenImag(S1)	eigen(S2)	eigenReal(S2)	eigenImag(S2)	eigen(S3)	eigenReal(S3)	eigenImag(S3)	eigen(S4)	eigenReal(S4)	eigenImag(S4)
-# %g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g
-1	3.15087	1.54712	1.0105	0.948797	10	0	3.15087	1.54712	1.0105	0.948797	10	0	1	10	10	10	2.5	0.5	6.56958	4.98675	3.78067	2.77343	1.63722	0	0	0	0	0	0	0	0	1.58283	1.20608	1.00725	1.13621	0	-0.545008	0.708568	0	-0.545008	-0.708568	0	-3.61427	0	0	-5.93877	0
-2	1.4231	1.87524	1.79238	2.00936	10	0	1.4231	1.87524	1.79238	2.00936	10	0	1	10	10	10	2.5	0.5	0.0908234	2.43828	3.24954	2.89584	2.00187	0	0	0	0	0	0	0	0	-2.34746	-0.811256	0.353695	0.893978	0	-4.32885	0	0	-2.88955	0	0	-0.42144	0.12732	0	-0.42144	-0.12732
-3	0.23736	0.765993	1.559	2.50695	10	0	0.23736	0.765993	1.559	2.50695	10	0	1	10	10	10	2.5	0.5	0.010163	0.420104	1.36599	2.08769	2.0843	0	0	0	0	0	0	0	0	-0.409941	-0.945889	-0.721698	0.00339591	0	-7.00809	0	0	-3.63495	0	0	-1.36335	0	0	-0.0996989	0
-4	0.0626202	0.227289	0.777257	1.97528	10	0	0.0626202	0.227289	0.777257	1.97528	10	0	1	10	10	10	2.5	0.5	0.109183	0.133051	0.358375	1.01851	1.99501	0	0	0	0	0	0	0	0	-0.0238673	-0.225324	-0.660136	-0.976496	0	-10.9185	0	0	-5.78325	0	0	-0.995338	0.228864	0	-0.995338	-0.228864
-5	1.26783	0.730205	0.707624	1.1401	10	0	1.26783	0.730205	0.707624	1.1401	10	0	1	10	10	10	2.5	0.5	6.17918	3.74175	2.41477	1.68417	1.73785	0	0	0	0	0	0	0	0	2.43743	1.32697	0.730603	-0.0536798	0	0.0124502	2.20185	0	0.0124502	-2.20185	0	-7.03892	1.64289	0	-7.03892	-1.64289
+# time	S1	S2	S3	S4	X0	X1	[S1]	[S2]	[S3]	[S4]	[X0]	[X1]	compartment	J0_VM1	J0_Keq1	J0_h	J4_V4	J4_KS4	J0	J1	J2	J3	J4	S1'	S2'	S3'	S4'	eigen(S1)	eigenReal(S1)	eigenImag(S1)	eigen(S2)	eigenReal(S2)	eigenImag(S2)	eigen(S3)	eigenReal(S3)	eigenImag(S3)	eigen(S4)	eigenReal(S4)	eigenImag(S4)
+# %g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g
+1	3.15087	1.54712	1.0105	0.948797	10	0	3.15087	1.54712	1.0105	0.948797	10	0	1	10	10	10	2.5	0.5	6.56958	4.98675	3.78067	2.77343	1.63722	1.58283	1.20608	1.00725	1.13621	0	-0.545008	0.708568	0	-0.545008	-0.708568	0	-3.61427	0	0	-5.93877	0
+2	1.4231	1.87524	1.79238	2.00936	10	0	1.4231	1.87524	1.79238	2.00936	10	0	1	10	10	10	2.5	0.5	0.0908234	2.43828	3.24954	2.89584	2.00187	-2.34746	-0.811256	0.353695	0.893978	0	-4.32885	0	0	-2.88955	0	0	-0.42144	0.12732	0	-0.42144	-0.12732
+3	0.23736	0.765993	1.559	2.50695	10	0	0.23736	0.765993	1.559	2.50695	10	0	1	10	10	10	2.5	0.5	0.010163	0.420104	1.36599	2.08769	2.0843	-0.409941	-0.945889	-0.721698	0.00339591	0	-7.00809	0	0	-3.63495	0	0	-1.36335	0	0	-0.0996989	0
+4	0.0626202	0.227289	0.777257	1.97528	10	0	0.0626202	0.227289	0.777257	1.97528	10	0	1	10	10	10	2.5	0.5	0.109183	0.133051	0.358375	1.01851	1.99501	-0.0238673	-0.225324	-0.660136	-0.976496	0	-10.9185	0	0	-5.78325	0	0	-0.995338	0.228864	0	-0.995338	-0.228864
+5	1.26783	0.730205	0.707624	1.1401	10	0	1.26783	0.730205	0.707624	1.1401	10	0	1	10	10	10	2.5	0.5	6.17918	3.74175	2.41477	1.68417	1.73785	2.43743	1.32697	0.730603	-0.0536798	0	0.0124502	2.20185	0	0.0124502	-2.20185	0	-7.03892	1.64289	0	-7.03892	-1.64289
diff --git a/yggdrasil/examples/sbml2/Output/expected1.txt b/yggdrasil/examples/sbml2/Output/expected1.txt
index 52ca61887..7a0f37d5a 100644
--- a/yggdrasil/examples/sbml2/Output/expected1.txt
+++ b/yggdrasil/examples/sbml2/Output/expected1.txt
@@ -1,7 +1,7 @@
-# time	S1	S2	S3	S4	X0	X1	[S1]	[S2]	[S3]	[S4]	[X0]	[X1]	compartment	J0_VM1	J0_Keq1	J0_h	J4_V4	J4_KS4	J0	J1	J2	J3	J4	init([S1])	init([S2])	init([S3])	init([S4])	init(S1)	init(S2)	init(S3)	init(S4)	S1'	S2'	S3'	S4'	eigen(S1)	eigenReal(S1)	eigenImag(S1)	eigen(S2)	eigenReal(S2)	eigenImag(S2)	eigen(S3)	eigenReal(S3)	eigenImag(S3)	eigen(S4)	eigenReal(S4)	eigenImag(S4)
-# %g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g
-2	1.4231	1.87524	1.79238	2.00936	10	0	1.4231	1.87524	1.79238	2.00936	10	0	1	10	10	10	2.5	0.5	0.0908234	2.43828	3.24954	2.89584	2.00187	0	0	0	0	0	0	0	0	-2.34746	-0.811256	0.353695	0.893978	0	-4.32885	0	0	-2.88955	0	0	-0.42144	0.12732	0	-0.42144	-0.12732
-4	0.0626202	0.227289	0.777257	1.97528	10	0	0.0626202	0.227289	0.777257	1.97528	10	0	1	10	10	10	2.5	0.5	0.109183	0.133051	0.358375	1.01851	1.99501	0	0	0	0	0	0	0	0	-0.0238673	-0.225324	-0.660136	-0.976496	0	-10.9185	0	0	-5.78325	0	0	-0.995338	0.228864	0	-0.995338	-0.228864
-6	1.10813	1.34748	1.36601	1.65329	10	0	1.10813	1.34748	1.36601	1.65329	10	0	1	10	10	10	2.5	0.5	0.600504	2.42687	2.89291	2.57596	1.91949	0	0	0	0	0	0	0	0	-1.82637	-0.466031	0.316941	0.656473	0	-0.245455	1.08378	0	-0.245455	-1.08378	0	-4.73903	0.621603	0	-4.73903	-0.621603
-8	0.441469	0.3727	0.603703	1.31728	10	0	0.441469	0.3727	0.603703	1.31728	10	0	1	10	10	10	2.5	0.5	3.66378	2.02257	1.27484	1.16483	1.81216	0	0	0	0	0	0	0	0	1.64121	0.747733	0.110007	-0.647327	0	0.412044	3.25937	0	0.412044	-3.25937	0	-9.08432	2.69667	0	-9.08432	-2.69667
-10	0.269431	0.678094	1.19936	1.86825	10	0	0.269431	0.678094	1.19936	1.86825	10	0	1	10	10	10	2.5	0.5	0.18842	0.687087	1.52296	2.02995	1.97218	0	0	0	0	0	0	0	0	-0.498667	-0.83587	-0.506996	0.0577685	0	-0.652073	0.714291	0	-0.652073	-0.714291	0	-7.36911	0	0	-4.83401	0
+# time	S1	S2	S3	S4	X0	X1	[S1]	[S2]	[S3]	[S4]	[X0]	[X1]	compartment	J0_VM1	J0_Keq1	J0_h	J4_V4	J4_KS4	J0	J1	J2	J3	J4	S1'	S2'	S3'	S4'	eigen(S1)	eigenReal(S1)	eigenImag(S1)	eigen(S2)	eigenReal(S2)	eigenImag(S2)	eigen(S3)	eigenReal(S3)	eigenImag(S3)	eigen(S4)	eigenReal(S4)	eigenImag(S4)
+# %g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g	%g
+2	1.4231	1.87524	1.79238	2.00936	10	0	1.4231	1.87524	1.79238	2.00936	10	0	1	10	10	10	2.5	0.5	0.0908234	2.43828	3.24954	2.89584	2.00187	-2.34746	-0.811256	0.353695	0.893978	0	-4.32885	0	0	-2.88955	0	0	-0.42144	0.12732	0	-0.42144	-0.12732
+4	0.0626202	0.227289	0.777257	1.97528	10	0	0.0626202	0.227289	0.777257	1.97528	10	0	1	10	10	10	2.5	0.5	0.109183	0.133051	0.358375	1.01851	1.99501	-0.0238673	-0.225324	-0.660136	-0.976496	0	-10.9185	0	0	-5.78325	0	0	-0.995338	0.228864	0	-0.995338	-0.228864
+6	1.10813	1.34748	1.36601	1.65329	10	0	1.10813	1.34748	1.36601	1.65329	10	0	1	10	10	10	2.5	0.5	0.600504	2.42687	2.89291	2.57596	1.91949	-1.82637	-0.466031	0.316941	0.656473	0	-0.245455	1.08378	0	-0.245455	-1.08378	0	-4.73903	0.621603	0	-4.73903	-0.621603
+8	0.441469	0.3727	0.603703	1.31728	10	0	0.441469	0.3727	0.603703	1.31728	10	0	1	10	10	10	2.5	0.5	3.66378	2.02257	1.27484	1.16483	1.81216	1.64121	0.747733	0.110007	-0.647327	0	0.412044	3.25937	0	0.412044	-3.25937	0	-9.08432	2.69667	0	-9.08432	-2.69667
+10	0.269431	0.678094	1.19936	1.86825	10	0	0.269431	0.678094	1.19936	1.86825	10	0	1	10	10	10	2.5	0.5	0.18842	0.687087	1.52296	2.02995	1.97218	-0.498667	-0.83587	-0.506996	0.0577685	0	-0.652073	0.714291	0	-0.652073	-0.714291	0	-7.36911	0	0	-4.83401	0

From b302d57914e9637575c165a3a15a3dc4850b7633 Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Tue, 12 Oct 2021 17:01:08 -0700
Subject: [PATCH 65/73] SBML change to dict not necessary after removing
 'init(*)' parameters from test Add tests and remove unsed code for coverage

---
 yggdrasil/drivers/CModelDriver.py    | 12 +-----------
 yggdrasil/drivers/SBMLModelDriver.py |  8 ++++----
 yggdrasil/tests/test_runner.py       |  2 +-
 3 files changed, 6 insertions(+), 16 deletions(-)

diff --git a/yggdrasil/drivers/CModelDriver.py b/yggdrasil/drivers/CModelDriver.py
index 14871dfeb..478036324 100755
--- a/yggdrasil/drivers/CModelDriver.py
+++ b/yggdrasil/drivers/CModelDriver.py
@@ -143,17 +143,7 @@ def get_search_path(cls, *args, **kwargs):
             list: List of paths that the tools will search.
 
         """
-        cfg = kwargs.get('cfg', ygg_cfg)
-        libtype = kwargs.get('libtype', None)
-        out = super(CompilerBase, cls).get_search_path(*args, **kwargs)
-        macos_sdkroot = cfg.get('c', 'macos_sdkroot', _osx_sysroot)
-        if platform._is_mac and (macos_sdkroot is not None):
-            base_path = os.path.join(macos_sdkroot, 'usr')
-            assert(libtype == 'include')
-            new = os.path.join(base_path, 'include')
-            if new not in out:
-                out.append(new)
-        return out
+        return super(CompilerBase, cls).get_search_path(*args, **kwargs)
 
 
 class GCCCompiler(CCompilerBase):
diff --git a/yggdrasil/drivers/SBMLModelDriver.py b/yggdrasil/drivers/SBMLModelDriver.py
index 5debb6e8e..8c49e3e62 100644
--- a/yggdrasil/drivers/SBMLModelDriver.py
+++ b/yggdrasil/drivers/SBMLModelDriver.py
@@ -194,8 +194,8 @@ def call_model(cls, model, curr_time, end_time, steps,
                              steps=int(steps))
         # Unsupported?
         # variableStep=variable_step)
-        try:
-            out = {k: out[k] for k in out.colnames}
-        except IndexError:
-            out = {k: out[:, i] for i, k in enumerate(out.colnames)}
+        # try:
+        #     out = {k: out[k] for k in out.colnames}
+        # except IndexError:
+        #     out = {k: out[:, i] for i, k in enumerate(out.colnames)}
         return end_time, out
diff --git a/yggdrasil/tests/test_runner.py b/yggdrasil/tests/test_runner.py
index f61abd77c..3b024a2be 100644
--- a/yggdrasil/tests/test_runner.py
+++ b/yggdrasil/tests/test_runner.py
@@ -13,7 +13,7 @@ def test_get_runner():
     r"""Use get_runner to start a run."""
     namespace = "test_get_runner_%s" % str(uuid.uuid4)
     cr = runner.get_runner([ex_yamls['hello']['python']],
-                           namespace=namespace)
+                           namespace=namespace, validate=True)
     cr.run()
     cr.sleep()
 

From 35b2ef732c28697e4752f50eaba1194cbdf5498e Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Tue, 12 Oct 2021 18:21:16 -0700
Subject: [PATCH 66/73] Add test for creation of schema/constants from scratch

---
 yggdrasil/.ygg_schema.yml      |  8 ++++----
 yggdrasil/command_line.py      | 35 ++++++++++++++++++----------------
 yggdrasil/components.py        | 15 ++++++---------
 yggdrasil/constants.py         |  2 +-
 yggdrasil/schema.py            | 29 ++++++++++++++++------------
 yggdrasil/tests/test_schema.py | 26 +++++++++++++++++++++++--
 6 files changed, 71 insertions(+), 44 deletions(-)

diff --git a/yggdrasil/.ygg_schema.yml b/yggdrasil/.ygg_schema.yml
index 6e4a4d057..549faf439 100644
--- a/yggdrasil/.ygg_schema.yml
+++ b/yggdrasil/.ygg_schema.yml
@@ -180,9 +180,9 @@ definitions:
             type: string
           type: array
       required:
-      - name
-      - datatype
       - commtype
+      - datatype
+      - name
       title: comm_base
       type: object
     - anyOf:
@@ -704,9 +704,9 @@ definitions:
             is used.
           type: string
       required:
+      - filetype
       - name
       - working_dir
-      - filetype
       title: file_base
       type: object
     - anyOf:
@@ -1856,8 +1856,8 @@ definitions:
             is used.
           type: string
       required:
-      - language
       - args
+      - language
       - name
       - working_dir
       title: model_base
diff --git a/yggdrasil/command_line.py b/yggdrasil/command_line.py
index 3a024008f..405714334 100755
--- a/yggdrasil/command_line.py
+++ b/yggdrasil/command_line.py
@@ -7,7 +7,9 @@
 import argparse
 import pprint
 import shutil
-from yggdrasil.constants import LANGUAGES, LANGUAGES_WITH_ALIASES
+from yggdrasil import constants
+LANGUAGES = getattr(constants, 'LANGUAGES', {})
+LANGUAGES_WITH_ALIASES = getattr(constants, 'LANGUAGES_WITH_ALIASES', {})
 
 
 logger = logging.getLogger(__name__)
@@ -139,7 +141,7 @@ def parse_args(cls, parser, args=None, allow_unknown=False,
                 if isinstance(v_flag, list):
                     v.extend(v_flag)
                 if (len(v) == 0) or ('all' in v):
-                    setattr(args, k, LANGUAGES['all'])
+                    setattr(args, k, LANGUAGES.get('all', []))
                     args.all_languages = True
         return args
 
@@ -468,7 +470,7 @@ class ygginfo(SubCommand):
             description='Compilation tool types to get info about.',
             arguments=[
                 (('language', ),
-                 {'choices': LANGUAGES_WITH_ALIASES['compiled'],
+                 {'choices': LANGUAGES_WITH_ALIASES.get('compiled', []),
                   'type': str.lower,
                   'help': 'Language to get tool information for.'}),
                 (('--toolname', ),
@@ -878,7 +880,8 @@ class yggcc(SubCommand):
                    "R package.")}),
         (('--language', ),
          {'default': None,
-          'choices': [None] + LANGUAGES_WITH_ALIASES['compiled'] + ['R', 'r'],
+          'choices': ([None] + LANGUAGES_WITH_ALIASES.get('compiled', [])
+                      + ['R', 'r']),
           'help': ("Language of the source code. If not provided, "
                    "the language will be determined from the "
                    "source extension.")}),
@@ -921,8 +924,8 @@ class yggcompile(SubCommand):
     arguments = [
         (('language', ),
          {'nargs': '*', 'default': ['all'],
-          # 'choices': (['all'] + LANGUAGES_WITH_ALIASES['compiled']
-          #             + LANGUAGES_WITH_ALIASES['compiled_dsl']),
+          # 'choices': (['all'] + LANGUAGES_WITH_ALIASES.get('compiled', [])
+          #             + LANGUAGES_WITH_ALIASES.get('compiled_dsl', [])),
           'help': ("One or more languages to compile dependencies "
                    "for.")}),
         (('--toolname', ),
@@ -957,7 +960,7 @@ class yggclean(SubCommand):
     arguments = [
         (('language', ),
          {'nargs': '*', 'default': ['all'],
-          # 'choices': ['all'] + LANGUAGES_WITH_ALIASES['all'],
+          # 'choices': ['all'] + LANGUAGES_WITH_ALIASES.get('all', []),
           'help': ("One or more languages to clean up dependencies "
                    "for.")})]
 
@@ -1098,12 +1101,12 @@ class update_config(SubCommand):
     arguments = (
         [(('languages', ),
           {'nargs': '*',
-           # 'choices': ['all'] + LANGUAGES_WITH_ALIASES['all'],
+           # 'choices': ['all'] + LANGUAGES_WITH_ALIASES.get('all', []),
            'default': [],
            'help': 'One or more languages that should be configured.'}),
          (('--languages', ),
           {'nargs': '+', 'dest': 'languages_flag',
-           # 'choices': ['all'] + LANGUAGES_WITH_ALIASES['all'],
+           # 'choices': ['all'] + LANGUAGES_WITH_ALIASES.get('all', []),
            'default': [],
            'help': 'One or more languages that should be configured.'}),
          (('--show-file', ),
@@ -1117,11 +1120,11 @@ class update_config(SubCommand):
            'help': 'Overwrite the existing file.'}),
          (('--disable-languages', ),
           {'nargs': '+', 'default': [],
-           'choices': LANGUAGES_WITH_ALIASES['all'],
+           'choices': LANGUAGES_WITH_ALIASES.get('all', []),
            'help': 'One or more languages that should be disabled.'}),
          (('--enable-languages', ),
           {'nargs': '+', 'default': [],
-           'choices': LANGUAGES_WITH_ALIASES['all'],
+           'choices': LANGUAGES_WITH_ALIASES.get('all', []),
            'help': 'One or more languages that should be enabled.'}),
          (('--quiet', '-q'),
           {'action': 'store_true',
@@ -1142,15 +1145,15 @@ class update_config(SubCommand):
         + [(('--%s-compiler' % k, ),
             {'help': ('Name or path to compiler that should be used to compile '
                       'models written in %s.' % k)})
-           for k in LANGUAGES['compiled']]
+           for k in LANGUAGES.get('compiled', [])]
         + [(('--%s-linker' % k, ),
             {'help': ('Name or path to linker that should be used to link '
                       'models written in %s.' % k)})
-           for k in LANGUAGES['compiled']]
+           for k in LANGUAGES.get('compiled', [])]
         + [(('--%s-archiver' % k, ),
             {'help': ('Name or path to archiver that should be used to create '
                       'static libraries for models written in %s.' % k)})
-           for k in LANGUAGES['compiled']]
+           for k in LANGUAGES.get('compiled', [])]
     )
     # TODO: Move these into the language directories?
     language_arguments = {
@@ -1211,7 +1214,7 @@ def add_arguments(cls, parser, **kwargs):
         if preargs.languages_flag:
             prelang += preargs.languages_flag
         if (len(prelang) == 0) or ('all' in prelang):
-            prelang = LANGUAGES['all']
+            prelang = LANGUAGES.get('all', [])
         # TODO: The languages could be subparsers
         for k, v in cls.language_arguments.items():
             if k in prelang:
@@ -1243,7 +1246,7 @@ def func(cls, args):
                     lang_kwargs.setdefault(k, {})
                     lang_kwargs[k][name] = getattr(args, name)
         for x in ['compiler', 'linker', 'archiver']:
-            for k in LANGUAGES['compiled']:
+            for k in LANGUAGES.get('compiled', []):
                 if getattr(args, '%s_%s' % (k, x), None):
                     lang_kwargs.setdefault(k, {})
                     lang_kwargs[k][x] = getattr(args, '%s_%s' % (k, x))
diff --git a/yggdrasil/components.py b/yggdrasil/components.py
index dd0ed6d6e..fe42bf131 100644
--- a/yggdrasil/components.py
+++ b/yggdrasil/components.py
@@ -88,11 +88,15 @@ def registering(recurse=False):
     if not recurse:
         assert(not registration_in_progress())
     try:
+        previous = os.environ.get('YGGDRASIL_REGISTRATION_IN_PROGRESS', None)
         os.environ['YGGDRASIL_REGISTRATION_IN_PROGRESS'] = '1'
         yield
     finally:
-        if 'YGGDRASIL_REGISTRATION_IN_PROGRESS' in os.environ:
-            del os.environ['YGGDRASIL_REGISTRATION_IN_PROGRESS']
+        if previous is None:
+            if 'YGGDRASIL_REGISTRATION_IN_PROGRESS' in os.environ:
+                del os.environ['YGGDRASIL_REGISTRATION_IN_PROGRESS']
+        else:
+            os.environ['YGGDRASIL_REGISTRATION_IN_PROGRESS'] = previous
 
 
 def init_registry(recurse=False):
@@ -598,13 +602,6 @@ def __init__(self, skip_component_schema_normalization=None, **kwargs):
             #                   % (k, v, getattr(self, k)))
         self.extra_kwargs = kwargs
 
-    @staticmethod
-    def before_schema(cls):
-        r"""Operations that should be performed before the schema is generated.
-        These actions will only be performed if a new schema is being generated.
-        """
-        pass
-
     @staticmethod
     def before_registration(cls):
         r"""Operations that should be performed to modify class attributes prior
diff --git a/yggdrasil/constants.py b/yggdrasil/constants.py
index 1e6175810..e755846c5 100644
--- a/yggdrasil/constants.py
+++ b/yggdrasil/constants.py
@@ -259,7 +259,7 @@
     'compiled': [
         'c', 'c++', 'cpp', 'cxx', 'fortran'],
     'interpreted': [
-        'R', 'r', 'matlab', 'python'],
+        'R', 'matlab', 'python', 'r'],
     'build': [
         'cmake', 'make'],
     'dsl': [
diff --git a/yggdrasil/schema.py b/yggdrasil/schema.py
index 85c511af6..5c446137e 100644
--- a/yggdrasil/schema.py
+++ b/yggdrasil/schema.py
@@ -12,6 +12,11 @@
 _schema_fname = os.path.abspath(os.path.join(
     os.path.dirname(__file__), '.ygg_schema.yml'))
 _schema = None
+_constants_separator = (
+    "\n# ======================================================\n"
+    "# Do not edit this file past this point as the following\n"
+    "# is generated by yggdrasil.schema.update_constants\n"
+    "# ======================================================\n")
 
 
 class SchemaDict(OrderedDict):
@@ -244,11 +249,6 @@ def as_lines(x, newline='\n', key_order=None):
             out += repr(x)
         return out
 
-    separator = (
-        "\n# ======================================================\n"
-        "# Do not edit this file past this point as the following\n"
-        "# is generated by yggdrasil.schema.update_constants\n"
-        "# ======================================================\n")
     filename = os.path.join(os.path.dirname(__file__), 'constants.py')
     # Component information
     component_registry = {}
@@ -260,8 +260,8 @@ def as_lines(x, newline='\n', key_order=None):
             'key': v.subtype_key,
             'subtypes': v.subtype2class}
     # Language driver information
-    drivers = {k: import_component('model', k)
-               for k in schema['model'].subtypes}
+    drivers = {k: import_component('model', v)
+               for k, v in component_registry['model']['subtypes'].items()}
     language_cat = ['compiled', 'interpreted', 'build', 'dsl', 'other']
     typemap = {'compiler': 'compiled', 'interpreter': 'interpreted'}
     lang2ext = {'yaml': '.yml', 'executable': '.exe'}
@@ -286,6 +286,9 @@ def as_lines(x, newline='\n', key_order=None):
         languages_with_aliases.setdefault(drv_type, [])
         languages_with_aliases[drv_type].append(drv.language)
         languages_with_aliases[drv_type] += drv.language_aliases
+    languages = {k: sorted(v) for k, v in languages.items()}
+    languages_with_aliases = {k: sorted(v) for k, v in
+                              languages_with_aliases.items()}
     for x in ['compiler', 'linker', 'archiver']:
         reg = get_compilation_tool_registry(x).get('by_language', {})
         for lang, tools in reg.items():
@@ -300,7 +303,8 @@ def as_lines(x, newline='\n', key_order=None):
                     compiler_env_vars[lang] = compilation_tool_vars[k].copy()
     language_cat = list(languages.keys())
     with open(filename, 'r') as fd:
-        lines = [fd.read().split(separator)[0], separator[1:]]
+        lines = [fd.read().split(_constants_separator)[0],
+                 _constants_separator[1:]]
     lines += [
         "", "# Component registry",
         f"COMPONENT_REGISTRY = {as_lines(component_registry)}"]
@@ -442,8 +446,9 @@ def get_subtype_schema(self, subtype, unique=False, relaxed=False,
             if unique:
                 out['additionalProperties'] = True
                 if 'required' in out:
-                    out['required'] = list(set(out['required'])
-                                           - set(self._base_schema.get('required', [])))
+                    out['required'] = sorted(
+                        list(set(out['required'])
+                             - set(self._base_schema.get('required', []))))
                     if not out['required']:
                         del out['required']
                 for k in self._base_schema['properties'].keys():
@@ -745,9 +750,9 @@ def append(self, comp_cls, verify=False):
         # Update base schema, checking for compatiblity
         if not is_base:
             if 'required' in self._base_schema:
-                self._base_schema['required'] = list(
+                self._base_schema['required'] = sorted(list(
                     set(self._base_schema['required'])
-                    & set(new_schema.get('required', [])))
+                    & set(new_schema.get('required', []))))
                 if not self._base_schema['required']:  # pragma: no cover
                     del self._base_schema['required']
             prop_overlap = list(
diff --git a/yggdrasil/tests/test_schema.py b/yggdrasil/tests/test_schema.py
index 32d8eff6b..f7033ac2c 100644
--- a/yggdrasil/tests/test_schema.py
+++ b/yggdrasil/tests/test_schema.py
@@ -1,6 +1,7 @@
 import os
 import pprint
 import tempfile
+import subprocess
 from jsonschema import ValidationError
 from yggdrasil import schema, components
 from yggdrasil.tests import assert_raises, assert_equal
@@ -102,12 +103,33 @@ def test_default_schema():
 
 
 def test_create_schema():
-    r"""Test creating new schema."""
+    r"""Test re-creating the schema."""
+    f_schema = schema._schema_fname
+    f_consts = os.path.join(os.path.dirname(schema.__file__), 'constants.py')
+    old_schema = open(f_schema, 'r').read()
+    old_consts = open(f_consts, 'r').read()
+    try:
+        os.remove(f_schema)
+        open(f_consts, 'w').write(
+            old_consts.split(schema._constants_separator)[0]
+            + schema._constants_separator)
+        subprocess.check_call(['yggschema'])
+        new_schema = open(f_schema, 'r').read()
+        new_consts = open(f_consts, 'r').read()
+        assert(new_consts == old_consts)
+        assert(new_schema == old_schema)
+    finally:
+        open(f_schema, 'w').write(old_schema)
+        open(f_consts, 'w').write(old_consts)
+
+
+def test_save_load_schema():
+    r"""Test saving & loading schema."""
     fname = 'test_schema.yml'
     if os.path.isfile(fname):  # pragma: debug
         os.remove(fname)
     # Test saving/loading schema
-    s0 = schema.create_schema()
+    s0 = schema.load_schema()
     s0.save(fname)
     assert(s0 is not None)
     assert(os.path.isfile(fname))

From c314c1844afdb9dfdc19472247a0697fcd755d74 Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Tue, 12 Oct 2021 18:33:29 -0700
Subject: [PATCH 67/73] Remove unused code

---
 yggdrasil/schema.py | 41 ++++++++++++++++-------------------------
 yggdrasil/tools.py  |  2 +-
 2 files changed, 17 insertions(+), 26 deletions(-)

diff --git a/yggdrasil/schema.py b/yggdrasil/schema.py
index 5c446137e..3c8ab0a9f 100644
--- a/yggdrasil/schema.py
+++ b/yggdrasil/schema.py
@@ -639,27 +639,21 @@ def base_subtype_class_name(self):
     def base_subtype_class(self):
         r"""ComponentClass: Base class for the subtype."""
         if not getattr(self, '_base_subtype_class', None):
-            if getattr(self, '_base_subtype_class_name', None):
-                self._base_subtype_class = getattr(
-                    importlib.import_module(
-                        f"{self.module}.{self._base_subtype_class_name}"),
-                    self._base_subtype_class_name)
-            else:
-                default_class = list(self.schema_subtypes.keys())[0]
-                cls = getattr(
-                    importlib.import_module(f"{self.module}.{default_class}"),
-                    default_class)
-                base_class = cls
-                for i, x in enumerate(cls.__mro__):
-                    if x._schema_type != cls._schema_type:
-                        break
-                    base_class = x
-                else:  # pragma: debug
-                    raise RuntimeError(
-                        f"Could not determine a base class for "
-                        f"{self.schema_type} (using class {cls})")
-                self._base_subtype_class = base_class
-                self._base_subtype_class_name = base_class.__name__
+            default_class = list(self.schema_subtypes.keys())[0]
+            cls = getattr(
+                importlib.import_module(f"{self.module}.{default_class}"),
+                default_class)
+            base_class = cls
+            for i, x in enumerate(cls.__mro__):
+                if x._schema_type != cls._schema_type:
+                    break
+                base_class = x
+            else:  # pragma: debug
+                raise RuntimeError(
+                    f"Could not determine a base class for "
+                    f"{self.schema_type} (using class {cls})")
+            self._base_subtype_class = base_class
+            self._base_subtype_class_name = base_class.__name__
         return self._base_subtype_class
 
     @property
@@ -701,10 +695,7 @@ def append(self, comp_cls, verify=False):
             subtype_list = [subtype_list]
         subtype_list += getattr(comp_cls, '_%s_aliases' % self.subtype_key, [])
         self.schema_subtypes[name] = subtype_list
-        if self.module is None:
-            self.module = subtype_module
-        else:
-            assert(subtype_module == self.module)
+        assert(subtype_module == self.module)
         # Create new schema for subtype
         new_schema = {'title': fullname,
                       'description': ('Schema for %s component %s subtype.'
diff --git a/yggdrasil/tools.py b/yggdrasil/tools.py
index b87ab4537..49d8a66c0 100644
--- a/yggdrasil/tools.py
+++ b/yggdrasil/tools.py
@@ -1292,7 +1292,7 @@ def import_all_modules(base=None, exclude=None, do_first=None):
             continue
         importlib.import_module(x_mod)
     for x in sorted(glob.glob(os.path.join(directory, '*', ''))):
-        if x.startswith('__') and x.endswith('__'):
+        if x.startswith('__') and x.endswith('__'):  # pragma: debug
             continue
         next_module = os.path.basename(os.path.dirname(x))
         import_all_modules(f"{base}.{next_module}",

From 12c03c25aad7aad977cf51bf89ce95bb902b8f62 Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Wed, 13 Oct 2021 12:00:29 -0700
Subject: [PATCH 68/73] Change equality assertion in schema test to use ==

---
 yggdrasil/tests/test_schema.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yggdrasil/tests/test_schema.py b/yggdrasil/tests/test_schema.py
index f7033ac2c..a72f63c24 100644
--- a/yggdrasil/tests/test_schema.py
+++ b/yggdrasil/tests/test_schema.py
@@ -140,7 +140,7 @@ def test_save_load_schema():
     # Test getting schema
     s2 = schema.load_schema(fname)
     assert(os.path.isfile(fname))
-    assert_equal(s2, s0)
+    assert(s2 == s0)
     os.remove(fname)
 
 

From 4be941b040d130d8f7dbc2afa890e19f95048ece Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Wed, 13 Oct 2021 13:43:54 -0700
Subject: [PATCH 69/73] Check comparison of schemas directly

---
 yggdrasil/tests/test_schema.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/yggdrasil/tests/test_schema.py b/yggdrasil/tests/test_schema.py
index a72f63c24..498b62160 100644
--- a/yggdrasil/tests/test_schema.py
+++ b/yggdrasil/tests/test_schema.py
@@ -140,6 +140,7 @@ def test_save_load_schema():
     # Test getting schema
     s2 = schema.load_schema(fname)
     assert(os.path.isfile(fname))
+    assert(s2.schema == s0.schema)
     assert(s2 == s0)
     os.remove(fname)
 

From 59b0c55de8588d8eb69b62631199397f0a18946d Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Fri, 15 Oct 2021 10:09:33 -0700
Subject: [PATCH 70/73] Sort subtype schemas

---
 yggdrasil/schema.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/yggdrasil/schema.py b/yggdrasil/schema.py
index 3c8ab0a9f..97fa18034 100644
--- a/yggdrasil/schema.py
+++ b/yggdrasil/schema.py
@@ -496,7 +496,7 @@ def get_schema(self, relaxed=False, allow_instance=False, for_form=False):
         else:
             out['allOf'] = [self.get_subtype_schema('base', relaxed=relaxed),
                             {'anyOf': [self.get_subtype_schema(x, unique=True)
-                                       for x in self._storage.keys()]}]
+                                       for x in sorted(self._storage.keys())]}]
         if allow_instance:
             out['oneOf'] = [{'allOf': out.pop('allOf')},
                             {'type': 'instance',
@@ -519,7 +519,7 @@ def full_schema(self):
                'title': self.schema_type,
                'allOf': [self.get_subtype_schema('base', unique=True),
                          {'anyOf': [self.get_subtype_schema(x)
-                                    for x in self._storage.keys()]}]}
+                                    for x in sorted(self._storage.keys())]}]}
         return out
 
     @classmethod

From ef9e400efead636e3d28b37f4d93663f026dba8e Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Fri, 15 Oct 2021 10:23:15 -0700
Subject: [PATCH 71/73] Update schema

---
 yggdrasil/.ygg_schema.yml | 412 +++++++++++++++++++-------------------
 1 file changed, 206 insertions(+), 206 deletions(-)

diff --git a/yggdrasil/.ygg_schema.yml b/yggdrasil/.ygg_schema.yml
index 549faf439..aac3b4a81 100644
--- a/yggdrasil/.ygg_schema.yml
+++ b/yggdrasil/.ygg_schema.yml
@@ -256,26 +256,26 @@ definitions:
         title: yggdrasil.communication.RESTComm.RESTComm
         type: object
       - additionalProperties: true
-        description: Schema for comm component ['rmq'] subtype.
+        description: Schema for comm component ['rmq_async'] subtype.
         properties:
           commtype:
             default: default
-            description: RabbitMQ connection.
+            description: Asynchronous RabbitMQ connection.
             enum:
-            - rmq
+            - rmq_async
             type: string
-        title: yggdrasil.communication.RMQComm.RMQComm
+        title: yggdrasil.communication.RMQAsyncComm.RMQAsyncComm
         type: object
       - additionalProperties: true
-        description: Schema for comm component ['rmq_async'] subtype.
+        description: Schema for comm component ['rmq'] subtype.
         properties:
           commtype:
             default: default
-            description: Asynchronous RabbitMQ connection.
+            description: RabbitMQ connection.
             enum:
-            - rmq_async
+            - rmq
             type: string
-        title: yggdrasil.communication.RMQAsyncComm.RMQAsyncComm
+        title: yggdrasil.communication.RMQComm.RMQComm
         type: object
       - additionalProperties: true
         description: Schema for comm component ['value'] subtype.
@@ -387,66 +387,66 @@ definitions:
         title: yggdrasil.drivers.ConnectionDriver.ConnectionDriver
         type: object
       - additionalProperties: true
-        description: Schema for connection component ['input'] subtype.
+        description: Schema for connection component ['file_input'] subtype.
         properties:
           connection_type:
-            description: Connection between one or more comms/files and a model.
+            description: Connection between a file and a model.
             enum:
-            - input
+            - file_input
             type: string
-        title: yggdrasil.drivers.InputDriver.InputDriver
+        title: yggdrasil.drivers.FileInputDriver.FileInputDriver
         type: object
       - additionalProperties: true
-        description: Schema for connection component ['file_input'] subtype.
+        description: Schema for connection component ['file_output'] subtype.
         properties:
           connection_type:
-            description: Connection between a file and a model.
+            description: Connection between a model and a file.
             enum:
-            - file_input
+            - file_output
             type: string
-        title: yggdrasil.drivers.FileInputDriver.FileInputDriver
+        title: yggdrasil.drivers.FileOutputDriver.FileOutputDriver
         type: object
       - additionalProperties: true
-        description: Schema for connection component ['output'] subtype.
+        description: Schema for connection component ['input'] subtype.
         properties:
           connection_type:
-            description: Connection between a model and one or more comms/files.
+            description: Connection between one or more comms/files and a model.
             enum:
-            - output
+            - input
             type: string
-        title: yggdrasil.drivers.OutputDriver.OutputDriver
+        title: yggdrasil.drivers.InputDriver.InputDriver
         type: object
       - additionalProperties: true
-        description: Schema for connection component ['file_output'] subtype.
+        description: Schema for connection component ['output'] subtype.
         properties:
           connection_type:
-            description: Connection between a model and a file.
+            description: Connection between a model and one or more comms/files.
             enum:
-            - file_output
+            - output
             type: string
-        title: yggdrasil.drivers.FileOutputDriver.FileOutputDriver
+        title: yggdrasil.drivers.OutputDriver.OutputDriver
         type: object
       - additionalProperties: true
-        description: Schema for connection component ['rpc_response'] subtype.
+        description: Schema for connection component ['rpc_request'] subtype.
         properties:
           connection_type:
             description: Connection between one or more comms/files and one or more
               comms/files.
             enum:
-            - rpc_response
+            - rpc_request
             type: string
-        title: yggdrasil.drivers.RPCResponseDriver.RPCResponseDriver
+        title: yggdrasil.drivers.RPCRequestDriver.RPCRequestDriver
         type: object
       - additionalProperties: true
-        description: Schema for connection component ['rpc_request'] subtype.
+        description: Schema for connection component ['rpc_response'] subtype.
         properties:
           connection_type:
             description: Connection between one or more comms/files and one or more
               comms/files.
             enum:
-            - rpc_request
+            - rpc_response
             type: string
-        title: yggdrasil.drivers.RPCRequestDriver.RPCRequestDriver
+        title: yggdrasil.drivers.RPCResponseDriver.RPCResponseDriver
         type: object
     description: Schema for connection components.
     title: connection
@@ -710,37 +710,6 @@ definitions:
       title: file_base
       type: object
     - anyOf:
-      - additionalProperties: true
-        description: Schema for file component ['binary'] subtype.
-        properties:
-          filetype:
-            default: binary
-            description: The entire file is read/written all at once as bytes.
-            enum:
-            - binary
-            type: string
-          read_meth:
-            default: read
-            description: Method that should be used to read data from the file. Defaults
-              to 'read'. Ignored if direction is 'send'.
-            enum:
-            - read
-            - readline
-            type: string
-          serializer:
-            default:
-              seritype: direct
-            description: Class with serialize and deserialize methods that should
-              be used to process sent and received messages. Defaults to None and
-              is constructed using provided 'serializer_kwargs'.
-            oneOf:
-            - $ref: '#/definitions/serializer'
-            - class: yggdrasil.serialize.SerializeBase:SerializeBase
-              type: instance
-        required:
-        - serializer
-        title: yggdrasil.communication.FileComm.FileComm
-        type: object
       - additionalProperties: true
         description: Schema for file component ['ascii'] subtype.
         properties:
@@ -840,6 +809,37 @@ definitions:
             type: boolean
         title: yggdrasil.communication.AsciiTableComm.AsciiTableComm
         type: object
+      - additionalProperties: true
+        description: Schema for file component ['binary'] subtype.
+        properties:
+          filetype:
+            default: binary
+            description: The entire file is read/written all at once as bytes.
+            enum:
+            - binary
+            type: string
+          read_meth:
+            default: read
+            description: Method that should be used to read data from the file. Defaults
+              to 'read'. Ignored if direction is 'send'.
+            enum:
+            - read
+            - readline
+            type: string
+          serializer:
+            default:
+              seritype: direct
+            description: Class with serialize and deserialize methods that should
+              be used to process sent and received messages. Defaults to None and
+              is constructed using provided 'serializer_kwargs'.
+            oneOf:
+            - $ref: '#/definitions/serializer'
+            - class: yggdrasil.serialize.SerializeBase:SerializeBase
+              type: instance
+        required:
+        - serializer
+        title: yggdrasil.communication.FileComm.FileComm
+        type: object
       - additionalProperties: true
         description: Schema for file component ['json'] subtype.
         properties:
@@ -953,34 +953,6 @@ definitions:
             type: integer
         title: yggdrasil.communication.NetCDFFileComm.NetCDFFileComm
         type: object
-      - additionalProperties: true
-        description: Schema for file component ['ply'] subtype.
-        properties:
-          comment:
-            default: '# '
-            description: One or more characters indicating a comment. Defaults to
-              '# '.
-            type: string
-          datatype:
-            description: JSON schema defining the type of object that the serializer
-              will be used to serialize/deserialize. Defaults to default_datatype.
-            type: schema
-          filetype:
-            default: binary
-            description: The file is in the `Ply <http://paulbourke.net/dataformats/ply/>`_
-              data format for 3D structures.
-            enum:
-            - ply
-            type: string
-          newline:
-            default: '
-
-              '
-            description: One or more characters indicating a newline. Defaults to
-              '\n'.
-            type: string
-        title: yggdrasil.communication.PlyFileComm.PlyFileComm
-        type: object
       - additionalProperties: true
         description: Schema for file component ['obj'] subtype.
         properties:
@@ -1083,6 +1055,34 @@ definitions:
             type: string
         title: yggdrasil.communication.PickleFileComm.PickleFileComm
         type: object
+      - additionalProperties: true
+        description: Schema for file component ['ply'] subtype.
+        properties:
+          comment:
+            default: '# '
+            description: One or more characters indicating a comment. Defaults to
+              '# '.
+            type: string
+          datatype:
+            description: JSON schema defining the type of object that the serializer
+              will be used to serialize/deserialize. Defaults to default_datatype.
+            type: schema
+          filetype:
+            default: binary
+            description: The file is in the `Ply <http://paulbourke.net/dataformats/ply/>`_
+              data format for 3D structures.
+            enum:
+            - ply
+            type: string
+          newline:
+            default: '
+
+              '
+            description: One or more characters indicating a newline. Defaults to
+              '\n'.
+            type: string
+        title: yggdrasil.communication.PlyFileComm.PlyFileComm
+        type: object
       - additionalProperties: true
         description: Schema for file component ['wofost'] subtype.
         properties:
@@ -1863,55 +1863,6 @@ definitions:
       title: model_base
       type: object
     - anyOf:
-      - additionalProperties: true
-        description: Schema for model component ['c'] subtype.
-        properties:
-          compiler:
-            description: Command or path to executable that should be used to compile
-              the model. If not provided, the compiler will be determined based on
-              configuration options for the language (if present) and the registered
-              compilers that are available on the current operating system.
-            type: string
-          compiler_flags:
-            default: []
-            description: Flags that should be passed to the compiler during compilation.
-              If nto provided, the compiler flags will be determined based on configuration
-              options for the language (if present), the compiler defaults, and the
-              default_compiler_flags class attribute.
-            items:
-              type: string
-            type: array
-          language:
-            default: executable
-            description: Model is written in C.
-            enum:
-            - c
-            type: string
-          linker:
-            description: Command or path to executable that should be used to link
-              the model. If not provided, the linker will be determined based on configuration
-              options for the language (if present) and the registered linkers that
-              are available on the current operating system
-            type: string
-          linker_flags:
-            default: []
-            description: Flags that should be passed to the linker during compilation.
-              If nto provided, the linker flags will be determined based on configuration
-              options for the language (if present), the linker defaults, and the
-              default_linker_flags class attribute.
-            items:
-              type: string
-            type: array
-          source_files:
-            default: []
-            description: Source files that should be compiled into an executable.
-              Defaults to an empty list and the driver will search for a source file
-              based on the model executable (the first model argument).
-            items:
-              type: string
-            type: array
-        title: yggdrasil.drivers.CModelDriver.CModelDriver
-        type: object
       - additionalProperties: true
         description: Schema for model component ['cmake'] subtype.
         properties:
@@ -2026,6 +1977,55 @@ definitions:
             type: array
         title: yggdrasil.drivers.CMakeModelDriver.CMakeModelDriver
         type: object
+      - additionalProperties: true
+        description: Schema for model component ['c'] subtype.
+        properties:
+          compiler:
+            description: Command or path to executable that should be used to compile
+              the model. If not provided, the compiler will be determined based on
+              configuration options for the language (if present) and the registered
+              compilers that are available on the current operating system.
+            type: string
+          compiler_flags:
+            default: []
+            description: Flags that should be passed to the compiler during compilation.
+              If nto provided, the compiler flags will be determined based on configuration
+              options for the language (if present), the compiler defaults, and the
+              default_compiler_flags class attribute.
+            items:
+              type: string
+            type: array
+          language:
+            default: executable
+            description: Model is written in C.
+            enum:
+            - c
+            type: string
+          linker:
+            description: Command or path to executable that should be used to link
+              the model. If not provided, the linker will be determined based on configuration
+              options for the language (if present) and the registered linkers that
+              are available on the current operating system
+            type: string
+          linker_flags:
+            default: []
+            description: Flags that should be passed to the linker during compilation.
+              If nto provided, the linker flags will be determined based on configuration
+              options for the language (if present), the linker defaults, and the
+              default_linker_flags class attribute.
+            items:
+              type: string
+            type: array
+          source_files:
+            default: []
+            description: Source files that should be compiled into an executable.
+              Defaults to an empty list and the driver will search for a source file
+              based on the model executable (the first model argument).
+            items:
+              type: string
+            type: array
+        title: yggdrasil.drivers.CModelDriver.CModelDriver
+        type: object
       - additionalProperties: true
         description: Schema for model component ['c++', 'cpp', 'cxx'] subtype.
         properties:
@@ -2177,38 +2177,6 @@ definitions:
             type: string
         title: yggdrasil.drivers.FortranModelDriver.FortranModelDriver
         type: object
-      - additionalProperties: true
-        description: Schema for model component ['python'] subtype.
-        properties:
-          interpreter:
-            description: Name or path of interpreter executable that should be used
-              to run the model. If not provided, the interpreter will be determined
-              based on configuration options for the language (if present) and the
-              default_interpreter class attribute.
-            type: string
-          interpreter_flags:
-            default: []
-            description: Flags that should be passed to the interpreter when running
-              the model. If not provided, the flags are determined based on configuration
-              options for the language (if present) and the default_interpreter_flags
-              class attribute.
-            items:
-              type: string
-            type: array
-          language:
-            default: executable
-            description: Model is written in Python.
-            enum:
-            - python
-            type: string
-          skip_interpreter:
-            default: false
-            description: If True, no interpreter will be added to the arguments. This
-              should only be used for subclasses that will not be invoking the model
-              via the command line. Defaults to False.
-            type: boolean
-        title: yggdrasil.drivers.PythonModelDriver.PythonModelDriver
-        type: object
       - additionalProperties: true
         description: Schema for model component ['lpy'] subtype.
         properties:
@@ -2444,6 +2412,38 @@ definitions:
             type: object
         title: yggdrasil.drivers.OSRModelDriver.OSRModelDriver
         type: object
+      - additionalProperties: true
+        description: Schema for model component ['python'] subtype.
+        properties:
+          interpreter:
+            description: Name or path of interpreter executable that should be used
+              to run the model. If not provided, the interpreter will be determined
+              based on configuration options for the language (if present) and the
+              default_interpreter class attribute.
+            type: string
+          interpreter_flags:
+            default: []
+            description: Flags that should be passed to the interpreter when running
+              the model. If not provided, the flags are determined based on configuration
+              options for the language (if present) and the default_interpreter_flags
+              class attribute.
+            items:
+              type: string
+            type: array
+          language:
+            default: executable
+            description: Model is written in Python.
+            enum:
+            - python
+            type: string
+          skip_interpreter:
+            default: false
+            description: If True, no interpreter will be added to the arguments. This
+              should only be used for subclasses that will not be invoking the model
+              via the command line. Defaults to False.
+            type: boolean
+        title: yggdrasil.drivers.PythonModelDriver.PythonModelDriver
+        type: object
       - additionalProperties: true
         description: Schema for model component ['R', 'r'] subtype.
         properties:
@@ -2802,19 +2802,6 @@ definitions:
             type: string
         title: yggdrasil.serialize.AsciiMapSerialize.AsciiMapSerialize
         type: object
-      - additionalProperties: true
-        description: Schema for serializer component ['default'] subtype.
-        properties:
-          seritype:
-            default: default
-            description: Default serializer that uses |yggdrasil|'s extended JSON
-              serialization based on a provided type definition (See discussion :ref:`here
-              <serialization_rst>`).
-            enum:
-            - default
-            type: string
-        title: yggdrasil.serialize.DefaultSerialize.DefaultSerialize
-        type: object
       - additionalProperties: true
         description: Schema for serializer component ['table'] subtype.
         properties:
@@ -2860,6 +2847,19 @@ definitions:
             type: boolean
         title: yggdrasil.serialize.AsciiTableSerialize.AsciiTableSerialize
         type: object
+      - additionalProperties: true
+        description: Schema for serializer component ['default'] subtype.
+        properties:
+          seritype:
+            default: default
+            description: Default serializer that uses |yggdrasil|'s extended JSON
+              serialization based on a provided type definition (See discussion :ref:`here
+              <serialization_rst>`).
+            enum:
+            - default
+            type: string
+        title: yggdrasil.serialize.DefaultSerialize.DefaultSerialize
+        type: object
       - additionalProperties: true
         description: Schema for serializer component ['direct'] subtype.
         properties:
@@ -2933,17 +2933,6 @@ definitions:
             type: string
         title: yggdrasil.serialize.MatSerialize.MatSerialize
         type: object
-      - additionalProperties: true
-        description: Schema for serializer component ['ply'] subtype.
-        properties:
-          seritype:
-            default: default
-            description: Serialize 3D structures using Ply format.
-            enum:
-            - ply
-            type: string
-        title: yggdrasil.serialize.PlySerialize.PlySerialize
-        type: object
       - additionalProperties: true
         description: Schema for serializer component ['obj'] subtype.
         properties:
@@ -3015,6 +3004,17 @@ definitions:
             type: string
         title: yggdrasil.serialize.PickleSerialize.PickleSerialize
         type: object
+      - additionalProperties: true
+        description: Schema for serializer component ['ply'] subtype.
+        properties:
+          seritype:
+            default: default
+            description: Serialize 3D structures using Ply format.
+            enum:
+            - ply
+            type: string
+        title: yggdrasil.serialize.PlySerialize.PlySerialize
+        type: object
       - additionalProperties: true
         description: Schema for serializer component ['wofost'] subtype.
         properties:
@@ -3146,18 +3146,6 @@ definitions:
             - array
         title: yggdrasil.communication.transforms.ArrayTransform.ArrayTransform
         type: object
-      - additionalProperties: true
-        description: Schema for transform component ['pandas'] subtype.
-        properties:
-          field_names:
-            items:
-              type: string
-            type: array
-          transformtype:
-            enum:
-            - pandas
-        title: yggdrasil.communication.transforms.PandasTransform.PandasTransform
-        type: object
       - additionalProperties: true
         description: Schema for transform component ['direct'] subtype.
         properties:
@@ -3219,6 +3207,18 @@ definitions:
         - map
         title: yggdrasil.communication.transforms.MapFieldsTransform.MapFieldsTransform
         type: object
+      - additionalProperties: true
+        description: Schema for transform component ['pandas'] subtype.
+        properties:
+          field_names:
+            items:
+              type: string
+            type: array
+          transformtype:
+            enum:
+            - pandas
+        title: yggdrasil.communication.transforms.PandasTransform.PandasTransform
+        type: object
       - additionalProperties: true
         description: Schema for transform component ['select_fields'] subtype.
         properties:

From b6da1087d15cfde8630e352845be92d74f5051cf Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Fri, 15 Oct 2021 15:53:32 -0700
Subject: [PATCH 72/73] Wait for values to be queued when testing async
 ValueComm

---
 yggdrasil/communication/tests/test_ValueComm.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/yggdrasil/communication/tests/test_ValueComm.py b/yggdrasil/communication/tests/test_ValueComm.py
index cf6a51933..6e5db52cc 100644
--- a/yggdrasil/communication/tests/test_ValueComm.py
+++ b/yggdrasil/communication/tests/test_ValueComm.py
@@ -43,6 +43,9 @@ def test_send_recv(self):
         n_recv = self.testing_options['kwargs']['count']
         msg_recv = self.test_msg
         if self.use_async:
+            from yggdrasil.multitasking import wait_on_function
+            wait_on_function(lambda: self.recv_instance.n_msg_recv > 0,
+                             timeout=self.timeout)
             assert(self.recv_instance.n_msg_recv > 0)
         else:
             self.assert_equal(self.recv_instance.n_msg_recv, n_recv)

From a330fbabf66b87fa7f603ddf5e0f13827ea67455 Mon Sep 17 00:00:00 2001
From: Meagan Lang <langmm.astro@gmail.com>
Date: Fri, 15 Oct 2021 19:10:39 -0700
Subject: [PATCH 73/73] Coverage pragma for uncaught exception

---
 yggdrasil/metaschema/encoder.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yggdrasil/metaschema/encoder.py b/yggdrasil/metaschema/encoder.py
index 3402ba9f9..2c59b5bbd 100644
--- a/yggdrasil/metaschema/encoder.py
+++ b/yggdrasil/metaschema/encoder.py
@@ -40,7 +40,7 @@ def string2import(s):
         try:
             mod = importlib.import_module(pkg_mod[0])
             s = getattr(mod, pkg_mod[1])
-        except (ImportError, AttributeError):
+        except (ImportError, AttributeError):  # pragma: debug
             pass
     return s