Merge pull request #85 from cropsinsilico/topic/forms

Topic/forms

.github/workflows/test-install.yml

@@ -44,7 +44,7 @@ jobs:
         - macos-latest
         - windows-latest
         python-version:
-        - 3.6
+        - 3.7
         install-method:
         - pip
         test-flags1:
@@ -61,7 +61,7 @@ jobs:
         - true
         include:
         - os: windows-latest
-          python-version: 3.6
+          python-version: 3.7
           install-method: pip
           test-flags1: --long-running --languages cpp
           test-flags2: --long-running --skip-languages c++
@@ -351,7 +351,7 @@ jobs:
         - macos-latest
         - windows-latest
         python-version:
-        - 3.6
+        - 3.7
         install-method:
         - conda
         test-flags1:
@@ -364,7 +364,7 @@ jobs:
         - true
         include:
         - os: ubuntu-latest
-          python-version: 3.7
+          python-version: 3.6
           install-method: conda
           test-flags1: --long-running
           test-flags2: ''
@@ -679,7 +679,7 @@ jobs:
         os:
         - ubuntu-latest
         python-version:
-        - 3.6
+        - 3.7
         install-method:
         - pip
         test-flags1:
@@ -981,7 +981,7 @@ jobs:
         os:
         - ubuntu-latest
         python-version:
-        - 3.6
+        - 3.7
         install-method:
         - conda
         test-flags1:
@@ -1295,7 +1295,7 @@ jobs:
         - macos-latest
         - windows-latest
         python-version:
-        - 3.6
+        - 3.7
         install-method:
         - pip
         test-flags1:
@@ -1672,3 +1672,32 @@ jobs:
       run: python utils/build_docker.py --push
     - name: Build and push executable docker images for the new release
       run: python utils/build_docker.py --push executable
+  schema:
+    name: Update the schema used by the model submission form
+    needs:
+    - test_pip
+    - test_conda
+    - test_rmq_pip
+    - test_rmq_conda
+    - test_stripped
+    - docs
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: Install dependencies
+      run: 'python -m pip install PyGithub
+
+        python utils/setup_test_env.py install pip
+
+        '
+    - name: Verify installation
+      run: python utils/setup_test_env.py verify
+    - name: Update the form
+      run: python utils/update_form_schema.py

HISTORY.rst

@@ -2,6 +2,17 @@
 History
 =======
 
+1.8.1 (2021-10-15) Minor updates to support model submission form development
+------------------
+
+* Added --model_repository option to the integration-service-manager CLI
+* Preload models from the model repository into the service manager registry
+* Added validation_command option to model schema that can be used to validate a run on completion via the 'validate' runner option
+* Added dependencies and additional_dependencies options to model schema that can be used to specify packages that should be installed for a model
+* Refactor registration to use constants to prevent failure on circular import and speed up import time
+* Migrate constants into the constants module
+* Fix various bugs introduced by updates to GHA images and dependencies by pinning or updating use (e.g. jsonschema, libroadrunner, mac version, R version)
+
 1.8.0 (2021-09-15) Support for REST API based communicators, running integrations as services, and connecting to remote integration services
 ------------------
 

MANIFEST.in

@@ -20,6 +20,7 @@ recursive-include yggdrasil *.json
 recursive-include yggdrasil *.f90
 recursive-include yggdrasil *.xml
 recursive-include yggdrasil/examples *.py
+recursive-include yggdrasil/examples *.rst
 recursive-include yggdrasil/demos *.py
 recursive-include yggdrasil/tests/scripts *.py
 recursive-include yggdrasil/examples Makefile_linux

docs/source/docker.rst

@@ -83,7 +83,7 @@ Then the Dockerfile that will be able to run those services is
 
 .. code-block:: docker
 
-   FROM cropsinsilico/yggdrasil-service:1.7.0
+   FROM cropsinsilico/yggdrasil-service:v1.8.0
    COPY services.yml .
    COPY foo ./foo
    COPY bar ./bar

recipe/meta.yaml

@@ -60,7 +60,7 @@ requirements:
     - git
     - GitPython <=3.1.20  # [py==36]
     - GitPython  # [py>36]
-    - jsonschema
+    - jsonschema==3.2.0
     - matplotlib-base
     - numpy >=1.13.0
     - pandas

requirements.txt

@@ -1,6 +1,6 @@
 chevron
 flask
-jsonschema>=3
+jsonschema==3.2.0
 matplotlib
 numpy>=1.13.0
 pandas

requirements_development.txt

@@ -1,2 +1,3 @@
 black
 git  # [conda]
+PyGithub

utils/build_docker.py

@@ -7,7 +7,7 @@
 
 
 def build(dockerfile, tag, flags=[], repo='cropsinsilico/yggdrasil',
-          context=_utils_dir):
+          context=_utils_dir, disable_latest=False):
     r"""Build a docker image.
 
     Args:
@@ -20,11 +20,17 @@ def build(dockerfile, tag, flags=[], repo='cropsinsilico/yggdrasil',
         context (str, optional): Directory that should be provided as the
             context for the image. Defaults to the directory containing this
             script.
+        disable_latest (bool, optional): If True, the new image will not
+            be tagged 'latest' in addition to the provided tag value. Defaults
+            to False.
 
     """
     args = ['docker', 'build', '-t', f'{repo}:{tag}', '-f', dockerfile] + flags
     args.append(context)
     subprocess.call(args)
+    if not disable_latest:
+        args = ['docker', 'tag', f'{repo}:{tag}', f"{repo}:latest"]
+        subprocess.call(args)
 
 
 def push_image(tag, repo='cropsinsilico/yggdrasil'):
@@ -140,6 +146,10 @@ def params_service(params):
     parser.add_argument(
         "--push", action="store_true",
         help="After successfully building the image, push it to DockerHub.")
+    parser.add_argument(
+        "--disable-latest", action="store_true",
+        help=("Don't tag the new image as 'latest' in addition to the "
+              "version/commit specific tag."))
     subparsers = parser.add_subparsers(
         dest="type",
         help=("Type of docker image that should be built "
@@ -163,8 +173,11 @@ def params_service(params):
         params = params_executable(params)
     elif args.type == 'service':
         params = params_service(params)
+    params.setdefault('disable_latest', args.disable_latest)
     dockerfile = params.pop('dockerfile')
     tag = params.pop('tag')
     build(dockerfile, tag, **params)
     if args.push:
         push_image(tag, repo=params['repo'])
+        if not params['disable_latest']:
+            push_image('latest', repo=params['repo'])

utils/setup_test_env.py

@@ -329,7 +329,7 @@ def create_env(method, python, name=None, packages=None, init=_on_ci):
 
     """
     cmds = ["echo Creating test environment using %s..." % method]
-    major, minor = [int(x) for x in python.split('.')]
+    major, minor = [int(x) for x in python.split('.')][:2]
     if name is None:
         name = '%s%s' % (method, python.replace('.', ''))
     if packages is None:
@@ -782,7 +782,7 @@ def install_deps(method, return_commands=False, verbose=False,
         ]
     if fallback_to_conda:
         cmds.append("%s update --all" % CONDA_CMD)
-    if install_opts['R'] and (not fallback_to_conda):
+    if install_opts['R'] and (not fallback_to_conda) and (not only_python):
         # TODO: Test split installation where r-base is installed from
         # conda and the R dependencies are installed from CRAN?
         if _is_linux:

utils/test-install-base.yml

@@ -38,15 +38,15 @@ jobs:
       max-parallel: 20
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: [3.6]
+        python-version: [3.7]
         install-method: [pip]
         test-flags1: [--long-running --test-suite=timing --test-suite=demos --test-suite=top, --test-suite=examples --separate-test='--test-suite=mpi --write-script=run_mpi.sh --with-mpi=3', --test-suite=types --skip-languages c c++, --test-suite=types --languages c cpp]
         test-flags2: [""]
         install-c: [true]
         install-mpi: [true]
         include:
           - os: windows-latest
-            python-version: 3.6
+            python-version: 3.7
             install-method: pip
             test-flags1: --long-running --languages cpp
             test-flags2: --long-running --skip-languages c++
@@ -348,15 +348,15 @@ jobs:
       max-parallel: 20
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: [3.6]
+        python-version: [3.7]
         install-method: [conda]
         test-flags1: [--long-running --separate-test='--test-suite=mpi --write-script=run_mpi.sh']
         test-flags2: [--with-examples examples/tests/test_backwards.py]
         install-c: [true]
         install-mpi: [true]
         include:
           - os: ubuntu-latest
-            python-version: 3.7
+            python-version: 3.6
             install-method: conda
             test-flags1: --long-running
             test-flags2: ""
@@ -400,7 +400,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python-version: [3.6]
+        python-version: [3.7]
         install-method: [pip]
         test-flags1: [--long-running --languages python R matlab]
         test-flags2: [""]
@@ -425,7 +425,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python-version: [3.6]
+        python-version: [3.7]
         install-method: [conda]
         test-flags1: [--long-running --languages python R matlab]
         test-flags2: [tests/test_services.py --nocapture]
@@ -455,7 +455,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: [3.6]
+        python-version: [3.7]
         install-method: [pip]
         test-flags1: [--long-running]
         test-flags2: [""]
@@ -543,3 +543,25 @@ jobs:
       run: python utils/build_docker.py --push
     - name: Build and push executable docker images for the new release
       run: python utils/build_docker.py --push executable
+
+  schema:
+    name: Update the schema used by the model submission form
+    needs: [test_pip, test_conda, test_rmq_pip, test_rmq_conda, test_stripped, docs]
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: Install dependencies
+      run: |
+        python -m pip install PyGithub
+        python utils/setup_test_env.py install pip
+    - name: Verify installation
+      run: python utils/setup_test_env.py verify
+    - name: Update the form
+      run: python utils/update_form_schema.py

utils/update_form_schema.py

@@ -0,0 +1,46 @@
+import os
+import json
+import argparse
+from github import Github
+from yggdrasil import __version__
+from yggdrasil.schema import get_model_form_schema
+
+
+def update_schema(token):
+    r"""Update the model form schema via pull request to the
+    cropsinsilico/model_submission_form repository.
+
+    Args:
+        token (str): Github authentication token that should be used.
+
+    """
+    contents = json.dumps(get_model_form_schema(), indent='    ')
+    ver = __version__
+    repo_name = "cropsinsilico/model_submission_form"
+    msg = f"Update model form schema to version for yggdrasil {ver}"
+    branch = f"schema-update-{ver}"
+    schema = "static/model.json"
+    gh = Github(token)
+    repo = gh.get_repo(repo_name)
+    main = repo.get_branch('main')
+    schema_sha = repo.get_contents(schema).sha
+    repo.create_git_ref(f"refs/heads/{branch}", main.commit.sha)
+    repo.update_file(schema, msg, contents, schema_sha, branch=branch)
+    repo.create_pull(title=f"Update schema to {ver}",
+                     body=msg, head=branch, base="main")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        "Update the model form schema via pull request to the"
+        "cropsinsilico/model_submission_form repository.")
+    parser.add_argument(
+        '--token',
+        help=("Github authentication token that should be used to open the "
+              "pull request. The token must have write access to the "
+              "cropsinsilico/model_submission_form repository."))
+    args = parser.parse_args()
+    if args.token is None:
+        args.token = os.environ.get('YGGDRASIL_UPDATE_TOKEN', None)
+    assert(args.token)
+    update_schema = update_schema(args.token)