Merge pull request #732 from the-virtual-brain/EBR-78-windows_build_fix

EBR-78 Switch to Python 3.11. Fix Distribution builds

.github/workflows/build.yml

@@ -29,17 +29,18 @@ jobs:
 #        uses: actions/cache@v3
 #        with:
 #          path: ~/.local
-#          key: pip-${{ steps.setPy.outputs.version }}-${{ hashFiles('tvb_framework/requirements.txt') }}
+#          key: pip-${{ steps.setPy.outputs.version }}-${{ hashFiles('tvb_build/docker/requirements_group') }}
 
       - name: install tools and dependencies
 #        if: steps.cache-local.outputs.cache-hit != 'true'
         run: |
           sudo apt-get update
           sudo apt install libbz2-dev libhdf5-serial-dev liblzo2-dev
           python3 -m pip install --upgrade setuptools==59.8.0 pip wheel
-          pip3 install --user --upgrade numpy
-          python3 -m pip install scikit-build
-          pip3 install --user -r tvb_framework/requirements.txt
+          python3 -m pip install cryptography
+          python3 -m pip install lockfile
+          python3 -m pip install syncrypto
+          python3 -m pip install -r tvb_build/docker/requirements_group
           pip3 install --user --no-build-isolation tvb-gdist
 
       - name: setup tvb

.github/workflows/notebooks.yml

@@ -5,8 +5,6 @@ jobs:
   build:
     name: Test and Inspect
     runs-on: windows-latest
-    strategy:
-      fail-fast: false
     steps:
       - uses: actions/checkout@v3
         with:
@@ -24,9 +22,11 @@ jobs:
       - name: install tools and dependencies
         run: |
           python3 -m pip install --upgrade setuptools==59.8.0 pip wheel
-          pip3 install --user --upgrade numpy
           python3 -m pip install scikit-build
-          pip3 install --user -r tvb_framework/requirements.txt
+          python3 -m pip install cryptography
+          python3 -m pip install lockfile
+          python3 -m pip install syncrypto
+          python3 -m pip install -r tvb_build/docker/requirements_group
           pip3 install --user --no-build-isolation tvb-gdist
           python3 -m pip install elasticsearch
           python3 -m pip install nbformat nbconvert

.github/workflows/pg-tests.yml

@@ -36,17 +36,18 @@ jobs:
 #        uses: actions/cache@v3
 #        with:
 #          path: ~/.local
-#          key: pip--${{ hashFiles('tvb_framework/requirements.txt') }}
+#          key: pip--${{ hashFiles('tvb_build/docker/requirements_group') }}
 
       - name: install tools and dependencies
 #        if: steps.cache-local.outputs.cache-hit != 'true'
         run: |
           sudo apt-get update
           sudo apt install libbz2-dev libhdf5-serial-dev liblzo2-dev
           python3 -m pip install --upgrade setuptools==59.8.0 pip wheel
-          pip3 install --user --upgrade numpy psycopg2
-          python3 -m pip install scikit-build
-          pip3 install --user -r tvb_framework/requirements.txt
+          python3 -m pip install cryptography
+          python3 -m pip install lockfile
+          python3 -m pip install syncrypto
+          python3 -m pip install -r tvb_build/docker/requirements_group
           pip3 install --user --no-build-isolation tvb-gdist
 
       - name: setup tvb
@@ -56,19 +57,19 @@ jobs:
         id: cache-data
         uses: actions/cache@v3
         with:
-          path: tvb_data
-          key: tvb-data
+          path: demo_data
+          key: tvb-data-10128131
 
       - name: download data
         if: steps.cache-data.outputs.cache-hit != 'true'
         run: |
           wget -q https://zenodo.org/record/10128131/files/tvb_data.zip?download=1 -O tvb_data.zip
-          mkdir tvb_data
-          unzip tvb_data.zip -d tvb_data
+          mkdir demo_data
+          unzip tvb_data.zip -d demo_data
           rm tvb_data.zip
 
       - name: setup data
-        run: cd tvb_data && python3 setup.py develop
+        run: cd demo_data && python3 setup.py develop
 
       - name: run framework tests
         run: |

.github/workflows/win-tests.yml

@@ -24,14 +24,17 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.local
-          key: pip-${{ hashFiles('tvb_framework/requirements.txt') }}
+          key: pip-${{ hashFiles('tvb_build/docker/requirements_group') }}
 
       - name: install tools and dependencies
         if: steps.cache-local.outputs.cache-hit != 'true'
         run: |
           python -m pip install --upgrade setuptools pip wheel 
-          pip install -U numpy cython scikit-build
-          pip install -r tvb_framework/requirements.txt
+          pip install -U numpy<2 cython scikit-build pytest 
+          python3 -m pip install cryptography
+          python3 -m pip install syncrypto
+          python3 -m pip install lockfile
+          python3 -m pip install -r tvb_build/docker/requirements_group --verbose
           pip install --no-build-isolation tvb-gdist
 
       - name: cache data

tvb_build/Jenkinsfile-Mac

@@ -23,7 +23,7 @@ pipeline {
                     cd tvb_build
                     conda update -n base -c defaults conda
                     conda env remove --name mac-distribution
-                    conda create -y --name mac-distribution python=3.10 nomkl numba scipy numpy cython psycopg2 pip
+                    conda create -y --name mac-distribution python=3.11 nomkl numba scipy "numpy<2" cython psycopg2 pip
                     conda install -y --name mac-distribution -c conda-forge jupyterlab tvb-gdist
 
                     /Applications/anaconda3/envs/mac-distribution/bin/pip install --upgrade pip

tvb_build/Jenkinsfile-Windows

@@ -16,6 +16,14 @@ pipeline {
                 }
             }
         }
+        stage ('Tests on SqLite') {
+            steps {
+                withDockerContainer(image: "${FULL_DOCKER_IMAGE_NAME}") {
+                    powershell 'cd tvb_build; cmd /k "activate tvb-run & install_full_tvb.bat & cd ../tvb_bin & run_tests.bat"; exit 0'
+                }
+                junit 'tvb_bin/TEST_OUTPUT/results_*.xml'
+            }
+        }
         stage('Copy Step 1') {
              steps {
                  bat '''
@@ -44,14 +52,6 @@ pipeline {
                 archiveArtifacts artifacts: 'tvb_build/build/TVB_Windows*.zip'
             }
         }
-        stage ('Tests on SqLite') {
-            steps {
-                withDockerContainer(image: "${FULL_DOCKER_IMAGE_NAME}") {
-                    powershell 'cd tvb_build; cmd /k "activate tvb-run & install_full_tvb.bat & cd ../tvb_bin & run_tests.bat"; exit 0'
-                }
-                junit 'tvb_bin/TEST_OUTPUT/results_*.xml'
-            }
-        }
     }
     post {
         changed {

tvb_build/build_from_conda.py

@@ -84,8 +84,8 @@ def win64():
             'bin\\tvb_start.bat': 'distribution start',
             'bin\\tvb_clean.bat': 'distribution clean',
             'bin\\tvb_stop.bat': 'distribution stop',
-            'bin\\jupyter_notebook.bat': set_path + 'cd ..\\bin\n..\\tvb_data\\Scripts\\jupyter notebook ..\\demo_scripts',
-            'demo_scripts\\jupyter_notebook.bat': set_path + 'cd ..\\demo_scripts\n..\\tvb_data\\Scripts\\jupyter notebook'
+            'bin\\jupyter_notebook.bat': set_path + 'cd ..\\bin\n..\\tvb_data\\Scripts\\jupyter lab ..\\demo_scripts',
+            'demo_scripts\\jupyter_notebook.bat': set_path + 'cd ..\\demo_scripts\n..\\tvb_data\\Scripts\\jupyter lab'
         }
 
         return Config("Windows", "C:\\miniconda\\envs\\tvb-run",
@@ -113,8 +113,8 @@ def linux64():
             'bin/tvb_start.sh': 'bash ./distribution.sh start',
             'bin/tvb_clean.sh': 'bash ./distribution.sh clean',
             'bin/tvb_stop.sh': 'bash ./distribution.sh stop',
-            'bin/jupyter_notebook.sh': set_path + 'cd ../bin\n../tvb_data/bin/python -m notebook ../demo_scripts',
-            'demo_scripts/jupyter_notebook.sh': set_path + 'cd ../demo_scripts\n../tvb_data/bin/python -m notebook'
+            'bin/jupyter_notebook.sh': set_path + 'cd ../bin\n../tvb_data/bin/python -m jupyterlab ../demo_scripts',
+            'demo_scripts/jupyter_notebook.sh': set_path + 'cd ../demo_scripts\n../tvb_data/bin/python -m jupyterlab'
         }
 
         return Config("Linux", "/opt/conda/envs/tvb-run", join("lib", Environment.PYTHON_FOLDER, "site-packages"),

tvb_build/conda_env_to_app.py

@@ -56,15 +56,15 @@
 VERSION = TvbProfile.current.version.BASE_VERSION
 # Name of the app
 APP_NAME = "tvb-{}".format(VERSION)
-# The website in reversered order (domain first, etc.)
+# The website in reversed order (domain first, etc.)
 IDENTIFIER = "org.thevirtualbrain"
 # The author of this package
-AUTHOR = "TVB Team"
+AUTHOR = "Brainiacs Team"
 # Full path to the anaconda environment folder to package
 # Make sure it is the full path (and not a relative one, also to the homedir with ~) so this can be
-# correctly replaced later. Conda usßes hardcoded paths, which we convert to `/Applications/<APP_NAME>`
+# correctly replaced later. Conda uses hardcoded paths, which we convert to `/Applications/<APP_NAME>`
 CONDA_ENV_PATH = "/Applications/anaconda3/envs/mac-distribution"
-# Folders to include from Anaconda environment, if ommitted everything will be copied
+# Folders to include from Anaconda environment, if omitted everything will be copied
 # CONDA_FOLDERS = ["lib", "bin", "share", "qsci", "ssl", "translations"]
 # Paths of files and folders to remove from the copied anaconda environment,
 # relative to the environment's root.
@@ -209,13 +209,8 @@ def _find_and_replace(path, search, replace, exclusions=None):
 
 def replace_conda_abs_paths():
     app_path = os.path.join(os.path.sep, 'Applications', APP_NAME + '.app', 'Contents', 'Resources')
-    print('Replacing occurences of {} with {}'.format(CONDA_ENV_PATH, app_path))
-    _find_and_replace(
-        RESOURCE_DIR,
-        CONDA_ENV_PATH,
-        app_path,
-        exclusions=['site-packages', 'doc']
-    )
+    print('Replacing occurrences of {} with {}'.format(CONDA_ENV_PATH, app_path))
+    _find_and_replace(RESOURCE_DIR, CONDA_ENV_PATH, app_path, exclusions=['site-packages', 'doc'])
 
 
 def create_app():
@@ -279,10 +274,7 @@ def copy_anaconda_env():
         if "CONDA_FOLDERS" in globals():
             # IF conda folders is specified, copy only those folders.
             for item in CONDA_FOLDERS:
-                shutil.copytree(
-                    os.path.join(CONDA_ENV_PATH, item),
-                    os.path.join(RESOURCE_DIR, item),
-                    symlinks=True)
+                shutil.copytree(os.path.join(CONDA_ENV_PATH, item), os.path.join(RESOURCE_DIR, item), symlinks=True)
         else:
             # Copy everything
             shutil.copytree(CONDA_ENV_PATH, RESOURCE_DIR, True)
@@ -316,8 +308,8 @@ def copy_icon():
     print("Copying icon file")
     try:
         shutil.copy(ICON_PATH, os.path.join(RESOURCE_DIR, ICON_FILE))
-    except OSError as e:
-        logger("Error copying icon file from: {}".format(ICON_PATH))
+    except OSError:
+        logger.error("Error copying icon file from: {}".format(ICON_PATH))
 
 
 def create_plist():
@@ -393,8 +385,7 @@ def create_dmg():
     print("Creating disk image of {}".format(app_size))
 
     # Create a dmgbuild config file in same folder as
-    dmgbuild_config_file = os.path.join(os.getcwd(),
-                                        'dmgbuild_settings.py')
+    dmgbuild_config_file = os.path.join(os.getcwd(), 'dmgbuild_settings.py')
 
     dmg_config = {
         'filename': dmg_file,

tvb_build/docker/Dockerfile-build

@@ -18,14 +18,14 @@ RUN service postgresql start && createdb -O postgres tvb-test && psql --command
 
 USER root
 RUN conda update -n base -c defaults conda; conda init bash
-RUN conda create -y --name tvb-run python=3.10 nomkl numba scipy numpy cython psycopg2
+RUN conda create -y --name tvb-run python=3.11 nomkl numba scipy numpy cython psycopg2
 RUN conda install -y --name tvb-run -c conda-forge jupyterlab tvb-gdist
 RUN /opt/conda/envs/tvb-run/bin/pip install --upgrade pip
 RUN /opt/conda/envs/tvb-run/bin/pip install lockfile scikit-build
 RUN /opt/conda/envs/tvb-run/bin/pip install cryptography
 RUN /opt/conda/envs/tvb-run/bin/pip install syncrypto
 
-RUN conda create -y --name tvb-docs python=3.10 nomkl numba scipy numpy cython psycopg2
+RUN conda create -y --name tvb-docs python=3.11 nomkl numba scipy numpy cython psycopg2
 RUN conda install -y --name tvb-docs -c conda-forge jupyterlab tvb-gdist
 RUN /opt/conda/envs/tvb-docs/bin/pip install --upgrade pip
 RUN /opt/conda/envs/tvb-docs/bin/pip install lockfile scikit-build

tvb_build/docker/Dockerfile-run

@@ -18,7 +18,7 @@ RUN service postgresql start && createdb -O postgres tvb-test && psql --command
 
 USER root
 RUN conda update -n base -c defaults conda; conda init bash
-RUN conda create -y --name tvb-run python=3.10 nomkl numba scipy numpy cython psycopg2
+RUN conda create -y --name tvb-run python=3.11 nomkl numba scipy numpy cython psycopg2
 RUN conda install -y --name tvb-run -c conda-forge jupyterlab tvb-gdist
 RUN $PIP install --upgrade pip
 RUN $PIP install lockfile scikit-build

tvb_build/docker/Dockerfile-win

@@ -1,4 +1,4 @@
-FROM python:3.10.5-windowsservercore-1809
+FROM python:3.11-windowsservercore-1809
 
 WORKDIR /tmp
 # Download and install miniconda
@@ -10,20 +10,20 @@ RUN conda init powershell
 
 # Prepare tvb-run env
 RUN activate && conda update -n base -c defaults conda
-RUN activate && conda create -y --name tvb-run python=3.10 numba scipy numpy cython psycopg2
+RUN activate && conda create -y --name tvb-run python=3.11 numba scipy numpy cython psycopg2
 RUN activate && conda install -y --name tvb-run -c conda-forge jupyterlab tvb-gdist
 RUN activate tvb-run && python -m pip install --upgrade pip
 RUN activate tvb-run && pip install lockfile scikit-build
 RUN activate tvb-run && pip install cryptography
 RUN activate tvb-run && pip install syncrypto
 
-
 # Download and install tvb data
 RUN mkdir C:\\TVB_CODE
 WORKDIR /TVB_CODE
+
 ADD https://zenodo.org/record/10128131/files/tvb_data.zip?download=1 tvb_data.zip
-RUN tar -xf tvb_data.zip && dir && del tvb_data.zip
-RUN activate tvb-run && python setup.py develop
+RUN mkdir -p zenodo_data && tar -xf tvb_data.zip -C zenodo_data && dir && del tvb_data.zip
+RUN activate tvb-run && cd zenodo_data && python setup.py develop
 
 COPY requirements_group requirements.txt
 RUN activate tvb-run && pip install -r requirements.txt

tvb_build/docker/requirements_group

@@ -1,8 +1,14 @@
+numpy<2
+cryptography
+sqlalchemy
+pyAesCrypt
+Deprecated
+lockfile
 numba
-numpy
 scipy
+syncrypto
 networkx
-scikit-learn<=1.1.2
+scikit-learn<=1.1.3
 scikit-image
 cython
 numexpr
@@ -19,16 +25,13 @@ BeautifulSoup4
 subprocess32
 python-keycloak
 mako
-pyAesCrypt
 pyunicore
 formencode
 flask
 flask-restx
 jinja2
 nibabel>=4.0
-sqlalchemy
 alembic
-allensdk
 werkzeug
 gevent
 cherrypy
@@ -41,19 +44,17 @@ six
 ipywidgets
 ipympl
 autopep8
-lockfile
 psycopg2
-syncrypto
 lxml
 pylems
 docutils
 Pillow
 siibra
 bctpy
-Deprecated
 kubernetes
 watchdog
 requests-toolbelt>=0.10
 elasticsearch
 urllib3<2.0
 pycodestyle==2.10.0
+

tvb_build/install_full_tvb.bat

@@ -9,11 +9,11 @@ pip install -e . --no-deps
 cd ..
 
 cd tvb_storage
-pip install -e .
+pip install -e . --no-deps
 cd ..
 
 cd tvb_library
-pip install -e .
+pip install -e . --no-deps
 cd ..
 
 cd tvb_contrib

tvb_build/install_full_tvb.sh

@@ -9,11 +9,11 @@ pip install -e . --no-deps --user
 cd ..
 
 cd tvb_storage
-pip install -e . --user
+pip install -e . --no-deps --user
 cd ..
 
 cd tvb_library
-pip install -e . --user
+pip install -e . --no-deps --user
 cd ..
 
 cd tvb_contrib