[TVB-1999] Integrate Zenodo API

.github/workflows/build.yml

@@ -1,5 +1,7 @@
 name: Test Py
-on: [push]
+on: 
+  pull_request:
+  push:
 
 jobs:
   build:
@@ -47,25 +49,7 @@ jobs:
           cd tvb_build
           bash install_full_tvb.sh
 
-      - name: cache data
-        id: cache-data
-        uses: actions/cache@v3
-        with:
-          path: tvb_data
-          key: tvb-data
-
-      - name: download data
-        if: steps.cache-data.outputs.cache-hit != 'true'
-        run: |
-          wget -q https://zenodo.org/record/7574266/files/tvb_data.zip?download=1 -O tvb_data.zip
-          mkdir tvb_data
-          unzip tvb_data.zip -d tvb_data
-          rm tvb_data.zip
 
-      - name: setup data
-        run: |
-          cd tvb_data
-          python3 setup.py develop
 
       - name: run library tests
         run: pytest -v tvb_library --cov --cov-report=xml && mv coverage.xml coverage-library.xml

.github/workflows/lib-tests.yml_disabled

@@ -1,5 +1,7 @@
 name: Test lib
-on: [push]
+on: 
+  pull_request:
+  push:
 jobs:
   lib-tests:
 
@@ -27,16 +29,7 @@ jobs:
           pip3 install pipenv
           cd tvb_library && pipenv install -d --python $(which python3)
 
-      - name: download data
-        run: |
-          wget -q https://zenodo.org/record/7574266/files/tvb_data.zip?download=1 -O tvb_data.zip
-          mkdir tvb_data
-          unzip tvb_data.zip -d tvb_data
-          rm tvb_data.zip
 
-      - name: setup data
-        run: |
-          cd tvb_library && pipenv run bash -c 'cd ../tvb_data && python3 setup.py develop'
 
       - name: importlib_metadata?
         run: cd tvb_library && pipenv install importlib_metadata

.github/workflows/notebooks.yml

@@ -1,5 +1,7 @@
 name: Test Notebooks
-on: [push]
+on: 
+  pull_request:
+  push:
 
 jobs:
   build:
@@ -40,22 +42,7 @@ jobs:
           cd tvb_build
           cmd /k "install_full_tvb.bat"
 
-      - name: cache data
-        id: cache-data
-        uses: actions/cache@v3
-        with:
-          path: tvb_data
-          key: tvbdata
 
-      - name: download data
-        if: steps.cache-data.outputs.cache-hit != 'true'
-        shell: pwsh
-        run: |
-          Invoke-WebRequest -OutFile C:\\TEMP\\tvb_data.zip -Uri "https://zenodo.org/record/7574266/files/tvb_data.zip?download=1"
-          Expand-Archive 'C:\\TEMP\\tvb_data.zip' C:\\tvb_data
-          del C:\\TEMP\\tvb_data.zip
-          cd C:\\tvb_data
-          python setup.py develop
 
       - name: run notebooks
         env:

.github/workflows/pg-tests.yml

@@ -1,5 +1,7 @@
 name: Test PG
-on: [push]
+on: 
+  pull_request:
+  push:
 
 jobs:
   build:
@@ -52,23 +54,7 @@ jobs:
       - name: setup tvb
         run: cd tvb_build && bash install_full_tvb.sh
 
-      - name: cache data
-        id: cache-data
-        uses: actions/cache@v3
-        with:
-          path: tvb_data
-          key: tvb-data
-
-      - name: download data
-        if: steps.cache-data.outputs.cache-hit != 'true'
-        run: |
-          wget -q https://zenodo.org/record/7574266/files/tvb_data.zip?download=1 -O tvb_data.zip
-          mkdir tvb_data
-          unzip tvb_data.zip -d tvb_data
-          rm tvb_data.zip
 
-      - name: setup data
-        run: cd tvb_data && python3 setup.py develop
 
       - name: run framework tests
         run: |

.github/workflows/win-tests.yml

@@ -1,5 +1,7 @@
 name: Test Win
-on: [push]
+on: 
+  pull_request:
+  push:
 
 jobs:
   build:
@@ -34,22 +36,7 @@ jobs:
           pip install -r tvb_framework/requirements.txt
           pip install --no-build-isolation tvb-gdist
 
-      - name: cache data
-        id: cache-data
-        uses: actions/cache@v3
-        with:
-          path: tvb_data
-          key: tvbdata
 
-      - name: download data
-        if: steps.cache-data.outputs.cache-hit != 'true'
-        shell: pwsh
-        run: |
-          Invoke-WebRequest -OutFile C:\\TEMP\\tvb_data.zip -Uri "https://zenodo.org/record/7574266/files/tvb_data.zip?download=1"
-          Expand-Archive 'C:\\TEMP\\tvb_data.zip' C:\\tvb_data
-          del C:\\TEMP\\tvb_data.zip
-          cd C:\\tvb_data
-          pip install -e .
 
       - name: run framework tests
         shell: pwsh

.gitignore

@@ -2,6 +2,9 @@
 *.py[cod]
 __pycache__
 
+#vim binaries
+*.swp
+
 # packaging
 *.egg-info/
 dist/

tvb_build/build_step1.py

@@ -44,7 +44,6 @@
 import requests
 
 import tvb_bin
-import tvb_data
 from subprocess import Popen, PIPE
 
 # source paths
@@ -53,90 +52,13 @@
 FW_FOLDER = os.path.join(TVB_ROOT, 'tvb_framework')
 LICENSE_PATH = os.path.join(FW_FOLDER, 'LICENSE')
 RELEASE_NOTES_PATH = os.path.join(TVB_ROOT, 'tvb_documentation', 'RELEASE_NOTES')
-DATA_SRC_FOLDER = os.path.dirname(tvb_data.__file__)
+
+
 DEMOS_MATLAB_FOLDER = os.path.join(TVB_ROOT, 'tvb_documentation', 'matlab')
 
 # dest paths
 DIST_FOLDER = os.path.join(os.path.dirname(__file__), 'build', 'TVB_Distribution')
 
-DATA_INSIDE_FOLDER = os.path.join(DIST_FOLDER, '_tvb_data')
-
-INCLUDED_INSIDE_DATA = [
-    "__init__.py",
-    "Default_Project.zip",
-
-    "connectivity/connectivity_76.zip",
-    "connectivity/paupau.zip",
-    "connectivity/connectivity_66.zip",
-    "connectivity/connectivity_192.zip",
-    "connectivity/__init__.py",
-
-    "projectionMatrix/projection_eeg_62_surface_16k.mat",
-    "projectionMatrix/projection_eeg_65_surface_16k.npy",
-    "projectionMatrix/projection_meg_276_surface_16k.npy",
-    "projectionMatrix/projection_seeg_588_surface_16k.npy",
-    "projectionMatrix/__init__.py",
-
-    "regionMapping/__init__.py",
-    "regionMapping/regionMapping_16k_76.txt",
-    "regionMapping/regionMapping_80k_80.txt",
-    "regionMapping/regionMapping_16k_192.txt",
-
-    "sensors/eeg_unitvector_62.txt.bz2",
-    "sensors/eeg_brainstorm_65.txt",
-    "sensors/meg_151.txt.bz2",
-    "sensors/meg_brainstorm_276.txt",
-    "sensors/seeg_39.txt.bz2",
-    "sensors/seeg_brainstorm_960.txt",
-    "sensors/seeg_588.txt",
-    "sensors/__init__.py",
-
-    "surfaceData/__init__.py",
-    "surfaceData/cortex_80k.zip",
-    "surfaceData/cortex_16384.zip",
-    "surfaceData/outer_skin_4096.zip",
-    "surfaceData/inner_skull_4096.zip",
-    "surfaceData/outer_skull_4096.zip",
-    "surfaceData/scalp_1082.zip",
-    "surfaceData/face_8614.zip",
-
-    "local_connectivity/__init__.py",
-    "local_connectivity/local_connectivity_16384.mat",
-    "local_connectivity/local_connectivity_80k.mat",
-
-    "obj/__init__.py",
-    "obj/face_surface.obj",
-    "obj/eeg_cap.obj",
-
-    "mouse/allen_2mm/Connectivity.h5",
-    "mouse/allen_2mm/Volume.h5",
-    "mouse/allen_2mm/StructuralMRI.h5",
-    "mouse/allen_2mm/RegionVolumeMapping.h5",
-]
-
-
-def _copy_dataset(dataset_files, dataset_destination):
-    for pth in dataset_files:
-        rel_pth = pth.split('/')
-        origin = os.path.join(DATA_SRC_FOLDER, *rel_pth)
-        destination = os.path.join(dataset_destination, *rel_pth)
-        destination_folder = os.path.dirname(destination)
-        if not os.path.exists(destination_folder):
-            os.makedirs(destination_folder)
-        print("Copying %s into %s" % (origin, destination))
-        shutil.copyfile(origin, destination)
-
-
-def copy_distribution_dataset():
-    """
-    Copy the required data file from tvb_data folder:
-    - inside TVB library package (for internal usage).
-        Will be used during TVB functioning: import default project,
-        load default for console profile, or code update events
-    - in tvb_data folder, as example for users.
-    """
-    _copy_dataset(INCLUDED_INSIDE_DATA, DATA_INSIDE_FOLDER)
-
 
 def _copy_demos_collapsed(to_copy):
     """
@@ -230,7 +152,6 @@ def build_step1():
     shutil.copytree(DEMOS_MATLAB_FOLDER, os.path.join(DIST_FOLDER, 'matlab'),
                     ignore=shutil.ignore_patterns('.svn', '*.rst'))
 
-    copy_distribution_dataset()
 
     _copy_demos_collapsed({os.path.join("..", "tvb_documentation", "demos"): os.path.join(DIST_FOLDER, "demo_scripts"),
                            os.path.join("..", "tvb_documentation", "tutorials"):

tvb_build/docker/Dockerfile-build

@@ -38,11 +38,7 @@ RUN /bin/bash -c "source activate tvb-run"; \
     /opt/conda/envs/tvb-run/bin/jupyter notebook --generate-config; \
     echo "c.NotebookApp.password='sha1:12bff019c253:9daecd92c2e9bdb10b3b8a06767a74a0fe078d7c'">>$JUPYTER_CONFIG/jupyter_notebook_config.py
 
-RUN wget https://zenodo.org/record/7574266/files/tvb_data.zip?download=1 -O tvb_data.zip; \
-    mkdir tvb_data; unzip tvb_data.zip -d tvb_data; rm tvb_data.zip; \
-    cd tvb_data; \
-    /opt/conda/envs/tvb-run/bin/python setup.py develop;\
-    /opt/conda/envs/tvb-docs/bin/python setup.py develop
+
 
 WORKDIR $USER_HOME
 COPY requirements_group requirements.txt

tvb_build/docker/Dockerfile-run

@@ -31,10 +31,7 @@ RUN /bin/bash -c "source activate tvb-run"; \
     $ENV_BIN/jupyter notebook --generate-config; \
     echo "c.NotebookApp.password='sha1:12bff019c253:9daecd92c2e9bdb10b3b8a06767a74a0fe078d7c'">>$JUPYTER_CONFIG/jupyter_notebook_config.py
 
-RUN wget https://zenodo.org/record/7574266/files/tvb_data.zip?download=1 -O tvb_data.zip; \
-    mkdir tvb_data; unzip tvb_data.zip -d tvb_data; rm tvb_data.zip; \
-    cd tvb_data; \
-    $ENV_BIN/python setup.py develop
+
 
 WORKDIR $USER_HOME
 COPY requirements_group requirements.txt

tvb_build/docker/Dockerfile-win

@@ -17,12 +17,6 @@ RUN activate tvb-run && pip install lockfile scikit-build
 RUN activate tvb-run && pip install syncrypto
 
 
-# Download and install tvb data
-RUN mkdir C:\\TVB_CODE
-WORKDIR /TVB_CODE
-ADD https://zenodo.org/record/7574266/files/tvb_data.zip?download=1 tvb_data.zip
-RUN tar -xf tvb_data.zip && dir && del tvb_data.zip
-RUN activate tvb-run && python setup.py develop
 
 COPY requirements_group requirements.txt
 RUN activate tvb-run && pip install -r requirements.txt

tvb_contrib/tvb/contrib/scripts/datatypes/lookup_tables.py

@@ -31,6 +31,7 @@
 """
 
 import numpy
+from tvb.datasets import TVBZenodoDataset
 from tvb.basic.readers import try_get_absolute_path
 from tvb.basic.neotraits.api import HasTraits, Attr, NArray, Int, Float
 
@@ -74,7 +75,7 @@ class LookUpTable(HasTraits):
 
     @staticmethod
     def populate_table(result, source_file):
-        source_full_path = try_get_absolute_path("tvb_data.tables", source_file)
+        source_full_path = TVBZenodoDataset().fetch_data(source_file)
         zip_data = numpy.load(source_full_path)
 
         result.df = zip_data['df']

tvb_documentation/demos/encrypt_data.ipynb

@@ -113,10 +113,12 @@
    "outputs": [],
    "source": [
     "# EDIT paths to data files here inside the list\n",
-    "import tvb_data, os, tvb\n",
-    "from tvb.basic.readers import try_get_absolute_path\n",
+    "import  os, tvb\n",
+    "from tvb.datasets import TVBZenodoDataset\n",
+    "dataset = TVBZenodoDataset()\n",
     "\n",
-    "paths_to_files = [try_get_absolute_path(\"tvb_data.connectivity\", \"connectivity_76.zip\")]\n",
+    "\n",
+    "paths_to_files = [dataset.fetch_data(\"connectivity_76.zip\")]\n",
     "import_export_encryption_handler = StorageInterface.get_import_export_encryption_handler()\n",
     "\n",
     "buffer_size = TvbProfile.current.hpc.CRYPT_BUFFER_SIZE\n",
@@ -151,7 +153,7 @@
     "encrypted_password = import_export_encryption_handler.encrypt_password(public_key, password_bytes)\n",
     "\n",
     "# EDIT path for saving the encrypted password\n",
-    "encrypted_password_path = os.path.join(tvb_data.__path__[0], 'connectivity')\n",
+    "encrypted_password_path = os.path.join(dataset.extract_dir, 'tvb_data','connectivity')\n",
     "\n",
     "import_export_encryption_handler.save_encrypted_password(encrypted_password, encrypted_password_path)\n",
     "\n",