diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 08dc688957..9ec10866ff 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,5 +1,7 @@ name: Test Py -on: [push] +on: + pull_request: + push: jobs: build: @@ -47,25 +49,7 @@ jobs: cd tvb_build bash install_full_tvb.sh - - name: cache data - id: cache-data - uses: actions/cache@v3 - with: - path: tvb_data - key: tvb-data - - - name: download data - if: steps.cache-data.outputs.cache-hit != 'true' - run: | - wget -q https://zenodo.org/record/7574266/files/tvb_data.zip?download=1 -O tvb_data.zip - mkdir tvb_data - unzip tvb_data.zip -d tvb_data - rm tvb_data.zip - - name: setup data - run: | - cd tvb_data - python3 setup.py develop - name: run library tests run: pytest -v tvb_library --cov --cov-report=xml && mv coverage.xml coverage-library.xml diff --git a/.github/workflows/lib-tests.yml_disabled b/.github/workflows/lib-tests.yml_disabled index 07dff59116..78d1e44e4e 100644 --- a/.github/workflows/lib-tests.yml_disabled +++ b/.github/workflows/lib-tests.yml_disabled @@ -1,5 +1,7 @@ name: Test lib -on: [push] +on: + pull_request: + push: jobs: lib-tests: @@ -27,16 +29,7 @@ jobs: pip3 install pipenv cd tvb_library && pipenv install -d --python $(which python3) - - name: download data - run: | - wget -q https://zenodo.org/record/7574266/files/tvb_data.zip?download=1 -O tvb_data.zip - mkdir tvb_data - unzip tvb_data.zip -d tvb_data - rm tvb_data.zip - - name: setup data - run: | - cd tvb_library && pipenv run bash -c 'cd ../tvb_data && python3 setup.py develop' - name: importlib_metadata? run: cd tvb_library && pipenv install importlib_metadata diff --git a/.github/workflows/notebooks.yml b/.github/workflows/notebooks.yml index da9eab305a..d5590e62bf 100644 --- a/.github/workflows/notebooks.yml +++ b/.github/workflows/notebooks.yml @@ -1,5 +1,7 @@ name: Test Notebooks -on: [push] +on: + pull_request: + push: jobs: build: @@ -40,22 +42,7 @@ jobs: cd tvb_build cmd /k "install_full_tvb.bat" - - name: cache data - id: cache-data - uses: actions/cache@v3 - with: - path: tvb_data - key: tvbdata - - name: download data - if: steps.cache-data.outputs.cache-hit != 'true' - shell: pwsh - run: | - Invoke-WebRequest -OutFile C:\\TEMP\\tvb_data.zip -Uri "https://zenodo.org/record/7574266/files/tvb_data.zip?download=1" - Expand-Archive 'C:\\TEMP\\tvb_data.zip' C:\\tvb_data - del C:\\TEMP\\tvb_data.zip - cd C:\\tvb_data - python setup.py develop - name: run notebooks env: diff --git a/.github/workflows/pg-tests.yml b/.github/workflows/pg-tests.yml index b360fd2605..b9fc618c7d 100644 --- a/.github/workflows/pg-tests.yml +++ b/.github/workflows/pg-tests.yml @@ -1,5 +1,7 @@ name: Test PG -on: [push] +on: + pull_request: + push: jobs: build: @@ -52,23 +54,7 @@ jobs: - name: setup tvb run: cd tvb_build && bash install_full_tvb.sh - - name: cache data - id: cache-data - uses: actions/cache@v3 - with: - path: tvb_data - key: tvb-data - - - name: download data - if: steps.cache-data.outputs.cache-hit != 'true' - run: | - wget -q https://zenodo.org/record/7574266/files/tvb_data.zip?download=1 -O tvb_data.zip - mkdir tvb_data - unzip tvb_data.zip -d tvb_data - rm tvb_data.zip - - name: setup data - run: cd tvb_data && python3 setup.py develop - name: run framework tests run: | diff --git a/.github/workflows/win-tests.yml b/.github/workflows/win-tests.yml index b01ae36d52..a1b75b93a4 100644 --- a/.github/workflows/win-tests.yml +++ b/.github/workflows/win-tests.yml @@ -1,5 +1,7 @@ name: Test Win -on: [push] +on: + pull_request: + push: jobs: build: @@ -34,22 +36,7 @@ jobs: pip install -r tvb_framework/requirements.txt pip install --no-build-isolation tvb-gdist - - name: cache data - id: cache-data - uses: actions/cache@v3 - with: - path: tvb_data - key: tvbdata - - name: download data - if: steps.cache-data.outputs.cache-hit != 'true' - shell: pwsh - run: | - Invoke-WebRequest -OutFile C:\\TEMP\\tvb_data.zip -Uri "https://zenodo.org/record/7574266/files/tvb_data.zip?download=1" - Expand-Archive 'C:\\TEMP\\tvb_data.zip' C:\\tvb_data - del C:\\TEMP\\tvb_data.zip - cd C:\\tvb_data - pip install -e . - name: run framework tests shell: pwsh diff --git a/.gitignore b/.gitignore index 330ded9df8..8a03e8377c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,9 @@ *.py[cod] __pycache__ +#vim binaries +*.swp + # packaging *.egg-info/ dist/ diff --git a/tvb_build/build_step1.py b/tvb_build/build_step1.py index 885b1ab54f..44532af9b5 100644 --- a/tvb_build/build_step1.py +++ b/tvb_build/build_step1.py @@ -44,7 +44,6 @@ import requests import tvb_bin -import tvb_data from subprocess import Popen, PIPE # source paths @@ -53,90 +52,13 @@ FW_FOLDER = os.path.join(TVB_ROOT, 'tvb_framework') LICENSE_PATH = os.path.join(FW_FOLDER, 'LICENSE') RELEASE_NOTES_PATH = os.path.join(TVB_ROOT, 'tvb_documentation', 'RELEASE_NOTES') -DATA_SRC_FOLDER = os.path.dirname(tvb_data.__file__) + + DEMOS_MATLAB_FOLDER = os.path.join(TVB_ROOT, 'tvb_documentation', 'matlab') # dest paths DIST_FOLDER = os.path.join(os.path.dirname(__file__), 'build', 'TVB_Distribution') -DATA_INSIDE_FOLDER = os.path.join(DIST_FOLDER, '_tvb_data') - -INCLUDED_INSIDE_DATA = [ - "__init__.py", - "Default_Project.zip", - - "connectivity/connectivity_76.zip", - "connectivity/paupau.zip", - "connectivity/connectivity_66.zip", - "connectivity/connectivity_192.zip", - "connectivity/__init__.py", - - "projectionMatrix/projection_eeg_62_surface_16k.mat", - "projectionMatrix/projection_eeg_65_surface_16k.npy", - "projectionMatrix/projection_meg_276_surface_16k.npy", - "projectionMatrix/projection_seeg_588_surface_16k.npy", - "projectionMatrix/__init__.py", - - "regionMapping/__init__.py", - "regionMapping/regionMapping_16k_76.txt", - "regionMapping/regionMapping_80k_80.txt", - "regionMapping/regionMapping_16k_192.txt", - - "sensors/eeg_unitvector_62.txt.bz2", - "sensors/eeg_brainstorm_65.txt", - "sensors/meg_151.txt.bz2", - "sensors/meg_brainstorm_276.txt", - "sensors/seeg_39.txt.bz2", - "sensors/seeg_brainstorm_960.txt", - "sensors/seeg_588.txt", - "sensors/__init__.py", - - "surfaceData/__init__.py", - "surfaceData/cortex_80k.zip", - "surfaceData/cortex_16384.zip", - "surfaceData/outer_skin_4096.zip", - "surfaceData/inner_skull_4096.zip", - "surfaceData/outer_skull_4096.zip", - "surfaceData/scalp_1082.zip", - "surfaceData/face_8614.zip", - - "local_connectivity/__init__.py", - "local_connectivity/local_connectivity_16384.mat", - "local_connectivity/local_connectivity_80k.mat", - - "obj/__init__.py", - "obj/face_surface.obj", - "obj/eeg_cap.obj", - - "mouse/allen_2mm/Connectivity.h5", - "mouse/allen_2mm/Volume.h5", - "mouse/allen_2mm/StructuralMRI.h5", - "mouse/allen_2mm/RegionVolumeMapping.h5", -] - - -def _copy_dataset(dataset_files, dataset_destination): - for pth in dataset_files: - rel_pth = pth.split('/') - origin = os.path.join(DATA_SRC_FOLDER, *rel_pth) - destination = os.path.join(dataset_destination, *rel_pth) - destination_folder = os.path.dirname(destination) - if not os.path.exists(destination_folder): - os.makedirs(destination_folder) - print("Copying %s into %s" % (origin, destination)) - shutil.copyfile(origin, destination) - - -def copy_distribution_dataset(): - """ - Copy the required data file from tvb_data folder: - - inside TVB library package (for internal usage). - Will be used during TVB functioning: import default project, - load default for console profile, or code update events - - in tvb_data folder, as example for users. - """ - _copy_dataset(INCLUDED_INSIDE_DATA, DATA_INSIDE_FOLDER) - def _copy_demos_collapsed(to_copy): """ @@ -230,7 +152,6 @@ def build_step1(): shutil.copytree(DEMOS_MATLAB_FOLDER, os.path.join(DIST_FOLDER, 'matlab'), ignore=shutil.ignore_patterns('.svn', '*.rst')) - copy_distribution_dataset() _copy_demos_collapsed({os.path.join("..", "tvb_documentation", "demos"): os.path.join(DIST_FOLDER, "demo_scripts"), os.path.join("..", "tvb_documentation", "tutorials"): diff --git a/tvb_build/docker/Dockerfile-build b/tvb_build/docker/Dockerfile-build index 8fa3c0cbc2..4dc15045d6 100644 --- a/tvb_build/docker/Dockerfile-build +++ b/tvb_build/docker/Dockerfile-build @@ -38,11 +38,7 @@ RUN /bin/bash -c "source activate tvb-run"; \ /opt/conda/envs/tvb-run/bin/jupyter notebook --generate-config; \ echo "c.NotebookApp.password='sha1:12bff019c253:9daecd92c2e9bdb10b3b8a06767a74a0fe078d7c'">>$JUPYTER_CONFIG/jupyter_notebook_config.py -RUN wget https://zenodo.org/record/7574266/files/tvb_data.zip?download=1 -O tvb_data.zip; \ - mkdir tvb_data; unzip tvb_data.zip -d tvb_data; rm tvb_data.zip; \ - cd tvb_data; \ - /opt/conda/envs/tvb-run/bin/python setup.py develop;\ - /opt/conda/envs/tvb-docs/bin/python setup.py develop + WORKDIR $USER_HOME COPY requirements_group requirements.txt diff --git a/tvb_build/docker/Dockerfile-run b/tvb_build/docker/Dockerfile-run index d634c1d1d2..4ac08786a3 100644 --- a/tvb_build/docker/Dockerfile-run +++ b/tvb_build/docker/Dockerfile-run @@ -31,10 +31,7 @@ RUN /bin/bash -c "source activate tvb-run"; \ $ENV_BIN/jupyter notebook --generate-config; \ echo "c.NotebookApp.password='sha1:12bff019c253:9daecd92c2e9bdb10b3b8a06767a74a0fe078d7c'">>$JUPYTER_CONFIG/jupyter_notebook_config.py -RUN wget https://zenodo.org/record/7574266/files/tvb_data.zip?download=1 -O tvb_data.zip; \ - mkdir tvb_data; unzip tvb_data.zip -d tvb_data; rm tvb_data.zip; \ - cd tvb_data; \ - $ENV_BIN/python setup.py develop + WORKDIR $USER_HOME COPY requirements_group requirements.txt diff --git a/tvb_build/docker/Dockerfile-win b/tvb_build/docker/Dockerfile-win index 40d21eba5f..4152cea4fd 100644 --- a/tvb_build/docker/Dockerfile-win +++ b/tvb_build/docker/Dockerfile-win @@ -17,12 +17,6 @@ RUN activate tvb-run && pip install lockfile scikit-build RUN activate tvb-run && pip install syncrypto -# Download and install tvb data -RUN mkdir C:\\TVB_CODE -WORKDIR /TVB_CODE -ADD https://zenodo.org/record/7574266/files/tvb_data.zip?download=1 tvb_data.zip -RUN tar -xf tvb_data.zip && dir && del tvb_data.zip -RUN activate tvb-run && python setup.py develop COPY requirements_group requirements.txt RUN activate tvb-run && pip install -r requirements.txt diff --git a/tvb_contrib/tvb/contrib/scripts/datatypes/lookup_tables.py b/tvb_contrib/tvb/contrib/scripts/datatypes/lookup_tables.py index 478259c622..8396a739c1 100644 --- a/tvb_contrib/tvb/contrib/scripts/datatypes/lookup_tables.py +++ b/tvb_contrib/tvb/contrib/scripts/datatypes/lookup_tables.py @@ -31,6 +31,7 @@ """ import numpy +from tvb.datasets import TVBZenodoDataset from tvb.basic.readers import try_get_absolute_path from tvb.basic.neotraits.api import HasTraits, Attr, NArray, Int, Float @@ -74,7 +75,7 @@ class LookUpTable(HasTraits): @staticmethod def populate_table(result, source_file): - source_full_path = try_get_absolute_path("tvb_data.tables", source_file) + source_full_path = TVBZenodoDataset().fetch_data(source_file) zip_data = numpy.load(source_full_path) result.df = zip_data['df'] diff --git a/tvb_documentation/demos/encrypt_data.ipynb b/tvb_documentation/demos/encrypt_data.ipynb index f40bdab160..2197f07bb8 100644 --- a/tvb_documentation/demos/encrypt_data.ipynb +++ b/tvb_documentation/demos/encrypt_data.ipynb @@ -113,10 +113,12 @@ "outputs": [], "source": [ "# EDIT paths to data files here inside the list\n", - "import tvb_data, os, tvb\n", - "from tvb.basic.readers import try_get_absolute_path\n", + "import os, tvb\n", + "from tvb.datasets import TVBZenodoDataset\n", + "dataset = TVBZenodoDataset()\n", "\n", - "paths_to_files = [try_get_absolute_path(\"tvb_data.connectivity\", \"connectivity_76.zip\")]\n", + "\n", + "paths_to_files = [dataset.fetch_data(\"connectivity_76.zip\")]\n", "import_export_encryption_handler = StorageInterface.get_import_export_encryption_handler()\n", "\n", "buffer_size = TvbProfile.current.hpc.CRYPT_BUFFER_SIZE\n", @@ -151,7 +153,7 @@ "encrypted_password = import_export_encryption_handler.encrypt_password(public_key, password_bytes)\n", "\n", "# EDIT path for saving the encrypted password\n", - "encrypted_password_path = os.path.join(tvb_data.__path__[0], 'connectivity')\n", + "encrypted_password_path = os.path.join(dataset.extract_dir, 'tvb_data','connectivity')\n", "\n", "import_export_encryption_handler.save_encrypted_password(encrypted_password, encrypted_password_path)\n", "\n", diff --git a/tvb_documentation/demos/interacting_with_rest_api_launch_operations.ipynb b/tvb_documentation/demos/interacting_with_rest_api_launch_operations.ipynb index 21c88fb5f2..735b493f65 100644 --- a/tvb_documentation/demos/interacting_with_rest_api_launch_operations.ipynb +++ b/tvb_documentation/demos/interacting_with_rest_api_launch_operations.ipynb @@ -27,10 +27,10 @@ "import os\n", "import time\n", "\n", - "import tvb_data\n", "from keycloak import KeycloakOpenID\n", "from tvb.core.entities.model.model_operation import STATUS_ERROR, STATUS_CANCELED, STATUS_FINISHED\n", - "from tvb.interfaces.rest.client.tvb_client import TVBClient" + "from tvb.interfaces.rest.client.tvb_client import TVBClient\n", + "from tvb.datasets import TVBZenodoDataset" ] }, { @@ -39,8 +39,7 @@ "metadata": {}, "outputs": [], "source": [ - "def compute_tvb_data_path(folder, filename):\n", - " return os.path.join(os.path.dirname(tvb_data.__file__), folder, filename)\n", + "\n", "\n", "def monitor_operation(tvb_client, operation_gid):\n", " while True:\n", @@ -95,9 +94,13 @@ "source": [ "from tvb.adapters.uploaders.zip_connectivity_importer import ZIPConnectivityImporterModel, ZIPConnectivityImporter\n", "\n", + "#Loading TVB Zenodo data\n", + "\n", + "dataset = TVBZenodoDataset()\n", + "\n", "# Importing a connectivity from ZIP\n", "zip_connectivity_importer_model = ZIPConnectivityImporterModel()\n", - "zip_connectivity_importer_model.uploaded = compute_tvb_data_path('connectivity', 'connectivity_96.zip')\n", + "zip_connectivity_importer_model.uploaded = dataset.fetch_data(\"connectivity_96.zip\")\n", "zip_connectivity_importer_model.normalization = 'region'\n", "operation_gid = tvb_client.launch_operation(default_project_gid, ZIPConnectivityImporter,\n", " zip_connectivity_importer_model)\n", @@ -122,7 +125,7 @@ "\n", "# Importing a surface from ZIP\n", "zip_surface_importer_model = ZIPSurfaceImporterModel()\n", - "zip_surface_importer_model.uploaded = compute_tvb_data_path('surfaceData', 'cortex_16384.zip')\n", + "zip_surface_importer_model.uploaded = dataset.fetch_data('cortex_16384.zip')\n", "zip_surface_importer_model.surface_type = SurfaceTypesEnum.CORTICAL_SURFACE\n", "zip_surface_importer_model.should_center = False\n", "\n", @@ -146,7 +149,7 @@ "\n", "# Importing a region mapping\n", "rm_importer_model = RegionMappingImporterModel()\n", - "rm_importer_model.mapping_file = compute_tvb_data_path('regionMapping', 'regionMapping_16k_76.txt')\n", + "rm_importer_model.mapping_file = dataset.fetch_data('regionMapping_16k_76.txt')\n", "rm_importer_model.connectivity = connectivity_dto.gid\n", "rm_importer_model.surface = surface_gid\n", "\n", @@ -260,4 +263,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} \ No newline at end of file +} diff --git a/tvb_documentation/demos/interacting_with_the_framework.ipynb b/tvb_documentation/demos/interacting_with_the_framework.ipynb index d13afe11be..8d7d62590a 100644 --- a/tvb_documentation/demos/interacting_with_the_framework.ipynb +++ b/tvb_documentation/demos/interacting_with_the_framework.ipynb @@ -100,8 +100,8 @@ "outputs": [], "source": [ "import os\n", - "import tvb_data\n", - "p = os.path.join(os.path.dirname(tvb_data.__file__), 'connectivity/connectivity_66.zip')\n", + "from tvb.datasets import TVBZenodoDataset\n", + "p = TVBZenodoDataset().fetch_data('connectivity_66.zip')\n", "import_op = import_conn_zip(proj.id, p)\n", "\n", "import_op = wait_to_finish(import_op)\n", @@ -200,20 +200,6 @@ "source": [ "You can re-evaluate this cell multiple times while it's running to see how the results gradually show up as the simulation finishes." ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/tvb_documentation/demos/simulate_for_mouse.ipynb b/tvb_documentation/demos/simulate_for_mouse.ipynb index cfc5fda7f6..6ca7857928 100644 --- a/tvb_documentation/demos/simulate_for_mouse.ipynb +++ b/tvb_documentation/demos/simulate_for_mouse.ipynb @@ -69,8 +69,10 @@ }, "outputs": [], "source": [ - "from tvb.basic.readers import try_get_absolute_path\n", - "connectivity_path = try_get_absolute_path(\"tvb_data\",\"mouse/allen_2mm/Connectivity.h5\")\n", + "from tvb.datasets import TVBZenodoDataset\n", + "dataset = TVBZenodoDataset()\n", + "\n", + "connectivity_path = dataset.fetch_data(\"Connectivity.h5\")\n", "import_op = import_conn_h5(1, connectivity_path)\n", "import_op = wait_to_finish(import_op)\n", "import_op" @@ -401,6 +403,7 @@ "outputs": [], "source": [ "# copy all the ids of the ConnectivityMeasureIndexes obtained before\n", + "\n", "connectivity_measure_ids = [i.id for i in get_operation_results(launched_operation.id)[1:]]" ] }, @@ -419,16 +422,16 @@ "\n", "import h5py\n", "from mpl_toolkits.axes_grid1 import make_axes_locatable\n", - "from tvb.basic.readers import try_get_absolute_path\n", + "\n", "\n", "fig, axes = plt.subplots(1,3)\n", "slice_idy=73\n", "j=0\n", "for conn_measure_id in connectivity_measure_ids:\n", - " f_path = try_get_absolute_path(\"tvb_data\", \"mouse/allen_2mm/RegionVolumeMapping.h5\")\n", + " f_path = dataset.fetch_data(\"RegionVolumeMapping.h5\")\n", " f = h5py.File(f_path, 'r', libver='latest')\n", " Vol=f['array_data'][:,:,:]\n", - " f_path = try_get_absolute_path(\"tvb_data\", \"mouse/allen_2mm/StructuralMRI.h5\")\n", + " f_path = dataset.fetch_data('StructuralMRI.h5')\n", " f = h5py.File(f_path, 'r', libver='latest')\n", " template=f['array_data'][:,:,:]\n", " conn_measure = load_dt(conn_measure_id)\n", @@ -474,13 +477,6 @@ "\n", "[7] Newman, Mark EJ. \"The mathematics of networks.\" The new palgrave encyclopedia of economics 2, no. 2008 (2008): 1-12." ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/tvb_framework/tvb/core/code_versions/code_update_scripts/4455_update_code.py b/tvb_framework/tvb/core/code_versions/code_update_scripts/4455_update_code.py index cd9381b8df..00be86d4a3 100644 --- a/tvb_framework/tvb/core/code_versions/code_update_scripts/4455_update_code.py +++ b/tvb_framework/tvb/core/code_versions/code_update_scripts/4455_update_code.py @@ -29,15 +29,16 @@ """ import os -import tvb_data.obj +from tvb.datasets import TVBZenodoDataset from tvb.adapters.uploaders.obj_importer import ObjSurfaceImporter from tvb.basic.logger.builder import get_logger from tvb.core.services.operation_service import OperationService from tvb.core.entities.storage import dao from tvb.datatypes.surfaces import SurfaceTypesEnum -DATA_FILE_EEG_CAP = os.path.join(os.path.dirname(tvb_data.obj.__file__), "eeg_cap.obj") -DATA_FILE_FACE = os.path.join(os.path.dirname(tvb_data.obj.__file__), "face_surface.obj") +dataset = TVBZenodoDataset() +DATA_FILE_EEG_CAP = dataset.fetch_data("eeg_cap.obj") +DATA_FILE_FACE = dataset.fetch_data('face_surface.obj') LOGGER = get_logger(__name__) PAGE_SIZE = 20 diff --git a/tvb_framework/tvb/core/code_versions/code_update_scripts/4750_update_code.py b/tvb_framework/tvb/core/code_versions/code_update_scripts/4750_update_code.py index 7f107c564f..1cb0ac2dbc 100644 --- a/tvb_framework/tvb/core/code_versions/code_update_scripts/4750_update_code.py +++ b/tvb_framework/tvb/core/code_versions/code_update_scripts/4750_update_code.py @@ -28,13 +28,13 @@ .. moduleauthor:: Bogdan Neacsa """ import os -import tvb_data.sensors +from tvb.datasets import TVBZenodoDataset from tvb.adapters.uploaders.sensors_importer import SensorsImporter from tvb.basic.logger.builder import get_logger from tvb.core.entities.storage import dao from tvb.core.services.operation_service import OperationService -DATA_FILE = os.path.join(os.path.dirname(tvb_data.sensors.__file__), "seeg_39.txt.bz2") +DATA_FILE = TVBZenodoDataset().fetch_data('seeg_39.txt.bz2') LOGGER = get_logger(__name__) PAGE_SIZE = 20 diff --git a/tvb_framework/tvb/core/code_versions/code_update_scripts/6093_update_code.py b/tvb_framework/tvb/core/code_versions/code_update_scripts/6093_update_code.py index ffffdfe028..76e0a7dd79 100644 --- a/tvb_framework/tvb/core/code_versions/code_update_scripts/6093_update_code.py +++ b/tvb_framework/tvb/core/code_versions/code_update_scripts/6093_update_code.py @@ -30,14 +30,14 @@ .. moduleauthor:: Mihai Andrei """ import os -import tvb_data.obj +from tvb.datasets import TVBZenodoDataset from tvb.adapters.uploaders.obj_importer import ObjSurfaceImporter from tvb.basic.logger.builder import get_logger from tvb.core.entities.storage import dao from tvb.core.services.operation_service import OperationService from tvb.datatypes.surfaces import SurfaceTypesEnum -DATA_FILE_FACE = os.path.join(os.path.dirname(tvb_data.obj.__file__), "face_surface.obj") +DATA_FILE_FACE = TVBZenodoDataset().fetch_data('face_surface.obj') LOGGER = get_logger(__name__) PAGE_SIZE = 20 diff --git a/tvb_framework/tvb/core/code_versions/code_update_scripts/6600_update_code.py b/tvb_framework/tvb/core/code_versions/code_update_scripts/6600_update_code.py index a8a89a476f..03ef9089e6 100644 --- a/tvb_framework/tvb/core/code_versions/code_update_scripts/6600_update_code.py +++ b/tvb_framework/tvb/core/code_versions/code_update_scripts/6600_update_code.py @@ -33,10 +33,9 @@ from tvb.basic.logger.builder import get_logger from tvb.core.entities.storage import dao from tvb.core.services.import_service import ImportService -import tvb_data - -DATA_FILE = os.path.join(os.path.dirname(tvb_data.__file__), "Default_Project.zip") +from tvb.datasets import TVBZenodoDataset +DATA_FILE = TVBZenodoDataset().fetch_data('Default_Project.zip') LOGGER = get_logger(__name__) diff --git a/tvb_framework/tvb/core/services/user_service.py b/tvb_framework/tvb/core/services/user_service.py index 9a2de56dc6..0fe08e7415 100644 --- a/tvb_framework/tvb/core/services/user_service.py +++ b/tvb_framework/tvb/core/services/user_service.py @@ -33,7 +33,7 @@ import os import random import six -import tvb_data +from tvb.datasets import TVBZenodoDataset from tvb.basic.logger.builder import get_logger from tvb.basic.profile import TvbProfile from tvb.config import DEFAULT_PROJECT_GID @@ -120,7 +120,7 @@ def create_user(self, username=None, display_name=None, password=None, password2 user = dao.store_entity(user) if role == ROLE_ADMINISTRATOR and not skip_import: - to_upload = os.path.join(os.path.dirname(tvb_data.__file__), "Default_Project.zip") + to_upload = TVBZenodoDataset().fetch_data('Default_Project.zip') if not os.path.exists(to_upload): self.logger.warning("Could not find DEFAULT PROJECT at path %s. You might want to import it " "yourself. See TVB documentation about where to find it!" % to_upload) diff --git a/tvb_framework/tvb/interfaces/command/benchmark.py b/tvb_framework/tvb/interfaces/command/benchmark.py index ae736f87b9..02a536d459 100644 --- a/tvb_framework/tvb/interfaces/command/benchmark.py +++ b/tvb_framework/tvb/interfaces/command/benchmark.py @@ -31,7 +31,7 @@ from os import path -import tvb_data +from tvb.datasets import TVBZenodoDataset from tvb.adapters.datatypes.db.connectivity import ConnectivityIndex from tvb.core.entities.file.simulator.view_model import HeunDeterministicViewModel from tvb.interfaces.command.lab import * @@ -56,12 +56,12 @@ def _fire_simulation(project_id, simulator_vm): def _create_bench_project(): prj = new_project("benchmark_project_ %s" % datetime.now()) - data_dir = path.abspath(path.dirname(tvb_data.__file__)) - zip_path = path.join(data_dir, 'connectivity', 'connectivity_68.zip') + dataset = TVBZenodoDataset() + zip_path = dataset.fetch_data('connectivity_68.zip') import_conn_zip(prj.id, zip_path) - zip_path = path.join(data_dir, 'connectivity', 'connectivity_96.zip') + zip_path = dataset.fetch_data('connectivity_96.zip') import_conn_zip(prj.id, zip_path) - zip_path = path.join(data_dir, 'connectivity', 'connectivity_192.zip') + zip_path = dataset.fetch_data('connectivity_192.zip') import_conn_zip(prj.id, zip_path) conn68 = dao.get_generic_entity(ConnectivityIndex, 68, "number_of_regions")[0] diff --git a/tvb_framework/tvb/interfaces/command/demos/importers/brain_tumor_connectivity_importer.py b/tvb_framework/tvb/interfaces/command/demos/importers/brain_tumor_connectivity_importer.py index a69006cf0b..7bbb0a369a 100644 --- a/tvb_framework/tvb/interfaces/command/demos/importers/brain_tumor_connectivity_importer.py +++ b/tvb_framework/tvb/interfaces/command/demos/importers/brain_tumor_connectivity_importer.py @@ -31,6 +31,7 @@ .. moduleauthor:: Bogdan Valean """ import sys +from tvb.datasets import TVBZenodoDataset from tvb.adapters.uploaders.region_mapping_importer import RegionMappingImporter, RegionMappingImporterModel from tvb.adapters.uploaders.zip_surface_importer import ZIPSurfaceImporter, ZIPSurfaceImporterModel from tvb.basic.logger.builder import get_logger @@ -66,8 +67,9 @@ def import_tumor_connectivities(project_id, folder_path): def import_surface_rm(project_id, conn_gid): # Import surface and region mapping from tvb_data berlin subjects (68 regions) - rm_file = try_get_absolute_path("tvb_data", "berlinSubjects/DH_20120806/DH_20120806_RegionMapping.txt") - surface_zip_file = try_get_absolute_path("tvb_data", "berlinSubjects/DH_20120806/DH_20120806_Surface_Cortex.zip") + dataset = TVBZenodoDataset() + rm_file = dataset.fetch_data('DH_20120806_RegionMapping.txt') + surface_zip_file = dataset.fetch_data('DH_20120806_Surface_Cortex.zip') surface_importer = ABCAdapter.build_adapter_from_class(ZIPSurfaceImporter) surface_imp_model = ZIPSurfaceImporterModel() diff --git a/tvb_framework/tvb/interfaces/rest/client/examples/utils.py b/tvb_framework/tvb/interfaces/rest/client/examples/utils.py index bb1711a435..0111c74980 100644 --- a/tvb_framework/tvb/interfaces/rest/client/examples/utils.py +++ b/tvb_framework/tvb/interfaces/rest/client/examples/utils.py @@ -28,7 +28,7 @@ import sys import time -import tvb_data +from tvb.datasets import TVBZenodoDataset from tvb.basic.logger.builder import get_logger from tvb.core.entities.model.model_operation import STATUS_ERROR, STATUS_CANCELED, STATUS_FINISHED @@ -44,7 +44,7 @@ def compute_rest_url(): def compute_tvb_data_path(folder, filename): - return os.path.join(os.path.dirname(tvb_data.__file__), folder, filename) + return os.path.join(TVBZenodoDataset().extract_dir, folder, filename) logger = get_logger(__name__) diff --git a/tvb_framework/tvb/tests/framework/adapters/analyzers/bct_test.py b/tvb_framework/tvb/tests/framework/adapters/analyzers/bct_test.py index d26ca55b0e..b8aca921ff 100644 --- a/tvb_framework/tvb/tests/framework/adapters/analyzers/bct_test.py +++ b/tvb_framework/tvb/tests/framework/adapters/analyzers/bct_test.py @@ -29,7 +29,7 @@ """ import os -import tvb_data +from tvb.datasets import TVBZenodoDataset from tvb.adapters.analyzers.bct_adapters import BaseBCTModel from tvb.core.entities.model.model_operation import Algorithm from tvb.tests.framework.core.base_testcase import TransactionalTestCase @@ -56,7 +56,7 @@ def transactional_setup_method(self): self.test_user = TestFactory.create_user("BCT_User") self.test_project = TestFactory.create_project(self.test_user, "BCT-Project") # Make sure Connectivity is in DB - zip_path = os.path.join(os.path.dirname(tvb_data.__file__), 'connectivity', 'connectivity_76.zip') + zip_path = TVBZenodoDataset().fetch_data('connectivity_76.zip') self.connectivity = TestFactory.import_zip_connectivity(self.test_user, self.test_project, zip_path) algorithms = dao.get_generic_entity(Algorithm, 'Brain Connectivity Toolbox', 'group_description') diff --git a/tvb_framework/tvb/tests/framework/adapters/analyzers/timeseries_metrics_adapter_test.py b/tvb_framework/tvb/tests/framework/adapters/analyzers/timeseries_metrics_adapter_test.py index 9b68008bb2..20b089f39c 100644 --- a/tvb_framework/tvb/tests/framework/adapters/analyzers/timeseries_metrics_adapter_test.py +++ b/tvb_framework/tvb/tests/framework/adapters/analyzers/timeseries_metrics_adapter_test.py @@ -29,7 +29,7 @@ """ import os -import tvb_data +from tvb.datasets import TVBZenodoDataset import json from tvb.adapters.datatypes.db.mapped_value import DatatypeMeasureIndex @@ -51,7 +51,7 @@ def transactional_setup_method(self): """ self.test_user = TestFactory.create_user() self.test_project = TestFactory.create_project(self.test_user) - zip_path = os.path.join(os.path.dirname(tvb_data.__file__), 'connectivity', 'connectivity_66.zip') + zip_path = TVBZenodoDataset().fetch_data('connectivity_66.zip') TestFactory.import_zip_connectivity(self.test_user, self.test_project, zip_path) def test_adapter_launch(self, connectivity_factory, region_mapping_factory, diff --git a/tvb_framework/tvb/tests/framework/adapters/creators/stimulus_creator_test.py b/tvb_framework/tvb/tests/framework/adapters/creators/stimulus_creator_test.py index 77d623909c..64122a9dde 100644 --- a/tvb_framework/tvb/tests/framework/adapters/creators/stimulus_creator_test.py +++ b/tvb_framework/tvb/tests/framework/adapters/creators/stimulus_creator_test.py @@ -27,8 +27,7 @@ import json import os import numpy -import tvb_data -import tvb_data.surfaceData +from tvb.datasets import TVBZenodoDataset from tvb.adapters.creators.stimulus_creator import RegionStimulusCreator, SurfaceStimulusCreator from tvb.adapters.datatypes.db.connectivity import ConnectivityIndex @@ -52,11 +51,14 @@ def transactional_setup_method(self): self.test_project = TestFactory.create_project(self.test_user, "Stim_Project") self.storage_interface = StorageInterface() - zip_path = os.path.join(os.path.dirname(tvb_data.__file__), 'connectivity', 'connectivity_66.zip') + dataset = TVBZenodoDataset() + + zip_path = dataset.fetch_data('connectivity_66.zip') TestFactory.import_zip_connectivity(self.test_user, self.test_project, zip_path) self.connectivity = TestFactory.get_entity(self.test_project, ConnectivityIndex) - cortex = os.path.join(os.path.dirname(tvb_data.surfaceData.__file__), 'cortex_16384.zip') + cortex = dataset.fetch_data('cortex_16384.zip') + self.surface = TestFactory.import_surface_zip(self.test_user, self.test_project, cortex, SurfaceTypesEnum.CORTICAL_SURFACE) diff --git a/tvb_framework/tvb/tests/framework/adapters/simulator/simulator_adapter_test.py b/tvb_framework/tvb/tests/framework/adapters/simulator/simulator_adapter_test.py index 0db157f22c..5f9210c3c0 100644 --- a/tvb_framework/tvb/tests/framework/adapters/simulator/simulator_adapter_test.py +++ b/tvb_framework/tvb/tests/framework/adapters/simulator/simulator_adapter_test.py @@ -28,8 +28,7 @@ .. moduleauthor:: Lia Domide """ -import tvb_data.surfaceData -import tvb_data.regionMapping +from tvb.datasets import TVBZenodoDataset from os import path from tvb.adapters.datatypes.db.time_series import TimeSeriesRegionIndex @@ -111,11 +110,12 @@ def test_estimate_execution_time(self, connectivity_index_factory): self.simulator_adapter.configure(model) estimation1 = self.simulator_adapter.get_execution_time_approximation(model) + dataset = TVBZenodoDataset() # import surfaceData and region mapping - cortex_file = path.join(path.dirname(tvb_data.surfaceData.__file__), 'cortex_16384.zip') + cortex_file = dataset.fetch_data('cortex_16384.zip') surface = TestFactory.import_surface_zip(self.test_user, self.test_project, cortex_file, SurfaceTypesEnum.CORTICAL_SURFACE) - rm_file = path.join(path.dirname(tvb_data.regionMapping.__file__), 'regionMapping_16k_76.txt') + rm_file = dataset.fetch_data('regionMapping_16k_76.txt') region_mapping = TestFactory.import_region_mapping(self.test_user, self.test_project, rm_file, surface.gid, model.connectivity.hex) local_conn = TestFactory.create_local_connectivity(self.test_user, self.test_project, surface.gid) diff --git a/tvb_framework/tvb/tests/framework/adapters/uploaders/connectivity_measure_importer_test.py b/tvb_framework/tvb/tests/framework/adapters/uploaders/connectivity_measure_importer_test.py index e7003a17d7..bc0d58d340 100644 --- a/tvb_framework/tvb/tests/framework/adapters/uploaders/connectivity_measure_importer_test.py +++ b/tvb_framework/tvb/tests/framework/adapters/uploaders/connectivity_measure_importer_test.py @@ -32,7 +32,7 @@ import os.path import pytest -import tvb_data +from tvb.datasets import TVBZenodoDataset from tvb.adapters.datatypes.db.graph import ConnectivityMeasureIndex from tvb.adapters.uploaders.connectivity_measure_importer import ConnectivityMeasureImporter from tvb.adapters.uploaders.connectivity_measure_importer import ConnectivityMeasureImporterModel @@ -48,7 +48,7 @@ class TestConnectivityMeasureImporter(BaseTestCase): """ def setup_method(self): - zip_path = os.path.join(os.path.dirname(tvb_data.__file__), 'connectivity', 'connectivity_66.zip') + zip_path = TVBZenodoDataset().fetch_data('connectivity_66.zip') self.test_user = TestFactory.create_user('Test_User_CM') self.test_project = TestFactory.create_project(self.test_user, "Test_Project_CM") self.connectivity = TestFactory.import_zip_connectivity(self.test_user, self.test_project, zip_path, "John") diff --git a/tvb_framework/tvb/tests/framework/adapters/uploaders/connectivity_zip_importer_test.py b/tvb_framework/tvb/tests/framework/adapters/uploaders/connectivity_zip_importer_test.py index abe8b64008..4d30e57789 100644 --- a/tvb_framework/tvb/tests/framework/adapters/uploaders/connectivity_zip_importer_test.py +++ b/tvb_framework/tvb/tests/framework/adapters/uploaders/connectivity_zip_importer_test.py @@ -30,7 +30,8 @@ """ from os import path -import tvb_data +#import tvb_data +from tvb.datasets import TVBZenodoDataset from tvb.adapters.datatypes.db.connectivity import ConnectivityIndex from tvb.tests.framework.core.base_testcase import BaseTestCase from tvb.tests.framework.core.factory import TestFactory @@ -58,7 +59,7 @@ def test_happy_flow_import(self): """ Test that importing a CFF generates at least one DataType in DB. """ - zip_path = path.join(path.dirname(tvb_data.__file__), 'connectivity', 'connectivity_96.zip') + zip_path = TVBZenodoDataset().fetch_data('connectivity_96.zip') dt_count_before = TestFactory.get_entity_count(self.test_project, ConnectivityIndex) TestFactory.import_zip_connectivity(self.test_user, self.test_project, zip_path, "John", False) dt_count_after = TestFactory.get_entity_count(self.test_project, ConnectivityIndex) diff --git a/tvb_framework/tvb/tests/framework/adapters/uploaders/csv_importer_test.py b/tvb_framework/tvb/tests/framework/adapters/uploaders/csv_importer_test.py index 4db2d6475d..2c9addf2f5 100644 --- a/tvb_framework/tvb/tests/framework/adapters/uploaders/csv_importer_test.py +++ b/tvb_framework/tvb/tests/framework/adapters/uploaders/csv_importer_test.py @@ -29,8 +29,7 @@ """ import pytest -import tvb_data -from os import path +from tvb.datasets import TVBZenodoDataset from tvb.adapters.datatypes.db.connectivity import ConnectivityIndex from tvb.adapters.uploaders.csv_connectivity_importer import CSVConnectivityImporter from tvb.adapters.uploaders.csv_connectivity_importer import CSVConnectivityParser, CSVConnectivityImporterModel @@ -46,10 +45,10 @@ class TestCSVConnectivityParser(BaseTestCase): - BASE_PTH = path.join(path.dirname(tvb_data.__file__), 'dti_pipeline_toronto') - + + def test_parse_happy(self): - cap_pth = path.join(self.BASE_PTH, 'output_ConnectionDistanceMatrix.csv') + cap_pth = TVBZenodoDataset().fetch_data('output_ConnectionDistanceMatrix.csv') with open(cap_pth) as f: result_conn = CSVConnectivityParser(f).result_conn @@ -62,6 +61,7 @@ class TestCSVConnectivityImporter(BaseTestCase): """ Unit-tests for csv connectivity importer. """ + dataset = TVBZenodoDataset() def setup_method(self): self.test_user = TestFactory.create_user() @@ -75,15 +75,11 @@ def teardown_method(self): self.clean_database() def _import_csv_test_connectivity(self, reference_connectivity_gid, subject): - # First prepare the input data: - data_dir = path.abspath(path.dirname(tvb_data.__file__)) - - toronto_dir = path.join(data_dir, 'dti_pipeline_toronto') - weights = path.join(toronto_dir, 'output_ConnectionCapacityMatrix.csv') - tracts = path.join(toronto_dir, 'output_ConnectionDistanceMatrix.csv') - tmp_folder = self.storage_interface.get_temp_folder(self.test_project.name) - weights_tmp = path.join(tmp_folder, 'output_ConnectionCapacityMatrix.csv.tmp') - tracts_tmp = path.join(tmp_folder, 'output_ConnectionDistanceMatrix.csv.tmp') + ### First prepare input data: + weights = self.dataset.fetch_data('output_ConnectionCapacityMatrix.csv') + tracts = self.dataset.fetch_data('output_ConnectionDistanceMatrix.csv') + weights_tmp = weights + '.tmp' + tracts_tmp = tracts + '.tmp' self.storage_interface.copy_file(weights, weights_tmp) self.storage_interface.copy_file(tracts, tracts_tmp) @@ -99,7 +95,7 @@ def test_happy_flow_import(self): Test that importing a CFF generates at least one DataType in DB. """ - zip_path = path.join(path.dirname(tvb_data.__file__), 'connectivity', 'connectivity_96.zip') + zip_path = self.dataset.fetch_data('connectivity_96.zip') TestFactory.import_zip_connectivity(self.test_user, self.test_project, zip_path, subject=TEST_SUBJECT_A) field = FilterChain.datatype + '.subject' @@ -131,7 +127,7 @@ def test_happy_flow_import(self): assert (reference_connectivity.region_labels == imported_connectivity.region_labels).all() def test_bad_reference(self): - zip_path = path.join(path.dirname(tvb_data.__file__), 'connectivity', 'connectivity_66.zip') + zip_path = self.dataset.fetch_data('connectivity_66.zip') TestFactory.import_zip_connectivity(self.test_user, self.test_project, zip_path) field = FilterChain.datatype + '.subject' filters = FilterChain('', [field], [TEST_SUBJECT_A], ['!=']) diff --git a/tvb_framework/tvb/tests/framework/adapters/uploaders/encrypt_decrypt_test.py b/tvb_framework/tvb/tests/framework/adapters/uploaders/encrypt_decrypt_test.py index 990ffb7f85..3ea827175e 100644 --- a/tvb_framework/tvb/tests/framework/adapters/uploaders/encrypt_decrypt_test.py +++ b/tvb_framework/tvb/tests/framework/adapters/uploaders/encrypt_decrypt_test.py @@ -31,7 +31,8 @@ import os import pyAesCrypt import pytest -import tvb_data +from tvb.datasets import TVBZenodoDataset + import tempfile from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization @@ -46,14 +47,17 @@ class TestEncryptionDecryption(TransactionalTestCase): + dataset = TVBZenodoDataset() # noinspection PyTypeChecker - @pytest.mark.parametrize("dir_name, file_name", [('connectivity', 'connectivity_76.zip'), - ('surfaceData', 'cortex_2x120k.zip'), - ('projectionMatrix', 'projection_meg_276_surface_16k.npy'), - ('h5', 'TimeSeriesRegion.h5')]) - def test_encrypt_decrypt(self, dir_name, file_name): + + @pytest.mark.parametrize(" file_name", [('connectivity_76.zip'), + ( 'cortex_2x120k.zip'), + ( 'projection_meg_276_surface_16k.npy'), + ( 'TimeSeriesRegion.h5')]) + def test_encrypt_decrypt(self, file_name): handler = StorageInterface.get_import_export_encryption_handler() + # Generate a private key and public key private_key = rsa.generate_private_key( public_exponent=65537, @@ -74,7 +78,8 @@ def test_encrypt_decrypt(self, dir_name, file_name): with open(private_key_path, 'wb') as f: f.write(pem) - path_to_file = os.path.join(os.path.dirname(tvb_data.__file__), dir_name, file_name) + path_to_file = self.dataset.fetch_data(file_name) + # Create model for ABCUploader connectivity_model = ZIPConnectivityImporterModel() diff --git a/tvb_framework/tvb/tests/framework/adapters/uploaders/gifti_importer_test.py b/tvb_framework/tvb/tests/framework/adapters/uploaders/gifti_importer_test.py index 00f1e21662..c0735b4c45 100644 --- a/tvb_framework/tvb/tests/framework/adapters/uploaders/gifti_importer_test.py +++ b/tvb_framework/tvb/tests/framework/adapters/uploaders/gifti_importer_test.py @@ -29,7 +29,7 @@ """ import os -import tvb_data.gifti as demo_data +from tvb.datasets import TVBZenodoDataset from tvb.adapters.uploaders.gifti.parser import GIFTIParser from tvb.core.services.exceptions import OperationException from tvb.storage.storage_interface import StorageInterface @@ -42,9 +42,11 @@ class TestGIFTISurfaceImporter(BaseTestCase): Unit-tests for GIFTI Surface importer. """ - GIFTI_SURFACE_FILE = os.path.join(os.path.dirname(demo_data.__file__), 'sample.cortex.gii') - GIFTI_TIME_SERIES_FILE = os.path.join(os.path.dirname(demo_data.__file__), 'sample.time_series.gii') WRONG_GII_FILE = os.path.abspath(__file__) + + dataset = TVBZenodoDataset() + GIFTI_SURFACE_FILE = dataset.fetch_data('sample.cortex.gii') + GIFTI_TIME_SERIES_FILE = dataset.fetch_data( 'sample.time_series.gii') def setup_method(self): self.test_user = TestFactory.create_user('Gifti_User') diff --git a/tvb_framework/tvb/tests/framework/adapters/uploaders/mat_timeseries_importer_test.py b/tvb_framework/tvb/tests/framework/adapters/uploaders/mat_timeseries_importer_test.py index 5c8df773f2..855b08d504 100644 --- a/tvb_framework/tvb/tests/framework/adapters/uploaders/mat_timeseries_importer_test.py +++ b/tvb_framework/tvb/tests/framework/adapters/uploaders/mat_timeseries_importer_test.py @@ -32,7 +32,7 @@ import os -import tvb_data +from tvb.datasets import TVBZenodoDataset from tvb.adapters.datatypes.db.time_series import TimeSeriesRegionIndex from tvb.adapters.uploaders.mat_timeseries_importer import RegionMatTimeSeriesImporterModel, RegionTimeSeriesImporter from tvb.tests.framework.core.base_testcase import BaseTestCase @@ -40,9 +40,9 @@ class TestMatTimeSeriesImporter(BaseTestCase): - base_pth = os.path.join(os.path.dirname(tvb_data.__file__), 'berlinSubjects', 'QL_20120814') - bold_path = os.path.join(base_pth, 'QL_BOLD_regiontimecourse.mat') - connectivity_path = os.path.join(base_pth, 'QL_20120814_Connectivity.zip') + dataset = TVBZenodoDataset() + bold_path = dataset.fetch_data('QL_BOLD_regiontimecourse.mat') + connectivity_path = dataset.fetch_data('QL_20120814_Connectivity.zip') def setup_method(self): self.test_user = TestFactory.create_user('Mat_Timeseries_User') diff --git a/tvb_framework/tvb/tests/framework/adapters/uploaders/nifti_importer_test.py b/tvb_framework/tvb/tests/framework/adapters/uploaders/nifti_importer_test.py index d1c0572e6b..e494ce8bbd 100644 --- a/tvb_framework/tvb/tests/framework/adapters/uploaders/nifti_importer_test.py +++ b/tvb_framework/tvb/tests/framework/adapters/uploaders/nifti_importer_test.py @@ -31,8 +31,7 @@ import os import numpy -import tvb_data -import tvb_data.nifti as demo_data +from tvb.datasets import TVBZenodoDataset from tvb.adapters.datatypes.db.connectivity import ConnectivityIndex from tvb.adapters.datatypes.db.region_mapping import RegionVolumeMappingIndex from tvb.adapters.datatypes.db.structural import StructuralMRIIndex @@ -51,11 +50,13 @@ class TestNIFTIImporter(BaseTestCase): Unit-tests for NIFTI importer. """ - NII_FILE = os.path.join(os.path.dirname(demo_data.__file__), 'minimal.nii') - GZ_NII_FILE = os.path.join(os.path.dirname(demo_data.__file__), 'minimal.nii.gz') - TIMESERIES_NII_FILE = os.path.join(os.path.dirname(demo_data.__file__), 'time_series_152.nii.gz') - WRONG_NII_FILE = os.path.abspath(__file__) - TXT_FILE = os.path.join(os.path.dirname(demo_data.__file__), 'volume_mapping/mapping_FS_76.txt') + + dataset = TVBZenodoDataset() + NII_FILE = dataset.fetch_data('minimal.nii') + GZ_NII_FILE = dataset.fetch_data('minimal.nii.gz') + TIMESERIES_NII_FILE = dataset.fetch_data('time_series_152.nii.gz') + WRONG_NII_FILE = os.path.abspath(__file__) #? + TXT_FILE = dataset.fetch_data('mapping_FS_76.txt') DEFAULT_ORIGIN = [[0.0, 0.0, 0.0]] UNKNOWN_STR = "unknown" @@ -144,7 +145,7 @@ def test_import_region_mapping(self): """ This method tests import of a NIFTI file compressed in GZ format. """ - zip_path = os.path.join(os.path.dirname(tvb_data.__file__), 'connectivity', 'connectivity_76.zip') + zip_path = self.dataset.fetch_data('connectivity_76.zip') TestFactory.import_zip_connectivity(self.test_user, self.test_project, zip_path, "John") to_link_conn = TestFactory.get_entity(self.test_project, ConnectivityIndex) diff --git a/tvb_framework/tvb/tests/framework/adapters/uploaders/obj_importer_test.py b/tvb_framework/tvb/tests/framework/adapters/uploaders/obj_importer_test.py index 27b0776b10..45a3792b87 100644 --- a/tvb_framework/tvb/tests/framework/adapters/uploaders/obj_importer_test.py +++ b/tvb_framework/tvb/tests/framework/adapters/uploaders/obj_importer_test.py @@ -29,8 +29,7 @@ """ import os -import tvb_data.obj - +from tvb.datasets import TVBZenodoDataset from tvb.core.neocom import h5 from tvb.datatypes.surfaces import SurfaceTypesEnum from tvb.tests.framework.core.base_testcase import BaseTestCase @@ -41,9 +40,11 @@ class TestObjSurfaceImporter(BaseTestCase): """ Unit-tests for Obj Surface importer. """ + + dataset = TVBZenodoDataset() + torus = dataset.fetch_data('test_torus.obj') + face = dataset.fetch_data('face_surface.obj') - torus = os.path.join(os.path.dirname(tvb_data.obj.__file__), 'test_torus.obj') - face = os.path.join(os.path.dirname(tvb_data.obj.__file__), 'face_surface.obj') def setup_method(self): self.test_user = TestFactory.create_user('Obj_Importer_User') diff --git a/tvb_framework/tvb/tests/framework/adapters/uploaders/projection_matrix_importer_test.py b/tvb_framework/tvb/tests/framework/adapters/uploaders/projection_matrix_importer_test.py index 5723ec6913..1f413def77 100644 --- a/tvb_framework/tvb/tests/framework/adapters/uploaders/projection_matrix_importer_test.py +++ b/tvb_framework/tvb/tests/framework/adapters/uploaders/projection_matrix_importer_test.py @@ -31,9 +31,7 @@ import os -import tvb_data.projectionMatrix as dataset -import tvb_data.sensors -import tvb_data.surfaceData +from tvb.datasets import TVBZenodoDataset from tvb.adapters.datatypes.db.projections import ProjectionMatrixIndex from tvb.core.services.exceptions import OperationException from tvb.datatypes.sensors import SensorTypesEnum @@ -46,6 +44,7 @@ class TestProjectionMatrix(BaseTestCase): """ Unit-tests for CFF-importer. """ + dataset = TVBZenodoDataset() def setup_method(self): """ @@ -54,11 +53,11 @@ def setup_method(self): self.test_user = TestFactory.create_user("UserPM") self.test_project = TestFactory.create_project(self.test_user) - zip_path = os.path.join(os.path.dirname(tvb_data.sensors.__file__), 'eeg_brainstorm_65.txt') + zip_path = self.dataset.fetch_data('eeg_brainstorm_65.txt') self.sensors = TestFactory.import_sensors(self.test_user, self.test_project, zip_path, SensorTypesEnum.TYPE_EEG) - zip_path = os.path.join(os.path.dirname(tvb_data.surfaceData.__file__), 'cortex_16384.zip') + zip_path = self.dataset.fetch_data('cortex_16384.zip') self.surface = TestFactory.import_surface_zip(self.test_user, self.test_project, zip_path, SurfaceTypesEnum.CORTICAL_SURFACE, True) @@ -72,8 +71,7 @@ def test_wrong_shape(self): """ Verifies that importing a different shape throws exception """ - file_path = os.path.join(os.path.abspath(os.path.dirname(dataset.__file__)), - 'projection_eeg_62_surface_16k.mat') + file_path = self.dataset.fetch_data('projection_eeg_62_surface_16k.mat') try: TestFactory.import_projection_matrix(self.test_user, self.test_project, file_path, self.sensors.gid, @@ -87,8 +85,7 @@ def test_happy_flow_surface_import(self): Verifies the happy flow for importing a surface. """ dt_count_before = TestFactory.get_entity_count(self.test_project, ProjectionMatrixIndex) - file_path = os.path.join(os.path.abspath(os.path.dirname(dataset.__file__)), - 'projection_eeg_65_surface_16k.npy') + file_path = self.dataset.fetch_data('projection_eeg_65_surface_16k.npy') TestFactory.import_projection_matrix(self.test_user, self.test_project, file_path, self.sensors.gid, self.surface.gid, False) diff --git a/tvb_framework/tvb/tests/framework/adapters/uploaders/region_mapping_importer_test.py b/tvb_framework/tvb/tests/framework/adapters/uploaders/region_mapping_importer_test.py index 42e1bb6370..c27d7152d3 100644 --- a/tvb_framework/tvb/tests/framework/adapters/uploaders/region_mapping_importer_test.py +++ b/tvb_framework/tvb/tests/framework/adapters/uploaders/region_mapping_importer_test.py @@ -30,8 +30,7 @@ import os import tvb.tests.framework.adapters.uploaders.test_data as test_data -import tvb_data.regionMapping as demo_data -import tvb_data.surfaceData +from tvb.datasets import TVBZenodoDataset from tvb.adapters.datatypes.db.surface import SurfaceIndex from tvb.basic.neotraits.ex import TraitValueError from tvb.core.adapters.exceptions import LaunchException @@ -48,10 +47,12 @@ class TestRegionMappingImporter(BaseTestCase): """ Unit-tests for RegionMapping importer. """ + dataset = TVBZenodoDataset() - TXT_FILE = os.path.join(os.path.dirname(demo_data.__file__), 'regionMapping_16k_76.txt') - ZIP_FILE = os.path.join(os.path.dirname(demo_data.__file__), 'regionMapping_16k_76.zip') - BZ2_FILE = os.path.join(os.path.dirname(demo_data.__file__), 'regionMapping_16k_76.bz2') + + TXT_FILE = dataset.fetch_data('regionMapping_16k_76.txt') + ZIP_FILE = dataset.fetch_data('regionMapping_16k_76.zip') + BZ2_FILE = dataset.fetch_data('regionMapping_16k_76.bz2') # Wrong data WRONG_FILE_1 = os.path.join(os.path.dirname(test_data.__file__), 'region_mapping_wrong_1.txt') @@ -67,12 +68,12 @@ def setup_method(self): self.test_user = TestFactory.create_user("UserRM") self.test_project = TestFactory.create_project(self.test_user) - zip_path = os.path.join(os.path.dirname(tvb_data.__file__), 'connectivity', 'connectivity_76.zip') + zip_path = self.dataset.fetch_data("connectivity_76.zip") self.connectivity = TestFactory.import_zip_connectivity(self.test_user, self.test_project, zip_path, "John") field = FilterChain.datatype + '.surface_type' filters = FilterChain('', [field], [SurfaceTypesEnum.CORTICAL_SURFACE.value], ['==']) - cortex = os.path.join(os.path.dirname(tvb_data.surfaceData.__file__), 'cortex_16384.zip') + cortex = self.dataset.fetch_data('cortex_16384.zip') TestFactory.import_surface_zip(self.test_user, self.test_project, cortex, SurfaceTypesEnum.CORTICAL_SURFACE) self.surface = TestFactory.get_entity(self.test_project, SurfaceIndex, filters) diff --git a/tvb_framework/tvb/tests/framework/adapters/uploaders/sensors_importer_test.py b/tvb_framework/tvb/tests/framework/adapters/uploaders/sensors_importer_test.py index d05e14edfb..407484ed49 100644 --- a/tvb_framework/tvb/tests/framework/adapters/uploaders/sensors_importer_test.py +++ b/tvb_framework/tvb/tests/framework/adapters/uploaders/sensors_importer_test.py @@ -30,7 +30,7 @@ import os -import tvb_data.sensors as demo_data +from tvb.datasets import TVBZenodoDataset from tvb.adapters.uploaders.sensors_importer import SensorsImporter, SensorsImporterModel from tvb.core.neocom import h5 from tvb.core.services.exceptions import OperationException @@ -44,8 +44,9 @@ class TestSensorsImporter(BaseTestCase): """ Unit-tests for Sensors importer. """ - EEG_FILE = os.path.join(os.path.dirname(demo_data.__file__), 'eeg_unitvector_62.txt.bz2') - MEG_FILE = os.path.join(os.path.dirname(demo_data.__file__), 'meg_151.txt.bz2') + dataset = TVBZenodoDataset() + EEG_FILE = dataset.fetch_data('eeg_unitvector_62.txt.bz2') + MEG_FILE = dataset.fetch_data('meg_151.txt.bz2') def setup_method(self): """ diff --git a/tvb_framework/tvb/tests/framework/adapters/uploaders/zip_surface_importer_test.py b/tvb_framework/tvb/tests/framework/adapters/uploaders/zip_surface_importer_test.py index d047a63b56..65e1152efa 100644 --- a/tvb_framework/tvb/tests/framework/adapters/uploaders/zip_surface_importer_test.py +++ b/tvb_framework/tvb/tests/framework/adapters/uploaders/zip_surface_importer_test.py @@ -30,7 +30,7 @@ import os -import tvb_data.surfaceData +from tvb.datasets import TVBZenodoDataset from tvb.datatypes.surfaces import SurfaceTypesEnum from tvb.tests.framework.core.base_testcase import BaseTestCase from tvb.tests.framework.core.factory import TestFactory @@ -41,7 +41,7 @@ class TestZIPSurfaceImporter(BaseTestCase): Unit-tests for Zip Surface importer. """ - surf_skull = os.path.join(os.path.dirname(tvb_data.surfaceData.__file__), 'outer_skull_4096.zip') + surf_skull = TVBZenodoDataset().fetch_data('outer_skull_4096.zip') def setup_method(self): self.test_user = TestFactory.create_user('Zip_Surface_User') diff --git a/tvb_framework/tvb/tests/framework/adapters/visualizers/brainviewer_test.py b/tvb_framework/tvb/tests/framework/adapters/visualizers/brainviewer_test.py index 92db5dd4c6..c697ac2011 100644 --- a/tvb_framework/tvb/tests/framework/adapters/visualizers/brainviewer_test.py +++ b/tvb_framework/tvb/tests/framework/adapters/visualizers/brainviewer_test.py @@ -29,8 +29,7 @@ """ import os -import tvb_data.surfaceData -import tvb_data.regionMapping +from tvb.datasets import TVBZenodoDataset from tvb.core.neocom import h5 from tvb.tests.framework.core.base_testcase import TransactionalTestCase @@ -50,8 +49,9 @@ class TestBrainViewer(TransactionalTestCase): EXPECTED_EXTRA_KEYS = ['urlMeasurePointsLabels', 'urlMeasurePoints', 'pageSize', 'shellObject', 'extended_view', 'legendLabels', 'labelsStateVar', 'labelsModes', 'title'] - cortex = os.path.join(os.path.dirname(tvb_data.surfaceData.__file__), 'cortex_16384.zip') - region_mapping_path = os.path.join(os.path.dirname(tvb_data.regionMapping.__file__), 'regionMapping_16k_76.txt') + dataset = TVBZenodoDataset() + cortex = dataset.fetch_data('cortex_16384.zip') + region_mapping_path = dataset.fetch_data('regionMapping_16k_76.txt') def transactional_setup_method(self): """ @@ -62,7 +62,7 @@ def transactional_setup_method(self): self.test_user = TestFactory.create_user('Brain_Viewer_User') self.test_project = TestFactory.create_project(self.test_user, 'Brain_Viewer_Project') - zip_path = os.path.join(os.path.dirname(tvb_data.__file__), 'connectivity', 'connectivity_96.zip') + zip_path = self.dataset.fetch_data('connectivity_96.zip') TestFactory.import_zip_connectivity(self.test_user, self.test_project, zip_path, "John") connectivity_idx = TestFactory.get_entity(self.test_project, ConnectivityIndex) assert connectivity_idx is not None diff --git a/tvb_framework/tvb/tests/framework/adapters/visualizers/connectivityviewer_test.py b/tvb_framework/tvb/tests/framework/adapters/visualizers/connectivityviewer_test.py index 1db6322f11..502289adf5 100644 --- a/tvb_framework/tvb/tests/framework/adapters/visualizers/connectivityviewer_test.py +++ b/tvb_framework/tvb/tests/framework/adapters/visualizers/connectivityviewer_test.py @@ -28,7 +28,7 @@ """ import os -import tvb_data +from tvb.datasets import TVBZenodoDataset from tvb.adapters.datatypes.db.connectivity import ConnectivityIndex from tvb.tests.framework.core.base_testcase import TransactionalTestCase @@ -51,7 +51,7 @@ def transactional_setup_method(self): self.test_user = TestFactory.create_user("UserCVV") self.test_project = TestFactory.create_project(self.test_user) - zip_path = os.path.join(os.path.dirname(tvb_data.__file__), 'connectivity', 'connectivity_66.zip') + zip_path = TVBZenodoDataset().fetch_data('connectivity_66.zip') TestFactory.import_zip_connectivity(self.test_user, self.test_project, zip_path) self.connectivity_index = TestFactory.get_entity(self.test_project, ConnectivityIndex) assert self.connectivity_index is not None diff --git a/tvb_framework/tvb/tests/framework/adapters/visualizers/sensorsviewer_test.py b/tvb_framework/tvb/tests/framework/adapters/visualizers/sensorsviewer_test.py index f6f53d93bc..39bff3dac1 100644 --- a/tvb_framework/tvb/tests/framework/adapters/visualizers/sensorsviewer_test.py +++ b/tvb_framework/tvb/tests/framework/adapters/visualizers/sensorsviewer_test.py @@ -29,8 +29,7 @@ """ import os -import tvb_data.obj -import tvb_data.sensors +from tvb.datasets import TVBZenodoDataset from tvb.adapters.datatypes.db.sensors import SensorsIndex from tvb.adapters.datatypes.db.surface import SurfaceIndex from tvb.adapters.uploaders.sensors_importer import SensorsImporterModel @@ -46,6 +45,7 @@ class TestSensorViewers(TransactionalTestCase): """ Unit-tests for Sensors viewers. """ + dataset = TVBZenodoDataset() EXPECTED_KEYS_INTERNAL = {'urlMeasurePoints': None, 'urlMeasurePointsLabels': None, 'noOfMeasurePoints': 103, 'minMeasure': 0, 'maxMeasure': 103, 'urlMeasure': None, 'shellObject': None} @@ -71,7 +71,7 @@ def test_launch_eeg(self): Check that all required keys are present in output from EegSensorViewer launch. """ # Import Sensors - zip_path = os.path.join(os.path.dirname(tvb_data.sensors.__file__), 'eeg_unitvector_62.txt.bz2') + zip_path = self.dataset.fetch_data('eeg_unitvector_62.txt.bz2') TestFactory.import_sensors(self.test_user, self.test_project, zip_path, SensorTypesEnum.TYPE_EEG) field = FilterChain.datatype + '.sensors_type' @@ -79,7 +79,7 @@ def test_launch_eeg(self): sensors_index = TestFactory.get_entity(self.test_project, SensorsIndex, filters) # Import EEGCap - cap_path = os.path.join(os.path.dirname(tvb_data.obj.__file__), 'eeg_cap.obj') + cap_path = self.dataset.fetch_data('eeg_cap.obj') TestFactory.import_surface_obj(self.test_user, self.test_project, cap_path, SurfaceTypesEnum.EEG_CAP_SURFACE) field = FilterChain.datatype + '.surface_type' filters = FilterChain('', [field], [SurfaceTypesEnum.EEG_CAP_SURFACE.value], ['==']) @@ -106,7 +106,7 @@ def test_launch_meg(self): Check that all required keys are present in output from MEGSensorViewer launch. """ - zip_path = os.path.join(os.path.dirname(tvb_data.sensors.__file__), 'meg_151.txt.bz2') + zip_path = self.dataset.fetch_data('meg_151.txt.bz2') TestFactory.import_sensors(self.test_user, self.test_project, zip_path, SensorTypesEnum.TYPE_MEG) @@ -126,7 +126,7 @@ def test_launch_internal(self): """ Check that all required keys are present in output from InternalSensorViewer launch. """ - zip_path = os.path.join(os.path.dirname(tvb_data.sensors.__file__), 'seeg_39.txt.bz2') + zip_path = self.dataset.fetch_data('seeg_39.txt.bz2') sensors_index = TestFactory.import_sensors(self.test_user, self.test_project, zip_path, SensorTypesEnum.TYPE_INTERNAL) viewer = SensorsViewer() diff --git a/tvb_framework/tvb/tests/framework/adapters/visualizers/surfaceviewer_test.py b/tvb_framework/tvb/tests/framework/adapters/visualizers/surfaceviewer_test.py index c6715dbbc7..c1ff9220bf 100644 --- a/tvb_framework/tvb/tests/framework/adapters/visualizers/surfaceviewer_test.py +++ b/tvb_framework/tvb/tests/framework/adapters/visualizers/surfaceviewer_test.py @@ -29,8 +29,7 @@ """ import os -import tvb_data.surfaceData -import tvb_data.regionMapping as demo_data +from tvb.datasets import TVBZenodoDataset from uuid import UUID from tvb.adapters.datatypes.db.connectivity import ConnectivityIndex @@ -56,20 +55,21 @@ def transactional_setup_method(self): creates a test user, a test project, a connectivity and a surface; imports a CFF data-set """ + dataset = TVBZenodoDataset() test_user = TestFactory.create_user('Surface_Viewer_User') self.test_project = TestFactory.create_project(test_user, 'Surface_Viewer_Project') - surf_skull = os.path.join(os.path.dirname(tvb_data.surfaceData.__file__), 'cortex_16384.zip') + surf_skull = dataset.fetch_data('cortex_16384.zip') self.surface = TestFactory.import_surface_zip(test_user, self.test_project, surf_skull, SurfaceTypesEnum.CORTICAL_SURFACE) assert self.surface is not None - zip_path = os.path.join(os.path.dirname(tvb_data.__file__), 'connectivity', 'connectivity_76.zip') + zip_path = dataset.fetch_data('connectivity_76.zip') TestFactory.import_zip_connectivity(test_user, self.test_project, zip_path, "John") connectivity_index = TestFactory.get_entity(self.test_project, ConnectivityIndex) assert connectivity_index is not None - TXT_FILE = os.path.join(os.path.dirname(demo_data.__file__), 'regionMapping_16k_76.txt') + TXT_FILE = dataset.fetch_data('regionMapping_16k_76.txt') self.region_mapping = TestFactory.import_region_mapping(test_user, self.test_project, TXT_FILE, self.surface.gid, connectivity_index.gid) assert self.region_mapping is not None diff --git a/tvb_framework/tvb/tests/framework/core/factory.py b/tvb_framework/tvb/tests/framework/core/factory.py index f41d970496..0fa6e7a171 100644 --- a/tvb_framework/tvb/tests/framework/core/factory.py +++ b/tvb_framework/tvb/tests/framework/core/factory.py @@ -37,8 +37,7 @@ import os import random import uuid -import tvb_data - +from tvb.datasets import TVBZenodoDataset from tvb.adapters.datatypes.db.connectivity import ConnectivityIndex from tvb.adapters.datatypes.db.local_connectivity import LocalConnectivityIndex from tvb.adapters.datatypes.db.projections import ProjectionMatrixIndex @@ -77,7 +76,7 @@ class TestFactory(object): """ Expose mostly static methods for creating different entities used in tests. """ - + @staticmethod def get_entity(project, expected_data, filters=None): """ @@ -220,7 +219,7 @@ def import_default_project(admin_user=None): if not admin_user: admin_user = TestFactory.create_user() - project_path = os.path.join(os.path.dirname(tvb_data.__file__), 'Default_Project.zip') + project_path = TVBZenodoDataset().fetch_data('Default_Project.zip') import_service = ImportService() import_service.import_project_structure(project_path, admin_user.id) return import_service.created_projects[0] @@ -312,7 +311,7 @@ def import_zip_connectivity(user, project, zip_path=None, subject=DataTypeMetaDa same_process=True): if zip_path is None: - zip_path = os.path.join(os.path.dirname(tvb_data.__file__), 'connectivity', 'connectivity_76.zip') + zip_path = TVBZenodoDataset().fetch_data('connectivity_76.zip') count = dao.count_datatypes(project.id, ConnectivityIndex) view_model = ZIPConnectivityImporterModel() diff --git a/tvb_framework/tvb/tests/framework/core/neotraits/forms_test.py b/tvb_framework/tvb/tests/framework/core/neotraits/forms_test.py index fb49b1ae1c..a9bef543e3 100644 --- a/tvb_framework/tvb/tests/framework/core/neotraits/forms_test.py +++ b/tvb_framework/tvb/tests/framework/core/neotraits/forms_test.py @@ -27,7 +27,7 @@ import uuid import numpy import pytest -import tvb_data +from tvb.datasets import TVBZenodoDataset from tvb.basic.neotraits.api import Attr, Float, Int, NArray, List from tvb.core.entities.file.simulator.view_model import SimulatorAdapterModel @@ -54,7 +54,7 @@ def teardown_method(self): self.clean_database() def test_upload_field(self): - connectivity_file = os.path.join(os.path.dirname(tvb_data.__file__), 'connectivity', 'connectivity_96.zip') + connectivity_file = TVBZenodoDataset().fetch_data('connectivity_96.zip') data_file = Str('Test Upload Field') required_type = '.zip' upload_field = TraitUploadField(data_file, required_type, self.name) diff --git a/tvb_framework/tvb/tests/framework/core/services/import_service_test.py b/tvb_framework/tvb/tests/framework/core/services/import_service_test.py index 54a904c6df..4315b787ee 100644 --- a/tvb_framework/tvb/tests/framework/core/services/import_service_test.py +++ b/tvb_framework/tvb/tests/framework/core/services/import_service_test.py @@ -31,7 +31,7 @@ import os import pytest -import tvb_data +from tvb.datasets import TVBZenodoDataset from PIL import Image from time import sleep from tvb.adapters.datatypes.db.mapped_value import ValueWrapperIndex @@ -58,7 +58,7 @@ class TestImportService(BaseTestCase): """ This class contains tests for the tvb.core.services.import_service module. """ - + dataset = TVBZenodoDataset() def setup_method(self): """ Reset the database before each test. @@ -88,7 +88,7 @@ def test_import_export(self, user_factory, project_factory, value_wrapper_factor """ test_user = user_factory() test_project = project_factory(test_user, "TestImportExport", "test_desc") - zip_path = os.path.join(os.path.dirname(tvb_data.__file__), 'connectivity', 'connectivity_66.zip') + zip_path = self.dataset.fetch_data('connectivity_66.zip') TestFactory.import_zip_connectivity(test_user, test_project, zip_path) value_wrapper = value_wrapper_factory(test_user, test_project) ProjectService.set_datatype_visibility(value_wrapper.gid, False) @@ -138,7 +138,7 @@ def test_import_export_existing(self, user_factory, project_factory): """ test_user = user_factory() test_project = project_factory(test_user, "TestImportExport2") - zip_path = os.path.join(os.path.dirname(tvb_data.__file__), 'connectivity', 'connectivity_66.zip') + zip_path = self.dataset.fetch_data('connectivity_66.zip') TestFactory.import_zip_connectivity(test_user, test_project, zip_path) count_operations = dao.get_filtered_operations(test_project.id, None, is_count=True) @@ -182,7 +182,7 @@ def test_export_import_figures(self, user_factory, project_factory): # Prepare data user = user_factory() project = project_factory(user, "TestImportExportFigures") - zip_path = os.path.join(os.path.dirname(tvb_data.__file__), 'connectivity', 'paupau.zip') + zip_path = self.dataset.fetch_data('paupau.zip') TestFactory.import_zip_connectivity(user, project, zip_path) figure_service = FigureService() diff --git a/tvb_framework/tvb/tests/framework/core/services/links_test.py b/tvb_framework/tvb/tests/framework/core/services/links_test.py index 5a08dffdb5..5c317f2520 100644 --- a/tvb_framework/tvb/tests/framework/core/services/links_test.py +++ b/tvb_framework/tvb/tests/framework/core/services/links_test.py @@ -32,7 +32,7 @@ """ import pytest import os -import tvb_data +from tvb.datasets import TVBZenodoDataset from tvb.adapters.datatypes.db.connectivity import ConnectivityIndex from tvb.adapters.datatypes.db.sensors import SensorsIndex from tvb.adapters.exporters.export_manager import ExportManager @@ -49,6 +49,8 @@ class _BaseLinksTest(TransactionalTestCase): + dataset = TVBZenodoDataset() + @pytest.fixture() def initialize_two_projects(self, dummy_datatype_index_factory, project_factory, user_factory): """ @@ -66,9 +68,9 @@ def initialize_two_projects(self, dummy_datatype_index_factory, project_factory, src_user = user_factory(username="Links Test") self.src_usr_id = src_user.id self.src_project = project_factory(src_user, "Src_Project") - zip_path = os.path.join(os.path.dirname(tvb_data.__file__), 'connectivity', 'paupau.zip') + zip_path = self.dataset.fetch_data("paupau.zip") self.red_datatype = TestFactory.import_zip_connectivity(src_user, self.src_project, zip_path, "John") - zip_path = os.path.join(os.path.dirname(tvb_data.__file__), 'sensors', 'eeg_unitvector_62.txt.bz2') + zip_path = self.dataset.fetch_data('eeg_unitvector_62.txt.bz2') self.blue_datatype = TestFactory.import_sensors(src_user, self.src_project, zip_path, SensorTypesEnum.TYPE_EEG) assert 1 == self.red_datatypes_in(self.src_project.id) @@ -207,7 +209,7 @@ def build(): Project dest will have the derived VW and links """ # add a connectivity to src project and link it to dest project - zip_path = os.path.join(os.path.dirname(tvb_data.__file__), 'connectivity', 'connectivity_96.zip') + zip_path = self.dataset.fetch_data('connectivity_96.zip') conn = TestFactory.import_zip_connectivity(self.dst_user, self.src_project, zip_path, "John") self.algorithm_service.create_link(conn.id, self.dest_project.id) diff --git a/tvb_framework/tvb/tests/framework/core/services/project_service_test.py b/tvb_framework/tvb/tests/framework/core/services/project_service_test.py index 7b876beb64..324c91beb0 100644 --- a/tvb_framework/tvb/tests/framework/core/services/project_service_test.py +++ b/tvb_framework/tvb/tests/framework/core/services/project_service_test.py @@ -33,7 +33,7 @@ import pytest import sqlalchemy -import tvb_data +from tvb.datasets import TVBZenodoDataset from tvb.basic.profile import TvbProfile from tvb.core.entities.model import model_datatype, model_project, model_operation from tvb.core.entities.storage import dao @@ -331,11 +331,12 @@ def test_empty_project_has_zero_disk_size(self): def test_project_disk_size(self): project1 = TestFactory.create_project(self.test_user, 'test_proj1') - zip_path = os.path.join(os.path.dirname(tvb_data.__file__), 'connectivity', 'connectivity_66.zip') + dataset = TVBZenodoDataset() + zip_path = dataset.fetch_data('connectivity_66.zip') TestFactory.import_zip_connectivity(self.test_user, project1, zip_path, 'testSubject') project2 = TestFactory.create_project(self.test_user, 'test_proj2') - zip_path = os.path.join(os.path.dirname(tvb_data.__file__), 'connectivity', 'connectivity_76.zip') + zip_path = dataset.fetch_data('connectivity_76.zip') TestFactory.import_zip_connectivity(self.test_user, project2, zip_path, 'testSubject') projects = self.project_service.retrieve_projects_for_user(self.test_user.id)[0] diff --git a/tvb_framework/tvb/tests/framework/core/services/serialization_manager_test.py b/tvb_framework/tvb/tests/framework/core/services/serialization_manager_test.py index a7c48010b5..84611b3637 100644 --- a/tvb_framework/tvb/tests/framework/core/services/serialization_manager_test.py +++ b/tvb_framework/tvb/tests/framework/core/services/serialization_manager_test.py @@ -29,7 +29,7 @@ """ from os import path -import tvb_data +from tvb.datasets import TVBZenodoDataset from tvb.adapters.datatypes.db.connectivity import ConnectivityIndex from tvb.adapters.forms.model_forms import ModelsEnum @@ -45,7 +45,7 @@ class TestSerializationManager(TransactionalTestCase): def transactional_setup_method(self): self.test_user = TestFactory.create_user(username="test_user") self.test_project = TestFactory.create_project(self.test_user, "Test") - zip_path = path.join(path.dirname(tvb_data.__file__), 'connectivity', 'connectivity_66.zip') + zip_path = TVBZenodoDataset().fetch_data('connectivity_66.zip') TestFactory.import_zip_connectivity(self.test_user, self.test_project, zip_path, "John") self.connectivity = TestFactory.get_entity(self.test_project, ConnectivityIndex) diff --git a/tvb_framework/tvb/tests/framework/interfaces/rest/datatype_resource_test.py b/tvb_framework/tvb/tests/framework/interfaces/rest/datatype_resource_test.py index e811994a36..5b1563c494 100644 --- a/tvb_framework/tvb/tests/framework/interfaces/rest/datatype_resource_test.py +++ b/tvb_framework/tvb/tests/framework/interfaces/rest/datatype_resource_test.py @@ -27,7 +27,7 @@ import os import flask import pytest -import tvb_data +from tvb.datasets import TVBZenodoDataset from tvb.adapters.datatypes.db.connectivity import ConnectivityIndex from tvb.interfaces.rest.commons.exceptions import InvalidIdentifierException from tvb.interfaces.rest.server.resources.datatype.datatype_resource import RetrieveDatatypeResource @@ -39,6 +39,8 @@ class TestDatatypeResource(RestResourceTest): + dataset = TVBZenodoDataset() + def transactional_setup_method(self): self.test_user = TestFactory.create_user('Rest_User') self.test_project = TestFactory.create_project(self.test_user, 'Rest_Project', users=[self.test_user.id]) @@ -53,7 +55,7 @@ def test_server_retrieve_datatype_inexistent_gid(self, mocker): def test_server_retrieve_datatype(self, mocker): self._mock_user(mocker) - zip_path = os.path.join(os.path.dirname(tvb_data.__file__), 'connectivity', 'connectivity_96.zip') + zip_path = self.dataset.fetch_data('connectivity_96.zip') TestFactory.import_zip_connectivity(self.test_user, self.test_project, zip_path) datatypes_in_project = self.get_data_in_project_resource.get(project_gid=self.test_project.gid) @@ -79,7 +81,7 @@ def send_file_dummy(path, as_attachment, attachment_filename): def test_server_get_operations_for_datatype(self, mocker): self._mock_user(mocker) - zip_path = os.path.join(os.path.dirname(tvb_data.__file__), 'connectivity', 'connectivity_96.zip') + zip_path = self.dataset.fetch_data('connectivity_96.zip') TestFactory.import_zip_connectivity(self.test_user, self.test_project, zip_path) datatypes_in_project = self.get_data_in_project_resource.get(project_gid=self.test_project.gid) diff --git a/tvb_framework/tvb/tests/framework/interfaces/rest/operation_resource_test.py b/tvb_framework/tvb/tests/framework/interfaces/rest/operation_resource_test.py index 4600fc6686..5950e51a0b 100644 --- a/tvb_framework/tvb/tests/framework/interfaces/rest/operation_resource_test.py +++ b/tvb_framework/tvb/tests/framework/interfaces/rest/operation_resource_test.py @@ -29,8 +29,7 @@ from uuid import UUID import flask import pytest -import tvb_data - +from tvb.datasets import TVBZenodoDataset from tvb.adapters.analyzers.fourier_adapter import FFTAdapterModel from tvb.basic.exceptions import TVBException from tvb.core.neocom import h5 @@ -49,6 +48,8 @@ class TestOperationResource(RestResourceTest): + dataset = TVBZenodoDataset() + def transactional_setup_method(self): self.test_user = TestFactory.create_user('Rest_User') self.test_project = TestFactory.create_project(self.test_user, 'Rest_Project', users=[self.test_user.id]) @@ -65,7 +66,7 @@ def test_server_get_operation_status_inexistent_gid(self, mocker): def test_server_get_operation_status(self, mocker): self._mock_user(mocker) - zip_path = os.path.join(os.path.dirname(tvb_data.__file__), 'connectivity', 'connectivity_96.zip') + zip_path = self.dataset.fetch_data('connectivity_96.zip') TestFactory.import_zip_connectivity(self.test_user, self.test_project, zip_path) request_mock = mocker.patch.object(flask, 'request', spec={}) @@ -84,7 +85,7 @@ def test_server_get_operation_results_inexistent_gid(self, mocker): def test_server_get_operation_results(self, mocker): self._mock_user(mocker) - zip_path = os.path.join(os.path.dirname(tvb_data.__file__), 'connectivity', 'connectivity_96.zip') + zip_path = self.dataset.fetch_data('connectivity_96.zip') TestFactory.import_zip_connectivity(self.test_user, self.test_project, zip_path) request_mock = mocker.patch.object(flask, 'request', spec={}) @@ -98,7 +99,10 @@ def test_server_get_operation_results(self, mocker): def test_server_get_operation_results_failed_operation(self, mocker): self._mock_user(mocker) - zip_path = os.path.join(os.path.dirname(tvb_data.__file__), 'connectivity', 'connectivity_90.zip') + with pytest.raises(KeyError): + zip_path = self.dataset.fetch_data('connectivity_90.zip') + zip_path = self.dataset.fetch_data('connectivity_96.zip') + zip_path = zip_path.replace("connectivity_96", "connectivity_90") with pytest.raises(TVBException): TestFactory.import_zip_connectivity(self.test_user, self.test_project, zip_path) diff --git a/tvb_framework/tvb/tests/framework/interfaces/rest/project_resource_test.py b/tvb_framework/tvb/tests/framework/interfaces/rest/project_resource_test.py index a1320bc645..b6da293795 100644 --- a/tvb_framework/tvb/tests/framework/interfaces/rest/project_resource_test.py +++ b/tvb_framework/tvb/tests/framework/interfaces/rest/project_resource_test.py @@ -27,7 +27,7 @@ import os import flask import pytest -import tvb_data +from tvb.datasets import TVBZenodoDataset from tvb.interfaces.rest.commons.exceptions import InvalidIdentifierException from tvb.interfaces.rest.commons.strings import Strings @@ -45,7 +45,8 @@ def transactional_setup_method(self): self.test_user = TestFactory.create_user('Rest_User') self.test_project_without_data = TestFactory.create_project(self.test_user, 'Rest_Project', users=[self.test_user.id]) self.test_project_with_data = TestFactory.create_project(self.test_user, 'Rest_Project2', users=[self.test_user.id]) - zip_path = os.path.join(os.path.dirname(tvb_data.__file__), 'connectivity', 'connectivity_96.zip') + dataset = TVBZenodoDataset() + zip_path = dataset.fetch_data('connectivity_96.zip') TestFactory.import_zip_connectivity(self.test_user, self.test_project_with_data, zip_path) def test_server_get_data_in_project_inexistent_gid(self, mocker): diff --git a/tvb_framework/tvb/tests/framework/interfaces/web/controllers/simulator_controller_test.py b/tvb_framework/tvb/tests/framework/interfaces/web/controllers/simulator_controller_test.py index 1443c9db3d..1158476fd6 100644 --- a/tvb_framework/tvb/tests/framework/interfaces/web/controllers/simulator_controller_test.py +++ b/tvb_framework/tvb/tests/framework/interfaces/web/controllers/simulator_controller_test.py @@ -25,11 +25,8 @@ # import numpy -import tvb_data.connectivity -import tvb_data.surfaceData -import tvb_data.sensors -import tvb_data.regionMapping -import tvb_data.projectionMatrix +from tvb.datasets import TVBZenodoDataset + from os import path from uuid import UUID from unittest.mock import patch @@ -63,7 +60,7 @@ class TestSimulationController(BaseTransactionalControllerTest): - + dataset = TVBZenodoDataset() def transactional_setup_method(self): self.simulator_controller = SimulatorController() self.test_user = TestFactory.create_user('SimulationController_User') @@ -90,7 +87,7 @@ def test_index(self): assert not result_dict['errors'], 'Some errors were encountered!' def test_set_connectivity(self): - zip_path = path.join(path.dirname(tvb_data.connectivity.__file__), 'connectivity_66.zip') + zip_path = self.dataset.fetch_data('connectivity_66.zip') connectivity = TestFactory.import_zip_connectivity(self.test_user, self.test_project, zip_path, "John") self.sess_mock['connectivity'] = connectivity.gid @@ -126,7 +123,7 @@ def test_set_coupling_params(self): assert self.session_stored_simulator.coupling.b[0] == [0.0], "b value was not set correctly." def test_set_surface(self): - zip_path = path.join(path.dirname(tvb_data.surfaceData.__file__), 'cortex_16384.zip') + zip_path = self.dataset.fetch_data('cortex_16384.zip') TestFactory.import_surface_zip(self.test_user, self.test_project, zip_path, SurfaceTypesEnum.CORTICAL_SURFACE, True) surface = TestFactory.get_entity(self.test_project, SurfaceIndex) @@ -147,14 +144,14 @@ def test_set_surface_none(self): assert self.session_stored_simulator.surface is None, "Surface should not be set." def test_set_cortex_without_local_connectivity(self): - zip_path = path.join(path.dirname(tvb_data.connectivity.__file__), 'connectivity_76.zip') + zip_path = self.dataset.fetch_data('connectivity_76.zip') connectivity = TestFactory.import_zip_connectivity(self.test_user, self.test_project, zip_path, "John") - zip_path = path.join(path.dirname(tvb_data.surfaceData.__file__), 'cortex_16384.zip') + zip_path = self.dataset.fetch_data('cortex_16384.zip') surface = TestFactory.import_surface_zip(self.test_user, self.test_project, zip_path, SurfaceTypesEnum.CORTICAL_SURFACE, True) - text_file = path.join(path.dirname(tvb_data.regionMapping.__file__), 'regionMapping_16k_76.txt') + text_file = self.dataset.fetch_data('regionMapping_16k_76.txt') region_mapping = TestFactory.import_region_mapping(self.test_user, self.test_project, text_file, surface.gid, connectivity.gid) @@ -176,14 +173,14 @@ def test_set_cortex_without_local_connectivity(self): "coupling_strength was not set correctly." def test_set_cortex_with_local_connectivity(self, local_connectivity_index_factory): - zip_path = path.join(path.dirname(tvb_data.connectivity.__file__), 'connectivity_76.zip') + zip_path = self.dataset.fetch_data('connectivity_76.zip') connectivity = TestFactory.import_zip_connectivity(self.test_user, self.test_project, zip_path, "John") - zip_path = path.join(path.dirname(tvb_data.surfaceData.__file__), 'cortex_16384.zip') + zip_path = self.dataset.fetch_data('cortex_16384.zip') surface = TestFactory.import_surface_zip(self.test_user, self.test_project, zip_path, SurfaceTypesEnum.CORTICAL_SURFACE, True) - text_file = path.join(path.dirname(tvb_data.regionMapping.__file__), 'regionMapping_16k_76.txt') + text_file = self.dataset.fetch_data('regionMapping_16k_76.txt' ) region_mapping = TestFactory.import_region_mapping(self.test_user, self.test_project, text_file, surface.gid, connectivity.gid) @@ -214,7 +211,7 @@ def test_set_stimulus_none(self): assert self.session_stored_simulator.stimulus is None, "Stimulus should not be set." def test_set_stimulus(self): - zip_path = path.join(path.dirname(tvb_data.connectivity.__file__), 'connectivity_66.zip') + zip_path = self.dataset.fetch_data('connectivity_66.zip') connectivity_index = TestFactory.import_zip_connectivity(self.test_user, self.test_project, zip_path) weight_array = numpy.zeros(connectivity_index.number_of_regions) @@ -443,29 +440,28 @@ def test_set_monitor_params(self): assert not rendering_rules['renderer'].include_next_button, 'Next button should not be displayed!' def set_region_mapping(self): - zip_path = path.join(path.dirname(tvb_data.connectivity.__file__), 'connectivity_76.zip') + zip_path = self.dataset.fetch_data('connectivity_76.zip') connectivity = TestFactory.import_zip_connectivity(self.test_user, self.test_project, zip_path, "John") - zip_path = path.join(path.dirname(tvb_data.surfaceData.__file__), 'cortex_16384.zip') + zip_path = self.dataset.fetch_data('cortex_16384.zip') surface = TestFactory.import_surface_zip(self.test_user, self.test_project, zip_path, SurfaceTypesEnum.CORTICAL_SURFACE, True) - text_file = path.join(path.dirname(tvb_data.regionMapping.__file__), 'regionMapping_16k_76.txt') + text_file = self.dataset.fetch_data('regionMapping_16k_76.txt') region_mapping = TestFactory.import_region_mapping(self.test_user, self.test_project, text_file, surface.gid, connectivity.gid) return region_mapping def set_eeg(self): - eeg_sensors_file = path.join(path.dirname(tvb_data.sensors.__file__), 'eeg_unitvector_62.txt') + eeg_sensors_file = self.dataset.fetch_data('eeg_unitvector_62.txt.bz2') eeg_sensors = TestFactory.import_sensors(self.test_user, self.test_project, eeg_sensors_file, SensorTypesEnum.TYPE_EEG) - surface_file = path.join(path.dirname(tvb_data.surfaceData.__file__), 'cortex_16384.zip') + surface_file = self.dataset.fetch_data('cortex_16384.zip') surface = TestFactory.import_surface_zip(self.test_user, self.test_project, surface_file, SurfaceTypesEnum.CORTICAL_SURFACE, True) - eeg_projection_file = path.join(path.dirname(tvb_data.projectionMatrix.__file__), - 'projection_eeg_62_surface_16k.mat') + eeg_projection_file = self.dataset.fetch_data('projection_eeg_62_surface_16k.mat') eeg_projection = TestFactory.import_projection_matrix(self.test_user, self.test_project, eeg_projection_file, eeg_sensors.gid, surface.gid) return eeg_sensors, eeg_projection @@ -502,16 +498,15 @@ def test_set_eeg_monitor_params(self): "Projection wasn't stored correctly." def set_meg(self): - meg_sensors_file = path.join(path.dirname(tvb_data.sensors.__file__), 'meg_brainstorm_276.txt') + meg_sensors_file = self.dataset.fetch_data('meg_brainstorm_276.txt') meg_sensors = TestFactory.import_sensors(self.test_user, self.test_project, meg_sensors_file, SensorTypesEnum.TYPE_MEG) - surface_file = path.join(path.dirname(tvb_data.surfaceData.__file__), 'cortex_16384.zip') + surface_file = self.dataset.fetch_data('cortex_16384.zip') surface = TestFactory.import_surface_zip(self.test_user, self.test_project, surface_file, SurfaceTypesEnum.CORTICAL_SURFACE, True) - meg_projection_file = path.join(path.dirname(tvb_data.projectionMatrix.__file__), - 'projection_meg_276_surface_16k.npy') + meg_projection_file = self.dataset.fetch_data('projection_meg_276_surface_16k.npy') meg_projection = TestFactory.import_projection_matrix(self.test_user, self.test_project, meg_projection_file, meg_sensors.gid, surface.gid) return meg_sensors, meg_projection @@ -549,16 +544,15 @@ def test_set_meg_monitor_params(self): "Projection wasn't stored correctly." def set_seeg(self): - seeg_sensors_file = path.join(path.dirname(tvb_data.sensors.__file__), 'seeg_588.txt') + seeg_sensors_file = self.dataset.fetch_data('seeg_588.txt') seeg_sensors = TestFactory.import_sensors(self.test_user, self.test_project, seeg_sensors_file, SensorTypesEnum.TYPE_INTERNAL) - surface_file = path.join(path.dirname(tvb_data.surfaceData.__file__), 'cortex_16384.zip') + surface_file = self.dataset.fetch_data('cortex_16384.zip') surface = TestFactory.import_surface_zip(self.test_user, self.test_project, surface_file, SurfaceTypesEnum.CORTICAL_SURFACE, True) - seeg_projection_file = path.join(path.dirname(tvb_data.projectionMatrix.__file__), - 'projection_seeg_588_surface_16k.npy') + seeg_projection_file = self.dataset.fetch_data('projection_seeg_588_surface_16k.npy') seeg_projection = TestFactory.import_projection_matrix(self.test_user, self.test_project, seeg_projection_file, seeg_sensors.gid, surface.gid) return seeg_sensors, seeg_projection @@ -697,7 +691,7 @@ def test_load_burst_history(self): assert len(burst_parameters['burst_list']) == 3, "The burst configurations where not stored." def test_reset_simulator_configuration(self): - zip_path = path.join(path.dirname(tvb_data.connectivity.__file__), 'connectivity_66.zip') + zip_path = self.dataset.fetch_data('connectivity_66.zip') connectivity = TestFactory.import_zip_connectivity(self.test_user, self.test_project, zip_path, "John") self.sess_mock['connectivity'] = connectivity.gid @@ -753,7 +747,7 @@ def test_rename_burst(self): assert dao.get_bursts_for_project(self.test_project.id)[0].name == new_name, "Name wasn't actually changed." def test_copy_simulator_configuration(self): - zip_path = path.join(path.dirname(tvb_data.connectivity.__file__), 'connectivity_66.zip') + zip_path = self.dataset.fetch_data('connectivity_66.zip') connectivity = TestFactory.import_zip_connectivity(self.test_user, self.test_project, zip_path, "John") op = TestFactory.create_operation(test_user=self.test_user, test_project=self.test_project) @@ -787,7 +781,7 @@ def test_copy_simulator_configuration(self): assert rendering_rules['renderer'].disable_fields, 'Fragments should be read-only!' def test_load_burst(self): - zip_path = path.join(path.dirname(tvb_data.connectivity.__file__), 'connectivity_66.zip') + zip_path = self.dataset.fetch_data('connectivity_66.zip') connectivity = TestFactory.import_zip_connectivity(self.test_user, self.test_project, zip_path, "John") op = TestFactory.create_operation(test_user=self.test_user, test_project=self.test_project) @@ -837,7 +831,7 @@ def test_launch_simulation(self): assert burst_config.status == 'running', 'Simulation launching has failed!' def test_launch_branch_simulation(self): - zip_path = path.join(path.dirname(tvb_data.connectivity.__file__), 'connectivity_66.zip') + zip_path = self.dataset.fetch_data('connectivity_66.zip') connectivity = TestFactory.import_zip_connectivity(self.test_user, self.test_project, zip_path, "John") self.sess_mock['input_simulation_name_id'] = 'HappySimulation' diff --git a/tvb_library/setup.py b/tvb_library/setup.py index 7e40129348..f5acab92d7 100644 --- a/tvb_library/setup.py +++ b/tvb_library/setup.py @@ -40,7 +40,7 @@ LIBRARY_TEAM = "Marmaduke Woodman, Jan Fousek, Stuart Knock, Paula Sanz Leon, Viktor Jirsa" LIBRARY_REQUIRED_PACKAGES = ["autopep8", "Deprecated", "docutils", "ipywidgets", "lxml", "mako>=1.1.4", "matplotlib", - "networkx", "numba", "numexpr", "numpy", "pylems", "scipy", "six"] + "networkx", "numba", "numexpr", "numpy", "pooch", "pylems", "scipy", "six"] LIBRARY_REQUIRED_EXTRA = ["h5py", "pytest", "pytest-benchmark", "pytest-xdist", "tvb-gdist", "tvb-data"] diff --git a/tvb_library/tvb/basic/config/profile_settings.py b/tvb_library/tvb/basic/config/profile_settings.py index 052ef6a7ab..880cb6303b 100644 --- a/tvb_library/tvb/basic/config/profile_settings.py +++ b/tvb_library/tvb/basic/config/profile_settings.py @@ -95,6 +95,10 @@ def __init__(self): # The number of logs in a message batch that are sent to the server self.ELASTICSEARCH_BUFFER_THRESHOLD = self.manager.get_attribute(stored.KEY_ELASTICSEARCH_BUFFER_THRESHOLD, 1000000, int) + # Directory where all the datasets will be extracted/unzipped. + self.DATASETS_FOLDER = os.path.join(self.TVB_STORAGE, "DATASETS") + + @property def BIN_FOLDER(self): """ diff --git a/tvb_library/tvb/datasets/__init__.py b/tvb_library/tvb/datasets/__init__.py new file mode 100644 index 0000000000..995b08e09f --- /dev/null +++ b/tvb_library/tvb/datasets/__init__.py @@ -0,0 +1,2 @@ +from .zenodo import Record, Zenodo +from .tvb_data import TVBZenodoDataset diff --git a/tvb_library/tvb/datasets/base.py b/tvb_library/tvb/datasets/base.py new file mode 100644 index 0000000000..ae030098f0 --- /dev/null +++ b/tvb_library/tvb/datasets/base.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- +# +# +# TheVirtualBrain-Scientific Package. This package holds all simulators, and +# analysers necessary to run brain-simulations. You can use it stand alone or +# in conjunction with TheVirtualBrain-Framework Package. See content of the +# documentation-folder for more details. See also http://www.thevirtualbrain.org +# +# (c) 2012-2023, Baycrest Centre for Geriatric Care ("Baycrest") and others +# +# This program is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software Foundation, +# either version 3 of the License, or (at your option) any later version. +# This program is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. See the GNU General Public License for more details. +# You should have received a copy of the GNU General Public License along with this +# program. If not, see . +# +# +# CITATION: +# When using The Virtual Brain for scientific publications, please cite it as explained here: +# https://www.thevirtualbrain.org/tvb/zwei/neuroscience-publications +# +# + +""" +.. moduleauthor:: Abhijit Deo +""" + + + +from tvb.basic.logger.builder import get_logger +from tvb.basic.profile import TvbProfile +from pathlib import Path +from zipfile import ZipFile + +class BaseDataset: + + def __init__(self, version : str , extract_dir : str =None) -> None: + + self.log = get_logger(self.__class__.__module__) + self.cached_files = None + self.version = version + + if (extract_dir==None): + extract_dir = TvbProfile.current.DATASETS_FOLDER + + self.extract_dir = Path(extract_dir).expanduser() + + + def fetch_data(self) : + raise NotImplementedError + def get_version(self) -> str: + return self.version + + def delete_data(self): + raise NotImplementedError + + def _download(self): + raise NotImplementedError diff --git a/tvb_library/tvb/datasets/tvb_data.py b/tvb_library/tvb/datasets/tvb_data.py new file mode 100644 index 0000000000..e749320cde --- /dev/null +++ b/tvb_library/tvb/datasets/tvb_data.py @@ -0,0 +1,224 @@ +# -*- coding: utf-8 -*- +# +# +# TheVirtualBrain-Scientific Package. This package holds all simulators, and +# analysers necessary to run brain-simulations. You can use it stand alone or +# in conjunction with TheVirtualBrain-Framework Package. See content of the +# documentation-folder for more details. See also http://www.thevirtualbrain.org +# +# (c) 2012-2023, Baycrest Centre for Geriatric Care ("Baycrest") and others +# +# This program is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software Foundation, +# either version 3 of the License, or (at your option) any later version. +# This program is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. See the GNU General Public License for more details. +# You should have received a copy of the GNU General Public License along with this +# program. If not, see . +# +# +# CITATION: +# When using The Virtual Brain for scientific publications, please cite it as explained here: +# https://www.thevirtualbrain.org/tvb/zwei/neuroscience-publications +# +# + +""" +.. moduleauthor:: Abhijit Deo +""" + +import requests +import json + +from pathlib import Path +from zipfile import ZipFile +import shutil + + +from .base import BaseDataset +from .zenodo import Record, BASE_URL, Zenodo + +class TVBZenodoDataset(BaseDataset): + + CONCEPTID = "3417206" + + def __init__(self, version : str= "2.7", extract_dir : str = None): + """ + Constructor for TVB_Data class + + parameters + ----------- + + version: str + - Version number of the dataset, Default value is 2.7 + + extract_dir: str + - path where you want to extract the archive. + - If `extract_dir` is None, Dataset is downloaded at location according to your profile settings. + + """ + super().__init__(version, extract_dir) + self.cached_dir = self.extract_dir / ".cache" + self.cached_file = self.cached_dir / "tvb_cached_responses.txt" + self.files_in_zip_dict = None + + if not self.cached_dir.is_dir(): + self.cached_dir.mkdir(parents=True) + + try: + self.recid = self._read_cached_response()[version]['conceptrecid'] + + except : + self.log.warning(f"Failed to read data from cached response.") + self.recid = Zenodo().get_versions_info(self.CONCEPTID)[version] + self._update_cached_response() + + self.rec = Record(self._read_cached_response()[self.version]) + + def fetch_data(self, file_name:str)->str: + """ + Function to fetch the file having `file_name` as name of the file. The function checks if the dataset is downloaded or not. If not, function downloads the dataset and then extracts/unzip the file. + + parameters: + ----------- + file_name: str + - Name of the file from the downloaded zip file to fetch. Also accepts relative path of the file with respect to tvb_data.zip. This is useful when having multiple files with same name. + + returns: str + path of the extracted/Unzipped file. + """ + if Path(file_name).is_absolute(): + self.log.warning("Given `file_name` is an absolute path. No operations are done. The `file_name` is returned as it is") + return file_name + + extract_dir = self.extract_dir + + try: + file_path = self.rec.file_loc['tvb_data.zip'] + except: + self._download(path = self.cached_dir, fname=f"tvb_data_{self.version}.zip") + file_path = self.rec.file_loc['tvb_data.zip'] + + if self.files_in_zip_dict == None: + self.files_in_zip_dict = self._read_zipfile_structure(file_path=file_path) + + file_name = file_name.strip() + + + if file_name.startswith("tvb_data"): + if file_name in self.files_in_zip_dict[str(Path(file_name).name)] : + ZipFile(file_path).extract(file_name, path=extract_dir) + + if extract_dir.is_absolute(): + return str(extract_dir / file_name) + return str(Path.cwd()/ extract_dir / file_name) + else: + self.log.error("file_name not found, please mention correct relative file path") + + elif len(self.files_in_zip_dict[file_name]) == 1: + ZipFile(file_path).extract(self.files_in_zip_dict[file_name][0], path=extract_dir) + + if extract_dir.is_absolute(): + return str(extract_dir / self.files_in_zip_dict[file_name][0]) + return str(Path.cwd()/ extract_dir / self.files_in_zip_dict[file_name][0]) + + + elif len(self.files_in_zip_dict[file_name]) > 1: + + self.log.error(f"""There are more than 1 files with same names in the zip file. Please mention relative path of the file with respect to the tvb_data.zip. + file_name should be one of the following paths: {self.files_in_zip_dict[file_name]}""") + raise NameError(f"file name should be one of the {self.files_in_zip_dict[file_name]}, but got {file_name}") + + def describe(self)-> str: + """ + Returns the project description mentioned on the zenodo website. + """ + return self.rec.describe() + + def get_recordid(self) -> str: + """ + returns record id of the dataset + """ + return self.recid + + + def delete_data(self)->None: + """ + Deletes the `tvb_data` folder in the `self.extract_dir` directory. + """ + _dir = self.extract_dir / "tvb_data" + shutil.rmtree(_dir) + self.log.info(f"deleting {self.extract_dir/'tvb_data'} directory.") + + def _download(self, path=None, fname=None): + """ + Downloads the dataset to `path` + parameters + ----------- + path: + - path where you want to download the Dataset. + - If `path` is None, Dataset is downloaded at location according to your profile settings. + fname: + - The name that will be used to save the file. Should NOT include the full the path, just the file name (it will be appended to path). + - If fname is None, file will be saved with a unique name that contains hash of the file and the last part of the url from where the file would be fetched. + """ + if path == None: + path = self.cached_dir + self.rec.download(path, fname) + + def _update_cached_response(self): + """ + Gets responses from zenodo server and saves them to a cache file. + """ + + file_dir = self.cached_file + + responses = {} + + url = f"{BASE_URL}records?q=conceptrecid:{self.CONCEPTID}&all_versions=true" + + for hit in requests.get(url).json()['hits']['hits']: + version = hit['metadata']['version'] + response = hit + + responses[version] = response + + Path(file_dir).touch(exist_ok=True) + + with open(file_dir, "w") as fp: + json.dump(responses, fp) + fp.close() + self.log.warning("Updated the cache response file") + return + + def _read_cached_response(self): + """ + Reads responses from the cache file. + """ + + file_dir = self.cached_file + + with open(file_dir) as fp: + responses = json.load(fp) + fp.close() + + responses = dict(responses) + return responses + + def _read_zipfile_structure(self, file_path): + """ + Reads the zipfile structure and returns the dictionary containing file_names as keys and list of relative paths having same file name. + """ + with ZipFile(file_path) as zf: + file_names_in_zip = zf.namelist() + zf.close() + + file_names_dict = {} + for i in file_names_in_zip: + if str(Path(i).name) not in file_names_dict.keys(): + file_names_dict[str(Path(i).name)] = [i] + else: + file_names_dict[str(Path(i).name)].append(i) + return file_names_dict + diff --git a/tvb_library/tvb/datasets/zenodo.py b/tvb_library/tvb/datasets/zenodo.py new file mode 100644 index 0000000000..9a3c95eb07 --- /dev/null +++ b/tvb_library/tvb/datasets/zenodo.py @@ -0,0 +1,151 @@ +## -*- coding: utf-8 -*- +# +# +# TheVirtualBrain-Scientific Package. This package holds all simulators, and +# analysers necessary to run brain-simulations. You can use it stand alone or +# in conjunction with TheVirtualBrain-Framework Package. See content of the +# documentation-folder for more details. See also http://www.thevirtualbrain.org +# +# (c) 2012-2023, Baycrest Centre for Geriatric Care ("Baycrest") and others +# +# This program is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software Foundation, +# either version 3 of the License, or (at your option) any later version. +# This program is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. See the GNU General Public License for more details. +# You should have received a copy of the GNU General Public License along with this +# program. If not, see . +# +# +# CITATION: +# When using The Virtual Brain for scientific publications, please cite it as explained here: +# https://www.thevirtualbrain.org/tvb/zwei/neuroscience-publications +# +# + +""" +.. moduleauthor:: Abhijit Deo +""" + +# code from https://github.com/space-physics/pyzenodo3 and https://github.com/space-physics/pyzenodo3/pull/9 + + +import requests +import pooch +from typing import List, Dict +from pathlib import Path +import json + +BASE_URL = "https://zenodo.org/api/" + + +class Record: + def __init__(self, data, base_url: str = BASE_URL) -> None: + """ + Record represents the repsonse from the Zenodo. + """ + + self.base_url = base_url + self.data = data + self.file_loc = {} + + + + def download(self, path: str = None, fname=None) -> None: + """ + Download the files entity from the json response at `path`. If the `path` is None, the data is downloaded at os caches. + + For more info about os cache, have a look at https://www.fatiando.org/pooch/latest/api/generated/pooch.os_cache.html. + In our use case, the is `tvb`. + """ + + if 'files' not in self.data: + raise AttributeError("No files to download! Please check if the record id entered is correct! or the data is publically accessible") + + + if path == None: + path = pooch.os_cache("tvb") + + path = str(path) + + for file in self.data["files"]: + url = file['links']['self'] + known_hash = file['checksum'] + file_name = file['key'] + + file_path = pooch.retrieve(url= url, known_hash= known_hash, path = path, fname=fname ,progressbar = True) + + self.file_loc[f'{file_name}'] = file_path + + print(f"file {file_name} is downloaded at {file_path}") + + + def get_latest_version(self): + return Zenodo().get_record(self.data['links']['latest'].split("/")[-1]) + + def describe(self) -> str: + return self.data['metadata']['description'] + + def get_record_id(self) -> str: + return self.data['conceptrecid'] + + def is_open_access(self) -> str: + return self.data['metadata']['access_right'] == "open" + + def __eq__(self, record_b) -> bool: + return (self.data == record_b.data) + + def __str__(self) -> str: + return json.dumps(self.data, indent=2) + + + +class Zenodo: + def __init__(self, api_key: str = "", base_url: str = BASE_URL) -> None: + """ + This class handles all the interactions of the user to the zenodo platform. + + + """ + self.base_url = base_url + self._api_key = api_key + + + def get_record(self, recid: str) -> Record: + """ + recid: unique id of the data repository + """ + url = self.base_url + "records/" + recid + + return Record(requests.get(url).json()) + + + def _get_records(self, params: Dict[str, str]) -> List[Record]: + url = self.base_url + "records?" + urlencode(params) + + return [Record(hit) for hit in requests.get(url).json()["hits"]["hits"]] + + + + + def get_versions_info(self, recid) -> Dict: + """ + recid: unique id of the data repository + + """ + # needs ineternet + + recid = self.get_record(recid).data['metadata']['relations']['version'][0]['parent']['pid_value'] + versions = {} + url = f"{self.base_url}records?q=conceptrecid:{recid}&all_versions=true" + + for hit in requests.get(url).json()['hits']['hits']: + + version = hit['metadata']['version'] + recid = hit['doi'].split(".")[-1] + if hit['metadata']['access_right'] == "closed": + continue + versions[version] = recid + + return versions diff --git a/tvb_library/tvb/datatypes/connectivity.py b/tvb_library/tvb/datatypes/connectivity.py index fec76be6ad..ce6d84a582 100644 --- a/tvb_library/tvb/datatypes/connectivity.py +++ b/tvb_library/tvb/datatypes/connectivity.py @@ -40,7 +40,7 @@ from tvb.basic.neotraits.api import Attr, NArray, List, HasTraits, Int, narray_summary_info from tvb.basic.neotraits.ex import TraitAttributeError from tvb.basic.readers import ZipReader, H5Reader, try_get_absolute_path - +from tvb.datasets import TVBZenodoDataset class Connectivity(HasTraits): region_labels = NArray( @@ -719,8 +719,10 @@ def _read(cls, reader): def from_file(cls, source_file="connectivity_76.zip"): result = Connectivity() - source_full_path = try_get_absolute_path("tvb_data.connectivity", source_file) + #source_full_path = try_get_absolute_path("tvb_data.connectivity", source_file) + source_full_path = TVBZenodoDataset().fetch_data(source_file) + if source_file.endswith(".h5"): reader = H5Reader(source_full_path) diff --git a/tvb_library/tvb/datatypes/local_connectivity.py b/tvb_library/tvb/datatypes/local_connectivity.py index 6c60ceb3f2..e50bd6dcc1 100644 --- a/tvb_library/tvb/datatypes/local_connectivity.py +++ b/tvb_library/tvb/datatypes/local_connectivity.py @@ -31,7 +31,7 @@ from tvb.basic.neotraits.api import HasTraits, Attr, Float, narray_summary_info from tvb.basic.readers import try_get_absolute_path, FileReader from tvb.datatypes import equations, surfaces - +from tvb.datasets import TVBZenodoDataset class LocalConnectivity(HasTraits): """ @@ -109,7 +109,7 @@ def from_file(source_file="local_connectivity_16384.mat"): result = LocalConnectivity() - source_full_path = try_get_absolute_path("tvb_data.local_connectivity", source_file) + source_full_path = TVBZenodoDataset().fetch_data(source_file) reader = FileReader(source_full_path) result.matrix = reader.read_array(matlab_data_name="LocalCoupling") diff --git a/tvb_library/tvb/datatypes/projections.py b/tvb_library/tvb/datatypes/projections.py index 713472b3af..fac6dbe8aa 100644 --- a/tvb_library/tvb/datatypes/projections.py +++ b/tvb_library/tvb/datatypes/projections.py @@ -32,7 +32,7 @@ from tvb.basic.readers import try_get_absolute_path, FileReader from tvb.datatypes import surfaces, sensors from tvb.basic.neotraits.api import HasTraits, TVBEnum, Attr, NArray, Final - +from tvb.datasets import TVBZenodoDataset class ProjectionsTypeEnum(TVBEnum): EEG = "projEEG" @@ -88,7 +88,7 @@ def from_file(cls, source_file, matlab_data_name=None, is_brainstorm=False): proj = cls() - source_full_path = try_get_absolute_path("tvb_data.projectionMatrix", source_file) + source_full_path = TVBZenodoDataset().fetch_data(source_file) reader = FileReader(source_full_path) if is_brainstorm: proj.projection_data = reader.read_gain_from_brainstorm() diff --git a/tvb_library/tvb/datatypes/region_mapping.py b/tvb_library/tvb/datatypes/region_mapping.py index 12eca3cd53..0c59c37fca 100644 --- a/tvb_library/tvb/datatypes/region_mapping.py +++ b/tvb_library/tvb/datatypes/region_mapping.py @@ -39,7 +39,7 @@ from tvb.datatypes.surfaces import Surface from tvb.datatypes.volumes import Volume from tvb.basic.neotraits.api import HasTraits, Attr, NArray - +from tvb.datasets import TVBZenodoDataset class RegionMapping(HasTraits): """ @@ -55,7 +55,7 @@ class RegionMapping(HasTraits): @staticmethod def from_file(source_file="regionMapping_16k_76.txt"): - source_full_path = try_get_absolute_path("tvb_data.regionMapping", source_file) + source_full_path = TVBZenodoDataset().fetch_data(source_file) reader = FileReader(source_full_path) result = RegionMapping() diff --git a/tvb_library/tvb/datatypes/sensors.py b/tvb_library/tvb/datatypes/sensors.py index 250a190e46..c88d7a831c 100644 --- a/tvb_library/tvb/datatypes/sensors.py +++ b/tvb_library/tvb/datatypes/sensors.py @@ -40,7 +40,7 @@ from tvb.basic.readers import FileReader, try_get_absolute_path from tvb.basic.neotraits.api import HasTraits, Attr, NArray, Int, TVBEnum, Final - +from tvb.datasets import TVBZenodoDataset class SensorTypesEnum(TVBEnum): TYPE_EEG = "EEG" @@ -77,7 +77,7 @@ class Sensors(HasTraits): def from_file(cls, source_file="eeg_brainstorm_65.txt"): result = cls() - source_full_path = try_get_absolute_path("tvb_data.sensors", source_file) + source_full_path = TVBZenodoDataset().fetch_data(source_file) reader = FileReader(source_full_path) result.labels = reader.read_array(dtype=numpy.str_, use_cols=(0,)) @@ -236,7 +236,7 @@ class SensorsMEG(Sensors): def from_file(cls, source_file="meg_151.txt.bz2"): result = super(SensorsMEG, cls).from_file(source_file) - source_full_path = try_get_absolute_path("tvb_data.sensors", source_file) + source_full_path = TVBZenodoDataset().fetch_data(source_file) reader = FileReader(source_full_path) result.orientations = reader.read_array(use_cols=(4, 5, 6)) diff --git a/tvb_library/tvb/datatypes/surfaces.py b/tvb_library/tvb/datatypes/surfaces.py index e7d011dbce..aae6cbada6 100644 --- a/tvb_library/tvb/datatypes/surfaces.py +++ b/tvb_library/tvb/datatypes/surfaces.py @@ -41,6 +41,7 @@ from tvb.basic.neotraits.api import TVBEnum from tvb.basic.neotraits.api import HasTraits, Attr, NArray, Final, Int, Float, narray_describe from tvb.basic.readers import ZipReader, try_get_absolute_path +from tvb.datasets import TVBZenodoDataset try: import gdist @@ -162,7 +163,7 @@ def _read(cls, reader): @classmethod def from_file(cls, source_file="cortex_16384.zip"): """Construct a Surface from source_file.""" - source_full_path = try_get_absolute_path("tvb_data.surfaceData", source_file) + source_full_path = TVBZenodoDataset().fetch_data(source_file) reader = ZipReader(source_full_path) return cls._read(reader) diff --git a/tvb_library/tvb/tests/library/datasets/__init__.py b/tvb_library/tvb/tests/library/datasets/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tvb_library/tvb/tests/library/datasets/tvb_zenodo_dataset_test.py b/tvb_library/tvb/tests/library/datasets/tvb_zenodo_dataset_test.py new file mode 100644 index 0000000000..b06e01cd5d --- /dev/null +++ b/tvb_library/tvb/tests/library/datasets/tvb_zenodo_dataset_test.py @@ -0,0 +1,135 @@ + +# -*- coding: utf-8 -*- +# +# +# TheVirtualBrain-Scientific Package. This package holds all simulators, and +# analysers necessary to run brain-simulations. You can use it stand alone or +# in conjunction with TheVirtualBrain-Framework Package. See content of the +# documentation-folder for more details. See also http://www.thevirtualbrain.org +# +# (c) 2012-2023, Baycrest Centre for Geriatric Care ("Baycrest") and others +# +# This program is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software Foundation, +# either version 3 of the License, or (at your option) any later version. +# This program is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. See the GNU General Public License for more details. +# You should have received a copy of the GNU General Public License along with this +# program. If not, see . +# +# +# CITATION: +# When using The Virtual Brain for scientific publications, please cite it as explained here: +# https://www.thevirtualbrain.org/tvb/zwei/neuroscience-publications +# +# + +""" +.. moduleauthor:: Abhijit Deo +""" + +from tvb.datasets import TVBZenodoDataset +from pathlib import Path +from tvb.tests.library.base_testcase import BaseTestCase +import zipfile +import pytest + +class Test_TVBZenodoDataset(BaseTestCase): + + + def test_extract(self): + + dataset = TVBZenodoDataset() + connectivity66_dir = Path(dataset.fetch_data("connectivity_66.zip")) + + assert str(connectivity66_dir).endswith(".zip") + assert connectivity66_dir.is_file() + dataset.delete_data() + assert not connectivity66_dir.is_file() + + dataset = TVBZenodoDataset(version="2.0.3", extract_dir="dataset") + connectivity66_dir = Path(dataset.fetch_data("connectivity_66.zip")) + + assert str(connectivity66_dir).endswith(".zip") + assert "dataset" in str(connectivity66_dir) + assert (Path.cwd()/"dataset").is_dir() + assert (Path.cwd()/"dataset"/"tvb_data").is_dir() + dataset.delete_data() + assert not connectivity66_dir.is_file() + + dataset = TVBZenodoDataset(version="2.0.3", extract_dir="~/dataset") + matfile_dir = Path(dataset.fetch_data("local_connectivity_80k.mat")) + + assert str(matfile_dir).endswith(".mat") + assert matfile_dir.is_file() + dataset.delete_data() + assert not matfile_dir.is_file() + + + excel_extract = Path(dataset.fetch_data(" ConnectivityTable_regions.xls")) + assert excel_extract.is_file() + dataset.delete_data() + assert not excel_extract.is_file() + + + def test_check_content(self): + + #check if connectivity_66 contains expected files. + dataset = TVBZenodoDataset() + connectivity66_dir = Path(dataset.fetch_data("connectivity_66.zip")) + + assert "centres.txt" in zipfile.ZipFile(connectivity66_dir).namelist() + assert "info.txt" in zipfile.ZipFile(connectivity66_dir).namelist() + assert "tract_lengths.txt" in zipfile.ZipFile(connectivity66_dir).namelist() + assert "weights.txt" in zipfile.ZipFile(connectivity66_dir).namelist() + dataset.delete_data() + + + dataset = TVBZenodoDataset(version= "2.0.3", extract_dir="~/dataset") + connectivity66_dir = Path(dataset.fetch_data("connectivity_66.zip")) + assert "centres.txt" in zipfile.ZipFile(connectivity66_dir).namelist() + assert "info.txt" in zipfile.ZipFile(connectivity66_dir).namelist() + assert "tract_lengths.txt" in zipfile.ZipFile(connectivity66_dir).namelist() + assert "weights.txt" in zipfile.ZipFile(connectivity66_dir).namelist() + dataset.delete_data() + + + def test_file_name_variants(self): + dataset = TVBZenodoDataset(version= "2.0.3", extract_dir="~/dataset") + connectivity66_dir_1 = Path(dataset.fetch_data("connectivity_66.zip")) + connectivity66_dir_2 = Path(dataset.fetch_data('tvb_data/connectivity/connectivity_66.zip')) + assert connectivity66_dir_1 == connectivity66_dir_2 + + dataset.delete_data() + + dataset = TVBZenodoDataset() + connectivity66_dir_1 = Path(dataset.fetch_data("connectivity_66.zip")) + connectivity66_dir_2 = Path(dataset.fetch_data('tvb_data/connectivity/connectivity_66.zip')) + assert connectivity66_dir_1 == connectivity66_dir_2 + + dataset.delete_data() + + + dataset = TVBZenodoDataset(version= "2.0.3", extract_dir="dataset") + connectivity66_dir_1 = Path(dataset.fetch_data("connectivity_66.zip")) + connectivity66_dir_2 = Path(dataset.fetch_data('tvb_data/connectivity/connectivity_66.zip')) + assert connectivity66_dir_1 == connectivity66_dir_2 + + dataset.delete_data() + + + # should raise error cause there are two files with name mapping_FS_84.txt + with pytest.raises(NameError): + dataset = TVBZenodoDataset() + data = dataset.fetch_data("mapping_FS_84.txt") + + # no error when relative path given + dataset = TVBZenodoDataset() + data = Path(dataset.fetch_data(" tvb_data/macaque/mapping_FS_84.txt")) + assert data.is_file() + + data = Path(dataset.fetch_data('tvb_data/nifti/volume_mapping/mapping_FS_84.txt')) + assert data.is_file() + + dataset.delete_data() diff --git a/tvb_library/tvb/tests/library/datasets/zenodo_test.py b/tvb_library/tvb/tests/library/datasets/zenodo_test.py new file mode 100644 index 0000000000..59e82b974e --- /dev/null +++ b/tvb_library/tvb/tests/library/datasets/zenodo_test.py @@ -0,0 +1,81 @@ + +# -*- coding: utf-8 -*- +# +# +# TheVirtualBrain-Scientific Package. This package holds all simulators, and +# analysers necessary to run brain-simulations. You can use it stand alone or +# in conjunction with TheVirtualBrain-Framework Package. See content of the +# documentation-folder for more details. See also http://www.thevirtualbrain.org +# +# (c) 2012-2023, Baycrest Centre for Geriatric Care ("Baycrest") and others +# +# This program is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software Foundation, +# either version 3 of the License, or (at your option) any later version. +# This program is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. See the GNU General Public License for more details. +# You should have received a copy of the GNU General Public License along with this +# program. If not, see . +# +# +# CITATION: +# When using The Virtual Brain for scientific publications, please cite it as explained here: +# https://www.thevirtualbrain.org/tvb/zwei/neuroscience-publications +# +# + +""" +.. moduleauthor:: Abhijit Deo +""" + + +#TODO : change the record id to the latest when done testing on local machine. :) :wq + +from tvb.datasets import Zenodo, Record +from pathlib import Path +from tvb.tests.library.base_testcase import BaseTestCase + + +class TestZenodo(BaseTestCase): + + def test_get_record(self): + + zenodo = Zenodo() + rec = zenodo.get_record("4263723") + + assert type(rec) == Record + assert rec.data["doi"] == "10.5281/zenodo.4263723" + + del rec + del zenodo + + + def test_get_versions(self): + + zenodo = Zenodo() + versions = zenodo.get_versions_info("3491055") + + assert type(versions) == dict + assert versions == {'1.5.9': '3457454', '2.0.3': '4263723', '2.0.1': '3497545', '2.0.2': '3688773', '2.0.0': '3491055', '1.5.9.b': '3474071', '2.7': '7574266'} + + del zenodo + del versions + +class TestRecord(BaseTestCase): + + + def test_download(self): + + zen = Zenodo() + + rec = zen.get_record("7929679") + + + rec.download() + print(rec.file_loc) + for file_name, file_path in rec.file_loc.items(): + assert Path(file_path).is_file() + Path(file_path).unlink() + +