Merge pull request #690 from abhi-glitchhg/pyzenodo

Update TVB-1999-deo branch

tvb_build/build_step1.py

@@ -44,7 +44,6 @@
 import requests
 
 import tvb_bin
-from tvb.datasets import TVBZenodoDataset
 from subprocess import Popen, PIPE
 
 # source paths
@@ -53,95 +52,13 @@
 FW_FOLDER = os.path.join(TVB_ROOT, 'tvb_framework')
 LICENSE_PATH = os.path.join(FW_FOLDER, 'LICENSE')
 RELEASE_NOTES_PATH = os.path.join(TVB_ROOT, 'tvb_documentation', 'RELEASE_NOTES')
-dataset = TVBZenodoDataset()
-DATA_SRC_FOLDER = dataset.extract_dir / 'tvb_data'
+
+
 DEMOS_MATLAB_FOLDER = os.path.join(TVB_ROOT, 'tvb_documentation', 'matlab')
 
 # dest paths
 DIST_FOLDER = os.path.join(os.path.dirname(__file__), 'build', 'TVB_Distribution')
 
-DATA_INSIDE_FOLDER = os.path.join(DIST_FOLDER, '_tvb_data')
-
-INCLUDED_INSIDE_DATA = [
-    "__init__.py",
-    "Default_Project.zip",
-
-    "connectivity/connectivity_76.zip",
-    "connectivity/paupau.zip",
-    "connectivity/connectivity_66.zip",
-    "connectivity/connectivity_192.zip",
-    "connectivity/__init__.py",
-
-    "projectionMatrix/projection_eeg_62_surface_16k.mat",
-    "projectionMatrix/projection_eeg_65_surface_16k.npy",
-    "projectionMatrix/projection_meg_276_surface_16k.npy",
-    "projectionMatrix/projection_seeg_588_surface_16k.npy",
-    "projectionMatrix/__init__.py",
-
-    "regionMapping/__init__.py",
-    "regionMapping/regionMapping_16k_76.txt",
-    "regionMapping/regionMapping_80k_80.txt",
-    "regionMapping/regionMapping_16k_192.txt",
-
-    "sensors/eeg_unitvector_62.txt.bz2",
-    "sensors/eeg_brainstorm_65.txt",
-    "sensors/meg_151.txt.bz2",
-    "sensors/meg_brainstorm_276.txt",
-    "sensors/seeg_39.txt.bz2",
-    "sensors/seeg_brainstorm_960.txt",
-    "sensors/seeg_588.txt",
-    "sensors/__init__.py",
-
-    "surfaceData/__init__.py",
-    "surfaceData/cortex_80k.zip",
-    "surfaceData/cortex_16384.zip",
-    "surfaceData/outer_skin_4096.zip",
-    "surfaceData/inner_skull_4096.zip",
-    "surfaceData/outer_skull_4096.zip",
-    "surfaceData/scalp_1082.zip",
-    "surfaceData/face_8614.zip",
-
-    "local_connectivity/__init__.py",
-    "local_connectivity/local_connectivity_16384.mat",
-    "local_connectivity/local_connectivity_80k.mat",
-
-    "obj/__init__.py",
-    "obj/face_surface.obj",
-    "obj/eeg_cap.obj",
-
-    "mouse/allen_2mm/Connectivity.h5",
-    "mouse/allen_2mm/Volume.h5",
-    "mouse/allen_2mm/StructuralMRI.h5",
-    "mouse/allen_2mm/RegionVolumeMapping.h5",
-]
-
-def fetch_data_to_include(filenames_list, dataset):
-    for i in filenames_list:
-        dataset.fetch_data("tvb_data/"+i)
-
-
-def _copy_dataset(dataset_files, dataset_destination):
-    for pth in dataset_files:
-        rel_pth = pth.split('/')
-        origin = os.path.join(DATA_SRC_FOLDER, *rel_pth)
-        destination = os.path.join(dataset_destination, *rel_pth)
-        destination_folder = os.path.dirname(destination)
-        if not os.path.exists(destination_folder):
-            os.makedirs(destination_folder)
-        print("Copying %s into %s" % (origin, destination))
-        shutil.copyfile(origin, destination)
-
-
-def copy_distribution_dataset():
-    """
-    Copy the required data file from tvb_data folder:
-    - inside TVB library package (for internal usage).
-        Will be used during TVB functioning: import default project,
-        load default for console profile, or code update events
-    - in tvb_data folder, as example for users.
-    """
-    _copy_dataset(INCLUDED_INSIDE_DATA, DATA_INSIDE_FOLDER)
-
 
 def _copy_demos_collapsed(to_copy):
     """
@@ -235,8 +152,6 @@ def build_step1():
     shutil.copytree(DEMOS_MATLAB_FOLDER, os.path.join(DIST_FOLDER, 'matlab'),
                     ignore=shutil.ignore_patterns('.svn', '*.rst'))
 
-    fetch_data_to_include(INCLUDED_INSIDE_DATA, dataset)
-    copy_distribution_dataset()
 
     _copy_demos_collapsed({os.path.join("..", "tvb_documentation", "demos"): os.path.join(DIST_FOLDER, "demo_scripts"),
                            os.path.join("..", "tvb_documentation", "tutorials"):

tvb_contrib/setup.py

@@ -40,6 +40,7 @@
                  install_requires=CONTRIB_DEPENDENCIES,
                  description='A package with TVB contributed additions to the simulator, useful for scripting.',
                  long_description=DESCRIPTION,
+                 long_description_content_type="text/x-rst",
                  license="GPL-3.0-or-later",
                  author=TEAM,
                  author_email='[email protected]',

tvb_deprecated/setup.py

@@ -40,12 +40,18 @@
       install_requires=["tvb-library"],
       description='This namespace is only redirecting to tvb-library',
       long_description=DESCRIPTION,
+      long_description_content_type="text/x-rst",
       license="GPL-3.0-or-later",
       author=TVB_TEAM,
       author_email='[email protected]',
       url='https://www.thevirtualbrain.org',
       download_url='https://github.com/the-virtual-brain/tvb-root',
-      keywords='tvb brain simulator neuroscience human animal neuronal dynamics models delay'
+      keywords='tvb brain simulator neuroscience human animal neuronal dynamics models delay',
+      classifiers=["Development Status :: 6 - Mature",
+                   "Intended Audience :: Science/Research",
+                   "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)",
+                   "Programming Language :: Python :: 3"
+                   ]
       )
 
 # Cleanup

tvb_documentation/demos/Demos.rst

@@ -113,7 +113,6 @@ examples.
 .. _Analyze Region Corrcoef: https://nbviewer.thevirtualbrain.org/url/docs.thevirtualbrain.org/demos/analyze_region_corrcoef.ipynb
 .. _Compare Connectivity Normalization: https://nbviewer.thevirtualbrain.org/url/docs.thevirtualbrain.org/demos/compare_connectivity_normalization.ipynb
 .. _Display Source Sensor Geometry: https://nbviewer.thevirtualbrain.org/url/docs.thevirtualbrain.org/demos/display_source_sensor_geometry.ipynb
-.. _Display Surface Local Connectivity: https://nbviewer.thevirtualbrain.org/url/docs.thevirtualbrain.org/demos/display_surface_local_connectivity.ipynb
 .. _Encrypt Files before upload in TVB Web GUI: https://nbviewer.thevirtualbrain.org/url/docs.thevirtualbrain.org/demos/encrypt_data.ipynb
 .. _Exploring Epileptor 2D: https://nbviewer.thevirtualbrain.org/url/docs.thevirtualbrain.org/demos/exploring_epileptor_2d.ipynb
 .. _Exploring Longer Time Series: https://nbviewer.thevirtualbrain.org/url/docs.thevirtualbrain.org/demos/exploring_longer_time_series.ipynb
@@ -167,14 +166,6 @@ examples.
       `Display Source Sensor Geometry`_
 
 
-.. figure:: figures/display_surface_local_connectivity.png
-      :width: 200px
-      :figclass: demo-figure
-      :target: `Display Surface Local Connectivity`_
-
-      `Display Surface Local Connectivity`_
-
-
 .. figure:: figures/exploring_longer_time_series.png
       :width: 200px
       :figclass: demo-figure

tvb_framework/setup.py

@@ -64,12 +64,19 @@
                                  'test': ["pytest", "pytest-benchmark", "pytest-mock", "BeautifulSoup4"]},
                  description='A package for performing whole brain simulations',
                  long_description=DESCRIPTION,
+                 long_description_content_type="text/x-rst",
                  license="GPL-3.0-or-later",
                  author=TVB_TEAM,
                  author_email='[email protected]',
                  url='https://www.thevirtualbrain.org',
                  download_url='https://github.com/the-virtual-brain/tvb-root',
-                 keywords='tvb brain simulator neuroscience human animal neuronal dynamics models delay')
+                 keywords='tvb brain simulator neuroscience human animal neuronal dynamics models delay',
+                 classifiers=["Development Status :: 6 - Mature",
+                              "Intended Audience :: Science/Research",
+                              "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)",
+                              "Programming Language :: Python :: 3"
+                              ]
+                 )
 
 # Clean after install
 shutil.rmtree('tvb_framework.egg-info', True)

tvb_framework/setup_bids_monitor.py

@@ -50,18 +50,19 @@
                  version=VERSION,
                  packages=setuptools.find_packages(
                      exclude=['tvb.interfaces.web', 'tvb.interfaces.web.*', 'tvb.interfaces.command',
-                              'tvb.interfaces.command.*', 'tvb.tests', 'tvb.tests.*', 'tvb.interfaces.rest'
+                              'tvb.interfaces.command.*', 'tvb.tests', 'tvb.tests.*', 'tvb.interfaces.rest',
                               'tvb.interfaces.rest.*', 'tvb.adapters', 'tvb.adapters.*', 'tvb.core',
                               'tvb.core.*']),
                  include_package_data=True,
                  install_requires=TVB_INSTALL_REQUIREMENTS,
                  extras_require={'test': ["pytest", "pytest-benchmark"]},
                  description='A helper package containing BIDS directory monitor module',
                  long_description=DESCRIPTION,
+                 long_description_content_type="text/x-rst",
                  license="GPL-3.0-or-later",
                  author=TVB_TEAM,
                  author_email='[email protected]',
-                 url='http://www.thevirtualbrain.org',
+                 url='https://www.thevirtualbrain.org',
                  download_url='https://github.com/the-virtual-brain/tvb-root',
                  keywords='tvb rest client brain simulator neuroscience human animal neuronal dynamics models delay')
 

tvb_framework/setup_rest_client.py

@@ -59,6 +59,7 @@
                                  'test': ["pytest", "pytest-benchmark"]},
                  description='A helper package for preparing and sending requests towards the TVB REST API',
                  long_description=DESCRIPTION,
+                 long_description_content_type="text/x-rst",
                  license="GPL-3.0-or-later",
                  author=TVB_TEAM,
                  author_email='[email protected]',

tvb_library/setup.py

@@ -42,12 +42,11 @@
 LIBRARY_REQUIRED_PACKAGES = ["autopep8", "Deprecated", "docutils", "ipywidgets", "lxml", "mako>=1.1.4", "matplotlib",
                              "networkx", "numba", "numexpr", "numpy", "pooch", "pylems", "scipy", "six"]
 
-LIBRARY_REQUIRED_EXTRA = ["h5py",  "pytest", "pytest-benchmark", "pytest-xdist", "tvb-gdist", "tvb-data"]
+LIBRARY_REQUIRED_EXTRA = ["h5py", "pytest", "pytest-benchmark", "pytest-xdist", "tvb-gdist", "tvb-data"]
 
 with open(os.path.join(os.path.dirname(__file__), 'README.rst')) as fd:
     DESCRIPTION = fd.read()
 
-
 setuptools.setup(name='tvb-library',
                  version=LIBRARY_VERSION,
                  packages=setuptools.find_packages(),
@@ -56,12 +55,19 @@
                  extras_require={"test": LIBRARY_REQUIRED_EXTRA},
                  description='A package for performing whole brain simulations',
                  long_description=DESCRIPTION,
+                 long_description_content_type="text/x-rst",
                  license="GPL-3.0-or-later",
                  author=LIBRARY_TEAM,
                  author_email='[email protected]',
                  url='https://www.thevirtualbrain.org',
                  download_url='https://github.com/the-virtual-brain/tvb-root',
-                 keywords='tvb brain simulator neuroscience human animal neuronal dynamics models delay')
+                 keywords='tvb brain simulator neuroscience human animal neuronal dynamics models delay',
+                 classifiers=["Development Status :: 6 - Mature",
+                              "Intended Audience :: Science/Research",
+                              "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)",
+                              "Programming Language :: Python :: 3"
+                              ]
+                 )
 
 # Cleanup after EGG install. These are created by running setup.py in the source tree
 shutil.rmtree('tvb_library.egg-info', True)

tvb_library/tvb/datasets/base.py

@@ -49,37 +49,13 @@ def __init__(self, version : str , extract_dir : str =None) -> None:
         self.extract_dir = Path(extract_dir).expanduser()
 
 
-    def fetch_data(self, file_name:str) -> str:
-        if Path(file_name).is_absolute():
-            self.log.warning("Given file name is an absolute path. No operations are done. The path is returned as it is")
-            return file_name
-
-        return self._fetch_data(file_name)
-
+    def fetch_data(self) :
+        raise NotImplemented
     def get_version(self) -> str:
         return self.version
 
     def delete_data(self):
         raise NotImplemented
-
-    def _read_zipfile_structure(self, file_path):
-        """
-        Reads the zipfile structure and returns the dictionary containing file_names as keys and list of relative paths having same file name. 
-        """
-        with ZipFile(file_path) as zf:
-            file_names_in_zip = zf.namelist()
-        zf.close()      
-
-        file_names_dict = {}
-        for i in file_names_in_zip:
-            if str(Path(i).name) not in file_names_dict.keys():
-                file_names_dict[str(Path(i).name)] = [i]
-            else:
-                file_names_dict[str(Path(i).name)].append(i)
-        return file_names_dict
-
-    def _fetch_data(self,file_name):
-        raise NotImplemented
 
     def _download(self):
         raise NotImplemented

tvb_library/tvb/datasets/tvb_data.py

@@ -28,7 +28,6 @@
 .. moduleauthor:: Abhijit Deo <[email protected]>
 """
 
-import os
 import requests
 import json
 
@@ -77,7 +76,7 @@ def __init__(self, version : str= "2.7", extract_dir : str = None):
 
         self.rec = Record(self._read_cached_response()[self.version])
 
-    def _fetch_data(self, file_name):        
+    def fetch_data(self, file_name:str)->str:        
         """
         Function to fetch the file having `file_name` as name of the file. The function checks if the dataset is downloaded or not. If not, function downloads the dataset and then extracts/unzip the file.
 
@@ -89,7 +88,10 @@ def _fetch_data(self, file_name):
         returns: str
             path of the extracted/Unzipped file.
         """
-
+        if Path(file_name).is_absolute():
+            self.log.warning("Given `file_name` is an absolute path. No operations are done. The `file_name` is returned as it is")
+            return file_name
+
         extract_dir = self.extract_dir 
 
         try:
@@ -128,35 +130,20 @@ def _fetch_data(self, file_name):
                            file_name should be one of the following paths: {self.files_in_zip_dict[file_name]}""")
             raise NameError(f"file name should be one of the {self.files_in_zip_dict[file_name]}, but got {file_name}")
 
-    def describe(self):
+    def describe(self)-> str:
         """
         Returns the project description mentioned on the zenodo website. 
         """
         return self.rec.describe()
 
-    def get_recordid(self):
+    def get_recordid(self) -> str:
         """
         returns record id of the dataset  
         """
         return self.recid
 
-    def fetch_all_data(self):
-
-        if self.files_in_zip_dict == None:
-            self._download(path = self.cached_dir, fname=f"tvb_data_{self.version}.zip")
-            self.files_in_zip_dict = self._read_zipfile_structure(self.rec.file_loc['tvb_data.zip'])
-
-
-        for  file_paths in self.files_in_zip_dict.values():
-            for file_path in file_paths:
-                self.fetch_data(file_path)
-
-        if self.extract_dir.is_absolute():
-            return str(self.extract_dir)
-        return str(Path.cwd()/self.extract_dir)
 
-
-    def delete_data(self):
+    def delete_data(self)->None:
         """
         Deletes the `tvb_data` folder in the `self.extract_dir` directory. 
         """
@@ -218,3 +205,20 @@ def _read_cached_response(self):
 
         responses = dict(responses)
         return responses
+
+    def _read_zipfile_structure(self, file_path):
+        """
+        Reads the zipfile structure and returns the dictionary containing file_names as keys and list of relative paths having same file name. 
+        """
+        with ZipFile(file_path) as zf:
+            file_names_in_zip = zf.namelist()
+        zf.close()      
+
+        file_names_dict = {}
+        for i in file_names_in_zip:
+            if str(Path(i).name) not in file_names_dict.keys():
+                file_names_dict[str(Path(i).name)] = [i]
+            else:
+                file_names_dict[str(Path(i).name)].append(i)
+        return file_names_dict
+