From 61bfbb9a222049477d8f8d91ddaf9ac625f18f3e Mon Sep 17 00:00:00 2001 From: Valerie Hayot-Sasson Date: Thu, 31 Aug 2017 13:14:37 -0700 Subject: [PATCH 01/32] added spark docker container to circle.yml --- circle.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/circle.yml b/circle.yml index 4fd67ba..53272de 100644 --- a/circle.yml +++ b/circle.yml @@ -3,8 +3,7 @@ machine: - sudo curl -L -o /usr/bin/docker 'https://s3-external-1.amazonaws.com/circle-downloads/docker-1.9.1-circleci' - sudo chmod 0755 /usr/bin/docker -services: - pre: + services: - docker dependencies: @@ -12,10 +11,17 @@ dependencies: - pip install --upgrade pip - npm install -g bids-validator - gem install bundler - - pip install boutiques - - pip install pytest + - docker pull sequenceiq/spark:1.6.0 test: override: - bosh-validate demo/bids-app-example.json - bids-validator demo/ds001 + - docker run --rm -v $PWD:/sim -it sequenceiq/spark:1.6.0 bash + - yum install python-pip + - pip install boutiques + - pip install pytest + - cd $HADOOP_PREFIX + - bin/hdfs dfs -copyFromLocal /sim/demo/ds001 / + - python/spark-bids.py demo/bids-app-example.json /ds001 output + - exit From 0a7ab62c963ebb80c8cec03cf2fd22ec00a6f681 Mon Sep 17 00:00:00 2001 From: Valerie Hayot-Sasson Date: Thu, 31 Aug 2017 13:27:19 -0700 Subject: [PATCH 02/32] testing modification made in circle.yml --- circle.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/circle.yml b/circle.yml index 53272de..8a43eee 100644 --- a/circle.yml +++ b/circle.yml @@ -17,11 +17,4 @@ test: override: - bosh-validate demo/bids-app-example.json - bids-validator demo/ds001 - - docker run --rm -v $PWD:/sim -it sequenceiq/spark:1.6.0 bash - - yum install python-pip - - pip install boutiques - - pip install pytest - - cd $HADOOP_PREFIX - - bin/hdfs dfs -copyFromLocal /sim/demo/ds001 / - - python/spark-bids.py demo/bids-app-example.json /ds001 output - - exit + - docker run --rm -v $PWD:/sim sequenceiq/spark:1.6.0 bash "yum install python-pip; pip install boutiques; pip install pytest; cd $HADOOP_PREFIX; bin/hdfs dfs -copyFromLocal /sim/demo/ds001 /;python/spark-bids.py demo/bids-app-example.json /ds001 output; exit" From 0f68c58d656ff5a37e7159e888e5bd788ba32a36 Mon Sep 17 00:00:00 2001 From: Tristan Glatard Date: Sun, 3 Sep 2017 22:47:13 -0700 Subject: [PATCH 03/32] Test coverage --- setup.cfg | 2 ++ setup.py | 7 ++++--- sim/tests/README.md | 10 ++++++++++ {test => sim/tests}/demo/bids-app-example.json | 0 {test => sim/tests}/demo/ds001/CHANGES | 0 {test => sim/tests}/demo/ds001/README | 0 .../tests}/demo/ds001/dataset_description.json | 0 {test => sim/tests}/demo/ds001/participants.tsv | 0 .../tests}/demo/ds001/sub-01/anat/sub-01_T1w.nii.gz | Bin .../demo/ds001/sub-01/anat/sub-01_inplaneT2.nii.gz | Bin .../tests}/demo/ds001/sub-02/anat/sub-02_T1w.nii.gz | Bin .../demo/ds001/sub-02/anat/sub-02_inplaneT2.nii.gz | Bin .../demo/ds001/task-balloonanalogrisktask_bold.json | 0 {test => sim/tests}/test_all.py | 5 +---- 14 files changed, 17 insertions(+), 7 deletions(-) create mode 100644 sim/tests/README.md rename {test => sim/tests}/demo/bids-app-example.json (100%) rename {test => sim/tests}/demo/ds001/CHANGES (100%) rename {test => sim/tests}/demo/ds001/README (100%) rename {test => sim/tests}/demo/ds001/dataset_description.json (100%) rename {test => sim/tests}/demo/ds001/participants.tsv (100%) rename {test => sim/tests}/demo/ds001/sub-01/anat/sub-01_T1w.nii.gz (100%) rename {test => sim/tests}/demo/ds001/sub-01/anat/sub-01_inplaneT2.nii.gz (100%) rename {test => sim/tests}/demo/ds001/sub-02/anat/sub-02_T1w.nii.gz (100%) rename {test => sim/tests}/demo/ds001/sub-02/anat/sub-02_inplaneT2.nii.gz (100%) rename {test => sim/tests}/demo/ds001/task-balloonanalogrisktask_bold.json (100%) rename {test => sim/tests}/test_all.py (94%) diff --git a/setup.cfg b/setup.cfg index b88034e..0115b61 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,4 @@ [metadata] description-file = README.md +[aliases] +test=pytest \ No newline at end of file diff --git a/setup.py b/setup.py index 527b79a..b88df27 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,8 @@ DEPS = [ "boutiques", "pybids", - "pyspark" + "pyspark", + "pytest-runner" ] setup(name="simtools", @@ -16,8 +17,8 @@ license="GPL3.0", packages=["sim"], include_package_data=True, - test_suite="nose.collector", - tests_require=["nose"], + test_suite="pytest", + tests_require=["pytest"], setup_requires=DEPS, install_requires=DEPS, entry_points = { diff --git a/sim/tests/README.md b/sim/tests/README.md new file mode 100644 index 0000000..c33fb17 --- /dev/null +++ b/sim/tests/README.md @@ -0,0 +1,10 @@ +# Tests + +To check coverage, install coveralls: +```pip install coveralls``` + +Run the tests through coveral: +```coverage run --source sim setup.py test``` + +Print the coverage report: +```coverage report -m``` \ No newline at end of file diff --git a/test/demo/bids-app-example.json b/sim/tests/demo/bids-app-example.json similarity index 100% rename from test/demo/bids-app-example.json rename to sim/tests/demo/bids-app-example.json diff --git a/test/demo/ds001/CHANGES b/sim/tests/demo/ds001/CHANGES similarity index 100% rename from test/demo/ds001/CHANGES rename to sim/tests/demo/ds001/CHANGES diff --git a/test/demo/ds001/README b/sim/tests/demo/ds001/README similarity index 100% rename from test/demo/ds001/README rename to sim/tests/demo/ds001/README diff --git a/test/demo/ds001/dataset_description.json b/sim/tests/demo/ds001/dataset_description.json similarity index 100% rename from test/demo/ds001/dataset_description.json rename to sim/tests/demo/ds001/dataset_description.json diff --git a/test/demo/ds001/participants.tsv b/sim/tests/demo/ds001/participants.tsv similarity index 100% rename from test/demo/ds001/participants.tsv rename to sim/tests/demo/ds001/participants.tsv diff --git a/test/demo/ds001/sub-01/anat/sub-01_T1w.nii.gz b/sim/tests/demo/ds001/sub-01/anat/sub-01_T1w.nii.gz similarity index 100% rename from test/demo/ds001/sub-01/anat/sub-01_T1w.nii.gz rename to sim/tests/demo/ds001/sub-01/anat/sub-01_T1w.nii.gz diff --git a/test/demo/ds001/sub-01/anat/sub-01_inplaneT2.nii.gz b/sim/tests/demo/ds001/sub-01/anat/sub-01_inplaneT2.nii.gz similarity index 100% rename from test/demo/ds001/sub-01/anat/sub-01_inplaneT2.nii.gz rename to sim/tests/demo/ds001/sub-01/anat/sub-01_inplaneT2.nii.gz diff --git a/test/demo/ds001/sub-02/anat/sub-02_T1w.nii.gz b/sim/tests/demo/ds001/sub-02/anat/sub-02_T1w.nii.gz similarity index 100% rename from test/demo/ds001/sub-02/anat/sub-02_T1w.nii.gz rename to sim/tests/demo/ds001/sub-02/anat/sub-02_T1w.nii.gz diff --git a/test/demo/ds001/sub-02/anat/sub-02_inplaneT2.nii.gz b/sim/tests/demo/ds001/sub-02/anat/sub-02_inplaneT2.nii.gz similarity index 100% rename from test/demo/ds001/sub-02/anat/sub-02_inplaneT2.nii.gz rename to sim/tests/demo/ds001/sub-02/anat/sub-02_inplaneT2.nii.gz diff --git a/test/demo/ds001/task-balloonanalogrisktask_bold.json b/sim/tests/demo/ds001/task-balloonanalogrisktask_bold.json similarity index 100% rename from test/demo/ds001/task-balloonanalogrisktask_bold.json rename to sim/tests/demo/ds001/task-balloonanalogrisktask_bold.json diff --git a/test/test_all.py b/sim/tests/test_all.py similarity index 94% rename from test/test_all.py rename to sim/tests/test_all.py index 37ed71b..f890ae5 100755 --- a/test/test_all.py +++ b/sim/tests/test_all.py @@ -1,14 +1,11 @@ import os, pytest, random, subprocess, time from unittest import TestCase -#def test_bids_validator(): - # subprocess.call(["bids-validator","../demo/ds001"]) - class TestSim(TestCase): ## UTILITY METHODS def get_sim_dir(self): - return os.path.join(os.path.dirname(__file__),"../sim") + return os.path.join(os.path.dirname(__file__),"..") def get_demo_dir(self): return os.path.join(os.path.dirname(__file__),"demo") From d0edfbe597c564aad28db7dd2fb93f6a75570e13 Mon Sep 17 00:00:00 2001 From: Tristan Glatard Date: Sun, 3 Sep 2017 22:58:23 -0700 Subject: [PATCH 04/32] Added pypi badge --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 45f38ca..0a2b526 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +[![PyPI](https://img.shields.io/pypi/v/sim.svg)](https://pypi.python.org/pypi/sim) +[![PyPI](https://img.shields.io/pypi/pyversions/sim.svg)](https://pypi.python.org/pypi/sim) [![CircleCI](https://circleci.com/gh/big-data-lab-team/sim/tree/master.svg?style=svg)](https://circleci.com/gh/big-data-lab-team/sim/tree/master) # Spark for neuroIMaging (sim) From 12466195dce4d99f4278b9ff9dd91770c08ee33e Mon Sep 17 00:00:00 2001 From: Tristan Glatard Date: Sun, 3 Sep 2017 23:00:24 -0700 Subject: [PATCH 05/32] Added pypi badge --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 0a2b526..980d018 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ -[![PyPI](https://img.shields.io/pypi/v/sim.svg)](https://pypi.python.org/pypi/sim) -[![PyPI](https://img.shields.io/pypi/pyversions/sim.svg)](https://pypi.python.org/pypi/sim) +[![PyPI](https://img.shields.io/pypi/v/simtools.svg)](https://pypi.python.org/pypi/simtools) [![CircleCI](https://circleci.com/gh/big-data-lab-team/sim/tree/master.svg?style=svg)](https://circleci.com/gh/big-data-lab-team/sim/tree/master) # Spark for neuroIMaging (sim) From 1773a49650717dd1450f7de7116b33993b57e056 Mon Sep 17 00:00:00 2001 From: Greg Kiar Date: Wed, 27 Sep 2017 14:46:41 -0400 Subject: [PATCH 06/32] updated sparkbids for new boutiques invocations (consistent with invoc. schema --- sim/SparkBIDS.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/sim/SparkBIDS.py b/sim/SparkBIDS.py index 607cd42..9cfd5c9 100644 --- a/sim/SparkBIDS.py +++ b/sim/SparkBIDS.py @@ -109,14 +109,13 @@ def write_invocation_file(self, analysis_level, participant_label, invocation_fi # Creates invocation object invocation = {} - invocation["inputs"] = [ ] - invocation["inputs"].append({"bids_dir": self.bids_dataset}) - invocation["inputs"].append({"output_dir_name": self.output_dir}) + invocation["bids_dir"] = self.bids_dataset + invocation["output_dir_name"] = self.output_dir if analysis_level == "participant": - invocation["inputs"].append({"analysis_level": "participant"}) - invocation["inputs"].append({"participant_label": participant_label}) + invocation["analysis_level"] = "participant" + invocation["participant_label"] = participant_label elif analysis_level == "group": - invocation["inputs"].append({"analysis_level": "group"}) + invocation["analysis_level"] = "group" json_invocation = json.dumps(invocation) From c1325d932677d271f0653981488af54612fb729f Mon Sep 17 00:00:00 2001 From: Greg Kiar Date: Wed, 27 Sep 2017 14:56:51 -0400 Subject: [PATCH 07/32] specified version to which this change pertains --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 527b79a..3be84b8 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ VERSION = "0.1.3" DEPS = [ - "boutiques", + "boutiques=>0.5.3", "pybids", "pyspark" ] From 1e4f5c776a910bf0c24f33c9c0ee44116684cc2c Mon Sep 17 00:00:00 2001 From: Greg Kiar Date: Wed, 27 Sep 2017 15:09:23 -0400 Subject: [PATCH 08/32] removed out-dated flag --- sim/SparkBIDS.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sim/SparkBIDS.py b/sim/SparkBIDS.py index 9cfd5c9..44e5cb6 100644 --- a/sim/SparkBIDS.py +++ b/sim/SparkBIDS.py @@ -175,7 +175,7 @@ def run_group_analysis(self): return ("group", exec_result) def bosh_exec(self, invocation_file): - run_command = "bosh {0} -i {1} -e -d".format(self.boutiques_descriptor, invocation_file) + run_command = "bosh {0} -i {1} -e".format(self.boutiques_descriptor, invocation_file) result = None try: log = subprocess.check_output(run_command, shell=True, stderr=subprocess.STDOUT) From 0d9903f916f6ad8d3ab62dc23f6e74ec4959ac1f Mon Sep 17 00:00:00 2001 From: Tristan Glatard Date: Tue, 3 Oct 2017 17:55:35 -0400 Subject: [PATCH 09/32] Updates to support boutiques API --- sim/SparkBIDS.py | 20 +++++++------------- sim/tests/test_all.py | 5 ++--- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/sim/SparkBIDS.py b/sim/SparkBIDS.py index 44e5cb6..f2801cc 100644 --- a/sim/SparkBIDS.py +++ b/sim/SparkBIDS.py @@ -1,5 +1,5 @@ from bids.grabbids import BIDSLayout -import json, os, errno, subprocess, time, tarfile, shutil +import boutiques, errno, json, os, shutil, subprocess, tarfile, time class SparkBIDS(object): @@ -95,7 +95,7 @@ def create_tar_file(self, out_dir, tar_name, files): tar.add(f) def pretty_print(self, result): - (label, (log, returncode)) = result + (label, (returncode, log)) = result status = "SUCCESS" if returncode == 0 else "ERROR" timestamp = str(int(time.time() * 1000)) filename = "{0}.{1}.log".format(timestamp, label) @@ -175,18 +175,12 @@ def run_group_analysis(self): return ("group", exec_result) def bosh_exec(self, invocation_file): - run_command = "bosh {0} -i {1} -e".format(self.boutiques_descriptor, invocation_file) - result = None try: - log = subprocess.check_output(run_command, shell=True, stderr=subprocess.STDOUT) - result = (log, 0) - except subprocess.CalledProcessError as e: - result = (e.output, e.returncode) - try: - shutil.rmtree(label) - except: - pass - return result + boutiques.execute("launch",self.boutiques_descriptor,invocation_file, "-x") + result = 0 + except SystemExit as e: + result = e.code + return (result, "Empty log, Boutiques API doesn't return it yet.\n") def is_valid_file(parser, arg): if not os.path.exists(arg): diff --git a/sim/tests/test_all.py b/sim/tests/test_all.py index f890ae5..cd6da3e 100755 --- a/sim/tests/test_all.py +++ b/sim/tests/test_all.py @@ -1,5 +1,6 @@ import os, pytest, random, subprocess, time from unittest import TestCase +import boutiques class TestSim(TestCase): @@ -39,9 +40,7 @@ def run_spark_bids(self,checkOutputGroup=True,options=[],correctBrainSize="83053 ## TESTS def test_demo_descriptor_valid(self): - self.assertFalse(subprocess.call(["bosh-validate", - self.get_json_descriptor() - ,"-b"])) + self.assertFalse(boutiques.validate(self.get_json_descriptor(),"-b")) def test_spark_bids_no_option(self): self.run_spark_bids() From 366b698fe2c0896053417a30fbaa3cf1a65b59ba Mon Sep 17 00:00:00 2001 From: Tristan Glatard Date: Tue, 3 Oct 2017 18:01:27 -0400 Subject: [PATCH 10/32] Fixed tests --- setup.py | 2 +- sim/spark_bids.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index db8766b..b88df27 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ VERSION = "0.1.3" DEPS = [ - "boutiques=>0.5.3", + "boutiques", "pybids", "pyspark", "pytest-runner" diff --git a/sim/spark_bids.py b/sim/spark_bids.py index 7e956df..3e11d01 100755 --- a/sim/spark_bids.py +++ b/sim/spark_bids.py @@ -1,7 +1,7 @@ #!/usr/bin/env python from pyspark import SparkContext, SparkConf -from SparkBIDS import SparkBIDS +from sim import SparkBIDS import argparse, os def is_valid_file(parser, arg): From b5952792699f59f0852e9ddaf7da3870c430ade1 Mon Sep 17 00:00:00 2001 From: Tushar Gupta Date: Thu, 5 Oct 2017 22:42:15 -0400 Subject: [PATCH 11/32] Function to detect Participant analysis failure and abort the Group analysis --- sim/SparkBIDS.py | 9 +++++++++ 1 file changed, 9 insertions(+) mode change 100644 => 100755 sim/SparkBIDS.py diff --git a/sim/SparkBIDS.py b/sim/SparkBIDS.py old mode 100644 new mode 100755 index 607cd42..a8616f4 --- a/sim/SparkBIDS.py +++ b/sim/SparkBIDS.py @@ -42,6 +42,11 @@ def run(self, sc): for result in mapped.collect(): self.pretty_print(result) + if self.check_failure(result): + # Disable Group Analysis if Participant Analysis Fails + self.do_group_analysis = False + print("ERROR# Participant analysis failed. Group analysis will be aborted.") + # Group analysis if self.do_group_analysis: @@ -198,3 +203,7 @@ def is_valid_file(parser, arg): def get_participant_from_fn(self,filename): if filename.endswith(".tar"): return filename.split('-')[-1][:-4] return filename + def check_failure(self, result): + (label, (log, returncode)) = result + #return True if returncode !=0 else False + return True \ No newline at end of file From af36cfd89f627dbf5ca851cd8a5e2c97a82b68d6 Mon Sep 17 00:00:00 2001 From: Tushar Gupta Date: Thu, 5 Oct 2017 22:44:54 -0400 Subject: [PATCH 12/32] Function to detect Participant analysis failure and abort the Group analysis --- sim/SparkBIDS.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sim/SparkBIDS.py b/sim/SparkBIDS.py index a8616f4..4b8f143 100755 --- a/sim/SparkBIDS.py +++ b/sim/SparkBIDS.py @@ -205,5 +205,4 @@ def get_participant_from_fn(self,filename): return filename def check_failure(self, result): (label, (log, returncode)) = result - #return True if returncode !=0 else False - return True \ No newline at end of file + return True if returncode !=0 else False \ No newline at end of file From 3ba2929036f93e98ff2440c1442921eed37cc8f7 Mon Sep 17 00:00:00 2001 From: Tristan Glatard Date: Sat, 7 Oct 2017 15:05:12 -0400 Subject: [PATCH 13/32] Fixed bug in error detection for group analysis --- sim/SparkBIDS.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sim/SparkBIDS.py b/sim/SparkBIDS.py index 4fa7722..eff50fe 100755 --- a/sim/SparkBIDS.py +++ b/sim/SparkBIDS.py @@ -106,7 +106,7 @@ def pretty_print(self, result): filename = "{0}.{1}.log".format(timestamp, label) with open(filename,"w") as f: f.write(log) - print(" [ {3} ({0}) ] {1} - {2}".format(returncode, label, filename, status)) + print(" [ {0} ({1}) ] {2} - {3}".format(status, returncode, label, filename)) def write_invocation_file(self, analysis_level, participant_label, invocation_file): @@ -197,5 +197,5 @@ def get_participant_from_fn(self,filename): if filename.endswith(".tar"): return filename.split('-')[-1][:-4] return filename def check_failure(self, result): - (label, (log, returncode)) = result - return True if returncode !=0 else False \ No newline at end of file + (label, (returncode, log)) = result + return True if returncode !=0 else False From 56500d38d2521f8fe6b51d48f40790781bc00cf0 Mon Sep 17 00:00:00 2001 From: Valerie Hayot-Sasson Date: Sat, 7 Oct 2017 20:32:52 -0400 Subject: [PATCH 14/32] placed general functions in a separate class (Sim) --- sim/Sim.py | 71 ++++++++++++++++++++++++++++++++++++++++++++++++ sim/SparkBIDS.py | 67 ++++----------------------------------------- 2 files changed, 77 insertions(+), 61 deletions(-) create mode 100644 sim/Sim.py diff --git a/sim/Sim.py b/sim/Sim.py new file mode 100644 index 0000000..68abf8f --- /dev/null +++ b/sim/Sim.py @@ -0,0 +1,71 @@ +import boutiques, os, time, errno, tarfile, json + +class Sim(object): + + def __init__(self, boutiques_descriptor, input_path, output_dir): + + self.boutiques_descriptor = boutiques_descriptor + self.input_path = input_path + self.output_dir = output_dir + + def create_tar_file(self, out_dir, tar_name, files): + try: + os.makedirs(out_dir) + except OSError as e: + if e.errno != errno.EEXIST: + raise + with tarfile.open(os.path.join(out_dir, tar_name), "w") as tar: + for f in files: + tar.add(f) + + def write_invocation_file(self, analysis_level, participant_label, invocation_file): + + # Note: the invocation file format will change soon + + # Creates invocation object + invocation = {} + invocation["bids_dir"] = self.input_path + invocation["output_dir_name"] = self.output_dir + if analysis_level == "participant": + invocation["analysis_level"] = "participant" + invocation["participant_label"] = participant_label + elif analysis_level == "group": + invocation["analysis_level"] = "group" + + json_invocation = json.dumps(invocation) + + # Writes invocation + with open(invocation_file,"w") as f: + f.write(json_invocation) + + def bosh_exec(self, invocation_file, mount=None): + try: + + if mount is None: + boutiques.execute("launch",self.boutiques_descriptor,invocation_file, "-x") + else: + boutiques.execute("launch", self.boutiques_descriptor, invocation_file, "-v", + "{0}:{0}".format(mount), "-x") + result = 0 + except SystemExit as e: + result = e.code + return (result, "Empty log, Boutiques API doesn't return it yet.\n") + + def pretty_print(self, result): + (label, (returncode, log)) = result + status = "SUCCESS" if returncode == 0 else "ERROR" + timestamp = str(int(time.time() * 1000)) + filename = "{0}.{1}.log".format(timestamp, label) + with open(filename,"w") as f: + f.write(log) + print(" [ {0} ({1}) ] {2} - {3}".format(status, returncode, label, filename)) + + def check_failure(self, result): + (label, (returncode, log)) = result + return True if returncode !=0 else False + + def is_valid_file(parser, arg): + if not os.path.exists(arg): + parser.error("The file %s does not exist!" % arg) + else: + return open(arg, 'r') diff --git a/sim/SparkBIDS.py b/sim/SparkBIDS.py index eff50fe..7428f9a 100755 --- a/sim/SparkBIDS.py +++ b/sim/SparkBIDS.py @@ -1,13 +1,13 @@ from bids.grabbids import BIDSLayout import boutiques, errno, json, os, shutil, subprocess, tarfile, time +from Sim import Sim -class SparkBIDS(object): +class SparkBIDS(Sim): def __init__(self, boutiques_descriptor, bids_dataset, output_dir, options={}): - self.boutiques_descriptor = os.path.join(os.path.abspath(boutiques_descriptor)) - self.bids_dataset = bids_dataset - self.output_dir = output_dir + super(SparkBIDS, self).__init__(os.path.abspath(boutiques_descriptor), bids_dataset, output_dir) + # Includes: use_hdfs, skip_participant_analysis, # skip_group_analysis, skip_participants_file @@ -71,7 +71,7 @@ def create_RDD(self, sc): sub_dir="tar_files" - layout = BIDSLayout(self.bids_dataset) + layout = BIDSLayout(self.input_path) participants = layout.get_subjects() # Create RDD of file paths as key and tarred subject data as value @@ -89,45 +89,6 @@ def create_RDD(self, sc): return sc.parallelize(list_participants) - def create_tar_file(self, out_dir, tar_name, files): - try: - os.makedirs(out_dir) - except OSError as e: - if e.errno != errno.EEXIST: - raise - with tarfile.open(os.path.join(out_dir, tar_name), "w") as tar: - for f in files: - tar.add(f) - - def pretty_print(self, result): - (label, (returncode, log)) = result - status = "SUCCESS" if returncode == 0 else "ERROR" - timestamp = str(int(time.time() * 1000)) - filename = "{0}.{1}.log".format(timestamp, label) - with open(filename,"w") as f: - f.write(log) - print(" [ {0} ({1}) ] {2} - {3}".format(status, returncode, label, filename)) - - def write_invocation_file(self, analysis_level, participant_label, invocation_file): - - # Note: the invocation file format will change soon - - # Creates invocation object - invocation = {} - invocation["bids_dir"] = self.bids_dataset - invocation["output_dir_name"] = self.output_dir - if analysis_level == "participant": - invocation["analysis_level"] = "participant" - invocation["participant_label"] = participant_label - elif analysis_level == "group": - invocation["analysis_level"] = "group" - - json_invocation = json.dumps(invocation) - - # Writes invocation - with open(invocation_file,"w") as f: - f.write(json_invocation) - def get_bids_dataset(self, data, participant_label): filename = 'sub-{0}.tar'.format(participant_label) @@ -145,7 +106,7 @@ def get_bids_dataset(self, data, participant_label): os.remove(filename) - return os.path.join(tmp_dataset, os.path.abspath(self.bids_dataset)) + return os.path.join(tmp_dataset, os.path.abspath(self.input_path)) def run_participant_analysis(self, participant_label, data): @@ -179,23 +140,7 @@ def run_group_analysis(self): os.remove(invocation_file) return ("group", exec_result) - def bosh_exec(self, invocation_file): - try: - boutiques.execute("launch",self.boutiques_descriptor,invocation_file, "-x") - result = 0 - except SystemExit as e: - result = e.code - return (result, "Empty log, Boutiques API doesn't return it yet.\n") - - def is_valid_file(parser, arg): - if not os.path.exists(arg): - parser.error("The file %s does not exist!" % arg) - else: - return open(arg, 'r') def get_participant_from_fn(self,filename): if filename.endswith(".tar"): return filename.split('-')[-1][:-4] return filename - def check_failure(self, result): - (label, (returncode, log)) = result - return True if returncode !=0 else False From 418b423870e7e5c761c0ae7dd0ba4b7becb4652a Mon Sep 17 00:00:00 2001 From: Valerie Hayot-Sasson Date: Fri, 13 Oct 2017 20:15:27 -0400 Subject: [PATCH 15/32] some cleanup --- sim/Sim.py | 31 ++++++++++++++----------------- sim/SparkBIDS.py | 8 ++++---- 2 files changed, 18 insertions(+), 21 deletions(-) diff --git a/sim/Sim.py b/sim/Sim.py index 68abf8f..239426b 100644 --- a/sim/Sim.py +++ b/sim/Sim.py @@ -18,20 +18,9 @@ def create_tar_file(self, out_dir, tar_name, files): for f in files: tar.add(f) - def write_invocation_file(self, analysis_level, participant_label, invocation_file): + def write_invocation_file(self, invocation, invocation_file): # Note: the invocation file format will change soon - - # Creates invocation object - invocation = {} - invocation["bids_dir"] = self.input_path - invocation["output_dir_name"] = self.output_dir - if analysis_level == "participant": - invocation["analysis_level"] = "participant" - invocation["participant_label"] = participant_label - elif analysis_level == "group": - invocation["analysis_level"] = "group" - json_invocation = json.dumps(invocation) # Writes invocation @@ -64,8 +53,16 @@ def check_failure(self, result): (label, (returncode, log)) = result return True if returncode !=0 else False - def is_valid_file(parser, arg): - if not os.path.exists(arg): - parser.error("The file %s does not exist!" % arg) - else: - return open(arg, 'r') + + def write_BIDS_invocation(self, analysis_level, participant_label, invocation_file): + + invocation = {} + invocation["bids_dir"] = self.input_path + invocation["output_dir_name"] = self.output_dir + if analysis_level == "participant": + invocation["analysis_level"] = "participant" + invocation["participant_label"] = participant_label + elif analysis_level == "group": + invocation["analysis_level"] = "group" + + self.write_invocation_file(invocation, invocation_file) diff --git a/sim/SparkBIDS.py b/sim/SparkBIDS.py index 7428f9a..8817f7a 100755 --- a/sim/SparkBIDS.py +++ b/sim/SparkBIDS.py @@ -123,20 +123,20 @@ def run_participant_analysis(self, participant_label, data): raise invocation_file = "./invocation-{0}.json".format(participant_label) - self.write_invocation_file("participant", + self.write_BIDS_invocation("participant", participant_label, invocation_file) - exec_result = self.bosh_exec(invocation_file) + exec_result = self.bosh_exec(invocation_file, os.path.dirname(os.path.abspath(self.input_path))) os.remove(invocation_file) return (participant_label, exec_result) def run_group_analysis(self): invocation_file = "./invocation-group.json" - self.write_invocation_file("group", + self.write_BIDS_invocation("group", None, invocation_file) - exec_result = self.bosh_exec(invocation_file) + exec_result = self.bosh_exec(invocation_file, os.path.dirname(os.path.abspath(self.input_path))) os.remove(invocation_file) return ("group", exec_result) From 33e932e5a2063dac7e4ebad76f40e871d163b18d Mon Sep 17 00:00:00 2001 From: Valerie Hayot-Sasson Date: Fri, 13 Oct 2017 21:22:58 -0400 Subject: [PATCH 16/32] testing out travis ci --- .travis.yml | 31 +++++++++++++++++++++++++++++++ circle.yml | 21 --------------------- 2 files changed, 31 insertions(+), 21 deletions(-) create mode 100644 .travis.yml delete mode 100644 circle.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..a0fb457 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,31 @@ +language: python + +sudo: required + +services: +- docker +- privileged: true + +python: +- 2.7 +- 3.5 +- 3.6 +- nightly + +before_install: +- docker build -t simtool . +- docker run --privileged --rm=false -v /var/run/docker.sock:/var/run/docker.sock -v $PWD:$PWD -w $PWD simtool ./test + +install: +- pip install coveralls pytest pytest-runner +- pip install ./sim + +script: +- cd ./sim +- coverage run --source boutiques setup.py pytest + +after_success: +- coveralls + +deploy: + diff --git a/circle.yml b/circle.yml deleted file mode 100644 index bee873f..0000000 --- a/circle.yml +++ /dev/null @@ -1,21 +0,0 @@ -machine: - pre: - - sudo curl -L -o /usr/bin/docker 'https://s3-external-1.amazonaws.com/circle-downloads/docker-1.9.1-circleci' - - sudo chmod 0755 /usr/bin/docker - - services: - - docker - - privileged: true - -dependencies: - cache_directories: - - "~/docker" - - override: - - docker build -t simtool . : - timeout: 21600 - - mkdir -p ~/docker; docker save "simtool" > ~/docker/image.tar - -test: - override: - - docker run --privileged --rm=false -v /var/run/docker.sock:/var/run/docker.sock -v $PWD:$PWD -w $PWD simtool ./test From f80a7e8ce98f5e5312db90288457d677c6042fee Mon Sep 17 00:00:00 2001 From: Valerie Hayot-Sasson Date: Fri, 13 Oct 2017 21:53:43 -0400 Subject: [PATCH 17/32] travis testing --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9f4d50c..3056148 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM mesosphere/spark:2.0.0-2.2.0-1-hadoop-2.6 -RUN apt-get -y install docker.io \ +RUN sudo apt-get update && apt-get -y upgrade && apt-get -y install docker.io \ python-setuptools && \ easy_install pip From f151dfbef3e76c83823e0c0a542bbbc69e6a8ea7 Mon Sep 17 00:00:00 2001 From: Valerie Hayot-Sasson Date: Fri, 13 Oct 2017 21:59:38 -0400 Subject: [PATCH 18/32] fixed .travis.yml --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a0fb457..99b227f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ python: before_install: - docker build -t simtool . -- docker run --privileged --rm=false -v /var/run/docker.sock:/var/run/docker.sock -v $PWD:$PWD -w $PWD simtool ./test +- docker run --privileged --rm=false -v /var/run/docker.sock:/var/run/docker.sock -v $PWD:$PWD -w $PWD simtool ./sim/tests install: - pip install coveralls pytest pytest-runner From 32b57a741a9e5d5a2f29d8a5914f21720b64b323 Mon Sep 17 00:00:00 2001 From: Valerie Hayot-Sasson Date: Sat, 14 Oct 2017 08:45:03 -0400 Subject: [PATCH 19/32] debugging travis failure --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 99b227f..7e4fa83 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,14 +14,14 @@ python: before_install: - docker build -t simtool . -- docker run --privileged --rm=false -v /var/run/docker.sock:/var/run/docker.sock -v $PWD:$PWD -w $PWD simtool ./sim/tests +- docker run --privileged --rm=false -v /var/run/docker.sock:/var/run/docker.sock -v $PWD:$PWD -w $PWD simtool ./test install: - pip install coveralls pytest pytest-runner - pip install ./sim script: -- cd ./sim +- spark_bids.py ./test/demo/bids-app-example.json ./test/demo/ds001 output - coverage run --source boutiques setup.py pytest after_success: From 05dfe48940554fc5fd8ba636cd925f568bdd699d Mon Sep 17 00:00:00 2001 From: Valerie Hayot-Sasson Date: Sat, 14 Oct 2017 08:51:31 -0400 Subject: [PATCH 20/32] debugging travis failure --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7e4fa83..233d84e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ python: before_install: - docker build -t simtool . -- docker run --privileged --rm=false -v /var/run/docker.sock:/var/run/docker.sock -v $PWD:$PWD -w $PWD simtool ./test +- docker run --privileged --rm=false -v /var/run/docker.sock:/var/run/docker.sock -v $PWD:$PWD -w $PWD simtool ./sim/tests install: - pip install coveralls pytest pytest-runner From 21974dfb809c2cbb55e5dd6120368586f1fffca6 Mon Sep 17 00:00:00 2001 From: Valerie Hayot-Sasson Date: Sat, 14 Oct 2017 13:25:23 -0400 Subject: [PATCH 21/32] debugging travis failure --- .travis.yml | 2 +- Dockerfile | 20 ++++++++++++++------ sim/SparkBIDS.py | 2 ++ sim/spark_bids.py | 2 +- sim/tests/test_all.py | 12 +++++++----- 5 files changed, 25 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index 233d84e..f42c9b6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ python: before_install: - docker build -t simtool . -- docker run --privileged --rm=false -v /var/run/docker.sock:/var/run/docker.sock -v $PWD:$PWD -w $PWD simtool ./sim/tests +- docker run --privileged --rm=false -v /var/run/docker.sock:/var/run/docker.sock -v $PWD:$PWD -w $PWD/sim simtool ./tests install: - pip install coveralls pytest pytest-runner diff --git a/Dockerfile b/Dockerfile index 3056148..8a1802d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,19 @@ -FROM mesosphere/spark:2.0.0-2.2.0-1-hadoop-2.6 +FROM jupyter/pyspark-notebook:281505737f8a -RUN sudo apt-get update && apt-get -y upgrade && apt-get -y install docker.io \ - python-setuptools && \ - easy_install pip +USER root +RUN apt-get update && \ + echo 'Y' | apt-get install apt-utils && \ + echo 'Y' | apt-get install curl && \ + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - && \ + echo 'Y' | apt install --reinstall base-files lsb-release lsb-base && \ + echo 'Y' | apt-get install software-properties-common && \ + echo 'Y' | apt-get install apt-transport-https && \ + add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $( lsb_release -cs ) stable" && \ + apt-get update && \ + apt-get install -y docker-ce && \ + service docker start - -RUN pip install boutiques pytest pyspark pybids +RUN pip install boutiques pytest pyspark pybids duecredit ENTRYPOINT ["pytest"] diff --git a/sim/SparkBIDS.py b/sim/SparkBIDS.py index 8817f7a..719eb0c 100755 --- a/sim/SparkBIDS.py +++ b/sim/SparkBIDS.py @@ -20,6 +20,8 @@ def __init__(self, boutiques_descriptor, bids_dataset, output_dir, options={}): and not self.skip_group_analysis self.skipped_participants = self.skip_participants_file.read().split() if self.skip_participants_file else [] + print(self.skipped_participants) + # Print analysis summary print("Computed Analyses: Participant [ {0} ] - Group [ {1} ]".format(str(self.do_participant_analysis).upper(), str(self.do_group_analysis).upper())) diff --git a/sim/spark_bids.py b/sim/spark_bids.py index 3e11d01..7e956df 100755 --- a/sim/spark_bids.py +++ b/sim/spark_bids.py @@ -1,7 +1,7 @@ #!/usr/bin/env python from pyspark import SparkContext, SparkConf -from sim import SparkBIDS +from SparkBIDS import SparkBIDS import argparse, os def is_valid_file(parser, arg): diff --git a/sim/tests/test_all.py b/sim/tests/test_all.py index cd6da3e..cf758cd 100755 --- a/sim/tests/test_all.py +++ b/sim/tests/test_all.py @@ -15,6 +15,7 @@ def get_json_descriptor(self): return os.path.join(self.get_demo_dir(),"bids-app-example.json") def run_spark_bids(self,checkOutputGroup=True,options=[],correctBrainSize="830532",output_name=None): + os.system("service docker start;") millitime = int(time.time()*1000) if not output_name: output_name = "output"+str(random.SystemRandom().randint(0,int(millitime))) @@ -28,9 +29,10 @@ def run_spark_bids(self,checkOutputGroup=True,options=[],correctBrainSize="83053 try: stdout_string = subprocess.check_output(command, stderr=subprocess.STDOUT) - except: + except subprocess.CalledProcessError as e: + print(e.output.decode('utf8')) self.assertTrue(False,"Command-line execution failed {0}".format(str(command))) - self.assertTrue("ERROR" not in stdout_string) + self.assertTrue(bytes("ERROR", "utf8") not in stdout_string) if checkOutputGroup: assert(os.path.isfile(os.path.join(output_name,"avg_brain_size.txt"))) with open(os.path.join(output_name,"avg_brain_size.txt")) as f: @@ -53,9 +55,9 @@ def test_spark_bids_skip_participant(self): participant_file = "skip.txt" with open(participant_file,"w") as f: f.write("01") - self.run_spark_bids(options=["--skip-participants","skip.txt"],correctBrainSize="865472") + self.run_spark_bids(options=["--skip-participants", os.path.join(os.path.dirname(__file__),"skip.txt")],correctBrainSize="865472") - def test_spark_bids_hdfs(self): - self.run_spark_bids(options=["--hdfs"]) + #def test_spark_bids_hdfs(self): + # self.run_spark_bids(options=["--hdfs"]) From 7ebda454218a9f9c8a2502947346a319ebeecf74 Mon Sep 17 00:00:00 2001 From: Valerie Hayot-Sasson Date: Sat, 14 Oct 2017 16:59:45 -0400 Subject: [PATCH 22/32] forced python2.7 --- .travis.yml | 11 ----------- Dockerfile | 16 ++++++++++++++-- sim/tests/test_all.py | 19 +++++++++++++------ 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/.travis.yml b/.travis.yml index f42c9b6..979bcce 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,22 +8,11 @@ services: python: - 2.7 -- 3.5 -- 3.6 -- nightly before_install: - docker build -t simtool . - docker run --privileged --rm=false -v /var/run/docker.sock:/var/run/docker.sock -v $PWD:$PWD -w $PWD/sim simtool ./tests -install: -- pip install coveralls pytest pytest-runner -- pip install ./sim - -script: -- spark_bids.py ./test/demo/bids-app-example.json ./test/demo/ds001 output -- coverage run --source boutiques setup.py pytest - after_success: - coveralls diff --git a/Dockerfile b/Dockerfile index 8a1802d..0d678fd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM jupyter/pyspark-notebook:281505737f8a +FROM jupyter/pyspark-notebook:82b978b3ceeb USER root @@ -14,6 +14,18 @@ RUN apt-get update && \ apt-get install -y docker-ce && \ service docker start -RUN pip install boutiques pytest pyspark pybids duecredit + +#RUN echo "alias python='/usr/bin/python2.7'" >> ~/.bashrc && . ~/.bashrc + +#RUN pip install boutiques pytest pyspark pybids duecredit + +RUN conda create -n simenv python=2.7 pytest pyspark + +ENV PATH /opt/conda/envs/simenv/bin:$PATH + +RUN /bin/bash -c "source activate simenv" + +RUN pip install boutiques pybids duecredit +#RUN pip2 install boutiques pytest pyspark pybids duecredit ENTRYPOINT ["pytest"] diff --git a/sim/tests/test_all.py b/sim/tests/test_all.py index cf758cd..0c32068 100755 --- a/sim/tests/test_all.py +++ b/sim/tests/test_all.py @@ -5,6 +5,14 @@ class TestSim(TestCase): ## UTILITY METHODS + def module_exists(module): + try: + __import__(module) + except: + return False + else: + return True + def get_sim_dir(self): return os.path.join(os.path.dirname(__file__),"..") @@ -15,7 +23,6 @@ def get_json_descriptor(self): return os.path.join(self.get_demo_dir(),"bids-app-example.json") def run_spark_bids(self,checkOutputGroup=True,options=[],correctBrainSize="830532",output_name=None): - os.system("service docker start;") millitime = int(time.time()*1000) if not output_name: output_name = "output"+str(random.SystemRandom().randint(0,int(millitime))) @@ -30,9 +37,8 @@ def run_spark_bids(self,checkOutputGroup=True,options=[],correctBrainSize="83053 stdout_string = subprocess.check_output(command, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: - print(e.output.decode('utf8')) self.assertTrue(False,"Command-line execution failed {0}".format(str(command))) - self.assertTrue(bytes("ERROR", "utf8") not in stdout_string) + self.assertTrue(bytes("ERROR") not in stdout_string) if checkOutputGroup: assert(os.path.isfile(os.path.join(output_name,"avg_brain_size.txt"))) with open(os.path.join(output_name,"avg_brain_size.txt")) as f: @@ -56,8 +62,9 @@ def test_spark_bids_skip_participant(self): with open(participant_file,"w") as f: f.write("01") self.run_spark_bids(options=["--skip-participants", os.path.join(os.path.dirname(__file__),"skip.txt")],correctBrainSize="865472") - - #def test_spark_bids_hdfs(self): - # self.run_spark_bids(options=["--hdfs"]) + + @pytest.mark.skipif(not module_exists("hdfs"), reason="HDFS not installed") + def test_spark_bids_hdfs(self): + self.run_spark_bids(options=["--hdfs"]) From 60c1641502c2d168b7c985a85199d0b7f7a9f2c9 Mon Sep 17 00:00:00 2001 From: Valerie Hayot-Sasson Date: Sat, 14 Oct 2017 17:14:46 -0400 Subject: [PATCH 23/32] fixed bug --- sim/tests/test_all.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sim/tests/test_all.py b/sim/tests/test_all.py index 0c32068..5e1c1d4 100755 --- a/sim/tests/test_all.py +++ b/sim/tests/test_all.py @@ -61,7 +61,7 @@ def test_spark_bids_skip_participant(self): participant_file = "skip.txt" with open(participant_file,"w") as f: f.write("01") - self.run_spark_bids(options=["--skip-participants", os.path.join(os.path.dirname(__file__),"skip.txt")],correctBrainSize="865472") + self.run_spark_bids(options=["--skip-participants", "skip.txt"],correctBrainSize="865472") @pytest.mark.skipif(not module_exists("hdfs"), reason="HDFS not installed") def test_spark_bids_hdfs(self): From e6330bc53afdea0fbf85eb98dbec0f0345d6e5df Mon Sep 17 00:00:00 2001 From: Valerie Hayot-Sasson Date: Sat, 14 Oct 2017 18:18:31 -0400 Subject: [PATCH 24/32] temporary install so travis wont complain --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 979bcce..55e22a2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,6 +13,9 @@ before_install: - docker build -t simtool . - docker run --privileged --rm=false -v /var/run/docker.sock:/var/run/docker.sock -v $PWD:$PWD -w $PWD/sim simtool ./tests +install: +- pip install ./sim + after_success: - coveralls From b27337f1976b8c3574fecb7396a1ab26ce013bae Mon Sep 17 00:00:00 2001 From: Valerie Hayot-Sasson Date: Sat, 14 Oct 2017 18:31:08 -0400 Subject: [PATCH 25/32] skip install --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 55e22a2..c6901d1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,8 +13,7 @@ before_install: - docker build -t simtool . - docker run --privileged --rm=false -v /var/run/docker.sock:/var/run/docker.sock -v $PWD:$PWD -w $PWD/sim simtool ./tests -install: -- pip install ./sim +install: true after_success: - coveralls From 43309956e515f97d7f0347725a1eea598e113372 Mon Sep 17 00:00:00 2001 From: Valerie Hayot-Sasson Date: Sat, 14 Oct 2017 18:45:31 -0400 Subject: [PATCH 26/32] changes to .travis.yml --- .travis.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index c6901d1..23e7625 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,12 +9,14 @@ services: python: - 2.7 -before_install: -- docker build -t simtool . -- docker run --privileged --rm=false -v /var/run/docker.sock:/var/run/docker.sock -v $PWD:$PWD -w $PWD/sim simtool ./tests - install: true +before_script: +- docker build -t simtool . + +script: +- docker run --privileged --rm=false -v /var/run/docker.sock:/var/run/docker.sock -v $PWD:$PWD -w $PWD/sim simtool ./tests + after_success: - coveralls From 30b71cc3d452c97ac358020c1b19abce1f641cdf Mon Sep 17 00:00:00 2001 From: Valerie Hayot-Sasson Date: Sat, 14 Oct 2017 19:06:56 -0400 Subject: [PATCH 27/32] removed commented code in Dockerfile --- Dockerfile | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0d678fd..74ff9af 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,11 +14,6 @@ RUN apt-get update && \ apt-get install -y docker-ce && \ service docker start - -#RUN echo "alias python='/usr/bin/python2.7'" >> ~/.bashrc && . ~/.bashrc - -#RUN pip install boutiques pytest pyspark pybids duecredit - RUN conda create -n simenv python=2.7 pytest pyspark ENV PATH /opt/conda/envs/simenv/bin:$PATH @@ -26,6 +21,5 @@ ENV PATH /opt/conda/envs/simenv/bin:$PATH RUN /bin/bash -c "source activate simenv" RUN pip install boutiques pybids duecredit -#RUN pip2 install boutiques pytest pyspark pybids duecredit ENTRYPOINT ["pytest"] From ea41fd2ac4dbd7926c7c31af478174465e3e43e4 Mon Sep 17 00:00:00 2001 From: Valerie Hayot-Sasson Date: Sun, 15 Oct 2017 11:59:53 -0400 Subject: [PATCH 28/32] added nipype example --- .travis.yml | 2 +- Dockerfile | 6 +- sim/other_wf_examples/nipype/NipBIDS.py | 139 ++++++++++++++++++ sim/other_wf_examples/nipype/nip_bids.py | 42 ++++++ .../nipype/tests/test_all.py | 58 ++++++++ sim/tests/execute_pytest.sh | 14 ++ 6 files changed, 257 insertions(+), 4 deletions(-) create mode 100644 sim/other_wf_examples/nipype/NipBIDS.py create mode 100755 sim/other_wf_examples/nipype/nip_bids.py create mode 100755 sim/other_wf_examples/nipype/tests/test_all.py create mode 100644 sim/tests/execute_pytest.sh diff --git a/.travis.yml b/.travis.yml index 23e7625..b3ae8ba 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,7 @@ before_script: - docker build -t simtool . script: -- docker run --privileged --rm=false -v /var/run/docker.sock:/var/run/docker.sock -v $PWD:$PWD -w $PWD/sim simtool ./tests +- docker run --privileged --rm=false -v /var/run/docker.sock:/var/run/docker.sock -v $PWD:$PWD -w $PWD/sim simtool ./tests/execute_pytest.sh after_success: - coveralls diff --git a/Dockerfile b/Dockerfile index 74ff9af..f1ccbb8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,12 +14,12 @@ RUN apt-get update && \ apt-get install -y docker-ce && \ service docker start -RUN conda create -n simenv python=2.7 pytest pyspark +RUN conda create -n simenv python=2.7 pytest py4j==0.10.4 pyspark pytest-cov ENV PATH /opt/conda/envs/simenv/bin:$PATH RUN /bin/bash -c "source activate simenv" -RUN pip install boutiques pybids duecredit +RUN pip install boutiques pybids duecredit nipype -ENTRYPOINT ["pytest"] +ENTRYPOINT ["/bin/bash"] diff --git a/sim/other_wf_examples/nipype/NipBIDS.py b/sim/other_wf_examples/nipype/NipBIDS.py new file mode 100644 index 0000000..2206208 --- /dev/null +++ b/sim/other_wf_examples/nipype/NipBIDS.py @@ -0,0 +1,139 @@ +from nipype import Workflow, MapNode, Node, Function +from nipype.interfaces.utility import IdentityInterface, Function +import os, json, time +from sim import Sim + +class NipBIDS(Sim.Sim): + + def __init__(self, boutiques_descriptor, bids_dataset, output_dir, options={}): + + super(NipBIDS, self).__init__(os.path.abspath(boutiques_descriptor), + os.path.abspath(bids_dataset), + os.path.abspath(output_dir)) + + # Includes: skip_participant_analysis, + # skip_group_analysis, skip_participants_file + for option in list(options.keys()): setattr(self, option, options.get(option)) + + # Check what will have to be done + self.do_participant_analysis = self.supports_analysis_level("participant") \ + and not self.skip_participant_analysis + self.do_group_analysis = self.supports_analysis_level("group") \ + and not self.skip_group_analysis + self.skipped_participants = open(self.skip_participants_file, "r").read().split() if self.skip_participants_file else [] + + + # Print analysis summary + print("Computed Analyses: Participant [ {0} ] - Group [ {1} ]".format(str(self.do_participant_analysis).upper(), + str(self.do_group_analysis).upper())) + + if len(self.skipped_participants): + print("Skipped participants: {0}".format(self.skipped_participants)) + + + def run(self): + + wf = Workflow('bapp') + wf.base_dir = os.getcwd() + + # group analysis can be executed if participant analysis is skipped + p_analysis = None + + # Participant analysis + if self.do_participant_analysis: + + participants = Node(Function(input_names=['nip'], + output_names=['out'], + function=get_participants), + name='get_participants') + participants.inputs.nip = self + + + + p_analysis = MapNode(Function(input_names=['nip', 'analysis_level', + 'participant_label','working_dir'], + output_names=['result'], + function=run_analysis), + iterfield=['participant_label'], + name='run_participant_analysis') + + wf.add_nodes([participants]) + wf.connect(participants, 'out', p_analysis, 'participant_label') + + p_analysis.inputs.analysis_level = 'participant' + p_analysis.inputs.nip = self + p_analysis.inputs.working_dir = os.getcwd() + + + # Group analysis + if self.do_group_analysis: + groups = Node(Function(input_names=['nip', 'analysis_level', + 'working_dir', 'dummy_token'], + output_names=['g_result'], + function=run_analysis), + name='run_group_analysis') + + groups.inputs.analysis_level = 'group' + groups.inputs.nip = self + groups.inputs.working_dir = os.getcwd() + + + if p_analysis is not None: + wf.connect(p_analysis, 'result', groups, 'dummy_token') + else: + wf.add_nodes([groups]) + + eg = wf.run() + + + # Convert to dictionary to more easily extract results + node_names = [i.name for i in eg.nodes()] + result_dict = dict(zip(node_names, eg.nodes())) + + if self.do_participant_analysis: + for res in result_dict['run_participant_analysis'].result.outputs.get('result'): + self.pretty_print(res) + + if self.do_group_analysis: + self.pretty_print(result_dict['run_group_analysis'].result.outputs.g_result) + + + def supports_analysis_level(self,level): + desc = json.load(open(self.boutiques_descriptor)) + analysis_level_input = None + for input in desc["inputs"]: + if input["id"] == "analysis_level": + analysis_level_input = input + break + assert(analysis_level_input),"BIDS app descriptor has no input with id 'analysis_level'" + assert(analysis_level_input.get("value-choices")),"Input 'analysis_level' of BIDS app descriptor has no 'value-choices' property" + return level in analysis_level_input["value-choices"] + +def run_analysis(nip, analysis_level, working_dir, participant_label=None, dummy_token=None): + import os + + out_key = None + + if analysis_level == "group": + invocation_file = "./invocation-group.json" + out_key = "group" + else: + invocation_file = "./invocation-{0}.json".format(participant_label) + out_key = "participant_label" + + nip.write_BIDS_invocation(analysis_level, participant_label, invocation_file) + exec_result = nip.bosh_exec(invocation_file, working_dir) + os.remove(invocation_file) + + return (out_key, exec_result) + + + +def get_participants(nip): + + from bids.grabbids import BIDSLayout + + layout = BIDSLayout(nip.input_path) + participants = layout.get_subjects() + + return list(set(participants) - set(nip.skipped_participants)) diff --git a/sim/other_wf_examples/nipype/nip_bids.py b/sim/other_wf_examples/nipype/nip_bids.py new file mode 100755 index 0000000..da3bbdc --- /dev/null +++ b/sim/other_wf_examples/nipype/nip_bids.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python + +from NipBIDS import NipBIDS +import argparse, os + +def is_valid_file(parser, arg): + if not os.path.exists(arg): + parser.error("The file %s does not exist!" % arg) + else: + return arg + +def main(): + + # Arguments parsing + parser=argparse.ArgumentParser() + + # Required inputs + parser.add_argument("bids_app_boutiques_descriptor", help="Boutiques descriptor of the BIDS App that will process the dataset.") + parser.add_argument("bids_dataset", help="BIDS dataset to be processed.") + parser.add_argument("output_dir", help="Output directory.") + + # Optional inputs + parser.add_argument("--skip-participant-analysis", action = 'store_true', help="Skips participant analysis.") + parser.add_argument("--skip-group-analysis", action = 'store_true', help="Skips groups analysis.") + parser.add_argument("--skip-participants", metavar="FILE", type=lambda x: is_valid_file(parser, x), help="Skips participant labels in the text file.") + args=parser.parse_args() + + nip_bids = NipBIDS(args.bids_app_boutiques_descriptor, + args.bids_dataset, + args.output_dir, + { 'skip_participant_analysis': args.skip_participant_analysis, + 'skip_group_analysis': args.skip_group_analysis, + 'skip_participants_file': args.skip_participants}) + + + + # Run! + nip_bids.run() + +# Execute program +if __name__ == "__main__": + main() diff --git a/sim/other_wf_examples/nipype/tests/test_all.py b/sim/other_wf_examples/nipype/tests/test_all.py new file mode 100755 index 0000000..7bd2d9d --- /dev/null +++ b/sim/other_wf_examples/nipype/tests/test_all.py @@ -0,0 +1,58 @@ +import os, pytest, random, subprocess, time +from unittest import TestCase +import boutiques + +class TestNip(TestCase): + + ## UTILITY METHODS + def get_nip_dir(self): + return os.path.join(os.path.dirname(__file__),"..") + + def get_demo_dir(self): + return os.path.join(os.path.dirname(__file__),"../../../tests/demo") + + def get_json_descriptor(self): + return os.path.join(self.get_demo_dir(),"bids-app-example.json") + + def run_nip_bids(self,checkOutputGroup=True,options=[],correctBrainSize="830532",output_name=None): + millitime = int(time.time()*1000) + if not output_name: + output_name = "output"+str(random.SystemRandom().randint(0,int(millitime))) + command = [os.path.join(self.get_nip_dir(), + "nip_bids.py"), + self.get_json_descriptor(), + os.path.join(self.get_demo_dir(),"ds001"), + output_name] + for option in options: + command.append(option) + try: + stdout_string = subprocess.check_output(command, + stderr=subprocess.STDOUT) + except: + self.assertTrue(False,"Command-line execution failed {0}".format(str(command))) + self.assertTrue("ERROR" not in stdout_string) + if checkOutputGroup: + assert(os.path.isfile(os.path.join(output_name,"avg_brain_size.txt"))) + with open(os.path.join(output_name,"avg_brain_size.txt")) as f: + output_content = f.read() + content = "Average brain size is {0} voxels".format(correctBrainSize) + self.assertTrue(output_content == content) + + ## TESTS + def test_demo_descriptor_valid(self): + self.assertFalse(boutiques.validate(self.get_json_descriptor(),"-b")) + + def test_nip_bids_no_option(self): + self.run_nip_bids() + + def test_nip_bids_separate_analyses(self): + self.run_nip_bids(options=["--skip-group-analysis"],checkOutputGroup=False,output_name="output") # just participant analysis + self.run_nip_bids(options=["--skip-participant-analysis"],output_name="output") # just the group analysis + + def test_nip_bids_skip_participant(self): + participant_file = "skip.txt" + with open(participant_file,"w") as f: + f.write("01") + self.run_nip_bids(options=["--skip-participants","skip.txt"],correctBrainSize="865472") + + diff --git a/sim/tests/execute_pytest.sh b/sim/tests/execute_pytest.sh new file mode 100644 index 0000000..99b7d3a --- /dev/null +++ b/sim/tests/execute_pytest.sh @@ -0,0 +1,14 @@ +# !/bin/bash + +export PYTHONPATH="/opt/conda/envs/python2/lib/python2.7/site-packages:/usr/local/spark-2.2.0-bin-hadoop2.7/python:/opt/conda/envs/python2/bin/" + +# install sim +pip install -e ../ + + +# sparkBIDS test +pytest --cov=./ tests + +# nipBIDS test +pytest other_wf_examples/nipype/tests + From e09da521835362cb0b21a131d695013b0d03d3d1 Mon Sep 17 00:00:00 2001 From: ValHayot Date: Sun, 15 Oct 2017 13:26:04 -0400 Subject: [PATCH 29/32] Create README.md --- sim/other_wf_examples/nipype/README.md | 34 ++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 sim/other_wf_examples/nipype/README.md diff --git a/sim/other_wf_examples/nipype/README.md b/sim/other_wf_examples/nipype/README.md new file mode 100644 index 0000000..a5b0e40 --- /dev/null +++ b/sim/other_wf_examples/nipype/README.md @@ -0,0 +1,34 @@ +# NipBIDS + +An example of a nipype workflow executing sim functions / A demonstration of how to rewrite Spark BIDS in nipype + +## Additional dependencies +* `pip install nipype` + +## Demo +``` +nip_bids ./sim/tests/demo/bids-app-example.json ./sim/tests/demo/ds001 output +``` + +It should produce the following output: +``` +Computed Analyses: Subject [ YES ] - Group [ YES ] + [ SUCCESS ] sub-01 + [ SUCCESS ] sub-02 + [ SUCCESS ] group +``` + +A directory named `output` should also be created with the following content: +``` +avg_brain_size.txt sub-01_brain.nii.gz sub-02_brain.nii.gz +``` + +The content of `avg_brain_size.txt` should be: +``` +Average brain size is 830532 voxels +``` + + + + + From 1d8f7865d342926a97b9428a87d5aa0fea943c12 Mon Sep 17 00:00:00 2001 From: Valerie Hayot-Sasson Date: Sun, 15 Oct 2017 16:15:07 -0400 Subject: [PATCH 30/32] cleaned up code --- Dockerfile | 4 ++++ sim/tests/execute_pytest.sh | 3 +-- sim/tests/test_all.py | 9 +-------- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index f1ccbb8..f7b5bd6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,6 +20,10 @@ ENV PATH /opt/conda/envs/simenv/bin:$PATH RUN /bin/bash -c "source activate simenv" +ENV PYTHONPATH /opt/conda/envs/python2/lib/python2.7/site-packages:\ + /usr/local/spark-2.2.0-bin-hadoop2.7/python:\ + /opt/conda/envs/python2/bin:$PYTHONPATH + RUN pip install boutiques pybids duecredit nipype ENTRYPOINT ["/bin/bash"] diff --git a/sim/tests/execute_pytest.sh b/sim/tests/execute_pytest.sh index 99b7d3a..535512b 100644 --- a/sim/tests/execute_pytest.sh +++ b/sim/tests/execute_pytest.sh @@ -1,7 +1,5 @@ # !/bin/bash -export PYTHONPATH="/opt/conda/envs/python2/lib/python2.7/site-packages:/usr/local/spark-2.2.0-bin-hadoop2.7/python:/opt/conda/envs/python2/bin/" - # install sim pip install -e ../ @@ -9,6 +7,7 @@ pip install -e ../ # sparkBIDS test pytest --cov=./ tests +echo $PYTHONPATH # nipBIDS test pytest other_wf_examples/nipype/tests diff --git a/sim/tests/test_all.py b/sim/tests/test_all.py index 5e1c1d4..e9e2f23 100755 --- a/sim/tests/test_all.py +++ b/sim/tests/test_all.py @@ -5,13 +5,6 @@ class TestSim(TestCase): ## UTILITY METHODS - def module_exists(module): - try: - __import__(module) - except: - return False - else: - return True def get_sim_dir(self): return os.path.join(os.path.dirname(__file__),"..") @@ -63,8 +56,8 @@ def test_spark_bids_skip_participant(self): f.write("01") self.run_spark_bids(options=["--skip-participants", "skip.txt"],correctBrainSize="865472") - @pytest.mark.skipif(not module_exists("hdfs"), reason="HDFS not installed") def test_spark_bids_hdfs(self): + pytest.importorskip("hdfs") self.run_spark_bids(options=["--hdfs"]) From 45fbe13516eddf6d71b9660e6a8655bf19b94785 Mon Sep 17 00:00:00 2001 From: Tristan Glatard Date: Sun, 15 Oct 2017 19:49:30 -0400 Subject: [PATCH 31/32] Updated badge to Travis --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 980d018..05fba52 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ [![PyPI](https://img.shields.io/pypi/v/simtools.svg)](https://pypi.python.org/pypi/simtools) -[![CircleCI](https://circleci.com/gh/big-data-lab-team/sim/tree/master.svg?style=svg)](https://circleci.com/gh/big-data-lab-team/sim/tree/master) +[![Build Status](https://travis-ci.org/big-data-lab-team/sim.svg?branch=master)](https://travis-ci.org/big-data-lab-team/sim) # Spark for neuroIMaging (sim) From 648e935e03e24ece5db11f590443f6d3643524ac Mon Sep 17 00:00:00 2001 From: Tristan Glatard Date: Sun, 15 Oct 2017 20:29:17 -0400 Subject: [PATCH 32/32] Added coveralls badge --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 05fba52..ca1c7d0 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ [![PyPI](https://img.shields.io/pypi/v/simtools.svg)](https://pypi.python.org/pypi/simtools) [![Build Status](https://travis-ci.org/big-data-lab-team/sim.svg?branch=master)](https://travis-ci.org/big-data-lab-team/sim) +[![Coverage Status](https://coveralls.io/repos/github/big-data-lab-team/sim/badge.svg?branch=master)](https://coveralls.io/github/big-data-lab-team/sim?branch=master) # Spark for neuroIMaging (sim)