From 68c3cbdbcce34aa2241a68c3bf20abb325ddd137 Mon Sep 17 00:00:00 2001 From: Alyssa Morrow Date: Tue, 8 Oct 2019 11:29:32 -0700 Subject: [PATCH 01/10] added receipe for bdgenomics.mang --- recipes/mango/build.sh | 14 ++++++++++++++ recipes/mango/meta.yaml | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 recipes/mango/build.sh create mode 100644 recipes/mango/meta.yaml diff --git a/recipes/mango/build.sh b/recipes/mango/build.sh new file mode 100644 index 0000000000000..7b2e56cc41db0 --- /dev/null +++ b/recipes/mango/build.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +BINARY_HOME=$PREFIX/bin +PACKAGE_HOME=$PREFIX/share/$PKG_NAME-$PKG_VERSION-$PKG_BUILDNUM + +mkdir -p $BINARY_HOME +mkdir -p $PACKAGE_HOME + +cp -R * $PACKAGE_HOME + +ln -s $PACKAGE_HOME/bin/mango-notebook $BINARY_HOME +ln -s $PACKAGE_HOME/bin/mango-submit $BINARY_HOME +ln -s $PACKAGE_HOME/bin/make_genome $BINARY_HOME + diff --git a/recipes/mango/meta.yaml b/recipes/mango/meta.yaml new file mode 100644 index 0000000000000..538bb8db79333 --- /dev/null +++ b/recipes/mango/meta.yaml @@ -0,0 +1,38 @@ +{% set version="0.0.5" %} + +package: + name: mango + version: {{ version }} + +source: + url: https://search.maven.org/remotecontent?filepath=org/bdgenomics/mango/mango-distribution/{{ version }}/mango-distribution-{{ version }}-bin.tar.gz + md5: f438bdbe93e855db1dae077fa28ff3f7 + +build: + number: 0 + noarch: generic + +requirements: + host: + - python >3 + - pip + run: + - python + - bdgenomics.mango =={{ version }} + - bdgenomics.mango.pileup =={{ version }} + - openjdk >=8,<9 + - pyspark + +test: + # Python imports + imports: + - bdgenomics.mango + - bdgenomics.mango.pileup + commands: + - mango-submit --help + - mango-notebook --help + - make_genome --help +about: + home: https://github.com/bigdatagenomics/mango + license: Apache 2 + summary: Scalable genomics visualization tool built on Apache Spark From ac7a8e75fc6ace1cd6527795503b3e9862c0ad80 Mon Sep 17 00:00:00 2001 From: Alyssa Morrow Date: Tue, 8 Oct 2019 14:32:59 -0700 Subject: [PATCH 02/10] updated mango recipe --- recipes/mango/meta.yaml | 55 ++++++++++++++++++++++++++++++----------- 1 file changed, 41 insertions(+), 14 deletions(-) diff --git a/recipes/mango/meta.yaml b/recipes/mango/meta.yaml index 538bb8db79333..fe6e8a54d3836 100644 --- a/recipes/mango/meta.yaml +++ b/recipes/mango/meta.yaml @@ -1,38 +1,65 @@ -{% set version="0.0.5" %} +{% set name = "bdgenomics.mango" %} +{% set version = "0.0.5" %} package: - name: mango - version: {{ version }} + name: "mango" + version: "{{ version }}" source: - url: https://search.maven.org/remotecontent?filepath=org/bdgenomics/mango/mango-distribution/{{ version }}/mango-distribution-{{ version }}-bin.tar.gz - md5: f438bdbe93e855db1dae077fa28ff3f7 + - url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name }}-{{ version }}.tar.gz + sha256: 12dc1b334597481b2caa3988417cb6742569585fd1f056c02adfce2b1f208467 + - url: https://search.maven.org/remotecontent?filepath=org/bdgenomics/mango/mango-distribution/{{ version }}/mango-distribution-{{ version }}-bin.tar.gz + md5: f438bdbe93e855db1dae077fa28ff3f7 + build: - number: 0 noarch: generic + number: 0 + script: "{{ PYTHON }} -m pip install . --no-deps --ignore-installed -vv " requirements: host: - - python >3 + - bdgenomics.adam ==0.30.0a0 + - bdgenomics.mango.pileup ==0.0.4 + - cigar ==0.1.3 + - ipython ==7.8.0 + - ipywidgets ==7.0.0 + - jupyter ==1.0.0 + - matplotlib ==2.0.2 - pip - run: + - psutil - python - - bdgenomics.mango =={{ version }} - - bdgenomics.mango.pileup =={{ version }} + run: - openjdk >=8,<9 - pyspark + - bdgenomics.adam ==0.30.0a0 + - bdgenomics.mango.pileup ==0.0.4 + - cigar ==0.1.3 + - ipython ==7.8.0 + - ipywidgets ==7.0.0 + - jupyter ==1.0.0 + - matplotlib ==2.0.2 + - psutil + - python test: - # Python imports imports: + - bdgenomics - bdgenomics.mango + - bdgenomics.mango.test - bdgenomics.mango.pileup commands: - mango-submit --help - mango-notebook --help - make_genome --help + about: - home: https://github.com/bigdatagenomics/mango - license: Apache 2 - summary: Scalable genomics visualization tool built on Apache Spark + home: "https://github.com/bdgenomics/mango" + license: "Apache 2" + summary: "A scalable genomic visualization tool" + doc_url: "https://bdg-mango.readthedocs.io/en/latest/" + dev_url: "https://github.com/bigdatagenomics/mango" + +extra: + recipe-maintainers: + - akmorrow13 From 6087fe9a10b5f593a300171e5de821d3540dd8fd Mon Sep 17 00:00:00 2001 From: Devon Ryan Date: Wed, 9 Oct 2019 10:20:07 +0200 Subject: [PATCH 03/10] Update meta.yaml --- recipes/mango/meta.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/recipes/mango/meta.yaml b/recipes/mango/meta.yaml index 538bb8db79333..3f00f220935b5 100644 --- a/recipes/mango/meta.yaml +++ b/recipes/mango/meta.yaml @@ -13,9 +13,6 @@ build: noarch: generic requirements: - host: - - python >3 - - pip run: - python - bdgenomics.mango =={{ version }} From 5d20ead6478958d56ece504381f6234e98219d05 Mon Sep 17 00:00:00 2001 From: Alyssa Morrow Date: Wed, 16 Oct 2019 12:00:17 -0700 Subject: [PATCH 04/10] update --- recipes/mango/build.sh | 23 ++++++++++++- recipes/mango/meta.yaml | 73 ++++++++++++++++++++++++++++------------- recipes/ray/build.sh | 11 ------- recipes/ray/meta.yaml | 36 -------------------- 4 files changed, 73 insertions(+), 70 deletions(-) delete mode 100644 recipes/ray/build.sh delete mode 100644 recipes/ray/meta.yaml diff --git a/recipes/mango/build.sh b/recipes/mango/build.sh index 7b2e56cc41db0..e934f3992c7fd 100644 --- a/recipes/mango/build.sh +++ b/recipes/mango/build.sh @@ -6,9 +6,30 @@ PACKAGE_HOME=$PREFIX/share/$PKG_NAME-$PKG_VERSION-$PKG_BUILDNUM mkdir -p $BINARY_HOME mkdir -p $PACKAGE_HOME -cp -R * $PACKAGE_HOME +ls $PACKAGE_HOME +cp -R core/* $PACKAGE_HOME ln -s $PACKAGE_HOME/bin/mango-notebook $BINARY_HOME ln -s $PACKAGE_HOME/bin/mango-submit $BINARY_HOME ln -s $PACKAGE_HOME/bin/make_genome $BINARY_HOME +echo "PYTHON VERSION----" +$PYTHON --version +echo $PY_VER +which $PYTHON +which python3.7 + +# modin +$PYTHON -m pip install python_modin/.[dask] --no-deps --ignore-installed -vv + +# adam +$PYTHON -m pip install python_adam/. --no-deps --ignore-installed -vv + +# mangos +$PYTHON -m pip install python/. --no-deps --ignore-installed -vv + +$PYTHON -m pip install python_pileup/. --no-deps --ignore-installed -vv +# # Enable widget +# jupyter nbextension enable --py widgetsnbextension +# jupyter nbextension install --py --symlink --user bdgenomics.mango.pileup +# jupyter nbextension enable bdgenomics.mango.pileup --user --py diff --git a/recipes/mango/meta.yaml b/recipes/mango/meta.yaml index fe6e8a54d3836..62e9f388720ab 100644 --- a/recipes/mango/meta.yaml +++ b/recipes/mango/meta.yaml @@ -1,56 +1,85 @@ {% set name = "bdgenomics.mango" %} +{% set pileup_name = "bdgenomics.mango.pileup" %} +{% set adam_name = "bdgenomics.adam" %} + {% set version = "0.0.5" %} +{% set pileup_version = "0.0.5" %} +{% set adam_version = "0.30.0a0" %} +{% set modin_name = "modin" %} +{% set modin_version = "0.6.1" %} + +# Current issues: +# 1: noarch not working for either python or generic +## noarch generic uses a different python version in test, so imports will fail +## noarch python makes symlinks not work from build script +# 2: modin and ray are not on conda +## current issue in modin is that non-ray build still tries to pull +## from ray modules, which causes an error because ray is not on conda +## solution: need to wait until modin is on conda package: name: "mango" version: "{{ version }}" source: + - url: https://search.maven.org/remotecontent?filepath=org/bdgenomics/mango/mango-distribution/{{ version }}/mango-distribution-{{ version }}-bin.tar.gz + md5: fb9c822dcc0ea1a13766d7faf3413af5 + folder: core + patches: + - mango-notebook.patch + - mango-submit.patch + - make_genome.patch - url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name }}-{{ version }}.tar.gz sha256: 12dc1b334597481b2caa3988417cb6742569585fd1f056c02adfce2b1f208467 - - url: https://search.maven.org/remotecontent?filepath=org/bdgenomics/mango/mango-distribution/{{ version }}/mango-distribution-{{ version }}-bin.tar.gz - md5: f438bdbe93e855db1dae077fa28ff3f7 - + folder: python + - url: https://pypi.io/packages/source/{{ pileup_name[0] }}/{{ pileup_name }}/{{ pileup_name }}-{{ pileup_version }}.tar.gz + sha256: bc35dae68a5541ba663d07f35ffdfb2534c33892e13a1dd8e68929a961b123d9 + folder: python_pileup + - url: https://pypi.io/packages/source/{{ adam_name[0] }}/{{ adam_name }}/{{ adam_name }}-{{ adam_version }}.tar.gz + sha256: bfb9e5f0f2bf887a63eebc994d9d6c3d7606e1e9504b88426972b170b7d6f4a9 + folder: python_adam + - url: https://pypi.io/packages/source/{{ modin_name[0] }}/{{ modin_name }}/{{ modin_name }}-{{ modin_version }}.tar.gz + sha256: 0b9b407ce9ba0907aae6c54d221e59e80d2d7a9b7a8510416625d5f4b9e83762 + folder: python_modin build: - noarch: generic number: 0 - script: "{{ PYTHON }} -m pip install . --no-deps --ignore-installed -vv " + noarch: python requirements: host: - - bdgenomics.adam ==0.30.0a0 - - bdgenomics.mango.pileup ==0.0.4 - - cigar ==0.1.3 - - ipython ==7.8.0 - - ipywidgets ==7.0.0 - - jupyter ==1.0.0 - - matplotlib ==2.0.2 - pip - psutil - - python + - python >3 + - jupyter ==1.0.0 run: + - dask + - distributed - openjdk >=8,<9 - - pyspark - - bdgenomics.adam ==0.30.0a0 - - bdgenomics.mango.pileup ==0.0.4 + - pyspark ==2.4.4 + - psutil + - python >3 - cigar ==0.1.3 - ipython ==7.8.0 - ipywidgets ==7.0.0 - - jupyter ==1.0.0 - matplotlib ==2.0.2 - - psutil - - python + - ipykernel >=5.1.2 + - widgetsnbextension >=3.5.0 + - pandas ==0.25.1 # required by modin + - traitlets >=4.3.0,<5.0 # required by jupyter widget + - traittypes >=0.0.6 # required by jupyter widget test: imports: + - modin - bdgenomics + - bdgenomics.adam - bdgenomics.mango - - bdgenomics.mango.test - - bdgenomics.mango.pileup + # - bdgenomics.mango.io needs modin fix in conda + # - bdgenomics.mango.pileup needs modin fix commands: - mango-submit --help - - mango-notebook --help + - SPARK_HOME=$SP_DIR/pyspark mango-notebook --help - make_genome --help about: diff --git a/recipes/ray/build.sh b/recipes/ray/build.sh deleted file mode 100644 index 495a4945daec0..0000000000000 --- a/recipes/ray/build.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -mkdir -p $PREFIX/bin - -mkdir build -cd build -export MPICXX=$PREFIX/bin -export OMPI_MCA_mpi_show_handle_leaks=1 -cmake . -DCMAKE_INSTALL_PREFIX=$PREFIX/bin .. -make -cp Ray $PREFIX/bin diff --git a/recipes/ray/meta.yaml b/recipes/ray/meta.yaml deleted file mode 100644 index 8fbc238c2987f..0000000000000 --- a/recipes/ray/meta.yaml +++ /dev/null @@ -1,36 +0,0 @@ -{% set name = "ray" %} -{% set version = "2.3.1" %} - -package: - name: {{ name }} - version: {{ version }} - -source: - url: https://downloads.sourceforge.net/project/denovoassembler/Ray-{{ version }}.tar.bz2 - md5: 82f693c4db60af4328263c9279701009 - -build: - number: 1 - skip: False - -requirements: - build: - - {{ compiler('c') }} - - openmpi >=2.0.2 - - cmake - - run: - - openmpi >=2.0.2 - -test: - commands: - - which Ray # Ray -help requires ssh - -about: - home: 'http://denovoassembler.sourceforge.net/index.html' - license: GPL3 - summary: "Parallel genome assemblies for parallel DNA sequencing" -extra: - identifiers: - - biotools:ray - - doi:10.1186/gb-2012-13-12-r122 From ea0d8b619de78ab81762d39f7aaa5602ed5b0879 Mon Sep 17 00:00:00 2001 From: Alyssa Morrow Date: Wed, 16 Oct 2019 12:01:07 -0700 Subject: [PATCH 05/10] added patches --- recipes/mango/make_genome.patch | 31 ++++++++++++++++++++++++++ recipes/mango/mango-notebook.patch | 35 ++++++++++++++++++++++++++++++ recipes/mango/mango-submit.patch | 33 ++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+) create mode 100644 recipes/mango/make_genome.patch create mode 100644 recipes/mango/mango-notebook.patch create mode 100644 recipes/mango/mango-submit.patch diff --git a/recipes/mango/make_genome.patch b/recipes/mango/make_genome.patch new file mode 100644 index 0000000000000..0146b94a470f2 --- /dev/null +++ b/recipes/mango/make_genome.patch @@ -0,0 +1,31 @@ +--- a/bin/make_genome ++++ b/bin/make_genome +@@ -20,12 +20,20 @@ + # Runs MakeGenome, which generates a .genome file containing reference, genes, cytoband, and chromsizes. + # Requires 2 arguments: genome name and output file to save .genome file to. + +-SOURCE_DIR=$(dirname ${BASH_SOURCE[0]}) ++# Find original directory of this script, resolving symlinks ++# https://stackoverflow.com/questions/59895/get-the-source-directory-of-a-bash-script-from-within-the-script-itself/246128#246128 ++SOURCE="${BASH_SOURCE[0]}" ++while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink ++ DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" ++ SOURCE="$(readlink "$SOURCE")" ++ [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located ++done ++SCRIPT_DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" + + # get Mango jar +-MANGO_CLI_JAR=$(${SOURCE_DIR}/find-mango-assembly.sh) ++MANGO_CLI_JAR=$(${SCRIPT_DIR}/find-mango-assembly.sh) + +-SPARK_SUBMIT=$(${SOURCE_DIR}/find-spark.sh) ++SPARK_SUBMIT=$(${SCRIPT_DIR}/find-spark.sh) + + # submit the job to Spark + "$SPARK_SUBMIT" \ +@@ -34,4 +42,3 @@ + --conf spark.kryo.registrator=org.bdgenomics.mango.serialization.MangoKryoRegistrator \ + $MANGO_CLI_JAR \ + "MakeGenome" "$@" \ +- diff --git a/recipes/mango/mango-notebook.patch b/recipes/mango/mango-notebook.patch new file mode 100644 index 0000000000000..2ac93184cd938 --- /dev/null +++ b/recipes/mango/mango-notebook.patch @@ -0,0 +1,35 @@ +--- a/bin/mango-notebook ++++ b/bin/mango-notebook +@@ -62,6 +62,16 @@ + echo ${path} + } + ++# Find original directory of this script, resolving symlinks ++# https://stackoverflow.com/questions/59895/get-the-source-directory-of-a-bash-script-from-within-the-script-itself/246128#246128 ++SOURCE="${BASH_SOURCE[0]}" ++while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink ++ DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" ++ SOURCE="$(readlink "$SOURCE")" ++ [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located ++done ++SCRIPT_DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" ++ + # Retrieve the path of py4j zip archive used by current spark + get_py4jzip(){ + local -a py4j_zip_all_versions +@@ -112,14 +122,13 @@ + # variable declarations + trap 'err_report ${LINENO} $?' ERR + declare -r PY4J_ZIP=$(get_py4jzip) +-declare -r SOURCE_DIR=$(dirname ${BASH_SOURCE[0]}) + declare mango_cli_jar="" + declare wrapper_path + declare pyspark="" + parse_args "$@" + + # get Mango jar +-mango_cli_jar=$(${SOURCE_DIR}/find-mango-assembly.sh) ++mango_cli_jar=$(${SCRIPT_DIR}/find-mango-assembly.sh) + + # get pyspark path + if [[ -z "${SPARK_HOME}" ]]; then diff --git a/recipes/mango/mango-submit.patch b/recipes/mango/mango-submit.patch new file mode 100644 index 0000000000000..2c4b1c96e6d79 --- /dev/null +++ b/recipes/mango/mango-submit.patch @@ -0,0 +1,33 @@ +--- a/bin/mango-submit ++++ b/bin/mango-submit +@@ -20,7 +20,15 @@ + # usage: mango-submit [ --] + set -e + +-SOURCE_DIR=$(dirname ${BASH_SOURCE[0]}) ++# Find original directory of this script, resolving symlinks ++# https://stackoverflow.com/questions/59895/get-the-source-directory-of-a-bash-script-from-within-the-script-itself/246128#246128 ++SOURCE="${BASH_SOURCE[0]}" ++while [ -L "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink ++ DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" ++ SOURCE="$(readlink "$SOURCE")" ++ [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located ++done ++SCRIPT_DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" + + # Split args into Spark and ADAM args + DD=False # DD is "double dash" +@@ -52,10 +60,11 @@ + + + # get Mango jar +-MANGO_CLI_JAR=$(${SOURCE_DIR}/find-mango-assembly.sh) ++MANGO_CLI_JAR=$(${SCRIPT_DIR}/find-mango-assembly.sh) + +-SPARK_SUBMIT=$(${SOURCE_DIR}/find-spark.sh) ++SPARK_SUBMIT=$(${SCRIPT_DIR}/find-spark.sh) + echo "Using spark-submit=$SPARK_SUBMIT" 1>&2 ++ls "$(dirname "$( dirname "$SPARK_SUBMIT" )" )" + + # submit the job to Spark + "$SPARK_SUBMIT" \ From 8ab8b68a64ca4e520e3ebd94a1884ea91768e619 Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Thu, 17 Oct 2019 10:32:54 +0200 Subject: [PATCH 06/10] some adjustments --- recipes/mango/build.sh | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/recipes/mango/build.sh b/recipes/mango/build.sh index e934f3992c7fd..3db291748c6c2 100644 --- a/recipes/mango/build.sh +++ b/recipes/mango/build.sh @@ -9,15 +9,20 @@ mkdir -p $PACKAGE_HOME ls $PACKAGE_HOME cp -R core/* $PACKAGE_HOME -ln -s $PACKAGE_HOME/bin/mango-notebook $BINARY_HOME -ln -s $PACKAGE_HOME/bin/mango-submit $BINARY_HOME -ln -s $PACKAGE_HOME/bin/make_genome $BINARY_HOME - -echo "PYTHON VERSION----" -$PYTHON --version -echo $PY_VER -which $PYTHON -which python3.7 +for f in $PACKAGE_HOME/bin/* ; do + if [[ -f $f ]] ; then + ln -s $f $BINARY_HOME/ + fi +done +#ln -s $PACKAGE_HOME/bin/mango-notebook $BINARY_HOME +#ln -s $PACKAGE_HOME/bin/mango-submit $BINARY_HOME +#ln -s $PACKAGE_HOME/bin/make_genome $BINARY_HOME + +#echo "PYTHON VERSION----" +#$PYTHON --version +#echo $PY_VER +#which $PYTHON +#which python3.7 # modin $PYTHON -m pip install python_modin/.[dask] --no-deps --ignore-installed -vv @@ -33,3 +38,8 @@ $PYTHON -m pip install python_pileup/. --no-deps --ignore-installed -vv # jupyter nbextension enable --py widgetsnbextension # jupyter nbextension install --py --symlink --user bdgenomics.mango.pileup # jupyter nbextension enable bdgenomics.mango.pileup --user --py + +mkdir -p $PREFIX/mango-assembly/target +for f in $PACKAGE_HOME/*.jar ; do + ln -s $f $PREFIX/mango-assembly/target/ +done From 628f5fe98c53ef0e83366adba264a9bf0a6497a5 Mon Sep 17 00:00:00 2001 From: dpryan79 Date: Thu, 17 Oct 2019 10:43:33 +0200 Subject: [PATCH 07/10] oops --- recipes/mango/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/mango/build.sh b/recipes/mango/build.sh index 3db291748c6c2..efed721736e13 100644 --- a/recipes/mango/build.sh +++ b/recipes/mango/build.sh @@ -40,6 +40,6 @@ $PYTHON -m pip install python_pileup/. --no-deps --ignore-installed -vv # jupyter nbextension enable bdgenomics.mango.pileup --user --py mkdir -p $PREFIX/mango-assembly/target -for f in $PACKAGE_HOME/*.jar ; do +for f in $PACKAGE_HOME/repo/*.jar ; do ln -s $f $PREFIX/mango-assembly/target/ done From 0eb359ed4fd5c21c034e940e36f314e941db2928 Mon Sep 17 00:00:00 2001 From: Alyssa Morrow Date: Thu, 17 Oct 2019 09:46:49 -0700 Subject: [PATCH 08/10] remove patches and cleanup --- recipes/mango/build.sh | 9 -------- recipes/mango/make_genome.patch | 31 -------------------------- recipes/mango/mango-notebook.patch | 35 ------------------------------ recipes/mango/mango-submit.patch | 33 ---------------------------- recipes/mango/meta.yaml | 9 +------- 5 files changed, 1 insertion(+), 116 deletions(-) delete mode 100644 recipes/mango/make_genome.patch delete mode 100644 recipes/mango/mango-notebook.patch delete mode 100644 recipes/mango/mango-submit.patch diff --git a/recipes/mango/build.sh b/recipes/mango/build.sh index efed721736e13..14d7462e70205 100644 --- a/recipes/mango/build.sh +++ b/recipes/mango/build.sh @@ -14,15 +14,6 @@ for f in $PACKAGE_HOME/bin/* ; do ln -s $f $BINARY_HOME/ fi done -#ln -s $PACKAGE_HOME/bin/mango-notebook $BINARY_HOME -#ln -s $PACKAGE_HOME/bin/mango-submit $BINARY_HOME -#ln -s $PACKAGE_HOME/bin/make_genome $BINARY_HOME - -#echo "PYTHON VERSION----" -#$PYTHON --version -#echo $PY_VER -#which $PYTHON -#which python3.7 # modin $PYTHON -m pip install python_modin/.[dask] --no-deps --ignore-installed -vv diff --git a/recipes/mango/make_genome.patch b/recipes/mango/make_genome.patch deleted file mode 100644 index 0146b94a470f2..0000000000000 --- a/recipes/mango/make_genome.patch +++ /dev/null @@ -1,31 +0,0 @@ ---- a/bin/make_genome -+++ b/bin/make_genome -@@ -20,12 +20,20 @@ - # Runs MakeGenome, which generates a .genome file containing reference, genes, cytoband, and chromsizes. - # Requires 2 arguments: genome name and output file to save .genome file to. - --SOURCE_DIR=$(dirname ${BASH_SOURCE[0]}) -+# Find original directory of this script, resolving symlinks -+# https://stackoverflow.com/questions/59895/get-the-source-directory-of-a-bash-script-from-within-the-script-itself/246128#246128 -+SOURCE="${BASH_SOURCE[0]}" -+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink -+ DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" -+ SOURCE="$(readlink "$SOURCE")" -+ [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -+done -+SCRIPT_DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - - # get Mango jar --MANGO_CLI_JAR=$(${SOURCE_DIR}/find-mango-assembly.sh) -+MANGO_CLI_JAR=$(${SCRIPT_DIR}/find-mango-assembly.sh) - --SPARK_SUBMIT=$(${SOURCE_DIR}/find-spark.sh) -+SPARK_SUBMIT=$(${SCRIPT_DIR}/find-spark.sh) - - # submit the job to Spark - "$SPARK_SUBMIT" \ -@@ -34,4 +42,3 @@ - --conf spark.kryo.registrator=org.bdgenomics.mango.serialization.MangoKryoRegistrator \ - $MANGO_CLI_JAR \ - "MakeGenome" "$@" \ -- diff --git a/recipes/mango/mango-notebook.patch b/recipes/mango/mango-notebook.patch deleted file mode 100644 index 2ac93184cd938..0000000000000 --- a/recipes/mango/mango-notebook.patch +++ /dev/null @@ -1,35 +0,0 @@ ---- a/bin/mango-notebook -+++ b/bin/mango-notebook -@@ -62,6 +62,16 @@ - echo ${path} - } - -+# Find original directory of this script, resolving symlinks -+# https://stackoverflow.com/questions/59895/get-the-source-directory-of-a-bash-script-from-within-the-script-itself/246128#246128 -+SOURCE="${BASH_SOURCE[0]}" -+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink -+ DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" -+ SOURCE="$(readlink "$SOURCE")" -+ [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -+done -+SCRIPT_DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" -+ - # Retrieve the path of py4j zip archive used by current spark - get_py4jzip(){ - local -a py4j_zip_all_versions -@@ -112,14 +122,13 @@ - # variable declarations - trap 'err_report ${LINENO} $?' ERR - declare -r PY4J_ZIP=$(get_py4jzip) --declare -r SOURCE_DIR=$(dirname ${BASH_SOURCE[0]}) - declare mango_cli_jar="" - declare wrapper_path - declare pyspark="" - parse_args "$@" - - # get Mango jar --mango_cli_jar=$(${SOURCE_DIR}/find-mango-assembly.sh) -+mango_cli_jar=$(${SCRIPT_DIR}/find-mango-assembly.sh) - - # get pyspark path - if [[ -z "${SPARK_HOME}" ]]; then diff --git a/recipes/mango/mango-submit.patch b/recipes/mango/mango-submit.patch deleted file mode 100644 index 2c4b1c96e6d79..0000000000000 --- a/recipes/mango/mango-submit.patch +++ /dev/null @@ -1,33 +0,0 @@ ---- a/bin/mango-submit -+++ b/bin/mango-submit -@@ -20,7 +20,15 @@ - # usage: mango-submit [ --] - set -e - --SOURCE_DIR=$(dirname ${BASH_SOURCE[0]}) -+# Find original directory of this script, resolving symlinks -+# https://stackoverflow.com/questions/59895/get-the-source-directory-of-a-bash-script-from-within-the-script-itself/246128#246128 -+SOURCE="${BASH_SOURCE[0]}" -+while [ -L "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink -+ DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" -+ SOURCE="$(readlink "$SOURCE")" -+ [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -+done -+SCRIPT_DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - - # Split args into Spark and ADAM args - DD=False # DD is "double dash" -@@ -52,10 +60,11 @@ - - - # get Mango jar --MANGO_CLI_JAR=$(${SOURCE_DIR}/find-mango-assembly.sh) -+MANGO_CLI_JAR=$(${SCRIPT_DIR}/find-mango-assembly.sh) - --SPARK_SUBMIT=$(${SOURCE_DIR}/find-spark.sh) -+SPARK_SUBMIT=$(${SCRIPT_DIR}/find-spark.sh) - echo "Using spark-submit=$SPARK_SUBMIT" 1>&2 -+ls "$(dirname "$( dirname "$SPARK_SUBMIT" )" )" - - # submit the job to Spark - "$SPARK_SUBMIT" \ diff --git a/recipes/mango/meta.yaml b/recipes/mango/meta.yaml index 62e9f388720ab..2fce39e52eda5 100644 --- a/recipes/mango/meta.yaml +++ b/recipes/mango/meta.yaml @@ -9,10 +9,7 @@ {% set modin_version = "0.6.1" %} # Current issues: -# 1: noarch not working for either python or generic -## noarch generic uses a different python version in test, so imports will fail -## noarch python makes symlinks not work from build script -# 2: modin and ray are not on conda +# 1: modin and ray are not on conda ## current issue in modin is that non-ray build still tries to pull ## from ray modules, which causes an error because ray is not on conda ## solution: need to wait until modin is on conda @@ -25,10 +22,6 @@ source: - url: https://search.maven.org/remotecontent?filepath=org/bdgenomics/mango/mango-distribution/{{ version }}/mango-distribution-{{ version }}-bin.tar.gz md5: fb9c822dcc0ea1a13766d7faf3413af5 folder: core - patches: - - mango-notebook.patch - - mango-submit.patch - - make_genome.patch - url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name }}-{{ version }}.tar.gz sha256: 12dc1b334597481b2caa3988417cb6742569585fd1f056c02adfce2b1f208467 folder: python From dc38a37f24104cac7b89740370df5e07a5c48836 Mon Sep 17 00:00:00 2001 From: Alyssa Morrow Date: Thu, 17 Oct 2019 09:52:18 -0700 Subject: [PATCH 09/10] added back deleted recipe --- recipes/ray/build.sh | 11 +++++++++++ recipes/ray/meta.yaml | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 recipes/ray/build.sh create mode 100644 recipes/ray/meta.yaml diff --git a/recipes/ray/build.sh b/recipes/ray/build.sh new file mode 100644 index 0000000000000..495a4945daec0 --- /dev/null +++ b/recipes/ray/build.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +mkdir -p $PREFIX/bin + +mkdir build +cd build +export MPICXX=$PREFIX/bin +export OMPI_MCA_mpi_show_handle_leaks=1 +cmake . -DCMAKE_INSTALL_PREFIX=$PREFIX/bin .. +make +cp Ray $PREFIX/bin diff --git a/recipes/ray/meta.yaml b/recipes/ray/meta.yaml new file mode 100644 index 0000000000000..8fbc238c2987f --- /dev/null +++ b/recipes/ray/meta.yaml @@ -0,0 +1,36 @@ +{% set name = "ray" %} +{% set version = "2.3.1" %} + +package: + name: {{ name }} + version: {{ version }} + +source: + url: https://downloads.sourceforge.net/project/denovoassembler/Ray-{{ version }}.tar.bz2 + md5: 82f693c4db60af4328263c9279701009 + +build: + number: 1 + skip: False + +requirements: + build: + - {{ compiler('c') }} + - openmpi >=2.0.2 + - cmake + + run: + - openmpi >=2.0.2 + +test: + commands: + - which Ray # Ray -help requires ssh + +about: + home: 'http://denovoassembler.sourceforge.net/index.html' + license: GPL3 + summary: "Parallel genome assemblies for parallel DNA sequencing" +extra: + identifiers: + - biotools:ray + - doi:10.1186/gb-2012-13-12-r122 From a01382088832405b17caae8f316a197d47d2a0a9 Mon Sep 17 00:00:00 2001 From: Alyssa Morrow Date: Tue, 22 Oct 2019 16:24:30 -0700 Subject: [PATCH 10/10] removed SP_DIR for mulled tests --- recipes/mango/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/mango/meta.yaml b/recipes/mango/meta.yaml index 2fce39e52eda5..19a109ac1bc88 100644 --- a/recipes/mango/meta.yaml +++ b/recipes/mango/meta.yaml @@ -72,7 +72,7 @@ test: # - bdgenomics.mango.pileup needs modin fix commands: - mango-submit --help - - SPARK_HOME=$SP_DIR/pyspark mango-notebook --help + - SPARK_HOME=$(python -c "import site,os; print(os.path.join(site.getsitepackages()[0], \"pyspark\"))" ) mango-notebook --help - make_genome --help about: