From 842638d04cec031e8cba793265f81b9a312ca6a5 Mon Sep 17 00:00:00 2001 From: Espen Hagen <2492641+espenhgn@users.noreply.github.com> Date: Thu, 14 Dec 2023 14:00:00 +0100 Subject: [PATCH] Add/update requested gwas.sif binaries (#214) * add gdb, ldak, snptest; update metal, qctool * update changelog * fixed tests * updated changelog * rebuilt gwas.sif --- CHANGELOG.md | 15 ++++- docker/README.md | 6 +- docker/dockerfiles/gwas/Dockerfile | 22 +++++-- docker/scripts/apt_get_essential.sh | 15 ++--- docker/scripts/install_bcftools.sh | 4 +- docker/scripts/install_htslib.sh | 2 +- docker/scripts/install_ldak.sh | 6 ++ docker/scripts/install_metal.sh | 12 ++-- docker/scripts/install_minimac4.sh | 2 +- docker/scripts/install_qctools.sh | 30 ++++++--- docker/scripts/install_snptest.sh | 7 +++ singularity/gwas.sif | 4 +- tests/test_gwas.py | 95 ++++++++++++++++++++--------- 13 files changed, 159 insertions(+), 61 deletions(-) create mode 100644 docker/scripts/install_ldak.sh create mode 100644 docker/scripts/install_snptest.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index f489730e..372e8ad2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,7 +39,20 @@ If MD5 sum is not listed for a certain release then it means that the container * Miscellaneous goes here -## [1.5.1] - 2023-10.20 +## [1.6.0] - 2023-12-12 + +### Added + +- Added `gdb` debugger, `ldak` and `snptest` binaries to `gwas.sif` container +- Added tests for `ldak` and `snptest` binaries in `gwas.sif` container + +### Updated + +- updated `metal` to version `2020-05-05` in `gwas.sif` +- updated `qctool` to `v2.2.2` and added related binaries `inthinnerator`, `hptest`, `ldbird` and `selfmap` to `gwas.sif` +- rebuilt `gwas.sif` (md5 checksum b6104b58d21f862f9d61a86d9d4802a6) + +## [1.5.1] - 2023-10-20 ### Fixed diff --git a/docker/README.md b/docker/README.md index 7bc6e68f..4f0a1614 100644 --- a/docker/README.md +++ b/docker/README.md @@ -48,11 +48,12 @@ The detailed description of the available container [files](https://github.com/c | gwas.sif | flashpca_x86-64 | 2.0 | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) | gwas.sif | gcta64 | version 1.93.3 beta 2 Linux | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) | gwas.sif | gctb | 2.02 | [MIT](https://opensource.org/licenses/MIT) - | gwas.sif | qctool | 2.0.6, revision 18b8f17 | [Boost](https://www.boost.org/LICENSE_1_0.txt) + | gwas.sif | qctool | 2.2.2, revision e5723df2c0c85959 | [Boost](https://www.boost.org/LICENSE_1_0.txt) | gwas.sif | GWAMA | 2.2.2 | [BSD-3-Clause](https://opensource.org/licenses/BSD-3-Clause) | gwas.sif | HTSlib | 1.12 | [MIT/Expat/Modified-BSD](https://github.com/samtools/htslib/blob/develop/LICENSE) | gwas.sif | king | 2.2.9 - (c) | [permissive](https://www.kingrelatedness.com/Download.shtml) - | gwas.sif | metal | version released on 2011-03-25 | - + | gwas.sif | ldak | 5.2 | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) + | gwas.sif | metal | 2020-05-05 | - | gwas.sif | minimac4 | v4.1.0 | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) | gwas.sif | plink | v1.90b6.18 64-bit (16 Jun 2020) | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) | gwas.sif | plink2 | v2.00a3.6LM 64-bit Intel (14 Aug 2022) | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) @@ -66,6 +67,7 @@ The detailed description of the available container [files](https://github.com/c | gwas.sif | shapeit5 switch | v5.1.1 | [MIT](https://opensource.org/licenses/MIT) | gwas.sif | shapeit5 xcftools | v5.1.1 | [MIT](https://opensource.org/licenses/MIT) | gwas.sif | simu_linux | v0.9.4 | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) + | gwas.sif | snptest | v2.5.6 | [permissive](https://www.chg.ox.ac.uk/~gav/snptest/#download) | gwas.sif | switchError | 6e688b1 | [MIT](https://opensource.org/licenses/MIT) | gwas.sif | vcftools | 0.1.17 | [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html) | python3.sif | ubuntu | 20.04 (LTS) | [Creative Commons CC-BY-SA version 3.0 UK licence](https://ubuntu.com/legal/intellectual-property-policy) diff --git a/docker/dockerfiles/gwas/Dockerfile b/docker/dockerfiles/gwas/Dockerfile index 3274a69a..9387a222 100644 --- a/docker/dockerfiles/gwas/Dockerfile +++ b/docker/dockerfiles/gwas/Dockerfile @@ -59,11 +59,17 @@ COPY /scripts/install_gctb.sh /tmp/gctb/ RUN bash /tmp/gctb/install_gctb.sh && \ rm -rf /tmp/gctb -# qctools -WORKDIR /tmp/qctools -COPY /scripts/install_qctools.sh /tmp/qctools/ -RUN bash /tmp/qctools/install_qctools.sh && \ - rm -rf /tmp/qctools +# qctool +WORKDIR /tmp/qctool +COPY /scripts/install_qctools.sh /tmp/qctool/ +RUN bash /tmp/qctool/install_qctools.sh && \ + rm -rf /tmp/qctool + +# snptest +WORKDIR /tmp/snptest +COPY /scripts/install_snptest.sh /tmp/snptest/ +RUN bash /tmp/snptest/install_snptest.sh && \ + rm -rf /tmp/snptest # king WORKDIR /tmp/king @@ -71,6 +77,12 @@ COPY /scripts/install_king.sh /tmp/king/ RUN bash /tmp/king/install_king.sh && \ rm -rf /tmp/king +# ldak +WORKDIR /tmp/ldak +COPY /scripts/install_ldak.sh /tmp/ldak/ +RUN bash /tmp/ldak/install_ldak.sh && \ + rm -rf /tmp/ldak + # metal WORKDIR /tmp/metal COPY /scripts/install_metal.sh /tmp/metal/ diff --git a/docker/scripts/apt_get_essential.sh b/docker/scripts/apt_get_essential.sh index fc085ad9..62997405 100644 --- a/docker/scripts/apt_get_essential.sh +++ b/docker/scripts/apt_get_essential.sh @@ -1,7 +1,7 @@ #!/bin/sh set -euo pipefail -apt-get update && apt-get install -y --no-install-recommends apt-utils=2.0.9 +apt-get update && apt-get install -y --no-install-recommends apt-utils=2.0.10 apt-get update && apt-get install -y --no-install-recommends ca-certificates=20230311ubuntu0.20.04.1 && \ update-ca-certificates @@ -12,27 +12,28 @@ apt-get update && apt-get install -y --no-install-recommends \ build-essential=12.8ubuntu1 \ bzip2=1.0.8-2 \ cmake=3.16.3-1ubuntu1.20.04.1 \ - curl=7.68.0-1ubuntu2.19 \ + curl=7.68.0-1ubuntu2.21 \ dos2unix=7.4.0-2 \ + gdb=9.1-0ubuntu1 \ gfortran=4:9.3.0-1ubuntu2 \ git=1:2.25.1-1ubuntu3.11 \ less=551-1ubuntu0.1 \ libatlas-base-dev=3.10.3-8ubuntu7 \ - libcurl4-openssl-dev=7.68.0-1ubuntu2.19 \ + libcurl4-openssl-dev=7.68.0-1ubuntu2.21 \ libgomp1=10.5.0-1ubuntu1~20.04 \ - libgsl-dev=2.5+dfsg-6build1 \ + libgsl-dev=2.5+dfsg-6+deb10u1build0.20.04.1 \ libnss3=2:3.49.1-1ubuntu1.9 \ libpcre2-dev=10.34-7ubuntu0.1 \ libxt-dev=1:1.1.5-1 \ pandoc=2.5-3build2 \ pandoc-citeproc=0.15.0.1-1build4 \ parallel=20161222-1.1 \ - perl=5.30.0-9ubuntu0.4 \ + perl=5.30.0-9ubuntu0.5 \ pkg-config=0.29.1-0ubuntu4 \ - tar=1.30+dfsg-7ubuntu0.20.04.2 \ + tar=1.30+dfsg-7ubuntu0.20.04.4 \ tofrodos=1.7.13+ds-4 \ unzip=6.0-25ubuntu1.1 \ - vim=2:8.1.2269-1ubuntu5.17 \ + vim=2:8.1.2269-1ubuntu5.20 \ wget=1.20.3-1ubuntu2 \ zlib1g-dev=1:1.2.11.dfsg-2ubuntu1.5 \ && \ diff --git a/docker/scripts/install_bcftools.sh b/docker/scripts/install_bcftools.sh index c056e6a3..42835aab 100644 --- a/docker/scripts/install_bcftools.sh +++ b/docker/scripts/install_bcftools.sh @@ -3,8 +3,8 @@ set -euo pipefail # deps apt-get update && apt-get install -y --no-install-recommends \ - libcurl4-gnutls-dev=7.68.0-1ubuntu2.19 \ - libperl-dev=5.30.0-9ubuntu0.4 \ + libcurl4-gnutls-dev=7.68.0-1ubuntu2.21 \ + libperl-dev=5.30.0-9ubuntu0.5 \ && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* diff --git a/docker/scripts/install_htslib.sh b/docker/scripts/install_htslib.sh index 50a1da53..2d7d5191 100644 --- a/docker/scripts/install_htslib.sh +++ b/docker/scripts/install_htslib.sh @@ -5,7 +5,7 @@ set -euo pipefail apt-get update && apt-get install --no-install-recommends \ libbz2-dev=1.0.8-2 \ liblzma-dev=5.2.4-1ubuntu1.1 \ - libssl-dev=1.1.1f-1ubuntu2.19 \ + libssl-dev=1.1.1f-1ubuntu2.20 \ -y && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* diff --git a/docker/scripts/install_ldak.sh b/docker/scripts/install_ldak.sh new file mode 100644 index 00000000..af078cdf --- /dev/null +++ b/docker/scripts/install_ldak.sh @@ -0,0 +1,6 @@ +#!/bin/sh +set -euo pipefail + +wget --no-check-certificate https://dougspeed.com/wp-content/uploads/ldak5.2.linux_.zip +unzip ldak5.2.linux_.zip +cp ldak5.2.linux /bin/ldak diff --git a/docker/scripts/install_metal.sh b/docker/scripts/install_metal.sh index 78340444..eeff568a 100644 --- a/docker/scripts/install_metal.sh +++ b/docker/scripts/install_metal.sh @@ -2,9 +2,11 @@ set -euo pipefail # metal -wget --no-check-certificate http://csg.sph.umich.edu/abecasis/metal/download/Linux-metal.tar.gz && \ - tar -xvzf Linux-metal.tar.gz && \ - rm -rf Linux-metal.tar.gz +wget --no-check-certificate https://github.com/statgen/METAL/archive/refs/tags/2020-05-05.tar.gz +tar -xvzf 2020-05-05.tar.gz -mv generic-metal/* . -cp metal /bin +mkdir build && cd build +cmake -DCMAKE_BUILD_TYPE=Release ../METAL-2020-05-05/. +make -j4 +make install +cp bin/metal /bin diff --git a/docker/scripts/install_minimac4.sh b/docker/scripts/install_minimac4.sh index f68f9aef..657b5f67 100644 --- a/docker/scripts/install_minimac4.sh +++ b/docker/scripts/install_minimac4.sh @@ -4,7 +4,7 @@ set -euo pipefail # install some deps for installing cget apt-get update && \ apt-get install --no-install-recommends \ - python3-pip=20.0.2-5ubuntu1.9 \ + python3-pip=20.0.2-5ubuntu1.10 \ python3-click=7.0-3 \ python3-six=1.14.0-2 \ -y && \ diff --git a/docker/scripts/install_qctools.sh b/docker/scripts/install_qctools.sh index e68931cf..639919ad 100644 --- a/docker/scripts/install_qctools.sh +++ b/docker/scripts/install_qctools.sh @@ -1,13 +1,29 @@ #!/bin/sh set -euo pipefail -# qctools - wget --no-check-certificate https://www.well.ox.ac.uk/~gav/resources/qctool_v2.0.6-Ubuntu16.04-x86_64.tgz && \ - tar -xvzf qctool_v2.0.6-Ubuntu16.04-x86_64.tgz && \ - rm -rf qctool_v2.0.6-Ubuntu16.04-x86_64.tgz +# python appears to be a build time dependency +apt-get update && apt-get install -y --no-install-recommends python3=3.8.2-0ubuntu2 && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# /usr/bin/python must exist +update-alternatives --install /usr/bin/python python /usr/bin/python3 10 +# qctools +wget --no-check-certificate https://code.enkre.net/qctool/zip/e5723df2c0c85959/qctool.zip +unzip qctool.zip +cd qctool +./waf configure +./waf -mv qctool_v2.0.6-Ubuntu16.04-x86_64/* . -cp qctool /bin +# copy binaries to /bin +cp build/release/apps/inthinnerator_v2.2.2 /bin/inthinnerator +cp build/release/apps/hptest_v2.2.2 /bin/hptest +cp build/release/apps/ldbird_v2.2.2 /bin/ldbird +cp build/release/apps/qctool_v2.2.2 /bin/qctool +cp build/release/apps/selfmap_v2.2.2 /bin/selfmap -chmod 755 /bin/qctool +# remove python +apt-get purge \ + python3 -y && \ + apt-get autoremove --purge -y diff --git a/docker/scripts/install_snptest.sh b/docker/scripts/install_snptest.sh new file mode 100644 index 00000000..59c16717 --- /dev/null +++ b/docker/scripts/install_snptest.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -euo pipefail + +# snptest +wget --no-check-certificate http://www.well.ox.ac.uk/~gav/resources/snptest_v2.5.6_CentOS_Linux7.8-x86_64_dynamic.tgz +tar -xvzf snptest_v2.5.6_CentOS_Linux7.8-x86_64_dynamic.tgz +cp snptest_v2.5.6_CentOS_Linux7.8.2003-x86_64_dynamic/snptest_v2.5.6 /bin/snptest diff --git a/singularity/gwas.sif b/singularity/gwas.sif index 35cb9c66..0ff66e96 100644 --- a/singularity/gwas.sif +++ b/singularity/gwas.sif @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6499b892f4bc586167cdcef28a7c7ba253a472080eed19c64b13d664e7ae511b -size 573915136 +oid sha256:f4b569b3d39fb8d9a0b903966fa0e13be2addd8fec84011c0d2dffe3c275e7b6 +size 560619520 diff --git a/tests/test_gwas.py b/tests/test_gwas.py index 8763c4d7..974f8de2 100644 --- a/tests/test_gwas.py +++ b/tests/test_gwas.py @@ -14,71 +14,81 @@ def test_gwas_bcftools(): + """test bcftools""" call = f'singularity run {pth} bcftools --version' - out = subprocess.run(call.split(' ')) + out = subprocess.run(call.split(' '), check=False) assert out.returncode == 0 def test_gwas_beagle(): + """test beagle""" call = f'singularity run {pth} beagle' - out = subprocess.run(call.split(' ')) + out = subprocess.run(call.split(' '), check=False) assert out.returncode == 0 def test_gwas_bgenix(): - for bin in ['bgenix', 'cat-bgen', 'edit-bgen']: - call = f'singularity run {pth} {bin} -help' - out = subprocess.run(call.split(' ')) + """test bgenix, cat-bgen, edit-bgen binaries""" + for binary in ['bgenix', 'cat-bgen', 'edit-bgen']: + call = f'singularity run {pth} {binary} -help' + out = subprocess.run(call.split(' '), check=False) assert out.returncode == 0 def test_gwas_bolt(): + """test bolt""" call = f'singularity run {pth} bolt -h' - out = subprocess.run(call.split(' ')) + out = subprocess.run(call.split(' '), check=False) assert out.returncode == 0 def test_gwas_eagle(): + """test eagle""" call = f'singularity run {pth} eagle -h' - out = subprocess.run(call.split(' ')) + out = subprocess.run(call.split(' '), check=False) assert out.returncode == 0 def test_gwas_flashpca(): + """test flashpca""" call = f'singularity run {pth} flashpca_x86-64 --version' - out = subprocess.run(call.split(' ')) + out = subprocess.run(call.split(' '), check=False) assert out.returncode == 0 def test_gwas_gcta(): + """test gcta""" cwd = os.getcwd() with tempfile.TemporaryDirectory() as d: os.chdir(d) os.system(f'tar -xvf {cwd}/tests/extras/ex.tar.gz') os.chdir(cwd) call = f'singularity run {pth} gcta64 --bfile {d}/ex --out {d}' - out = subprocess.run(call.split(' ')) + out = subprocess.run(call.split(' '), check=False) assert out.returncode == 0 def test_gwas_gctb(): + """test gctb""" cwd = os.getcwd() with tempfile.TemporaryDirectory() as d: os.chdir(d) os.system(f'tar -xvf {cwd}/tests/extras/ex.tar.gz') os.chdir(cwd) call = f'singularity run {pth} gctb --bfile {d}/ex --out {d}' - out = subprocess.run(call.split(' ')) + out = subprocess.run(call.split(' '), check=False) assert out.returncode == 0 def test_gwas_gwama(): + """test gwama""" call = f'singularity run --home=/tmp/:/home/ {pth} GWAMA --version' - out = subprocess.run(call.split(' ')) + out = subprocess.run(call.split(' '), check=False) assert out.returncode == 0 def test_gwas_king(): + """test king""" cwd = os.getcwd() with tempfile.TemporaryDirectory() as d: os.chdir(d) @@ -87,11 +97,19 @@ def test_gwas_king(): call = ' '.join( [f'singularity run --home={d}:/home/ {pth} king -b', f'{d}/ex.bed --fam {d}/ex.fam --bim {d}/ex.bim --related']) - out = subprocess.run(call.split(' ')) + out = subprocess.run(call.split(' '), check=False) assert out.returncode == 0 +def test_gwas_ldak(): + """test ldak""" + call = f'singularity run {pth} ldak --make-snps 1 --num-samples 1 --num-snps 1' + out = subprocess.run(call.split(' '), check=False) + assert out.returncode == 0 + + def test_gwas_metal(): + """test metal""" cwd = os.getcwd() with tempfile.TemporaryDirectory() as d: os.chdir(d) @@ -99,7 +117,7 @@ def test_gwas_metal(): os.chdir('GlucoseExample') call = \ f'singularity run --home=$PWD:/home/ {cwd}/{pth} metal metal.txt' - out = subprocess.run(call.split(' '), capture_output=True) + out = subprocess.run(call.split(' '), capture_output=True, check=False) assert out.returncode == 0 # software may not crash on error, checking captured output assert out.stdout.decode('utf-8').rfind('Error') <= 0 @@ -110,69 +128,88 @@ def test_gwas_metal(): def test_gwas_minimac4(): + """test minimac4""" call = f'singularity run {pth} minimac4 --version' - out = subprocess.run(call.split(' ')) + out = subprocess.run(call.split(' '), check=False) assert out.returncode == 0 def test_gwas_plink(): + """test plink""" call = f'singularity run {pth} plink --version' - out = subprocess.run(call.split(' ')) + out = subprocess.run(call.split(' '), check=False) assert out.returncode == 0 def test_gwas_plink2(): + """test plink2""" call = f'singularity run {pth} plink2 --version' - out = subprocess.run(call.split(' ')) + out = subprocess.run(call.split(' '), check=False) assert out.returncode == 0 def test_gwas_plink2_avx2(): + """test plink2_avx2""" call = f'singularity run {pth} plink2_avx2 --version' - out = subprocess.run(call.split(' ')) + out = subprocess.run(call.split(' '), check=False) assert out.returncode == 0 def test_gwas_prsice(): + """test prsice""" call = f'singularity run {pth} PRSice_linux --version' - out = subprocess.run(call.split(' ')) + out = subprocess.run(call.split(' '), check=False) assert out.returncode == 0 def test_gwas_qctools(): - call = f'singularity run {pth} qctool -help' - out = subprocess.run(call.split(' ')) - assert out.returncode == 0 + """test qctools""" + for binary in ['inthinnerator', 'hptest', 'ldbird', 'qctool', 'selfmap']: + call = f'singularity run {pth} {binary} -help' + out = subprocess.run(call.split(' '), check=False) + assert out.returncode == 0 def test_gwas_regenie(): + """test regenie""" call = f'singularity run {pth} regenie option --help' - out = subprocess.run(call.split(' ')) + out = subprocess.run(call.split(' '), check=False) assert out.returncode == 0 def test_gwas_shapeit4(): + """test shapeit4""" call = f'singularity run {pth} shapeit4.2 --help' - out = subprocess.run(call.split(' ')) + out = subprocess.run(call.split(' '), check=False) assert out.returncode == 0 def test_gwas_shapeit5(): + """test shapeit5""" for binary in ['phase_common', 'phase_rare', 'ligate', 'switch', 'xcftools']: call = f'singularity run {pth} {binary} --help' - out = subprocess.run(call.split(' ')) + out = subprocess.run(call.split(' '), check=False) assert out.returncode == 0 +def test_gwas_snptest(): + """test snptest""" + call = f'singularity run {pth} snptest -help' + out = subprocess.run(call.split(' '), check=False) + assert out.returncode == 0 + + @pytest.mark.xfail(reason="no help function for switchError") def test_gwas_switchError(): + """test switchError""" call = f'singularity run {pth} switchError --reg foo --gen foo --hap foo --fam foo --ps foo --out foo --maf 0.0' - out = subprocess.run(call.split(' ')) + out = subprocess.run(call.split(' '), check=False) assert out.returncode == 0 def test_gwas_simu(): + """test simu""" cwd = os.getcwd() with tempfile.TemporaryDirectory() as d: os.chdir(d) @@ -182,17 +219,19 @@ def test_gwas_simu(): [f'singularity run --home=/tmp/:/home/ {pth}', f'simu_linux --bfile {d}/ex --qt ', '--causal-pi 0.01 --num-traits 2 --hsq 0.2 0.6 --rg 0.8']) - out = subprocess.run(call.split(' ')) + out = subprocess.run(call.split(' '), check=False) assert out.returncode == 0 def test_gwas_vcftools(): + """test vcftools""" call = f'singularity run {pth} vcftools --version' - out = subprocess.run(call.split(' ')) + out = subprocess.run(call.split(' '), check=False) assert out.returncode == 0 def test_gwas_duohmm(): + """test duohmm""" call = f'singularity run {pth} duohmm' - out = subprocess.run(call.split(' ')) + out = subprocess.run(call.split(' '), check=False) assert out.returncode == 0