app: docker improvements

opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/liftover/LiftoverWrapperAnalysis.java

@@ -17,11 +17,9 @@
 package org.opencb.opencga.analysis.wrappers.liftover;
 
 
-import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.opencb.opencga.analysis.tools.OpenCgaToolScopeStudy;
-import org.opencb.opencga.analysis.variant.manager.CatalogUtils;
-import org.opencb.opencga.analysis.wrappers.plink.PlinkWrapperAnalysisExecutor;
 import org.opencb.opencga.core.exceptions.ToolException;
 import org.opencb.opencga.core.models.common.Enums;
 import org.opencb.opencga.core.models.variant.LiftoverWrapperParams;
@@ -40,17 +38,23 @@ public class LiftoverWrapperAnalysis extends OpenCgaToolScopeStudy {
     protected void check() throws Exception {
         super.check();
 
-        if (StringUtils.isEmpty(analysisParams.getFile())) {
+        if (CollectionUtils.isEmpty(analysisParams.getFiles())) {
             throw new ToolException("Liftover 'file' parameter is mandatory.");
         }
 
-        if (StringUtils.isEmpty(analysisParams.getOutdir())) {
-            String file = analysisParams.getFile();
-            if (file.contains("/")) {
-                analysisParams.setOutdir(file.substring(0, file.lastIndexOf('/')));
-            } else {
-                analysisParams.setOutdir("");
-            }
+        if (StringUtils.isEmpty(analysisParams.getTargetAssembly())) {
+            throw new ToolException("Liftover 'targetDirectory' parameter is mandatory, valid options are 'GRCh38' and 'hg38'.");
+        }
+
+        if (StringUtils.isEmpty(analysisParams.getVcfOutdir())) {
+//            String file = analysisParams.getFiles();
+//            if (file.contains("/")) {
+//                // Set output directory to the parent directory of the input file
+//                analysisParams.setOutdir(file.substring(0, file.lastIndexOf('/')));
+//            } else {
+//                // Set output directory to the study root directory
+//                analysisParams.setOutdir("");
+//            }
         }
     }
 
@@ -59,9 +63,11 @@ protected void run() throws Exception {
 //        setUpStorageEngineExecutor(study);
 
         step(() -> {
+            executorParams.append("opencgaHome", getOpencgaHome().toString());
             executorParams.append("study", study);
-            executorParams.append("file", analysisParams.getFile());
+            executorParams.append("files", analysisParams.getFiles());
             executorParams.append("targetAssembly", analysisParams.getTargetAssembly());
+            executorParams.append("vcfOutdir", analysisParams.getVcfOutdir());
             executorParams.append("outdir", analysisParams.getOutdir());
 
             getToolExecutor(LiftoverWrapperAnalysisExecutor.class)

opencga-analysis/src/main/java/org/opencb/opencga/analysis/wrappers/liftover/LiftoverWrapperAnalysisExecutor.java

@@ -1,5 +1,7 @@
 package org.opencb.opencga.analysis.wrappers.liftover;
 
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.tuple.ImmutablePair;
 import org.apache.commons.lang3.tuple.Pair;
 import org.opencb.opencga.analysis.wrappers.executors.DockerWrapperAnalysisExecutor;
@@ -8,6 +10,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.File;
 import java.util.*;
 
 @ToolExecutor(id = LiftoverWrapperAnalysisExecutor.ID,
@@ -26,13 +29,22 @@ public class LiftoverWrapperAnalysisExecutor extends DockerWrapperAnalysisExecut
     protected void run() throws Exception {
         StringBuilder sb = initCommandLine();
 
-        // Append mounts
+        // 1. Wrapper analysis sets the path to the liftover executable
+        Object liftoverPath = getExecutorParams().get("liftoverPath");
+        if (StringUtils.isEmpty(liftoverPath.toString())) {
+            throw new IllegalArgumentException("Missing liftoverPath parameter");
+        }
+
+        // 2. Copy liftover executable to the job output directory
+        FileUtils.copyFile(new File(liftoverPath.toString()), getOutDir().toFile());
+
+        // 3. Append mounts
         List<Pair<String, String>> inputFilenames = DockerWrapperAnalysisExecutor.getInputFilenames(null,
                 PlinkWrapperAnalysis.FILE_PARAM_NAMES, getExecutorParams());
         Map<String, String> mountMap = appendMounts(inputFilenames, sb);
 
         // Append docker image, version and command
-        appendCommand("bcftools +liftover", sb);
+        appendCommand("liftover.sh", sb);
 
         // Append input file params
         appendInputFiles(inputFilenames, mountMap, sb);
@@ -47,7 +59,7 @@ protected void run() throws Exception {
         appendOtherParams(skipParams, sb);
 
         // Execute command and redirect stdout and stderr to the files
-        logger.info("Docker command line: " + sb.toString());
+        logger.info("Docker command line: {}", sb);
         runCommandLine(sb.toString());
     }
 

opencga-app/app/analysis/liftover/liftover.sh

@@ -17,7 +17,7 @@
 #/bin/bash
 
 INPUT_FILE=$1
-TARGET_ASSEMBLY=$2
+TARGET_ASSEMBLY=$2  ## Values accepted are: GRCh38, hg38
 OUTPUT_FILE=$3
 
 if [ -z "$INPUT_FILE" ] || [ -z "$TARGET_ASSEMBLY" ] || [ -z "$OUTPUT_FILE" ]; then
@@ -74,5 +74,6 @@ fi
 bcftools +liftover --no-version -Oz $INPUT_FILE -- -s $SOURCE_REFERENCE_FILE -f $TARGET_REFERENCE_FILE -c $CHAIN_FILE --reject rejected.vcf --reject-type v --write-reject --write-src > $OUTPUT_FILE
 
 ## Clean folders
+rm *.fa
 rm *.fai
-rm *.chain.gz
+rm *.chain.gz

opencga-app/app/cloud/docker/opencga-ext-tools/Dockerfile

@@ -1,4 +1,4 @@
-FROM debian:bullseye
+FROM ubuntu:22.04
 
 LABEL org.label-schema.vendor="OpenCB" \
       org.label-schema.name="opencga-ext-tools" \
@@ -7,31 +7,31 @@ LABEL org.label-schema.vendor="OpenCB" \
       maintainer="Joaquin Tarraga <[email protected]>" \
       org.label-schema.schema-version="1.0"
 
-# WORKDIR /opt/opencga/signature.tools.lib
-
-# run update and install necessary packages
+## Run update and install necessary packages
 RUN apt-get update -y && \
     ## 1. Install system dependencies
-    DEBIAN_FRONTEND="noninteractive" TZ="Europe/London" apt-get install -y libcurl4 git libgmp-dev libcurl4-openssl-dev libgit2-dev \
+    DEBIAN_FRONTEND="noninteractive" TZ="Europe/London" apt-get install -y libcurl4 git libgmp-dev libcurl4-openssl-dev libgit2-dev build-essential \
     libssl-dev libssh-dev libxml2-dev libfontconfig1-dev libharfbuzz-dev libfribidi-dev libfreetype6-dev libpng-dev libtiff5-dev libjpeg-dev \
-    gnuplot pandoc samtools bcftools tabix fastqc plink1.9 bwa r-base && \
+    gnuplot pandoc samtools bcftools tabix fastqc plink1.9 bwa r-base wget libopenblas0-openmp libcholmod3 libsuitesparse-dev && \
+    apt-get remove libopenblas0-pthread && \
 
     ## 2. Install samtools and bcftools 1.21 and plugins
+#     apt-get update -y && apt-get install -y wget && \
     wget https://github.com/samtools/samtools/releases/download/1.21/samtools-1.21.tar.bz2 && \
     tar xjvf samtools-1.21.tar.bz2 && \
     cd samtools-1.21 && \
-    ./configure --prefix=/usr/local/bin && \
+    ./configure --prefix=/usr/local && \
     make && make install && \
     cd .. && \
 
     wget http://github.com/samtools/bcftools/releases/download/1.21/bcftools-1.21.tar.bz2 && \
     tar xjvf bcftools-1.21.tar.bz2 && \
-    wget -P bcftools-1.21 http://raw.githubusercontent.com/DrTimothyAldenDavis/SuiteSparse/stable/{SuiteSparse_config/SuiteSparse_config,CHOLMOD/Include/cholmod}.h && \
+    bash -c 'wget -P bcftools-1.21 http://raw.githubusercontent.com/DrTimothyAldenDavis/SuiteSparse/stable/{SuiteSparse_config/SuiteSparse_config,CHOLMOD/Include/cholmod}.h' && \
     cd bcftools-1.21/ && \
-    /bin/rm -f plugins/{score.{c,h},{munge,liftover,metal,blup}.c,pgs.{c,mk}} && \
-    wget -P plugins http://raw.githubusercontent.com/freeseek/score/master/{score.{c,h},{munge,liftover,metal,blup}.c,pgs.{c,mk}} && \
+    bash -c '/bin/rm -f plugins/{score.{c,h},{munge,liftover,metal,blup}.c,pgs.{c,mk}}' && \
+    bash -c 'wget -P plugins http://raw.githubusercontent.com/freeseek/score/master/{score.{c,h},{munge,liftover,metal,blup}.c,pgs.{c,mk}}' && \
     make && \
-    /bin/cp bcftools plugins/{munge,liftover,score,metal,pgs,blup}.so /usr/local/bin/ && \
+    bash -c '/bin/cp bcftools plugins/{munge,liftover,score,metal,pgs,blup}.so /usr/local/bin/' && \
     wget -P /usr/local/bin http://raw.githubusercontent.com/freeseek/score/master/assoc_plot.R && \
     chmod a+x /usr/local/bin/assoc_plot.R && \
     cd .. && \
@@ -42,33 +42,21 @@ RUN apt-get update -y && \
     R -e "BiocManager::install('BSgenome.Hsapiens.UCSC.hg38')" && \
 
     ## 4. Install signature.tools.lib installation \
-    R -e 'install.packages(c("devtools", "getopt"), repos=c("http://cran.rstudio.com/", "https://www.stats.bris.ac.uk/R/"))' && \
+    R -e "install.packages(c('devtools', 'getopt'), repos=c('http://cran.rstudio.com/', 'https://www.stats.bris.ac.uk/R/'))" && \
     git clone https://github.com/Nik-Zainal-Group/signature.tools.lib.git /opt/opencga/signature.tools.lib && \
     cd /opt/opencga/signature.tools.lib && \
     git fetch origin --tags && \
     git checkout tags/v2.4.4 && \
     sed -i '/Mmusculus/d' DESCRIPTION && \
     sed -i '/Cfamiliaris/d' DESCRIPTION && \
     sed -i '/1000genomes/d' DESCRIPTION && \
-    R -e 'options(timeout = 3600);devtools::install(repos="https://www.stats.bris.ac.uk/R/")' && \
+    R -e "options(timeout = 3600);devtools::install(repos='https://www.stats.bris.ac.uk/R/')" && \
     cd .. && \
 
     ## 5. Clean up
     rm -rf samtools-1.21 bcftools-1.21 /var/lib/apt/lists/* /tmp/* /opt/opencga/signature.tools.lib/.git && \
     strip --remove-section=.note.ABI-tag /usr/lib/x86_64-linux-gnu/libQt5Core.so.5
 
-# WORKDIR /opt/opencga/signature.tools.lib
-
-# RUN git fetch origin --tags && \
-#     git checkout tags/v2.4.4 && \
-#     sed -i '/Mmusculus/d' DESCRIPTION && \
-#     sed -i '/Cfamiliaris/d' DESCRIPTION && \
-#     sed -i '/1000genomes/d' DESCRIPTION && \
-#     R -e 'options(timeout = 3600);devtools::install(repos="https://www.stats.bris.ac.uk/R/")' && \
-#     ## Clean up
-#     rm -rf /var/lib/apt/lists/* /tmp/* /opt/opencga/signature.tools.lib/.git && \
-#     strip --remove-section=.note.ABI-tag /usr/lib/x86_64-linux-gnu/libQt5Core.so.5
-
 ENV BCFTOOLS_PLUGINS="/usr/local/bin"
 
 WORKDIR /opt/opencga

opencga-core/src/main/java/org/opencb/opencga/core/models/variant/LiftoverWrapperParams.java

@@ -18,39 +18,44 @@
 
 import org.opencb.opencga.core.tools.ToolParams;
 
+import java.util.List;
+
 public class LiftoverWrapperParams extends ToolParams {
 
     public static final String DESCRIPTION = "BCFtools +liftover plugin params";
 
-    private String file;
+    private List<String> files;
     private String targetAssembly;
+    private String vcfOutdir;
     private String outdir;
 
     public LiftoverWrapperParams() {
     }
 
-    public LiftoverWrapperParams(String file, String targetAssembly, String outdir) {
-        this.file = file;
+    public LiftoverWrapperParams(List<String> files, String targetAssembly, String vcfOutdir, String outdir) {
+        this.files = files;
         this.targetAssembly = targetAssembly;
+        this.vcfOutdir = vcfOutdir;
         this.outdir = outdir;
     }
 
     @Override
     public String toString() {
         final StringBuilder sb = new StringBuilder("LiftoverWrapperParams{");
-        sb.append("file='").append(file).append('\'');
+        sb.append("files=").append(files);
         sb.append(", targetAssembly='").append(targetAssembly).append('\'');
+        sb.append(", vcfOutdir='").append(vcfOutdir).append('\'');
         sb.append(", outdir='").append(outdir).append('\'');
         sb.append('}');
         return sb.toString();
     }
 
-    public String getFile() {
-        return file;
+    public List<String> getFiles() {
+        return files;
     }
 
-    public LiftoverWrapperParams setFile(String file) {
-        this.file = file;
+    public LiftoverWrapperParams setFiles(List<String> files) {
+        this.files = files;
         return this;
     }
 
@@ -63,6 +68,15 @@ public LiftoverWrapperParams setTargetAssembly(String targetAssembly) {
         return this;
     }
 
+    public String getVcfOutdir() {
+        return vcfOutdir;
+    }
+
+    public LiftoverWrapperParams setVcfOutdir(String vcfOutdir) {
+        this.vcfOutdir = vcfOutdir;
+        return this;
+    }
+
     public String getOutdir() {
         return outdir;
     }