Merge branch 'develop' of https://github.com/opencb/cellbase into dev…

…elop

.github/workflows/junit.yml

@@ -22,6 +22,7 @@ jobs:
     - name: Clone and build java-common-libs
       run: cd ~ && git clone -b develop https://github.com/opencb/java-common-libs.git && cd ~/java-common-libs && mvn -T 2 clean install -DskipTests && cd $GITHUB_WORKSPACE
     - name: Clone and build biodata
-      run: cd ~ && git clone -b develop https://github.com/opencb/biodata.git && cd ~/biodata && mvn -T 2 clean install -DskipTests && cd $GITHUB_WORKSPACE
+      # remove copying over the .mvn directory once the HTTP repo is removed
+      run: cd ~ && git clone -b develop https://github.com/opencb/biodata.git && cd ~/biodata && cp -ar $GITHUB_WORKSPACE/.mvn . && mvn -T 2 clean install -DskipTests && cd $GITHUB_WORKSPACE
     - name: Build with Maven
       run: mvn -T 2 clean install

.github/workflows/release.yml

@@ -1,6 +1,7 @@
 name: CellBase Release workflow
 
 on:
+  workflow_dispatch: {}
   push:
     tags:
       - '*'
@@ -22,33 +23,33 @@ jobs:
         run: mvn clean install -DskipTests
       - name: Maven Validate
         run: mvn validate
-  test:
-    name: CellBase Test
-    runs-on: ubuntu-18.04
-    needs: build
-    strategy:
-      matrix:
-        java: ["1.8"]
-        mongodb: ["4.0", "4.2"]
-    services:
-      mongodb:
-        image: mongo:${{ matrix.mongodb }}
-        ports:
-          - 27017:27017
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: '10'
-      - name: Set up JDK ${{ matrix.java }}
-        uses: actions/setup-java@v1
-        with:
-          java-version: ${{ matrix.java }}
-      - name: Build with Maven
-        run: mvn clean install -Dcheckstyle.skip
+#  test:
+#    name: CellBase Test
+#    runs-on: ubuntu-18.04
+#    needs: build
+#    strategy:
+#      matrix:
+#        java: ["1.8"]
+#        mongodb: ["4.0", "4.2"]
+#    services:
+#      mongodb:
+#        image: mongo:${{ matrix.mongodb }}
+#        ports:
+#          - 27017:27017
+#    steps:
+#      - uses: actions/checkout@v2
+#        with:
+#          fetch-depth: '10'
+#      - name: Set up JDK ${{ matrix.java }}
+#        uses: actions/setup-java@v1
+#        with:
+#          java-version: ${{ matrix.java }}
+#      - name: Build with Maven
+#        run: mvn clean install -Dcheckstyle.skip
   deploy:
     name: Build and deploy CellBase libs, Java docs and Docker images in Maven Central and DockerHub repositories
     runs-on: ubuntu-18.04
-    needs: test
+#    needs: test
     steps:
       - uses: actions/checkout@v2
         with:

cellbase-app/app/cloud/docker/cellbase-base/Dockerfile

@@ -10,7 +10,10 @@ LABEL org.label-schema.vendor="OpenCB" \
 ENV CELLBASE_USER cellbase
 ENV CELLBASE_HOME /opt/cellbase/
 
-RUN addgroup -S $CELLBASE_USER && adduser -S $CELLBASE_USER -G $CELLBASE_USER -u 1001
+RUN apk update && apk upgrade && apk add ca-certificates openssl wget bash \
+    && update-ca-certificates \
+    && addgroup -S $CELLBASE_USER && adduser -S $CELLBASE_USER -G $CELLBASE_USER -u 1001
+
 USER $CELLBASE_USER
 
 VOLUME /opt/cellbase/conf

cellbase-app/app/cloud/docker/cellbase-base/Dockerfile.ubuntu

@@ -0,0 +1,21 @@
+FROM ubuntu:20.04
+
+LABEL org.label-schema.vendor="OpenCB" \
+      org.label-schema.name="cellbase-base" \
+      org.label-schema.url="http://docs.opencb.org/display/cellbase" \
+      org.label-schema.description="An Open Computational Genomics Analysis platform for big data processing and analysis in genomics" \
+      maintainer="Julie Sullivan <[email protected]>" \
+      org.label-schema.schema-version="1.0"
+
+ENV CELLBASE_USER cellbase
+ENV CELLBASE_HOME /opt/cellbase/
+
+RUN apt-get update && apt-get install -y openjdk-8-jre wget htop vim \
+    && adduser --uid 1001 $CELLBASE_USER
+#&& addgroup --gid 1001 $CELLBASE_USER &&
+USER $CELLBASE_USER
+
+VOLUME /opt/cellbase/conf
+
+COPY . /opt/cellbase
+WORKDIR /opt/cellbase

cellbase-app/app/cloud/docker/docker-build.py

@@ -12,7 +12,7 @@
 ## Configure command-line options
 parser = argparse.ArgumentParser()
 parser.add_argument('action', help="Action to execute", choices=["build", "push", "delete"], default="build")
-parser.add_argument('--images', help="comma separated list of images to be made, e.g. base,rest,python,builder", default="base,rest,python")
+parser.add_argument('--images', help="comma separated list of images to be made, e.g. base,rest,python,builder", default="base,rest")
 parser.add_argument('--tag', help="the tag for this code, e.g. v5.0.0")
 parser.add_argument('--build-folder', help="the location of the build folder, if not default location")
 parser.add_argument('--username', help="credentials for dockerhub (REQUIRED if deleting from DockerHub)")
@@ -139,7 +139,8 @@ def delete():
 
 # 4. init images: get a list with all images
 if args.images is None:
-    images = ["base", "rest", "python"]
+    # TODO think about "python"
+    images = ["base", "rest"]
 else:
     images = args.images.split(",")
 

cellbase-app/app/cloud/kubernetes/charts/cellbase/templates/worker-deployment.yaml

@@ -6,6 +6,8 @@ metadata:
     app: {{  .Values.worker.name  }}
 spec:
   replicas: {{ .Values.worker.replicaCount }}
+  strategy:
+    type: Recreate
   selector:
     matchLabels:
       app: {{  .Values.worker.name  }}

cellbase-app/pom.xml

@@ -6,7 +6,7 @@
     <parent>
         <groupId>org.opencb.cellbase</groupId>
         <artifactId>cellbase</artifactId>
-        <version>5.0.0-SNAPSHOT</version>
+        <version>5.1.0-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

cellbase-app/src/main/java/org/opencb/cellbase/app/cli/main/annotation/PopulationFrequenciesAnnotator.java

@@ -127,8 +127,8 @@ private CellBaseDataResult<Variant> getPopulationFrequencies(Variant variant) {
                 flagVisitedVariant(variantKey, variant1);
 
                 populationFrequencyCellBaseDataResult.setResults(Collections.singletonList(variant1));
-                populationFrequencyCellBaseDataResult.setNumTotalResults(1);
                 populationFrequencyCellBaseDataResult.setNumResults(1);
+                populationFrequencyCellBaseDataResult.setNumMatches(1);
             }
         } catch (RocksDBException | IOException e) {
             e.printStackTrace();

cellbase-app/src/main/java/org/opencb/cellbase/app/cli/main/annotation/VcfVariantAnnotator.java

@@ -145,8 +145,8 @@ private CellBaseDataResult<Variant> getCustomAnnotation(Variant variant) {
                 Variant variant1 = mapper.readValue(dbContent, Variant.class);
 
                 customAnnotationCellBaseDataResult.setResults(Collections.singletonList(variant1));
-                customAnnotationCellBaseDataResult.setNumTotalResults(1);
                 customAnnotationCellBaseDataResult.setNumResults(1);
+                customAnnotationCellBaseDataResult.setNumMatches(1);
             }
         } catch (RocksDBException | IOException e) {
             e.printStackTrace();

cellbase-client/pom.xml

@@ -6,7 +6,7 @@
     <parent>
         <groupId>org.opencb.cellbase</groupId>
         <artifactId>cellbase</artifactId>
-        <version>5.0.0-SNAPSHOT</version>
+        <version>5.1.0-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

cellbase-client/src/main/java/org/opencb/cellbase/client/config/RestConfig.java

@@ -24,7 +24,7 @@
 public class RestConfig {
 
     private List<String> hosts;
-    private int timeout;
+    private int timeout = 60000;
 
     public RestConfig() {
     }

cellbase-client/src/main/java/org/opencb/cellbase/client/rest/ParentRestClient.java

@@ -24,6 +24,7 @@
 import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
 import com.fasterxml.jackson.databind.util.StdConverter;
 import org.apache.commons.lang3.StringUtils;
+import org.glassfish.jersey.client.ClientProperties;
 import org.opencb.biodata.models.variant.avro.ConsequenceType;
 import org.opencb.biodata.models.variant.avro.DrugResponseClassification;
 import org.opencb.biodata.models.variant.avro.GeneCancerAssociation;
@@ -92,9 +93,13 @@ public ParentRestClient(String species, String assembly, ClientConfiguration con
         this.species = species;
         this.assembly = assembly;
         this.configuration = configuration;
+        logger = LoggerFactory.getLogger(this.getClass().toString());
 
         this.client = ClientBuilder.newClient();
-        logger = LoggerFactory.getLogger(this.getClass().toString());
+        client.property(ClientProperties.CONNECT_TIMEOUT, 1000);
+        client.property(ClientProperties.READ_TIMEOUT, configuration.getRest().getTimeout());
+
+        logger.debug("Configure read timeout : " + configuration.getRest().getTimeout() + "ms");
     }
 
     static {
@@ -204,6 +209,7 @@ protected <U> CellBaseDataResponse<U> execute(List<String> idList, String resour
                 ? options.getInt("numThreads", DEFAULT_NUM_THREADS)
                 : DEFAULT_NUM_THREADS;
 
+        // TODO: Use cached thread pool
         ExecutorService executorService = Executors.newFixedThreadPool(numThreads);
         List<Future<CellBaseDataResponse<U>>> futureList = new ArrayList<>((idList.size() / REST_CALL_BATCH_SIZE) + 1);
         for (int i = 0; i < idList.size(); i += REST_CALL_BATCH_SIZE) {
@@ -223,8 +229,11 @@ protected <U> CellBaseDataResponse<U> execute(List<String> idList, String resour
                     Thread.sleep(5);
                 }
                 cellBaseDataResults.addAll(responseFuture.get().getResponses());
-            } catch (InterruptedException | ExecutionException e) {
-                e.printStackTrace();
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                throw new IOException(e);
+            } catch (ExecutionException e) {
+                throw new IOException(e);
             }
         }
 
@@ -248,10 +257,9 @@ private <U> CellBaseDataResponse<U> fetchData(List<String> idList, String resour
         List<String> newIdsList = null;
         boolean call = true;
         int skip = 0;
-        CellBaseDataResponse<U> queryResponse = null;
         CellBaseDataResponse<U> finalDataResponse = null;
         while (call) {
-            queryResponse = robustRestCall(idList, resource, options, clazz, post);
+            CellBaseDataResponse<U> queryResponse = robustRestCall(idList, resource, options, clazz, post);
 
             // First iteration we set the response object, no merge needed
             // Create id -> finalDataResponse-position map, so that we can know in forthcoming iterations where to
@@ -296,8 +304,12 @@ private <U> CellBaseDataResponse<U> fetchData(List<String> idList, String resour
                 options.put("skip", skip);
             }
         }
+        logger.debug("queryResponse: {"
+                + "time: " + finalDataResponse.getTime() + ", "
+                + "apiVersion: " + finalDataResponse.getApiVersion() + ", "
+                + "responses: " + finalDataResponse.getResponses().size() + ", "
+                + "events: " + finalDataResponse.getEvents() + "}");
 
-        logger.debug("queryResponse = " + queryResponse);
         return finalDataResponse;
     }
 

cellbase-core/pom.xml

@@ -6,7 +6,7 @@
     <parent>
         <groupId>org.opencb.cellbase</groupId>
         <artifactId>cellbase</artifactId>
-        <version>5.0.0-SNAPSHOT</version>
+        <version>5.1.0-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

cellbase-core/src/main/java/org/opencb/cellbase/core/variant/PhasedQueryManager.java

@@ -224,7 +224,7 @@ private boolean alternateAlleleMatch(String allele, String allele1) {
     protected void reset(CellBaseDataResult<Variant> variantCellBaseDataResult) {
         variantCellBaseDataResult.setResults(Collections.emptyList());
         variantCellBaseDataResult.setNumResults(0);
-        variantCellBaseDataResult.setNumTotalResults(0);
+        variantCellBaseDataResult.setNumMatches(0);
     }
 
 

cellbase-lib/pom.xml

@@ -6,7 +6,7 @@
     <parent>
         <groupId>org.opencb.cellbase</groupId>
         <artifactId>cellbase</artifactId>
-        <version>5.0.0-SNAPSHOT</version>
+        <version>5.1.0-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 

cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/GeneBuilder.java

@@ -89,7 +89,7 @@ public GeneBuilder(Path geneDirectoryPath, Path genomeSequenceFastaFile, Species
                        boolean flexibleGTFParsing, CellBaseSerializer serializer) throws CellBaseException {
         this(null, geneDirectoryPath.resolve("description.txt"),
                 geneDirectoryPath.resolve("xrefs.txt"),
-                geneDirectoryPath.resolve("MANE.GRCh38.v0.91.summary.txt.gz"),
+                geneDirectoryPath.resolve("MANE.GRCh38.v0.93.summary.txt.gz"),
                 geneDirectoryPath.resolve("list_LRGs_transcripts_xrefs.txt"),
                 geneDirectoryPath.resolve("idmapping_selected.tab.gz"),
                 geneDirectoryPath.getParent().resolve("regulation/motif_features.gff.gz"),
@@ -448,13 +448,11 @@ private Transcript getTranscript(Gene gene, EnsemblGeneBuilderIndexer indexer, T
         if (StringUtils.isNotEmpty(maneRefSeq)) {
             String tso500Flag = indexer.getTSO500(maneRefSeq.split("\\.")[0]);
             if (StringUtils.isNotEmpty(tso500Flag)) {
-                System.out.println("tso500Flag = " + tso500Flag);
                 transcript.getFlags().add(tso500Flag);
             }
 
             String eglhHaemOncFlag = indexer.getEGLHHaemOnc(maneRefSeq.split("\\.")[0]);
             if (StringUtils.isNotEmpty(eglhHaemOncFlag)) {
-                System.out.println("eglhHaemOncFlag = " + eglhHaemOncFlag);
                 transcript.getFlags().add(eglhHaemOncFlag);
             }
         }
@@ -528,10 +526,15 @@ private void addGtfXrefs(Transcript transcript, Gene gene, Map<String, String> g
         if (transcript.getXrefs() == null) {
             transcript.setXrefs(new ArrayList<>());
         }
+
         transcript.getXrefs().add(new Xref(gene.getId(), "ensembl_gene", "Ensembl Gene"));
-        transcript.getXrefs().add(new Xref(gene.getName(), "hgnc_symbol", "HGNC Symbol"));
         transcript.getXrefs().add(new Xref(transcript.getId(), "ensembl_transcript", "Ensembl Transcript"));
-        transcript.getXrefs().add(new Xref(transcript.getName(), "ensembl_transcript_name", "Ensembl Transcript Name"));
+
+        // Some non-coding genes do not have Gene names
+        if (StringUtils.isNotEmpty(gene.getName())) {
+            transcript.getXrefs().add(new Xref(gene.getName(), "hgnc_symbol", "HGNC Symbol"));
+            transcript.getXrefs().add(new Xref(transcript.getName(), "ensembl_transcript_name", "Ensembl Transcript Name"));
+        }
 
         if (gtfAttributes.get("ccds_id") != null) {
             transcript.getXrefs().add(new Xref(gtfAttributes.get("ccds_id"), "ccds_id", "CCDS"));

cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/RefSeqGeneBuilder.java

@@ -120,9 +120,9 @@ private void getCdnaFastaFileFromDirectoryPath(Path refSeqDirectoryPath) {
     public void parse() throws Exception {
         // Preparing the fasta file for fast accessing
         FastaIndex fastaIndex = null;
-//        if (fastaFile != null) {
-//            fastaIndex = new FastaIndex(fastaFile);
-//        }
+        if (fastaFile != null) {
+            fastaIndex = new FastaIndex(fastaFile);
+        }
 
         // index protein sequences for later
         RefSeqGeneBuilderIndexer indexer = new RefSeqGeneBuilderIndexer(gtfFile.getParent());

cellbase-lib/src/main/java/org/opencb/cellbase/lib/builders/clinical/variant/ClinVarIndexer.java

@@ -837,6 +837,9 @@ private SequenceLocation parseSequenceLocation(String[] parts) {
         if (emptySequence(reference) && !emptySequence(alternate) && end == (start + 1)) {
             // NOTE! swapped start and end
             return new SequenceLocation(chromosome, end, start, reference, alternate);
+        } else if (alternate.length() == 1 && reference.length() > 1 && reference.startsWith(alternate)) {
+            // variant summary file has the wrong location for deletions. Adjust!
+            return new SequenceLocation(chromosome, start - 1, end, reference, alternate);
         } else {
             return new SequenceLocation(chromosome, start, end, reference, alternate);
         }