Skip to content

Commit

Permalink
analysis: add a data model for the clinical analysis load result, #TA…
Browse files Browse the repository at this point in the history
…SK-4610, #TASK-4158
  • Loading branch information
jtarraga committed Sep 18, 2023
1 parent f9c2e30 commit 5ac1346
Show file tree
Hide file tree
Showing 7 changed files with 125 additions and 69 deletions.
Original file line number Diff line number Diff line change
@@ -1,19 +1,19 @@
package org.opencb.opencga.analysis.clinical;

import org.apache.commons.lang3.StringUtils;
import org.opencb.commons.datastore.core.QueryOptions;
import org.opencb.opencga.analysis.tools.OpenCgaToolScopeStudy;
import org.opencb.opencga.catalog.managers.FileManager;
import org.opencb.opencga.catalog.models.ClinicalAnalysisLoadResult;
import org.opencb.opencga.core.exceptions.ToolException;
import org.opencb.opencga.core.models.clinical.ClinicalAnalysisLoadParams;
import org.opencb.opencga.core.models.common.Enums;
import org.opencb.opencga.core.models.file.File;
import org.opencb.opencga.core.models.panel.PanelImportParams;
import org.opencb.opencga.core.tools.annotations.Tool;
import org.opencb.opencga.core.tools.annotations.ToolParams;

import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Map;

@Tool(id = ClinicalAnalysisLoadTask.ID, resource = Enums.Resource.DISEASE_PANEL, description = ClinicalAnalysisLoadTask.DESCRIPTION)
public class ClinicalAnalysisLoadTask extends OpenCgaToolScopeStudy {
Expand Down Expand Up @@ -43,6 +43,21 @@ protected void check() throws Exception {

@Override
protected void run() throws Exception {
step(() -> catalogManager.getClinicalAnalysisManager().load(getStudy(), filePath, token));
step(() -> {
ClinicalAnalysisLoadResult loadResult = catalogManager.getClinicalAnalysisManager().load(getStudy(), filePath, token);

// Add results as attributes
addAttribute("Num. clinical analyses loaded", loadResult.getNumLoaded());
addAttribute("Num. clinical analyses not loaded", loadResult.getFailures().size());
addAttribute("Loading time (in sec.)", loadResult.getTime());
addAttribute("Clinical analyses file name", loadResult.getFilename());

// Add warnings with the not loaded clinical analysis
if (loadResult.getFailures().size() > 0) {
for (Map.Entry<String, String> entry : loadResult.getFailures().entrySet()) {
addWarning("Clinical analysis " + entry.getKey() + " could not be loaded due to error: " + entry.getValue());
}
}
});
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -1062,7 +1062,7 @@ public void testPedigreeGraph() throws CatalogException {

@Test
public void testClinicalAnalysisLoading() throws IOException, ToolException, CatalogException {
String fileStr = "ca2.json.gz";
String fileStr = "clinical_analyses.json.gz";

String gzFile = getClass().getResource("/biofiles/" + fileStr).getFile();
File file = catalogManager.getFileManager().link(CANCER_STUDY, new FileLinkParams(gzFile, "ca", "", "", null, null, null, null,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import com.fasterxml.jackson.databind.ObjectReader;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.StopWatch;
import org.opencb.biodata.models.clinical.ClinicalAnalyst;
import org.opencb.biodata.models.clinical.ClinicalAudit;
import org.opencb.biodata.models.clinical.ClinicalComment;
Expand All @@ -38,6 +39,7 @@
import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException;
import org.opencb.opencga.catalog.exceptions.CatalogException;
import org.opencb.opencga.catalog.exceptions.CatalogParameterException;
import org.opencb.opencga.catalog.models.ClinicalAnalysisLoadResult;
import org.opencb.opencga.catalog.models.InternalGetDataResult;
import org.opencb.opencga.catalog.utils.Constants;
import org.opencb.opencga.catalog.utils.ParamUtils;
Expand Down Expand Up @@ -76,6 +78,7 @@
import java.io.IOException;
import java.nio.file.Path;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
Expand Down Expand Up @@ -595,13 +598,13 @@ public OpenCGAResult<ClinicalAnalysis> create(String studyStr, ClinicalAnalysis
}
}

public int load(String studyStr, Path filePath, String token) throws CatalogException, IOException {
String userId = catalogManager.getUserManager().getUserId(token);
public ClinicalAnalysisLoadResult load(String studyStr, Path filePath, String token) throws CatalogException, IOException {
ClinicalAnalysisLoadResult result = new ClinicalAnalysisLoadResult();

// Check gzip format
int counter = 0;
ObjectReader objectReader = JacksonUtils.getDefaultObjectMapper().readerFor(ClinicalAnalysis.class);

StopWatch stopWatch = StopWatch.createStarted();
try (BufferedReader br = FileUtils.newBufferedReader(filePath)) {
while (true) {
String line = br.readLine();
Expand All @@ -618,10 +621,17 @@ public int load(String studyStr, Path filePath, String token) throws CatalogExce
} catch (Exception e) {
logger.error("Error loading clinical analysis" + (clinicalAnalysis != null ? (": " + clinicalAnalysis.getId()) : "")
+ ": " + e.getMessage());
result.getFailures().put(clinicalAnalysis.getId(), e.getMessage());
}
}
}
return counter;
stopWatch.stop();

result.setNumLoaded(counter)
.setFilename(filePath.getFileName().toString())
.setTime((int) stopWatch.getTime(TimeUnit.SECONDS));

return result;
}

private void load(ClinicalAnalysis clinicalAnalysis, String study, String token) throws CatalogException {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
/*
* Copyright 2015-2020 OpenCB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.opencb.opencga.catalog.models;

import java.util.HashMap;
import java.util.Map;

public class ClinicalAnalysisLoadResult {
private int numLoaded;
private Map<String, String> failures;
private String filename;
private int time;

public ClinicalAnalysisLoadResult() {
this.numLoaded = 0;
failures = new HashMap<>();
}

public ClinicalAnalysisLoadResult(int numLoaded, Map<String, String> failures, String filename, int time) {
this.numLoaded = numLoaded;
this.failures = failures;
this.filename = filename;
this.time = time;
}

@Override
public String toString() {
final StringBuilder sb = new StringBuilder("ClinicalAnalysisLoadResult{");
sb.append("numLoaded=").append(numLoaded);
sb.append(", failures=").append(failures);
sb.append(", filename='").append(filename).append('\'');
sb.append(", time='").append(time).append('\'');
sb.append('}');
return sb.toString();
}

public int getNumLoaded() {
return numLoaded;
}

public ClinicalAnalysisLoadResult setNumLoaded(int numLoaded) {
this.numLoaded = numLoaded;
return this;
}

public Map<String, String> getFailures() {
return failures;
}

public ClinicalAnalysisLoadResult setFailures(Map<String, String> failures) {
this.failures = failures;
return this;
}

public String getFilename() {
return filename;
}

public ClinicalAnalysisLoadResult setFilename(String filename) {
this.filename = filename;
return this;
}

public int getTime() {
return time;
}

public ClinicalAnalysisLoadResult setTime(int time) {
this.time = time;
return this;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
import org.opencb.opencga.catalog.db.api.ClinicalAnalysisDBAdaptor;
import org.opencb.opencga.catalog.db.api.InterpretationDBAdaptor;
import org.opencb.opencga.catalog.exceptions.CatalogException;
import org.opencb.opencga.catalog.models.ClinicalAnalysisLoadResult;
import org.opencb.opencga.catalog.utils.Constants;
import org.opencb.opencga.catalog.utils.ParamUtils;
import org.opencb.opencga.core.api.ParamConstants;
Expand Down Expand Up @@ -3496,15 +3497,17 @@ public void fetchCasesWithSameProbandAndDifferentSample() throws CatalogExceptio

@Test
public void loadClinicalAnalysesTest() throws CatalogException, IOException {
String gzFile = getClass().getResource("/biofiles/ca2.json.gz").getFile();
String gzFile = getClass().getResource("/biofiles/clinical_analyses.json.gz").getFile();
File file = catalogManager.getFileManager().link(STUDY, new FileLinkParams(gzFile, "", "", "", null, null, null, null,
null), false, sessionIdUser).first();

Path filePath = Paths.get(file.getUri());

System.out.println("Loading clinical analyses file: " + filePath + " ....");
int numLoaded = catalogManager.getClinicalAnalysisManager().load(STUDY, filePath, sessionIdUser);
System.out.println("\t\t.... " + numLoaded + " clinical analyses loaded from file " + filePath);
ClinicalAnalysisLoadResult loadResult = catalogManager.getClinicalAnalysisManager().load(STUDY, filePath, sessionIdUser);
System.out.println(loadResult);

Assert.assertEquals(1, loadResult.getFailures().size());

String ca1Id = "SAP-45016-1";
String ca2Id = "OPA-6607-1";
Expand Down

This file was deleted.

Binary file not shown.

0 comments on commit 5ac1346

Please sign in to comment.