PEPPER-923 . cleanup/refactor

pepper-apis/dsm-core/src/main/java/org/broadinstitute/dsm/service/onchistory/ESParticipantIdProvider.java

@@ -7,6 +7,7 @@
 import org.broadinstitute.dsm.exception.DSMBadRequestException;
 import org.broadinstitute.dsm.exception.DsmInternalError;
 import org.broadinstitute.dsm.model.elastic.Dsm;
+import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto;
 import org.broadinstitute.dsm.service.elastic.ElasticSearchService;
 
 @Slf4j
@@ -42,4 +43,14 @@ public int getParticipantIdForShortId(String shortId) {
             throw new DsmInternalError("Invalid dsm.participant.participantId for shortId " + shortId);
         }
     }
+
+    /**
+     * Given a participant short ID return a participant ID
+     * @throws DsmInternalError for bad ES behavior
+     * @throws DSMBadRequestException when no participant ID is found for short ID
+     */
+    public Optional<ElasticSearchParticipantDto> getParticipantDataForShortId(String shortId) {
+        return elasticSearchService.getParticipantDocumentByShortId(shortId, participantIndex);
+    }
+
 }

pepper-apis/dsm-core/src/main/java/org/broadinstitute/dsm/service/onchistory/OncHistoryUploadService.java

@@ -36,7 +36,6 @@
 import org.broadinstitute.dsm.model.elastic.Dsm;
 import org.broadinstitute.dsm.model.elastic.converters.camelcase.CamelCaseConverter;
 import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto;
-import org.broadinstitute.dsm.service.elastic.ElasticSearchService;
 import org.broadinstitute.lddp.db.SimpleResult;
 
 @Slf4j
@@ -50,7 +49,6 @@ public class OncHistoryUploadService {
     private String participantIndex;
     private int ddpInstanceId;
     private OncHistoryElasticUpdater elasticUpdater;
-    private final ElasticSearchService elasticSearchService = new ElasticSearchService();
     private boolean initialized;
     protected static final String ID_COLUMN = "RECORD_ID";
 
@@ -112,8 +110,8 @@ public void upload(String fileContent) {
         log.info("Validated {} rows for onc history upload", rows.size());
 
         // verify each participant ID for the study and get an associated medical record ID
-        Map<Integer, Integer> participantMedIds = getParticipantIds(rows, true);
-
+        Map<Integer, Integer> participantMedIds = getParticipantIds(rows,
+                new ESParticipantIdProvider(realm, participantIndex), true);
         log.info("Processing {} participants for onc history upload", participantMedIds.size());
 
         // ensure oncHistory record for each participant
@@ -135,17 +133,17 @@ public void upload(String fileContent) {
      *
      * @throws OncHistoryValidationException for failed verifications
      */
-    protected Map<Integer, Integer> getParticipantIds(List<OncHistoryRecord> oncHistoryRecords,
+    protected Map<Integer, Integer> getParticipantIds(List<OncHistoryRecord> oncHistoryRecords, ParticipantIdProvider participantIdProvider,
                                                       boolean updateElastic) {
         Map<Integer, Integer> medIds = new HashMap<>();
         List<String> exitedParticipants = new ArrayList<>();
 
         ParticipantDao participantDao = ParticipantDao.of();
 
-        String esIndex = new DDPInstanceDao().getDDPInstanceByInstanceName(realm).orElseThrow().getEsParticipantIndex();
+        //String esIndex = new DDPInstanceDao().getDDPInstanceByInstanceName(realm).orElseThrow().getEsParticipantIndex();
         for (OncHistoryRecord rec : oncHistoryRecords) {
-            ElasticSearchParticipantDto ptpData = elasticSearchService.getParticipantDocumentByShortId(
-                    rec.getParticipantTextId(), esIndex).orElseThrow(() -> new OncHistoryValidationException("Invalid short ID " + rec.getParticipantTextId()));
+            ElasticSearchParticipantDto ptpData = participantIdProvider.getParticipantDataForShortId(
+                    rec.getParticipantTextId()).orElseThrow(() -> new OncHistoryValidationException("Invalid short ID " + rec.getParticipantTextId()));
             if (ptpData.getStatus().isPresent() && ptpData.getStatus().get().startsWith("EXITED")) {
                 exitedParticipants.add(rec.getParticipantTextId());
             }
@@ -155,7 +153,7 @@ protected Map<Integer, Integer> getParticipantIds(List<OncHistoryRecord> oncHist
                 continue;
             }
 
-            int participantId = getParticipantIdFromElasticDoc(ptpData);
+            int participantId = getParticipantIdFromElasticDoc(ptpData, rec.getParticipantTextId());
             try {
                 ParticipantDto participant = participantDao.get(participantId).orElseThrow();
                 rec.setDdpParticipantId(participant.getDdpParticipantId().orElseThrow());
@@ -181,8 +179,7 @@ protected Map<Integer, Integer> getParticipantIds(List<OncHistoryRecord> oncHist
         return medIds;
     }
 
-    private int getParticipantIdFromElasticDoc(ElasticSearchParticipantDto ptpData) {
-        String shortId = ptpData.getProfile().get().getHruid();
+    private int getParticipantIdFromElasticDoc(ElasticSearchParticipantDto ptpData, String shortId) {
         Optional<Dsm> dsm = ptpData.getDsm();
         if (dsm.isEmpty()) {
             throw new DsmInternalError("Dsm object is empty for shortId " + shortId);

pepper-apis/dsm-core/src/main/java/org/broadinstitute/dsm/service/onchistory/ParticipantIdProvider.java

@@ -1,9 +1,15 @@
 package org.broadinstitute.dsm.service.onchistory;
 
+import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto;
+
+import java.util.Optional;
+
 public interface ParticipantIdProvider {
 
     /**
      * Given a participant short ID return a participant ID
      */
     int getParticipantIdForShortId(String shortId);
+
+    Optional<ElasticSearchParticipantDto> getParticipantDataForShortId(String shortId);
 }

pepper-apis/dsm-core/src/test/java/org/broadinstitute/dsm/service/onchistory/OncHistoryUploadServiceTest.java

@@ -21,6 +21,7 @@
 import org.broadinstitute.dsm.DbTxnBaseTest;
 import org.broadinstitute.dsm.db.MedicalRecord;
 import org.broadinstitute.dsm.db.OncHistoryDetail;
+import org.broadinstitute.dsm.db.Participant;
 import org.broadinstitute.dsm.db.dao.ddp.instance.DDPInstanceDao;
 import org.broadinstitute.dsm.db.dao.ddp.institution.DDPInstitutionDao;
 import org.broadinstitute.dsm.db.dao.ddp.medical.records.MedicalRecordDao;
@@ -33,6 +34,8 @@
 import org.broadinstitute.dsm.db.dto.onchistory.OncHistoryDto;
 import org.broadinstitute.dsm.files.parser.onchistory.OncHistoryParser;
 import org.broadinstitute.dsm.files.parser.onchistory.OncHistoryParserTest;
+import org.broadinstitute.dsm.model.elastic.Dsm;
+import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto;
 import org.broadinstitute.dsm.util.TestUtil;
 import org.junit.AfterClass;
 import org.junit.Assert;
@@ -71,7 +74,7 @@ public void testGetParticipantIds() {
         shortIdToId.put("ABC", participantId);
 
         List<OncHistoryRecord> records = createOncHistoryRecords(shortIdToId);
-        getParticipantIds(records, DEFAULT_REALM);
+        getParticipantIds(records, shortIdToId, DEFAULT_REALM);
     }
 
     //@Test
@@ -101,6 +104,11 @@ public void testLmsWriteToDb() {
         writeToDb(LMS_REALM, "onchistory/lmsOncHistory.txt");
     }
 
+    @Test
+    public void testLmsWriteToDbExit() {
+        writeToDb(LMS_REALM, "onchistory/lmsOncHistoryExited.txt");
+    }
+
     @Test
     public void testCreateOncHistoryRecords() {
         setupInstance(DEFAULT_REALM);
@@ -175,7 +183,11 @@ private void writeToDb(String realm, String testFile) {
         }
 
         // verify each participant ID for the study and get an associated medical record ID
-        Map<Integer, Integer> participantMedIds = getParticipantIds(rows, realm);
+        Map<Integer, Integer> participantMedIds = getParticipantIds(rows, shortIdToId, realm);
+        if (rows.get(0).getParticipantTextId().startsWith("xyz-exit")) {
+            // expecting an exit record and exception which was checked in getParticipantIds
+            return;
+        }
         Assert.assertEquals(2, participantMedIds.size());
 
         Map<String, OncHistoryUploadColumn> studyColumns = uploadService.getStudyColumns();
@@ -322,18 +334,27 @@ private static void compareRecord(OncHistoryRecord row, OncHistoryDetailDto reco
      * Verify the participant short ID, and get and record associated participant IDs and medical record IDs
      *
      * @param records list of records to update with participant IDs
+     * @param shortIdToId map of short ID to participant ID
      * @return map of participant ID to med record ID
      */
-    private static Map<Integer, Integer> getParticipantIds(List<OncHistoryRecord> records, String realm) {
+    private static Map<Integer, Integer> getParticipantIds(List<OncHistoryRecord> records,
+                                                           Map<String, Integer> shortIdToId, String realm) {
+        TestParticipantIdProvider participantIdProvider = new TestParticipantIdProvider(shortIdToId);
 
         OncHistoryUploadService uploadService =
                 new OncHistoryUploadService(realm, TEST_USER, new CodeStudyColumnsProvider());
 
         try {
-            return uploadService.getParticipantIds(records, false);
+            return uploadService.getParticipantIds(records, participantIdProvider, false);
         } catch (Exception e) {
-            Assert.fail("Exception from OncHistoryUploadService.getParticipantIds: " + e.toString());
-            return null;
+            if (records.get(0).getParticipantTextId().startsWith("xyz-exit")) {
+                // expecting an exit record and exception
+                Assert.assertTrue(e.getMessage().contains("One or more of the uploaded onc histories is associated with a withdrawn participant"));
+                return null;
+            } else {
+                Assert.fail("Exception from OncHistoryUploadService.getParticipantIds: " + e.toString());
+                return null;
+            }
         }
     }
 
@@ -408,6 +429,24 @@ public TestParticipantIdProvider(Map<String, Integer> shortIdToId) {
         public int getParticipantIdForShortId(String shortId) {
             return shortIdToId.get(shortId);
         }
+
+        @Override
+        public Optional<ElasticSearchParticipantDto> getParticipantDataForShortId(String shortId) {
+            Dsm dsm = new Dsm();
+            Participant participant = new Participant();
+            participant.setParticipantId(shortIdToId.get(shortId).longValue());
+            dsm.setParticipant(participant);
+
+            String participantStatus = "ENROLLED";
+            if (shortId.startsWith("xyz-exit")) {
+                participantStatus = "EXITED_AFTER_ENROLLMENT";
+            }
+            ElasticSearchParticipantDto dto = new ElasticSearchParticipantDto.Builder()
+                    .withDsm(dsm)
+                    .withStatus(participantStatus).build();
+            return Optional.of(dto);
+        }
+
     }
 
     private static class ParticipantInfo {

pepper-apis/dsm-core/src/test/resources/onchistory/lmsOncHistoryExited.txt

@@ -0,0 +1,4 @@
+RECORD_ID	FACILITY_PATH_REVIEW	FACILITY_PX	ACCESSION	DATE_PX	TYPE_PX	LOCATION_PX	HISTOLOGY	TUMOR_SIZE	VIABLE_TUMOR	NECROSIS	TX_EFFECT	BLOCKS_TO_REQUEST	SLIDES_TO_REQUEST	SLIDES_TOTAL	REQUEST_STATUS
+xyz-exit		my place	4	3/15/2005		Office		5cm x 6cm x 1cm	100	50		abc	"a,b"		review
+lmn	Office		5	2/12/2019	Unknown		None			94	None	abc	a	3	sent
+lmn	Clinic	your place		12/12/2022		Hospital		5cm x 6cm x 1cm	50			abc	"c,d"	1