PEPPER-923 Restrict onchistory upload for exited participants

pepper-apis/dsm-core/src/main/java/org/broadinstitute/dsm/service/onchistory/ESParticipantIdProvider.java

@@ -7,6 +7,7 @@
 import org.broadinstitute.dsm.exception.DSMBadRequestException;
 import org.broadinstitute.dsm.exception.DsmInternalError;
 import org.broadinstitute.dsm.model.elastic.Dsm;
+import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto;
 import org.broadinstitute.dsm.service.elastic.ElasticSearchService;
 
 @Slf4j
@@ -27,6 +28,18 @@ public ESParticipantIdProvider(String realm, String participantIndex) {
      * @throws DSMBadRequestException when no participant ID is found for short ID
      */
     public int getParticipantIdForShortId(String shortId) {
+        //getParticipantDataForShortId does all the validations to make sure participant ID exists
+        return getParticipantDataForShortId(shortId).getDsm().get().getParticipant().get().getParticipantId().intValue();
+    }
+
+    /**
+     * Given a participant short ID return a participant ID
+     * @throws DsmInternalError for bad ES behavior
+     * @throws DSMBadRequestException when no participant ID is found for short ID
+     */
+    public ElasticSearchParticipantDto getParticipantDataForShortId(String shortId) {
+        ElasticSearchParticipantDto ptpData = elasticSearchService.getParticipantDocumentByShortId(shortId, participantIndex)
+                .orElseThrow(() -> new DSMBadRequestException("No participant found for shortId " + shortId));
         Dsm dsm = elasticSearchService.getParticipantDsmByShortId(shortId, participantIndex);
         Optional<Participant> dsmParticipant = dsm.getParticipant();
         if (dsmParticipant.isEmpty()) {
@@ -37,9 +50,10 @@ public int getParticipantIdForShortId(String shortId) {
             throw new DsmInternalError("ES returned empty dsm.participant.participantId object for shortId " + shortId);
         }
         try {
-            return participantID.intValue();
+            return ptpData;
         } catch (Exception e) {
             throw new DsmInternalError("Invalid dsm.participant.participantId for shortId " + shortId);
         }
     }
+
 }

pepper-apis/dsm-core/src/main/java/org/broadinstitute/dsm/service/onchistory/OncHistoryUploadService.java

@@ -32,6 +32,7 @@
 import org.broadinstitute.dsm.exception.DsmInternalError;
 import org.broadinstitute.dsm.files.parser.onchistory.OncHistoryParser;
 import org.broadinstitute.dsm.model.elastic.converters.camelcase.CamelCaseConverter;
+import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto;
 import org.broadinstitute.lddp.db.SimpleResult;
 
 @Slf4j
@@ -125,23 +126,35 @@ public void upload(String fileContent) {
     /**
      * Given participant short IDs in uploaded rows, verify the short ID, and get and record associated
      * participant IDs and medical record IDs
+     * verify if any exited participant(s) exists in passed onc history records.
      *
-     * @throws OncHistoryValidationException for failed verification
+     * @throws OncHistoryValidationException for failed verifications
      */
-    protected Map<Integer, Integer> getParticipantIds(List<OncHistoryRecord> oncHistoryRecords,
-                                                      ParticipantIdProvider participantIdProvider, boolean updateElastic) {
+    protected Map<Integer, Integer> getParticipantIds(List<OncHistoryRecord> oncHistoryRecords, ParticipantIdProvider participantIdProvider,
+                                                      boolean updateElastic) {
         Map<Integer, Integer> medIds = new HashMap<>();
+        List<String> exitedParticipants = new ArrayList<>();
 
         ParticipantDao participantDao = ParticipantDao.of();
 
         for (OncHistoryRecord rec : oncHistoryRecords) {
-            int participantId = participantIdProvider.getParticipantIdForShortId(rec.getParticipantTextId());
+            ElasticSearchParticipantDto ptpData = participantIdProvider.getParticipantDataForShortId(rec.getParticipantTextId());
+            if (ptpData.getStatus().isPresent() && ptpData.getStatus().get().startsWith("EXITED")) {
+                exitedParticipants.add(rec.getParticipantTextId());
+            }
 
+            if (!exitedParticipants.isEmpty()) {
+                //skip MR verify/creation. continuing to collect any other exited participant hruids.
+                continue;
+            }
+
+            //since participantIdProvider.getParticipantDataForShortId already made sure participant ID exists.. its ok to get it directly
+            int participantId = ptpData.getDsm().get().getParticipant().get().getParticipantId().intValue();
             try {
                 ParticipantDto participant = participantDao.get(participantId).orElseThrow();
                 rec.setDdpParticipantId(participant.getDdpParticipantId().orElseThrow());
             } catch (Exception e) {
-                throw new DsmInternalError("Participant not found for id " + participantId, e);
+                throw new DsmInternalError("Failed to get Participant/Participant Id for " + rec.getParticipantTextId(), e);
             }
 
             rec.setParticipantId(participantId);
@@ -152,6 +165,13 @@ protected Map<Integer, Integer> getParticipantIds(List<OncHistoryRecord> oncHist
                     this.realm, updateElastic);
             medIds.put(participantId, medId);
         }
+
+        if (!exitedParticipants.isEmpty()) {
+            log.error("Found {} exited participants: {} in Onc History Upload", exitedParticipants.size(), exitedParticipants);
+            throw new OncHistoryValidationException("One or more of the uploaded onc histories is associated with a withdrawn participant. "
+                    + "Please remove onc histories for these withdrawn participants from the file and upload it again: " + exitedParticipants);
+        }
+
         return medIds;
     }
 
@@ -383,4 +403,6 @@ public Map<String, OncHistoryUploadColumn> initColumnsForStudy() {
     protected Map<String, OncHistoryUploadColumn> getStudyColumns() {
         return studyColumns;
     }
+
+
 }

pepper-apis/dsm-core/src/main/java/org/broadinstitute/dsm/service/onchistory/ParticipantIdProvider.java

@@ -1,9 +1,15 @@
 package org.broadinstitute.dsm.service.onchistory;
 
+import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto;
+
+import java.util.Optional;
+
 public interface ParticipantIdProvider {
 
     /**
      * Given a participant short ID return a participant ID
      */
     int getParticipantIdForShortId(String shortId);
+
+    ElasticSearchParticipantDto getParticipantDataForShortId(String shortId);
 }

pepper-apis/dsm-core/src/test/java/org/broadinstitute/dsm/service/onchistory/OncHistoryUploadServiceTest.java

@@ -21,6 +21,7 @@
 import org.broadinstitute.dsm.DbTxnBaseTest;
 import org.broadinstitute.dsm.db.MedicalRecord;
 import org.broadinstitute.dsm.db.OncHistoryDetail;
+import org.broadinstitute.dsm.db.Participant;
 import org.broadinstitute.dsm.db.dao.ddp.instance.DDPInstanceDao;
 import org.broadinstitute.dsm.db.dao.ddp.institution.DDPInstitutionDao;
 import org.broadinstitute.dsm.db.dao.ddp.medical.records.MedicalRecordDao;
@@ -33,6 +34,8 @@
 import org.broadinstitute.dsm.db.dto.onchistory.OncHistoryDto;
 import org.broadinstitute.dsm.files.parser.onchistory.OncHistoryParser;
 import org.broadinstitute.dsm.files.parser.onchistory.OncHistoryParserTest;
+import org.broadinstitute.dsm.model.elastic.Dsm;
+import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto;
 import org.broadinstitute.dsm.util.TestUtil;
 import org.junit.AfterClass;
 import org.junit.Assert;
@@ -101,6 +104,11 @@ public void testLmsWriteToDb() {
         writeToDb(LMS_REALM, "onchistory/lmsOncHistory.txt");
     }
 
+    @Test
+    public void testLmsWriteToDbExit() {
+        writeToDb(LMS_REALM, "onchistory/lmsOncHistoryExited.txt");
+    }
+
     @Test
     public void testCreateOncHistoryRecords() {
         setupInstance(DEFAULT_REALM);
@@ -176,6 +184,10 @@ private void writeToDb(String realm, String testFile) {
 
         // verify each participant ID for the study and get an associated medical record ID
         Map<Integer, Integer> participantMedIds = getParticipantIds(rows, shortIdToId, realm);
+        if (rows.get(0).getParticipantTextId().startsWith("xyz-exit")) {
+            // expecting an exit record and exception which was checked in getParticipantIds
+            return;
+        }
         Assert.assertEquals(2, participantMedIds.size());
 
         Map<String, OncHistoryUploadColumn> studyColumns = uploadService.getStudyColumns();
@@ -335,8 +347,14 @@ private static Map<Integer, Integer> getParticipantIds(List<OncHistoryRecord> re
         try {
             return uploadService.getParticipantIds(records, participantIdProvider, false);
         } catch (Exception e) {
-            Assert.fail("Exception from OncHistoryUploadService.getParticipantIds: " + e.toString());
-            return null;
+            if (records.get(0).getParticipantTextId().startsWith("xyz-exit")) {
+                // expecting an exit record and exception
+                Assert.assertTrue(e.getMessage().contains("One or more of the uploaded onc histories is associated with a withdrawn participant"));
+                return null;
+            } else {
+                Assert.fail("Exception from OncHistoryUploadService.getParticipantIds: " + e.toString());
+                return null;
+            }
         }
     }
 
@@ -411,6 +429,24 @@ public TestParticipantIdProvider(Map<String, Integer> shortIdToId) {
         public int getParticipantIdForShortId(String shortId) {
             return shortIdToId.get(shortId);
         }
+
+        @Override
+        public ElasticSearchParticipantDto getParticipantDataForShortId(String shortId) {
+            Dsm dsm = new Dsm();
+            Participant participant = new Participant();
+            participant.setParticipantId(shortIdToId.get(shortId).longValue());
+            dsm.setParticipant(participant);
+
+            String participantStatus = "ENROLLED";
+            if (shortId.startsWith("xyz-exit")) {
+                participantStatus = "EXITED_AFTER_ENROLLMENT";
+            }
+            ElasticSearchParticipantDto dto = new ElasticSearchParticipantDto.Builder()
+                    .withDsm(dsm)
+                    .withStatus(participantStatus).build();
+            return dto;
+        }
+
     }
 
     private static class ParticipantInfo {

pepper-apis/dsm-core/src/test/resources/onchistory/lmsOncHistoryExited.txt

@@ -0,0 +1,4 @@
+RECORD_ID	FACILITY_PATH_REVIEW	FACILITY_PX	ACCESSION	DATE_PX	TYPE_PX	LOCATION_PX	HISTOLOGY	TUMOR_SIZE	VIABLE_TUMOR	NECROSIS	TX_EFFECT	BLOCKS_TO_REQUEST	SLIDES_TO_REQUEST	SLIDES_TOTAL	REQUEST_STATUS
+xyz-exit		my place	4	3/15/2005		Office		5cm x 6cm x 1cm	100	50		abc	"a,b"		review
+lmn	Office		5	2/12/2019	Unknown		None			94	None	abc	a	3	sent
+lmn	Clinic	your place		12/12/2022		Hospital		5cm x 6cm x 1cm	50			abc	"c,d"	1