PEPPER-665 . participant list down Race's ethnicity (nested options) …

…fix (#2927)

pepper-apis/dsm-core/src/main/java/org/broadinstitute/dsm/model/elastic/export/tabular/TabularParticipantParser.java

@@ -224,8 +224,16 @@ private void addModuleDataToParticipantMap(ModuleExportConfig moduleConfig, Map<
                                                Map<String, Object> esModuleMap, int moduleRepeatNum) {
         for (FilterExportConfig filterConfig : moduleConfig.getQuestions()) {
             List<Map<String, Object>> options = Collections.singletonList(null);
+            List<Map<String, Object>> nestedOptions = new ArrayList<>();
             if (filterConfig.getOptions() != null && filterConfig.isSplitOptionsIntoColumns()) {
                 options = filterConfig.getOptions();
+                //add nestedOptions as options
+                options.stream().filter(option -> option.get(ESObjectConstants.NESTED_OPTIONS) != null).forEach(option -> {
+                    nestedOptions.addAll((List<Map<String, Object>>) option.get(ESObjectConstants.NESTED_OPTIONS));
+                });
+                if (!nestedOptions.isEmpty()) {
+                    options.addAll(nestedOptions);
+                }
             }
             TextValueProvider valueProvider =
                     ValueProviderFactory.getValueProvider(filterConfig.getColumn().getName(), filterConfig.getType());

pepper-apis/dsm-core/src/main/java/org/broadinstitute/dsm/model/elastic/export/tabular/renderer/PickListValueProvider.java

@@ -1,5 +1,6 @@
 package org.broadinstitute.dsm.model.elastic.export.tabular.renderer;
 
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
@@ -28,7 +29,7 @@ public List<String> formatRawValues(List<?> rawValues, FilterExportConfig filter
         }).collect(Collectors.toList());
     }
 
-    // adds extra logic to handle grouped options
+    // adds extra logic to handle grouped options & nested options
     protected Object extractValuesFromAnswer(Map<String, Object> targetAnswer, FilterExportConfig filterConfig) {
         if (targetAnswer == null) {
             return null;
@@ -40,6 +41,18 @@ protected Object extractValuesFromAnswer(Map<String, Object> targetAnswer, Filte
             List<?> allValues = (List<String>) optGroups.values().stream().flatMap(Collection::stream).collect(Collectors.toList());
             return allValues;
         }
+
+        Object nestedOptions = targetAnswer.get(ESObjectConstants.NESTED_OPTIONS);
+        // nestedOptions is a map of parent option -> array of nested option choices
+        if (nestedOptions instanceof Map && (!((Map) nestedOptions).isEmpty())) {
+            Map<String, List> nestedOptionsMap = (Map<String, List>) nestedOptions;
+            //get main options selected
+            List<String> allValues = (List<String>) super.extractValuesFromAnswer(targetAnswer, filterConfig);
+            //get nested options selected for the main options
+            List<String> allNestedValues = (List<String>) nestedOptionsMap.values().stream().flatMap(Collection::stream).collect(Collectors.toList());
+            allValues.addAll(allNestedValues);
+            return allValues;
+        }
         return super.extractValuesFromAnswer(targetAnswer, filterConfig);
 
     }

pepper-apis/dsm-core/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java

@@ -115,6 +115,7 @@ public class ESObjectConstants {
     public static final String ALLOW_MULTIPLE = "allowMultiple";
     public static final String OPTION_GROUPS = "groups";
     public static final String GROUPED_OPTIONS = "groupedOptions";
+    public static final String NESTED_OPTIONS = "nestedOptions";
     public static final String CHILD_QUESTIONS = "childQuestions";
     public static final String DETAIL = "detail";
     public static final String DETAILS = "details";

pepper-apis/dsm-core/src/test/java/org/broadinstitute/dsm/service/download/DownloadParticipantListServiceTest.java

@@ -39,9 +39,12 @@ public class DownloadParticipantListServiceTest extends DbAndElasticBaseTest {
 
     private static final String instanceName = "download_test_instance";
     private static final String shortId = "PT_SHORT";
+    private static final String FIRST_NAME = "testDataDownloadFromElastic";
+    private static final String LAST_NAME = "lastName";
+    private static final String AY_RACE_DOWNLOAD_FILE = "elastic/participantListAboutYouRaceDownload.json";
     private static String esIndex;
     private static DDPInstanceDto ddpInstanceDto;
-    private static String ddpParticipantId = "PT_SAMPLE_QUEUE_TEST";
+    private static String ddpParticipantId = "PT_DOWNLOAD_TEST";
     private static ParticipantDto participantDto = null;
 
     @BeforeClass
@@ -51,10 +54,11 @@ public static void doFirst() {
         ddpParticipantId = TestParticipantUtil.genDDPParticipantId(ddpParticipantId);
         participantDto = TestParticipantUtil.createParticipant(ddpParticipantId, ddpInstanceDto.getDdpInstanceId());
         ElasticTestUtil.createParticipant(esIndex, participantDto);
-        ElasticTestUtil.addParticipantProfileFromTemplate(esIndex, ddpParticipantId, shortId, "testDataDownloadFromElastic",
-                "lastName", "email");
+        ElasticTestUtil.addParticipantProfileFromTemplate(esIndex, ddpParticipantId, shortId, FIRST_NAME,
+                LAST_NAME, "email");
         ElasticTestUtil.addDsmEntityFromFile(esIndex, "elastic/dsmKitRequestShippingClinicalOrders.json", ddpParticipantId, "1990-10-10",
                 null);
+        ElasticTestUtil.addActivitiesFromFile(esIndex, "elastic/lmsAboutYouActivity.json", ddpParticipantId);
         log.debug("ES participant record with DSM for {}: {}", ddpParticipantId,
                 ElasticTestUtil.getParticipantDocumentAsString(esIndex, ddpParticipantId));
     }
@@ -128,6 +132,52 @@ public void testClinicalOrdersDownloadFromElastic() {
 
     }
 
+    @Test
+    public void testParticipantRaceDownloadFromElastic() {
+        ManualFilterParticipantList filterable = getFilterFromFile(AY_RACE_DOWNLOAD_FILE);
+        List<Filter> columnNames = getColumnNames(AY_RACE_DOWNLOAD_FILE);
+        Assert.assertNotNull(filterable);
+        Assert.assertNotNull(columnNames);
+        Assert.assertNotEquals(0, columnNames.size());
+
+        QueryParamsMap queryParamsMap = buildMockQueryParams(false, true, "tsv");
+        List<ParticipantWrapperDto> downloadList = DownloadParticipantListService.fetchParticipantEsData(filterable, queryParamsMap);
+        DownloadParticipantListParams downloadParticipantListParams = new DownloadParticipantListParams(queryParamsMap);
+
+        //load activity definition
+        String activityDefJson = null;
+        try {
+            activityDefJson = TestUtil.readFile("elastic/lmsAboutYouActivityDef.json");
+        } catch (Exception e) {
+            Assert.fail("Failed to read activity definition file " + e.getMessage());
+        }
+        Map<String, Object> activityDef = new Gson().fromJson(activityDefJson, Map.class);
+        Map<String, Map<String, Object>> lmsActivityDefs = Map.of("LMS_ACTIVITY_DEFS", activityDef);
+
+        TabularParticipantParser parser = new TabularParticipantParser(columnNames, ddpInstanceDto,
+                downloadParticipantListParams.isHumanReadable(), downloadParticipantListParams.isOnlyMostRecent(), lmsActivityDefs);
+        List<ModuleExportConfig> exportConfigs = parser.generateExportConfigs();
+        List<Map<String, Object>> participantEsDataMaps = downloadList.stream().map(dto ->
+                ((UnparsedESParticipantDto) dto.getEsData()).getDataAsMap()).toList();
+        List<Map<String, String>> participantValueMaps = parser.parse(exportConfigs, participantEsDataMaps);
+        TabularParticipantExporter participantExporter =
+                TabularParticipantExporter.getExporter(exportConfigs, participantValueMaps, downloadParticipantListParams.getFileFormat());
+        assertParticipantRaceExporterMap(participantExporter);
+    }
+
+    private void assertParticipantRaceExporterMap(TabularParticipantExporter participantExporter) {
+        assertNotNull(participantExporter);
+        assertNotNull(participantExporter.getExportFilename());
+        assertEquals(1, participantExporter.participantValueMaps.size());
+        Map<String, String> participantValues = participantExporter.participantValueMaps.get(0);
+        assertNotNull(participantValues);
+        assertEquals(shortId, participantValues.get("PROFILE.HRUID"));
+        assertEquals(FIRST_NAME, participantValues.get("PROFILE.FIRSTNAME"));
+        assertEquals(LAST_NAME, participantValues.get("PROFILE.LASTNAME"));
+        assertEquals("Hispanic, Latino, or Spanish (For example: Colombian, Cuban, Dominican, Mexican or Mexican American, Puerto Rican, Salvadoran, etc.), None of these fully describe my child/me, HISPANIC_SPANISH", participantValues.get("ABOUT_YOU.RACE"));
+        assertEquals("Other Ethnicity", participantValues.get("ABOUT_YOU.RACE_DETAIL"));
+    }
+
     private void assertParticipantExporterMap(TabularParticipantExporter participantExporter) {
         // matching the values with the values in the file elastic/filtersWithClinicalOrderColumns.json
 

pepper-apis/dsm-core/src/test/resources/elastic/lmsAboutYouActivity.json

@@ -0,0 +1,39 @@
+[
+  {
+    "activityCode": "ABOUT_YOU",
+    "createdAt": "2024-07-30",
+    "completedAt": null,
+    "activityVersion": "v1",
+    "lastUpdatedAt": "2024-07-30",
+    "parentInstanceGuid": null,
+    "questionsAnswers": [
+      {
+        "stableId": "RACE",
+        "answer": [
+          "HISPANIC_OR_SPANISH",
+          "NONE"
+        ],
+        "groupedOptions": {},
+        "RACE": [
+          "HISPANIC_OR_SPANISH",
+          "NONE"
+        ],
+        "optionDetails": [
+          {
+            "details": "Other Ethnicity",
+            "option": "NONE"
+          }
+        ],
+        "nestedOptions": {
+          "HISPANIC_OR_SPANISH": [
+            "HISPANIC_SPANISH"
+          ]
+        },
+        "questionType": "PICKLIST"
+      }
+    ],
+    "guid": "49QOWOYBKB",
+    "attributes": {},
+    "status": "IN_PROGRESS"
+  }
+]