NickAkhmetov/CAT-777 Processed Data section (#3495)

hubmapconsortium · Aug 20, 2024 · dda7257 · dda7257
1 parent 4e4f286
commit dda7257
Show file tree

Hide file tree

Showing 164 changed files with 3,236 additions and 1,535 deletions.
diff --git a/CHANGELOG-cat-831.md b/CHANGELOG-cat-831.md
@@ -0,0 +1,3 @@
+- Extend provenance table logic to handle missing entities.
+- Fix handling of large search requests.
+- Request less data for provenance table tiles.
diff --git a/CHANGELOG-processed-datasets.md b/CHANGELOG-processed-datasets.md
@@ -0,0 +1 @@
+- Added processed datasets section to display all visualizations for a given raw dataset.
diff --git a/context/app/routes_browse.py b/context/app/routes_browse.py
@@ -1,5 +1,5 @@
 import json
-from urllib.parse import urlparse
+from urllib.parse import urlparse, quote
 
 from flask import (
     render_template, jsonify,
@@ -47,22 +47,49 @@ def details(type, uuid):
     client = get_client()
     entity = client.get_entity(uuid)
     actual_type = entity['entity_type'].lower()
+
+    # Redirect to primary dataset if this is
+    # - a support entity (e.g. an image pyramid)
+    # - a processed or component dataset
+    is_support = actual_type == 'support'
+    is_processed = entity.get('processing') != 'raw' and actual_type == 'dataset'
+    is_component = entity.get('is_component', False) is True
+    if (is_support or is_processed or is_component):
+        supported_entity = client.get_entities(
+            'datasets',
+            query_override={
+                "bool": {
+                    "must": {
+                        "terms": {
+                            "descendant_ids": [uuid]
+                        }
+                    }
+                }
+            },
+            non_metadata_fields=['hubmap_id', 'uuid']
+        )
+
+        pipeline_anchor = entity.get('pipeline', entity.get('hubmap_id')).replace(' ', '')
+        anchor = quote(f'section-{pipeline_anchor}-{entity.get("status")}').lower()
+
+        if len(supported_entity) > 0:
+            return redirect(
+                url_for('routes_browse.details',
+                        type='dataset',
+                        uuid=supported_entity[0]['uuid'],
+                        _anchor=anchor,
+                        redirected=True))
+
     if type != actual_type:
         return redirect(url_for('routes_browse.details', type=actual_type, uuid=uuid))
 
+    redirected = request.args.get('redirected') == 'True'
+
     flask_data = {
         **get_default_flask_data(),
         'entity': entity,
+        'redirected': redirected,
     }
-    marker = request.args.get('marker')
-
-    if type == 'dataset':
-        conf_cells_uuid = client.get_vitessce_conf_cells_and_lifted_uuid(entity, marker=marker)
-        flask_data.update({
-            'vitessce_conf': conf_cells_uuid.vitessce_conf.conf,
-            'has_notebook': conf_cells_uuid.vitessce_conf.cells is not None,
-            'vis_lifted_uuid': conf_cells_uuid.vis_lifted_uuid
-        })
 
     if type == 'publication':
         publication_ancillary_data = client.get_publication_ancillary_json(entity)
@@ -93,7 +120,11 @@ def details_vitessce(type, uuid):
         abort(404)
     client = get_client()
     entity = client.get_entity(uuid)
-    vitessce_conf = client.get_vitessce_conf_cells_and_lifted_uuid(entity).vitessce_conf
+    parent_uuid = request.args.get('parent') or None
+    marker = request.args.get('marker') or None
+    parent = client.get_entity(parent_uuid) if parent_uuid else None
+    vitessce_conf = client.get_vitessce_conf_cells_and_lifted_uuid(
+        entity, marker=marker, parent=parent).vitessce_conf
     # Returns a JSON null if there is no visualization.
     response = jsonify(vitessce_conf.conf)
     response.headers.add("Access-Control-Allow-Origin", "*")

diff --git a/context/app/static/js/components/Contexts.tsx b/context/app/static/js/components/Contexts.tsx
@@ -9,6 +9,7 @@ export interface FlaskDataContextType {
   [key: string]: unknown;
   title: string; // preview page title
   vis_lifted_uuid?: string;
+  redirected?: boolean;
 }
 
 export const FlaskDataContext = createContext<FlaskDataContextType>('FlaskDataContext');

diff --git a/context/app/static/js/components/Providers.jsx b/context/app/static/js/components/Providers.jsx
@@ -9,6 +9,7 @@ import { FlaskDataContext, AppContext } from 'js/components/Contexts';
 import GlobalStyles from 'js/components/globalStyles';
 import { ProtocolAPIContext } from 'js/components/detailPage/Protocol/ProtocolAPIContext';
 import { EntityStoreProvider } from 'js/stores/useEntityStore';
+import { InitialHashContextProvider } from 'js/hooks/useInitialHash';
 import theme from '../theme';
 import GlobalFonts from '../fonts';
 import { useEntityHeaderSprings } from './detailPage/entityHeader/EntityHeader/hooks';
@@ -63,22 +64,24 @@ export default function Providers({
 
   return (
     <SWRConfig value={swrConfig}>
-      <GlobalFonts />
-      <MuiThemeProvider theme={theme}>
-        <SCThemeProvider theme={theme}>
-          <AppContext.Provider value={appContext}>
-            <FlaskDataContext.Provider value={flaskDataWithDefaults}>
-              <EntityStoreProvider springs={springs} assayMetadata={flaskData?.entity ?? {}}>
-                <ProtocolAPIContext.Provider value={protocolsContext}>
-                  <CssBaseline />
-                  <GlobalStyles />
-                  {children}
-                </ProtocolAPIContext.Provider>
-              </EntityStoreProvider>
-            </FlaskDataContext.Provider>
-          </AppContext.Provider>
-        </SCThemeProvider>
-      </MuiThemeProvider>
+      <InitialHashContextProvider>
+        <GlobalFonts />
+        <MuiThemeProvider theme={theme}>
+          <SCThemeProvider theme={theme}>
+            <AppContext.Provider value={appContext}>
+              <FlaskDataContext.Provider value={flaskDataWithDefaults}>
+                <EntityStoreProvider springs={springs} assayMetadata={flaskData?.entity ?? {}}>
+                  <ProtocolAPIContext.Provider value={protocolsContext}>
+                    <CssBaseline />
+                    <GlobalStyles />
+                    {children}
+                  </ProtocolAPIContext.Provider>
+                </EntityStoreProvider>
+              </FlaskDataContext.Provider>
+            </AppContext.Provider>
+          </SCThemeProvider>
+        </MuiThemeProvider>
+      </InitialHashContextProvider>
     </SWRConfig>
   );
 }
diff --git a/context/app/static/js/components/Routes/Routes.jsx b/context/app/static/js/components/Routes/Routes.jsx
@@ -45,8 +45,6 @@ function Routes({ flaskData }) {
     markdown,
     errorCode,
     list_uuid,
-    has_notebook,
-    vis_lifted_uuid,
     entities,
     organs,
     organs_count,
@@ -86,12 +84,7 @@ function Routes({ flaskData }) {
   if (urlPath.startsWith('/browse/dataset/') || urlPath.startsWith('/browse/support/')) {
     return (
       <Route>
-        <Dataset
-          assayMetadata={entity}
-          vitData={vitessce_conf}
-          hasNotebook={has_notebook}
-          visLiftedUUID={vis_lifted_uuid}
-        />
+        <Dataset assayMetadata={entity} />
       </Route>
     );
   }
@@ -341,6 +334,7 @@ Routes.propTypes = {
     redirected_from: PropTypes.string,
     cell_type: PropTypes.string,
     globusGroups: PropTypes.object,
+    redirected: PropTypes.bool,
   }),
 };
 

diff --git a/context/app/static/js/components/cell-types/CellTypesBiomarkersTable.tsx b/context/app/static/js/components/cell-types/CellTypesBiomarkersTable.tsx
@@ -12,15 +12,14 @@ import TableCell from '@mui/material/TableCell';
 import TableRow from '@mui/material/TableRow';
 import { capitalize } from '@mui/material/utils';
 
-import SectionHeader from 'js/shared-styles/sections/SectionHeader';
 import Description from 'js/shared-styles/sections/Description';
 import { useTabs } from 'js/shared-styles/tabs';
 import { Tab, Tabs, TabPanel } from 'js/shared-styles/tables/TableTabs';
 import { StyledTableContainer } from 'js/shared-styles/tables';
 import { InternalLink } from 'js/shared-styles/Links';
 import { CellTypeBiomarkerInfo } from 'js/hooks/useUBKG';
 
-import DetailPageSection from '../detailPage/DetailPageSection';
+import { CollapsibleDetailPageSection } from '../detailPage/DetailPageSection';
 import { useCellTypeBiomarkers } from './hooks';
 
 const tableKeys = ['genes', 'proteins'] as const;
@@ -80,8 +79,7 @@ export default function CellTypesBiomarkersTable() {
   const { openTabIndex, handleTabChange } = useTabs();
 
   return (
-    <DetailPageSection id="biomarkers">
-      <SectionHeader>Biomarkers</SectionHeader>
+    <CollapsibleDetailPageSection id="biomarkers" title="Biomarkers">
       <Stack direction="column" spacing={2}>
         <Description>
           This is a list of identified biomarkers that are validated from the listed source. Explore other sources in
@@ -118,6 +116,6 @@ export default function CellTypesBiomarkersTable() {
           ))}
         </div>
       </Stack>
-    </DetailPageSection>
+    </CollapsibleDetailPageSection>
   );
 }
diff --git a/context/app/static/js/components/cell-types/CellTypesEntitiesTables.tsx b/context/app/static/js/components/cell-types/CellTypesEntitiesTables.tsx
@@ -1,4 +1,3 @@
-import SectionHeader from 'js/shared-styles/sections/SectionHeader';
 import React, { Fragment } from 'react';
 import Description from 'js/shared-styles/sections/Description';
 import { useTabs } from 'js/shared-styles/tabs';
@@ -13,16 +12,15 @@ import { format } from 'date-fns/format';
 import { InternalLink } from 'js/shared-styles/Links';
 import Paper from '@mui/material/Paper';
 import Stack from '@mui/material/Stack';
-import DetailPageSection from '../detailPage/DetailPageSection';
+import { CollapsibleDetailPageSection } from '../detailPage/DetailPageSection';
 import { useCellTypeDetails } from './hooks';
 
 export default function CellTypesEntitiesTables() {
   const { datasets, samples, organs } = useCellTypeDetails();
 
   const { openTabIndex, handleTabChange } = useTabs();
   return (
-    <DetailPageSection id="organs">
-      <SectionHeader>Organs</SectionHeader>
+    <CollapsibleDetailPageSection title="Organs" id="organs">
       <Description>
         This is the list of organs and its associated data that is dependent on the data available within HuBMAP. To
         filter the list of data in the table below by organ, select organ(s) from the list below. Multiple organs can be
@@ -71,6 +69,6 @@ export default function CellTypesEntitiesTables() {
           ))}
         </StyledTableContainer>
       </TabPanel>
-    </DetailPageSection>
+    </CollapsibleDetailPageSection>
   );
 }
diff --git a/context/app/static/js/components/cell-types/CellTypesVisualization.tsx b/context/app/static/js/components/cell-types/CellTypesVisualization.tsx
@@ -5,12 +5,11 @@ import Box from '@mui/material/Box';
 import Skeleton from '@mui/material/Skeleton';
 import { LegendItem, LegendLabel, LegendOrdinal } from '@visx/legend';
 
-import SectionHeader from 'js/shared-styles/sections/SectionHeader';
 import Description from 'js/shared-styles/sections/Description';
 
 import VerticalStackedBarChart from 'js/shared-styles/charts/VerticalStackedBarChart';
 import { useBandScale, useLogScale, useOrdinalScale } from 'js/shared-styles/charts/hooks';
-import DetailPageSection from 'js/components/detailPage/DetailPageSection';
+import { CollapsibleDetailPageSection } from 'js/components/detailPage/DetailPageSection';
 import { TooltipData } from 'js/shared-styles/charts/types';
 import { CellTypeOrgan } from 'js/hooks/useCrossModalityApi';
 import { useCellTypeDetails, useCellTypeName } from './hooks';
@@ -158,10 +157,9 @@ export default function CellTypesVisualization() {
   const { organs = [] } = useCellTypeDetails();
 
   return (
-    <DetailPageSection id="distribution-across-organs">
-      <SectionHeader>Distribution Across Organs</SectionHeader>
+    <CollapsibleDetailPageSection id="distribution-across-organs" title="Distribution Across Organs">
       <Description>Cell counts in this visualization are dependent on the data available within HuBMAP.</Description>
       <CellTypeOrgansGraph organs={organs} />
-    </DetailPageSection>
+    </CollapsibleDetailPageSection>
   );
 }
diff --git a/...alysisDetails/ProvAnalysisDetails.spec.js → .../AnalysisDetails/AnalysisDetails.spec.tsx b/...alysisDetails/ProvAnalysisDetails.spec.js → .../AnalysisDetails/AnalysisDetails.spec.tsx
@@ -1,15 +1,16 @@
 import React from 'react';
 import { render, screen } from 'test-utils/functions';
 
-import ProvAnalysisDetails from './ProvAnalysisDetails';
+import { DagProvenanceType } from 'js/components/types';
+import AnalysisDetails from './AnalysisDetails';
 
 test('should display ingest and cwl lists', () => {
   const dagListData = [
     { origin: 'https://github.com/fake1/fake1.git', hash: 'aaaaaaa' },
     { origin: 'https://github.com/fake2/fake2.git', hash: 'bbbbbbb' },
     { origin: 'https://github.com/fake3/fake3.git', hash: 'ccccccc', name: 'fake3.cwl' },
   ];
-  render(<ProvAnalysisDetails dagListData={dagListData} />);
+  render(<AnalysisDetails dagListData={dagListData} />);
 
   expect(screen.getByText('Ingest Pipelines')).toBeInTheDocument();
   expect(screen.getByText('CWL Pipelines')).toBeInTheDocument();
@@ -19,8 +20,8 @@ test('should display ingest and cwl lists', () => {
 });
 
 test('should not display pipelines when pipelines do not exist', () => {
-  const dagListData = [];
-  render(<ProvAnalysisDetails dagListData={dagListData} />);
+  const dagListData: DagProvenanceType[] = [];
+  render(<AnalysisDetails dagListData={dagListData} />);
 
   expect(screen.queryByText('Ingest Pipelines')).not.toBeInTheDocument();
   expect(screen.queryByText('CWL Pipelines')).not.toBeInTheDocument();

diff --git a/context/app/static/js/components/detailPage/AnalysisDetails/AnalysisDetails.tsx b/context/app/static/js/components/detailPage/AnalysisDetails/AnalysisDetails.tsx
@@ -0,0 +1,34 @@
+import React from 'react';
+
+import { DagProvenanceType } from 'js/components/types';
+import AnalysisDetailsList from './AnalysisDetailsList';
+
+interface AnalysisDetails {
+  dagListData: DagProvenanceType[];
+}
+
+function AnalysisDetails({ dagListData }: AnalysisDetails) {
+  const ingestPipelines = dagListData.filter((pipeline) => !('name' in pipeline));
+  const cwlPipelines = dagListData.filter((pipeline) => 'name' in pipeline);
+
+  return (
+    <div>
+      {ingestPipelines.length > 0 && (
+        <AnalysisDetailsList
+          pipelines={ingestPipelines}
+          pipelineType="Ingest"
+          tooltip="Supplementary links for the data ingestion pipelines for this dataset"
+        />
+      )}
+      {cwlPipelines.length > 0 && (
+        <AnalysisDetailsList
+          pipelines={cwlPipelines}
+          pipelineType="CWL"
+          tooltip="Supplementary links for the CWL (Common Workflow Language) pipelines for this dataset"
+        />
+      )}
+    </div>
+  );
+}
+
+export default AnalysisDetails;
diff --git a/...tailsLink/ProvAnalysisDetailsLink.spec.js → ...lysisDetails/AnalysisDetailsLink.spec.tsx b/...tailsLink/ProvAnalysisDetailsLink.spec.js → ...lysisDetails/AnalysisDetailsLink.spec.tsx
@@ -1,7 +1,7 @@
 import React from 'react';
 import { render, screen } from 'test-utils/functions';
 
-import ProvAnalysisDetailsLink from './ProvAnalysisDetailsLink';
+import ProvAnalysisDetailsLink from './AnalysisDetailsLink';
 
 test('should display ingest pipeline link', () => {
   const fakePipeline = { origin: 'https://github.com/fake/fake.git', hash: 'aabbccd' };

diff --git a/...isDetailsLink/ProvAnalysisDetailsLink.jsx → ...e/AnalysisDetails/AnalysisDetailsLink.tsx b/...isDetailsLink/ProvAnalysisDetailsLink.jsx → ...e/AnalysisDetails/AnalysisDetailsLink.tsx
@@ -1,9 +1,13 @@
 import React from 'react';
-import PropTypes from 'prop-types';
 
+import { DagProvenanceType } from 'js/components/types';
 import { CwlIcon, FlexOutboundLink, PrimaryTextDivider, StyledListItem } from './style';
 
-function ProvAnalysisDetailsLink({ data }) {
+interface ProvAnalysisDetailsLinkProps {
+  data: DagProvenanceType;
+}
+
+function ProvAnalysisDetailsLink({ data }: ProvAnalysisDetailsLinkProps) {
   const trimmedOrigin = data.origin.replace(/\.git$/, '');
   const githubUrl =
     'name' in data ? `${trimmedOrigin}/blob/${data.hash}/${data.name}` : `${trimmedOrigin}/tree/${data.hash}`;
@@ -26,12 +30,4 @@ function ProvAnalysisDetailsLink({ data }) {
   );
 }
 
-ProvAnalysisDetailsLink.propTypes = {
-  data: PropTypes.shape({
-    hash: PropTypes.string,
-    name: PropTypes.string,
-    origin: PropTypes.string,
-  }).isRequired,
-};
-
 export default ProvAnalysisDetailsLink;