NickAkhmetov/CAT-777 Processed Data section

CHANGELOG-processed-datasets.md

@@ -0,0 +1 @@
+- Added processed datasets section to display all visualizations for a given raw dataset.

context/app/routes_browse.py

@@ -54,15 +54,6 @@ def details(type, uuid):
         **get_default_flask_data(),
         'entity': entity,
     }
-    marker = request.args.get('marker')
-
-    if type == 'dataset':
-        conf_cells_uuid = client.get_vitessce_conf_cells_and_lifted_uuid(entity, marker=marker)
-        flask_data.update({
-            'vitessce_conf': conf_cells_uuid.vitessce_conf.conf,
-            'has_notebook': conf_cells_uuid.vitessce_conf.cells is not None,
-            'vis_lifted_uuid': conf_cells_uuid.vis_lifted_uuid
-        })
 
     if type == 'publication':
         publication_ancillary_data = client.get_publication_ancillary_json(entity)
@@ -93,7 +84,11 @@ def details_vitessce(type, uuid):
         abort(404)
     client = get_client()
     entity = client.get_entity(uuid)
-    vitessce_conf = client.get_vitessce_conf_cells_and_lifted_uuid(entity).vitessce_conf
+    parent_uuid = request.args.get('parent') or None
+    marker = request.args.get('marker') or None
+    parent = client.get_entity(parent_uuid) if parent_uuid else None
+    vitessce_conf = client.get_vitessce_conf_cells_and_lifted_uuid(
+        entity, marker=marker, parent=parent).vitessce_conf
     # Returns a JSON null if there is no visualization.
     response = jsonify(vitessce_conf.conf)
     response.headers.add("Access-Control-Allow-Origin", "*")

context/app/static/js/components/Routes/Routes.jsx

@@ -45,8 +45,6 @@ function Routes({ flaskData }) {
     markdown,
     errorCode,
     list_uuid,
-    has_notebook,
-    vis_lifted_uuid,
     entities,
     organs,
     organs_count,
@@ -86,12 +84,7 @@ function Routes({ flaskData }) {
   if (urlPath.startsWith('/browse/dataset/') || urlPath.startsWith('/browse/support/')) {
     return (
       <Route>
-        <Dataset
-          assayMetadata={entity}
-          vitData={vitessce_conf}
-          hasNotebook={has_notebook}
-          visLiftedUUID={vis_lifted_uuid}
-        />
+        <Dataset assayMetadata={entity} />
       </Route>
     );
   }

context/app/static/js/components/detailPage/provenance/ProvAnalysisDetails/ProvAnalysisDetails.spec.js → context/app/static/js/components/detailPage/AnalysisDetails/AnalysisDetails.spec.tsx

@@ -1,15 +1,16 @@
 import React from 'react';
 import { render, screen } from 'test-utils/functions';
 
-import ProvAnalysisDetails from './ProvAnalysisDetails';
+import { DagProvenanceType } from 'js/components/types';
+import AnalysisDetails from './AnalysisDetails';
 
 test('should display ingest and cwl lists', () => {
   const dagListData = [
     { origin: 'https://github.com/fake1/fake1.git', hash: 'aaaaaaa' },
     { origin: 'https://github.com/fake2/fake2.git', hash: 'bbbbbbb' },
     { origin: 'https://github.com/fake3/fake3.git', hash: 'ccccccc', name: 'fake3.cwl' },
   ];
-  render(<ProvAnalysisDetails dagListData={dagListData} />);
+  render(<AnalysisDetails dagListData={dagListData} />);
 
   expect(screen.getByText('Ingest Pipelines')).toBeInTheDocument();
   expect(screen.getByText('CWL Pipelines')).toBeInTheDocument();
@@ -19,8 +20,8 @@ test('should display ingest and cwl lists', () => {
 });
 
 test('should not display pipelines when pipelines do not exist', () => {
-  const dagListData = [];
-  render(<ProvAnalysisDetails dagListData={dagListData} />);
+  const dagListData: DagProvenanceType[] = [];
+  render(<AnalysisDetails dagListData={dagListData} />);
 
   expect(screen.queryByText('Ingest Pipelines')).not.toBeInTheDocument();
   expect(screen.queryByText('CWL Pipelines')).not.toBeInTheDocument();

context/app/static/js/components/detailPage/AnalysisDetails/AnalysisDetails.tsx

@@ -0,0 +1,34 @@
+import React from 'react';
+
+import { DagProvenanceType } from 'js/components/types';
+import AnalysisDetailsList from './AnalysisDetailsList';
+
+interface AnalysisDetails {
+  dagListData: DagProvenanceType[];
+}
+
+function AnalysisDetails({ dagListData }: AnalysisDetails) {
+  const ingestPipelines = dagListData.filter((pipeline) => !('name' in pipeline));
+  const cwlPipelines = dagListData.filter((pipeline) => 'name' in pipeline);
+
+  return (
+    <div>
+      {ingestPipelines.length > 0 && (
+        <AnalysisDetailsList
+          pipelines={ingestPipelines}
+          pipelineType="Ingest"
+          tooltip="Supplementary links for the data ingestion pipelines for this dataset"
+        />
+      )}
+      {cwlPipelines.length > 0 && (
+        <AnalysisDetailsList
+          pipelines={cwlPipelines}
+          pipelineType="CWL"
+          tooltip="Supplementary links for the CWL (Common Workflow Language) pipelines for this dataset"
+        />
+      )}
+    </div>
+  );
+}
+
+export default AnalysisDetails;

context/app/static/js/components/detailPage/provenance/ProvAnalysisDetailsLink/ProvAnalysisDetailsLink.spec.js → context/app/static/js/components/detailPage/AnalysisDetails/AnalysisDetailsLink.spec.tsx

@@ -1,7 +1,7 @@
 import React from 'react';
 import { render, screen } from 'test-utils/functions';
 
-import ProvAnalysisDetailsLink from './ProvAnalysisDetailsLink';
+import ProvAnalysisDetailsLink from './AnalysisDetailsLink';
 
 test('should display ingest pipeline link', () => {
   const fakePipeline = { origin: 'https://github.com/fake/fake.git', hash: 'aabbccd' };

context/app/static/js/components/detailPage/provenance/ProvAnalysisDetailsLink/ProvAnalysisDetailsLink.jsx → context/app/static/js/components/detailPage/AnalysisDetails/AnalysisDetailsLink.tsx

@@ -1,9 +1,23 @@
 import React from 'react';
-import PropTypes from 'prop-types';
 
 import { CwlIcon, FlexOutboundLink, PrimaryTextDivider, StyledListItem } from './style';
 
-function ProvAnalysisDetailsLink({ data }) {
+interface CWLPipelineLink {
+  hash: string;
+  name: string;
+  origin: string;
+}
+
+interface IngestPipelineLink {
+  hash: string;
+  origin: string;
+}
+
+interface ProvAnalysisDetailsLinkProps {
+  data: CWLPipelineLink | IngestPipelineLink;
+}
+
+function ProvAnalysisDetailsLink({ data }: ProvAnalysisDetailsLinkProps) {
   const trimmedOrigin = data.origin.replace(/\.git$/, '');
   const githubUrl =
     'name' in data ? `${trimmedOrigin}/blob/${data.hash}/${data.name}` : `${trimmedOrigin}/tree/${data.hash}`;
@@ -26,12 +40,4 @@ function ProvAnalysisDetailsLink({ data }) {
   );
 }
 
-ProvAnalysisDetailsLink.propTypes = {
-  data: PropTypes.shape({
-    hash: PropTypes.string,
-    name: PropTypes.string,
-    origin: PropTypes.string,
-  }).isRequired,
-};
-
 export default ProvAnalysisDetailsLink;

context/app/static/js/components/detailPage/AnalysisDetails/AnalysisDetailsList.tsx

@@ -0,0 +1,36 @@
+import React from 'react';
+import List from '@mui/material/List';
+import Typography from '@mui/material/Typography';
+
+import InfoTooltipIcon from 'js/shared-styles/icons/TooltipIcon';
+import ProvAnalysisDetailsLink from './AnalysisDetailsLink';
+
+interface ProvAnalysisDetailsListProps {
+  pipelines: {
+    hash: string;
+    name?: string;
+    origin: string;
+  }[];
+  pipelineType: string;
+  tooltip?: string;
+}
+
+function ProvAnalysisDetailsList({ pipelines, pipelineType, tooltip }: ProvAnalysisDetailsListProps) {
+  return (
+    <>
+      <Typography variant="subtitle2">
+        {`${pipelineType} Pipelines`} <InfoTooltipIcon iconTooltipText={tooltip} />
+      </Typography>
+      <List data-testid={pipelineType}>
+        {pipelines.map((item) => (
+          <ProvAnalysisDetailsLink
+            data={item}
+            key={`provenance-analysis-details-${pipelineType.toLowerCase()}-pipeline-${item.hash}`}
+          />
+        ))}
+      </List>
+    </>
+  );
+}
+
+export default ProvAnalysisDetailsList;

context/app/static/js/components/detailPage/AnalysisDetails/index.ts

@@ -0,0 +1,3 @@
+import AnalysisDetails from './AnalysisDetails';
+
+export default AnalysisDetails;

context/app/static/js/components/detailPage/AnalysisDetails/style.ts

@@ -0,0 +1,29 @@
+import { styled } from '@mui/material/styles';
+import LaunchRoundedIcon from '@mui/icons-material/LaunchRounded';
+import Divider from '@mui/material/Divider';
+
+import OutboundLink from 'js/shared-styles/Links/OutboundLink';
+
+const CwlIcon = styled(LaunchRoundedIcon)(({ theme }) => ({
+  marginLeft: theme.spacing(0.5),
+  fontSize: '1rem',
+  alignSelf: 'center',
+}));
+
+const FlexOutboundLink = styled(OutboundLink)({
+  display: 'flex',
+});
+
+const PrimaryTextDivider = styled(Divider)(({ theme }) => ({
+  marginLeft: theme.spacing(0.5),
+  marginRight: theme.spacing(0.5),
+  height: '15px',
+  backgroundColor: theme.palette.text.primary,
+  alignSelf: 'center',
+}));
+
+const StyledListItem = styled('li')({
+  display: 'flex',
+});
+
+export { CwlIcon, FlexOutboundLink, PrimaryTextDivider, StyledListItem };

context/app/static/js/components/detailPage/BulkDataTransfer/BulkDataTransferSection.tsx

@@ -2,24 +2,20 @@ import React from 'react';
 
 import SectionHeader from 'js/shared-styles/sections/SectionHeader';
 import { DetailPageSection } from 'js/components/detailPage/style';
-import { useFlaskDataContext } from 'js/components/Contexts';
 import { FilesContextProvider } from 'js/components/detailPage/files/FilesContext';
 import BulkDataTransferPanels from './BulkDataTransferPanels';
 import { StyledContainer } from './style';
+import { SectionDescription } from '../ProcessedData/ProcessedDataset/SectionDescription';
 
 function BulkDataTransfer() {
-  const {
-    entity: { entity_type },
-  } = useFlaskDataContext();
-
   return (
     <FilesContextProvider>
       <DetailPageSection id="bulk-data-transfer" data-testid="bulk-data-transfer">
-        <SectionHeader
-          iconTooltipText={`Information about how to bulk download all files related to this ${entity_type?.toLowerCase()}.`}
-        >
-          Bulk Data Transfer
-        </SectionHeader>
+        <SectionHeader>Bulk Data Transfer</SectionHeader>
+        <SectionDescription>
+          This section explains how to download data in bulk from raw and processed datasets. Processed datasets have
+          separate download directories in Globus or dbGaP, distinct from the raw dataset.
+        </SectionDescription>
         <StyledContainer>
           <BulkDataTransferPanels />
         </StyledContainer>

context/app/static/js/components/detailPage/CollectionsSection/CollectionsSection.tsx

@@ -2,10 +2,10 @@ import React from 'react';
 
 import SectionHeader from 'js/shared-styles/sections/SectionHeader';
 import PanelList from 'js/shared-styles/panels/PanelList';
-import { useFlaskDataContext } from 'js/components/Contexts';
 import { DetailPageSection } from 'js/components/detailPage/style';
 import { buildCollectionsPanelsProps } from 'js/pages/Collections/utils';
 import { CollectionHit } from 'js/pages/Collections/types';
+import { SectionDescription } from '../ProcessedData/ProcessedDataset/SectionDescription';
 
 interface CollectionsSectionProps {
   collectionsData: CollectionHit[];
@@ -14,15 +14,13 @@ interface CollectionsSectionProps {
 function CollectionsSection({ collectionsData }: CollectionsSectionProps) {
   const panelsProps = buildCollectionsPanelsProps(collectionsData);
 
-  const {
-    entity: { entity_type },
-  } = useFlaskDataContext();
-
   return (
     <DetailPageSection id="collections">
-      <SectionHeader iconTooltipText={`List of collections that contain this ${entity_type?.toLowerCase()}.`}>
-        Collections
-      </SectionHeader>
+      <SectionHeader>Collections</SectionHeader>
+      <SectionDescription>
+        Collections may contain references to either raw or processed datasets. If a processed dataset is not included
+        in any collection, there will be no corresponding tabs in the table below.
+      </SectionDescription>
       <PanelList panelsProps={panelsProps} />
     </DetailPageSection>
   );

context/app/static/js/components/detailPage/DatasetRelationships/hooks.ts

@@ -40,7 +40,14 @@ async function fetchPipelineInfo({ url, datasets, groupsToken }: PipelineInfoReq
   if (datasets.length !== 1) {
     return Promise.resolve('');
   }
-  return (await fetchSoftAssay({ url, dataset: datasets[0], groupsToken }))['pipeline-shorthand'];
+  const result = await fetchSoftAssay({ url, dataset: datasets[0], groupsToken });
+
+  // Handle image pyramids separately since their pipeline-shorthand is blank
+  if (result.assaytype === 'image_pyramid') {
+    return 'Image Pyramid Generation';
+  }
+
+  return result['pipeline-shorthand'];
 }
 
 export function usePipelineInfo(datasets: string[]) {

context/app/static/js/components/detailPage/DatasetRelationships/nodeTypes.tsx

@@ -6,8 +6,10 @@ import Typography from '@mui/material/Typography';
 import { AccountTreeRounded, ExtensionRounded, SvgIconComponent } from '@mui/icons-material';
 import Skeleton from '@mui/material/Skeleton';
 import { Box } from '@mui/system';
+import { datasetSectionId } from 'js/pages/Dataset/utils';
 import StatusIcon from '../StatusIcon';
 import { usePipelineInfo } from './hooks';
+import { useProcessedDatasetDetails } from '../ProcessedData/ProcessedDataset/hooks';
 
 interface CommonNodeInfo extends Record<string, unknown> {
   name: string;
@@ -21,6 +23,7 @@ interface PipelineNodeInfo extends CommonNodeInfo {
 
 interface DatasetNodeInfo extends CommonNodeInfo {
   datasetType?: string;
+  uuid: string;
   status: string;
 }
 
@@ -122,13 +125,15 @@ function PipelineNode({
 type ProcessedDatasetNodeProps = Node<DatasetNodeInfo, 'processedDataset'>;
 
 function ProcessedDatasetNode({ data }: NodeProps<ProcessedDatasetNodeProps>) {
+  const { datasetDetails, isLoading } = useProcessedDatasetDetails(data.uuid);
   return (
     <NodeTemplate
       rounded
       target
-      href={`#${data.name}-section`}
+      href={datasetDetails ? `#${datasetSectionId(datasetDetails, 'section')}` : undefined}
       icon={nodeIcons.processedDataset}
       bgColor={nodeColors.processedDataset}
+      isLoading={isLoading}
       {...data}
     >
       {data.datasetType}
@@ -139,13 +144,15 @@ function ProcessedDatasetNode({ data }: NodeProps<ProcessedDatasetNodeProps>) {
 type ComponentDatasetNodeProps = Node<DatasetNodeInfo, 'componentDataset'>;
 
 function ComponentDatasetNode({ data }: NodeProps<ComponentDatasetNodeProps>) {
+  const { datasetDetails, isLoading } = useProcessedDatasetDetails(data.uuid);
   return (
     <NodeTemplate
       rounded
       target
       icon={nodeIcons.componentDataset}
-      href={`#${data.name}-section`}
+      href={datasetDetails ? `#${datasetSectionId(datasetDetails, 'section')}` : undefined}
       bgColor={nodeColors.componentDataset}
+      isLoading={isLoading}
       {...data}
     >
       {data.datasetType}

context/app/static/js/components/detailPage/DatasetRelationships/prov.fixtures.ts

@@ -473,6 +473,7 @@ export const nodes = [
       name: 'HBM749.SMWP.555',
       status: 'Published',
       datasetType: 'CODEX',
+      uuid: '6df2f796ad72307d04dc94d688b725c5',
     },
   },
   {
@@ -500,6 +501,7 @@ export const nodes = [
       name: 'HBM878.HRPG.642',
       status: 'QA',
       datasetType: 'CODEX [Cytokit + SPRM]',
+      uuid: 'd8f851efb54f84d7ee0952e10d4c449e',
     },
   },
   {
@@ -509,6 +511,7 @@ export const nodes = [
       name: 'HBM398.SWKV.256',
       status: 'Published',
       datasetType: 'CODEX [Cytokit + SPRM]',
+      uuid: 'ff77fcae7f6d9b5b7b8741c282677eef',
     },
   },
 ];