From 14fdb40ecea095c9a80adcedd2762514a98d3efe Mon Sep 17 00:00:00 2001 From: Nikolay Akhmetov Date: Mon, 4 Nov 2024 10:36:42 -0500 Subject: [PATCH 1/2] CAT-634 Fix selection of samples on organ page when more than 500 are present --- ...GELOG-cat-634-fix-infinite-scroll-selection.md | 1 + context/app/static/js/hooks/useSearchData.ts | 15 ++++----------- 2 files changed, 5 insertions(+), 11 deletions(-) create mode 100644 CHANGELOG-cat-634-fix-infinite-scroll-selection.md diff --git a/CHANGELOG-cat-634-fix-infinite-scroll-selection.md b/CHANGELOG-cat-634-fix-infinite-scroll-selection.md new file mode 100644 index 0000000000..d78e88bdc9 --- /dev/null +++ b/CHANGELOG-cat-634-fix-infinite-scroll-selection.md @@ -0,0 +1 @@ +- Adjust ID fetching for infinite scroll table to ensure "select all" functionality works as expected. diff --git a/context/app/static/js/hooks/useSearchData.ts b/context/app/static/js/hooks/useSearchData.ts index 7a6f3a73f7..ac71cf1df3 100644 --- a/context/app/static/js/hooks/useSearchData.ts +++ b/context/app/static/js/hooks/useSearchData.ts @@ -234,7 +234,7 @@ export function useAllSearchIDs( const { elasticsearchEndpoint, groupsToken } = useAppContext(); const { searchData } = useSearchData( - { ...query, track_total_hits: true, size: 0, ...sharedIDsQueryClauses }, + { ...query, track_total_hits: true, size: 10000, ...sharedIDsQueryClauses }, { useDefaultQuery, fetcher, @@ -243,21 +243,14 @@ export function useAllSearchIDs( ); const totalHitsCount = getTotalHitsCount(searchData); - const numberOfPagesToRequest = totalHitsCount ? Math.ceil(10000 / totalHitsCount) : undefined; const getKey: SWRInfiniteKeyLoader = useCallback(() => { - if (numberOfPagesToRequest === undefined) { + if (totalHitsCount === undefined) { return null; } - return [ - { ...query, ...sharedIDsQueryClauses }, - elasticsearchEndpoint, - groupsToken, - useDefaultQuery, - numberOfPagesToRequest, - ]; - }, [query, elasticsearchEndpoint, groupsToken, useDefaultQuery, numberOfPagesToRequest]); + return [{ ...query, ...sharedIDsQueryClauses, size: 10000 }, elasticsearchEndpoint, groupsToken, useDefaultQuery]; + }, [totalHitsCount, query, elasticsearchEndpoint, groupsToken, useDefaultQuery]); // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-argument // @ts-expect-error - revisit to make these keys more type safe From 7ef1016396d4fe5a52b8c0585e616bfb8ac5e5c7 Mon Sep 17 00:00:00 2001 From: Nikolay Akhmetov Date: Tue, 5 Nov 2024 12:20:00 -0500 Subject: [PATCH 2/2] Fix "all IDs fetch" functionality to work for result sets of all sizes --- context/app/static/js/hooks/useSearchData.ts | 103 ++++++++++++++++--- 1 file changed, 86 insertions(+), 17 deletions(-) diff --git a/context/app/static/js/hooks/useSearchData.ts b/context/app/static/js/hooks/useSearchData.ts index ac71cf1df3..4c421e5dc2 100644 --- a/context/app/static/js/hooks/useSearchData.ts +++ b/context/app/static/js/hooks/useSearchData.ts @@ -211,13 +211,78 @@ function getTotalHitsCount(results?: SearchResponseBody) { return total?.value; } -function extractIDs(results?: SearchResponseBody) { - return results?.hits?.hits?.map((hit) => hit._id); +function extractIDs(results?: SearchResponseBody): string[] { + return results?.hits?.hits?.map((hit) => hit._id) ?? []; } -async function fetchAllIDs(...args: Parameters) { - const results = await fetchSearchData(...args); - return extractIDs(results); +// Get the sort array from the last hit. https://www.elastic.co/guide/en/elasticsearch/reference/current/paginate-search-results.html#search-after. +function getSearchAfterSort(hits: SearchResponseBody['hits']['hits']) { + const { sort } = hits.slice(-1)[0]; + return sort; +} + +/** + * Generator for sequentially fetching multiple pages of search data from the API + * while using the search_after parameter to paginate. + * @param query The search request to fetch + * @param elasticsearchEndpoint The endpoint to fetch the data from + * @param groupsToken The auth token to use for the request + * @param numberOfPagesToRequest The number of pages to fetch + */ +async function* fetchAllPages( + query: SearchRequest, + elasticsearchEndpoint: string, + groupsToken: string, + numberOfPagesToRequest: number, +) { + const q = query; + + try { + let i = 0; + while (i < numberOfPagesToRequest) { + // disabling eslint rule because that's the whole point of this generator + // eslint-disable-next-line no-await-in-loop + const firstPageResults = await fetchSearchData(q, elasticsearchEndpoint, groupsToken); + yield firstPageResults; + q.search_after = getSearchAfterSort(firstPageResults.hits.hits); + i += 1; + } + } catch (error) { + console.error("Error fetching all pages' data", error); + } +} + +/** + * Fetcher for useAllSearchIDs + * + * @param args.query The search request to fetch + * @param args.elasticsearchEndpoint The endpoint to fetch the data from + * @param args.groupsToken The auth token to use for the request + * @param args.useDefaultQuery Whether to apply the default query restrictions + * @param args.numberOfPagesToRequest The number of pages to fetch + * @returns + */ +async function fetchAllIDs({ + query: q, + elasticsearchEndpoint, + groupsToken, + useDefaultQuery, + numberOfPagesToRequest, +}: { + query: SearchRequest; + elasticsearchEndpoint: string; + groupsToken: string; + useDefaultQuery: boolean; + numberOfPagesToRequest: number; +}) { + const query = useDefaultQuery ? addRestrictionsToQuery(q) : q; + const ids = new Set(); + // For await loop is the clearest way to fetch all pages sequentially. + // eslint-disable-next-line no-restricted-syntax + for await (const results of fetchAllPages(query, elasticsearchEndpoint, groupsToken, numberOfPagesToRequest)) { + extractIDs(results).forEach((id) => ids.add(id)); + } + return Array.from(ids); } // We do not want the query to revalidate when _source or sort change. @@ -234,7 +299,7 @@ export function useAllSearchIDs( const { elasticsearchEndpoint, groupsToken } = useAppContext(); const { searchData } = useSearchData( - { ...query, track_total_hits: true, size: 10000, ...sharedIDsQueryClauses }, + { ...query, track_total_hits: true, size: 0, ...sharedIDsQueryClauses }, { useDefaultQuery, fetcher, @@ -244,17 +309,27 @@ export function useAllSearchIDs( const totalHitsCount = getTotalHitsCount(searchData); - const getKey: SWRInfiniteKeyLoader = useCallback(() => { + // Creates a key object for useSWR to fetch the IDs + // The key is null if the totalHitsCount is undefined + // Otherwise, it returns an object with the query, endpoint, and token + const getKey = useCallback(() => { if (totalHitsCount === undefined) { return null; } - return [{ ...query, ...sharedIDsQueryClauses, size: 10000 }, elasticsearchEndpoint, groupsToken, useDefaultQuery]; + const numberOfPagesToRequest = Math.ceil(totalHitsCount / 10_000); + + const q = { ...query, ...sharedIDsQueryClauses, size: 10_000 } as SearchRequest; + return { + query: q, + elasticsearchEndpoint, + groupsToken, + useDefaultQuery, + numberOfPagesToRequest, + }; }, [totalHitsCount, query, elasticsearchEndpoint, groupsToken, useDefaultQuery]); - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-argument - // @ts-expect-error - revisit to make these keys more type safe - const { data } = useSWRInfinite(getKey, (args) => fetchAllIDs(...args), { + const { data } = useSWR(getKey, (args) => fetchAllIDs(args), { fallbackData: [], ...swrConfigRest, }); @@ -262,12 +337,6 @@ export function useAllSearchIDs( return { allSearchIDs: data?.flat?.() ?? [], totalHitsCount }; } -// Get the sort array from the last hit. https://www.elastic.co/guide/en/elasticsearch/reference/current/paginate-search-results.html#search-after. -function getSearchAfterSort(hits: SearchResponseBody['hits']['hits']) { - const { sort } = hits.slice(-1)[0]; - return sort; -} - function getCombinedHits(pagesResults: SearchResponseBody[]) { const hasData = pagesResults.length > 0;