-
Notifications
You must be signed in to change notification settings - Fork 20
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #125 from powersync-ja/feat/full-text-search
feat: Full text search
- Loading branch information
Showing
13 changed files
with
10,123 additions
and
8,148 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
39 changes: 39 additions & 0 deletions
39
demos/react-supabase-todolist/src/app/utils/fts_helpers.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,39 @@ | ||
| import { db } from '@/components/providers/SystemProvider'; | ||
|
|
||
| /** | ||
| * adding * to the end of the search term will match any word that starts with the search term | ||
| * e.g. searching bl will match blue, black, etc. | ||
| * consult FTS5 Full-text Query Syntax documentation for more options | ||
| * @param searchTerm | ||
| * @returns a modified search term with options. | ||
| */ | ||
| function createSearchTermWithOptions(searchTerm: string): string { | ||
| const searchTermWithOptions: string = `${searchTerm}*`; | ||
| return searchTermWithOptions; | ||
| } | ||
|
|
||
| /** | ||
| * Search the FTS table for the given searchTerm | ||
| * @param searchTerm | ||
| * @param tableName | ||
| * @returns results from the FTS table | ||
| */ | ||
| export async function searchTable(searchTerm: string, tableName: string): Promise<any[]> { | ||
| const searchTermWithOptions = createSearchTermWithOptions(searchTerm); | ||
| return await db.getAll(`SELECT * FROM fts_${tableName} WHERE fts_${tableName} MATCH ? ORDER BY rank`, [ | ||
| searchTermWithOptions | ||
| ]); | ||
| } | ||
|
|
||
| //Used to display the search results in the autocomplete text field | ||
| export class SearchResult { | ||
| id: string; | ||
| todoName: string | null; | ||
| listName: string; | ||
|
|
||
| constructor(id: string, listName: string, todoName: string | null = null) { | ||
| this.id = id; | ||
| this.listName = listName; | ||
| this.todoName = todoName; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,65 @@ | ||
| import { AppSchema } from '../../library/powersync/AppSchema'; | ||
| import { Table } from '@journeyapps/powersync-sdk-web'; | ||
| import { db } from '@/components/providers/SystemProvider'; | ||
| import { ExtractType, generateJsonExtracts } from './helpers'; | ||
|
|
||
| /** | ||
| * Create a Full Text Search table for the given table and columns | ||
| * with an option to use a different tokenizer otherwise it defaults | ||
| * to unicode61. It also creates the triggers that keep the FTS table | ||
| * and the PowerSync table in sync. | ||
| * @param tableName | ||
| * @param columns | ||
| * @param tokenizationMethod | ||
| */ | ||
| async function createFtsTable(tableName: string, columns: string[], tokenizationMethod = 'unicode61'): Promise<void> { | ||
| const internalName = (AppSchema.tables as Table[]).find((table) => table.name === tableName)?.internalName; | ||
| const stringColumns = columns.join(', '); | ||
|
|
||
| return await db.writeTransaction(async (tx) => { | ||
| // Add FTS table | ||
| await tx.execute(` | ||
| CREATE VIRTUAL TABLE IF NOT EXISTS fts_${tableName} | ||
| USING fts5(id UNINDEXED, ${stringColumns}, tokenize='${tokenizationMethod}'); | ||
| `); | ||
| // Copy over records already in table | ||
| await tx.execute(` | ||
| INSERT OR REPLACE INTO fts_${tableName}(rowid, id, ${stringColumns}) | ||
| SELECT rowid, id, ${generateJsonExtracts(ExtractType.columnOnly, 'data', columns)} FROM ${internalName}; | ||
| `); | ||
| // Add INSERT, UPDATE and DELETE and triggers to keep fts table in sync with table | ||
| await tx.execute(` | ||
| CREATE TRIGGER IF NOT EXISTS fts_insert_trigger_${tableName} AFTER INSERT ON ${internalName} | ||
| BEGIN | ||
| INSERT INTO fts_${tableName}(rowid, id, ${stringColumns}) | ||
| VALUES ( | ||
| NEW.rowid, | ||
| NEW.id, | ||
| ${generateJsonExtracts(ExtractType.columnOnly, 'NEW.data', columns)} | ||
| ); | ||
| END; | ||
| `); | ||
| await tx.execute(` | ||
| CREATE TRIGGER IF NOT EXISTS fts_update_trigger_${tableName} AFTER UPDATE ON ${internalName} BEGIN | ||
| UPDATE fts_${tableName} | ||
| SET ${generateJsonExtracts(ExtractType.columnInOperation, 'NEW.data', columns)} | ||
| WHERE rowid = NEW.rowid; | ||
| END; | ||
| `); | ||
| await tx.execute(` | ||
| CREATE TRIGGER IF NOT EXISTS fts_delete_trigger_${tableName} AFTER DELETE ON ${internalName} BEGIN | ||
| DELETE FROM fts_${tableName} WHERE rowid = OLD.rowid; | ||
| END; | ||
| `); | ||
| }); | ||
| } | ||
|
|
||
| /** | ||
| * This is where you can add more methods to generate FTS tables in this demo | ||
| * that correspond to the tables in your schema and populate them | ||
| * with the data you would like to search on | ||
| */ | ||
| export async function configureFts(): Promise<void> { | ||
| await createFtsTable('lists', ['name'], 'porter unicode61'); | ||
| await createFtsTable('todos', ['description', 'list_id']); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| type ExtractGenerator = (jsonColumnName: string, columnName: string) => string; | ||
|
|
||
| export enum ExtractType { | ||
| columnOnly, | ||
| columnInOperation | ||
| } | ||
|
|
||
| type ExtractGeneratorMap = Map<ExtractType, ExtractGenerator>; | ||
|
|
||
| function _createExtract(jsonColumnName: string, columnName: string): string { | ||
| return `json_extract(${jsonColumnName}, '$.${columnName}')`; | ||
| } | ||
|
|
||
| const extractGeneratorsMap: ExtractGeneratorMap = new Map<ExtractType, ExtractGenerator>([ | ||
| [ExtractType.columnOnly, (jsonColumnName: string, columnName: string) => _createExtract(jsonColumnName, columnName)], | ||
| [ | ||
| ExtractType.columnInOperation, | ||
| (jsonColumnName: string, columnName: string) => { | ||
| let extract = _createExtract(jsonColumnName, columnName); | ||
| return `${columnName} = ${extract}`; | ||
| } | ||
| ] | ||
| ]); | ||
|
|
||
| export const generateJsonExtracts = (type: ExtractType, jsonColumnName: string, columns: string[]): string => { | ||
| const generator = extractGeneratorsMap.get(type); | ||
| if (generator == null) { | ||
| throw new Error('Unexpected null generator for key: $type'); | ||
| } | ||
|
|
||
| if (columns.length == 1) { | ||
| return generator(jsonColumnName, columns[0]); | ||
| } | ||
|
|
||
| return columns.map((column) => generator(jsonColumnName, column)).join(', '); | ||
| }; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
105 changes: 105 additions & 0 deletions
105
demos/react-supabase-todolist/src/components/widgets/SearchBarWidget.tsx
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,105 @@ | ||
| import { SearchResult, searchTable } from '@/app/utils/fts_helpers'; | ||
| import { Autocomplete, Box, Card, CardContent, FormControl, Paper, TextField, Typography } from '@mui/material'; | ||
| import React from 'react'; | ||
| import { useNavigate } from 'react-router-dom'; | ||
| import { TODO_LISTS_ROUTE } from '@/app/router'; | ||
| import { usePowerSync, usePowerSyncQuery } from '@journeyapps/powersync-react'; | ||
| import { LISTS_TABLE, ListRecord } from '@/library/powersync/AppSchema'; | ||
| import { todo } from 'node:test'; | ||
|
|
||
| // This is a simple search bar widget that allows users to search for lists and todo items | ||
| export const SearchBarWidget: React.FC<any> = (props) => { | ||
| const [searchResults, setSearchResults] = React.useState<SearchResult[]>([]); | ||
| const [value, setValue] = React.useState<SearchResult | null>(null); | ||
|
|
||
| const navigate = useNavigate(); | ||
| const powersync = usePowerSync(); | ||
|
|
||
| const handleInputChange = async (value: string) => { | ||
| if (value.length !== 0) { | ||
| let listsSearchResults: any[] = []; | ||
| let todoItemsSearchResults = await searchTable(value, 'todos'); | ||
| for (let i = 0; i < todoItemsSearchResults.length; i++) { | ||
| let res = await powersync.get<ListRecord>(`SELECT * FROM ${LISTS_TABLE} WHERE id = ?`, [ | ||
| todoItemsSearchResults[i]['list_id'] | ||
| ]); | ||
| todoItemsSearchResults[i]['list_name'] = res.name; | ||
| } | ||
| if (!todoItemsSearchResults.length) { | ||
| listsSearchResults = await searchTable(value, 'lists'); | ||
| } | ||
| let formattedListResults: SearchResult[] = listsSearchResults.map( | ||
| (result) => new SearchResult(result['id'], result['name']) | ||
| ); | ||
| let formattedTodoItemsResults: SearchResult[] = todoItemsSearchResults.map((result) => { | ||
| return new SearchResult(result['list_id'], result['list_name'] ?? '', result['description']); | ||
| }); | ||
| setSearchResults([...formattedTodoItemsResults, ...formattedListResults]); | ||
| } | ||
| }; | ||
|
|
||
| return ( | ||
| <div> | ||
| <FormControl sx={{ my: 1, display: 'flex' }}> | ||
| <Autocomplete | ||
| freeSolo | ||
| id="autocomplete-search" | ||
| options={searchResults} | ||
| value={value?.id} | ||
| getOptionLabel={(option) => { | ||
| if (option instanceof SearchResult) { | ||
| return option.todoName ?? option.listName; | ||
| } | ||
| return option; | ||
| }} | ||
| renderOption={(props, option) => ( | ||
| <Box component="li" {...props}> | ||
| <Card variant="outlined" sx={{ display: 'flex', width: '100%' }}> | ||
| <CardContent> | ||
| {option.listName && ( | ||
| <Typography sx={{ fontSize: 18 }} color="text.primary" gutterBottom> | ||
| {option.listName} | ||
| </Typography> | ||
| )} | ||
| {option.todoName && ( | ||
| <Typography sx={{ fontSize: 14 }} color="text.secondary"> | ||
| {'\u2022'} {option.todoName} | ||
| </Typography> | ||
| )} | ||
| </CardContent> | ||
| </Card> | ||
| </Box> | ||
| )} | ||
| filterOptions={(x) => x} | ||
| onInputChange={(event, newInputValue, reason) => { | ||
| if (reason === 'clear') { | ||
| setValue(null); | ||
| setSearchResults([]); | ||
| return; | ||
| } | ||
| handleInputChange(newInputValue); | ||
| }} | ||
| onChange={(event, newValue, reason) => { | ||
| if (reason === 'selectOption') { | ||
| if (newValue instanceof SearchResult) { | ||
| navigate(TODO_LISTS_ROUTE + '/' + newValue.id); | ||
| } | ||
| } | ||
| }} | ||
| selectOnFocus | ||
| clearOnBlur | ||
| handleHomeEndKeys | ||
| renderInput={(params) => ( | ||
| <TextField | ||
| {...params} | ||
| label="Search..." | ||
| InputProps={{ | ||
| ...params.InputProps | ||
| }} | ||
| /> | ||
| )} | ||
| /> | ||
| </FormControl> | ||
| </div> | ||
| ); | ||
| }; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.