feat: supporting multi dataset

README.md

@@ -254,7 +254,6 @@ export interface IGWProps {
 	i18nLang?: string;
 	i18nResources?: { [lang: string]: Record<string, string | any> };
 	keepAlive?: boolean | string;
-    fieldKeyGuard?: boolean;
     vizThemeConfig?: IThemeKey;
     apperence?: IDarkMode;
     storeRef?: React.MutableRefObject<IGlobalStore | null>;

packages/duckdb-wasm-computation/src/index.ts

@@ -5,7 +5,7 @@ import duckdb_wasm from '@duckdb/duckdb-wasm/dist/duckdb-mvp.wasm?url';
 import mvp_worker from '@duckdb/duckdb-wasm/dist/duckdb-browser-mvp.worker.js?url';
 import duckdb_wasm_eh from '@duckdb/duckdb-wasm/dist/duckdb-eh.wasm?url';
 import eh_worker from '@duckdb/duckdb-wasm/dist/duckdb-browser-eh.worker.js?url';
-import initWasm, { parser_dsl_with_table } from '@kanaries/gw-dsl-parser';
+import initWasm, { parser_dsl_with_meta, parser_dsl_with_table } from '@kanaries/gw-dsl-parser';
 import dslWasm from '@kanaries/gw-dsl-parser/gw_dsl_parser_bg.wasm?url';
 import { nanoid } from 'nanoid';
 import type { IDataSourceProvider, IMutField, IDataSourceListener } from '@kanaries/graphic-walker';
@@ -51,7 +51,7 @@ const ArrowToJSON = (v: any): any => {
     if (typeof v === 'object') {
         if (v instanceof Vector) {
             return Array.from(v).map(ArrowToJSON);
-        } else {
+        } else if (v !== null) {
             return parseInt(bigNumToString(v as any));
         }
     }
@@ -68,6 +68,7 @@ const transformData = (table: Table) => {
 export async function getMemoryProvider(): Promise<IDataSourceProvider> {
     await init();
     const conn = await db.connect();
+    const files: { id: string; content: any }[] = [];
     const datasets: { name: string; id: string }[] = [];
     const metaDict = new Map<string, IMutField[]>();
     const specDict = new Map<string, string>();
@@ -82,6 +83,7 @@ export async function getMemoryProvider(): Promise<IDataSourceProvider> {
             const filename = `${id}.json`;
             await db.registerFileText(filename, JSON.stringify(data));
             await conn.insertJSONFromPath(filename, { name: id });
+            files.push({ id, content: data });
             datasets.push({ id, name });
             metaDict.set(id, meta);
             specDict.set(id, JSON.stringify([exportFullRaw(fromFields(meta, 'Chart 1'))]));
@@ -101,7 +103,11 @@ export async function getMemoryProvider(): Promise<IDataSourceProvider> {
         async getSpecs(datasetId) {
             const specs = specDict.get(datasetId);
             if (!specs) {
-                throw new Error('cannot find specs');
+                const selectedDatasets: string[] = JSON.parse(datasetId);
+                const fields = selectedDatasets.flatMap((dataset) => metaDict.get(dataset)?.map((x) => ({ ...x, dataset })) ?? []);
+                const specs = JSON.stringify([exportFullRaw(fromFields(fields, 'Chart 1'))]);
+                specDict.set(datasetId, specs);
+                return specs;
             }
             return specs;
         },
@@ -110,7 +116,13 @@ export async function getMemoryProvider(): Promise<IDataSourceProvider> {
             listeners.forEach((cb) => cb(4, datasetId));
         },
         async queryData(query, datasetIds) {
-            const sql = parser_dsl_with_table(datasetIds[0], JSON.stringify(query));
+            let sql: string;
+            if (datasetIds.length === 1) {
+                sql = parser_dsl_with_table(datasetIds[0], JSON.stringify(query));
+            } else {
+                const metas = Object.fromEntries(datasetIds.map((id) => [id, metaDict.get(id)!.map((x) => ({ key: x.fid, type: 'string' }))]));
+                sql = parser_dsl_with_meta(query.datasets[0], JSON.stringify(query), JSON.stringify(metas));
+            }
             if (process.env.NODE_ENV !== 'production') {
                 console.log(query, sql);
             }
@@ -123,6 +135,40 @@ export async function getMemoryProvider(): Promise<IDataSourceProvider> {
                 listeners.filter((x) => x !== cb);
             };
         },
+        async onExportFile() {
+            const data = {
+                files,
+                datasets,
+                metaDict: Array.from(metaDict.entries()),
+                specDict: Array.from(specDict.entries()),
+            };
+            const result = new Blob([JSON.stringify(data)], { type: 'text/plain' });
+            return result;
+        },
+        async onImportFile(file) {
+            const data = JSON.parse(await file.text()) as {
+                files: {
+                    id: string;
+                    content: any;
+                }[];
+                datasets: {
+                    name: string;
+                    id: string;
+                }[];
+                metaDict: [string, IMutField[]][];
+                specDict: [string, string][];
+            };
+            files.push(...data.files);
+            for (const { id, content } of data.files) {
+                const filename = `${id}.json`;
+                await db.registerFileText(filename, JSON.stringify(content));
+                await conn.insertJSONFromPath(filename, { name: id });
+            }
+            data.datasets.forEach((x) => datasets.push(x));
+            data.metaDict.forEach(([k, v]) => metaDict.set(k, v));
+            data.specDict.forEach(([k, v]) => specDict.set(k, v));
+            listeners.forEach((cb) => cb(1, ''));
+        },
     };
 }
 

packages/graphic-walker/package.json

@@ -48,7 +48,7 @@
   },
   "types": "./dist/index.d.ts",
   "dependencies": {
-    "@headlessui-float/react": "^0.11.4",
+    "@headlessui-float/react": "^0.13.2",
     "@headlessui/react": "1.7.12",
     "@heroicons/react": "^2.0.8",
     "@kanaries/react-beautiful-dnd": "^0.1.1",
@@ -58,6 +58,7 @@
     "@radix-ui/react-context-menu": "^2.1.5",
     "@radix-ui/react-dialog": "^1.0.5",
     "@radix-ui/react-dropdown-menu": "^2.0.6",
+    "@radix-ui/react-hover-card": "^1.0.7",
     "@radix-ui/react-icons": "^1.3.0",
     "@radix-ui/react-label": "^2.0.2",
     "@radix-ui/react-popover": "^1.0.7",

packages/graphic-walker/src/App.tsx

@@ -20,7 +20,6 @@ import GeoConfigPanel from './components/leafletRenderer/geoConfigPanel';
 import AskViz from './components/askViz';
 import { renderSpec } from './store/visualSpecStore';
 import FieldsContextWrapper from './fields/fieldsContext';
-import { guardDataKeys } from './utils/dataPrep';
 import { getComputation } from './computation/clientComputation';
 import LogPanel from './fields/datasetFields/logPanel';
 import BinPanel from './fields/datasetFields/binPanel';
@@ -41,7 +40,9 @@ import { VizAppContext } from './store/context';
 import { Tabs, TabsList, TabsTrigger } from './components/ui/tabs';
 import { ChartPieIcon, CircleStackIcon, ChatBubbleLeftRightIcon } from '@heroicons/react/24/outline';
 import { TabsContent } from '@radix-ui/react-tabs';
+import MultiDatasetFields from './fields/datasetFields/multi';
 import { VegaliteChat } from './components/chat';
+import { LinkDataset } from './components/linkDataset';
 
 export type BaseVizProps = IAppI18nProps &
     IVizProps &
@@ -68,6 +69,7 @@ export const VizApp = observer(function VizApp(props: BaseVizProps) {
         chart,
         vlSpec,
         onError,
+        datasetNames,
     } = props;
 
     const { t, i18n } = useTranslation();
@@ -145,12 +147,13 @@ export const VizApp = observer(function VizApp(props: BaseVizProps) {
 
     return (
         <ErrorContext value={{ reportError }}>
-            <ErrorBoundary fallback={<div>Something went wrong</div>} onError={props.onError}>
+            <ErrorBoundary fallback={<div>Something went wrong</div>} onError={console.log}>
                 <VizAppContext
                     ComputationContext={wrappedComputation}
                     themeContext={darkMode}
                     vegaThemeContext={{ vizThemeConfig: props.vizThemeConfig ?? props.themeConfig ?? props.themeKey }}
                     portalContainerContext={portal}
+                    DatasetNamesContext={props.datasetNames}
                 >
                     <div className={classNames(`App font-sans bg-background text-foreground m-0 p-0`, darkMode === 'dark' ? 'dark' : '')}>
                         <FieldsContextWrapper>
@@ -214,6 +217,7 @@ export const VizApp = observer(function VizApp(props: BaseVizProps) {
                                             <RenamePanel />
                                             <ComputedFieldDialog />
                                             <Painter themeConfig={themeConfig} themeKey={themeKey} />
+                                            <LinkDataset />
                                             {vizStore.showGeoJSONConfigPanel && <GeoConfigPanel geoList={props.geoList} />}
                                             <div className="sm:flex">
                                                 <SideResize
@@ -222,7 +226,8 @@ export const VizApp = observer(function VizApp(props: BaseVizProps) {
                                                     className="min-w-[100%] max-w-full sm:min-w-[96px] sm:max-w-[35%] flex-shrink-0"
                                                     handlerClassName="hidden sm:block"
                                                 >
-                                                    <DatasetFields />
+                                                    {!vizStore.isMultiDataset && <DatasetFields />}
+                                                    {vizStore.isMultiDataset && <MultiDatasetFields />}
                                                 </SideResize>
                                                 <SideResize
                                                     defaultWidth={180}
@@ -284,7 +289,7 @@ export const VizApp = observer(function VizApp(props: BaseVizProps) {
 });
 
 export function VizAppWithContext(props: IVizAppProps & IComputationProps) {
-    const { computation, onMetaChange, fieldKeyGuard, keepAlive, storeRef, defaultConfig, ...rest } = props;
+    const { computation, onMetaChange, keepAlive, storeRef, defaultConfig, ...rest } = props;
     // @TODO remove deprecated props
     const appearance = props.appearance ?? props.dark;
     const data = props.data ?? props.dataSource;
@@ -295,19 +300,6 @@ export function VizAppWithContext(props: IVizAppProps & IComputationProps) {
         onMetaChange: safeOnMetaChange,
     } = useMemo(() => {
         if (data) {
-            if (props.fieldKeyGuard) {
-                const { safeData, safeMetas } = guardDataKeys(data, fields);
-                return {
-                    safeMetas,
-                    computation: getComputation(safeData),
-                    onMetaChange: (safeFID, meta) => {
-                        const index = safeMetas.findIndex((x) => x.fid === safeFID);
-                        if (index >= 0) {
-                            props.onMetaChange?.(fields[index].fid, meta);
-                        }
-                    },
-                };
-            }
             return {
                 safeMetas: fields,
                 computation: getComputation(data),
@@ -319,7 +311,7 @@ export function VizAppWithContext(props: IVizAppProps & IComputationProps) {
             computation: props.computation,
             onMetaChange: props.onMetaChange,
         };
-    }, [fields, data ? data : props.computation, props.fieldKeyGuard, props.onMetaChange]);
+    }, [fields, data ? data : props.computation, props.onMetaChange]);
 
     const darkMode = useCurrentMediaTheme(appearance);
 

packages/graphic-walker/src/Renderer.tsx

@@ -18,13 +18,12 @@ import ReactiveRenderer from './renderer/index';
 import { ComputationContext, VizStoreWrapper, useCompututaion, useVizStore, withErrorReport, withTimeout } from './store';
 import { mergeLocaleRes, setLocaleLanguage } from './locales/i18n';
 import { renderSpec } from './store/visualSpecStore';
-import { guardDataKeys } from './utils/dataPrep';
 import { getComputation } from './computation/clientComputation';
 import { ErrorContext } from './utils/reportError';
 import { ErrorBoundary } from 'react-error-boundary';
 import Errorpanel from './components/errorpanel';
 import { useCurrentMediaTheme } from './utils/media';
-import { classNames, getFilterMeaAggKey, parseErrorMessage } from './utils';
+import { classNames, getFieldIdentifier, getFilterMeaAggKey, isSameField, parseErrorMessage } from './utils';
 import { VegaliteMapper } from './lib/vl2gw';
 import { newChart } from './models/visSpecHistory';
 import { SimpleOneOfSelector, SimpleRange, SimpleSearcher, SimpleTemporalRange } from './fields/filterField/simple';
@@ -138,6 +137,7 @@ export const RendererApp = observer(function VizApp(props: BaseVizProps) {
                     themeContext={darkMode}
                     vegaThemeContext={{ vizThemeConfig: vizThemeConfig ?? themeConfig ?? themeKey }}
                     portalContainerContext={portal}
+                    DatasetNamesContext={props.datasetNames}
                 >
                     <div className={`${darkMode === 'dark' ? 'dark' : ''} App font-sans bg-background text-foreground m-0 p-0`}>
                         <div className="flex flex-col space-y-2 bg-background text-foreground">
@@ -170,12 +170,12 @@ const FilterItem = observer(function FilterItem({ filter, onChange }: { filter:
 
     const computation = useCompututaion();
 
-    const originalField = filter.enableAgg ? allFields.find((x) => x.fid === filter.fid) : undefined;
+    const originalField = filter.enableAgg ? allFields.find(isSameField(filter)) : undefined;
     const filterAggName = filter?.enableAgg ? filter.aggName : undefined;
 
     const transformedComputation = useMemo((): IComputationFunction => {
         if (originalField && viewDimensions.length > 0) {
-            const preWorkflow = toWorkflow(
+            const { workflow, datasets } = toWorkflow(
                 [],
                 allFields,
                 viewDimensions,
@@ -185,24 +185,26 @@ const FilterItem = observer(function FilterItem({ filter, onChange }: { filter:
                 [],
                 undefined,
                 timezoneDisplayOffset
-            ).map((x) => {
-                if (x.type === 'view') {
-                    return {
-                        ...x,
-                        query: x.query.map((q) => {
-                            if (q.op === 'aggregate') {
-                                return { ...q, measures: q.measures.map((m) => ({ ...m, asFieldKey: m.field })) };
-                            }
-                            return q;
-                        }),
-                    };
-                }
-                return x;
-            });
+            );
             return (query) =>
                 computation({
                     ...query,
-                    workflow: preWorkflow.concat(query.workflow.filter((x) => x.type !== 'transform')),
+                    workflow: workflow
+                        .map((x) => {
+                            if (x.type === 'view') {
+                                return {
+                                    ...x,
+                                    query: x.query.map((q) => {
+                                        if (q.op === 'aggregate') {
+                                            return { ...q, measures: q.measures.map((m) => ({ ...m, asFieldKey: m.field })) };
+                                        }
+                                        return q;
+                                    }),
+                                };
+                            }
+                            return x;
+                        })
+                        .concat(query.workflow.filter((x) => x.type !== 'transform')),
                 });
         } else {
             return computation;
@@ -264,7 +266,7 @@ const FilterSection = observer(function FilterSection() {
     return (
         <div className={classNames('grid gap-2 px-2', cols)} ref={ref}>
             {vizStore.viewFilters.map((filter, idx) => (
-                <FilterItem key={filter.fid} filter={filter} onChange={(rule) => handleWriteFilter(idx, rule)} />
+                <FilterItem key={getFieldIdentifier(filter)} filter={filter} onChange={(rule) => handleWriteFilter(idx, rule)} />
             ))}
         </div>
     );
@@ -273,7 +275,7 @@ const FilterSection = observer(function FilterSection() {
 export function RendererAppWithContext(
     props: IVizAppProps & IComputationProps & { overrideSize?: IVisualLayout['size']; containerClassName?: string; containerStyle?: React.CSSProperties }
 ) {
-    const { dark, dataSource, computation, onMetaChange, fieldKeyGuard, keepAlive, storeRef, defaultConfig, ...rest } = props;
+    const { dark, dataSource, computation, onMetaChange, keepAlive, storeRef, defaultConfig, ...rest } = props;
     // @TODO remove deprecated props
     const appearance = props.appearance ?? props.dark;
     const data = props.data ?? props.dataSource;
@@ -285,19 +287,6 @@ export function RendererAppWithContext(
         onMetaChange: safeOnMetaChange,
     } = useMemo(() => {
         if (data) {
-            if (props.fieldKeyGuard) {
-                const { safeData, safeMetas } = guardDataKeys(data, fields);
-                return {
-                    safeMetas,
-                    computation: getComputation(safeData),
-                    onMetaChange: (safeFID, meta) => {
-                        const index = safeMetas.findIndex((x) => x.fid === safeFID);
-                        if (index >= 0) {
-                            props.onMetaChange?.(fields[index].fid, meta);
-                        }
-                    },
-                };
-            }
             return {
                 safeMetas: fields,
                 computation: getComputation(data),
@@ -309,7 +298,7 @@ export function RendererAppWithContext(
             computation: props.computation,
             onMetaChange: props.onMetaChange,
         };
-    }, [fields, data ? data : props.computation, props.fieldKeyGuard, props.onMetaChange]);
+    }, [fields, data ? data : props.computation, props.onMetaChange]);
 
     const darkMode = useCurrentMediaTheme(appearance);