From f335dfe5eed64829020c41ec81e001765767c9e4 Mon Sep 17 00:00:00 2001 From: Eugene Molodkin Date: Fri, 12 Jul 2024 17:53:15 +0200 Subject: [PATCH 1/7] wip: Update operation for vector store node --- .../VectorStorePinecone.node.ts | 89 +++++++-- .../shared/createVectorStoreNode.ts | 171 +++++++++++++++--- packages/workflow/src/Interfaces.ts | 1 + 3 files changed, 216 insertions(+), 45 deletions(-) diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePinecone/VectorStorePinecone.node.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePinecone/VectorStorePinecone.node.ts index 08ad2b18dc21f..db6165c0b3308 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePinecone/VectorStorePinecone.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePinecone/VectorStorePinecone.node.ts @@ -9,6 +9,15 @@ import { pineconeIndexSearch } from '../shared/methods/listSearch'; const sharedFields: INodeProperties[] = [pineconeIndexRLC]; +const pineconeNamespaceField: INodeProperties = { + displayName: 'Pinecone Namespace', + name: 'pineconeNamespace', + type: 'string', + description: + 'Partition the records in an index into namespaces. Queries and other operations are then limited to one namespace, so different requests can search different subsets of your index.', + default: '', +}; + const retrieveFields: INodeProperties[] = [ { displayName: 'Options', @@ -16,17 +25,7 @@ const retrieveFields: INodeProperties[] = [ type: 'collection', placeholder: 'Add Option', default: {}, - options: [ - { - displayName: 'Pinecone Namespace', - name: 'pineconeNamespace', - type: 'string', - description: - 'Partition the records in an index into namespaces. Queries and other operations are then limited to one namespace, so different requests can search different subsets of your index.', - default: '', - }, - metadataFilterField, - ], + options: [pineconeNamespaceField, metadataFilterField], }, ]; @@ -45,17 +44,22 @@ const insertFields: INodeProperties[] = [ default: false, description: 'Whether to clear the namespace before inserting new data', }, - { - displayName: 'Pinecone Namespace', - name: 'pineconeNamespace', - type: 'string', - description: - 'Partition the records in an index into namespaces. Queries and other operations are then limited to one namespace, so different requests can search different subsets of your index.', - default: '', - }, + pineconeNamespaceField, ], }, ]; + +const updateFields: INodeProperties[] = [ + { + displayName: 'Options', + name: 'options', + type: 'collection', + placeholder: 'Add Option', + default: {}, + options: [pineconeNamespaceField, metadataFilterField], + }, +]; + export const VectorStorePinecone = createVectorStoreNode({ meta: { displayName: 'Pinecone Vector Store', @@ -75,6 +79,7 @@ export const VectorStorePinecone = createVectorStoreNode({ retrieveFields, loadFields: retrieveFields, insertFields, + updateFields, sharedFields, async getVectorStoreClient(context, filter, embeddings, itemIndex) { const index = context.getNodeParameter('pineconeIndex', itemIndex, '', { @@ -138,4 +143,50 @@ export const VectorStorePinecone = createVectorStoreNode({ pineconeIndex, }); }, + async updateVectorStore() { + return; + }, + // async updateVectorStore(context, embeddings, document, itemIndex) { + // const index = context.getNodeParameter('pineconeIndex', itemIndex, '', { + // extractValue: true, + // }) as string; + // + // const documentId = context.getNodeParameter('id', itemIndex, '', { + // extractValue: true, + // }) as string; + // + // if (!documentId) { + // throw new NodeOperationError(context.getNode(), 'ID is required'); + // } + // + // const credentials = await context.getCredentials('pineconeApi'); + // + // const client = new Pinecone({ + // apiKey: credentials.apiKey as string, + // }); + // + // const indexes = ((await client.listIndexes()).indexes ?? []).map((i) => i.name); + // + // if (!indexes.includes(index)) { + // throw new NodeOperationError(context.getNode(), `Index ${index} not found`, { + // itemIndex, + // description: 'Please check that the index exists in your vector store', + // }); + // } + // + // const pineconeIndex = client.Index(index); + // + // const text = document.pageContent; + // + // console.log('UPDATE WILL HAPPEN HERE'); + // + // const config: PineconeStoreParams = { + // pineconeIndex, + // }; + // const vectorStore = await PineconeStore.fromExistingIndex(embeddings, config); + // + // await vectorStore.addVectors(await embeddings.embedDocuments([text]), [document], { + // ids: [documentId], + // }); + // }, }); diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/shared/createVectorStoreNode.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/shared/createVectorStoreNode.ts index 39617d873dbe2..c70759e372d4c 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/shared/createVectorStoreNode.ts +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/shared/createVectorStoreNode.ts @@ -13,11 +13,12 @@ import type { ILoadOptionsFunctions, INodeListSearchResult, Icon, + INodePropertyOptions, } from 'n8n-workflow'; import type { Embeddings } from '@langchain/core/embeddings'; import type { Document } from '@langchain/core/documents'; import { logWrapper } from '../../../utils/logWrapper'; -import type { N8nJsonLoader } from '../../../utils/N8nJsonLoader'; +import { N8nJsonLoader } from '../../../utils/N8nJsonLoader'; import type { N8nBinaryLoader } from '../../../utils/N8nBinaryLoader'; import { getMetadataFiltersValues, logAiEvent } from '../../../utils/helpers'; import { getConnectionHintNoticeField } from '../../../utils/sharedFields'; @@ -42,10 +43,12 @@ interface VectorStoreNodeConstructorArgs { ) => Promise; }; }; + sharedFields: INodeProperties[]; insertFields?: INodeProperties[]; loadFields?: INodeProperties[]; retrieveFields?: INodeProperties[]; + updateFields?: INodeProperties[]; populateVectorStore: ( context: IExecuteFunctions, embeddings: Embeddings, @@ -58,17 +61,78 @@ interface VectorStoreNodeConstructorArgs { embeddings: Embeddings, itemIndex: number, ) => Promise; + updateVectorStore?: ( + context: IExecuteFunctions, + embeddings: Embeddings, + document: Document>, + itemIndex: number, + ) => Promise; +} + +interface VectorStoreNodeUpdatableConstructorArgs extends VectorStoreNodeConstructorArgs { + updateVectorStore: ( + context: IExecuteFunctions, + embeddings: Embeddings, + document: Document>, + itemIndex: number, + ) => Promise; } function transformDescriptionForOperationMode( fields: INodeProperties[], - mode: 'insert' | 'load' | 'retrieve', + mode: 'insert' | 'load' | 'retrieve' | 'update', ) { return fields.map((field) => ({ ...field, displayOptions: { show: { mode: [mode] } }, })); } + +function isUpdateImplemented( + args: VectorStoreNodeConstructorArgs, +): args is VectorStoreNodeUpdatableConstructorArgs { + return Boolean((args as VectorStoreNodeUpdatableConstructorArgs).updateVectorStore); +} + +function getOperationModes(args: VectorStoreNodeConstructorArgs): INodePropertyOptions[] { + const commonOperationModes = [ + { + name: 'Get Many', + value: 'load', + description: 'Get many ranked documents from vector store for query', + action: 'Get many ranked documents from vector store for query', + }, + { + name: 'Insert Documents', + value: 'insert', + description: 'Insert documents into vector store', + action: 'Insert documents into vector store', + }, + { + name: 'Retrieve Documents (For Agent/Chain)', + value: 'retrieve', + description: 'Retrieve documents from vector store to be used with AI nodes', + action: 'Retrieve documents from vector store to be used with AI nodes', + }, + ]; + + // The update operation mode needs to be implemented for each specific vector store separately + // Enable this mode only if the implementation was provided in args + if (isUpdateImplemented(args)) { + return [ + ...commonOperationModes, + { + name: 'Update Documents', + value: 'update', + description: 'Update documents in vector store by ID', + action: 'Update documents in vector store by ID', + }, + ]; + } + + return commonOperationModes; +} + export const createVectorStoreNode = (args: VectorStoreNodeConstructorArgs) => class VectorStoreNodeType implements INodeType { description: INodeTypeDescription = { @@ -101,11 +165,11 @@ export const createVectorStoreNode = (args: VectorStoreNodeConstructorArgs) => const mode = parameters?.mode; const inputs = [{ displayName: "Embedding", type: "${NodeConnectionType.AiEmbedding}", required: true, maxConnections: 1}] - if (['insert', 'load'].includes(mode)) { + if (['insert', 'load', 'update'].includes(mode)) { inputs.push({ displayName: "", type: "${NodeConnectionType.Main}"}) } - if (mode === 'insert') { + if (['insert'].includes(mode)) { inputs.push({ displayName: "Document", type: "${NodeConnectionType.AiDocument}", required: true, maxConnections: 1}) } return inputs @@ -127,26 +191,7 @@ export const createVectorStoreNode = (args: VectorStoreNodeConstructorArgs) => type: 'options', noDataExpression: true, default: 'retrieve', - options: [ - { - name: 'Get Many', - value: 'load', - description: 'Get many ranked documents from vector store for query', - action: 'Get many ranked documents from vector store for query', - }, - { - name: 'Insert Documents', - value: 'insert', - description: 'Insert documents into vector store', - action: 'Insert documents into vector store', - }, - { - name: 'Retrieve Documents (For Agent/Chain)', - value: 'retrieve', - description: 'Retrieve documents from vector store to be used with AI nodes', - action: 'Retrieve documents from vector store to be used with AI nodes', - }, - ], + options: getOperationModes(args), }, { ...getConnectionHintNoticeField([NodeConnectionType.AiRetriever]), @@ -185,15 +230,30 @@ export const createVectorStoreNode = (args: VectorStoreNodeConstructorArgs) => }, }, }, + // ID is always used for update operation + { + displayName: 'ID', + name: 'id', + type: 'string', + default: '', + required: true, + description: 'ID of an embedding entry', + displayOptions: { + show: { + mode: ['update'], + }, + }, + }, ...transformDescriptionForOperationMode(args.loadFields ?? [], 'load'), ...transformDescriptionForOperationMode(args.retrieveFields ?? [], 'retrieve'), + ...transformDescriptionForOperationMode(args.updateFields ?? [], 'update'), ], }; methods = args.methods; async execute(this: IExecuteFunctions): Promise { - const mode = this.getNodeParameter('mode', 0) as 'load' | 'insert' | 'retrieve'; + const mode = this.getNodeParameter('mode', 0) as 'load' | 'insert' | 'retrieve' | 'update'; const embeddings = (await this.getInputConnectionData( NodeConnectionType.AiEmbedding, @@ -208,7 +268,7 @@ export const createVectorStoreNode = (args: VectorStoreNodeConstructorArgs) => const filter = getMetadataFiltersValues(this, itemIndex); const vectorStore = await args.getVectorStoreClient( this, - // We'll pass filter to similaritySearchVectorWithScore instaed of getVectorStoreClient + // We'll pass filter to similaritySearchVectorWithScore instead of getVectorStoreClient undefined, embeddings, itemIndex, @@ -274,6 +334,65 @@ export const createVectorStoreNode = (args: VectorStoreNodeConstructorArgs) => return [resultData]; } + if (mode === 'update') { + if (!isUpdateImplemented(args)) { + throw new NodeOperationError( + this.getNode(), + 'Update operation is not implemented for this Vector Store', + ); + } + + const items = this.getInputData(); + + // const documentInput = (await this.getInputConnectionData( + // NodeConnectionType.AiDocument, + // 0, + // )) as N8nJsonLoader | N8nBinaryLoader | Array>>; + + const loader = new N8nJsonLoader(this, 'options.'); + + const resultData = []; + for (let itemIndex = 0; itemIndex < items.length; itemIndex++) { + const itemData = items[itemIndex]; + + const documentId = this.getNodeParameter('id', itemIndex, '', { + extractValue: true, + }) as string; + + const { processedDocuments, serializedDocuments } = await processDocument( + loader, + itemData, + itemIndex, + ); + + if (processedDocuments?.length !== 1) { + throw new NodeOperationError(this.getNode(), 'Single document expected per item'); + } + + resultData.push(...serializedDocuments); + + const vectorStore = await args.getVectorStoreClient( + this, + undefined, + embeddings, + itemIndex, + ); + + try { + await vectorStore.addDocuments(processedDocuments, { + ids: [documentId], + }); + // await args.updateVectorStore(this, embeddings, processedDocuments[0], itemIndex); + + void logAiEvent(this, 'n8n.ai.vector.store.updated'); + } catch (error) { + throw error; + } + } + + return [resultData]; + } + throw new NodeOperationError( this.getNode(), 'Only the "load" and "insert" operation modes are supported with execute', diff --git a/packages/workflow/src/Interfaces.ts b/packages/workflow/src/Interfaces.ts index 184b00144ff81..a1a6ea9d6de1b 100644 --- a/packages/workflow/src/Interfaces.ts +++ b/packages/workflow/src/Interfaces.ts @@ -2127,6 +2127,7 @@ export const eventNamesAiNodes = [ 'n8n.ai.llm.generated', 'n8n.ai.llm.error', 'n8n.ai.vector.store.populated', + 'n8n.ai.vector.store.updated', ] as const; export type EventNamesAiNodesType = (typeof eventNamesAiNodes)[number]; From 5b148e39167ebd6353c5164d7acc332835b1fc77 Mon Sep 17 00:00:00 2001 From: Eugene Molodkin Date: Mon, 15 Jul 2024 11:57:27 +0200 Subject: [PATCH 2/7] wip: Fix vector store nodes without options --- packages/@n8n/nodes-langchain/utils/helpers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@n8n/nodes-langchain/utils/helpers.ts b/packages/@n8n/nodes-langchain/utils/helpers.ts index b1cd9a5960cd1..a548ab93c0250 100644 --- a/packages/@n8n/nodes-langchain/utils/helpers.ts +++ b/packages/@n8n/nodes-langchain/utils/helpers.ts @@ -15,7 +15,7 @@ export function getMetadataFiltersValues( ctx: IExecuteFunctions, itemIndex: number, ): Record | undefined { - const options = ctx.getNodeParameter('options', itemIndex); + const options = ctx.getNodeParameter('options', itemIndex, {}); if (options.metadata) { const { metadataValues: metadata } = options.metadata as { From 40c151a51a570af30dda8a51c23003e193087fa1 Mon Sep 17 00:00:00 2001 From: Eugene Molodkin Date: Mon, 15 Jul 2024 11:57:54 +0200 Subject: [PATCH 3/7] wip: Refactor vector store update operation --- .../VectorStorePinecone.node.ts | 69 +++----------- .../shared/createVectorStoreNode.ts | 93 +++++++------------ 2 files changed, 46 insertions(+), 116 deletions(-) diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePinecone/VectorStorePinecone.node.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePinecone/VectorStorePinecone.node.ts index db6165c0b3308..ca45fceeb41d2 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePinecone/VectorStorePinecone.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePinecone/VectorStorePinecone.node.ts @@ -49,16 +49,16 @@ const insertFields: INodeProperties[] = [ }, ]; -const updateFields: INodeProperties[] = [ - { - displayName: 'Options', - name: 'options', - type: 'collection', - placeholder: 'Add Option', - default: {}, - options: [pineconeNamespaceField, metadataFilterField], - }, -]; +// const updateFields: INodeProperties[] = [ +// { +// displayName: 'Options', +// name: 'options', +// type: 'collection', +// placeholder: 'Add Option', +// default: {}, +// options: [pineconeNamespaceField, metadataFilterField], +// }, +// ]; export const VectorStorePinecone = createVectorStoreNode({ meta: { @@ -74,12 +74,13 @@ export const VectorStorePinecone = createVectorStoreNode({ required: true, }, ], + operationModes: ['load', 'insert', 'retrieve', 'update'], }, methods: { listSearch: { pineconeIndexSearch } }, retrieveFields, loadFields: retrieveFields, insertFields, - updateFields, + // updateFields, sharedFields, async getVectorStoreClient(context, filter, embeddings, itemIndex) { const index = context.getNodeParameter('pineconeIndex', itemIndex, '', { @@ -143,50 +144,4 @@ export const VectorStorePinecone = createVectorStoreNode({ pineconeIndex, }); }, - async updateVectorStore() { - return; - }, - // async updateVectorStore(context, embeddings, document, itemIndex) { - // const index = context.getNodeParameter('pineconeIndex', itemIndex, '', { - // extractValue: true, - // }) as string; - // - // const documentId = context.getNodeParameter('id', itemIndex, '', { - // extractValue: true, - // }) as string; - // - // if (!documentId) { - // throw new NodeOperationError(context.getNode(), 'ID is required'); - // } - // - // const credentials = await context.getCredentials('pineconeApi'); - // - // const client = new Pinecone({ - // apiKey: credentials.apiKey as string, - // }); - // - // const indexes = ((await client.listIndexes()).indexes ?? []).map((i) => i.name); - // - // if (!indexes.includes(index)) { - // throw new NodeOperationError(context.getNode(), `Index ${index} not found`, { - // itemIndex, - // description: 'Please check that the index exists in your vector store', - // }); - // } - // - // const pineconeIndex = client.Index(index); - // - // const text = document.pageContent; - // - // console.log('UPDATE WILL HAPPEN HERE'); - // - // const config: PineconeStoreParams = { - // pineconeIndex, - // }; - // const vectorStore = await PineconeStore.fromExistingIndex(embeddings, config); - // - // await vectorStore.addVectors(await embeddings.embedDocuments([text]), [document], { - // ids: [documentId], - // }); - // }, }); diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/shared/createVectorStoreNode.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/shared/createVectorStoreNode.ts index c70759e372d4c..05faed9e8f030 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/shared/createVectorStoreNode.ts +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/shared/createVectorStoreNode.ts @@ -24,6 +24,10 @@ import { getMetadataFiltersValues, logAiEvent } from '../../../utils/helpers'; import { getConnectionHintNoticeField } from '../../../utils/sharedFields'; import { processDocument } from './processDocuments'; +type NodeOperationMode = 'insert' | 'load' | 'retrieve' | 'update'; + +const DEFAULT_OPERATION_MODES: NodeOperationMode[] = ['load', 'insert', 'retrieve']; + interface NodeMeta { displayName: string; name: string; @@ -31,7 +35,9 @@ interface NodeMeta { docsUrl: string; icon: Icon; credentials?: INodeCredentialDescription[]; + operationModes?: NodeOperationMode[]; } + interface VectorStoreNodeConstructorArgs { meta: NodeMeta; methods?: { @@ -48,7 +54,6 @@ interface VectorStoreNodeConstructorArgs { insertFields?: INodeProperties[]; loadFields?: INodeProperties[]; retrieveFields?: INodeProperties[]; - updateFields?: INodeProperties[]; populateVectorStore: ( context: IExecuteFunctions, embeddings: Embeddings, @@ -61,41 +66,23 @@ interface VectorStoreNodeConstructorArgs { embeddings: Embeddings, itemIndex: number, ) => Promise; - updateVectorStore?: ( - context: IExecuteFunctions, - embeddings: Embeddings, - document: Document>, - itemIndex: number, - ) => Promise; } -interface VectorStoreNodeUpdatableConstructorArgs extends VectorStoreNodeConstructorArgs { - updateVectorStore: ( - context: IExecuteFunctions, - embeddings: Embeddings, - document: Document>, - itemIndex: number, - ) => Promise; -} - -function transformDescriptionForOperationMode( - fields: INodeProperties[], - mode: 'insert' | 'load' | 'retrieve' | 'update', -) { +function transformDescriptionForOperationMode(fields: INodeProperties[], mode: NodeOperationMode) { return fields.map((field) => ({ ...field, displayOptions: { show: { mode: [mode] } }, })); } -function isUpdateImplemented( - args: VectorStoreNodeConstructorArgs, -): args is VectorStoreNodeUpdatableConstructorArgs { - return Boolean((args as VectorStoreNodeUpdatableConstructorArgs).updateVectorStore); +function isUpdateSupported(args: VectorStoreNodeConstructorArgs): boolean { + return args.meta.operationModes?.includes('update') ?? false; } -function getOperationModes(args: VectorStoreNodeConstructorArgs): INodePropertyOptions[] { - const commonOperationModes = [ +function getOperationModeOptions(args: VectorStoreNodeConstructorArgs): INodePropertyOptions[] { + const enabledOperationModes = args.meta.operationModes ?? DEFAULT_OPERATION_MODES; + + const allOptions = [ { name: 'Get Many', value: 'load', @@ -114,23 +101,17 @@ function getOperationModes(args: VectorStoreNodeConstructorArgs): INodePropertyO description: 'Retrieve documents from vector store to be used with AI nodes', action: 'Retrieve documents from vector store to be used with AI nodes', }, + { + name: 'Update Documents', + value: 'update', + description: 'Update documents in vector store by ID', + action: 'Update documents in vector store by ID', + }, ]; - // The update operation mode needs to be implemented for each specific vector store separately - // Enable this mode only if the implementation was provided in args - if (isUpdateImplemented(args)) { - return [ - ...commonOperationModes, - { - name: 'Update Documents', - value: 'update', - description: 'Update documents in vector store by ID', - action: 'Update documents in vector store by ID', - }, - ]; - } - - return commonOperationModes; + return allOptions.filter(({ value }) => + enabledOperationModes.includes(value as NodeOperationMode), + ); } export const createVectorStoreNode = (args: VectorStoreNodeConstructorArgs) => @@ -191,7 +172,7 @@ export const createVectorStoreNode = (args: VectorStoreNodeConstructorArgs) => type: 'options', noDataExpression: true, default: 'retrieve', - options: getOperationModes(args), + options: getOperationModeOptions(args), }, { ...getConnectionHintNoticeField([NodeConnectionType.AiRetriever]), @@ -246,7 +227,6 @@ export const createVectorStoreNode = (args: VectorStoreNodeConstructorArgs) => }, ...transformDescriptionForOperationMode(args.loadFields ?? [], 'load'), ...transformDescriptionForOperationMode(args.retrieveFields ?? [], 'retrieve'), - ...transformDescriptionForOperationMode(args.updateFields ?? [], 'update'), ], }; @@ -335,7 +315,7 @@ export const createVectorStoreNode = (args: VectorStoreNodeConstructorArgs) => } if (mode === 'update') { - if (!isUpdateImplemented(args)) { + if (!isUpdateSupported(args)) { throw new NodeOperationError( this.getNode(), 'Update operation is not implemented for this Vector Store', @@ -344,12 +324,7 @@ export const createVectorStoreNode = (args: VectorStoreNodeConstructorArgs) => const items = this.getInputData(); - // const documentInput = (await this.getInputConnectionData( - // NodeConnectionType.AiDocument, - // 0, - // )) as N8nJsonLoader | N8nBinaryLoader | Array>>; - - const loader = new N8nJsonLoader(this, 'options.'); + const loader = new N8nJsonLoader(this); const resultData = []; for (let itemIndex = 0; itemIndex < items.length; itemIndex++) { @@ -359,6 +334,13 @@ export const createVectorStoreNode = (args: VectorStoreNodeConstructorArgs) => extractValue: true, }) as string; + const vectorStore = await args.getVectorStoreClient( + this, + undefined, + embeddings, + itemIndex, + ); + const { processedDocuments, serializedDocuments } = await processDocument( loader, itemData, @@ -366,23 +348,16 @@ export const createVectorStoreNode = (args: VectorStoreNodeConstructorArgs) => ); if (processedDocuments?.length !== 1) { - throw new NodeOperationError(this.getNode(), 'Single document expected per item'); + throw new NodeOperationError(this.getNode(), 'Single document per item expected'); } resultData.push(...serializedDocuments); - const vectorStore = await args.getVectorStoreClient( - this, - undefined, - embeddings, - itemIndex, - ); - try { + // Use ids option to upsert instead of insert await vectorStore.addDocuments(processedDocuments, { ids: [documentId], }); - // await args.updateVectorStore(this, embeddings, processedDocuments[0], itemIndex); void logAiEvent(this, 'n8n.ai.vector.store.updated'); } catch (error) { From 80b9d770248c74cda620ecd0decd6c38369b43a2 Mon Sep 17 00:00:00 2001 From: Eugene Molodkin Date: Mon, 15 Jul 2024 11:58:14 +0200 Subject: [PATCH 4/7] wip: Supabase Vector Store update operation --- .../vector_store/VectorStoreSupabase/VectorStoreSupabase.node.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreSupabase/VectorStoreSupabase.node.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreSupabase/VectorStoreSupabase.node.ts index ebeacd9a33f87..91e572f66d17b 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreSupabase/VectorStoreSupabase.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreSupabase/VectorStoreSupabase.node.ts @@ -58,6 +58,7 @@ export const VectorStoreSupabase = createVectorStoreNode({ required: true, }, ], + operationModes: ['load', 'insert', 'retrieve', 'update'], }, methods: { listSearch: { supabaseTableNameSearch }, From 5ddab328f262f45cd5300b2c9057a1f1c2731830 Mon Sep 17 00:00:00 2001 From: Eugene Molodkin Date: Mon, 15 Jul 2024 12:14:49 +0200 Subject: [PATCH 5/7] wip: Refactor update fields for vector store nodes --- .../VectorStoreSupabase.node.ts | 34 ++++++++----------- .../shared/createVectorStoreNode.ts | 2 ++ 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreSupabase/VectorStoreSupabase.node.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreSupabase/VectorStoreSupabase.node.ts index 91e572f66d17b..0269a0cef2e60 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreSupabase/VectorStoreSupabase.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStoreSupabase/VectorStoreSupabase.node.ts @@ -6,6 +6,14 @@ import { metadataFilterField } from '../../../utils/sharedFields'; import { supabaseTableNameRLC } from '../shared/descriptions'; import { supabaseTableNameSearch } from '../shared/methods/listSearch'; +const queryNameField: INodeProperties = { + displayName: 'Query Name', + name: 'queryName', + type: 'string', + default: 'match_documents', + description: 'Name of the query to use for matching documents', +}; + const sharedFields: INodeProperties[] = [supabaseTableNameRLC]; const insertFields: INodeProperties[] = [ { @@ -14,17 +22,10 @@ const insertFields: INodeProperties[] = [ type: 'collection', placeholder: 'Add Option', default: {}, - options: [ - { - displayName: 'Query Name', - name: 'queryName', - type: 'string', - default: 'match_documents', - description: 'Name of the query to use for matching documents', - }, - ], + options: [queryNameField], }, ]; + const retrieveFields: INodeProperties[] = [ { displayName: 'Options', @@ -32,18 +33,12 @@ const retrieveFields: INodeProperties[] = [ type: 'collection', placeholder: 'Add Option', default: {}, - options: [ - { - displayName: 'Query Name', - name: 'queryName', - type: 'string', - default: 'match_documents', - description: 'Name of the query to use for matching documents', - }, - metadataFilterField, - ], + options: [queryNameField, metadataFilterField], }, ]; + +const updateFields: INodeProperties[] = [...insertFields]; + export const VectorStoreSupabase = createVectorStoreNode({ meta: { description: 'Work with your data in Supabase Vector Store', @@ -67,6 +62,7 @@ export const VectorStoreSupabase = createVectorStoreNode({ insertFields, loadFields: retrieveFields, retrieveFields, + updateFields, async getVectorStoreClient(context, filter, embeddings, itemIndex) { const tableName = context.getNodeParameter('tableName', itemIndex, '', { extractValue: true, diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/shared/createVectorStoreNode.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/shared/createVectorStoreNode.ts index 05faed9e8f030..17e319d875fde 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/shared/createVectorStoreNode.ts +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/shared/createVectorStoreNode.ts @@ -54,6 +54,7 @@ interface VectorStoreNodeConstructorArgs { insertFields?: INodeProperties[]; loadFields?: INodeProperties[]; retrieveFields?: INodeProperties[]; + updateFields?: INodeProperties[]; populateVectorStore: ( context: IExecuteFunctions, embeddings: Embeddings, @@ -227,6 +228,7 @@ export const createVectorStoreNode = (args: VectorStoreNodeConstructorArgs) => }, ...transformDescriptionForOperationMode(args.loadFields ?? [], 'load'), ...transformDescriptionForOperationMode(args.retrieveFields ?? [], 'retrieve'), + ...transformDescriptionForOperationMode(args.updateFields ?? [], 'update'), ], }; From 4302c7c92e80259d2aca4edec7a820176430976f Mon Sep 17 00:00:00 2001 From: Eugene Molodkin Date: Mon, 15 Jul 2024 12:16:01 +0200 Subject: [PATCH 6/7] wip: Clean up --- .../VectorStorePinecone/VectorStorePinecone.node.ts | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePinecone/VectorStorePinecone.node.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePinecone/VectorStorePinecone.node.ts index ca45fceeb41d2..85509a6287064 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePinecone/VectorStorePinecone.node.ts +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/VectorStorePinecone/VectorStorePinecone.node.ts @@ -49,17 +49,6 @@ const insertFields: INodeProperties[] = [ }, ]; -// const updateFields: INodeProperties[] = [ -// { -// displayName: 'Options', -// name: 'options', -// type: 'collection', -// placeholder: 'Add Option', -// default: {}, -// options: [pineconeNamespaceField, metadataFilterField], -// }, -// ]; - export const VectorStorePinecone = createVectorStoreNode({ meta: { displayName: 'Pinecone Vector Store', @@ -80,7 +69,6 @@ export const VectorStorePinecone = createVectorStoreNode({ retrieveFields, loadFields: retrieveFields, insertFields, - // updateFields, sharedFields, async getVectorStoreClient(context, filter, embeddings, itemIndex) { const index = context.getNodeParameter('pineconeIndex', itemIndex, '', { From fa78722ca76658888cc1415a7c1c29c93c4d1ac6 Mon Sep 17 00:00:00 2001 From: Eugene Molodkin Date: Mon, 15 Jul 2024 12:18:58 +0200 Subject: [PATCH 7/7] wip: Clean up --- .../nodes/vector_store/shared/createVectorStoreNode.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@n8n/nodes-langchain/nodes/vector_store/shared/createVectorStoreNode.ts b/packages/@n8n/nodes-langchain/nodes/vector_store/shared/createVectorStoreNode.ts index 17e319d875fde..4fbe9eace5ae8 100644 --- a/packages/@n8n/nodes-langchain/nodes/vector_store/shared/createVectorStoreNode.ts +++ b/packages/@n8n/nodes-langchain/nodes/vector_store/shared/createVectorStoreNode.ts @@ -235,7 +235,7 @@ export const createVectorStoreNode = (args: VectorStoreNodeConstructorArgs) => methods = args.methods; async execute(this: IExecuteFunctions): Promise { - const mode = this.getNodeParameter('mode', 0) as 'load' | 'insert' | 'retrieve' | 'update'; + const mode = this.getNodeParameter('mode', 0) as NodeOperationMode; const embeddings = (await this.getInputConnectionData( NodeConnectionType.AiEmbedding,