Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(Pinecone Vector Store Node, Supabase Vector Store Node): Add update operation to vector store nodes #10060

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -9,24 +9,23 @@ import { pineconeIndexSearch } from '../shared/methods/listSearch';

const sharedFields: INodeProperties[] = [pineconeIndexRLC];

const pineconeNamespaceField: INodeProperties = {
displayName: 'Pinecone Namespace',
name: 'pineconeNamespace',
type: 'string',
description:
'Partition the records in an index into namespaces. Queries and other operations are then limited to one namespace, so different requests can search different subsets of your index.',
default: '',
};

const retrieveFields: INodeProperties[] = [
{
displayName: 'Options',
name: 'options',
type: 'collection',
placeholder: 'Add Option',
default: {},
options: [
{
displayName: 'Pinecone Namespace',
name: 'pineconeNamespace',
type: 'string',
description:
'Partition the records in an index into namespaces. Queries and other operations are then limited to one namespace, so different requests can search different subsets of your index.',
default: '',
},
metadataFilterField,
],
options: [pineconeNamespaceField, metadataFilterField],
},
];

Expand All @@ -45,17 +44,11 @@ const insertFields: INodeProperties[] = [
default: false,
description: 'Whether to clear the namespace before inserting new data',
},
{
displayName: 'Pinecone Namespace',
name: 'pineconeNamespace',
type: 'string',
description:
'Partition the records in an index into namespaces. Queries and other operations are then limited to one namespace, so different requests can search different subsets of your index.',
default: '',
},
pineconeNamespaceField,
],
},
];

export const VectorStorePinecone = createVectorStoreNode({
meta: {
displayName: 'Pinecone Vector Store',
Expand All @@ -70,6 +63,7 @@ export const VectorStorePinecone = createVectorStoreNode({
required: true,
},
],
operationModes: ['load', 'insert', 'retrieve', 'update'],
},
methods: { listSearch: { pineconeIndexSearch } },
retrieveFields,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,14 @@ import { metadataFilterField } from '../../../utils/sharedFields';
import { supabaseTableNameRLC } from '../shared/descriptions';
import { supabaseTableNameSearch } from '../shared/methods/listSearch';

const queryNameField: INodeProperties = {
displayName: 'Query Name',
name: 'queryName',
type: 'string',
default: 'match_documents',
description: 'Name of the query to use for matching documents',
};

const sharedFields: INodeProperties[] = [supabaseTableNameRLC];
const insertFields: INodeProperties[] = [
{
Expand All @@ -14,36 +22,23 @@ const insertFields: INodeProperties[] = [
type: 'collection',
placeholder: 'Add Option',
default: {},
options: [
{
displayName: 'Query Name',
name: 'queryName',
type: 'string',
default: 'match_documents',
description: 'Name of the query to use for matching documents',
},
],
options: [queryNameField],
},
];

const retrieveFields: INodeProperties[] = [
{
displayName: 'Options',
name: 'options',
type: 'collection',
placeholder: 'Add Option',
default: {},
options: [
{
displayName: 'Query Name',
name: 'queryName',
type: 'string',
default: 'match_documents',
description: 'Name of the query to use for matching documents',
},
metadataFilterField,
],
options: [queryNameField, metadataFilterField],
},
];

const updateFields: INodeProperties[] = [...insertFields];

export const VectorStoreSupabase = createVectorStoreNode({
meta: {
description: 'Work with your data in Supabase Vector Store',
Expand All @@ -58,6 +53,7 @@ export const VectorStoreSupabase = createVectorStoreNode({
required: true,
},
],
operationModes: ['load', 'insert', 'retrieve', 'update'],
},
methods: {
listSearch: { supabaseTableNameSearch },
Expand All @@ -66,6 +62,7 @@ export const VectorStoreSupabase = createVectorStoreNode({
insertFields,
loadFields: retrieveFields,
retrieveFields,
updateFields,
async getVectorStoreClient(context, filter, embeddings, itemIndex) {
const tableName = context.getNodeParameter('tableName', itemIndex, '', {
extractValue: true,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,24 +13,31 @@ import type {
ILoadOptionsFunctions,
INodeListSearchResult,
Icon,
INodePropertyOptions,
} from 'n8n-workflow';
import type { Embeddings } from '@langchain/core/embeddings';
import type { Document } from '@langchain/core/documents';
import { logWrapper } from '../../../utils/logWrapper';
import type { N8nJsonLoader } from '../../../utils/N8nJsonLoader';
import { N8nJsonLoader } from '../../../utils/N8nJsonLoader';
import type { N8nBinaryLoader } from '../../../utils/N8nBinaryLoader';
import { getMetadataFiltersValues, logAiEvent } from '../../../utils/helpers';
import { getConnectionHintNoticeField } from '../../../utils/sharedFields';
import { processDocument } from './processDocuments';

type NodeOperationMode = 'insert' | 'load' | 'retrieve' | 'update';

const DEFAULT_OPERATION_MODES: NodeOperationMode[] = ['load', 'insert', 'retrieve'];

interface NodeMeta {
displayName: string;
name: string;
description: string;
docsUrl: string;
icon: Icon;
credentials?: INodeCredentialDescription[];
operationModes?: NodeOperationMode[];
}

interface VectorStoreNodeConstructorArgs {
meta: NodeMeta;
methods?: {
Expand All @@ -42,10 +49,12 @@ interface VectorStoreNodeConstructorArgs {
) => Promise<INodeListSearchResult>;
};
};

sharedFields: INodeProperties[];
insertFields?: INodeProperties[];
loadFields?: INodeProperties[];
retrieveFields?: INodeProperties[];
updateFields?: INodeProperties[];
populateVectorStore: (
context: IExecuteFunctions,
embeddings: Embeddings,
Expand All @@ -60,15 +69,52 @@ interface VectorStoreNodeConstructorArgs {
) => Promise<VectorStore>;
}

function transformDescriptionForOperationMode(
fields: INodeProperties[],
mode: 'insert' | 'load' | 'retrieve',
) {
function transformDescriptionForOperationMode(fields: INodeProperties[], mode: NodeOperationMode) {
return fields.map((field) => ({
...field,
displayOptions: { show: { mode: [mode] } },
}));
}

function isUpdateSupported(args: VectorStoreNodeConstructorArgs): boolean {
return args.meta.operationModes?.includes('update') ?? false;
}

function getOperationModeOptions(args: VectorStoreNodeConstructorArgs): INodePropertyOptions[] {
const enabledOperationModes = args.meta.operationModes ?? DEFAULT_OPERATION_MODES;

const allOptions = [
{
name: 'Get Many',
value: 'load',
description: 'Get many ranked documents from vector store for query',
action: 'Get many ranked documents from vector store for query',
},
{
name: 'Insert Documents',
value: 'insert',
description: 'Insert documents into vector store',
action: 'Insert documents into vector store',
},
{
name: 'Retrieve Documents (For Agent/Chain)',
value: 'retrieve',
description: 'Retrieve documents from vector store to be used with AI nodes',
action: 'Retrieve documents from vector store to be used with AI nodes',
},
{
name: 'Update Documents',
value: 'update',
description: 'Update documents in vector store by ID',
action: 'Update documents in vector store by ID',
},
];

return allOptions.filter(({ value }) =>
enabledOperationModes.includes(value as NodeOperationMode),
);
}

export const createVectorStoreNode = (args: VectorStoreNodeConstructorArgs) =>
class VectorStoreNodeType implements INodeType {
description: INodeTypeDescription = {
Expand Down Expand Up @@ -101,11 +147,11 @@ export const createVectorStoreNode = (args: VectorStoreNodeConstructorArgs) =>
const mode = parameters?.mode;
const inputs = [{ displayName: "Embedding", type: "${NodeConnectionType.AiEmbedding}", required: true, maxConnections: 1}]

if (['insert', 'load'].includes(mode)) {
if (['insert', 'load', 'update'].includes(mode)) {
inputs.push({ displayName: "", type: "${NodeConnectionType.Main}"})
}

if (mode === 'insert') {
if (['insert'].includes(mode)) {
inputs.push({ displayName: "Document", type: "${NodeConnectionType.AiDocument}", required: true, maxConnections: 1})
}
return inputs
Expand All @@ -127,26 +173,7 @@ export const createVectorStoreNode = (args: VectorStoreNodeConstructorArgs) =>
type: 'options',
noDataExpression: true,
default: 'retrieve',
options: [
{
name: 'Get Many',
value: 'load',
description: 'Get many ranked documents from vector store for query',
action: 'Get many ranked documents from vector store for query',
},
{
name: 'Insert Documents',
value: 'insert',
description: 'Insert documents into vector store',
action: 'Insert documents into vector store',
},
{
name: 'Retrieve Documents (For Agent/Chain)',
value: 'retrieve',
description: 'Retrieve documents from vector store to be used with AI nodes',
action: 'Retrieve documents from vector store to be used with AI nodes',
},
],
options: getOperationModeOptions(args),
},
{
...getConnectionHintNoticeField([NodeConnectionType.AiRetriever]),
Expand Down Expand Up @@ -185,15 +212,30 @@ export const createVectorStoreNode = (args: VectorStoreNodeConstructorArgs) =>
},
},
},
// ID is always used for update operation
{
displayName: 'ID',
name: 'id',
type: 'string',
default: '',
required: true,
description: 'ID of an embedding entry',
displayOptions: {
show: {
mode: ['update'],
},
},
},
...transformDescriptionForOperationMode(args.loadFields ?? [], 'load'),
...transformDescriptionForOperationMode(args.retrieveFields ?? [], 'retrieve'),
...transformDescriptionForOperationMode(args.updateFields ?? [], 'update'),
],
};

methods = args.methods;

async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
const mode = this.getNodeParameter('mode', 0) as 'load' | 'insert' | 'retrieve';
const mode = this.getNodeParameter('mode', 0) as NodeOperationMode;

const embeddings = (await this.getInputConnectionData(
NodeConnectionType.AiEmbedding,
Expand All @@ -208,7 +250,7 @@ export const createVectorStoreNode = (args: VectorStoreNodeConstructorArgs) =>
const filter = getMetadataFiltersValues(this, itemIndex);
const vectorStore = await args.getVectorStoreClient(
this,
// We'll pass filter to similaritySearchVectorWithScore instaed of getVectorStoreClient
// We'll pass filter to similaritySearchVectorWithScore instead of getVectorStoreClient
undefined,
embeddings,
itemIndex,
Expand Down Expand Up @@ -274,6 +316,60 @@ export const createVectorStoreNode = (args: VectorStoreNodeConstructorArgs) =>
return [resultData];
}

if (mode === 'update') {
if (!isUpdateSupported(args)) {
throw new NodeOperationError(
this.getNode(),
'Update operation is not implemented for this Vector Store',
);
}

const items = this.getInputData();

const loader = new N8nJsonLoader(this);

const resultData = [];
for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
const itemData = items[itemIndex];

const documentId = this.getNodeParameter('id', itemIndex, '', {
extractValue: true,
}) as string;

const vectorStore = await args.getVectorStoreClient(
this,
undefined,
embeddings,
itemIndex,
);

const { processedDocuments, serializedDocuments } = await processDocument(
loader,
itemData,
itemIndex,
);

if (processedDocuments?.length !== 1) {
throw new NodeOperationError(this.getNode(), 'Single document per item expected');
}

resultData.push(...serializedDocuments);

try {
// Use ids option to upsert instead of insert
await vectorStore.addDocuments(processedDocuments, {
ids: [documentId],
});

void logAiEvent(this, 'n8n.ai.vector.store.updated');
} catch (error) {
throw error;
}
}

return [resultData];
}

throw new NodeOperationError(
this.getNode(),
'Only the "load" and "insert" operation modes are supported with execute',
Expand Down
2 changes: 1 addition & 1 deletion packages/@n8n/nodes-langchain/utils/helpers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ export function getMetadataFiltersValues(
ctx: IExecuteFunctions,
itemIndex: number,
): Record<string, never> | undefined {
const options = ctx.getNodeParameter('options', itemIndex);
const options = ctx.getNodeParameter('options', itemIndex, {});

if (options.metadata) {
const { metadataValues: metadata } = options.metadata as {
Expand Down
Loading
Loading