Merge pull request #239 from voyage-ai/main

Introducing VoyageAI's new multimodal embedding model
weaviate · Dec 9, 2024 · d197556 · d197556
2 parents f3e6ddf + 3f7de11
commit d197556
Show file tree

Hide file tree

Showing 3 changed files with 62 additions and 0 deletions.
diff --git a/src/collections/config/types/vectorizer.ts b/src/collections/config/types/vectorizer.ts
@@ -24,6 +24,7 @@ export type Vectorizer =
   | 'multi2vec-bind'
   | Multi2VecPalmVectorizer
   | 'multi2vec-google'
+  | 'multi2vec-voyageai'
   | 'ref2vec-centroid'
   | 'text2vec-aws'
   | 'text2vec-azure-openai'
@@ -184,6 +185,24 @@ export type Multi2VecGoogleConfig = {
   };
 };
 
+/** The configuration for multi-media vectorization using the VoyageAI module.
+ *
+ * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/transformers/embeddings-multimodal) for detailed usage.
+ */
+export type Multi2VecVoyageAIConfig = {
+  /** The image fields used when vectorizing. */
+  imageFields?: string[];
+  /** The text fields used when vectorizing. */
+  textFields?: string[];
+  /** The weights of the fields used for vectorization. */
+  weights?: {
+    /** The weights of the image fields. */
+    imageFields?: number[];
+    /** The weights of the text fields. */
+    textFields?: number[];
+  };
+};
+
 /** The configuration for reference-based vectorization using the centroid method.
  *
  * See the [documentation](https://weaviate.io/developers/weaviate/modules/ref2vec-centroid) for detailed usage.
@@ -431,6 +450,7 @@ export type VectorizerConfig =
   | Multi2VecBindConfig
   | Multi2VecGoogleConfig
   | Multi2VecPalmConfig
+  | Multi2VecVoyageAIConfig
   | Ref2VecCentroidConfig
   | Text2VecAWSConfig
   | Text2VecAzureOpenAIConfig
@@ -460,6 +480,8 @@ export type VectorizerConfigType<V> = V extends 'img2vec-neural'
   ? Multi2VecGoogleConfig
   : V extends Multi2VecPalmVectorizer
   ? Multi2VecPalmConfig
+  : V extends 'multi2vec-voyageai'
+  ? Multi2VecVoyageAIConfig | undefined
   : V extends 'ref2vec-centroid'
   ? Ref2VecCentroidConfig
   : V extends 'text2vec-aws'

diff --git a/src/collections/configure/types/vectorizer.ts b/src/collections/configure/types/vectorizer.ts
@@ -150,6 +150,13 @@ export type Multi2VecGoogleConfigCreate = {
   vectorizeCollectionName?: boolean;
 };
 
+export type Multi2VecVoyageAIConfigCreate = {
+  /** The image fields to use in vectorization. Can be string of `Multi2VecField` type. If string, weight 0 will be assumed. */
+  imageFields?: string[] | Multi2VecField[];
+  /** The text fields to use in vectorization. Can be string of `Multi2VecField` type. If string, weight 0 will be assumed. */
+  textFields?: string[] | Multi2VecField[];
+};
+
 export type Ref2VecCentroidConfigCreate = Ref2VecCentroidConfig;
 
 export type Text2VecAWSConfigCreate = Text2VecAWSConfig;
@@ -197,6 +204,8 @@ export type VectorizerConfigCreateType<V> = V extends 'img2vec-neural'
   ? Multi2VecPalmConfigCreate
   : V extends 'multi2vec-google'
   ? Multi2VecGoogleConfigCreate
+  : V extends 'multi2vec-voyageai'
+  ? Multi2VecVoyageAIConfigCreate | undefined
   : V extends 'ref2vec-centroid'
   ? Ref2VecCentroidConfigCreate
   : V extends 'text2vec-aws'

diff --git a/src/collections/configure/vectorizer.ts b/src/collections/configure/vectorizer.ts
@@ -3,6 +3,7 @@ import {
   Multi2VecClipConfig,
   Multi2VecField,
   Multi2VecPalmConfig,
+  Multi2VecVoyageAIConfig,
   VectorIndexType,
   Vectorizer,
   VectorizerConfigType,
@@ -263,6 +264,36 @@ export const vectorizer = {
       },
     });
   },
+  /**
+   * Create a `VectorConfigCreate` object with the vectorizer set to `'multi2vec-clip'`.
+   *
+   * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/transformers/embeddings-multimodal) for detailed usage.
+   *
+   * @param {ConfigureNonTextVectorizerOptions<N, I, 'multi2vec-voyageai'>} [opts] The configuration options for the `multi2vec-voyageai` vectorizer.
+   * @returns {VectorConfigCreate<PrimitiveKeys<T>[], N, I, 'multi2vec-voyageai'>} The configuration object.
+   */
+  multi2VecVoyageAI: <N extends string | undefined = undefined, I extends VectorIndexType = 'hnsw'>(
+    opts?: ConfigureNonTextVectorizerOptions<N, I, 'multi2vec-voyageai'>
+  ): VectorConfigCreate<never, N, I, 'multi2vec-voyageai'> => {
+    const { name, vectorIndexConfig, ...config } = opts || {};
+    const imageFields = config.imageFields?.map(mapMulti2VecField);
+    const textFields = config.textFields?.map(mapMulti2VecField);
+    let weights: Multi2VecVoyageAIConfig['weights'] = {};
+    weights = formatMulti2VecFields(weights, 'imageFields', imageFields);
+    weights = formatMulti2VecFields(weights, 'textFields', textFields);
+    return makeVectorizer(name, {
+      vectorIndexConfig,
+      vectorizerConfig: {
+        name: 'multi2vec-voyageai',
+        config: {
+          ...config,
+          imageFields: imageFields?.map((f) => f.name),
+          textFields: textFields?.map((f) => f.name),
+          weights: Object.keys(weights).length === 0 ? undefined : weights,
+        },
+      },
+    });
+  },
   /**
    * Create a `VectorConfigCreate` object with the vectorizer set to `'ref2vec-centroid'`.
    *