feat: support local storage

.dockerignore

@@ -5,4 +5,7 @@
 .env
 
 *.key
-*.key.pub
+*.key.pub
+
+# upload files
+/uploads

.env.template

@@ -50,4 +50,4 @@ DISABLE_FAST_LINK=
 # (optional)
 # Default: 1
 # If your project is not deployed on Vercel, set this value to 1.
-NEXT_PUBLIC_ENABLE_NODEJS_PLUGIN=1
+NEXT_PUBLIC_ENABLE_NODEJS_PLUGIN=1

.gitignore

@@ -45,5 +45,5 @@ dev
 *.key
 *.key.pub
 
-/public/uploads
+/uploads
 .vercel

README.md

@@ -105,6 +105,11 @@
     - 配置密钥 `GOOGLE_API_KEY` ，key 可以在这里获取：https://ai.google.dev/tutorials/setup
     - 配置自定义接口地址（可选） `GOOGLE_BASE_URL`，可以使用我的这个项目搭建一个基于 vercel 的代理服务：[google-gemini-vercel-proxy](https://github.com/Hk-Gosuto/google-gemini-vercel-proxy)
   - 常见问题参考：[Gemini Prompting FAQs](https://js.langchain.com/docs/integrations/chat/google_generativeai#gemini-prompting-faqs)
+
+- 非 Vercel 运行环境下支持本地存储
+
+  - 如果你的程序运行在非 Vercel 环境，不配置 `S3_ENDPOINT` 和 `R2_ACCOUNT_ID` 参数，默认上传的文件将存储在 `/app/uploads` 文件夹中
+
 
 ## 开发计划
 
@@ -113,25 +118,16 @@
   不配置时默认使用 `DuckDuckGo` 作为搜索插件。
 
 - [x] 插件列表页面开发
+
 - [x] 支持开关指定插件
-- [ ] 支持添加自定义插件
+
 - [x] 支持 Agent 参数配置（ ~~agentType~~, maxIterations, returnIntermediateSteps 等）
+
 - [x] 支持 ChatSession 级别插件功能开关
 
   仅在使用非 `0301` 和 `0314` 版本模型时会出现插件开关，其它模型默认为关闭状态，开关也不会显示。
-
-## 已知问题
-- [x] ~~使用插件时需将模型切换为 `0613` 版本模型，如：`gpt-3.5-turbo-0613`~~
-
-  尝试使用 `chat-conversational-react-description` 等类型的 `agent` 使用插件时效果并不理想，不再考虑支持其它版本的模型。
-
-  限制修改为非 `0301` 和 `0314` 模型均可调用插件。 [#10](https://github.com/Hk-Gosuto/ChatGPT-Next-Web-LangChain/issues/10)
-- [x] `SERPAPI_API_KEY` 目前为必填，后续会支持使用 DuckDuckGo 替换搜索插件
-- [x] Agent 不支持自定义接口地址
-- [x] ~~部分场景下插件会调用失败~~
-
-  问题出现在使用 [Calculator](https://js.langchain.com/docs/api/tools_calculator/classes/Calculator) 进行计算时的参数错误，暂时无法干预。
-- [x] 插件调用失败后无反馈
+
+- [ ] 支持添加自定义插件
 
 ## 最新动态
 

app/api/file/[...path]/route.ts

@@ -1,5 +1,7 @@
+import { getServerSideConfig } from "@/app/config/server";
+import LocalFileStorage from "@/app/utils/local_file_storage";
+import S3FileStorage from "@/app/utils/s3_file_storage";
 import { NextRequest, NextResponse } from "next/server";
-import S3FileStorage from "../../../utils/s3_file_storage";
 
 async function handle(
   req: NextRequest,
@@ -10,12 +12,22 @@ async function handle(
   }
 
   try {
-    var file = await S3FileStorage.get(params.path[0]);
-    return new Response(file?.transformToWebStream(), {
-      headers: {
-        "Content-Type": "image/png",
-      },
-    });
+    const serverConfig = getServerSideConfig();
+    if (serverConfig.isStoreFileToLocal) {
+      var fileBuffer = await LocalFileStorage.get(params.path[0]);
+      return new Response(fileBuffer, {
+        headers: {
+          "Content-Type": "image/png",
+        },
+      });
+    } else {
+      var file = await S3FileStorage.get(params.path[0]);
+      return new Response(file?.transformToWebStream(), {
+        headers: {
+          "Content-Type": "image/png",
+        },
+      });
+    }
   } catch (e) {
     return new Response("not found", {
       status: 404,
@@ -25,5 +37,5 @@ async function handle(
 
 export const GET = handle;
 
-export const runtime = "edge";
+export const runtime = "nodejs";
 export const revalidate = 0;

app/api/file/upload/route.ts

@@ -1,7 +1,9 @@
 import { NextRequest, NextResponse } from "next/server";
-import { auth } from "../../auth";
-import S3FileStorage from "../../../utils/s3_file_storage";
 import { ModelProvider } from "@/app/constant";
+import { auth } from "@/app/api/auth";
+import LocalFileStorage from "@/app/utils/local_file_storage";
+import { getServerSideConfig } from "@/app/config/server";
+import S3FileStorage from "@/app/utils/s3_file_storage";
 
 async function handle(req: NextRequest) {
   if (req.method === "OPTIONS") {
@@ -31,10 +33,17 @@ async function handle(req: NextRequest) {
     const buffer = Buffer.from(imageData);
 
     var fileName = `${Date.now()}.png`;
-    await S3FileStorage.put(fileName, buffer);
+    var filePath = "";
+    const serverConfig = getServerSideConfig();
+    if (serverConfig.isStoreFileToLocal) {
+      filePath = await LocalFileStorage.put(fileName, buffer);
+    } else {
+      filePath = await S3FileStorage.put(fileName, buffer);
+    }
     return NextResponse.json(
       {
         fileName: fileName,
+        filePath: filePath,
       },
       {
         status: 200,
@@ -55,4 +64,4 @@ async function handle(req: NextRequest) {
 
 export const POST = handle;
 
-export const runtime = "edge";
+export const runtime = "nodejs";

app/api/langchain-tools/dalle_image_generator_node.ts

@@ -0,0 +1,22 @@
+import { getServerSideConfig } from "@/app/config/server";
+import { DallEAPIWrapper } from "./dalle_image_generator";
+import S3FileStorage from "@/app/utils/s3_file_storage";
+import LocalFileStorage from "@/app/utils/local_file_storage";
+
+export class DallEAPINodeWrapper extends DallEAPIWrapper {
+  async saveImageFromUrl(url: string) {
+    const response = await fetch(url);
+    const content = await response.arrayBuffer();
+    const buffer = Buffer.from(content);
+
+    var filePath = "";
+    const serverConfig = getServerSideConfig();
+    var fileName = `${Date.now()}.png`;
+    if (serverConfig.isStoreFileToLocal) {
+      filePath = await LocalFileStorage.put(fileName, buffer);
+    } else {
+      filePath = await S3FileStorage.put(fileName, buffer);
+    }
+    return filePath;
+  }
+}

app/api/langchain-tools/edge_tools.ts

@@ -4,13 +4,8 @@ import { StableDiffusionWrapper } from "@/app/api/langchain-tools/stable_diffusi
 import { BaseLanguageModel } from "langchain/dist/base_language";
 import { Calculator } from "langchain/tools/calculator";
 import { WebBrowser } from "langchain/tools/webbrowser";
-import { BaiduSearch } from "@/app/api/langchain-tools/baidu_search";
-import { DuckDuckGo } from "@/app/api/langchain-tools/duckduckgo_search";
-import { GoogleSearch } from "@/app/api/langchain-tools/google_search";
-import { Tool, DynamicTool } from "langchain/tools";
-import * as langchainTools from "langchain/tools";
 import { Embeddings } from "langchain/dist/embeddings/base.js";
-import { WolframAlphaTool } from "./wolframalpha";
+import { WolframAlphaTool } from "@/app/api/langchain-tools/wolframalpha";
 
 export class EdgeTool {
   private apiKey: string | undefined;
@@ -52,7 +47,6 @@ export class EdgeTool {
     const arxivAPITool = new ArxivAPIWrapper();
     const wolframAlphaTool = new WolframAlphaTool();
     let tools = [
-      // searchTool,
       calculatorTool,
       webBrowserTool,
       dallEAPITool,

app/api/langchain-tools/nodejs_tools.ts

@@ -1,7 +1,12 @@
 import { BaseLanguageModel } from "langchain/dist/base_language";
 import { PDFBrowser } from "@/app/api/langchain-tools/pdf_browser";
-
 import { Embeddings } from "langchain/dist/embeddings/base.js";
+import { ArxivAPIWrapper } from "@/app/api/langchain-tools/arxiv";
+import { DallEAPINodeWrapper } from "@/app/api/langchain-tools/dalle_image_generator_node";
+import { StableDiffusionNodeWrapper } from "@/app/api/langchain-tools/stable_diffusion_image_generator_node";
+import { Calculator } from "langchain/tools/calculator";
+import { WebBrowser } from "langchain/tools/webbrowser";
+import { WolframAlphaTool } from "@/app/api/langchain-tools/wolframalpha";
 
 export class NodeJSTool {
   private apiKey: string | undefined;
@@ -29,7 +34,29 @@ export class NodeJSTool {
   }
 
   async getCustomTools(): Promise<any[]> {
+    const webBrowserTool = new WebBrowser({
+      model: this.model,
+      embeddings: this.embeddings,
+    });
+    const calculatorTool = new Calculator();
+    const dallEAPITool = new DallEAPINodeWrapper(
+      this.apiKey,
+      this.baseUrl,
+      this.callback,
+    );
+    const stableDiffusionTool = new StableDiffusionNodeWrapper();
+    const arxivAPITool = new ArxivAPIWrapper();
+    const wolframAlphaTool = new WolframAlphaTool();
     const pdfBrowserTool = new PDFBrowser(this.model, this.embeddings);
-    return [pdfBrowserTool];
+    let tools = [
+      calculatorTool,
+      webBrowserTool,
+      dallEAPITool,
+      stableDiffusionTool,
+      arxivAPITool,
+      wolframAlphaTool,
+      pdfBrowserTool,
+    ];
+    return tools;
   }
 }

app/api/langchain-tools/stable_diffusion_image_generator.ts

@@ -8,6 +8,13 @@ export class StableDiffusionWrapper extends Tool {
     super();
   }
 
+  async saveImage(imageBase64: string) {
+    const buffer = Buffer.from(imageBase64, "base64");
+    var fileName = `${Date.now()}.png`;
+    const filePath = await S3FileStorage.put(fileName, buffer);
+    return filePath;
+  }
+
   /** @ignore */
   async _call(prompt: string) {
     let url = process.env.STABLE_DIFFUSION_API_URL;
@@ -40,8 +47,7 @@ export class StableDiffusionWrapper extends Tool {
     const json = await response.json();
     let imageBase64 = json.images[0];
     if (!imageBase64) return "No image was generated";
-    const buffer = Buffer.from(imageBase64, "base64");
-    const filePath = await S3FileStorage.put(`${Date.now()}.png`, buffer);
+    const filePath = await this.saveImage(imageBase64);
     console.log(`[${this.name}]`, filePath);
     return filePath;
   }

app/api/langchain-tools/stable_diffusion_image_generator_node.ts

@@ -0,0 +1,19 @@
+import S3FileStorage from "@/app/utils/s3_file_storage";
+import { StableDiffusionWrapper } from "./stable_diffusion_image_generator";
+import { getServerSideConfig } from "@/app/config/server";
+import LocalFileStorage from "@/app/utils/local_file_storage";
+
+export class StableDiffusionNodeWrapper extends StableDiffusionWrapper {
+  async saveImage(imageBase64: string) {
+    var filePath = "";
+    var fileName = `${Date.now()}.png`;
+    const buffer = Buffer.from(imageBase64, "base64");
+    const serverConfig = getServerSideConfig();
+    if (serverConfig.isStoreFileToLocal) {
+      filePath = await LocalFileStorage.put(fileName, buffer);
+    } else {
+      filePath = await S3FileStorage.put(fileName, buffer);
+    }
+    return filePath;
+  }
+}

app/api/langchain/tool/agent/nodejs/route.ts

@@ -55,23 +55,15 @@ async function handle(req: NextRequest) {
       );
     };
 
-    var edgeTool = new EdgeTool(
-      apiKey,
-      baseUrl,
-      model,
-      embeddings,
-      dalleCallback,
-    );
     var nodejsTool = new NodeJSTool(
       apiKey,
       baseUrl,
       model,
       embeddings,
       dalleCallback,
     );
-    var edgeTools = await edgeTool.getCustomTools();
     var nodejsTools = await nodejsTool.getCustomTools();
-    var tools = [...edgeTools, ...nodejsTools];
+    var tools = [...nodejsTools];
     return await agentApi.getApiHandler(req, reqBody, tools);
   } catch (e) {
     return new Response(JSON.stringify({ error: (e as any).message }), {

app/client/platforms/utils.ts

@@ -1,11 +1,12 @@
 import { getHeaders } from "../api";
 
 export class FileApi {
-  async upload(file: any): Promise<void> {
+  async upload(file: any): Promise<any> {
     const formData = new FormData();
     formData.append("file", file);
     var headers = getHeaders(true);
-    var res = await fetch("/api/file/upload", {
+    const api = "/api/file/upload";
+    var res = await fetch(api, {
       method: "POST",
       body: formData,
       headers: {
@@ -14,6 +15,9 @@ export class FileApi {
     });
     const resJson = await res.json();
     console.log(resJson);
-    return resJson.fileName;
+    return {
+      fileName: resJson.fileName,
+      filePath: resJson.filePath,
+    };
   }
 }

app/components/chat.tsx

@@ -465,10 +465,10 @@ export function ChatActions(props: {
   const onImageSelected = async (e: any) => {
     const file = e.target.files[0];
     const api = new ClientApi();
-    const fileName = await api.file.upload(file);
+    const uploadFile = await api.file.upload(file);
     props.imageSelected({
-      fileName,
-      fileUrl: `/api/file/${fileName}`,
+      fileName: uploadFile.fileName,
+      fileUrl: uploadFile.filePath,
     });
     e.target.value = null;
   };

app/config/server.ts

@@ -101,5 +101,10 @@ export const getServerSideConfig = () => {
     hideBalanceQuery: !process.env.ENABLE_BALANCE_QUERY,
     disableFastLink: !!process.env.DISABLE_FAST_LINK,
     customModels,
+
+    isStoreFileToLocal:
+      !!process.env.NEXT_PUBLIC_ENABLE_NODEJS_PLUGIN &&
+      !process.env.R2_ACCOUNT_ID &&
+      !process.env.S3_ENDPOINT,
   };
 };

app/utils/local_file_storage.ts

@@ -0,0 +1,29 @@
+import fs from "fs";
+import path from "path";
+
+export default class LocalFileStorage {
+  static async get(fileName: string) {
+    const filePath = path.resolve(`./uploads`, fileName);
+    const file = fs.readFileSync(filePath);
+    if (!file) {
+      throw new Error("not found.");
+    }
+    return file;
+  }
+
+  static async put(fileName: string, data: Buffer) {
+    try {
+      const filePath = path.resolve(`./uploads`, fileName);
+      const dir = path.dirname(filePath);
+      if (!fs.existsSync(dir)) {
+        fs.mkdirSync(dir, { recursive: true });
+      }
+      await fs.promises.writeFile(filePath, data);
+      console.log("[LocalFileStorage]", filePath);
+      return `/api/file/${fileName}`;
+    } catch (e) {
+      console.error("[LocalFileStorage]", e);
+      throw e;
+    }
+  }
+}

app/utils/s3_file_storage.ts

@@ -55,7 +55,7 @@ export default class S3FileStorage {
       { expiresIn: 60 },
     );
 
-    console.log(signedUrl);
+    console.log("[S3]", signedUrl);
 
     try {
       await fetch(signedUrl, {