Skip to content

Commit

Permalink
Tool call support interactive node (labring#2903)
Browse files Browse the repository at this point in the history 
* feat: tool call support interactive node

* feat: interactive node tool response

* fix: tool call concat

* fix: llm history concat
  • Loading branch information
@c121914yu
c121914yu authored Oct 14, 2024
1 parent 2a2b919 commit 4f1ce64
Show file tree
Hide file tree
Showing 29 changed files with 831 additions and 347 deletions.
4 changes: 3 additions & 1 deletion docSite/content/zh-cn/docs/development/upgrading/4812.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,6 @@ weight: 812

1. 新增 - 全局变量支持更多数据类型
2. 新增 - FE_DOMAIN 环境变量，配置该环境变量后，上传文件/图片会补全后缀后得到完整地址。（可解决 docx 文件图片链接，有时会无法被模型识别问题）
3. 修复 - 文件后缀判断，去除 query 影响。
3. 新增 - 工具调用支持交互模式
4. 修复 - 文件后缀判断，去除 query 影响。
5. 修复 - AI 响应为空时，会造成 LLM 历史记录合并。
8 changes: 4 additions & 4 deletions packages/global/common/string/markdown.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,10 +70,10 @@ export const uploadMarkdownBase64 = async ({
}

// Remove white space on both sides of the picture
const trimReg = /(!\[.*\]\(.*\))\s*/g;
if (trimReg.test(rawText)) {
rawText = rawText.replace(trimReg, '$1');
}
// const trimReg = /(!\[.*\]\(.*\))\s*/g;
// if (trimReg.test(rawText)) {
// rawText = rawText.replace(trimReg, '$1');
// }

return rawText;
};
Expand Down
40 changes: 28 additions & 12 deletions packages/global/core/ai/type.d.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,14 @@ import type {
ChatCompletionChunk,
ChatCompletionMessageParam as SdkChatCompletionMessageParam,
ChatCompletionToolMessageParam,
ChatCompletionAssistantMessageParam,
ChatCompletionContentPart as SdkChatCompletionContentPart,
ChatCompletionUserMessageParam as SdkChatCompletionUserMessageParam
ChatCompletionUserMessageParam as SdkChatCompletionUserMessageParam,
ChatCompletionToolMessageParam as SdkChatCompletionToolMessageParam,
ChatCompletionAssistantMessageParam as SdkChatCompletionAssistantMessageParam,
ChatCompletionContentPartText
} from 'openai/resources';
import { ChatMessageTypeEnum } from './constants';
import { InteractiveNodeResponseItemType } from '../workflow/template/system/interactive/type';
import { WorkflowInteractiveResponseType } from '../workflow/template/system/interactive/type';
export * from 'openai/resources';

// Extension of ChatCompletionMessageParam, Add file url type
Expand All @@ -22,18 +24,31 @@ export type ChatCompletionContentPartFile = {
export type ChatCompletionContentPart =
| SdkChatCompletionContentPart
| ChatCompletionContentPartFile;
type CustomChatCompletionUserMessageParam = {
content: string | Array<ChatCompletionContentPart>;
type CustomChatCompletionUserMessageParam = Omit<ChatCompletionUserMessageParam, 'content'> & {
role: 'user';
content: string | Array<ChatCompletionContentPart>;
};
type CustomChatCompletionToolMessageParam = SdkChatCompletionToolMessageParam & {
role: 'tool';
name?: string;
};
type CustomChatCompletionAssistantMessageParam = SdkChatCompletionAssistantMessageParam & {
role: 'assistant';
interactive?: WorkflowInteractiveResponseType;
};

export type ChatCompletionMessageParam = (
| Exclude<SdkChatCompletionMessageParam, SdkChatCompletionUserMessageParam>
| Exclude<
SdkChatCompletionMessageParam,
| SdkChatCompletionUserMessageParam
| SdkChatCompletionToolMessageParam
| SdkChatCompletionAssistantMessageParam
>
| CustomChatCompletionUserMessageParam
| CustomChatCompletionToolMessageParam
| CustomChatCompletionAssistantMessageParam
) & {
dataId?: string;
interactive?: InteractiveNodeResponseItemType;
};
export type SdkChatCompletionMessageParam = SdkChatCompletionMessageParam;

Expand All @@ -47,11 +62,12 @@ export type ChatCompletionMessageToolCall = ChatCompletionMessageToolCall & {
toolName?: string;
toolAvatar?: string;
};
export type ChatCompletionMessageFunctionCall = ChatCompletionAssistantMessageParam.FunctionCall & {
id?: string;
toolName?: string;
toolAvatar?: string;
};
export type ChatCompletionMessageFunctionCall =
SdkChatCompletionAssistantMessageParam.FunctionCall & {
id?: string;
toolName?: string;
toolAvatar?: string;
};

// Stream response
export type StreamChatType = Stream<ChatCompletionChunk>;
Expand Down
75 changes: 53 additions & 22 deletions packages/global/core/chat/adapt.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,8 +90,9 @@ export const chats2GPTMessages = ({
});
}
} else {
const aiResults: ChatCompletionMessageParam[] = [];
//AI
item.value.forEach((value) => {
item.value.forEach((value, i) => {
if (value.type === ChatItemValueTypeEnum.tool && value.tools && reserveTool) {
const tool_calls: ChatCompletionMessageToolCall[] = [];
const toolResponse: ChatCompletionToolMessageParam[] = [];
Expand All @@ -111,28 +112,53 @@ export const chats2GPTMessages = ({
content: tool.response
});
});
results = results
.concat({
dataId,
role: ChatCompletionRequestMessageRoleEnum.Assistant,
tool_calls
})
.concat(toolResponse);
} else if (value.text?.content) {
results.push({
aiResults.push({
dataId,
role: ChatCompletionRequestMessageRoleEnum.Assistant,
content: value.text.content
tool_calls
});
aiResults.push(...toolResponse);
} else if (
value.type === ChatItemValueTypeEnum.text &&
typeof value.text?.content === 'string'
) {
// Concat text
const lastValue = item.value[i - 1];
const lastResult = aiResults[aiResults.length - 1];
if (
lastValue &&
lastValue.type === ChatItemValueTypeEnum.text &&
typeof lastResult.content === 'string'
) {
lastResult.content += value.text.content;
} else {
aiResults.push({
dataId,
role: ChatCompletionRequestMessageRoleEnum.Assistant,
content: value.text.content
});
}
} else if (value.type === ChatItemValueTypeEnum.interactive) {
results = results.concat({
aiResults.push({
dataId,
role: ChatCompletionRequestMessageRoleEnum.Assistant,
interactive: value.interactive,
content: ''
interactive: value.interactive
});
}
});

// Auto add empty assistant message
results = results.concat(
aiResults.length > 0
? aiResults
: [
{
dataId,
role: ChatCompletionRequestMessageRoleEnum.Assistant,
content: ''
}
]
);
}
});

Expand Down Expand Up @@ -215,14 +241,7 @@ export const GPTMessages2Chats = (
obj === ChatRoleEnum.AI &&
item.role === ChatCompletionRequestMessageRoleEnum.Assistant
) {
if (item.content && typeof item.content === 'string') {
value.push({
type: ChatItemValueTypeEnum.text,
text: {
content: item.content
}
});
} else if (item.tool_calls && reserveTool) {
if (item.tool_calls && reserveTool) {
// save tool calls
const toolCalls = item.tool_calls as ChatCompletionMessageToolCall[];
value.push({
Expand Down Expand Up @@ -278,6 +297,18 @@ export const GPTMessages2Chats = (
type: ChatItemValueTypeEnum.interactive,
interactive: item.interactive
});
} else if (typeof item.content === 'string') {
const lastValue = value[value.length - 1];
if (lastValue && lastValue.type === ChatItemValueTypeEnum.text && lastValue.text) {
lastValue.text.content += item.content;
} else {
value.push({
type: ChatItemValueTypeEnum.text,
text: {
content: item.content
}
});
}
}
}

Expand Down
4 changes: 2 additions & 2 deletions packages/global/core/chat/type.d.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import type { AppSchema as AppType } from '@fastgpt/global/core/app/type.d';
import { DatasetSearchModeEnum } from '../dataset/constants';
import { DispatchNodeResponseType } from '../workflow/runtime/type.d';
import { ChatBoxInputType } from '../../../../projects/app/src/components/core/chat/ChatContainer/ChatBox/type';
import { InteractiveNodeResponseItemType } from '../workflow/template/system/interactive/type';
import { WorkflowInteractiveResponseType } from '../workflow/template/system/interactive/type';

export type ChatSchema = {
_id: string;
Expand Down Expand Up @@ -73,7 +73,7 @@ export type AIChatItemValueItemType = {
content: string;
};
tools?: ToolModuleResponseItemType[];
interactive?: InteractiveNodeResponseItemType;
interactive?: WorkflowInteractiveResponseType;
};
export type AIChatItemType = {
obj: ChatRoleEnum.AI;
Expand Down
26 changes: 26 additions & 0 deletions packages/global/core/chat/utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,3 +143,29 @@ export const getChatSourceByPublishChannel = (publishChannel: PublishChannelEnum
return ChatSourceEnum.online;
}
};

/*
Merge chat responseData
1. Same tool mergeSignId (Interactive tool node)
*/
export const mergeChatResponseData = (responseDataList: ChatHistoryItemResType[]) => {
let lastResponse: ChatHistoryItemResType | undefined = undefined;

return responseDataList.reduce<ChatHistoryItemResType[]>((acc, curr) => {
if (lastResponse && lastResponse.mergeSignId && curr.mergeSignId === lastResponse.mergeSignId) {
// 替换 lastResponse
const concatResponse: ChatHistoryItemResType = {
...curr,
runningTime: +((lastResponse.runningTime || 0) + (curr.runningTime || 0)).toFixed(2),
totalPoints: (lastResponse.totalPoints || 0) + (curr.totalPoints || 0),
childTotalPoints: (lastResponse.childTotalPoints || 0) + (curr.childTotalPoints || 0),
toolCallTokens: (lastResponse.toolCallTokens || 0) + (curr.toolCallTokens || 0),
toolDetail: [...(lastResponse.toolDetail || []), ...(curr.toolDetail || [])]
};
return [...acc.slice(0, -1), concatResponse];
} else {
lastResponse = curr;
return [...acc, curr];
}
}, []);
};
4 changes: 3 additions & 1 deletion packages/global/core/workflow/runtime/type.d.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ export type RuntimeNodeItemType = {
intro?: StoreNodeItemType['intro'];
flowNodeType: StoreNodeItemType['flowNodeType'];
showStatus?: StoreNodeItemType['showStatus'];
isEntry?: StoreNodeItemType['isEntry'];
isEntry?: boolean;

inputs: FlowNodeInputItemType[];
outputs: FlowNodeOutputItemType[];
Expand Down Expand Up @@ -108,12 +108,14 @@ export type DispatchNodeResponseType = {
customOutputs?: Record<string, any>;
nodeInputs?: Record<string, any>;
nodeOutputs?: Record<string, any>;
mergeSignId?: string;

// bill
tokens?: number;
model?: string;
contextTotalLen?: number;
totalPoints?: number;
childTotalPoints?: number;

// chat
temperature?: number;
Expand Down
4 changes: 2 additions & 2 deletions packages/global/core/workflow/runtime/utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ export const initWorkflowEdgeStatus = (
histories?: ChatItemType[]
): RuntimeEdgeItemType[] => {
// If there is a history, use the last interactive value
if (!!histories) {
if (histories && histories.length > 0) {
const memoryEdges = getLastInteractiveValue(histories)?.memoryEdges;

if (memoryEdges && memoryEdges.length > 0) {
Expand All @@ -90,7 +90,7 @@ export const getWorkflowEntryNodeIds = (
histories?: ChatItemType[]
) => {
// If there is a history, use the last interactive entry node
if (!!histories) {
if (histories && histories.length > 0) {
const entryNodeIds = getLastInteractiveValue(histories)?.entryNodeIds;

if (Array.isArray(entryNodeIds) && entryNodeIds.length > 0) {
Expand Down
2 changes: 1 addition & 1 deletion packages/global/core/workflow/template/system/interactive/formInput.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ export const FormInputNode: FlowNodeTemplateType = {
avatar: 'core/workflow/template/formInput',
name: i18nT('app:workflow.form_input'),
intro: i18nT(`app:workflow.form_input_tip`),
showStatus: true,
isTool: true,
version: '4811',
inputs: [
{
Expand Down
17 changes: 12 additions & 5 deletions packages/global/core/workflow/template/system/interactive/type.d.ts
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
import { NodeOutputItemType } from '../../../../chat/type';
import { FlowNodeOutputItemType } from '../../../type/io';
import { RuntimeEdgeItemType } from '../../../runtime/type';
import type { NodeOutputItemType } from '../../../../chat/type';
import type { FlowNodeOutputItemType } from '../../../type/io';
import type { RuntimeEdgeItemType } from '../../../runtime/type';
import { FlowNodeInputTypeEnum } from 'core/workflow/node/constant';
import { WorkflowIOValueTypeEnum } from 'core/workflow/constants';
import type { ChatCompletionMessageParam } from '../../../../ai/type';

export type UserSelectOptionItemType = {
key: string;
Expand Down Expand Up @@ -32,6 +33,12 @@ type InteractiveBasicType = {
entryNodeIds: string[];
memoryEdges: RuntimeEdgeItemType[];
nodeOutputs: NodeOutputItemType[];

toolParams?: {
entryNodeIds: string[]; // 记录工具中，交互节点的 Id，而不是起始工作流的入口
memoryMessages: ChatCompletionMessageParam[]; // 这轮工具中，产生的新的 messages
toolCallId: string; // 记录对应 tool 的id，用于后续交互节点可以替换掉 tool 的 response
};
};

type UserSelectInteractive = {
Expand All @@ -52,5 +59,5 @@ type UserInputInteractive = {
};
};

export type InteractiveNodeResponseItemType = InteractiveBasicType &
(UserSelectInteractive | UserInputInteractive);
export type InteractiveNodeResponseType = UserSelectInteractive | UserInputInteractive;
export type WorkflowInteractiveResponseType = InteractiveBasicType & InteractiveNodeResponseType;
2 changes: 1 addition & 1 deletion packages/global/core/workflow/template/system/interactive/userSelect.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ export const UserSelectNode: FlowNodeTemplateType = {
diagram: '/imgs/app/userSelect.svg',
name: i18nT('app:workflow.user_select'),
intro: i18nT(`app:workflow.user_select_tip`),
showStatus: true,
isTool: true,
version: '489',
inputs: [
{
Expand Down
Loading

0 comments on commit 4f1ce64

Please sign in to comment.