From f5172e07470f534d2584e9dbc6d49d269116dd08 Mon Sep 17 00:00:00 2001 From: Ali Salimli <67149699+elisalimli@users.noreply.github.com> Date: Thu, 7 Mar 2024 21:08:39 +0400 Subject: [PATCH 1/2] Just update tool when description is changed (#886) * fix: change semantic to by_title needs to be fixed in superrag, we will use by_title until it gets fixed * feat: just update the tool when description is changed --- .../api/api_datasource_superrag_manager.py | 28 +++++++++++++++++++ .../api/workflow_configs/data_transformer.py | 6 ++-- .../workflow_configs/processors/superagent.py | 11 ++++++-- 3 files changed, 40 insertions(+), 5 deletions(-) diff --git a/libs/superagent/app/api/workflow_configs/api/api_datasource_superrag_manager.py b/libs/superagent/app/api/workflow_configs/api/api_datasource_superrag_manager.py index 5b13427bd..842140cfa 100644 --- a/libs/superagent/app/api/workflow_configs/api/api_datasource_superrag_manager.py +++ b/libs/superagent/app/api/workflow_configs/api/api_datasource_superrag_manager.py @@ -13,6 +13,9 @@ from app.api.tools import ( delete as api_delete_tool, ) +from app.api.tools import ( + update as api_update_tool, +) from app.api.workflow_configs.api.base import ( BaseApiAgentManager, BaseApiDatasourceManager, @@ -23,6 +26,9 @@ from app.models.request import ( Tool as ToolRequest, ) +from app.models.request import ( + ToolUpdate as ToolUpdateRequest, +) from prisma.enums import ToolType from services.superrag import SuperRagService @@ -76,6 +82,21 @@ async def _add_tool(self, assistant: dict, data: dict): except Exception as err: logger.error(f"Error adding tool: {new_tool} - {assistant} - Error: {err}") + async def _update_tool(self, assistant: dict, data: dict): + tool = await self.agent_manager.get_tool(assistant, data) + + try: + await api_update_tool( + tool_id=tool.id, + body=ToolUpdateRequest.parse_obj(data), + api_user=self.api_user, + ) + logger.info(f"Updated tool: {tool.name} - {assistant.get('name')}") + except Exception as err: + logger.error( + f"Error updating tool: {tool} - {data} - {assistant} - Error: {err}" + ) + async def _add_superrag_tool(self, assistant: dict, data: dict): new_tool = { **data, @@ -111,6 +132,13 @@ async def _get_unique_index_name(self, datasource: dict, assistant: dict): return unique_name + async def update_datasource(self, assistant: dict, data: dict): + """ + This method only updates the superrag tool, not the datasource in SuperRag. + To achive that, first delete the datasource and then add it again. + """ + await self._update_tool(assistant, data) + async def add_datasource(self, assistant: dict, data: dict): data["index_name"] = await self._get_unique_index_name(data, assistant) diff --git a/libs/superagent/app/api/workflow_configs/data_transformer.py b/libs/superagent/app/api/workflow_configs/data_transformer.py index 2f437660b..b488a2777 100644 --- a/libs/superagent/app/api/workflow_configs/data_transformer.py +++ b/libs/superagent/app/api/workflow_configs/data_transformer.py @@ -148,7 +148,7 @@ async def transform_superrags(self): "splitter": { "max_tokens": 400, "min_tokens": 30, - "name": "semantic", + "name": "by_title", "prefix_summary": True, "prefix_title": True, "rolling_window_size": 1, @@ -177,8 +177,8 @@ async def _set_database_provider(self, datasource: dict): } else: raise MissingVectorDatabaseProvider( - f"Vector database provider not found ({database_provider})." - f"Please configure it by going to the integrations page" + "Vector database provider not found." + "Please configure it by going to the integrations page" ) remove_key_if_present(datasource, "database_provider") diff --git a/libs/superagent/app/api/workflow_configs/processors/superagent.py b/libs/superagent/app/api/workflow_configs/processors/superagent.py index c7aa24583..11de6592a 100644 --- a/libs/superagent/app/api/workflow_configs/processors/superagent.py +++ b/libs/superagent/app/api/workflow_configs/processors/superagent.py @@ -40,8 +40,15 @@ async def process(self, old_data, new_data): if old_datasource_name and new_datasource_name: is_changed = compare_dicts(old_datasource, new_datasource) - - if is_changed: + if ( + is_changed.get("description") is not None + and len(is_changed.items()) == 1 + ): + await datasource_manager.update_datasource( + self.assistant, + new_datasource, + ) + else: await datasource_manager.delete_datasource( self.assistant, old_datasource, From b2ded8b0e103237b4fdf1238e56050942d1e9848 Mon Sep 17 00:00:00 2001 From: Ali Salimli <67149699+elisalimli@users.noreply.github.com> Date: Thu, 7 Mar 2024 21:09:09 +0400 Subject: [PATCH 2/2] Fixing Superrag (#887) * fix: small typo * fix: migrate superrag to the new api * chore: add field descriptions * chore: remove print statements --- .../api/api_datasource_superrag_manager.py | 1 - .../app/api/workflow_configs/data_transformer.py | 5 ++++- .../api/workflow_configs/processors/agent_processor.py | 2 +- libs/superagent/app/api/workflow_configs/saml_schema.py | 8 ++++++-- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/libs/superagent/app/api/workflow_configs/api/api_datasource_superrag_manager.py b/libs/superagent/app/api/workflow_configs/api/api_datasource_superrag_manager.py index 842140cfa..d69e2b95a 100644 --- a/libs/superagent/app/api/workflow_configs/api/api_datasource_superrag_manager.py +++ b/libs/superagent/app/api/workflow_configs/api/api_datasource_superrag_manager.py @@ -67,7 +67,6 @@ async def _create_tool(self, assistant: dict, data: dict): async def _add_tool(self, assistant: dict, data: dict): new_tool = await self._create_tool(assistant, data) - assistant = await self.agent_manager.get_assistant(assistant) try: diff --git a/libs/superagent/app/api/workflow_configs/data_transformer.py b/libs/superagent/app/api/workflow_configs/data_transformer.py index b488a2777..2518cfa65 100644 --- a/libs/superagent/app/api/workflow_configs/data_transformer.py +++ b/libs/superagent/app/api/workflow_configs/data_transformer.py @@ -137,9 +137,12 @@ async def transform_superrags(self): await self._set_superrag_files(datasource) await self._set_database_provider(datasource) + encoder = datasource.get("encoder") or DEFAULT_ENCODER_OPTIONS + rename_and_remove_keys(encoder, {"type": "provider"}) + rename_and_remove_keys(encoder, {"name": "model_name"}) datasource["document_processor"] = { - "encoder": datasource.get("encoder") or DEFAULT_ENCODER_OPTIONS, + "encoder": encoder, "unstructured": { "hi_res_model_name": "detectron2_onnx", "partition_strategy": "auto", diff --git a/libs/superagent/app/api/workflow_configs/processors/agent_processor.py b/libs/superagent/app/api/workflow_configs/processors/agent_processor.py index fab158eec..4d8f48714 100644 --- a/libs/superagent/app/api/workflow_configs/processors/agent_processor.py +++ b/libs/superagent/app/api/workflow_configs/processors/agent_processor.py @@ -81,7 +81,7 @@ async def process_assistant( new_superrag_processor = Processor( self.api_user, self.api_manager - ).get_superrag_processor(old_assistant) + ).get_superrag_processor(new_assistant) if old_assistant_type and new_assistant_type: if old_assistant_type != new_assistant_type: diff --git a/libs/superagent/app/api/workflow_configs/saml_schema.py b/libs/superagent/app/api/workflow_configs/saml_schema.py index 257c5691c..de746577f 100644 --- a/libs/superagent/app/api/workflow_configs/saml_schema.py +++ b/libs/superagent/app/api/workflow_configs/saml_schema.py @@ -12,8 +12,12 @@ class SuperragEncoderType(str, Enum): class SuperragEncoder(BaseModel): - type: SuperragEncoderType - name: str + type: SuperragEncoderType = Field( + description="The provider of encoder to use for the index. e.g. `openai`" + ) + name: str = Field( + description="The model name to use for the encoder. e.g. `text-embedding-3-small` for OpenAI's model" + ) dimensions: int