diff --git a/layers/langchain-common-layer/python/genai_core/adapters/__init__.py b/layers/langchain-common-layer/python/genai_core/adapters/__init__.py deleted file mode 100644 index 267543a4..00000000 --- a/layers/langchain-common-layer/python/genai_core/adapters/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .openai import * -from .sagemaker import * -from .bedrock import * -from .base import Mode diff --git a/layers/langchain-common-layer/python/genai_core/adapters/base/__init__.py b/layers/langchain-common-layer/python/genai_core/adapters/base/__init__.py deleted file mode 100644 index b31444cd..00000000 --- a/layers/langchain-common-layer/python/genai_core/adapters/base/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .base import ModelAdapter, Mode diff --git a/layers/langchain-common-layer/python/genai_core/adapters/base/base.py b/layers/langchain-common-layer/python/genai_core/adapters/base/base.py deleted file mode 100644 index 94ac5509..00000000 --- a/layers/langchain-common-layer/python/genai_core/adapters/base/base.py +++ /dev/null @@ -1,130 +0,0 @@ -# -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance -# with the License. A copy of the License is located at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions -# and limitations under the License. -# -import os -from enum import Enum -from langchain.chains import LLMChain -from langchain.callbacks.base import BaseCallbackHandler -from langchain.chains import ConversationalRetrievalChain, ConversationChain -from langchain.memory import ConversationBufferMemory -from langchain.prompts.prompt import PromptTemplate -from langchain.chains.conversational_retrieval.prompts import ( - QA_PROMPT, - CONDENSE_QUESTION_PROMPT, -) -from genai_core.utils import PredictionException - - -class Mode(Enum): - CHAIN = "qa_chain" - - -class ModelAdapter: - def __init__(self, session_id=None, user_id=None, model_kwargs={}, adapter_kwargs={}): - self.session_id = session_id - self.user_id = user_id - self._mode = adapter_kwargs.get("mode", "qa_chain") - self.model_kwargs = model_kwargs - - self.callback_handler = BaseCallbackHandler() - self.__bind_callbacks() - - self.llm = self.get_llm(model_kwargs) - - def __bind_callbacks(self): - callback_methods = [method for method in dir(self) if method.startswith("on_")] - valid_callback_names = [ - attr for attr in dir(self.callback_handler) if attr.startswith("on_") - ] - - for method in callback_methods: - if method in valid_callback_names: - setattr(self.callback_handler, method, getattr(self, method)) - - def get_llm(self, model_kwargs={}): - raise ValueError("llm must be implemented") - - def get_embeddings_model(self, embeddings): - raise ValueError("embeddings must be implemented") - - def get_prompt_no_history_no_context(self): - template = """The following is a friendly conversation between a human and an AI. If the AI does not know the answer to a question, it truthfully says it does not know. - - Question: {input}""" - input_variables = ["input"] - prompt_template_args = { - "input_variables": input_variables, - "template": template, - } - prompt_template = PromptTemplate(**prompt_template_args) - - return prompt_template - - def get_prompt(self): - template = """The following is a friendly conversation between a human and an AI. If the AI does not know the answer to a question, it truthfully says it does not know. - - Current conversation: - {chat_history} - - Question: {input}""" - input_variables = ["input", "chat_history"] - prompt_template_args = { - "chat_history": "{chat_history}", - "input_variables": input_variables, - "template": template, - } - prompt_template = PromptTemplate(**prompt_template_args) - - return prompt_template - - def get_condense_question_prompt(self): - return CONDENSE_QUESTION_PROMPT - - def get_qa_prompt(self): - return QA_PROMPT - - def run_with_chain(self, user_prompt): - if not self.llm: - raise ValueError("llm must be set") - - chain = LLMChain(llm=self.llm, prompt=self.get_prompt_no_history_no_context(), verbose=True) - try: - response = chain.predict(input=user_prompt, callbacks=[self.callback_handler]) - raise PredictionException(self.llm._llm_type) - except PredictionException as e: - print(e.message) - #TODO return response - - metadata = { - "modelId": self.model_id, - "modelKwargs": self.model_kwargs, - "mode": self._mode, - "sessionId": self.session_id, - "userId": self.user_id, - "documents": [], - } - - return { - "sessionId": self.session_id, - "type": "text", - "content": response, - "metadata": metadata, - } - - def run(self, prompt, *args, **kwargs): - print(f"run with {kwargs}") - print(f"mode: {self._mode}") - - if self._mode == "qa_chain": - return self.run_with_chain(prompt) - - raise ValueError(f"unknown mode {self._mode}") diff --git a/layers/langchain-common-layer/python/genai_core/adapters/bedrock/__init__.py b/layers/langchain-common-layer/python/genai_core/adapters/bedrock/__init__.py deleted file mode 100644 index acc198b8..00000000 --- a/layers/langchain-common-layer/python/genai_core/adapters/bedrock/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .claude import * -from .titan import * -from .ai21_j2 import * -from .cohere import * diff --git a/layers/langchain-common-layer/python/genai_core/adapters/bedrock/ai21_j2.py b/layers/langchain-common-layer/python/genai_core/adapters/bedrock/ai21_j2.py deleted file mode 100644 index 4afc5685..00000000 --- a/layers/langchain-common-layer/python/genai_core/adapters/bedrock/ai21_j2.py +++ /dev/null @@ -1,83 +0,0 @@ -# -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance -# with the License. A copy of the License is located at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions -# and limitations under the License. -# -import boto3 -from langchain.llms import Bedrock -from langchain.prompts.prompt import PromptTemplate - -from ..base import ModelAdapter -from ..registry import registry - - -class AI21J2Adapter(ModelAdapter): - def __init__(self, model_id, *args, **kwargs): - self.model_id = model_id - - super().__init__(*args, **kwargs) - - def get_llm(self, model_kwargs={}): - bedrock = boto3.client('bedrock-runtime') - - params = {} - if "temperature" in model_kwargs: - params["temperature"] = model_kwargs["temperature"] - if "topP" in model_kwargs: - params["topP"] = model_kwargs["topP"] - if "maxTokens" in model_kwargs: - params["maxTokens"] = model_kwargs["maxTokens"] - - return Bedrock( - client=bedrock, - model_id=self.model_id, - model_kwargs=params, - callbacks=[self.callback_handler], - ) - - def get_prompt(self): - template = """Human: The following is a friendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know. - -Current conversation: -{chat_history} - -Question: {input} - -Assistant:""" - - input_variables = ["input", "chat_history"] - prompt_template_args = { - "chat_history": "{chat_history}", - "input_variables": input_variables, - "template": template, - } - prompt_template = PromptTemplate(**prompt_template_args) - - return prompt_template - - def get_prompt_no_history_no_context(self): - template = """Human: The following is a friendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know. - - Question: {input} - - Assistant:""" - - input_variables = ["input"] - prompt_template_args = { - "input_variables": input_variables, - "template": template, - } - prompt_template = PromptTemplate(**prompt_template_args) - - return prompt_template - - -# Register the adapter -registry.register(r"^bedrock.ai21.j2*", AI21J2Adapter) diff --git a/layers/langchain-common-layer/python/genai_core/adapters/bedrock/claude.py b/layers/langchain-common-layer/python/genai_core/adapters/bedrock/claude.py deleted file mode 100644 index 8c890afa..00000000 --- a/layers/langchain-common-layer/python/genai_core/adapters/bedrock/claude.py +++ /dev/null @@ -1,119 +0,0 @@ -# -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance -# with the License. A copy of the License is located at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions -# and limitations under the License. -# -import boto3 - -from langchain.llms import Bedrock -from langchain.prompts.prompt import PromptTemplate - -from ..base import ModelAdapter -from ..registry import registry - - -class BedrockClaudeAdapter(ModelAdapter): - def __init__(self, model_id, *args, **kwargs): - self.model_id = model_id - - super().__init__(*args, **kwargs) - - def get_llm(self, model_kwargs={}): - bedrock = boto3.client('bedrock-runtime') - - params = {} - if "temperature" in model_kwargs: - params["temperature"] = model_kwargs["temperature"] - if "topP" in model_kwargs: - params["top_p"] = model_kwargs["topP"] - if "maxTokens" in model_kwargs: - params["max_tokens_to_sample"] = model_kwargs["maxTokens"] - - return Bedrock( - client=bedrock, - model_id=self.model_id, - model_kwargs=params, - streaming=model_kwargs.get("streaming", False), - callbacks=[self.callback_handler], - ) - - def get_qa_prompt(self): - template = """ - -Human: Use the following pieces of context to answer the question at the end. If you don't know the answer, just say that you don't know, don't try to make up an answer. - -{context} - -Question: {question} - -Assistant:""" - - return PromptTemplate( - template=template, input_variables=["context", "question"] - ) - - def get_prompt(self): - template = """ - -Human: The following is a friendly conversation between a human and an AI. If the AI does not know the answer to a question, it truthfully says it does not know. - -Current conversation: -{chat_history} - -Question: {input} - -Assistant:""" - - input_variables = ["input", "chat_history"] - prompt_template_args = { - "chat_history": "{chat_history}", - "input_variables": input_variables, - "template": template, - } - prompt_template = PromptTemplate(**prompt_template_args) - - return prompt_template - - def get_prompt_no_history_no_context(self): - template = """ - -Human: The following is a friendly conversation between a human and an AI. If the AI does not know the answer to a question, it truthfully says it does not know. - -Question: {input} - -Assistant:""" - - input_variables = ["input"] - prompt_template_args = { - "input_variables": input_variables, - "template": template, - } - prompt_template = PromptTemplate(**prompt_template_args) - - return prompt_template - - def get_condense_question_prompt(self): - template = """ -{chat_history} - -Human: Given the following conversation and a follow up question, rephrase the follow up question to be a standalone question, in its original language. -Follow Up Input: {question} - -Assistant:""" - - return PromptTemplate( - input_variables=["chat_history", "question"], - chat_history="{chat_history}", - template=template, - ) - - -# Register the adapter -registry.register(r"^bedrock.anthropic.claude*", BedrockClaudeAdapter) diff --git a/layers/langchain-common-layer/python/genai_core/adapters/bedrock/cohere.py b/layers/langchain-common-layer/python/genai_core/adapters/bedrock/cohere.py deleted file mode 100644 index 767295bc..00000000 --- a/layers/langchain-common-layer/python/genai_core/adapters/bedrock/cohere.py +++ /dev/null @@ -1,88 +0,0 @@ -# -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance -# with the License. A copy of the License is located at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions -# and limitations under the License. -# -import boto3 - -from langchain.llms import Bedrock -from langchain.prompts.prompt import PromptTemplate - -from ..base import ModelAdapter -from ..registry import registry - - -class BedrockCohereCommandAdapter(ModelAdapter): - def __init__(self, model_id, *args, **kwargs): - self.model_id = model_id - - super().__init__(*args, **kwargs) - - def get_llm(self, model_kwargs={}): - bedrock = boto3.client('bedrock-runtime') - - params = {} - if "temperature" in model_kwargs: - params["temperature"] = model_kwargs["temperature"] - if "maxTokens" in model_kwargs: - params["max_tokens"] = model_kwargs["maxTokens"] - params["return_likelihoods"] = "GENERATION" - - return Bedrock( - client=bedrock, - model_id=self.model_id, - model_kwargs=params, - streaming=model_kwargs.get("streaming", False), - callbacks=[self.callback_handler], - ) - - def get_prompt(self): - template = """ - -Human: The following is a friendly conversation between a human and an AI. If the AI does not know the answer to a question, it truthfully says it does not know. - -Current conversation: -{chat_history} - -Question: {input} - -Assistant:""" - - input_variables = ["input", "chat_history"] - prompt_template_args = { - "chat_history": "{chat_history}", - "input_variables": input_variables, - "template": template, - } - prompt_template = PromptTemplate(**prompt_template_args) - - return prompt_template - - def get_prompt_no_history_no_context(self): - template = """ - -Human: The following is a friendly conversation between a human and an AI. If the AI does not know the answer to a question, it truthfully says it does not know. - -Question: {input} - -Assistant:""" - - input_variables = ["input"] - prompt_template_args = { - "input_variables": input_variables, - "template": template, - } - prompt_template = PromptTemplate(**prompt_template_args) - - return prompt_template - - -# Register the adapter -registry.register(r"^bedrock.cohere.command-text*", BedrockCohereCommandAdapter) diff --git a/layers/langchain-common-layer/python/genai_core/adapters/bedrock/titan.py b/layers/langchain-common-layer/python/genai_core/adapters/bedrock/titan.py deleted file mode 100644 index f8115a62..00000000 --- a/layers/langchain-common-layer/python/genai_core/adapters/bedrock/titan.py +++ /dev/null @@ -1,85 +0,0 @@ -# -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance -# with the License. A copy of the License is located at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions -# and limitations under the License. -# -import boto3 -from langchain.prompts.prompt import PromptTemplate - -from langchain.llms import Bedrock - -from ..base import ModelAdapter -from ..registry import registry - - -class BedrockTitanAdapter(ModelAdapter): - def __init__(self, model_id, *args, **kwargs): - self.model_id = model_id - - super().__init__(*args, **kwargs) - - def get_llm(self, model_kwargs={}): - bedrock = boto3.client('bedrock-runtime') - - params = {} - if "temperature" in model_kwargs: - params["temperature"] = model_kwargs["temperature"] - if "topP" in model_kwargs: - params["topP"] = model_kwargs["topP"] - if "maxTokens" in model_kwargs: - params["maxTokenCount"] = model_kwargs["maxTokens"] - - return Bedrock( - client=bedrock, - model_id=self.model_id, - model_kwargs=params, - streaming=model_kwargs.get("streaming", False), - callbacks=[self.callback_handler], - ) - - def get_prompt(self): - template = """Human: The following is a friendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know. - -Current conversation: -{chat_history} - -Question: {input} - -Assistant:""" - - input_variables = ["input", "chat_history"] - prompt_template_args = { - "chat_history": "{chat_history}", - "input_variables": input_variables, - "template": template, - } - prompt_template = PromptTemplate(**prompt_template_args) - - return prompt_template - - def get_prompt_no_history_no_context(self): - template = """Human: The following is a friendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know. - -Question: {input} - -Assistant:""" - - input_variables = ["input"] - prompt_template_args = { - "input_variables": input_variables, - "template": template, - } - prompt_template = PromptTemplate(**prompt_template_args) - - return prompt_template - - -# Register the adapter -registry.register(r"^bedrock.amazon.titan-t*", BedrockTitanAdapter) diff --git a/layers/langchain-common-layer/python/genai_core/adapters/openai/__init__.py b/layers/langchain-common-layer/python/genai_core/adapters/openai/__init__.py deleted file mode 100644 index 0af1ca7a..00000000 --- a/layers/langchain-common-layer/python/genai_core/adapters/openai/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .gpt import * diff --git a/layers/langchain-common-layer/python/genai_core/adapters/openai/gpt.py b/layers/langchain-common-layer/python/genai_core/adapters/openai/gpt.py deleted file mode 100644 index 8a7fc5ed..00000000 --- a/layers/langchain-common-layer/python/genai_core/adapters/openai/gpt.py +++ /dev/null @@ -1,43 +0,0 @@ -# -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance -# with the License. A copy of the License is located at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions -# and limitations under the License. -# -import os -from langchain.chat_models import ChatOpenAI -from ..base import ModelAdapter -from ..registry import registry - - -class GPTAdapter(ModelAdapter): - def __init__(self, model_id, *args, **kwargs): - self.model_id = model_id - - super().__init__(*args, **kwargs) - - def get_llm(self, model_kwargs={}): - if not os.environ.get("OPENAI_API_KEY"): - raise Exception("OPENAI_API_KEY must be set in the environment") - - params = {} - if "streaming" in model_kwargs: - params["streaming"] = model_kwargs["streaming"] - if "temperature" in model_kwargs: - params["temperature"] = model_kwargs["temperature"] - if "maxTokens" in model_kwargs: - params["max_tokens"] = model_kwargs["maxTokens"] - - return ChatOpenAI( - model_name=self.model_id, callbacks=[self.callback_handler], **params - ) - - -# Register the adapter -registry.register(r"^openai*", GPTAdapter) diff --git a/layers/langchain-common-layer/python/genai_core/adapters/registry/__init__.py b/layers/langchain-common-layer/python/genai_core/adapters/registry/__init__.py deleted file mode 100644 index 8626c601..00000000 --- a/layers/langchain-common-layer/python/genai_core/adapters/registry/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .index import AdapterRegistry - -registry = AdapterRegistry() diff --git a/layers/langchain-common-layer/python/genai_core/adapters/registry/index.py b/layers/langchain-common-layer/python/genai_core/adapters/registry/index.py deleted file mode 100644 index 2fe4bfda..00000000 --- a/layers/langchain-common-layer/python/genai_core/adapters/registry/index.py +++ /dev/null @@ -1,35 +0,0 @@ -# -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance -# with the License. A copy of the License is located at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions -# and limitations under the License. -# -import re - -class AdapterRegistry: - def __init__(self): - # The registry is a dictionary where: - # Keys are compiled regular expressions - # Values are model IDs - self.registry = {} - - def register(self, regex, model_id): - # Compiles the regex and stores it in the registry - self.registry[re.compile(regex)] = model_id - - def get_adapter(self, model): - # Iterates over the registered regexes - for regex, adapter in self.registry.items(): - # If a match is found, returns the associated model ID - if regex.match(model): - return adapter - # If no match is found, returns None - raise ValueError( - f"Model {model} not found in registry. Available models: {self.registry}" - ) diff --git a/layers/langchain-common-layer/python/genai_core/adapters/sagemaker/__init__.py b/layers/langchain-common-layer/python/genai_core/adapters/sagemaker/__init__.py deleted file mode 100644 index fc9fb6c8..00000000 --- a/layers/langchain-common-layer/python/genai_core/adapters/sagemaker/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .meta import * -from .amazon import * diff --git a/layers/langchain-common-layer/python/genai_core/adapters/sagemaker/amazon/__init__.py b/layers/langchain-common-layer/python/genai_core/adapters/sagemaker/amazon/__init__.py deleted file mode 100644 index 70c5f045..00000000 --- a/layers/langchain-common-layer/python/genai_core/adapters/sagemaker/amazon/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .falconlite import * diff --git a/layers/langchain-common-layer/python/genai_core/adapters/sagemaker/amazon/falconlite.py b/layers/langchain-common-layer/python/genai_core/adapters/sagemaker/amazon/falconlite.py deleted file mode 100644 index 2b25e311..00000000 --- a/layers/langchain-common-layer/python/genai_core/adapters/sagemaker/amazon/falconlite.py +++ /dev/null @@ -1,102 +0,0 @@ -# -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance -# with the License. A copy of the License is located at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions -# and limitations under the License. -# -import json -import os - -from langchain.llms.sagemaker_endpoint import LLMContentHandler, SagemakerEndpoint -from langchain.prompts.prompt import PromptTemplate - -from ...base import ModelAdapter -from ...registry import registry - - -class FalconLiteContentHandler(LLMContentHandler): - content_type = "application/json" - accepts = "application/json" - - def transform_input(self, prompt, model_kwargs) -> bytes: - input_str = json.dumps( - { - "inputs": prompt, - "parameters": { - "max_new_tokens": model_kwargs.get("max_new_tokens", 512), - "do_sample": True, - "temperature": model_kwargs.get("temperature", 0.6), - "return_full_text": False, - "typical_p": 0.2, - "use_cache": True, - "seed": 1, - }, - } - ) - return input_str.encode("utf-8") - - def transform_output(self, output: bytes): - response_json = json.loads(output.read().decode("utf-8")) - return response_json[0]["generated_text"] - - -content_handler = FalconLiteContentHandler() - - -class SMFalconLiteAdapter(ModelAdapter): - def __init__(self, model_id, **kwargs): - self.model_id = model_id - - super().__init__(**kwargs) - - def get_llm(self, model_kwargs={}): - params = {} - if "temperature" in model_kwargs: - params["temperature"] = model_kwargs["temperature"] - if "maxTokens" in model_kwargs: - params["max_new_tokens"] = model_kwargs["maxTokens"] - - return SagemakerEndpoint( - endpoint_name=self.model_id, - region_name=os.environ["AWS_REGION"], - content_handler=content_handler, - model_kwargs=params, - callbacks=[self.callback_handler], - ) - - def get_prompt(self): - template = """<|prompter|>Our current conversation: {chat_history} - - {input}<|endoftext|><|assistant|>""" - - input_variables = ["input", "chat_history"] - prompt_template_args = { - "chat_history": "{chat_history}", - "input_variables": input_variables, - "template": template, - } - prompt_template = PromptTemplate(**prompt_template_args) - - return prompt_template - - def get_prompt_no_history_no_context(self): - template = """<|prompter|>{input}<|endoftext|><|assistant|>""" - - input_variables = ["input"] - prompt_template_args = { - "input_variables": input_variables, - "template": template, - } - prompt_template = PromptTemplate(**prompt_template_args) - - return prompt_template - - -# Register the adapter -registry.register(r"(?i)sagemaker\.amazon-FalconLite*", SMFalconLiteAdapter) diff --git a/layers/langchain-common-layer/python/genai_core/adapters/sagemaker/meta/__init__.py b/layers/langchain-common-layer/python/genai_core/adapters/sagemaker/meta/__init__.py deleted file mode 100644 index 7b127d57..00000000 --- a/layers/langchain-common-layer/python/genai_core/adapters/sagemaker/meta/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .llama2_base import * -from .llama2_chat import * diff --git a/layers/langchain-common-layer/python/genai_core/adapters/sagemaker/meta/llama2_base.py b/layers/langchain-common-layer/python/genai_core/adapters/sagemaker/meta/llama2_base.py deleted file mode 100644 index 13b45cb2..00000000 --- a/layers/langchain-common-layer/python/genai_core/adapters/sagemaker/meta/llama2_base.py +++ /dev/null @@ -1,69 +0,0 @@ -# -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance -# with the License. A copy of the License is located at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions -# and limitations under the License. -# -import json -import os - -from langchain.llms.sagemaker_endpoint import LLMContentHandler, SagemakerEndpoint - -from ...base import ModelAdapter -from ...registry import registry - - -class Llama2BaseContentHandler(LLMContentHandler): - content_type = "application/json" - accepts = "application/json" - - def transform_input(self, prompt, model_kwargs) -> bytes: - input_str = json.dumps( - { - "inputs": prompt, - "parameters": model_kwargs, - } - ) - return input_str.encode("utf-8") - - def transform_output(self, output: bytes): - response_json = json.loads(output.read().decode("utf-8")) - return response_json[0]["generation"] - - -content_handler = Llama2BaseContentHandler() - - -class SMLlama2BaseAdapter(ModelAdapter): - def __init__(self, model_id, **kwargs): - self.model_id = model_id - - super().__init__(**kwargs) - - def get_llm(self, model_kwargs={}): - params = {} - if "temperature" in model_kwargs: - params["temperature"] = model_kwargs["temperature"] - if "topP" in model_kwargs: - params["top_p"] = model_kwargs["topP"] - if "maxTokens" in model_kwargs: - params["max_new_tokens"] = model_kwargs["maxTokens"] - - return SagemakerEndpoint( - endpoint_name=self.model_id, - region_name=os.environ.get("AWS_REGION"), - model_kwargs=params, - endpoint_kwargs={"CustomAttributes": "accept_eula=true"}, - content_handler=content_handler, - callbacks=[self.callback_handler], - ) - - -# Register the adapter -registry.register(r"(?i)sagemaker\.meta-LLama.*base.*", SMLlama2BaseAdapter) diff --git a/layers/langchain-common-layer/python/genai_core/adapters/sagemaker/meta/llama2_chat.py b/layers/langchain-common-layer/python/genai_core/adapters/sagemaker/meta/llama2_chat.py deleted file mode 100644 index 5200f084..00000000 --- a/layers/langchain-common-layer/python/genai_core/adapters/sagemaker/meta/llama2_chat.py +++ /dev/null @@ -1,73 +0,0 @@ -# -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance -# with the License. A copy of the License is located at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions -# and limitations under the License. -# -import json -import os - -from langchain.llms.sagemaker_endpoint import LLMContentHandler, SagemakerEndpoint - -from ...base import ModelAdapter -from ...registry import registry - - -class Llama2ChatContentHandler(LLMContentHandler): - content_type = "application/json" - accepts = "application/json" - - def transform_input(self, prompt, model_kwargs) -> bytes: - input_str = json.dumps( - { - "inputs": [ - [ - {"role": "user", "content": prompt}, - ], - ], - "parameters": model_kwargs, - } - ) - return input_str.encode("utf-8") - - def transform_output(self, output: bytes): - response_json = json.loads(output.read().decode("utf-8")) - return response_json[0]["generation"]["content"] - - -content_handler = Llama2ChatContentHandler() - - -class SMLlama2ChatAdapter(ModelAdapter): - def __init__(self, model_id, **kwargs): - self.model_id = model_id - - super().__init__(**kwargs) - - def get_llm(self, model_kwargs={}): - params = {} - if "temperature" in model_kwargs: - params["temperature"] = model_kwargs["temperature"] - if "topP" in model_kwargs: - params["top_p"] = model_kwargs["topP"] - if "maxTokens" in model_kwargs: - params["max_new_tokens"] = model_kwargs["maxTokens"] - - return SagemakerEndpoint( - endpoint_name=self.model_id, - region_name=os.environ.get("AWS_REGION"), - model_kwargs=params, - endpoint_kwargs={"CustomAttributes": "accept_eula=true"}, - content_handler=content_handler, - callbacks=[self.callback_handler], - ) - - -# Register the adapter -registry.register(r"(?i)sagemaker\.meta-LLama.*\d+b.*chat.*", SMLlama2ChatAdapter) diff --git a/layers/langchain-common-layer/python/genai_core/clients.py b/layers/langchain-common-layer/python/genai_core/clients.py deleted file mode 100644 index fb658b22..00000000 --- a/layers/langchain-common-layer/python/genai_core/clients.py +++ /dev/null @@ -1,33 +0,0 @@ -# -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance -# with the License. A copy of the License is located at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions -# and limitations under the License. -# -import boto3 -import os -import openai -from botocore.config import Config - -def get_openai_client(): - api_key = os.environ['OPEN_API_KEY'] - if not api_key: - return None - - openai.api_key = api_key - - return openai - - -def get_sagemaker_client(): - config = Config(retries={"max_attempts": 15, "mode": "adaptive"}) - - client = boto3.client("sagemaker-runtime", config=config) - - return client diff --git a/layers/langchain-common-layer/python/genai_core/utils/__init__.py b/layers/langchain-common-layer/python/genai_core/utils/__init__.py deleted file mode 100644 index 96732050..00000000 --- a/layers/langchain-common-layer/python/genai_core/utils/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .error_codes import * \ No newline at end of file diff --git a/layers/langchain-common-layer/python/genai_core/utils/error_codes.py b/layers/langchain-common-layer/python/genai_core/utils/error_codes.py deleted file mode 100644 index 0bbc8eb4..00000000 --- a/layers/langchain-common-layer/python/genai_core/utils/error_codes.py +++ /dev/null @@ -1,25 +0,0 @@ -# -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance -# with the License. A copy of the License is located at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES -# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions -# and limitations under the License. -# - -class PredictionException(Exception): - """Exception raised for errors while running prediction. - - Attributes: - llm_name -- llm used for prediction - message -- explanation of the error - """ - - def __init__(self, llm_name, message="Exception during prediction"): - self.message = "601 -> "+llm_name+" -> "+message - super(PredictionException, self).__init__(message) - #super().__init__(self.message) \ No newline at end of file diff --git a/src/patterns/gen-ai/aws-langchain-common-layer/README.md b/src/patterns/gen-ai/aws-langchain-common-layer/README.md index 9179f80f..54af8d86 100644 --- a/src/patterns/gen-ai/aws-langchain-common-layer/README.md +++ b/src/patterns/gen-ai/aws-langchain-common-layer/README.md @@ -45,10 +45,23 @@ Thanks to the original authors: This construct provides an AWS Lambda layer which contains needed python pip packages to build generative AI applications based on the [LangChain](https://python.langchain.com/docs/get_started/introduction) client. The list of libraries installed and their version is available [here](../../../../layers/langchain-common-deps/requirements.txt) +--- + +### Building with Differing Architectures + +Use Docker's default platform environmental variable, [`DOCKER_DEFAULT_PLATFORM`](https://docs.docker.com/engine/reference/commandline/cli/), when synthesizing or deploying on different architectures. For example: ARM chips (such as Apple silicon series macOS M1, M2, and M3) are by default `arm64` architectures and may bundle unsupported packages into `x86_64` Lambda Layers. + +A solution is: + +```bash +DOCKER_DEFAULT_PLATFORM=linux/amd64 cdk deploy +``` + Here is a minimal deployable pattern definition: TypeScript -``` typescript + +```typescript import { Construct } from 'constructs'; import { Stack, StackProps, Aws } from 'aws-cdk-lib'; import { LangchainCommonDepsLayer } from '@cdklabs/generative-ai-cdk-constructs'; @@ -66,6 +79,7 @@ const lambdaDepsLayer = new LangchainCommonDepsLayer(this, 'lambdagenaidepslayer ``` Python + ``` python from constructs import Construct from aws_cdk import Aws, aws_lambda as lambda_ @@ -89,17 +103,17 @@ lambda_deps_layer = LangchainCommonDepsLayer( ## Initializer -``` +```typescript new LangchainCommonDepsLayer(scope: Construct, id: string, props: LangchainLayerProps) ``` -#### Parameters +### Parameters - scope [Construct](https://docs.aws.amazon.com/cdk/api/v2/docs/constructs.Construct.html) - id string - props [LangchainLayerProps](https://github.com/awslabs/generative-ai-cdk-constructs/blob/main/src/patterns/gen-ai/aws-langchain-common-layer/index.ts#L23) -#### Pattern Construct Props +### Pattern Construct Props | **Name** | **Type** | **Required** |**Description** | |:-------------|:----------------|-----------------|-----------------| @@ -112,12 +126,12 @@ new LangchainCommonDepsLayer(scope: Construct, id: string, props: LangchainLayer | license | string | ![Optional](https://img.shields.io/badge/optional-4169E1) | The SPDX licence identifier or URL to the license file for this layer | | removalPolicy | [RemovalPolicy](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.RemovalPolicy.html) | ![Optional](https://img.shields.io/badge/optional-4169E1) | Whether to retain this version of the layer when a new version is added or when the stack is deleted. Default: DESTROY | - ## Default properties Out-of-the-box implementation of the construct without any override will not set any default values. Depending on the features enabled, user will need to provide environmental variable values to the AWS Lambda function used by the LangchainCommonLayer. ## Architecture + ![Architecture Diagram](architecture.png) ## Cost