From 32273c0b53d0c7ea076866da65adbfe7c34ea4b3 Mon Sep 17 00:00:00 2001 From: Erick Friis Date: Mon, 28 Oct 2024 09:24:10 -0700 Subject: [PATCH 01/10] use staging for integration tests --- .github/workflows/integration_tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index e862470f7..4c5757d4b 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -47,7 +47,7 @@ jobs: uses: ./.github/actions/python-integration-tests with: python-version: 3.11 - langchain-endpoint: https://api.smith.langchain.com + langchain-endpoint: https://beta.api.smith.langchain.com langchain-api-key: ${{ secrets.LANGSMITH_API_KEY }} openai-api-key: ${{ secrets.OPENAI_API_KEY }} anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }} @@ -76,6 +76,6 @@ jobs: uses: ./.github/actions/js-integration-tests with: node-version: 20.x - langchain-endpoint: https://api.smith.langchain.com + langchain-endpoint: https://beta.api.smith.langchain.com langchain-api-key: ${{ secrets.LANGSMITH_API_KEY }} openai-api-key: ${{ secrets.OPENAI_API_KEY }} From 5ed29a0cfc71a4cac341f92b243f86446fbb20d0 Mon Sep 17 00:00:00 2001 From: Erick Friis Date: Mon, 28 Oct 2024 14:57:30 -0700 Subject: [PATCH 02/10] integration tests against staging --- .../integration_tests/test_async_client.py | 2 +- python/tests/integration_tests/test_client.py | 2 +- .../integration_tests/test_llm_evaluator.py | 2 +- .../integration_tests/wrappers/test_openai.py | 36 +++++++++---------- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/python/tests/integration_tests/test_async_client.py b/python/tests/integration_tests/test_async_client.py index 6338ec2ae..ecd03c87f 100644 --- a/python/tests/integration_tests/test_async_client.py +++ b/python/tests/integration_tests/test_async_client.py @@ -63,7 +63,7 @@ async def wait_for(condition, timeout=10): @pytest.fixture async def async_client(): ls_utils.get_env_var.cache_clear() - client = AsyncClient(api_url="https://api.smith.langchain.com") + client = AsyncClient() yield client await client.aclose() diff --git a/python/tests/integration_tests/test_client.py b/python/tests/integration_tests/test_client.py index ad4b2cd93..7b7b47ad3 100644 --- a/python/tests/integration_tests/test_client.py +++ b/python/tests/integration_tests/test_client.py @@ -43,7 +43,7 @@ def wait_for( @pytest.fixture def langchain_client() -> Client: get_env_var.cache_clear() - return Client(api_url="https://api.smith.langchain.com") + return Client() def test_datasets(langchain_client: Client) -> None: diff --git a/python/tests/integration_tests/test_llm_evaluator.py b/python/tests/integration_tests/test_llm_evaluator.py index 28b742096..1fffde4e9 100644 --- a/python/tests/integration_tests/test_llm_evaluator.py +++ b/python/tests/integration_tests/test_llm_evaluator.py @@ -147,7 +147,7 @@ def test_from_model() -> None: async def test_evaluate() -> None: client = Client() client.clone_public_dataset( - "https://smith.langchain.com/public/419dcab2-1d66-4b94-8901-0357ead390df/d" + "https://beta.smith.langchain.com/public/06785303-0f70-4466-b637-f23d38c0f28e/d" ) dataset_name = "Evaluate Examples" diff --git a/python/tests/integration_tests/wrappers/test_openai.py b/python/tests/integration_tests/wrappers/test_openai.py index 36fdf4ac9..396647a06 100644 --- a/python/tests/integration_tests/wrappers/test_openai.py +++ b/python/tests/integration_tests/wrappers/test_openai.py @@ -12,12 +12,12 @@ from langsmith.wrappers import wrap_openai -@mock.patch("langsmith.client.requests.Session") @pytest.mark.parametrize("stream", [False, True]) -def test_chat_sync_api(mock_session: mock.MagicMock, stream: bool): +def test_chat_sync_api(stream: bool): import openai # noqa - client = langsmith.Client(session=mock_session()) + mock_session = mock.MagicMock() + client = langsmith.Client(session=mock_session) original_client = openai.Client() patched_client = wrap_openai(openai.Client(), tracing_extra={"client": client}) messages = [{"role": "user", "content": "Say 'foo'"}] @@ -51,12 +51,12 @@ def test_chat_sync_api(mock_session: mock.MagicMock, stream: bool): assert call[0][0].upper() == "POST" -@mock.patch("langsmith.client.requests.Session") @pytest.mark.parametrize("stream", [False, True]) -async def test_chat_async_api(mock_session: mock.MagicMock, stream: bool): +async def test_chat_async_api(stream: bool): import openai # noqa - client = langsmith.Client(session=mock_session()) + mock_session = mock.MagicMock() + client = langsmith.Client(session=mock_session) original_client = openai.AsyncClient() patched_client = wrap_openai(openai.AsyncClient(), tracing_extra={"client": client}) messages = [{"role": "user", "content": "Say 'foo'"}] @@ -86,12 +86,12 @@ async def test_chat_async_api(mock_session: mock.MagicMock, stream: bool): assert call[0][0].upper() == "POST" -@mock.patch("langsmith.client.requests.Session") @pytest.mark.parametrize("stream", [False, True]) -def test_completions_sync_api(mock_session: mock.MagicMock, stream: bool): +def test_completions_sync_api(stream: bool): import openai - client = langsmith.Client(session=mock_session()) + mock_session = mock.MagicMock() + client = langsmith.Client(session=mock_session) original_client = openai.Client() patched_client = wrap_openai(openai.Client(), tracing_extra={"client": client}) prompt = ("Say 'Foo' then stop.",) @@ -129,12 +129,12 @@ def test_completions_sync_api(mock_session: mock.MagicMock, stream: bool): assert call[0][0].upper() == "POST" -@mock.patch("langsmith.client.requests.Session") @pytest.mark.parametrize("stream", [False, True]) -async def test_completions_async_api(mock_session: mock.MagicMock, stream: bool): +async def test_completions_async_api(stream: bool): import openai - client = langsmith.Client(session=mock_session()) + mock_session = mock.MagicMock() + client = langsmith.Client(session=mock_session) original_client = openai.AsyncClient() patched_client = wrap_openai( @@ -274,13 +274,13 @@ def _collect_requests(mock_session: mock.MagicMock, filename: str): @pytest.mark.parametrize("test_case", test_cases) -@mock.patch("langsmith.client.requests.Session") -def test_wrap_openai_chat_tokens(mock_session: mock.MagicMock, test_case): +def test_wrap_openai_chat_tokens(test_case): import openai from openai.types.chat import ChatCompletion, ChatCompletionChunk oai_client = openai.Client() - ls_client = langsmith.Client(session=mock_session()) + mock_session = mock.MagicMock() + ls_client = langsmith.Client(session=mock.mock_session) wrapped_oai_client = wrap_openai(oai_client, tracing_extra={"client": ls_client}) collect = Collect() @@ -323,13 +323,13 @@ def test_wrap_openai_chat_tokens(mock_session: mock.MagicMock, test_case): @pytest.mark.asyncio @pytest.mark.parametrize("test_case", test_cases) -@mock.patch("langsmith.client.requests.Session") -async def test_wrap_openai_chat_async_tokens(mock_session: mock.MagicMock, test_case): +async def test_wrap_openai_chat_async_tokens(test_case): import openai from openai.types.chat import ChatCompletion, ChatCompletionChunk oai_client = openai.AsyncClient() - ls_client = langsmith.Client(session=mock_session()) + mock_session = mock.MagicMock() + ls_client = langsmith.Client(session=mock_session) wrapped_oai_client = wrap_openai(oai_client, tracing_extra={"client": ls_client}) collect = Collect() From 7bdba1e1def89da68958498d631bfaef3f0710ba Mon Sep 17 00:00:00 2001 From: Erick Friis Date: Mon, 28 Oct 2024 15:00:15 -0700 Subject: [PATCH 03/10] x --- python/tests/integration_tests/wrappers/test_openai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/tests/integration_tests/wrappers/test_openai.py b/python/tests/integration_tests/wrappers/test_openai.py index 396647a06..970198507 100644 --- a/python/tests/integration_tests/wrappers/test_openai.py +++ b/python/tests/integration_tests/wrappers/test_openai.py @@ -280,7 +280,7 @@ def test_wrap_openai_chat_tokens(test_case): oai_client = openai.Client() mock_session = mock.MagicMock() - ls_client = langsmith.Client(session=mock.mock_session) + ls_client = langsmith.Client(session=mock_session) wrapped_oai_client = wrap_openai(oai_client, tracing_extra={"client": ls_client}) collect = Collect() From da0a3b298c74ca1829a0d909c81661176940e249 Mon Sep 17 00:00:00 2001 From: Erick Friis Date: Mon, 28 Oct 2024 15:19:16 -0700 Subject: [PATCH 04/10] x --- python/tests/integration_tests/wrappers/test_openai.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/tests/integration_tests/wrappers/test_openai.py b/python/tests/integration_tests/wrappers/test_openai.py index 970198507..85e52a19a 100644 --- a/python/tests/integration_tests/wrappers/test_openai.py +++ b/python/tests/integration_tests/wrappers/test_openai.py @@ -179,10 +179,10 @@ async def test_completions_async_api(stream: bool): # Give the thread a chance. for _ in range(10): time.sleep(0.1) - if mock_session.return_value.request.call_count >= 1: + if mock_session.request.call_count >= 1: break - assert mock_session.return_value.request.call_count >= 1 - for call in mock_session.return_value.request.call_args_list[1:]: + assert mock_session.request.call_count >= 1 + for call in mock_session.request.call_args_list[1:]: assert call[0][0].upper() == "POST" From ffd56747972841ed27fecd8ea3f142ec98a49813 Mon Sep 17 00:00:00 2001 From: Erick Friis Date: Mon, 28 Oct 2024 15:26:00 -0700 Subject: [PATCH 05/10] x --- python/tests/integration_tests/test_client.py | 6 ++---- python/tests/integration_tests/wrappers/test_openai.py | 10 +++++----- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/python/tests/integration_tests/test_client.py b/python/tests/integration_tests/test_client.py index 7b7b47ad3..ef289d502 100644 --- a/python/tests/integration_tests/test_client.py +++ b/python/tests/integration_tests/test_client.py @@ -356,11 +356,9 @@ def test_persist_update_run(langchain_client: Client) -> None: @pytest.mark.parametrize("uri", ["http://localhost:1981", "http://api.langchain.minus"]) -def test_error_surfaced_invalid_uri(monkeypatch: pytest.MonkeyPatch, uri: str) -> None: +def test_error_surfaced_invalid_uri(uri: str) -> None: get_env_var.cache_clear() - monkeypatch.setenv("LANGCHAIN_ENDPOINT", uri) - monkeypatch.setenv("LANGCHAIN_API_KEY", "test") - client = Client() + client = Client(api_url=uri, api_key="test") # expect connect error with pytest.raises(LangSmithConnectionError): client.create_run("My Run", inputs={"text": "hello world"}, run_type="llm") diff --git a/python/tests/integration_tests/wrappers/test_openai.py b/python/tests/integration_tests/wrappers/test_openai.py index 85e52a19a..d926583fa 100644 --- a/python/tests/integration_tests/wrappers/test_openai.py +++ b/python/tests/integration_tests/wrappers/test_openai.py @@ -47,7 +47,7 @@ def test_chat_sync_api(stream: bool): assert original.choices == patched.choices # Give the thread a chance. time.sleep(0.01) - for call in mock_session.return_value.request.call_args_list[1:]: + for call in mock_session.request.call_args_list[1:]: assert call[0][0].upper() == "POST" @@ -82,7 +82,7 @@ async def test_chat_async_api(stream: bool): assert original.choices == patched.choices # Give the thread a chance. time.sleep(0.1) - for call in mock_session.return_value.request.call_args_list[1:]: + for call in mock_session.request.call_args_list[1:]: assert call[0][0].upper() == "POST" @@ -125,7 +125,7 @@ def test_completions_sync_api(stream: bool): assert original.choices == patched.choices # Give the thread a chance. time.sleep(0.1) - for call in mock_session.return_value.request.call_args_list[1:]: + for call in mock_session.request.call_args_list[1:]: assert call[0][0].upper() == "POST" @@ -199,7 +199,7 @@ def __call__(self, run): def _collect_requests(mock_session: mock.MagicMock, filename: str): - mock_requests = mock_session.return_value.request.call_args_list + mock_requests = mock_session.request.call_args_list collected_requests = {} for _ in range(10): time.sleep(0.1) @@ -215,7 +215,7 @@ def _collect_requests(mock_session: mock.MagicMock, filename: str): # thread has finished processing the run if any(event.get("end_time") for event in all_events): break - mock_session.return_value.request.call_args_list.clear() + mock_session.request.call_args_list.clear() if os.environ.get("WRITE_TOKEN_COUNTING_TEST_DATA") == "1": dir_path = Path(__file__).resolve().parent.parent / "test_data" From 8df861e622b7f5a38132abe4ca20aa1f0dd897df Mon Sep 17 00:00:00 2001 From: Erick Friis Date: Mon, 28 Oct 2024 18:56:47 -0700 Subject: [PATCH 06/10] x --- .../python-integration-tests/action.yml | 18 +++++++++--------- .github/workflows/integration_tests.yml | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/actions/python-integration-tests/action.yml b/.github/actions/python-integration-tests/action.yml index 516af5b86..0cdf13f8a 100644 --- a/.github/actions/python-integration-tests/action.yml +++ b/.github/actions/python-integration-tests/action.yml @@ -4,11 +4,11 @@ inputs: python-version: description: "Python version" required: true - langchain-api-key: - description: "Langchain" + langchain-api-key-beta: + description: "LangSmith Beta Key" required: true - langchain-endpoint: - description: "LangSmith Endpoint" + langchain-api-key-prod: + description: "LangSmith Key" required: true openai-api-key: description: "OpenAI API key" @@ -43,7 +43,7 @@ runs: - name: Run integration tests env: LANGCHAIN_TRACING_V2: "true" - LANGCHAIN_ENDPOINT: ${{ inputs.langchain-endpoint }} + LANGCHAIN_ENDPOINT: https://beta.api.smith.langchain.com LANGCHAIN_API_KEY: ${{ inputs.langchain-api-key }} OPENAI_API_KEY: ${{ inputs.openai-api-key }} run: make integration_tests_fast @@ -53,8 +53,8 @@ runs: - name: Run doctest env: LANGCHAIN_TRACING_V2: "true" - LANGCHAIN_ENDPOINT: ${{ inputs.langchain-endpoint }} - LANGCHAIN_API_KEY: ${{ inputs.langchain-api-key }} + LANGCHAIN_ENDPOINT: https://api.smith.langchain.com + LANGCHAIN_API_KEY: ${{ inputs.langchain-api-key-prod }} OPENAI_API_KEY: ${{ inputs.openai-api-key }} ANTHROPIC_API_KEY: ${{ inputs.anthropic-api-key }} run: make doctest @@ -65,8 +65,8 @@ runs: - name: Run Evaluation env: LANGCHAIN_TRACING_V2: "true" - LANGCHAIN_ENDPOINT: ${{ inputs.langchain-endpoint }} - LANGCHAIN_API_KEY: ${{ inputs.langchain-api-key }} + LANGCHAIN_ENDPOINT: https://beta.api.smith.langchain.com + LANGCHAIN_API_KEY: ${{ inputs.langchain-api-key-beta }} OPENAI_API_KEY: ${{ inputs.openai-api-key }} ANTHROPIC_API_KEY: ${{ inputs.anthropic-api-key }} run: make evals diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index e862470f7..ea7e8ae4a 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -47,8 +47,8 @@ jobs: uses: ./.github/actions/python-integration-tests with: python-version: 3.11 - langchain-endpoint: https://api.smith.langchain.com - langchain-api-key: ${{ secrets.LANGSMITH_API_KEY }} + langchain-api-key-beta: ${{ secrets.LANGSMITH_API_KEY_BETA }} + langchain-api-key-prod: ${{ secrets.LANGSMITH_API_KEY_PROD }} openai-api-key: ${{ secrets.OPENAI_API_KEY }} anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }} From d14d140aa66b90284b7868af7ee5311714cc7a9f Mon Sep 17 00:00:00 2001 From: Erick Friis Date: Mon, 28 Oct 2024 21:46:27 -0700 Subject: [PATCH 07/10] x --- .github/actions/python-integration-tests/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/python-integration-tests/action.yml b/.github/actions/python-integration-tests/action.yml index 0cdf13f8a..7631570e2 100644 --- a/.github/actions/python-integration-tests/action.yml +++ b/.github/actions/python-integration-tests/action.yml @@ -44,7 +44,7 @@ runs: env: LANGCHAIN_TRACING_V2: "true" LANGCHAIN_ENDPOINT: https://beta.api.smith.langchain.com - LANGCHAIN_API_KEY: ${{ inputs.langchain-api-key }} + LANGCHAIN_API_KEY: ${{ inputs.langchain-api-key-beta }} OPENAI_API_KEY: ${{ inputs.openai-api-key }} run: make integration_tests_fast shell: bash From 15cde46fe8f909f0a7208f3a2c5c560578ba718e Mon Sep 17 00:00:00 2001 From: Erick Friis Date: Mon, 28 Oct 2024 21:55:12 -0700 Subject: [PATCH 08/10] x --- js/src/tests/client.int.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/js/src/tests/client.int.test.ts b/js/src/tests/client.int.test.ts index ddf160fa3..472f6b3c1 100644 --- a/js/src/tests/client.int.test.ts +++ b/js/src/tests/client.int.test.ts @@ -1112,7 +1112,7 @@ test("Test pull prompt include model", async () => { test("list shared examples can list shared examples", async () => { const client = new Client(); const multiverseMathPublicDatasetShareToken = - "620596ee-570b-4d2b-8c8f-f828adbe5242"; + "ce9c8a9-761a-4756-b159-58ed2640e274"; const sharedExamples = await client.listSharedExamples( multiverseMathPublicDatasetShareToken ); @@ -1123,7 +1123,7 @@ test("clonePublicDataset method can clone a dataset", async () => { const client = new Client(); const datasetName = "multiverse_math_public_testing"; const multiverseMathPublicDatasetURL = - "https://smith.langchain.com/public/620596ee-570b-4d2b-8c8f-f828adbe5242/d"; + "https://beta.smith.langchain.com/public/cce9c8a9-761a-4756-b159-58ed2640e274/d"; try { await client.clonePublicDataset(multiverseMathPublicDatasetURL, { From 747bc7ba2da58777f5cae4e9159f8ccdeb663395 Mon Sep 17 00:00:00 2001 From: Erick Friis Date: Mon, 28 Oct 2024 21:56:57 -0700 Subject: [PATCH 09/10] x --- .github/actions/js-integration-tests/action.yml | 9 +++------ .github/workflows/integration_tests.yml | 3 +-- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/actions/js-integration-tests/action.yml b/.github/actions/js-integration-tests/action.yml index fa55acf09..eb4ab072a 100644 --- a/.github/actions/js-integration-tests/action.yml +++ b/.github/actions/js-integration-tests/action.yml @@ -4,12 +4,9 @@ inputs: node-version: description: "Node version" required: true - langchain-api-key: + langchain-api-key-beta: description: "Langchain" required: true - langchain-endpoint: - description: "LangSmith Endpoint" - required: true openai-api-key: description: "OpenAI API key" required: false @@ -37,6 +34,6 @@ runs: working-directory: js env: LANGCHAIN_TRACING_V2: "true" - LANGCHAIN_ENDPOINT: ${{ inputs.langchain-endpoint }} - LANGCHAIN_API_KEY: ${{ inputs.langchain-api-key }} + LANGCHAIN_ENDPOINT: https://beta.api.smith.langchain.com + LANGCHAIN_API_KEY: ${{ inputs.langchain-api-key-beta }} OPENAI_API_KEY: ${{ inputs.openai-api-key }} \ No newline at end of file diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 2c41967e4..dd597ee45 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -76,6 +76,5 @@ jobs: uses: ./.github/actions/js-integration-tests with: node-version: 20.x - langchain-endpoint: https://beta.api.smith.langchain.com - langchain-api-key: ${{ secrets.LANGSMITH_API_KEY }} + langchain-api-key-beta: ${{ secrets.LANGSMITH_API_KEY_BETA }} openai-api-key: ${{ secrets.OPENAI_API_KEY }} From 437a5d5234adc3e98882dfb955b43f84a8875a3f Mon Sep 17 00:00:00 2001 From: Erick Friis Date: Mon, 28 Oct 2024 22:00:45 -0700 Subject: [PATCH 10/10] x --- js/src/tests/client.int.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/js/src/tests/client.int.test.ts b/js/src/tests/client.int.test.ts index 472f6b3c1..7b5d63f89 100644 --- a/js/src/tests/client.int.test.ts +++ b/js/src/tests/client.int.test.ts @@ -1112,7 +1112,7 @@ test("Test pull prompt include model", async () => { test("list shared examples can list shared examples", async () => { const client = new Client(); const multiverseMathPublicDatasetShareToken = - "ce9c8a9-761a-4756-b159-58ed2640e274"; + "cce9c8a9-761a-4756-b159-58ed2640e274"; const sharedExamples = await client.listSharedExamples( multiverseMathPublicDatasetShareToken );