diff --git a/.github/actions/js-integration-tests/action.yml b/.github/actions/js-integration-tests/action.yml index fa55acf09..eb4ab072a 100644 --- a/.github/actions/js-integration-tests/action.yml +++ b/.github/actions/js-integration-tests/action.yml @@ -4,12 +4,9 @@ inputs: node-version: description: "Node version" required: true - langchain-api-key: + langchain-api-key-beta: description: "Langchain" required: true - langchain-endpoint: - description: "LangSmith Endpoint" - required: true openai-api-key: description: "OpenAI API key" required: false @@ -37,6 +34,6 @@ runs: working-directory: js env: LANGCHAIN_TRACING_V2: "true" - LANGCHAIN_ENDPOINT: ${{ inputs.langchain-endpoint }} - LANGCHAIN_API_KEY: ${{ inputs.langchain-api-key }} + LANGCHAIN_ENDPOINT: https://beta.api.smith.langchain.com + LANGCHAIN_API_KEY: ${{ inputs.langchain-api-key-beta }} OPENAI_API_KEY: ${{ inputs.openai-api-key }} \ No newline at end of file diff --git a/.github/actions/python-integration-tests/action.yml b/.github/actions/python-integration-tests/action.yml index 516af5b86..7631570e2 100644 --- a/.github/actions/python-integration-tests/action.yml +++ b/.github/actions/python-integration-tests/action.yml @@ -4,11 +4,11 @@ inputs: python-version: description: "Python version" required: true - langchain-api-key: - description: "Langchain" + langchain-api-key-beta: + description: "LangSmith Beta Key" required: true - langchain-endpoint: - description: "LangSmith Endpoint" + langchain-api-key-prod: + description: "LangSmith Key" required: true openai-api-key: description: "OpenAI API key" @@ -43,8 +43,8 @@ runs: - name: Run integration tests env: LANGCHAIN_TRACING_V2: "true" - LANGCHAIN_ENDPOINT: ${{ inputs.langchain-endpoint }} - LANGCHAIN_API_KEY: ${{ inputs.langchain-api-key }} + LANGCHAIN_ENDPOINT: https://beta.api.smith.langchain.com + LANGCHAIN_API_KEY: ${{ inputs.langchain-api-key-beta }} OPENAI_API_KEY: ${{ inputs.openai-api-key }} run: make integration_tests_fast shell: bash @@ -53,8 +53,8 @@ runs: - name: Run doctest env: LANGCHAIN_TRACING_V2: "true" - LANGCHAIN_ENDPOINT: ${{ inputs.langchain-endpoint }} - LANGCHAIN_API_KEY: ${{ inputs.langchain-api-key }} + LANGCHAIN_ENDPOINT: https://api.smith.langchain.com + LANGCHAIN_API_KEY: ${{ inputs.langchain-api-key-prod }} OPENAI_API_KEY: ${{ inputs.openai-api-key }} ANTHROPIC_API_KEY: ${{ inputs.anthropic-api-key }} run: make doctest @@ -65,8 +65,8 @@ runs: - name: Run Evaluation env: LANGCHAIN_TRACING_V2: "true" - LANGCHAIN_ENDPOINT: ${{ inputs.langchain-endpoint }} - LANGCHAIN_API_KEY: ${{ inputs.langchain-api-key }} + LANGCHAIN_ENDPOINT: https://beta.api.smith.langchain.com + LANGCHAIN_API_KEY: ${{ inputs.langchain-api-key-beta }} OPENAI_API_KEY: ${{ inputs.openai-api-key }} ANTHROPIC_API_KEY: ${{ inputs.anthropic-api-key }} run: make evals diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index e862470f7..dd597ee45 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -47,8 +47,8 @@ jobs: uses: ./.github/actions/python-integration-tests with: python-version: 3.11 - langchain-endpoint: https://api.smith.langchain.com - langchain-api-key: ${{ secrets.LANGSMITH_API_KEY }} + langchain-api-key-beta: ${{ secrets.LANGSMITH_API_KEY_BETA }} + langchain-api-key-prod: ${{ secrets.LANGSMITH_API_KEY_PROD }} openai-api-key: ${{ secrets.OPENAI_API_KEY }} anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }} @@ -76,6 +76,5 @@ jobs: uses: ./.github/actions/js-integration-tests with: node-version: 20.x - langchain-endpoint: https://api.smith.langchain.com - langchain-api-key: ${{ secrets.LANGSMITH_API_KEY }} + langchain-api-key-beta: ${{ secrets.LANGSMITH_API_KEY_BETA }} openai-api-key: ${{ secrets.OPENAI_API_KEY }} diff --git a/js/src/tests/client.int.test.ts b/js/src/tests/client.int.test.ts index ddf160fa3..7b5d63f89 100644 --- a/js/src/tests/client.int.test.ts +++ b/js/src/tests/client.int.test.ts @@ -1112,7 +1112,7 @@ test("Test pull prompt include model", async () => { test("list shared examples can list shared examples", async () => { const client = new Client(); const multiverseMathPublicDatasetShareToken = - "620596ee-570b-4d2b-8c8f-f828adbe5242"; + "cce9c8a9-761a-4756-b159-58ed2640e274"; const sharedExamples = await client.listSharedExamples( multiverseMathPublicDatasetShareToken ); @@ -1123,7 +1123,7 @@ test("clonePublicDataset method can clone a dataset", async () => { const client = new Client(); const datasetName = "multiverse_math_public_testing"; const multiverseMathPublicDatasetURL = - "https://smith.langchain.com/public/620596ee-570b-4d2b-8c8f-f828adbe5242/d"; + "https://beta.smith.langchain.com/public/cce9c8a9-761a-4756-b159-58ed2640e274/d"; try { await client.clonePublicDataset(multiverseMathPublicDatasetURL, { diff --git a/python/tests/integration_tests/test_async_client.py b/python/tests/integration_tests/test_async_client.py index 6338ec2ae..ecd03c87f 100644 --- a/python/tests/integration_tests/test_async_client.py +++ b/python/tests/integration_tests/test_async_client.py @@ -63,7 +63,7 @@ async def wait_for(condition, timeout=10): @pytest.fixture async def async_client(): ls_utils.get_env_var.cache_clear() - client = AsyncClient(api_url="https://api.smith.langchain.com") + client = AsyncClient() yield client await client.aclose() diff --git a/python/tests/integration_tests/test_client.py b/python/tests/integration_tests/test_client.py index 059111e25..d5a39ab42 100644 --- a/python/tests/integration_tests/test_client.py +++ b/python/tests/integration_tests/test_client.py @@ -43,7 +43,7 @@ def wait_for( @pytest.fixture def langchain_client() -> Client: get_env_var.cache_clear() - return Client(api_url="https://api.smith.langchain.com") + return Client() def test_datasets(langchain_client: Client) -> None: @@ -356,11 +356,9 @@ def test_persist_update_run(langchain_client: Client) -> None: @pytest.mark.parametrize("uri", ["http://localhost:1981", "http://api.langchain.minus"]) -def test_error_surfaced_invalid_uri(monkeypatch: pytest.MonkeyPatch, uri: str) -> None: +def test_error_surfaced_invalid_uri(uri: str) -> None: get_env_var.cache_clear() - monkeypatch.setenv("LANGCHAIN_ENDPOINT", uri) - monkeypatch.setenv("LANGCHAIN_API_KEY", "test") - client = Client() + client = Client(api_url=uri, api_key="test") # expect connect error with pytest.raises(LangSmithConnectionError): client.create_run("My Run", inputs={"text": "hello world"}, run_type="llm") diff --git a/python/tests/integration_tests/test_llm_evaluator.py b/python/tests/integration_tests/test_llm_evaluator.py index 28b742096..1fffde4e9 100644 --- a/python/tests/integration_tests/test_llm_evaluator.py +++ b/python/tests/integration_tests/test_llm_evaluator.py @@ -147,7 +147,7 @@ def test_from_model() -> None: async def test_evaluate() -> None: client = Client() client.clone_public_dataset( - "https://smith.langchain.com/public/419dcab2-1d66-4b94-8901-0357ead390df/d" + "https://beta.smith.langchain.com/public/06785303-0f70-4466-b637-f23d38c0f28e/d" ) dataset_name = "Evaluate Examples" diff --git a/python/tests/integration_tests/wrappers/test_openai.py b/python/tests/integration_tests/wrappers/test_openai.py index 36fdf4ac9..d926583fa 100644 --- a/python/tests/integration_tests/wrappers/test_openai.py +++ b/python/tests/integration_tests/wrappers/test_openai.py @@ -12,12 +12,12 @@ from langsmith.wrappers import wrap_openai -@mock.patch("langsmith.client.requests.Session") @pytest.mark.parametrize("stream", [False, True]) -def test_chat_sync_api(mock_session: mock.MagicMock, stream: bool): +def test_chat_sync_api(stream: bool): import openai # noqa - client = langsmith.Client(session=mock_session()) + mock_session = mock.MagicMock() + client = langsmith.Client(session=mock_session) original_client = openai.Client() patched_client = wrap_openai(openai.Client(), tracing_extra={"client": client}) messages = [{"role": "user", "content": "Say 'foo'"}] @@ -47,16 +47,16 @@ def test_chat_sync_api(mock_session: mock.MagicMock, stream: bool): assert original.choices == patched.choices # Give the thread a chance. time.sleep(0.01) - for call in mock_session.return_value.request.call_args_list[1:]: + for call in mock_session.request.call_args_list[1:]: assert call[0][0].upper() == "POST" -@mock.patch("langsmith.client.requests.Session") @pytest.mark.parametrize("stream", [False, True]) -async def test_chat_async_api(mock_session: mock.MagicMock, stream: bool): +async def test_chat_async_api(stream: bool): import openai # noqa - client = langsmith.Client(session=mock_session()) + mock_session = mock.MagicMock() + client = langsmith.Client(session=mock_session) original_client = openai.AsyncClient() patched_client = wrap_openai(openai.AsyncClient(), tracing_extra={"client": client}) messages = [{"role": "user", "content": "Say 'foo'"}] @@ -82,16 +82,16 @@ async def test_chat_async_api(mock_session: mock.MagicMock, stream: bool): assert original.choices == patched.choices # Give the thread a chance. time.sleep(0.1) - for call in mock_session.return_value.request.call_args_list[1:]: + for call in mock_session.request.call_args_list[1:]: assert call[0][0].upper() == "POST" -@mock.patch("langsmith.client.requests.Session") @pytest.mark.parametrize("stream", [False, True]) -def test_completions_sync_api(mock_session: mock.MagicMock, stream: bool): +def test_completions_sync_api(stream: bool): import openai - client = langsmith.Client(session=mock_session()) + mock_session = mock.MagicMock() + client = langsmith.Client(session=mock_session) original_client = openai.Client() patched_client = wrap_openai(openai.Client(), tracing_extra={"client": client}) prompt = ("Say 'Foo' then stop.",) @@ -125,16 +125,16 @@ def test_completions_sync_api(mock_session: mock.MagicMock, stream: bool): assert original.choices == patched.choices # Give the thread a chance. time.sleep(0.1) - for call in mock_session.return_value.request.call_args_list[1:]: + for call in mock_session.request.call_args_list[1:]: assert call[0][0].upper() == "POST" -@mock.patch("langsmith.client.requests.Session") @pytest.mark.parametrize("stream", [False, True]) -async def test_completions_async_api(mock_session: mock.MagicMock, stream: bool): +async def test_completions_async_api(stream: bool): import openai - client = langsmith.Client(session=mock_session()) + mock_session = mock.MagicMock() + client = langsmith.Client(session=mock_session) original_client = openai.AsyncClient() patched_client = wrap_openai( @@ -179,10 +179,10 @@ async def test_completions_async_api(mock_session: mock.MagicMock, stream: bool) # Give the thread a chance. for _ in range(10): time.sleep(0.1) - if mock_session.return_value.request.call_count >= 1: + if mock_session.request.call_count >= 1: break - assert mock_session.return_value.request.call_count >= 1 - for call in mock_session.return_value.request.call_args_list[1:]: + assert mock_session.request.call_count >= 1 + for call in mock_session.request.call_args_list[1:]: assert call[0][0].upper() == "POST" @@ -199,7 +199,7 @@ def __call__(self, run): def _collect_requests(mock_session: mock.MagicMock, filename: str): - mock_requests = mock_session.return_value.request.call_args_list + mock_requests = mock_session.request.call_args_list collected_requests = {} for _ in range(10): time.sleep(0.1) @@ -215,7 +215,7 @@ def _collect_requests(mock_session: mock.MagicMock, filename: str): # thread has finished processing the run if any(event.get("end_time") for event in all_events): break - mock_session.return_value.request.call_args_list.clear() + mock_session.request.call_args_list.clear() if os.environ.get("WRITE_TOKEN_COUNTING_TEST_DATA") == "1": dir_path = Path(__file__).resolve().parent.parent / "test_data" @@ -274,13 +274,13 @@ def _collect_requests(mock_session: mock.MagicMock, filename: str): @pytest.mark.parametrize("test_case", test_cases) -@mock.patch("langsmith.client.requests.Session") -def test_wrap_openai_chat_tokens(mock_session: mock.MagicMock, test_case): +def test_wrap_openai_chat_tokens(test_case): import openai from openai.types.chat import ChatCompletion, ChatCompletionChunk oai_client = openai.Client() - ls_client = langsmith.Client(session=mock_session()) + mock_session = mock.MagicMock() + ls_client = langsmith.Client(session=mock_session) wrapped_oai_client = wrap_openai(oai_client, tracing_extra={"client": ls_client}) collect = Collect() @@ -323,13 +323,13 @@ def test_wrap_openai_chat_tokens(mock_session: mock.MagicMock, test_case): @pytest.mark.asyncio @pytest.mark.parametrize("test_case", test_cases) -@mock.patch("langsmith.client.requests.Session") -async def test_wrap_openai_chat_async_tokens(mock_session: mock.MagicMock, test_case): +async def test_wrap_openai_chat_async_tokens(test_case): import openai from openai.types.chat import ChatCompletion, ChatCompletionChunk oai_client = openai.AsyncClient() - ls_client = langsmith.Client(session=mock_session()) + mock_session = mock.MagicMock() + ls_client = langsmith.Client(session=mock_session) wrapped_oai_client = wrap_openai(oai_client, tracing_extra={"client": ls_client}) collect = Collect()