diff --git a/.github/workflows/autoblocks-simulations.yml b/.github/workflows/autoblocks-simulations.yml index e83586d..77f9cd8 100644 --- a/.github/workflows/autoblocks-simulations.yml +++ b/.github/workflows/autoblocks-simulations.yml @@ -9,6 +9,12 @@ env: POETRY_VERSION: "1.5.1" PYTHON_VERSION: "3.11" + # Use the simulation ingestion key so that Autoblocks knows we're sending simulated events + AUTOBLOCKS_INGESTION_KEY: ${{ secrets.AUTOBLOCKS_SIMULATION_INGESTION_KEY }} + + # Any other environment variables the application needs to run + OPENAI_API_KEY: ${{ secrets.DEMO_OPENAI_API_KEY }} + jobs: autoblocks-simulations: runs-on: ubuntu-latest @@ -32,25 +38,19 @@ jobs: - name: Install dependencies run: poetry install + - name: Run tests + run: poetry run pytest + - name: Start the app run: poetry run start & - env: - # Use the simulation ingestion key so that Autoblocks knows we're sending simulated events - AUTOBLOCKS_INGESTION_KEY: ${{ secrets.AUTOBLOCKS_SIMULATION_INGESTION_KEY }} - - # Any other environment variables the application needs to run - OPENAI_API_KEY: ${{ secrets.DEMO_OPENAI_API_KEY }} - name: Wait for the app to be ready run: | while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' http://localhost:5000/health)" != "200" ]]; do sleep 1; done - - name: Run simulation with static test cases - run: poetry run simulation-static - - name: Run simulation with production test cases run: poetry run simulation-production-replay env: # Production events are fetched from the Autoblocks API, - # so we need the API key to authenticate + # so we need the API key to authenticate. AUTOBLOCKS_API_KEY: ${{ secrets.AUTOBLOCKS_API_KEY }} diff --git a/README.md b/README.md index 54763dc..2c958af 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ This repository contains an example project that integrates Autoblocks Simulatio via GitHub Actions. See the [documentation](https://docs.autoblocks.ai/guides/simulations) for more information. -## Instructions +## Instructions for running locally ### 1. Install dependencies @@ -12,31 +12,34 @@ See the [documentation](https://docs.autoblocks.ai/guides/simulations) for more poetry install ``` -### 2. Start the application +### 2. Run the tests -Start the application with your simulation ingestion key, -a simulation id to uniquely identify your simulation run, -and any other environment variables needed to run your application: +Set the `AUTOBLOCKS_INGESTION_KEY` environment variable to your simulation ingestion key before running the tests +so that any events sent during the test run are sent as simulated events. ```bash AUTOBLOCKS_INGESTION_KEY= \ -AUTOBLOCKS_SIMULATION_ID=$(date +%Y%m%d%H%M%S) \ OPENAI_API_KEY= \ -poetry run start +poetry run pytest ``` -### 3. Run the simulation +[View your simulation](https://app.autoblocks.ai/simulations) -In another terminal, run either: +### 3. Run a simulation with production events -* `simulation-static`, which will replay a static set of test cases against your application: +To run a simulation that replays production events, first start the application: ```bash -poetry run simulation-static +AUTOBLOCKS_INGESTION_KEY= \ +AUTOBLOCKS_SIMULATION_ID=$(date +%Y%m%d%H%M%S) \ +OPENAI_API_KEY= \ +poetry run start ``` -* `simulation-production-replay`, which will replay a set of production events fetched from the Autoblocks API: +Then, in a separate terminal, run the simulation: ```bash AUTOBLOCKS_API_KEY= poetry run simulation-production-replay ``` + +[View your simulation](https://app.autoblocks.ai/simulations) diff --git a/demo_app/app.py b/demo_app/app.py index 64cbd00..ee317af 100644 --- a/demo_app/app.py +++ b/demo_app/app.py @@ -1,13 +1,10 @@ import uuid -from autoblocks.tracer import AutoblocksTracer from flask import Flask from flask import request from demo_app import bot from demo_app.settings import AUTOBLOCKS_SIMULATION_TRACE_ID_HEADER_NAME -from demo_app.settings import REQUEST_PAYLOAD_MESSAGE -from demo_app.settings import env app = Flask(__name__) @@ -29,19 +26,9 @@ def main(): # but in a simulation scenario we use the trace id passed in via the simulation trace id header trace_id = request.headers.get(AUTOBLOCKS_SIMULATION_TRACE_ID_HEADER_NAME) or str(uuid.uuid4()) - autoblocks = AutoblocksTracer( - env.AUTOBLOCKS_INGESTION_KEY, - trace_id=trace_id, - properties=dict(source="DEMO_SIMULATIONS"), - ) - autoblocks.send_event(REQUEST_PAYLOAD_MESSAGE, properties=dict(payload=payload)) + output = bot.get_response(trace_id, query) - output = bot.get_response(autoblocks, query) - - response = {"output": output} - autoblocks.send_event("request.response", properties=dict(response=response)) - - return response + return dict(output=output) def start(): diff --git a/demo_app/bot.py b/demo_app/bot.py index 6c6300c..22012b8 100644 --- a/demo_app/bot.py +++ b/demo_app/bot.py @@ -6,7 +6,13 @@ from demo_app.settings import env -def get_response(autoblocks: AutoblocksTracer, query: str) -> str: +def get_response(trace_id: str, query: str) -> str: + autoblocks = AutoblocksTracer( + env.AUTOBLOCKS_INGESTION_KEY, + trace_id=trace_id, + properties=dict(source="DEMO_SIMULATIONS"), + ) + ai = AIChat( api_key=env.OPENAI_API_KEY, model="gpt-3.5-turbo", diff --git a/demo_app/settings.py b/demo_app/settings.py index 3ec026d..cec0830 100644 --- a/demo_app/settings.py +++ b/demo_app/settings.py @@ -4,9 +4,9 @@ # the event being handled during the simulation. AUTOBLOCKS_SIMULATION_TRACE_ID_HEADER_NAME = "x-autoblocks-simulation-trace-id" -# The message for the request.payload event, pulled into a variable here +# The message for the user input event, pulled into a variable here # so that it's kept in sync between sending the events and replaying them. -REQUEST_PAYLOAD_MESSAGE = "request.payload" +USER_QUERY_MESSAGE = "user.query" # Environment variables diff --git a/demo_app/simulations.py b/demo_app/simulations.py index 1a9d89f..a97fe62 100644 --- a/demo_app/simulations.py +++ b/demo_app/simulations.py @@ -8,28 +8,10 @@ from autoblocks.api.models import TraceFilterOperator from demo_app.settings import AUTOBLOCKS_SIMULATION_TRACE_ID_HEADER_NAME -from demo_app.settings import REQUEST_PAYLOAD_MESSAGE +from demo_app.settings import USER_QUERY_MESSAGE from demo_app.settings import env -def static(): - """ - Test a static set of events against the locally-running app. - """ - for trace_id, query in [ - ("san-francisco-tourist-attractions", "San Francisco tourist attractions"), - ("paris-tourist-attractions", "Paris tourist attractions"), - ("lombard-street", "Lombard Street"), - ("eiffel-tower", "Eiffel Tower"), - ]: - print(f"Testing static event {trace_id} - {query}") - requests.post( - "http://localhost:5000", - json={"query": query}, - headers={AUTOBLOCKS_SIMULATION_TRACE_ID_HEADER_NAME: trace_id}, - ) - - def production_replay(): """ Replays production events fetched from the Autoblocks API against the locally-running app. @@ -46,7 +28,7 @@ def production_replay(): EventFilter( key=SystemEventFilterKey.MESSAGE, operator=EventFilterOperator.EQUALS, - value=REQUEST_PAYLOAD_MESSAGE, + value=USER_QUERY_MESSAGE, ), ], ), @@ -54,7 +36,7 @@ def production_replay(): ) for trace in page.traces: for event in trace.events: - if event.message == REQUEST_PAYLOAD_MESSAGE: + if event.message == USER_QUERY_MESSAGE: print(f"Replaying past event {event}") # The original payload diff --git a/poetry.lock b/poetry.lock index fece1c6..23d7aad 100644 --- a/poetry.lock +++ b/poetry.lock @@ -33,13 +33,13 @@ trio = ["trio (<0.22)"] [[package]] name = "autoblocksai" -version = "0.0.6" +version = "0.0.7" description = "" optional = false python-versions = ">=3.8,<4.0" files = [ - {file = "autoblocksai-0.0.6-py3-none-any.whl", hash = "sha256:d12af551ec5983781a684e474952b1bac0cc69afd7f3b35b21719749290896ec"}, - {file = "autoblocksai-0.0.6.tar.gz", hash = "sha256:23860611d9c59f16f31f8be273c33909357e3a2b2c633a22a0f4e379f667f5a6"}, + {file = "autoblocksai-0.0.7-py3-none-any.whl", hash = "sha256:65ba4fa6f2a4c2b7e828e273de946639fcde83d5bf562deed1c84ac9ec716f5d"}, + {file = "autoblocksai-0.0.7.tar.gz", hash = "sha256:603b32a56bc974db6a344247e10c0f26e23abddba3c33df7969ba915040962e1"}, ] [package.dependencies] @@ -329,6 +329,17 @@ files = [ {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, ] +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + [[package]] name = "itsdangerous" version = "2.1.2" @@ -520,6 +531,17 @@ files = [ {file = "orjson-3.9.2.tar.gz", hash = "sha256:24257c8f641979bf25ecd3e27251b5cc194cdd3a6e96004aac8446f5e63d9664"}, ] +[[package]] +name = "packaging" +version = "23.1" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, + {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, +] + [[package]] name = "platformdirs" version = "3.9.1" @@ -535,6 +557,21 @@ files = [ docs = ["furo (>=2023.5.20)", "proselint (>=0.13)", "sphinx (>=7.0.1)", "sphinx-autodoc-typehints (>=1.23,!=1.23.4)"] test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.3.1)", "pytest-cov (>=4.1)", "pytest-mock (>=3.10)"] +[[package]] +name = "pluggy" +version = "1.3.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.3.0-py3-none-any.whl", hash = "sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7"}, + {file = "pluggy-1.3.0.tar.gz", hash = "sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + [[package]] name = "pre-commit" version = "3.3.3" @@ -714,6 +751,26 @@ files = [ [package.extras] plugins = ["importlib-metadata"] +[[package]] +name = "pytest" +version = "7.4.0" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-7.4.0-py3-none-any.whl", hash = "sha256:78bf16451a2eb8c7a2ea98e32dc119fd2aa758f1d5d66dbf0a59d69a3969df32"}, + {file = "pytest-7.4.0.tar.gz", hash = "sha256:b4bf8c45bd59934ed84001ad51e11b4ee40d40a1229d2c79f9c592b0a3f6bd8a"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" + +[package.extras] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + [[package]] name = "python-dateutil" version = "2.8.2" @@ -969,4 +1026,4 @@ watchdog = ["watchdog (>=2.3)"] [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "fe5342ab6c115173e8f4f315a71d1341fa6664988f9806242231c84edf3c275b" +content-hash = "0a32aa36899e4e6af8ea4f26a27dcd588525b78f2857eff3511609d84bee3cf6" diff --git a/pyproject.toml b/pyproject.toml index e2ec6d4..4d5c71d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,10 +13,11 @@ requests = "^2.31.0" flask = "^2.3.2" pydantic-settings = "^2.0.2" simpleaichat = "^0.2.2" -autoblocksai = "0.0.6" +autoblocksai = "0.0.7" [tool.poetry.group.dev.dependencies] pre-commit = "^3.3.3" +pytest = "^7.4.0" [build-system] requires = ["poetry-core"] @@ -35,5 +36,4 @@ known-first-party = ["demo_app"] [tool.poetry.scripts] start = "demo_app.app:start" -simulation-static = "demo_app.simulations:static" simulation-production-replay = "demo_app.simulations:production_replay" diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..87d3559 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,11 @@ +import os +from datetime import datetime + +import pytest + + +@pytest.fixture(scope="session", autouse=True) +def set_autoblocks_simulation_id(): + os.environ["AUTOBLOCKS_SIMULATION_ID"] = "pytest-" + datetime.now().strftime("%Y%m%d-%H%M%S") + yield + del os.environ["AUTOBLOCKS_SIMULATION_ID"] diff --git a/tests/test_bot.py b/tests/test_bot.py new file mode 100644 index 0000000..8ad9f13 --- /dev/null +++ b/tests/test_bot.py @@ -0,0 +1,17 @@ +import pytest + +from demo_app import bot + + +@pytest.mark.parametrize( + "trace_id,query,expected_output", + [ + ("san-francisco-tourist-attractions", "San Francisco tourist attractions", "Lombard"), + ("paris-tourist-attractions", "Paris tourist attractions", "Eiffel"), + ("lombard-street", "Lombard Street", "San Francisco"), + ("eiffel-tower", "Eiffel Tower", "Paris"), + ], +) +def test_bot(trace_id: str, query: str, expected_output: str): + response = bot.get_response(trace_id, query) + assert expected_output in response