diff --git a/.github/workflows/autoblocks-replays-cleanup.yml b/.github/workflows/autoblocks-replays-cleanup.yml deleted file mode 100644 index 3a46833..0000000 --- a/.github/workflows/autoblocks-replays-cleanup.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Autoblocks Replays Cleanup - -on: delete - -permissions: - contents: write - -jobs: - autoblocks-replays-cleanup: - if: github.event.ref_type == 'branch' - - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - with: - # We need the full history to be able to find and delete branches. - fetch-depth: 0 - - - name: Delete Autoblocks replay branches for the deleted branch - run: | - git branch -r --list 'origin/autoblocks-replays/${{ github.event.ref }}/*' \ - | sed 's/origin\///' \ - | xargs git push origin --delete diff --git a/.github/workflows/autoblocks-replays.yml b/.github/workflows/autoblocks-replays.yml index 2a66578..5bc258c 100644 --- a/.github/workflows/autoblocks-replays.yml +++ b/.github/workflows/autoblocks-replays.yml @@ -1,22 +1,11 @@ name: Autoblocks Replays -on: - push: - branches-ignore: - - main - -permissions: - # This workflow needs to be given contents write permission since it creates branches and commits files - contents: write - # This workflow needs to be given pull requests write permission since it comments on pull requests - pull-requests: write +on: push env: POETRY_VERSION: "1.5.1" PYTHON_VERSION: "3.11" - AUTOBLOCKS_REPLAYS_ENABLED: "true" - jobs: autoblocks-replays: runs-on: ubuntu-latest @@ -33,47 +22,26 @@ jobs: run: curl -sSL https://install.python-poetry.org | python3 - - name: Install dependencies - run: | - poetry config virtualenvs.create true - poetry config virtualenvs.in-project true - poetry install + run: poetry install - name: Start the app run: poetry run start & env: + # Use the replay ingestion key so that Autoblocks knows we're sending replayed events + AUTOBLOCKS_INGESTION_KEY: ${{ secrets.DEMO_AUTOBLOCKS_REPLAY_INGESTION_KEY }} + + # Any other environment variables the application needs to run OPENAI_API_KEY: ${{ secrets.DEMO_OPENAI_API_KEY }} - name: Wait for the app to be ready run: | while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' http://localhost:5000/health)" != "200" ]]; do sleep 1; done - - name: Run Autoblocks replays - uses: autoblocksai/actions/replay@main - with: - # Replay entrypoint settings - replay-method: POST - replay-url: http://localhost:5000 - - # Autoblocks replay view settings - replay-view-id: clkeamsei0001l908cmjjtqrf - replay-num-traces: 3 - - # Configuration for filtering and transforming events for replay - replay-transform-config: | - filters: - message: request.payload - mappers: - query: properties.payload.query - - # Filter out properties that are expected to be different on each - # run to prevent the replay diffs from containing unnecessary noise - property-filter-config: | - ai.intermediate.response: - - response.id - - response.created + - name: Run static replays + run: poetry run replay-static - # Autoblocks API key - autoblocks-api-key: ${{ secrets.DEMO_AUTOBLOCKS_API_KEY }} - - # GitHub token - github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Run dynamic replays + run: poetry run replay-dynamic --view-id clkeamsei0001l908cmjjtqrf --num-traces 3 + env: + # Dynamic replays are fetched from the Autoblocks API + AUTOBLOCKS_API_KEY: ${{ secrets.DEMO_AUTOBLOCKS_API_KEY }} diff --git a/README.md b/README.md index 5a549e0..a31c9e8 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ This repository demonstrates how to integrate LLM chain replays into your code review process. It contains: -* a [`simpleaichat`](https://github.com/minimaxir/simpleaichat) application that uses the [Autoblocks Python SDK](https://github.com/autoblocksai/python-sdk) to send events to the Autoblocks API +* a [`simpleaichat`](https://github.com/minimaxir/simpleaichat) application that uses the [Autoblocks Python SDK](https://pypi.org/project/autoblocksai/) to send events to the Autoblocks API * a GitHub Actions workflow that **replays** real, past events from end users on every push to a feature branch With our GitHub integration enabled, your teammates are not only reviewing your code, but also the impact that code will have on your LLM chains, and therefore your end users. @@ -63,58 +63,26 @@ Autoblocks would easily surface this change during the code review process: ## Replaying Locally -Start the application with replays enabled: +Start the application with your replay ingestion key, a replay id to uniquely identify your replay run, and any other environment variables needed to run your application: ```bash -AUTOBLOCKS_REPLAYS_ENABLED=true poetry run start +AUTOBLOCKS_INGESTION_KEY= \ +AUTOBLOCKS_REPLAY_ID=$(date +%Y%m%d%H%M%S) \ +OPENAI_API_KEY= \ +poetry run start ``` -In another terminal, set the `AUTOBLOCKS_API_KEY` environment variable: +In another terminal, run either: -```bash -export AUTOBLOCKS_API_KEY=my-api-key -``` - -Then replay traces from a view: +* `replay-static`, which will replay a static set of test cases against your application ```bash -poetry run replay --view-id clkeamsei0001l908cmjjtqrf --num-traces 3 -``` - +poetry run replay-static ``` -Your replay id is 2023-07-23_09-36-36 -Replaying event {'id': 'geepag24zence2kbe0ppagt9', 'traceId': '7cb3ec98-b320-4e62-9a51-b15d0218ae4c', 'timestamp': '2023-07-22T18:32:51.862Z', 'message': 'request.payload', 'properties': {'payload': {'query': 'What are all of the airports in London?'}, 'source': 'DEMO_REPLAYS'}} -``` - -The original and replayed traces are written to the `autoblocks-replays/` folder with the structure: - -``` -autoblocks-replays/ - / - / - original/ - -.json - replayed/ - -.json -``` +* `replay-dynamic`, which will replay a set of real, past events fetched from the Autoblocks API: -Screenshot 2023-07-23 at 10 06 39 AM - -Inspect the original vs replayed output manually or use a CLI tool like `diff` to surface differences: - -``` -diff \ - autoblocks-replays/2023-07-23_09-36-36/7cb3ec98-b320-4e62-9a51-b15d0218ae4c/original \ - autoblocks-replays/2023-07-23_09-36-36/7cb3ec98-b320-4e62-9a51-b15d0218ae4c/replayed +```bash +AUTOBLOCKS_API_KEY= \ +poetry run replay-dynamic --view-id clkeamsei0001l908cmjjtqrf --num-traces 3 ``` - -Screenshot 2023-07-22 at 2 39 47 PM - -## Replaying in GitHub Actions - -Use the [`autoblocksai/actions/replay`](https://github.com/autoblocksai/actions/tree/main/replay) action to replay events in a GitHub Actions workflow. This is similar to replaying events locally but allows you to automate replays in your CI workflow and view results in the GitHub UI. - -The action will leave a comment on your pull request with a summary of the replay results: - -Screenshot 2023-07-26 at 6 50 31 PM diff --git a/demo_replays/app.py b/demo_replays/app.py index fbc426d..c671835 100644 --- a/demo_replays/app.py +++ b/demo_replays/app.py @@ -29,7 +29,9 @@ def main(): trace_id = request.headers.get(AUTOBLOCKS_REPLAY_TRACE_ID_HEADER_NAME) or str(uuid.uuid4()) autoblocks = AutoblocksTracer( - env.AUTOBLOCKS_INGESTION_KEY, trace_id=trace_id, properties=dict(source="DEMO_REPLAYS") + env.AUTOBLOCKS_INGESTION_KEY, + trace_id=trace_id, + properties=dict(source="DEMO_REPLAYS"), ) autoblocks.send_event("request.payload", properties=dict(payload=payload)) diff --git a/demo_replays/replay.py b/demo_replays/replay.py index 0e1f60a..58c47e5 100644 --- a/demo_replays/replay.py +++ b/demo_replays/replay.py @@ -1,8 +1,7 @@ import argparse import requests -from autoblocks.replays import replay_events_from_view -from autoblocks.replays import start_replay +from autoblocks.api.client import AutoblocksAPIClient from demo_replays.settings import AUTOBLOCKS_REPLAY_TRACE_ID_HEADER_NAME from demo_replays.settings import env @@ -12,13 +11,11 @@ def static(): """ Replays a static set of events against the locally-running app. """ - start_replay() - for trace_id, query in [ - ("sf", "San Francisco tourist attractions"), - ("paris", "Paris tourist attractions"), - ("lombard", "Lombard Street"), - ("eiffel", "Eiffel Tower"), + ("san-francisco-tourist-attractions", "San Francisco tourist attractions"), + ("paris-tourist-attractions", "Paris tourist attractions"), + ("lombard-stree", "Lombard Street"), + ("eiffel-tower", "Eiffel Tower"), ]: print(f"Testing static event {trace_id} - {query}") requests.post( @@ -42,22 +39,20 @@ def dynamic(): ) args = parser.parse_args() - start_replay() - - for event in replay_events_from_view( - api_key=env.AUTOBLOCKS_API_KEY, - view_id=args.view_id, - num_traces=args.num_traces, - ): - if event.message == "request.payload": - print(f"Replaying past event {event}") - - # The original payload - payload = event.properties["payload"] - - # Replay the request - requests.post( - "http://localhost:5000", - json=payload, - headers={AUTOBLOCKS_REPLAY_TRACE_ID_HEADER_NAME: event.trace_id}, - ) + ab_client = AutoblocksAPIClient(env.AUTOBLOCKS_API_KEY) + + resp = ab_client.get_traces_from_view(view_id=args.view_id, page_size=args.num_traces) + for trace in resp.traces: + for event in trace.events: + if event.message == "request.payload": + print(f"Replaying past event {event}") + + # The original payload + payload = event.properties["payload"] + + # Replay the request + requests.post( + "http://localhost:5000", + json=payload, + headers={AUTOBLOCKS_REPLAY_TRACE_ID_HEADER_NAME: event.trace_id}, + ) diff --git a/poetry.lock b/poetry.lock index afcce4e..ff228c3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -33,13 +33,13 @@ trio = ["trio (<0.22)"] [[package]] name = "autoblocksai" -version = "0.0.1" +version = "0.0.2" description = "" optional = false python-versions = ">=3.8,<4.0" files = [ - {file = "autoblocksai-0.0.1-py3-none-any.whl", hash = "sha256:353f12235dab4b7400b40f9e0f47fd448f6f7b59f23bd94df4a2b8dd04dd850b"}, - {file = "autoblocksai-0.0.1.tar.gz", hash = "sha256:a33b3592dd5204a7969d87ea6348df85ef887dfe780c832d4a07faf282eb216c"}, + {file = "autoblocksai-0.0.2-py3-none-any.whl", hash = "sha256:53faee4b35a5687b5984923234b89c842d883eeea1d7121d4616b86eea1d73d8"}, + {file = "autoblocksai-0.0.2.tar.gz", hash = "sha256:0a8901ebb19160cea28e58f9b1dda274c0463907d5b6477163de36e1f7f4a23b"}, ] [package.dependencies] @@ -969,4 +969,4 @@ watchdog = ["watchdog (>=2.3)"] [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "3f8570fdf16d701a88304bfb9aa1b6bc733c981cf61d72dd99ff65c6ae1129ec" +content-hash = "8e38290af25def0da4d18e14214ddd6728077387e27e944151935f4d3de7c89b" diff --git a/pyproject.toml b/pyproject.toml index 6f8fd8d..f14b0ab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ requests = "^2.31.0" flask = "^2.3.2" pydantic-settings = "^2.0.2" simpleaichat = "^0.2.2" -autoblocksai = "^0.0.1" +autoblocksai = "0.0.2" [tool.poetry.group.dev.dependencies] pre-commit = "^3.3.3" @@ -35,5 +35,5 @@ known-first-party = ["demo_replays"] [tool.poetry.scripts] start = "demo_replays.app:start" -replay = "demo_replays.replay:dynamic" replay-static = "demo_replays.replay:static" +replay-dynamic = "demo_replays.replay:dynamic"