diff --git a/.github/workflows/autoblocks-replays-cleanup.yml b/.github/workflows/autoblocks-replays-cleanup.yml deleted file mode 100644 index 3a46833..0000000 --- a/.github/workflows/autoblocks-replays-cleanup.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Autoblocks Replays Cleanup - -on: delete - -permissions: - contents: write - -jobs: - autoblocks-replays-cleanup: - if: github.event.ref_type == 'branch' - - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - with: - # We need the full history to be able to find and delete branches. - fetch-depth: 0 - - - name: Delete Autoblocks replay branches for the deleted branch - run: | - git branch -r --list 'origin/autoblocks-replays/${{ github.event.ref }}/*' \ - | sed 's/origin\///' \ - | xargs git push origin --delete diff --git a/.github/workflows/autoblocks-replays.yml b/.github/workflows/autoblocks-replays.yml index 2a66578..a57c2f2 100644 --- a/.github/workflows/autoblocks-replays.yml +++ b/.github/workflows/autoblocks-replays.yml @@ -5,18 +5,10 @@ on: branches-ignore: - main -permissions: - # This workflow needs to be given contents write permission since it creates branches and commits files - contents: write - # This workflow needs to be given pull requests write permission since it comments on pull requests - pull-requests: write - env: POETRY_VERSION: "1.5.1" PYTHON_VERSION: "3.11" - AUTOBLOCKS_REPLAYS_ENABLED: "true" - jobs: autoblocks-replays: runs-on: ubuntu-latest @@ -33,47 +25,26 @@ jobs: run: curl -sSL https://install.python-poetry.org | python3 - - name: Install dependencies - run: | - poetry config virtualenvs.create true - poetry config virtualenvs.in-project true - poetry install + run: poetry install - name: Start the app run: poetry run start & env: + # Use the replay ingestion key so that Autoblocks knows we're sending replayed events + AUTOBLOCKS_INGESTION_KEY: ${{ secrets.DEMO_AUTOBLOCKS_REPLAY_INGESTION_KEY }} + + # Any other environment variables the application needs to run OPENAI_API_KEY: ${{ secrets.DEMO_OPENAI_API_KEY }} - name: Wait for the app to be ready run: | while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' http://localhost:5000/health)" != "200" ]]; do sleep 1; done - - name: Run Autoblocks replays - uses: autoblocksai/actions/replay@main - with: - # Replay entrypoint settings - replay-method: POST - replay-url: http://localhost:5000 - - # Autoblocks replay view settings - replay-view-id: clkeamsei0001l908cmjjtqrf - replay-num-traces: 3 - - # Configuration for filtering and transforming events for replay - replay-transform-config: | - filters: - message: request.payload - mappers: - query: properties.payload.query - - # Filter out properties that are expected to be different on each - # run to prevent the replay diffs from containing unnecessary noise - property-filter-config: | - ai.intermediate.response: - - response.id - - response.created +# - name: Run static replays +# run: poetry run replay-static - # Autoblocks API key - autoblocks-api-key: ${{ secrets.DEMO_AUTOBLOCKS_API_KEY }} - - # GitHub token - github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Run dynamic replays + run: poetry run replay-dynamic --view-id clkeamsei0001l908cmjjtqrf --num-traces 3 + env: + # Dynamic replays are fetched from the Autoblocks API + AUTOBLOCKS_API_KEY: ${{ secrets.DEMO_AUTOBLOCKS_API_KEY }} diff --git a/README.md b/README.md index 84f5f76..a31c9e8 100644 --- a/README.md +++ b/README.md @@ -63,58 +63,26 @@ Autoblocks would easily surface this change during the code review process: ## Replaying Locally -Start the application with replays enabled: +Start the application with your replay ingestion key, a replay id to uniquely identify your replay run, and any other environment variables needed to run your application: ```bash -AUTOBLOCKS_REPLAYS_ENABLED=true poetry run start +AUTOBLOCKS_INGESTION_KEY= \ +AUTOBLOCKS_REPLAY_ID=$(date +%Y%m%d%H%M%S) \ +OPENAI_API_KEY= \ +poetry run start ``` -In another terminal, set the `AUTOBLOCKS_API_KEY` environment variable: +In another terminal, run either: -```bash -export AUTOBLOCKS_API_KEY=my-api-key -``` - -Then, replay traces from a view: +* `replay-static`, which will replay a static set of test cases against your application ```bash -poetry run replay --view-id clkeamsei0001l908cmjjtqrf --num-traces 3 -``` - +poetry run replay-static ``` -Your replay id is 2023-07-23_09-36-36 -Replaying event {'id': 'geepag24zence2kbe0ppagt9', 'traceId': '7cb3ec98-b320-4e62-9a51-b15d0218ae4c', 'timestamp': '2023-07-22T18:32:51.862Z', 'message': 'request.payload', 'properties': {'payload': {'query': 'What are all of the airports in London?'}, 'source': 'DEMO_REPLAYS'}} -``` - -The original and replayed traces are written to the `autoblocks-replays/` folder with the structure: - -``` -autoblocks-replays/ - / - / - original/ - -.json - replayed/ - -.json -``` +* `replay-dynamic`, which will replay a set of real, past events fetched from the Autoblocks API: -Screenshot 2023-07-23 at 10 06 39 AM - -Inspect the original vs replayed output manually or use a CLI tool like `diff` to surface differences: - -``` -diff \ - autoblocks-replays/2023-07-23_09-36-36/7cb3ec98-b320-4e62-9a51-b15d0218ae4c/original \ - autoblocks-replays/2023-07-23_09-36-36/7cb3ec98-b320-4e62-9a51-b15d0218ae4c/replayed +```bash +AUTOBLOCKS_API_KEY= \ +poetry run replay-dynamic --view-id clkeamsei0001l908cmjjtqrf --num-traces 3 ``` - -Screenshot 2023-07-22 at 2 39 47 PM - -## Replaying in GitHub Actions - -Use the [`autoblocksai/actions/replay`](https://github.com/autoblocksai/actions/tree/main/replay) action to replay events in a GitHub Actions workflow. This is similar to replaying events locally but allows you to automate replays in your CI workflow and view results in the GitHub UI. - -The action will leave a comment on your pull request with a summary of the replay results: - -Screenshot 2023-07-26 at 6 50 31 PM diff --git a/demo_replays/app.py b/demo_replays/app.py index fbc426d..c671835 100644 --- a/demo_replays/app.py +++ b/demo_replays/app.py @@ -29,7 +29,9 @@ def main(): trace_id = request.headers.get(AUTOBLOCKS_REPLAY_TRACE_ID_HEADER_NAME) or str(uuid.uuid4()) autoblocks = AutoblocksTracer( - env.AUTOBLOCKS_INGESTION_KEY, trace_id=trace_id, properties=dict(source="DEMO_REPLAYS") + env.AUTOBLOCKS_INGESTION_KEY, + trace_id=trace_id, + properties=dict(source="DEMO_REPLAYS"), ) autoblocks.send_event("request.payload", properties=dict(payload=payload)) diff --git a/demo_replays/replay.py b/demo_replays/replay.py index 0e1f60a..cae04aa 100644 --- a/demo_replays/replay.py +++ b/demo_replays/replay.py @@ -1,8 +1,7 @@ import argparse import requests -from autoblocks.replays import replay_events_from_view -from autoblocks.replays import start_replay +from autoblocks.api.client import AutoblocksAPIClient from demo_replays.settings import AUTOBLOCKS_REPLAY_TRACE_ID_HEADER_NAME from demo_replays.settings import env @@ -12,8 +11,6 @@ def static(): """ Replays a static set of events against the locally-running app. """ - start_replay() - for trace_id, query in [ ("sf", "San Francisco tourist attractions"), ("paris", "Paris tourist attractions"), @@ -42,22 +39,20 @@ def dynamic(): ) args = parser.parse_args() - start_replay() - - for event in replay_events_from_view( - api_key=env.AUTOBLOCKS_API_KEY, - view_id=args.view_id, - num_traces=args.num_traces, - ): - if event.message == "request.payload": - print(f"Replaying past event {event}") - - # The original payload - payload = event.properties["payload"] - - # Replay the request - requests.post( - "http://localhost:5000", - json=payload, - headers={AUTOBLOCKS_REPLAY_TRACE_ID_HEADER_NAME: event.trace_id}, - ) + ab_client = AutoblocksAPIClient(env.AUTOBLOCKS_API_KEY) + + resp = ab_client.get_traces_from_view(view_id=args.view_id, page_size=args.num_traces) + for trace in resp.traces: + for event in trace.events: + if event.message == "request.payload": + print(f"Replaying past event {event}") + + # The original payload + payload = event.properties["payload"] + + # Replay the request + requests.post( + "http://localhost:5000", + json=payload, + headers={AUTOBLOCKS_REPLAY_TRACE_ID_HEADER_NAME: event.trace_id}, + ) diff --git a/poetry.lock b/poetry.lock index afcce4e..ff228c3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -33,13 +33,13 @@ trio = ["trio (<0.22)"] [[package]] name = "autoblocksai" -version = "0.0.1" +version = "0.0.2" description = "" optional = false python-versions = ">=3.8,<4.0" files = [ - {file = "autoblocksai-0.0.1-py3-none-any.whl", hash = "sha256:353f12235dab4b7400b40f9e0f47fd448f6f7b59f23bd94df4a2b8dd04dd850b"}, - {file = "autoblocksai-0.0.1.tar.gz", hash = "sha256:a33b3592dd5204a7969d87ea6348df85ef887dfe780c832d4a07faf282eb216c"}, + {file = "autoblocksai-0.0.2-py3-none-any.whl", hash = "sha256:53faee4b35a5687b5984923234b89c842d883eeea1d7121d4616b86eea1d73d8"}, + {file = "autoblocksai-0.0.2.tar.gz", hash = "sha256:0a8901ebb19160cea28e58f9b1dda274c0463907d5b6477163de36e1f7f4a23b"}, ] [package.dependencies] @@ -969,4 +969,4 @@ watchdog = ["watchdog (>=2.3)"] [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "3f8570fdf16d701a88304bfb9aa1b6bc733c981cf61d72dd99ff65c6ae1129ec" +content-hash = "8e38290af25def0da4d18e14214ddd6728077387e27e944151935f4d3de7c89b" diff --git a/pyproject.toml b/pyproject.toml index 6f8fd8d..f14b0ab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ requests = "^2.31.0" flask = "^2.3.2" pydantic-settings = "^2.0.2" simpleaichat = "^0.2.2" -autoblocksai = "^0.0.1" +autoblocksai = "0.0.2" [tool.poetry.group.dev.dependencies] pre-commit = "^3.3.3" @@ -35,5 +35,5 @@ known-first-party = ["demo_replays"] [tool.poetry.scripts] start = "demo_replays.app:start" -replay = "demo_replays.replay:dynamic" replay-static = "demo_replays.replay:static" +replay-dynamic = "demo_replays.replay:dynamic"