Merge branch 'main' into change-temp

autoblocksai · Aug 17, 2023 · 4d691b3 · 4d691b3
2 parents 600e83c + 9896e2b
commit 4d691b3
Show file tree

Hide file tree

Showing 7 changed files with 57 additions and 149 deletions.
diff --git a/.github/workflows/autoblocks-replays-cleanup.yml b/.github/workflows/autoblocks-replays-cleanup.yml
diff --git a/.github/workflows/autoblocks-replays.yml b/.github/workflows/autoblocks-replays.yml
@@ -1,22 +1,11 @@
 name: Autoblocks Replays
 
-on:
-  push:
-    branches-ignore:
-      - main
-
-permissions:
-  # This workflow needs to be given contents write permission since it creates branches and commits files
-  contents: write
-  # This workflow needs to be given pull requests write permission since it comments on pull requests
-  pull-requests: write
+on: push
 
 env:
   POETRY_VERSION: "1.5.1"
   PYTHON_VERSION: "3.11"
 
-  AUTOBLOCKS_REPLAYS_ENABLED: "true"
-
 jobs:
   autoblocks-replays:
     runs-on: ubuntu-latest
@@ -33,47 +22,26 @@ jobs:
         run: curl -sSL https://install.python-poetry.org | python3 -
 
       - name: Install dependencies
-        run: |
-          poetry config virtualenvs.create true
-          poetry config virtualenvs.in-project true
-          poetry install
+        run: poetry install
 
       - name: Start the app
         run: poetry run start &
         env:
+          # Use the replay ingestion key so that Autoblocks knows we're sending replayed events
+          AUTOBLOCKS_INGESTION_KEY: ${{ secrets.DEMO_AUTOBLOCKS_REPLAY_INGESTION_KEY }}
+
+          # Any other environment variables the application needs to run
           OPENAI_API_KEY: ${{ secrets.DEMO_OPENAI_API_KEY }}
 
       - name: Wait for the app to be ready
         run: |
           while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' http://localhost:5000/health)" != "200" ]]; do sleep 1; done
 
-      - name: Run Autoblocks replays
-        uses: autoblocksai/actions/replay@main
-        with:
-          # Replay entrypoint settings
-          replay-method: POST
-          replay-url: http://localhost:5000
-
-          # Autoblocks replay view settings
-          replay-view-id: clkeamsei0001l908cmjjtqrf
-          replay-num-traces: 3
-
-          # Configuration for filtering and transforming events for replay
-          replay-transform-config: |
-            filters:
-              message: request.payload
-            mappers:
-              query: properties.payload.query
-
-          # Filter out properties that are expected to be different on each
-          # run to prevent the replay diffs from containing unnecessary noise
-          property-filter-config: |
-            ai.intermediate.response:
-              - response.id
-              - response.created
+      - name: Run static replays
+        run: poetry run replay-static
 
-          # Autoblocks API key
-          autoblocks-api-key: ${{ secrets.DEMO_AUTOBLOCKS_API_KEY }}
-
-          # GitHub token
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Run dynamic replays
+        run: poetry run replay-dynamic --view-id clkeamsei0001l908cmjjtqrf --num-traces 3
+        env:
+          # Dynamic replays are fetched from the Autoblocks API
+          AUTOBLOCKS_API_KEY: ${{ secrets.DEMO_AUTOBLOCKS_API_KEY }}
diff --git a/README.md b/README.md
@@ -2,7 +2,7 @@
 
 This repository demonstrates how to integrate LLM chain replays into your code review process. It contains:
 
-* a [`simpleaichat`](https://github.com/minimaxir/simpleaichat) application that uses the [Autoblocks Python SDK](https://github.com/autoblocksai/python-sdk) to send events to the Autoblocks API
+* a [`simpleaichat`](https://github.com/minimaxir/simpleaichat) application that uses the [Autoblocks Python SDK](https://pypi.org/project/autoblocksai/) to send events to the Autoblocks API
 * a GitHub Actions workflow that **replays** real, past events from end users on every push to a feature branch
 
 With our GitHub integration enabled, your teammates are not only reviewing your code, but also the impact that code will have on your LLM chains, and therefore your end users.
@@ -63,58 +63,26 @@ Autoblocks would easily surface this change during the code review process:
 
 ## Replaying Locally
 
-Start the application with replays enabled:
+Start the application with your replay ingestion key, a replay id to uniquely identify your replay run, and any other environment variables needed to run your application:
 
 ```bash
-AUTOBLOCKS_REPLAYS_ENABLED=true poetry run start
+AUTOBLOCKS_INGESTION_KEY=<replay-ingestion-key> \
+AUTOBLOCKS_REPLAY_ID=$(date +%Y%m%d%H%M%S) \
+OPENAI_API_KEY=<openai-api-key> \
+poetry run start
 ```
 
-In another terminal, set the `AUTOBLOCKS_API_KEY` environment variable:
+In another terminal, run either:
 
-```bash
-export AUTOBLOCKS_API_KEY=my-api-key
-```
-
-Then replay traces from a view:
+* `replay-static`, which will replay a static set of test cases against your application
 
 ```bash
-poetry run replay --view-id clkeamsei0001l908cmjjtqrf --num-traces 3
-```
-
+poetry run replay-static
 ```
-Your replay id is 2023-07-23_09-36-36
 
-Replaying event {'id': 'geepag24zence2kbe0ppagt9', 'traceId': '7cb3ec98-b320-4e62-9a51-b15d0218ae4c', 'timestamp': '2023-07-22T18:32:51.862Z', 'message': 'request.payload', 'properties': {'payload': {'query': 'What are all of the airports in London?'}, 'source': 'DEMO_REPLAYS'}}
-```
-
-The original and replayed traces are written to the `autoblocks-replays/` folder with the structure:
-
-```
-autoblocks-replays/
-    <replay-id>/
-        <trace-id>/
-            original/
-                <event-number>-<message>.json
-            replayed/
-                <event-number>-<message>.json
-```
+* `replay-dynamic`, which will replay a set of real, past events fetched from the Autoblocks API:
 
-<img width="981" alt="Screenshot 2023-07-23 at 10 06 39 AM" src="https://github.com/autoblocksai/actions/assets/7498009/ce20918f-ce75-4cd3-a5d2-66d3e1a00914">
-
-Inspect the original vs replayed output manually or use a CLI tool like `diff` to surface differences:
-
-```
-diff \
-  autoblocks-replays/2023-07-23_09-36-36/7cb3ec98-b320-4e62-9a51-b15d0218ae4c/original \
-  autoblocks-replays/2023-07-23_09-36-36/7cb3ec98-b320-4e62-9a51-b15d0218ae4c/replayed
+```bash
+AUTOBLOCKS_API_KEY=<autoblocks-api-key> \
+poetry run replay-dynamic --view-id clkeamsei0001l908cmjjtqrf --num-traces 3
 ```
-
-<img width="799" alt="Screenshot 2023-07-22 at 2 39 47 PM" src="https://github.com/autoblocksai/actions/assets/7498009/4279db15-26fc-4346-8e16-1ed060bcd9f7">
-
-## Replaying in GitHub Actions
-
-Use the [`autoblocksai/actions/replay`](https://github.com/autoblocksai/actions/tree/main/replay) action to replay events in a GitHub Actions workflow. This is similar to replaying events locally but allows you to automate replays in your CI workflow and view results in the GitHub UI.
-
-The action will leave a comment on your pull request with a summary of the replay results:
-
-<img width="857" alt="Screenshot 2023-07-26 at 6 50 31 PM" src="https://github.com/autoblocksai/actions/assets/7498009/ebfb31da-af70-45bf-b9b5-5e640a4fa104">
diff --git a/demo_replays/app.py b/demo_replays/app.py
@@ -29,7 +29,9 @@ def main():
     trace_id = request.headers.get(AUTOBLOCKS_REPLAY_TRACE_ID_HEADER_NAME) or str(uuid.uuid4())
 
     autoblocks = AutoblocksTracer(
-        env.AUTOBLOCKS_INGESTION_KEY, trace_id=trace_id, properties=dict(source="DEMO_REPLAYS")
+        env.AUTOBLOCKS_INGESTION_KEY,
+        trace_id=trace_id,
+        properties=dict(source="DEMO_REPLAYS"),
     )
     autoblocks.send_event("request.payload", properties=dict(payload=payload))
 

diff --git a/demo_replays/replay.py b/demo_replays/replay.py
@@ -1,8 +1,7 @@
 import argparse
 
 import requests
-from autoblocks.replays import replay_events_from_view
-from autoblocks.replays import start_replay
+from autoblocks.api.client import AutoblocksAPIClient
 
 from demo_replays.settings import AUTOBLOCKS_REPLAY_TRACE_ID_HEADER_NAME
 from demo_replays.settings import env
@@ -12,13 +11,11 @@ def static():
     """
     Replays a static set of events against the locally-running app.
     """
-    start_replay()
-
     for trace_id, query in [
-        ("sf", "San Francisco tourist attractions"),
-        ("paris", "Paris tourist attractions"),
-        ("lombard", "Lombard Street"),
-        ("eiffel", "Eiffel Tower"),
+        ("san-francisco-tourist-attractions", "San Francisco tourist attractions"),
+        ("paris-tourist-attractions", "Paris tourist attractions"),
+        ("lombard-stree", "Lombard Street"),
+        ("eiffel-tower", "Eiffel Tower"),
     ]:
         print(f"Testing static event {trace_id} - {query}")
         requests.post(
@@ -42,22 +39,20 @@ def dynamic():
     )
     args = parser.parse_args()
 
-    start_replay()
-
-    for event in replay_events_from_view(
-        api_key=env.AUTOBLOCKS_API_KEY,
-        view_id=args.view_id,
-        num_traces=args.num_traces,
-    ):
-        if event.message == "request.payload":
-            print(f"Replaying past event {event}")
-
-            # The original payload
-            payload = event.properties["payload"]
-
-            # Replay the request
-            requests.post(
-                "http://localhost:5000",
-                json=payload,
-                headers={AUTOBLOCKS_REPLAY_TRACE_ID_HEADER_NAME: event.trace_id},
-            )
+    ab_client = AutoblocksAPIClient(env.AUTOBLOCKS_API_KEY)
+
+    resp = ab_client.get_traces_from_view(view_id=args.view_id, page_size=args.num_traces)
+    for trace in resp.traces:
+        for event in trace.events:
+            if event.message == "request.payload":
+                print(f"Replaying past event {event}")
+
+                # The original payload
+                payload = event.properties["payload"]
+
+                # Replay the request
+                requests.post(
+                    "http://localhost:5000",
+                    json=payload,
+                    headers={AUTOBLOCKS_REPLAY_TRACE_ID_HEADER_NAME: event.trace_id},
+                )
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -13,7 +13,7 @@ requests = "^2.31.0"
 flask = "^2.3.2"
 pydantic-settings = "^2.0.2"
 simpleaichat = "^0.2.2"
-autoblocksai = "^0.0.1"
+autoblocksai = "0.0.2"
 
 [tool.poetry.group.dev.dependencies]
 pre-commit = "^3.3.3"
@@ -35,5 +35,5 @@ known-first-party = ["demo_replays"]
 
 [tool.poetry.scripts]
 start = "demo_replays.app:start"
-replay = "demo_replays.replay:dynamic"
 replay-static = "demo_replays.replay:static"
+replay-dynamic = "demo_replays.replay:dynamic"