From e8dcf056e88af8382e0225d761fe0ed526d84466 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=82=E3=81=A7?= Date: Tue, 24 Sep 2024 13:48:32 +0200 Subject: [PATCH] Use compact prompt when generating requests via Claude 3.5 --- Tiltfile | 79 +- api/package.json | 1 + .../generate-request.test.ts.snap | 1853 +++++++++++++++++ api/src/lib/ai/anthropic.ts | 5 +- api/src/lib/ai/generate-request.test.ts | 271 +++ api/src/lib/ai/prompts.ts | 30 +- api/src/routes/inference/expand-handler.ts | 2 +- pnpm-lock.yaml | 4 + studio/src/utils/index.ts | 1 + www/src/content/changelog/!canary.mdx | 2 + 10 files changed, 2230 insertions(+), 18 deletions(-) create mode 100644 api/src/lib/ai/__snapshots__/generate-request.test.ts.snap create mode 100644 api/src/lib/ai/generate-request.test.ts diff --git a/Tiltfile b/Tiltfile index 306fc0427..2e52221d9 100644 --- a/Tiltfile +++ b/Tiltfile @@ -1,10 +1,10 @@ -# Automagically install & update npm dependencies when package.json changes +# Automagically install & update pnpm dependencies when package.json changes local_resource( "node_modules", - labels=["api", "frontend"], - deps=["package.json", "api/package.json", "frontend/package.json"], + labels=["api", "studio"], + deps=["package.json", "api/package.json", "studio/package.json"], dir=".", - cmd="npm install", + cmd="pnpm install", ) # Ensure the api/dist directory exists @@ -14,22 +14,31 @@ local_resource( cmd="mkdir api/dist || true", ) -# Build & serve the frontend local_resource( - "frontend-build", - labels=["frontend"], - cmd="npm run clean:frontend && npm run build:frontend", - deps=["frontend/src"], + "packages-build", + labels=["studio"], + cmd="pnpm --filter @fiberplane/fpx-types build && pnpm --filter @fiberplane/fpx-utils build && pnpm --filter @fiberplane/hono-otel build", + deps=["packages"], + ignore=["packages/*/dist"], +) + +# Build & serve the studio +local_resource( + "studio-build", + labels=["studio"], + cmd="pnpm clean:frontend && pnpm build:frontend", + deps=["studio/src"], resource_deps=["node_modules", "api-dist"], ) local_resource( - "frontend-serve", - labels=["frontend"], + "studio-serve", + labels=["studio"], deps=["studio/src"], resource_deps=["node_modules", "api-dist"], - serve_cmd="npm run dev", + serve_cmd="pnpm dev", serve_dir="studio", + auto_init=False, trigger_mode=TRIGGER_MODE_MANUAL, ) @@ -38,7 +47,7 @@ local_resource( "db-generate", labels=["api"], dir="api", - cmd="npm run db:generate", + cmd="pnpm db:generate", deps=["api/drizzle.config.ts"], ) @@ -46,7 +55,7 @@ local_resource( "db-migrate", labels=["api"], dir="api", - cmd="npm run db:migrate", + cmd="pnpm db:migrate", deps=["api/migrate.ts"], ) @@ -55,6 +64,46 @@ local_resource( "api", labels=["api"], resource_deps=["node_modules", "db-generate", "db-migrate"], - serve_cmd="npm run dev", + serve_cmd="pnpm dev", serve_dir="api", ) + +local_resource( + "reset-db", + labels=["api"], + cmd="rm fpx.db", + dir="api", + auto_init=False, + trigger_mode=TRIGGER_MODE_MANUAL, +) + +local_resource( + "format", + labels=["api", "studio"], + cmd="pnpm format", + auto_init=False, + trigger_mode=TRIGGER_MODE_MANUAL, +) + + +# Examples + +local_resource( + "examples-node-api", + dir="examples/node-api", + labels=["examples"], + serve_dir="examples/node-api", + serve_cmd="pnpm dev", + auto_init=False, + trigger_mode=TRIGGER_MODE_MANUAL, +) + +local_resource( + "examples-goose-quotes", + dir="examples/goose-quotes", + labels=["examples"], + serve_dir="examples/goose-quotes", + serve_cmd="pnpm db:generate && pnpm db:migrate && pnpm dev", + auto_init=False, + trigger_mode=TRIGGER_MODE_MANUAL, +) \ No newline at end of file diff --git a/api/package.json b/api/package.json index 506e35e1b..57da50499 100644 --- a/api/package.json +++ b/api/package.json @@ -71,6 +71,7 @@ "@types/figlet": "^1.5.8", "@types/node": "^20.11.17", "@types/ws": "^8.5.10", + "ajv": "^8.17.1", "ts-to-zod": "^3.8.5", "tsx": "^4.10.5", "vitest": "^1.6.0" diff --git a/api/src/lib/ai/__snapshots__/generate-request.test.ts.snap b/api/src/lib/ai/__snapshots__/generate-request.test.ts.snap new file mode 100644 index 000000000..6f1794832 --- /dev/null +++ b/api/src/lib/ai/__snapshots__/generate-request.test.ts.snap @@ -0,0 +1,1853 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`generate request > geese handler (DEV / original) 1`] = ` +"I need to make a request to one of my Hono api handlers. + +Here are some recent requests/responses, which you can use as inspiration for future requests. +E.g., if we recently created a resource, you can look that resource up. + + +NO HISTORY + + +The request you make should be a POST request to route: /api/geese + +Here is the OpenAPI spec for the handler: +NO OPENAPI SPEC + +Here is the middleware that will be applied to the request: +NO MIDDLEWARE + +Here is some additional context for the middleware that will be applied to the request: +NO MIDDLEWARE CONTEXT + +Here is the code for the handler: +async (c) => { + const sql = neon(c.env.DATABASE_URL); + const db = drizzle(sql); + + console.log("Fetching flock leaders"); + + const flockLeaders = await measure("getFlockLeaders", () => + db.select().from(geese).where(eq(geese.isFlockLeader, true)), + )(); + + console.log(\`Found \${flockLeaders.length} flock leaders\`); + + return c.json(flockLeaders); +} + +Here is some additional context for the handler source code, if you need it: + + + neon + + #third-party-library-code + + @neondatabase+serverless@0.9.4/node_modules + + + drizzle + + #third-party-library-code + + drizzle-orm@0.32.2_@cloudflare+workers-types@4.20240821.1_@libsql+client@0.6.2_@neondatabase+_jgwx5w4as4cf27fa4hwacogio4 + + + measure + + #third-party-library-code + + + + geese + + pgTable("geese", { + id: serial("id").primaryKey(), + name: text("name").notNull(), + description: text("description"), + isFlockLeader: boolean("is_leader"), + programmingLanguage: text("programming_language"), + motivations: jsonb("motivations"), + location: text("location"), + bio: text("bio"), + avatar: text("avatar"), + honks: integer("honks").default(0), + createdAt: timestamp("created_at").defaultNow().notNull(), + updatedAt: timestamp("updated_at").defaultNow().notNull(), + }) + + + + eq + + #third-party-library-code + + drizzle-orm@0.32.2_@cloudflare+workers-types@4.20240821.1_@libsql+client@0.6.2_@neondatabase+_jgwx5w4as4cf27fa4hwacogio4 + +" +`; + +exports[`generate request > geese handler (DEV / original) 2`] = ` +"You are a friendly, expert full-stack engineer and an API testing assistant for apps that use Hono, +a typescript web framework similar to express. + +You need to help craft requests to route handlers. + +You will be provided the source code of a route handler for an API route, and you should generate +query parameters, a request body, and headers that will test the request. + +Be clever and creative with test data. Avoid just writing things like "test". + +For example, if you get a route like \`/users/:id\`, you should return a URL like +\`/users/10\` and a pathParams parameter like this: + +{ "path": "/users/10", "pathParams": { "key": ":id", "value": "10" } } + +*Remember to keep the colon in the pathParam key!* + +If you get a route like \`POST /users/:id\` with a handler like: + +\`\`\`ts +async (c) => { +const token = c.req.headers.get("authorization")?.split(" ")[1] + +const auth = c.get("authService"); +const isAuthorized = await auth.isAuthorized(token) +if (!isAuthorized) { +return c.json({ message: "Unauthorized" }, 401) +} + +const db = c.get("db"); + +const id = c.req.param('id'); +const { email } = await c.req.json() + +const user = (await db.update(user).set({ email }).where(eq(user.id, +id)).returning())?.[0]; + +if (!user) { +return c.json({ message: 'User not found' }, 404); +} + +return c.json(user); +} +\`\`\` + +You should return a URL like: + +\`/users/64\` and a pathParams like: + +{ "path": "/users/64", "pathParams": { "key": ":id", "value": "64" } } + +and a header like: + +{ "headers": { "key": "authorization", "value": "Bearer " } } + +and a body like: + +{ email: "paul@beatles.music" } + +with a body type of "json" + +It is, however, possible that the body type is JSON, text, or form data. If the body type is a file stream, return an empty body. +Only return bodyType "file" for obvious, singular file uploads. + +If it appears that more fields are coming alongside a file, return a body type of "form-data" with isMultipart set to true. + +For form data, you can return a body type of "form-data". You can still return a JSON object like above, +I will handle converting it to form data. + +Never add the x-fpx-trace-id header to the request. + +=== + +Use the tool "make_request". Always respond in valid JSON. Help the user test the happy path." +`; + +exports[`generate request > geese handler (DEV / original) 3`] = ` +{ + "body": "{}", + "bodyType": { + "type": "json", + }, + "path": "/api/geese", +} +`; + +exports[`generate request > geese handler (DEV / slim) 1`] = ` +"I need to make a request to one of my Hono api handlers. + +Here are some recent requests/responses, which you can use as inspiration for future requests. +E.g., if we recently created a resource, you can look that resource up. + + +NO HISTORY + + +The request you make should be a POST request to route: /api/geese + +Here is the OpenAPI spec for the handler: +NO OPENAPI SPEC + +Here is the middleware that will be applied to the request: +NO MIDDLEWARE + +Here is some additional context for the middleware that will be applied to the request: +NO MIDDLEWARE CONTEXT + +Here is the code for the handler: +async (c) => { + const sql = neon(c.env.DATABASE_URL); + const db = drizzle(sql); + + console.log("Fetching flock leaders"); + + const flockLeaders = await measure("getFlockLeaders", () => + db.select().from(geese).where(eq(geese.isFlockLeader, true)), + )(); + + console.log(\`Found \${flockLeaders.length} flock leaders\`); + + return c.json(flockLeaders); +} + +Here is some additional context for the handler source code, if you need it: + + + neon + + #third-party-library-code + + @neondatabase+serverless@0.9.4/node_modules + + + drizzle + + #third-party-library-code + + drizzle-orm@0.32.2_@cloudflare+workers-types@4.20240821.1_@libsql+client@0.6.2_@neondatabase+_jgwx5w4as4cf27fa4hwacogio4 + + + measure + + #third-party-library-code + + + + geese + + pgTable("geese", { + id: serial("id").primaryKey(), + name: text("name").notNull(), + description: text("description"), + isFlockLeader: boolean("is_leader"), + programmingLanguage: text("programming_language"), + motivations: jsonb("motivations"), + location: text("location"), + bio: text("bio"), + avatar: text("avatar"), + honks: integer("honks").default(0), + createdAt: timestamp("created_at").defaultNow().notNull(), + updatedAt: timestamp("updated_at").defaultNow().notNull(), + }) + + + + eq + + #third-party-library-code + + drizzle-orm@0.32.2_@cloudflare+workers-types@4.20240821.1_@libsql+client@0.6.2_@neondatabase+_jgwx5w4as4cf27fa4hwacogio4 + +" +`; + +exports[`generate request > geese handler (DEV / slim) 2`] = ` +"You are a friendly, expert full-stack engineer and an API testing assistant. Please help user to craft requests to route handlers. + + +For example, if you get a route like \`/users/:id\`, you should return a filled-in "path" field, +like \`/users/1234567890\` and a "pathParams" field like: + +{ "path": "/users/1234567890", "pathParams": { "key": ":id", "value": "1234567890" } } + +*Remember to keep the colon in the pathParam key!*" +`; + +exports[`generate request > geese handler (DEV / slim) 3`] = ` +{ + "bodyType": { + "isMultipart": false, + "type": "json", + }, + "headers": [ + { + "key": "Content-Type", + "value": "application/json", + }, + ], + "path": "/api/geese", +} +`; + +exports[`generate request > geese handler (QA / original) 1`] = ` +"I need to make a request to one of my Hono api handlers. + +Here are some recent requests and responses, which you can use as inspiration for future requests. + + +NO HISTORY + + +The request you make should be a POST request to route: /api/geese + +Here is the OpenAPI spec for the handler: +NO OPENAPI SPEC + +Here is the middleware that will be applied to the request: +NO MIDDLEWARE + +Here is some additional context for the middleware that will be applied to the request: +NO MIDDLEWARE CONTEXT + +Here is the code for the handler: +async (c) => { + const sql = neon(c.env.DATABASE_URL); + const db = drizzle(sql); + + console.log("Fetching flock leaders"); + + const flockLeaders = await measure("getFlockLeaders", () => + db.select().from(geese).where(eq(geese.isFlockLeader, true)), + )(); + + console.log(\`Found \${flockLeaders.length} flock leaders\`); + + return c.json(flockLeaders); +} + +Here is some additional context for the handler source code, if you need it: + + + neon + + #third-party-library-code + + @neondatabase+serverless@0.9.4/node_modules + + + drizzle + + #third-party-library-code + + drizzle-orm@0.32.2_@cloudflare+workers-types@4.20240821.1_@libsql+client@0.6.2_@neondatabase+_jgwx5w4as4cf27fa4hwacogio4 + + + measure + + #third-party-library-code + + + + geese + + pgTable("geese", { + id: serial("id").primaryKey(), + name: text("name").notNull(), + description: text("description"), + isFlockLeader: boolean("is_leader"), + programmingLanguage: text("programming_language"), + motivations: jsonb("motivations"), + location: text("location"), + bio: text("bio"), + avatar: text("avatar"), + honks: integer("honks").default(0), + createdAt: timestamp("created_at").defaultNow().notNull(), + updatedAt: timestamp("updated_at").defaultNow().notNull(), + }) + + + + eq + + #third-party-library-code + + drizzle-orm@0.32.2_@cloudflare+workers-types@4.20240821.1_@libsql+client@0.6.2_@neondatabase+_jgwx5w4as4cf27fa4hwacogio4 + + + +REMEMBER YOU ARE A QA. MISUSE THE API. BUT DO NOT MISUSE YOURSELF. +Keep your responses short-ish. Including your random data." +`; + +exports[`generate request > geese handler (QA / original) 2`] = ` +"You are an expert QA Engineer, a thorough API tester, and a code debugging assistant for web APIs that use Hono, +a typescript web framework similar to express. You have a generally hostile disposition. + +You need to help craft requests to route handlers. + +You will be provided the source code of a route handler for an API route, and you should generate +query parameters, a request body, and headers that will test the request. + +Be clever and creative with test data. Avoid just writing things like "test". + +For example, if you get a route like \`/users/:id\`, you should return a filled-in "path" field, +like \`/users/1234567890\` and a "pathParams" field like: + +{ "path": "/users/1234567890", "pathParams": { "key": ":id", "value": "1234567890" } } + +*Remember to keep the colon in the pathParam key!* + +If you get a route like \`POST /users/:id\` with a handler like: + +\`\`\`ts +async (c) => { +const token = c.req.headers.get("authorization")?.split(" ")[1] + +const auth = c.get("authService"); +const isAuthorized = await auth.isAuthorized(token) +if (!isAuthorized) { +return c.json({ message: "Unauthorized" }, 401) +} + +const db = c.get("db"); + +const id = c.req.param('id'); +const { email } = await c.req.json() + +const user = (await db.update(user).set({ email }).where(eq(user.id, +id)).returning())?.[0]; + +if (!user) { +return c.json({ message: 'User not found' }, 404); +} + +return c.json(user); +} +\`\`\` + +You should return a filled-in "path" field like \`/users/1234567890\` and a "pathParams" field like: + +{ "path": "/users/1234567890", "pathParams": { "key": ":id", "value": "1234567890" } } + +and a header like: + +{ "headers": { "key": "authorization", "value": "Bearer admin" } } + +and a body like: + +{ "body": { "email": "" } } + +It is possible that the body type is JSON, text, or form data. You can use the wrong body type to see what happens. +But if the body type is a file stream, just return an empty body. + +For form data, you can return a body type of "form-data". You can still return a JSON object like above, +I will handle converting it to form data. + +You should focus on trying to break things. You are a QA. + +You are the enemy of bugs. To protect quality, you must find bugs. + +Try strategies like specifying invalid data, missing data, or invalid data types (e.g., using strings instead of numbers). + +Try to break the system. But do not break yourself! +Keep your responses to a reasonable length. Including your random data. + +Never add the x-fpx-trace-id header to the request. + +Use the tool "make_request". Always respond in valid JSON. +***Don't make your responses too long, otherwise we cannot parse your JSON response.***" +`; + +exports[`generate request > geese handler (QA / original) 3`] = ` +{ + "body": "{ + "maliciousPayload": "DROP TABLE geese;", + "isFlockLeader": "NotABoolean", + "honks": "Over9000" +}", + "bodyType": { + "isMultipart": false, + "type": "json", + }, + "headers": [ + { + "key": "Content-Type", + "value": "application/xml", + }, + { + "key": "X-Hacker", + "value": "1337", + }, + ], + "path": "/api/geese", + "queryParams": [ + { + "key": "UNION", + "value": "SELECT * FROM users", + }, + { + "key": "limit", + "value": "-1", + }, + ], +} +`; + +exports[`generate request > geese handler (QA / slim) 1`] = ` +"I need to make a request to one of my Hono api handlers. + +Here are some recent requests and responses, which you can use as inspiration for future requests. + + +NO HISTORY + + +The request you make should be a POST request to route: /api/geese + +Here is the OpenAPI spec for the handler: +NO OPENAPI SPEC + +Here is the middleware that will be applied to the request: +NO MIDDLEWARE + +Here is some additional context for the middleware that will be applied to the request: +NO MIDDLEWARE CONTEXT + +Here is the code for the handler: +async (c) => { + const sql = neon(c.env.DATABASE_URL); + const db = drizzle(sql); + + console.log("Fetching flock leaders"); + + const flockLeaders = await measure("getFlockLeaders", () => + db.select().from(geese).where(eq(geese.isFlockLeader, true)), + )(); + + console.log(\`Found \${flockLeaders.length} flock leaders\`); + + return c.json(flockLeaders); +} + +Here is some additional context for the handler source code, if you need it: + + + neon + + #third-party-library-code + + @neondatabase+serverless@0.9.4/node_modules + + + drizzle + + #third-party-library-code + + drizzle-orm@0.32.2_@cloudflare+workers-types@4.20240821.1_@libsql+client@0.6.2_@neondatabase+_jgwx5w4as4cf27fa4hwacogio4 + + + measure + + #third-party-library-code + + + + geese + + pgTable("geese", { + id: serial("id").primaryKey(), + name: text("name").notNull(), + description: text("description"), + isFlockLeader: boolean("is_leader"), + programmingLanguage: text("programming_language"), + motivations: jsonb("motivations"), + location: text("location"), + bio: text("bio"), + avatar: text("avatar"), + honks: integer("honks").default(0), + createdAt: timestamp("created_at").defaultNow().notNull(), + updatedAt: timestamp("updated_at").defaultNow().notNull(), + }) + + + + eq + + #third-party-library-code + + drizzle-orm@0.32.2_@cloudflare+workers-types@4.20240821.1_@libsql+client@0.6.2_@neondatabase+_jgwx5w4as4cf27fa4hwacogio4 + + + +REMEMBER YOU ARE A QA. MISUSE THE API. BUT DO NOT MISUSE YOURSELF. +Keep your responses short-ish. Including your random data." +`; + +exports[`generate request > geese handler (QA / slim) 2`] = ` +"You are an expert QA Engineer, a thorough API tester with a generally hostile disposition. Please help user to craft requests to route handlers. + + +For example, if you get a route like \`/users/:id\`, you should return a filled-in "path" field, +like \`/users/1234567890\` and a "pathParams" field like: + +{ "path": "/users/1234567890", "pathParams": { "key": ":id", "value": "1234567890" } } + +*Remember to keep the colon in the pathParam key!* + + +You should focus on trying to break things. Be clever and creative with test data." +`; + +exports[`generate request > geese handler (QA / slim) 3`] = ` +{ + "body": "{ + "maliciousPayload": true, + "sql_injection": "'; DROP TABLE geese; --", + "overflowAttempt": "A".repeat(10000), + "nullByte": "Null\\u0000Byte", + "xss": "" +}", + "bodyType": { + "isMultipart": false, + "type": "json", + }, + "headers": [ + { + "key": "Content-Type", + "value": "application/json", + }, + { + "key": "X-Hacker", + "value": "true", + }, + { + "key": "User-Agent", + "value": "EvilGoose/1.0", + }, + ], + "path": "/api/geese", +} +`; + +exports[`generate request > goose update handler (DEV / original) 1`] = ` +"I need to make a request to one of my Hono api handlers. + +Here are some recent requests/responses, which you can use as inspiration for future requests. +E.g., if we recently created a resource, you can look that resource up. + + +NO HISTORY + + +The request you make should be a PUT request to route: /api/geese/:id + +Here is the OpenAPI spec for the handler: +NO OPENAPI SPEC + +Here is the middleware that will be applied to the request: +NO MIDDLEWARE + +Here is some additional context for the middleware that will be applied to the request: +NO MIDDLEWARE CONTEXT + +Here is the code for the handler: +async (c) => { + const sql = neon(c.env.DATABASE_URL); + const db = drizzle(sql); + + const id = c.req.param("id"); + const updateData = await c.req.json(); + + console.log(\`Updating goose \${id} with data:\`, updateData); + + const goose = await measure("getGooseById", () => getGooseById(db, +id))(); + + if (!goose) { + console.warn(\`Goose not found: \${id}\`); + return c.json({ message: "Goose not found" }, 404); + } + + // Simulate a race condition by splitting the update into multiple parts + const updatePromises = Object.entries(updateData).map( + async ([key, value]) => { + await new Promise((resolve) => setTimeout(resolve, Math.random() * 1000)); + return measure("updateGoose", () => + updateGoose(db, +id, { [key]: value }), + )(); + }, + ); + + await Promise.all(updatePromises); + + const updatedGoose = await measure("getGooseById", () => + getGooseById(db, +id), + )(); + + console.log(\`Goose \${id} updated successfully\`); + return c.json(updatedGoose); +} + +Here is some additional context for the handler source code, if you need it: + + + neon + + #third-party-library-code + + @neondatabase+serverless@0.9.4/node_modules + + + drizzle + + #third-party-library-code + + drizzle-orm@0.32.2_@cloudflare+workers-types@4.20240821.1_@libsql+client@0.6.2_@neondatabase+_jgwx5w4as4cf27fa4hwacogio4 + + + measure + + #third-party-library-code + + + + getGooseById + + async ( + db: ReturnType, + id: number, + ) => { + console.log(\`Fetching goose with id: \${id}\`); + return (await db.select().from(geese).where(eq(geese.id, id)))?.[0]; + } + + + + updateGoose + + async ( + db: ReturnType, + id: number, + updateData: Partial, + ) => { + console.log({ action: "updateGoose", id, updateData }); + + // Simulate a race condition by splitting the update into two parts + const updatePromises = Object.entries(updateData).map( + async ([key, value]) => { + // Introduce a random delay to increase the chance of interleaved updates + await new Promise((resolve) => setTimeout(resolve, Math.random() * 1000)); + + return db + .update(geese) + .set({ [key]: value }) + .where(eq(geese.id, id)) + .returning(); + }, + ); + + // Wait for all updates to complete + const results = await Promise.all(updatePromises); + + // Return the last result, which may not contain all updates + return results[results.length - 1][0]; + } + + +" +`; + +exports[`generate request > goose update handler (DEV / original) 2`] = ` +"You are a friendly, expert full-stack engineer and an API testing assistant for apps that use Hono, +a typescript web framework similar to express. + +You need to help craft requests to route handlers. + +You will be provided the source code of a route handler for an API route, and you should generate +query parameters, a request body, and headers that will test the request. + +Be clever and creative with test data. Avoid just writing things like "test". + +For example, if you get a route like \`/users/:id\`, you should return a URL like +\`/users/10\` and a pathParams parameter like this: + +{ "path": "/users/10", "pathParams": { "key": ":id", "value": "10" } } + +*Remember to keep the colon in the pathParam key!* + +If you get a route like \`POST /users/:id\` with a handler like: + +\`\`\`ts +async (c) => { +const token = c.req.headers.get("authorization")?.split(" ")[1] + +const auth = c.get("authService"); +const isAuthorized = await auth.isAuthorized(token) +if (!isAuthorized) { +return c.json({ message: "Unauthorized" }, 401) +} + +const db = c.get("db"); + +const id = c.req.param('id'); +const { email } = await c.req.json() + +const user = (await db.update(user).set({ email }).where(eq(user.id, +id)).returning())?.[0]; + +if (!user) { +return c.json({ message: 'User not found' }, 404); +} + +return c.json(user); +} +\`\`\` + +You should return a URL like: + +\`/users/64\` and a pathParams like: + +{ "path": "/users/64", "pathParams": { "key": ":id", "value": "64" } } + +and a header like: + +{ "headers": { "key": "authorization", "value": "Bearer " } } + +and a body like: + +{ email: "paul@beatles.music" } + +with a body type of "json" + +It is, however, possible that the body type is JSON, text, or form data. If the body type is a file stream, return an empty body. +Only return bodyType "file" for obvious, singular file uploads. + +If it appears that more fields are coming alongside a file, return a body type of "form-data" with isMultipart set to true. + +For form data, you can return a body type of "form-data". You can still return a JSON object like above, +I will handle converting it to form data. + +Never add the x-fpx-trace-id header to the request. + +=== + +Use the tool "make_request". Always respond in valid JSON. Help the user test the happy path." +`; + +exports[`generate request > goose update handler (DEV / original) 3`] = ` +{ + "body": "{ + "name": "Honkers McFeathers", + "age": 3, + "breed": "Canadian Goose", + "favoriteFood": "Bread crumbs", + "lastSeenAt": "2023-09-15T14:30:00Z" +}", + "bodyType": { + "type": "json", + }, + "path": "/api/geese/42", + "pathParams": [ + { + "key": ":id", + "value": "42", + }, + ], +} +`; + +exports[`generate request > goose update handler (DEV / slim) 1`] = ` +"I need to make a request to one of my Hono api handlers. + +Here are some recent requests/responses, which you can use as inspiration for future requests. +E.g., if we recently created a resource, you can look that resource up. + + +NO HISTORY + + +The request you make should be a PUT request to route: /api/geese/:id + +Here is the OpenAPI spec for the handler: +NO OPENAPI SPEC + +Here is the middleware that will be applied to the request: +NO MIDDLEWARE + +Here is some additional context for the middleware that will be applied to the request: +NO MIDDLEWARE CONTEXT + +Here is the code for the handler: +async (c) => { + const sql = neon(c.env.DATABASE_URL); + const db = drizzle(sql); + + const id = c.req.param("id"); + const updateData = await c.req.json(); + + console.log(\`Updating goose \${id} with data:\`, updateData); + + const goose = await measure("getGooseById", () => getGooseById(db, +id))(); + + if (!goose) { + console.warn(\`Goose not found: \${id}\`); + return c.json({ message: "Goose not found" }, 404); + } + + // Simulate a race condition by splitting the update into multiple parts + const updatePromises = Object.entries(updateData).map( + async ([key, value]) => { + await new Promise((resolve) => setTimeout(resolve, Math.random() * 1000)); + return measure("updateGoose", () => + updateGoose(db, +id, { [key]: value }), + )(); + }, + ); + + await Promise.all(updatePromises); + + const updatedGoose = await measure("getGooseById", () => + getGooseById(db, +id), + )(); + + console.log(\`Goose \${id} updated successfully\`); + return c.json(updatedGoose); +} + +Here is some additional context for the handler source code, if you need it: + + + neon + + #third-party-library-code + + @neondatabase+serverless@0.9.4/node_modules + + + drizzle + + #third-party-library-code + + drizzle-orm@0.32.2_@cloudflare+workers-types@4.20240821.1_@libsql+client@0.6.2_@neondatabase+_jgwx5w4as4cf27fa4hwacogio4 + + + measure + + #third-party-library-code + + + + getGooseById + + async ( + db: ReturnType, + id: number, + ) => { + console.log(\`Fetching goose with id: \${id}\`); + return (await db.select().from(geese).where(eq(geese.id, id)))?.[0]; + } + + + + updateGoose + + async ( + db: ReturnType, + id: number, + updateData: Partial, + ) => { + console.log({ action: "updateGoose", id, updateData }); + + // Simulate a race condition by splitting the update into two parts + const updatePromises = Object.entries(updateData).map( + async ([key, value]) => { + // Introduce a random delay to increase the chance of interleaved updates + await new Promise((resolve) => setTimeout(resolve, Math.random() * 1000)); + + return db + .update(geese) + .set({ [key]: value }) + .where(eq(geese.id, id)) + .returning(); + }, + ); + + // Wait for all updates to complete + const results = await Promise.all(updatePromises); + + // Return the last result, which may not contain all updates + return results[results.length - 1][0]; + } + + +" +`; + +exports[`generate request > goose update handler (DEV / slim) 2`] = ` +"You are a friendly, expert full-stack engineer and an API testing assistant. Please help user to craft requests to route handlers. + + +For example, if you get a route like \`/users/:id\`, you should return a filled-in "path" field, +like \`/users/1234567890\` and a "pathParams" field like: + +{ "path": "/users/1234567890", "pathParams": { "key": ":id", "value": "1234567890" } } + +*Remember to keep the colon in the pathParam key!*" +`; + +exports[`generate request > goose update handler (DEV / slim) 3`] = ` +{ + "body": "{ + "name": "Honker", + "age": 3, + "color": "gray" +}", + "bodyType": { + "isMultipart": false, + "type": "json", + }, + "headers": [ + { + "key": "Content-Type", + "value": "application/json", + }, + ], + "path": "/api/geese/1234567890", + "pathParams": [ + { + "key": ":id", + "value": "1234567890", + }, + ], +} +`; + +exports[`generate request > goose update handler (QA / original) 1`] = ` +"I need to make a request to one of my Hono api handlers. + +Here are some recent requests and responses, which you can use as inspiration for future requests. + + +NO HISTORY + + +The request you make should be a PUT request to route: /api/geese/:id + +Here is the OpenAPI spec for the handler: +NO OPENAPI SPEC + +Here is the middleware that will be applied to the request: +NO MIDDLEWARE + +Here is some additional context for the middleware that will be applied to the request: +NO MIDDLEWARE CONTEXT + +Here is the code for the handler: +async (c) => { + const sql = neon(c.env.DATABASE_URL); + const db = drizzle(sql); + + const id = c.req.param("id"); + const updateData = await c.req.json(); + + console.log(\`Updating goose \${id} with data:\`, updateData); + + const goose = await measure("getGooseById", () => getGooseById(db, +id))(); + + if (!goose) { + console.warn(\`Goose not found: \${id}\`); + return c.json({ message: "Goose not found" }, 404); + } + + // Simulate a race condition by splitting the update into multiple parts + const updatePromises = Object.entries(updateData).map( + async ([key, value]) => { + await new Promise((resolve) => setTimeout(resolve, Math.random() * 1000)); + return measure("updateGoose", () => + updateGoose(db, +id, { [key]: value }), + )(); + }, + ); + + await Promise.all(updatePromises); + + const updatedGoose = await measure("getGooseById", () => + getGooseById(db, +id), + )(); + + console.log(\`Goose \${id} updated successfully\`); + return c.json(updatedGoose); +} + +Here is some additional context for the handler source code, if you need it: + + + neon + + #third-party-library-code + + @neondatabase+serverless@0.9.4/node_modules + + + drizzle + + #third-party-library-code + + drizzle-orm@0.32.2_@cloudflare+workers-types@4.20240821.1_@libsql+client@0.6.2_@neondatabase+_jgwx5w4as4cf27fa4hwacogio4 + + + measure + + #third-party-library-code + + + + getGooseById + + async ( + db: ReturnType, + id: number, + ) => { + console.log(\`Fetching goose with id: \${id}\`); + return (await db.select().from(geese).where(eq(geese.id, id)))?.[0]; + } + + + + updateGoose + + async ( + db: ReturnType, + id: number, + updateData: Partial, + ) => { + console.log({ action: "updateGoose", id, updateData }); + + // Simulate a race condition by splitting the update into two parts + const updatePromises = Object.entries(updateData).map( + async ([key, value]) => { + // Introduce a random delay to increase the chance of interleaved updates + await new Promise((resolve) => setTimeout(resolve, Math.random() * 1000)); + + return db + .update(geese) + .set({ [key]: value }) + .where(eq(geese.id, id)) + .returning(); + }, + ); + + // Wait for all updates to complete + const results = await Promise.all(updatePromises); + + // Return the last result, which may not contain all updates + return results[results.length - 1][0]; + } + + + + +REMEMBER YOU ARE A QA. MISUSE THE API. BUT DO NOT MISUSE YOURSELF. +Keep your responses short-ish. Including your random data." +`; + +exports[`generate request > goose update handler (QA / original) 2`] = ` +"You are an expert QA Engineer, a thorough API tester, and a code debugging assistant for web APIs that use Hono, +a typescript web framework similar to express. You have a generally hostile disposition. + +You need to help craft requests to route handlers. + +You will be provided the source code of a route handler for an API route, and you should generate +query parameters, a request body, and headers that will test the request. + +Be clever and creative with test data. Avoid just writing things like "test". + +For example, if you get a route like \`/users/:id\`, you should return a filled-in "path" field, +like \`/users/1234567890\` and a "pathParams" field like: + +{ "path": "/users/1234567890", "pathParams": { "key": ":id", "value": "1234567890" } } + +*Remember to keep the colon in the pathParam key!* + +If you get a route like \`POST /users/:id\` with a handler like: + +\`\`\`ts +async (c) => { +const token = c.req.headers.get("authorization")?.split(" ")[1] + +const auth = c.get("authService"); +const isAuthorized = await auth.isAuthorized(token) +if (!isAuthorized) { +return c.json({ message: "Unauthorized" }, 401) +} + +const db = c.get("db"); + +const id = c.req.param('id'); +const { email } = await c.req.json() + +const user = (await db.update(user).set({ email }).where(eq(user.id, +id)).returning())?.[0]; + +if (!user) { +return c.json({ message: 'User not found' }, 404); +} + +return c.json(user); +} +\`\`\` + +You should return a filled-in "path" field like \`/users/1234567890\` and a "pathParams" field like: + +{ "path": "/users/1234567890", "pathParams": { "key": ":id", "value": "1234567890" } } + +and a header like: + +{ "headers": { "key": "authorization", "value": "Bearer admin" } } + +and a body like: + +{ "body": { "email": "" } } + +It is possible that the body type is JSON, text, or form data. You can use the wrong body type to see what happens. +But if the body type is a file stream, just return an empty body. + +For form data, you can return a body type of "form-data". You can still return a JSON object like above, +I will handle converting it to form data. + +You should focus on trying to break things. You are a QA. + +You are the enemy of bugs. To protect quality, you must find bugs. + +Try strategies like specifying invalid data, missing data, or invalid data types (e.g., using strings instead of numbers). + +Try to break the system. But do not break yourself! +Keep your responses to a reasonable length. Including your random data. + +Never add the x-fpx-trace-id header to the request. + +Use the tool "make_request". Always respond in valid JSON. +***Don't make your responses too long, otherwise we cannot parse your JSON response.***" +`; + +exports[`generate request > goose update handler (QA / original) 3`] = ` +{ + "body": "{ + "name": "Malicious Goose", + "age": "not_a_number", + "weight": -500, + "favoriteFood": null, + "id": 99999 +}", + "bodyType": { + "isMultipart": false, + "type": "json", + }, + "headers": [ + { + "key": "Content-Type", + "value": "application/json", + }, + ], + "path": "/api/geese/12345", + "pathParams": [ + { + "key": ":id", + "value": "12345", + }, + ], +} +`; + +exports[`generate request > goose update handler (QA / slim) 1`] = ` +"I need to make a request to one of my Hono api handlers. + +Here are some recent requests and responses, which you can use as inspiration for future requests. + + +NO HISTORY + + +The request you make should be a PUT request to route: /api/geese/:id + +Here is the OpenAPI spec for the handler: +NO OPENAPI SPEC + +Here is the middleware that will be applied to the request: +NO MIDDLEWARE + +Here is some additional context for the middleware that will be applied to the request: +NO MIDDLEWARE CONTEXT + +Here is the code for the handler: +async (c) => { + const sql = neon(c.env.DATABASE_URL); + const db = drizzle(sql); + + const id = c.req.param("id"); + const updateData = await c.req.json(); + + console.log(\`Updating goose \${id} with data:\`, updateData); + + const goose = await measure("getGooseById", () => getGooseById(db, +id))(); + + if (!goose) { + console.warn(\`Goose not found: \${id}\`); + return c.json({ message: "Goose not found" }, 404); + } + + // Simulate a race condition by splitting the update into multiple parts + const updatePromises = Object.entries(updateData).map( + async ([key, value]) => { + await new Promise((resolve) => setTimeout(resolve, Math.random() * 1000)); + return measure("updateGoose", () => + updateGoose(db, +id, { [key]: value }), + )(); + }, + ); + + await Promise.all(updatePromises); + + const updatedGoose = await measure("getGooseById", () => + getGooseById(db, +id), + )(); + + console.log(\`Goose \${id} updated successfully\`); + return c.json(updatedGoose); +} + +Here is some additional context for the handler source code, if you need it: + + + neon + + #third-party-library-code + + @neondatabase+serverless@0.9.4/node_modules + + + drizzle + + #third-party-library-code + + drizzle-orm@0.32.2_@cloudflare+workers-types@4.20240821.1_@libsql+client@0.6.2_@neondatabase+_jgwx5w4as4cf27fa4hwacogio4 + + + measure + + #third-party-library-code + + + + getGooseById + + async ( + db: ReturnType, + id: number, + ) => { + console.log(\`Fetching goose with id: \${id}\`); + return (await db.select().from(geese).where(eq(geese.id, id)))?.[0]; + } + + + + updateGoose + + async ( + db: ReturnType, + id: number, + updateData: Partial, + ) => { + console.log({ action: "updateGoose", id, updateData }); + + // Simulate a race condition by splitting the update into two parts + const updatePromises = Object.entries(updateData).map( + async ([key, value]) => { + // Introduce a random delay to increase the chance of interleaved updates + await new Promise((resolve) => setTimeout(resolve, Math.random() * 1000)); + + return db + .update(geese) + .set({ [key]: value }) + .where(eq(geese.id, id)) + .returning(); + }, + ); + + // Wait for all updates to complete + const results = await Promise.all(updatePromises); + + // Return the last result, which may not contain all updates + return results[results.length - 1][0]; + } + + + + +REMEMBER YOU ARE A QA. MISUSE THE API. BUT DO NOT MISUSE YOURSELF. +Keep your responses short-ish. Including your random data." +`; + +exports[`generate request > goose update handler (QA / slim) 2`] = ` +"You are an expert QA Engineer, a thorough API tester with a generally hostile disposition. Please help user to craft requests to route handlers. + + +For example, if you get a route like \`/users/:id\`, you should return a filled-in "path" field, +like \`/users/1234567890\` and a "pathParams" field like: + +{ "path": "/users/1234567890", "pathParams": { "key": ":id", "value": "1234567890" } } + +*Remember to keep the colon in the pathParam key!* + + +You should focus on trying to break things. Be clever and creative with test data." +`; + +exports[`generate request > goose update handler (QA / slim) 3`] = ` +{ + "body": "{ + "name": null, + "age": -1, + "weight": "not a number", + "favorite_food": {"malicious": "object"}, + "id": 9999 +}", + "bodyType": { + "isMultipart": false, + "type": "json", + }, + "path": "/api/geese/0", + "pathParams": [ + { + "key": ":id", + "value": "0", + }, + ], +} +`; + +exports[`generate request > home handler (DEV / original) 1`] = ` +"I need to make a request to one of my Hono api handlers. + +Here are some recent requests/responses, which you can use as inspiration for future requests. +E.g., if we recently created a resource, you can look that resource up. + + +NO HISTORY + + +The request you make should be a GET request to route: / + +Here is the OpenAPI spec for the handler: +NO OPENAPI SPEC + +Here is the middleware that will be applied to the request: +NO MIDDLEWARE + +Here is some additional context for the middleware that will be applied to the request: +NO MIDDLEWARE CONTEXT + +Here is the code for the handler: +(c) => { + const honk = shouldHonk(c.req) ? "Honk honk!" : ""; + console.log(\`Home page accessed. Honk: \${honk}\`); + return c.text(\`Hello Goose Quotes! \${honk}\`.trim()); +} + +Here is some additional context for the handler source code, if you need it: + + + shouldHonk + + export function shouldHonk(r: HonoRequest) { + const { shouldHonk } = r.query(); + return typeof shouldHonk !== "undefined"; + } + + +" +`; + +exports[`generate request > home handler (DEV / original) 2`] = ` +"You are a friendly, expert full-stack engineer and an API testing assistant for apps that use Hono, +a typescript web framework similar to express. + +You need to help craft requests to route handlers. + +You will be provided the source code of a route handler for an API route, and you should generate +query parameters, a request body, and headers that will test the request. + +Be clever and creative with test data. Avoid just writing things like "test". + +For example, if you get a route like \`/users/:id\`, you should return a URL like +\`/users/10\` and a pathParams parameter like this: + +{ "path": "/users/10", "pathParams": { "key": ":id", "value": "10" } } + +*Remember to keep the colon in the pathParam key!* + +If you get a route like \`POST /users/:id\` with a handler like: + +\`\`\`ts +async (c) => { +const token = c.req.headers.get("authorization")?.split(" ")[1] + +const auth = c.get("authService"); +const isAuthorized = await auth.isAuthorized(token) +if (!isAuthorized) { +return c.json({ message: "Unauthorized" }, 401) +} + +const db = c.get("db"); + +const id = c.req.param('id'); +const { email } = await c.req.json() + +const user = (await db.update(user).set({ email }).where(eq(user.id, +id)).returning())?.[0]; + +if (!user) { +return c.json({ message: 'User not found' }, 404); +} + +return c.json(user); +} +\`\`\` + +You should return a URL like: + +\`/users/64\` and a pathParams like: + +{ "path": "/users/64", "pathParams": { "key": ":id", "value": "64" } } + +and a header like: + +{ "headers": { "key": "authorization", "value": "Bearer " } } + +and a body like: + +{ email: "paul@beatles.music" } + +with a body type of "json" + +It is, however, possible that the body type is JSON, text, or form data. If the body type is a file stream, return an empty body. +Only return bodyType "file" for obvious, singular file uploads. + +If it appears that more fields are coming alongside a file, return a body type of "form-data" with isMultipart set to true. + +For form data, you can return a body type of "form-data". You can still return a JSON object like above, +I will handle converting it to form data. + +Never add the x-fpx-trace-id header to the request. + +=== + +Use the tool "make_request". Always respond in valid JSON. Help the user test the happy path." +`; + +exports[`generate request > home handler (DEV / original) 3`] = ` +{ + "path": "/", + "queryParams": [ + { + "key": "shouldHonk", + "value": "true", + }, + ], +} +`; + +exports[`generate request > home handler (DEV / slim) 1`] = ` +"I need to make a request to one of my Hono api handlers. + +Here are some recent requests/responses, which you can use as inspiration for future requests. +E.g., if we recently created a resource, you can look that resource up. + + +NO HISTORY + + +The request you make should be a GET request to route: / + +Here is the OpenAPI spec for the handler: +NO OPENAPI SPEC + +Here is the middleware that will be applied to the request: +NO MIDDLEWARE + +Here is some additional context for the middleware that will be applied to the request: +NO MIDDLEWARE CONTEXT + +Here is the code for the handler: +(c) => { + const honk = shouldHonk(c.req) ? "Honk honk!" : ""; + console.log(\`Home page accessed. Honk: \${honk}\`); + return c.text(\`Hello Goose Quotes! \${honk}\`.trim()); +} + +Here is some additional context for the handler source code, if you need it: + + + shouldHonk + + export function shouldHonk(r: HonoRequest) { + const { shouldHonk } = r.query(); + return typeof shouldHonk !== "undefined"; + } + + +" +`; + +exports[`generate request > home handler (DEV / slim) 2`] = ` +"You are a friendly, expert full-stack engineer and an API testing assistant. Please help user to craft requests to route handlers. + + +For example, if you get a route like \`/users/:id\`, you should return a filled-in "path" field, +like \`/users/1234567890\` and a "pathParams" field like: + +{ "path": "/users/1234567890", "pathParams": { "key": ":id", "value": "1234567890" } } + +*Remember to keep the colon in the pathParam key!*" +`; + +exports[`generate request > home handler (DEV / slim) 3`] = ` +{ + "path": "/", + "queryParams": [ + { + "key": "shouldHonk", + "value": "true", + }, + ], +} +`; + +exports[`generate request > home handler (QA / original) 1`] = ` +"I need to make a request to one of my Hono api handlers. + +Here are some recent requests and responses, which you can use as inspiration for future requests. + + +NO HISTORY + + +The request you make should be a GET request to route: / + +Here is the OpenAPI spec for the handler: +NO OPENAPI SPEC + +Here is the middleware that will be applied to the request: +NO MIDDLEWARE + +Here is some additional context for the middleware that will be applied to the request: +NO MIDDLEWARE CONTEXT + +Here is the code for the handler: +(c) => { + const honk = shouldHonk(c.req) ? "Honk honk!" : ""; + console.log(\`Home page accessed. Honk: \${honk}\`); + return c.text(\`Hello Goose Quotes! \${honk}\`.trim()); +} + +Here is some additional context for the handler source code, if you need it: + + + shouldHonk + + export function shouldHonk(r: HonoRequest) { + const { shouldHonk } = r.query(); + return typeof shouldHonk !== "undefined"; + } + + + + +REMEMBER YOU ARE A QA. MISUSE THE API. BUT DO NOT MISUSE YOURSELF. +Keep your responses short-ish. Including your random data." +`; + +exports[`generate request > home handler (QA / original) 2`] = ` +"You are an expert QA Engineer, a thorough API tester, and a code debugging assistant for web APIs that use Hono, +a typescript web framework similar to express. You have a generally hostile disposition. + +You need to help craft requests to route handlers. + +You will be provided the source code of a route handler for an API route, and you should generate +query parameters, a request body, and headers that will test the request. + +Be clever and creative with test data. Avoid just writing things like "test". + +For example, if you get a route like \`/users/:id\`, you should return a filled-in "path" field, +like \`/users/1234567890\` and a "pathParams" field like: + +{ "path": "/users/1234567890", "pathParams": { "key": ":id", "value": "1234567890" } } + +*Remember to keep the colon in the pathParam key!* + +If you get a route like \`POST /users/:id\` with a handler like: + +\`\`\`ts +async (c) => { +const token = c.req.headers.get("authorization")?.split(" ")[1] + +const auth = c.get("authService"); +const isAuthorized = await auth.isAuthorized(token) +if (!isAuthorized) { +return c.json({ message: "Unauthorized" }, 401) +} + +const db = c.get("db"); + +const id = c.req.param('id'); +const { email } = await c.req.json() + +const user = (await db.update(user).set({ email }).where(eq(user.id, +id)).returning())?.[0]; + +if (!user) { +return c.json({ message: 'User not found' }, 404); +} + +return c.json(user); +} +\`\`\` + +You should return a filled-in "path" field like \`/users/1234567890\` and a "pathParams" field like: + +{ "path": "/users/1234567890", "pathParams": { "key": ":id", "value": "1234567890" } } + +and a header like: + +{ "headers": { "key": "authorization", "value": "Bearer admin" } } + +and a body like: + +{ "body": { "email": "" } } + +It is possible that the body type is JSON, text, or form data. You can use the wrong body type to see what happens. +But if the body type is a file stream, just return an empty body. + +For form data, you can return a body type of "form-data". You can still return a JSON object like above, +I will handle converting it to form data. + +You should focus on trying to break things. You are a QA. + +You are the enemy of bugs. To protect quality, you must find bugs. + +Try strategies like specifying invalid data, missing data, or invalid data types (e.g., using strings instead of numbers). + +Try to break the system. But do not break yourself! +Keep your responses to a reasonable length. Including your random data. + +Never add the x-fpx-trace-id header to the request. + +Use the tool "make_request". Always respond in valid JSON. +***Don't make your responses too long, otherwise we cannot parse your JSON response.***" +`; + +exports[`generate request > home handler (QA / original) 3`] = ` +{ + "headers": [ + { + "key": "User-Agent", + "value": "EvilGoose/1.0", + }, + { + "key": "Accept", + "value": "text/html", + }, + ], + "path": "/", + "queryParams": [ + { + "key": "shouldHonk", + "value": "true", + }, + { + "key": "maliciousParam", + "value": "", + }, + ], +} +`; + +exports[`generate request > home handler (QA / slim) 1`] = ` +"I need to make a request to one of my Hono api handlers. + +Here are some recent requests and responses, which you can use as inspiration for future requests. + + +NO HISTORY + + +The request you make should be a GET request to route: / + +Here is the OpenAPI spec for the handler: +NO OPENAPI SPEC + +Here is the middleware that will be applied to the request: +NO MIDDLEWARE + +Here is some additional context for the middleware that will be applied to the request: +NO MIDDLEWARE CONTEXT + +Here is the code for the handler: +(c) => { + const honk = shouldHonk(c.req) ? "Honk honk!" : ""; + console.log(\`Home page accessed. Honk: \${honk}\`); + return c.text(\`Hello Goose Quotes! \${honk}\`.trim()); +} + +Here is some additional context for the handler source code, if you need it: + + + shouldHonk + + export function shouldHonk(r: HonoRequest) { + const { shouldHonk } = r.query(); + return typeof shouldHonk !== "undefined"; + } + + + + +REMEMBER YOU ARE A QA. MISUSE THE API. BUT DO NOT MISUSE YOURSELF. +Keep your responses short-ish. Including your random data." +`; + +exports[`generate request > home handler (QA / slim) 2`] = ` +"You are an expert QA Engineer, a thorough API tester with a generally hostile disposition. Please help user to craft requests to route handlers. + + +For example, if you get a route like \`/users/:id\`, you should return a filled-in "path" field, +like \`/users/1234567890\` and a "pathParams" field like: + +{ "path": "/users/1234567890", "pathParams": { "key": ":id", "value": "1234567890" } } + +*Remember to keep the colon in the pathParam key!* + + +You should focus on trying to break things. Be clever and creative with test data." +`; + +exports[`generate request > home handler (QA / slim) 3`] = ` +{ + "headers": [ + { + "key": "User-Agent", + "value": "EvilGoose/1.0", + }, + { + "key": "X-Forwarded-For", + "value": "192.168.1.1, 10.0.0.1, 172.16.0.1", + }, + ], + "path": "/", + "queryParams": [ + { + "key": "shouldHonk", + "value": "true", + }, + { + "key": "malicious", + "value": "", + }, + ], +} +`; diff --git a/api/src/lib/ai/anthropic.ts b/api/src/lib/ai/anthropic.ts index 24fdc6afb..af3d394cd 100644 --- a/api/src/lib/ai/anthropic.ts +++ b/api/src/lib/ai/anthropic.ts @@ -1,4 +1,5 @@ import Anthropic from "@anthropic-ai/sdk"; +import { CLAUDE_3_5_SONNET } from "@fiberplane/fpx-types"; import logger from "../../logger.js"; import { getSystemPrompt, invokeRequestGenerationPrompt } from "./prompts.js"; import { makeRequestTool as makeRequestToolBase } from "./tools.js"; @@ -81,11 +82,13 @@ export async function generateRequestWithAnthropic({ name: makeRequestTool.name, }; + const systemPrompt = getSystemPrompt(persona, model === CLAUDE_3_5_SONNET); + const response = await anthropicClient.messages.create({ model, tool_choice: toolChoice, tools: [makeRequestTool], - system: getSystemPrompt(persona), + system: systemPrompt, messages: [ { role: "user", diff --git a/api/src/lib/ai/generate-request.test.ts b/api/src/lib/ai/generate-request.test.ts new file mode 100644 index 000000000..b5de4751c --- /dev/null +++ b/api/src/lib/ai/generate-request.test.ts @@ -0,0 +1,271 @@ +import { spawn } from "node:child_process"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import Anthropic from "@anthropic-ai/sdk"; +import type { ToolUseBlock } from "@anthropic-ai/sdk/resources/messages.mjs"; +import { CLAUDE_3_5_SONNET } from "@fiberplane/fpx-types"; +import Ajv from "ajv"; +import { describe, expect } from "vitest"; +import { transformExpandedFunction } from "../../routes/inference/expand-handler.js"; +import { expandFunction } from "../expand-function/expand-function.js"; +import { getSystemPrompt, invokeRequestGenerationPrompt } from "./prompts.js"; +import { makeRequestTool as makeRequestToolBase } from "./tools.js"; + +// Shim __filename and __dirname since we're using esm +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +// Resolve the path and analyze the 'src' directory +const projectRoot = path.resolve( + __dirname, + "../../../../examples/goose-quotes", +); +const srcPath = path.resolve(projectRoot, "src"); + +const makeRequestTool = { + name: makeRequestToolBase.function.name, + description: makeRequestToolBase.function.description, + input_schema: makeRequestToolBase.function.parameters, +}; + +// Create json schema validator +const ajv = new Ajv(); +const matchesJsonSchema = ajv.compile(makeRequestToolBase.function.parameters); + +// We expect the path params to be prefixed with a colon +const validatePathParams = (input: unknown) => { + if (Object.prototype.hasOwnProperty.call(input, "pathParams")) { + const pathParams = (input as { pathParams: { key: string }[] }).pathParams; + + for (const { key } of pathParams) { + const firstChar = key.charAt(0); + if (firstChar !== ":") { + return false; + } + } + } + return true; +}; + +const apiGeeseHandler = { + title: "geese handler", + method: "POST", + path: "/api/geese", + handler: `async (c) => { + const sql = neon(c.env.DATABASE_URL); + const db = drizzle(sql); + + console.log("Fetching flock leaders"); + + const flockLeaders = await measure("getFlockLeaders", () => + db.select().from(geese).where(eq(geese.isFlockLeader, true)), + )(); + + console.log(${"`Found ${flockLeaders.length} flock leaders`"}); + + return c.json(flockLeaders); +}`, +}; + +const apiGooseUpdateHandler = { + title: "goose update handler", + method: "PUT", + path: "/api/geese/:id", + handler: `async (c) => { + const sql = neon(c.env.DATABASE_URL); + const db = drizzle(sql); + + const id = c.req.param("id"); + const updateData = await c.req.json(); + + console.log(${"`Updating goose ${id} with data:`"}, updateData); + + const goose = await measure("getGooseById", () => getGooseById(db, +id))(); + + if (!goose) { + console.warn(${"`Goose not found: ${id}`"}); + return c.json({ message: "Goose not found" }, 404); + } + + // Simulate a race condition by splitting the update into multiple parts + const updatePromises = Object.entries(updateData).map( + async ([key, value]) => { + await new Promise((resolve) => setTimeout(resolve, Math.random() * 1000)); + return measure("updateGoose", () => + updateGoose(db, +id, { [key]: value }), + )(); + }, + ); + + await Promise.all(updatePromises); + + const updatedGoose = await measure("getGooseById", () => + getGooseById(db, +id), + )(); + + console.log(${"`Goose ${id} updated successfully`"}); + return c.json(updatedGoose); +}`, +}; + +const homeHandler = { + title: "home handler", + method: "GET", + path: "/", + handler: `(c) => { + const honk = shouldHonk(c.req) ? "Honk honk!" : ""; + console.log(${"`Home page accessed. Honk: ${honk}`"}); + return c.text(${"`Hello Goose Quotes! ${honk}`"}.trim()); +}`, + validate: (input: unknown) => { + if (Object.prototype.hasOwnProperty.call(input, "queryParams")) { + const queryParams = (input as { queryParams: { key: string }[] }) + .queryParams; + + for (const { key } of queryParams) { + if (key === "shouldHonk") { + return true; + } + } + } + return false; + }, +}; + +const testCases: Array<{ + path: string; + title: string; + persona: string; + slimPrompt: boolean; + handler: string; + method: string; + validate?: (input: unknown) => boolean; +}> = [ + { persona: "DEV", slimPrompt: false, ...apiGeeseHandler }, + { persona: "QA", slimPrompt: false, ...apiGeeseHandler }, + { persona: "DEV", slimPrompt: true, ...apiGeeseHandler }, + { persona: "QA", slimPrompt: true, ...apiGeeseHandler }, + { persona: "DEV", slimPrompt: false, ...apiGooseUpdateHandler }, + { persona: "QA", slimPrompt: false, ...apiGooseUpdateHandler }, + { persona: "DEV", slimPrompt: true, ...apiGooseUpdateHandler }, + { persona: "QA", slimPrompt: true, ...apiGooseUpdateHandler }, + { persona: "DEV", slimPrompt: false, ...homeHandler }, + { persona: "QA", slimPrompt: false, ...homeHandler }, + { persona: "DEV", slimPrompt: true, ...homeHandler }, + { persona: "QA", slimPrompt: true, ...homeHandler }, +]; + +const KEY_REFERENCE = process.env.ANTHROPIC_API_KEY_REFERENCE; + +function getKeyFromOnePassword(reference: string) { + return new Promise((ok) => { + const handle = spawn("op", ["read", reference]); + handle.stdout.on("data", (data) => { + const token = data.toString(); + ok(token.trim()); + }); + + handle.stderr.on("data", (data) => { + console.error(data.toString()); + }); + + handle.on("close", (code) => { + if (code !== 0) { + console.error(`child process exited with code ${code}`); + ok(null); + } + }); + }); +} + +// this test is skipped by default because it is not completely deterministic and requires an API key. +// to securely pass key from 1password (requires cli installed) and run the test, +// pass credential reference as an environment variable: +// ANTHROPIC_API_KEY_REFERENCE="op://Organization/Anthropic API key/credential" pnpm test +describe("generate request", () => { + for (const { + persona, + slimPrompt, + handler, + method, + path, + title, + validate, + } of testCases) { + it( + `${title} (${persona} / ${slimPrompt ? "slim" : "original"})`, + { timeout: 20000, skip: !KEY_REFERENCE }, + async () => { + if (!KEY_REFERENCE) { + throw new Error("API key reference not set"); + } + const apiKey = await getKeyFromOnePassword(KEY_REFERENCE); + if (!apiKey) { + throw new Error("Failed to get API key"); + } + + const context = await expandFunction(srcPath, handler, { + skipSourceMap: true, + }); + const contextAsString = transformExpandedFunction(context); + + const anthropicClient = new Anthropic({ apiKey: String(apiKey) }); + const userPrompt = await invokeRequestGenerationPrompt({ + persona, + method, + path, + handler, + handlerContext: contextAsString || undefined, + history: undefined, + openApiSpec: undefined, + middleware: undefined, + middlewareContext: undefined, + }); + + const systemPrompt = getSystemPrompt(persona, slimPrompt); + + // check that inputs are same + expect(userPrompt).toMatchSnapshot(); + expect(systemPrompt).toMatchSnapshot(); + + const response = await anthropicClient.messages.create({ + model: CLAUDE_3_5_SONNET, + tool_choice: { + type: "tool", + name: makeRequestTool.name, + }, + tools: [makeRequestTool], + system: systemPrompt, + messages: [ + { + role: "user", + content: userPrompt, + }, + ], + temperature: 0, + max_tokens: 2048, + }); + + // Only one function call is expected + const { + content: [block], + } = response; + expect(block.type).toBe("tool_use"); + const toolUseBlock = block as ToolUseBlock; + expect(toolUseBlock.name).toBe("make_request"); + + // Check that function calls are same + expect(toolUseBlock.input).toMatchSnapshot(); + + expect(matchesJsonSchema(toolUseBlock.input)).toBe(true); + + expect(validatePathParams(toolUseBlock.input)).toBe(true); + + if (validate) { + expect(validate(toolUseBlock.input)).toBe(true); + } + }, + ); + } +}); diff --git a/api/src/lib/ai/prompts.ts b/api/src/lib/ai/prompts.ts index 2bd07c1e6..70c865c07 100644 --- a/api/src/lib/ai/prompts.ts +++ b/api/src/lib/ai/prompts.ts @@ -1,6 +1,11 @@ import { PromptTemplate } from "@langchain/core/prompts"; -export const getSystemPrompt = (persona: string) => { +export const getSystemPrompt = (persona: string, slimPrompt = false) => { + if (slimPrompt) { + return persona === "QA" + ? HOSTILE_SLIM_SYSTEM_PROMPT + : FRIENDLY_SLIM_SYSTEM_PROMPT; + } return persona === "QA" ? QA_PARAMETER_GENERATION_SYSTEM_PROMPT : FRIENDLY_PARAMETER_GENERATION_SYSTEM_PROMPT; @@ -302,6 +307,29 @@ Use the tool "make_request". Always respond in valid JSON. ***Don't make your responses too long, otherwise we cannot parse your JSON response.*** `); +const PATH_INSTRUCTION = ` +For example, if you get a route like \`/users/:id\`, you should return a filled-in "path" field, +like \`/users/1234567890\` and a "pathParams" field like: + +{ "path": "/users/1234567890", "pathParams": { "key": ":id", "value": "1234567890" } } + +*Remember to keep the colon in the pathParam key!* +`; + +export const FRIENDLY_SLIM_SYSTEM_PROMPT = cleanPrompt(` +You are a friendly, expert full-stack engineer and an API testing assistant. Please help user to craft requests to route handlers. + +${PATH_INSTRUCTION} +`); + +export const HOSTILE_SLIM_SYSTEM_PROMPT = cleanPrompt(` +You are an expert QA Engineer, a thorough API tester with a generally hostile disposition. Please help user to craft requests to route handlers. + +${PATH_INSTRUCTION} + +You should focus on trying to break things. Be clever and creative with test data. +`); + /** * Clean a prompt by trimming whitespace for each line and joining the lines. */ diff --git a/api/src/routes/inference/expand-handler.ts b/api/src/routes/inference/expand-handler.ts index d1ae1e61e..99212af13 100644 --- a/api/src/routes/inference/expand-handler.ts +++ b/api/src/routes/inference/expand-handler.ts @@ -160,7 +160,7 @@ async function expandFunctionInUserProject(handler: SourceFunctionResult) { * @param expandedFunction - The expanded function context * @returns The transformed expanded function context */ -function transformExpandedFunction( +export function transformExpandedFunction( expandedFunction: ExpandedFunctionResult | null, ): string | null { if (!expandedFunction || !expandedFunction.context?.length) { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5cb992bd8..8ad91b3cb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -53,6 +53,9 @@ importers: acorn-walk: specifier: ^8.3.2 version: 8.3.3 + ajv: + specifier: ^8.17.1 + version: 8.17.1 chalk: specifier: ^5.3.0 version: 5.3.0 @@ -5255,6 +5258,7 @@ packages: libsql@0.3.19: resolution: {integrity: sha512-Aj5cQ5uk/6fHdmeW0TiXK42FqUlwx7ytmMLPSaUQPin5HKKKuUPD62MAbN4OEweGBBI7q1BekoEN4gPUEL6MZA==} + cpu: [x64, arm64, wasm32] os: [darwin, linux, win32] lilconfig@2.1.0: diff --git a/studio/src/utils/index.ts b/studio/src/utils/index.ts index 2ecd1fecc..dcd8dd55c 100644 --- a/studio/src/utils/index.ts +++ b/studio/src/utils/index.ts @@ -123,6 +123,7 @@ export const SENSITIVE_HEADERS = [ "cookie", "set-cookie", "neon-connection-string", + "fpx-trace-id", ]; export function redactSensitiveHeaders( diff --git a/www/src/content/changelog/!canary.mdx b/www/src/content/changelog/!canary.mdx index 1ab032dac..5602c6579 100644 --- a/www/src/content/changelog/!canary.mdx +++ b/www/src/content/changelog/!canary.mdx @@ -14,6 +14,8 @@ draft: true - **Render audio files** When your api returns a binary audio response, we will render an audio player for you to listen to it. +- **Compact AI query generation prompts** Using latest model from Antrhopic, we were able to achieve a significant token reduction in the query generation prompts. Only affects Claude 3.5 Sonnet for now. + ### Bug fixes - **D1 autoinstrumentation** The client library, `@fiberplane/hono-otel`, now instruments D1 queries in latest versions of wrangler/miniflare.