Merge pull request #4 from outl1ne/pricing

feat: add cost metrics to OpenAI requests resource
outl1ne · Feb 26, 2024 · 1a48bd0 · 1a48bd0
2 parents a031dd4 + 9b80669
commit 1a48bd0
Show file tree

Hide file tree

Showing 18 changed files with 299 additions and 7 deletions.
diff --git a/README.md b/README.md
@@ -6,6 +6,10 @@ This is currently an **unstable** package, breaking changes are to be expected.
 
 If you need any features to be implemented or bump its priority in our backlog then feel free to make an inquiry via email at [email protected].
 
+## Screenshots
+
+![Screenshot](resources/media/screenshot1.png)
+
 ## Requirements
 
 - `php: >=8.1`

diff --git a/config/nova-openai.php b/config/nova-openai.php
@@ -36,4 +36,15 @@
     'headers' => [
         'OpenAI-Beta' => 'assistants=v1',
     ],
+
+    /*
+    |--------------------------------------------------------------------------
+    | Pricing configuration
+    |--------------------------------------------------------------------------
+    |
+    | Path to the pricing configuration file to be used to calculate
+    | the cost of requests made against OpenAI API.
+    */
+
+    'pricing' => null,
 ];
diff --git a/database/migrations/2024_02_05_000010_create_openai_requests_table.php b/database/migrations/2024_02_05_000010_create_openai_requests_table.php
@@ -16,6 +16,7 @@ public function up(): void
 
             $table->string('method')->nullable()->index();
             $table->string('status')->index();
+            $table->unsignedDecimal('cost', 16, 8)->nullable();
             $table->unsignedDecimal('time_sec', 10, 4)->nullable();
             $table->string('model_requested')->nullable();
             $table->string('model_used')->nullable();

diff --git a/resources/media/screenshot1.png b/resources/media/screenshot1.png
diff --git a/resources/openai-pricing.json b/resources/openai-pricing.json
@@ -0,0 +1,102 @@
+{
+  "models": {
+    "gpt-4-0125-preview": {
+      "input": 0.01,
+      "output": 0.03
+    },
+    "gpt-4-1106-preview": {
+      "input": 0.01,
+      "output": 0.03
+    },
+    "gpt-4-1106-vision-preview": {
+      "input": 0.01,
+      "output": 0.03
+    },
+    "gpt-4": {
+      "input": 0.03,
+      "output": 0.06
+    },
+    "gpt-4-32k": {
+      "input": 0.06,
+      "output": 0.12
+    },
+    "gpt-3.5-turbo-0125": {
+      "input": 0.0005,
+      "output": 0.0015
+    },
+    "gpt-3.5-turbo-instruct": {
+      "input": 0.0015,
+      "output": 0.002
+    },
+    "gpt-3.5-turbo-1106": {
+      "input": 0.001,
+      "output": 0.002
+    },
+    "gpt-3.5-turbo-0613": {
+      "input": 0.0015,
+      "output": 0.002
+    },
+    "gpt-3.5-turbo-16k-0613": {
+      "input": 0.003,
+      "output": 0.004
+    },
+    "gpt-3.5-turbo-0301": {
+      "input": 0.0015,
+      "output": 0.002
+    }
+  },
+  "assistants": {
+    "code-interpreter": {
+      "input": 0.03
+    },
+    "retrieval": {
+      "input": 0.2
+    }
+  },
+  "fine-tuning": {
+    "gpt-3.5-turbo": {
+      "training": 0.008,
+      "input": 0.003,
+      "output": 0.006
+    },
+    "davinci-002": {
+      "training": 0.006,
+      "input": 0.012,
+      "output": 0.012
+    },
+    "babbage-002": {
+      "training": 0.0004,
+      "input": 0.0016,
+      "output": 0.0016
+    }
+  },
+  "embedding": {
+    "text-embedding-3-small": 0.00002,
+    "text-embedding-3-large": 0.00013,
+    "ada-v2": 0.0001
+  },
+  "base-models": {
+    "davinci-002": 0.002,
+    "babbage-002": 0.0004
+  },
+  "image": {
+    "dall-e-3": {
+      "standard-1024-1024": 0.04,
+      "standard-1024-1792": 0.08,
+      "standard-1792-1024": 0.08,
+      "hd-1024-1024": 0.08,
+      "hd-1024-1792": 0.12,
+      "hd-1792-1024": 0.12
+    },
+    "dall-e-2": {
+      "1024-1024": 0.02,
+      "512-512": 0.018,
+      "256-256": 0.016
+    }
+  },
+  "audio": {
+    "whisper": 0.006,
+    "tts": 0.015,
+    "tts-hd": 0.03
+  }
+}
diff --git a/src/Capabilities/Capability.php b/src/Capabilities/Capability.php
@@ -3,11 +3,13 @@
 namespace Outl1ne\NovaOpenAI\Capabilities;
 
 use Illuminate\Http\Client\PendingRequest;
+use Outl1ne\NovaOpenAI\Pricing\Pricing;
 
 class Capability
 {
     public function __construct(
         protected readonly PendingRequest $http,
+        protected readonly Pricing $pricing,
     ) {
     }
 }
diff --git a/src/Capabilities/Chat/Chat.php b/src/Capabilities/Chat/Chat.php
@@ -27,7 +27,7 @@ public function create(
         ?array $tools = null,
         null|string|array $toolChoice = null,
     ) {
-        return (new CreateChat($this->http))->makeRequest(
+        return (new CreateChat($this->http, $this->pricing))->makeRequest(
             $model,
             $messages,
             $responseFormat,

diff --git a/src/Capabilities/Chat/CreateChat.php b/src/Capabilities/Chat/CreateChat.php
@@ -3,12 +3,13 @@
 namespace Outl1ne\NovaOpenAI\Capabilities\Chat;
 
 use Exception;
+use Outl1ne\NovaOpenAI\Pricing\Pricing;
 use Illuminate\Http\Client\PendingRequest;
 use Outl1ne\NovaOpenAI\Models\OpenAIRequest;
 use Outl1ne\NovaOpenAI\Capabilities\Measurable;
 use Outl1ne\NovaOpenAI\Capabilities\Chat\Parameters\Messages;
-use Outl1ne\NovaOpenAI\Capabilities\Chat\Parameters\ResponseFormat;
 use Outl1ne\NovaOpenAI\Capabilities\Chat\Responses\ChatResponse;
+use Outl1ne\NovaOpenAI\Capabilities\Chat\Parameters\ResponseFormat;
 use Outl1ne\NovaOpenAI\Capabilities\Embeddings\Responses\EmbeddingsResponse;
 
 class CreateChat
@@ -19,6 +20,7 @@ class CreateChat
 
     public function __construct(
         protected readonly PendingRequest $http,
+        protected readonly Pricing $pricing,
     ) {
         $this->request = new OpenAIRequest;
         $this->request->method = 'chat';
@@ -90,12 +92,14 @@ public function makeRequest(
 
     protected function handleResponse(ChatResponse $response)
     {
+        $this->request->cost = $this->pricing->models()->model($response->model)->calculate($response->usage->promptTokens, $response->usage->completionTokens);
         $this->request->time_sec = $this->measure();
         $this->request->status = 'success';
         $this->request->meta = $response->meta;
         $this->request->model_used = $response->model;
         $this->request->output = $response->choices;
         $this->request->usage_prompt_tokens = $response->usage->promptTokens;
+        $this->request->usage_completion_tokens = $response->usage->completionTokens;
         $this->request->usage_total_tokens = $response->usage->totalTokens;
         $this->request->save();
 

diff --git a/src/Capabilities/Embeddings/CreateEmbedding.php b/src/Capabilities/Embeddings/CreateEmbedding.php
@@ -3,6 +3,7 @@
 namespace Outl1ne\NovaOpenAI\Capabilities\Embeddings;
 
 use Exception;
+use Outl1ne\NovaOpenAI\Pricing\Pricing;
 use Illuminate\Http\Client\PendingRequest;
 use Outl1ne\NovaOpenAI\Models\OpenAIRequest;
 use Outl1ne\NovaOpenAI\Capabilities\Measurable;
@@ -16,6 +17,7 @@ class CreateEmbedding
 
     public function __construct(
         protected readonly PendingRequest $http,
+        protected readonly Pricing $pricing,
     ) {
         $this->request = new OpenAIRequest;
         $this->request->method = 'embeddings';
@@ -58,6 +60,7 @@ public function makeRequest(string $model, string $input, ?string $encodingForma
 
     protected function handleResponse(EmbeddingsResponse $response)
     {
+        $this->request->cost = $this->pricing->embedding()->model($response->model)->calculate($response->usage->promptTokens);
         $this->request->time_sec = $this->measure();
         $this->request->status = 'success';
         $this->request->meta = $response->meta;

diff --git a/src/Capabilities/Embeddings/Embeddings.php b/src/Capabilities/Embeddings/Embeddings.php
@@ -8,6 +8,6 @@ class Embeddings extends Capability
 {
     public function create(string $model, string $input, ?string $encodingFormat = null, ?int $dimensions = null, ?string $user = null)
     {
-        return (new CreateEmbedding($this->http))->makeRequest($model, $input, $encodingFormat, $dimensions, $user);
+        return (new CreateEmbedding($this->http, $this->pricing))->makeRequest($model, $input, $encodingFormat, $dimensions, $user);
     }
 }
diff --git a/src/Factory.php b/src/Factory.php
@@ -26,6 +26,11 @@ class Factory
      */
     protected array $headers = [];
 
+    /**
+     * The pricing configuration to calculate the cost of requests.
+     */
+    protected ?object $pricing = null;
+
     /**
      * Sets the API key for the requests.
      */
@@ -82,6 +87,16 @@ public function withHttpHeaders(?array $headers): self
         return $this;
     }
 
+    /**
+     * Adds a pricing configuration.
+     */
+    public function withPricing(object $pricing): self
+    {
+        $this->pricing = $pricing;
+
+        return $this;
+    }
+
     /**
      * Creates a new Open AI Client.
      */
@@ -99,6 +114,6 @@ public function make(): OpenAI
 
         $baseUrl = rtrim($this->baseUrl ?: 'https://api.openai.com/v1', '/') . '/';
 
-        return new OpenAI($baseUrl, $headers);
+        return new OpenAI($baseUrl, $headers, $this->pricing);
     }
 }
diff --git a/src/Nova/CostMetrics.php b/src/Nova/CostMetrics.php
@@ -0,0 +1,56 @@
+<?php
+
+namespace Outl1ne\NovaOpenAI\Nova;
+
+use Laravel\Nova\Nova;
+use Laravel\Nova\Metrics\Trend;
+use Laravel\Nova\Http\Requests\NovaRequest;
+use Outl1ne\NovaOpenAI\Models\OpenAIRequest;
+
+class CostMetrics extends Trend
+{
+    /**
+     * Calculate the value of the metric.
+     *
+     * @param  \Laravel\Nova\Http\Requests\NovaRequest  $request
+     * @return mixed
+     */
+    public function calculate(NovaRequest $request)
+    {
+        return $this->sumByDays($request, OpenAIRequest::class, 'cost')->prefix('$');
+    }
+
+    /**
+     * Get the ranges available for the metric.
+     *
+     * @return array
+     */
+    public function ranges()
+    {
+        return [
+            30 => Nova::__('30 Days'),
+            60 => Nova::__('60 Days'),
+            90 => Nova::__('90 Days'),
+        ];
+    }
+
+    /**
+     * Determine the amount of time the results of the metric should be cached.
+     *
+     * @return \DateTimeInterface|\DateInterval|float|int|null
+     */
+    public function cacheFor()
+    {
+        return now()->addMinutes(1);
+    }
+
+    /**
+     * Get the URI key for the metric.
+     *
+     * @return string
+     */
+    public function uriKey()
+    {
+        return 'cost-metrics';
+    }
+}
diff --git a/src/Nova/OpenAIRequest.php b/src/Nova/OpenAIRequest.php
@@ -9,6 +9,7 @@
 use Laravel\Nova\Fields\Code;
 use Laravel\Nova\Fields\Text;
 use Laravel\Nova\Fields\Badge;
+use Laravel\Nova\Fields\Number;
 use Laravel\Nova\Fields\DateTime;
 use Laravel\Nova\Http\Requests\NovaRequest;
 use Outl1ne\NovaOpenAI\Enums\OpenAIRequestMethod;
@@ -45,6 +46,16 @@ public static function label()
         return 'Requests';
     }
 
+    /**
+     * Get the URI key for the dashboard.
+     *
+     * @return string
+     */
+    public static function uriKey()
+    {
+        return 'openai-requests';
+    }
+
     /**
      * Get the fields displayed by the resource.
      *
@@ -69,6 +80,7 @@ public function fields(NovaRequest $request)
                     OpenAIRequestMethod::EMBEDDINGS->value => 'bg-amber-600 text-amber-200',
                     OpenAIRequestMethod::AUDIO->value => 'bg-fuchsia-600 text-fuchsia-200',
                 ])->sortable(),
+            Number::make('Cost', 'cost')->sortable()->displayUsing(fn ($value) => $value === null ? null : '$' . number_format($value, 4)),
             Text::make('Request time', 'time_sec')->sortable()->displayUsing(fn () => "{$this->time_sec} sec"),
             Text::make('Model requested', 'model_requested')->sortable(),
             Text::make('Model used', 'model_used')->sortable(),
@@ -92,7 +104,9 @@ public function fields(NovaRequest $request)
      */
     public function cards(NovaRequest $request)
     {
-        return [];
+        return [
+            new CostMetrics,
+        ];
     }
 
     /**

diff --git a/src/NovaOpenAIServiceProvider.php b/src/NovaOpenAIServiceProvider.php
@@ -77,6 +77,8 @@ public function register()
             $apiKey = config('nova-openai.api_key');
             $organization = config('nova-openai.organization');
             $headers = config('nova-openai.headers');
+            $pricingPath = config('nova-openai.pricing') ?? __DIR__ . '/../resources/openai-pricing.json';
+            $pricing = json_decode(file_get_contents($pricingPath));
 
             if (!is_string($apiKey)) {
                 throw new ApiKeyMissingException;
@@ -90,6 +92,7 @@ public function register()
                 ->withApiKey($apiKey)
                 ->withOrganization($organization)
                 ->withHttpHeaders($headers)
+                ->withPricing($pricing)
                 ->make();
         });
         $this->app->alias(OpenAI::class, 'nova-openai');