Autocomplete: Add feature flag for Sourcegraph specific Fireworks set…

…up (#1670)

Now that I have added support for
`fireworks/accounts/sourcegraph/models/starcoder-16b` and
`fireworks/accounts/sourcegraph/models/starcoder-7b` to the Cody
Gateway, we can start using it on the dotcom clients.

To roll this out, I’m adding a new flag
`cody-autocomplete-default-starcoder-hybrid-sourcegraph` that is only
used when `cody-autocomplete-default-starcoder-hybrid` is also true.
This way we can set
`cody-autocomplete-default-starcoder-hybrid-sourcegraph` to a 50:50
split and it will only affect the test group that is already in the
StarCoder branch.

I expect this to be very short lived but it would be great if we can
measure the affect of that migration.

I plan to roll this out as a patch release so we can turn this on in the
next days.

<img width="996" alt="Screenshot 2023-11-07 at 16 12 05"
src="https://github.com/sourcegraph/cody/assets/458591/2ebd0511-bc18-4d7a-bdf5-0ab285772a02">

<img width="1162" alt="Screenshot 2023-11-07 at 16 13 00"
src="https://github.com/sourcegraph/cody/assets/458591/7177c14d-f456-4c46-b2f6-198a0af99b39">

<!-- Required. See
https://docs.sourcegraph.com/dev/background-information/testing_principles.
-->

lib/shared/src/experimentation/FeatureFlagProvider.ts

@@ -13,6 +13,8 @@ export enum FeatureFlag {
     CodyAutocompleteStarCoder7B = 'cody-autocomplete-default-starcoder-7b',
     CodyAutocompleteStarCoder16B = 'cody-autocomplete-default-starcoder-16b',
     CodyAutocompleteStarCoderHybrid = 'cody-autocomplete-default-starcoder-hybrid',
+    // This flag is only used when `CodyAutocompleteStarCoderHybrid` is also true
+    CodyAutocompleteStarCoderHybridSourcegraph = 'cody-autocomplete-default-starcoder-hybrid-sourcegraph',
     CodyAutocompleteLlamaCode7B = 'cody-autocomplete-default-llama-code-7b',
     CodyAutocompleteLlamaCode13B = 'cody-autocomplete-default-llama-code-13b',
     CodyAutocompleteGraphContext = 'cody-autocomplete-graph-context',

lib/shared/src/sourcegraph-api/errors.ts

@@ -32,9 +32,10 @@ export class NetworkError extends Error {
 
     constructor(
         response: Response,
+        content: string,
         public traceId: string | undefined
     ) {
-        super(`Request to ${response.url} failed with ${response.status}: ${response.statusText}`)
+        super(`Request to ${response.url} failed with ${response.status} ${response.statusText}: ${content}`)
         this.status = response.status
     }
 }

vscode/CHANGELOG.md

@@ -12,6 +12,8 @@ Starting from `0.2.0`, Cody is using `major.EVEN_NUMBER.patch` for release versi
 
 ### Changed
 
+- Added support to test a Sourcegraph specific StarCoder setup for dotcom. [pull/1670]
+
 ## [0.14.4]
 
 ### Added

vscode/src/completions/client.ts

@@ -107,7 +107,7 @@ export function createClient(config: CompletionsClientConfig, logger?: Completio
         }
 
         if (!response.ok) {
-            throw new NetworkError(response, traceId)
+            throw new NetworkError(response, await response.text(), traceId)
         }
 
         if (response.body === null) {

vscode/src/completions/logger.ts

@@ -210,7 +210,7 @@ export function loaded(id: SuggestionID, params: RequestParams, items: InlineCom
 export function suggested(
     id: SuggestionID,
     source: InlineCompletionsResultSource,
-    completion: InlineCompletionItem
+    completion: InlineCompletionItemWithAnalytics
 ): void {
     const event = activeSuggestions.get(id)
     if (!event) {

vscode/src/completions/providers/createProvider.ts

@@ -119,6 +119,7 @@ async function resolveDefaultProviderFromVSCodeConfigOrFeatureFlags(configuredPr
         starCoder7b,
         starCoder16b,
         starCoderHybrid,
+        starCoderHybridSourcegraph,
         llamaCode7b,
         llamaCode13b,
         starcoderExtendedTokenWindow,
@@ -127,6 +128,7 @@ async function resolveDefaultProviderFromVSCodeConfigOrFeatureFlags(configuredPr
         featureFlagProvider.evaluateFeatureFlag(FeatureFlag.CodyAutocompleteStarCoder7B),
         featureFlagProvider.evaluateFeatureFlag(FeatureFlag.CodyAutocompleteStarCoder16B),
         featureFlagProvider.evaluateFeatureFlag(FeatureFlag.CodyAutocompleteStarCoderHybrid),
+        featureFlagProvider.evaluateFeatureFlag(FeatureFlag.CodyAutocompleteStarCoderHybridSourcegraph),
         featureFlagProvider.evaluateFeatureFlag(FeatureFlag.CodyAutocompleteLlamaCode7B),
         featureFlagProvider.evaluateFeatureFlag(FeatureFlag.CodyAutocompleteLlamaCode13B),
         featureFlagProvider.evaluateFeatureFlag(FeatureFlag.CodyAutocompleteStarCoderExtendedTokenWindow),
@@ -139,7 +141,9 @@ async function resolveDefaultProviderFromVSCodeConfigOrFeatureFlags(configuredPr
             : starCoder16b
             ? 'starcoder-16b'
             : starCoderHybrid
-            ? 'starcoder-hybrid'
+            ? starCoderHybridSourcegraph
+                ? 'starcoder-hybrid-sourcegraph'
+                : 'starcoder-hybrid'
             : llamaCode7b
             ? 'llama-code-7b'
             : 'llama-code-13b'

vscode/src/completions/providers/fireworks.ts

@@ -35,7 +35,9 @@ const EOT_LLAMA_CODE = ' <EOT>'
 // conversations
 const MODEL_MAP = {
     'starcoder-16b': 'fireworks/accounts/fireworks/models/starcoder-16b-w8a16',
+    'starcoder-16b-sourcegraph': 'fireworks/accounts/sourcegraph/models/starcoder-16b',
     'starcoder-7b': 'fireworks/accounts/fireworks/models/starcoder-7b-w8a16',
+    'starcoder-7b-sourcegraph': 'fireworks/accounts/sourcegraph/models/starcoder-7b',
     'starcoder-3b': 'fireworks/accounts/fireworks/models/starcoder-3b-w8a16',
     'starcoder-1b': 'fireworks/accounts/fireworks/models/starcoder-1b-w8a16',
     'wizardcoder-15b': 'fireworks/accounts/fireworks/models/wizardcoder-15b',
@@ -48,10 +50,12 @@ type FireworksModel =
     | keyof typeof MODEL_MAP
     // `starcoder-hybrid` uses the 16b model for multiline requests and the 7b model for single line
     | 'starcoder-hybrid'
+    | 'starcoder-hybrid-sourcegraph'
 
 function getMaxContextTokens(model: FireworksModel, starcoderExtendedTokenWindow?: boolean): number {
     switch (model) {
         case 'starcoder-hybrid':
+        case 'starcoder-hybrid-sourcegraph':
         case 'starcoder-16b':
         case 'starcoder-7b':
         case 'starcoder-3b':
@@ -152,6 +156,8 @@ export class FireworksProvider extends Provider {
         const model =
             this.model === 'starcoder-hybrid'
                 ? MODEL_MAP[multiline ? 'starcoder-16b' : 'starcoder-7b']
+                : this.model === 'starcoder-hybrid-sourcegraph'
+                ? MODEL_MAP[multiline ? 'starcoder-16b-sourcegraph' : 'starcoder-7b-sourcegraph']
                 : MODEL_MAP[this.model]
 
         const args: CodeCompletionsParams = {
@@ -252,6 +258,8 @@ export function createProviderConfig({
             ? 'starcoder-hybrid'
             : model === 'starcoder-hybrid'
             ? 'starcoder-hybrid'
+            : model === 'starcoder-hybrid-sourcegraph'
+            ? 'starcoder-hybrid-sourcegraph'
             : Object.prototype.hasOwnProperty.call(MODEL_MAP, model)
             ? (model as keyof typeof MODEL_MAP)
             : null