switch relace to use openrouter

Author: charleslien

backend/src/llm-apis/relace-api.ts

@@ -1,8 +1,6 @@
 import { models } from '@codebuff/common/old-constants'
-import {
-  createMarkdownFileBlock,
-  parseMarkdownCodeBlock,
-} from '@codebuff/common/util/file'
+import { buildArray } from '@codebuff/common/util/array'
+import { parseMarkdownCodeBlock } from '@codebuff/common/util/file'
 import { env } from '@codebuff/internal'
 
 import { saveMessage } from '../llm-apis/message-cost-tracker'
@@ -22,82 +20,31 @@ export async function promptRelaceAI(
     initialCode: string
     editSnippet: string
     instructions: string | undefined
-    messageId: string
-    userMessage?: string
     promptAiSdk: PromptAiSdkFn
     logger: Logger
-  } & ParamsExcluding<
-    typeof saveMessage,
-    | 'model'
-    | 'request'
-    | 'response'
-    | 'inputTokens'
-    | 'outputTokens'
-    | 'finishedAt'
-    | 'latencyMs'
-  > &
-    ParamsExcluding<PromptAiSdkFn, 'messages' | 'model'>,
+  } & ParamsExcluding<PromptAiSdkFn, 'messages' | 'model'>,
 ) {
-  const {
-    initialCode,
-    editSnippet,
-    instructions,
-    userMessage,
-    messageId,
-    promptAiSdk,
-    logger,
-  } = params
-  const startTime = Date.now()
+  const { initialCode, editSnippet, instructions, promptAiSdk, logger } = params
 
   try {
     // const model = 'relace-apply-2.5-lite'
-    const response = (await Promise.race([
-      fetch('https://instantapply.endpoint.relace.run/v1/code/apply', {
-        method: 'POST',
-        headers: {
-          'Content-Type': 'application/json',
-          Authorization: `Bearer ${env.RELACE_API_KEY}`,
-        },
-        body: JSON.stringify({
-          // model,
-          initialCode,
-          editSnippet,
-          ...(instructions ? { instructions } : {}),
-          stream: false,
-          'relace-metadata': {
-            'codebuff-id': messageId,
-            'codebuff-user-prompt': userMessage,
-          },
-        }),
-      }),
-      timeoutPromise(100_000),
-    ])) as Response
-
-    if (!response.ok) {
-      throw new Error(
-        `Relace API error: ${response.status} ${response.statusText}`,
-      )
-    }
-
-    const data = (await response.json()) as { mergedCode: string }
-    const content = data.mergedCode
-
-    const fakeRequestContent = `Initial code:${createMarkdownFileBlock('', initialCode)}\n\nEdit snippet${createMarkdownFileBlock('', editSnippet)}`
-    saveMessage({
+    const content = await promptAiSdk({
       ...params,
-      model: 'relace-fast-apply',
-      request: [
+      model: 'relace/relace-apply-3',
+      messages: [
         {
           role: 'user',
-          content: fakeRequestContent,
+          content: buildArray(
+            instructions && `<instruction>${instructions}</instruction>`,
+            `<code>${initialCode}</code>`,
+            `<update>${editSnippet}</update>`,
+          ).join('\n'),
         },
       ],
-      response: content,
-      inputTokens: countTokens(initialCode + editSnippet),
-      outputTokens: countTokens(content),
-      finishedAt: new Date(),
-      latencyMs: Date.now() - startTime,
+      system: undefined,
+      includeCacheControl: false,
     })
+
     return content + '\n'
   } catch (error) {
     logger.error(

backend/src/llm-apis/vercel-ai-sdk/ai-sdk.ts

@@ -8,7 +8,6 @@ import { getErrorObject } from '@codebuff/common/util/error'
 import { convertCbToModelMessages } from '@codebuff/common/util/messages'
 import { withTimeout } from '@codebuff/common/util/promise'
 import { StopSequenceHandler } from '@codebuff/common/util/stop-sequence'
-import { generateCompactId } from '@codebuff/common/util/string'
 import { APICallError, generateObject, generateText, streamText } from 'ai'
 
 import { checkLiveUserInput, getLiveUserInputIds } from '../../live-user-inputs'
@@ -252,19 +251,52 @@ export async function promptAiSdk(
     messages: convertCbToModelMessages(params),
   })
   const content = response.text
-  const inputTokens = response.usage.inputTokens || 0
-  const outputTokens = response.usage.inputTokens || 0
+
+  const messageId = response.response.id
+  const providerMetadata = response.providerMetadata ?? {}
+  const usage = response.usage
+  let inputTokens = usage.inputTokens || 0
+  const outputTokens = usage.outputTokens || 0
+  let cacheReadInputTokens: number = 0
+  let cacheCreationInputTokens: number = 0
+  let costOverrideDollars: number | undefined
+  if (providerMetadata.anthropic) {
+    cacheReadInputTokens =
+      typeof providerMetadata.anthropic.cacheReadInputTokens === 'number'
+        ? providerMetadata.anthropic.cacheReadInputTokens
+        : 0
+    cacheCreationInputTokens =
+      typeof providerMetadata.anthropic.cacheCreationInputTokens === 'number'
+        ? providerMetadata.anthropic.cacheCreationInputTokens
+        : 0
+  }
+  if (providerMetadata.openrouter) {
+    if (providerMetadata.openrouter.usage) {
+      const openrouterUsage = providerMetadata.openrouter
+        .usage as OpenRouterUsageAccounting
+      cacheReadInputTokens =
+        openrouterUsage.promptTokensDetails?.cachedTokens ?? 0
+      inputTokens = openrouterUsage.promptTokens - cacheReadInputTokens
+
+      costOverrideDollars =
+        (openrouterUsage.cost ?? 0) +
+        (openrouterUsage.costDetails?.upstreamInferenceCost ?? 0)
+    }
+  }
 
   const creditsUsedPromise = saveMessage({
     ...params,
-    messageId: generateCompactId(),
+    messageId,
     request: params.messages,
     response: content,
     inputTokens,
     outputTokens,
+    cacheCreationInputTokens,
+    cacheReadInputTokens,
     finishedAt: new Date(),
     latencyMs: Date.now() - startTime,
     chargeUser: params.chargeUser ?? true,
+    costOverrideDollars,
   })
 
   // Call the cost callback if provided
@@ -308,20 +340,52 @@ export async function promptAiSdkStructured<T>(
     ? responsePromise
     : withTimeout(responsePromise, params.timeout))
   const content = response.object
-  const inputTokens = response.usage.inputTokens || 0
-  const outputTokens = response.usage.inputTokens || 0
+
+  const messageId = response.response.id
+  const providerMetadata = response.providerMetadata ?? {}
+  const usage = response.usage
+  let inputTokens = usage.inputTokens || 0
+  const outputTokens = usage.outputTokens || 0
+  let cacheReadInputTokens: number = 0
+  let cacheCreationInputTokens: number = 0
+  let costOverrideDollars: number | undefined
+  if (providerMetadata.anthropic) {
+    cacheReadInputTokens =
+      typeof providerMetadata.anthropic.cacheReadInputTokens === 'number'
+        ? providerMetadata.anthropic.cacheReadInputTokens
+        : 0
+    cacheCreationInputTokens =
+      typeof providerMetadata.anthropic.cacheCreationInputTokens === 'number'
+        ? providerMetadata.anthropic.cacheCreationInputTokens
+        : 0
+  }
+  if (providerMetadata.openrouter) {
+    if (providerMetadata.openrouter.usage) {
+      const openrouterUsage = providerMetadata.openrouter
+        .usage as OpenRouterUsageAccounting
+      cacheReadInputTokens =
+        openrouterUsage.promptTokensDetails?.cachedTokens ?? 0
+      inputTokens = openrouterUsage.promptTokens - cacheReadInputTokens
+
+      costOverrideDollars =
+        (openrouterUsage.cost ?? 0) +
+        (openrouterUsage.costDetails?.upstreamInferenceCost ?? 0)
+    }
+  }
 
   const creditsUsedPromise = saveMessage({
     ...params,
-    messageId: generateCompactId(),
+    messageId,
     request: params.messages,
     response: JSON.stringify(content),
     inputTokens,
     outputTokens,
+    cacheCreationInputTokens,
+    cacheReadInputTokens,
     finishedAt: new Date(),
     latencyMs: Date.now() - startTime,
     chargeUser: params.chargeUser ?? true,
-    logger,
+    costOverrideDollars,
   })
 
   // Call the cost callback if provided