diff --git a/package.json b/package.json index e8b1eb1a..faa9ae9c 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,7 @@ "@thirdweb-dev/auth": "^4.1.87", "@thirdweb-dev/chains": "^0.1.77", "@thirdweb-dev/sdk": "^4.0.89", - "@thirdweb-dev/service-utils": "^0.4.28", + "@thirdweb-dev/service-utils": "^0.8.8-nightly-cad9c74befe33bef2cc6a12a8ade8d9c996ffe41-20250209100117", "@types/base-64": "^1.0.2", "aws-kms-signer": "^0.5.3", "base-64": "^1.0.0", diff --git a/src/shared/utils/env.ts b/src/shared/utils/env.ts index c048630d..04993f40 100644 --- a/src/shared/utils/env.ts +++ b/src/shared/utils/env.ts @@ -54,6 +54,7 @@ export const env = createEnv({ CLIENT_ANALYTICS_URL: z .union([UrlSchema, z.literal("")]) .default("https://c.thirdweb.com/event"), + ENABLE_USAGE_V2_ANALYTICS: boolEnvSchema(true), SDK_BATCH_TIME_LIMIT: z.coerce.number().default(0), SDK_BATCH_SIZE_LIMIT: z.coerce.number().default(100), REDIS_URL: z.string(), @@ -120,6 +121,7 @@ export const env = createEnv({ HTTPS_PASSPHRASE: process.env.HTTPS_PASSPHRASE, TRUST_PROXY: process.env.TRUST_PROXY, CLIENT_ANALYTICS_URL: process.env.CLIENT_ANALYTICS_URL, + ENABLE_USAGE_V2_ANALYTICS: process.env.ENABLE_USAGE_V2_ANALYTICS, SDK_BATCH_TIME_LIMIT: process.env.SDK_BATCH_TIME_LIMIT, SDK_BATCH_SIZE_LIMIT: process.env.SDK_BATCH_SIZE_LIMIT, REDIS_URL: process.env.REDIS_URL, diff --git a/src/shared/utils/usage.ts b/src/shared/utils/usage.ts index 64ec568e..33cd09fb 100644 --- a/src/shared/utils/usage.ts +++ b/src/shared/utils/usage.ts @@ -1,5 +1,9 @@ import type { Static } from "@sinclair/typebox"; -import type { UsageEvent } from "@thirdweb-dev/service-utils/cf-worker"; +import { + sendUsageV2Events, + type ClientUsageV2Event, + type UsageEvent, +} from "@thirdweb-dev/service-utils/cf-worker"; import type { FastifyInstance } from "fastify"; import type { Address, Hex } from "thirdweb"; import { ADMIN_QUEUES_BASEPATH } from "../../server/middleware/admin-routes"; @@ -11,7 +15,7 @@ import { env } from "./env"; import { logger } from "./logger"; import { thirdwebClientId } from "./sdk"; -export interface ReportUsageParams { +interface ReportTransactionUsageParams { action: | "queue_tx" | "not_send_tx" @@ -26,7 +30,7 @@ export interface ReportUsageParams { to?: Address; value?: bigint; transactionHash?: Hex; - onChainTxStatus?: number; + onchainStatus?: "success" | "reverted"; userOpHash?: Hex; functionName?: string; extension?: string; @@ -38,13 +42,6 @@ export interface ReportUsageParams { error?: string; } -const ANALYTICS_DEFAULT_HEADERS = { - "Content-Type": "application/json", - "x-sdk-version": process.env.ENGINE_VERSION, - "x-product-name": "engine", - "x-client-id": thirdwebClientId, -} as HeadersInit; - const SKIP_USAGE_PATHS = new Set([ "", "/", @@ -55,8 +52,8 @@ const SKIP_USAGE_PATHS = new Set([ ]); export function withServerUsageReporting(server: FastifyInstance) { - // Skip reporting if CLIENT_ANALYTICS_URL is unset. - if (env.CLIENT_ANALYTICS_URL === "") { + // Skip reporting if analytics reporting is disabled. + if (!env.CLIENT_ANALYTICS_URL && !env.ENABLE_USAGE_V2_ANALYTICS) { return; } @@ -79,66 +76,113 @@ export function withServerUsageReporting(server: FastifyInstance) { ? await getChainIdFromChain(requestParams.chain) : undefined; - const requestBody: UsageEvent = { - source: "engine", - action: "api_request", - clientId: thirdwebClientId, - pathname: reply.request.routeOptions.url, - chainId, - walletAddress: requestParams?.walletAddress, - contractAddress: requestParams?.contractAddress, - httpStatusCode: reply.statusCode, - msTotalDuration: Math.ceil(reply.elapsedTime), - httpMethod: request.method.toUpperCase() as UsageEvent["httpMethod"], - }; - - fetch(env.CLIENT_ANALYTICS_URL, { - method: "POST", - headers: ANALYTICS_DEFAULT_HEADERS, - body: JSON.stringify(requestBody), - }).catch(() => {}); // Catch uncaught exceptions since this fetch call is non-blocking. + await reportUsageV1([ + { + source: "engine", + action: "api_request", + clientId: thirdwebClientId, + pathname: reply.request.routeOptions.url, + chainId, + walletAddress: requestParams?.walletAddress, + contractAddress: requestParams?.contractAddress, + httpStatusCode: reply.statusCode, + msTotalDuration: Math.ceil(reply.elapsedTime), + httpMethod: request.method.toUpperCase() as UsageEvent["httpMethod"], + }, + ]); + // @TODO: UsageV2 reporting }); } -export const reportUsage = (usageEvents: ReportUsageParams[]) => { - // Skip reporting if CLIENT_ANALYTICS_URL is unset. - if (env.CLIENT_ANALYTICS_URL === "") { - return; +/** + * Reports usage events. + * This method must not throw uncaught exceptions and is safe to call non-blocking. + */ +export async function reportUsage(events: ReportTransactionUsageParams[]) { + try { + if (env.CLIENT_ANALYTICS_URL) { + await reportUsageV1( + events.map(({ action, input, error }) => ({ + source: "engine", + action, + clientId: thirdwebClientId, + chainId: input.chainId, + walletAddress: input.from, + contractAddress: input.to, + transactionValue: input.value?.toString(), + transactionHash: input.transactionHash, + userOpHash: input.userOpHash, + errorCode: + input.onchainStatus === "reverted" ? "EXECUTION_REVERTED" : error, + functionName: input.functionName, + extension: input.extension, + retryCount: input.retryCount, + provider: input.provider, + msSinceQueue: input.msSinceQueue, + msSinceSend: input.msSinceSend, + })), + ); + } + if (env.ENABLE_USAGE_V2_ANALYTICS) { + await reportUsageV2( + events.map((event) => ({ + action: event.action, + sdk_platform: env.ENGINE_TIER ?? "SELF_HOSTED", + sdk_version: env.ENGINE_VERSION, + product_name: "engine", + chain_id: event.input.chainId, + from_address: event.input.from, + to_address: event.input.to, + value: event.input.value?.toString(), + transaction_hash: event.input.transactionHash, + user_op_hash: event.input.userOpHash, + function_name: event.input.functionName, + retry_count: event.input.retryCount, + rpc_provider: event.input.provider, + ms_since_queue: event.input.msSinceQueue, + ms_since_send: event.input.msSinceSend, + onchain_status: event.input.onchainStatus, + })), + ); + } + } catch (error) { + logger({ + service: "server", + level: "error", + message: "reportUsage error", + error, + }); } +} - usageEvents.map(async ({ action, input, error }) => { - try { - const requestBody: UsageEvent = { - source: "engine", - action, - clientId: thirdwebClientId, - chainId: input.chainId, - walletAddress: input.from, - contractAddress: input.to, - transactionValue: input.value?.toString(), - transactionHash: input.transactionHash, - userOpHash: input.userOpHash, - errorCode: input.onChainTxStatus === 0 ? "EXECUTION_REVERTED" : error, - functionName: input.functionName, - extension: input.extension, - retryCount: input.retryCount, - provider: input.provider, - msSinceQueue: input.msSinceQueue, - msSinceSend: input.msSinceSend, - }; +// Generic usageV1 helper. +async function reportUsageV1(events: UsageEvent[]) { + const BATCH_SIZE = 20; + for (let i = 0; i < events.length; i += BATCH_SIZE) { + const batch = events.slice(i, i + BATCH_SIZE); - fetch(env.CLIENT_ANALYTICS_URL, { - method: "POST", - headers: ANALYTICS_DEFAULT_HEADERS, - body: JSON.stringify(requestBody), - }).catch(() => {}); // Catch uncaught exceptions since this fetch call is non-blocking. - } catch (e) { - logger({ - service: "worker", - level: "error", - message: "Error:", - error: e, - }); - } + await Promise.allSettled( + batch.map(async (event) => { + await fetch(env.CLIENT_ANALYTICS_URL, { + method: "POST", + headers: { + "Content-Type": "application/json", + "x-sdk-version": process.env.ENGINE_VERSION ?? "", + "x-product-name": "engine", + "x-client-id": thirdwebClientId, + }, + body: JSON.stringify(event), + }); + }), + ); + } +} + +// Generic usageV2 helper. +async function reportUsageV2(events: ClientUsageV2Event[]) { + await sendUsageV2Events(events, { + environment: env.NODE_ENV === "production" ? "production" : "development", + source: "engine", + thirdwebSecretKey: env.THIRDWEB_API_SECRET_KEY, }); -}; +} diff --git a/yarn.lock b/yarn.lock index 8e200fea..0a02b272 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3983,13 +3983,15 @@ yaml "^2.4.1" zod "^3.22.4" -"@thirdweb-dev/service-utils@^0.4.28": - version "0.4.52" - resolved "https://registry.yarnpkg.com/@thirdweb-dev/service-utils/-/service-utils-0.4.52.tgz#6473ab0fabf7e9ee138776855c5bb4729f88efdb" - integrity sha512-24g+juzurNBRQ4llwQBbfQE/msQplFTbnDsp2ihxkb9KThTwI5+d/jWvEoXumFDWm3Ji+FDBJIbNaY9H2Fz6AQ== +"@thirdweb-dev/service-utils@^0.8.8-nightly-cad9c74befe33bef2cc6a12a8ade8d9c996ffe41-20250209100117": + version "0.8.8-nightly-cad9c74befe33bef2cc6a12a8ade8d9c996ffe41-20250209100117" + resolved "https://registry.yarnpkg.com/@thirdweb-dev/service-utils/-/service-utils-0.8.8-nightly-cad9c74befe33bef2cc6a12a8ade8d9c996ffe41-20250209100117.tgz#b109e191e2e685977809447e7a6d80ca99594526" + integrity sha512-NZHxg/lBipF3dLl3P8HMA4FASdOLjmyAXQFlsFFaWpP9LgawPwauwzLOD79Ym+MnpHVbIFn30pZbus9Ob5WTqg== dependencies: aws4fetch "1.0.20" - zod "3.23.8" + kafkajs "2.2.4" + lz4js "0.2.0" + zod "3.24.1" "@thirdweb-dev/storage@2.0.15": version "2.0.15" @@ -7972,6 +7974,11 @@ jws@^4.0.0: jwa "^2.0.0" safe-buffer "^5.0.1" +kafkajs@2.2.4: + version "2.2.4" + resolved "https://registry.yarnpkg.com/kafkajs/-/kafkajs-2.2.4.tgz#59e6e16459d87fdf8b64be73970ed5aa42370a5b" + integrity sha512-j/YeapB1vfPT2iOIUn/vxdyKEuhuY2PxMBvf5JWux6iSaukAccrMtXEY/Lb7OvavDhOWME589bpLrEdnVHjfjA== + keccak@^3.0.0, keccak@^3.0.2, keccak@^3.0.3: version "3.0.4" resolved "https://registry.yarnpkg.com/keccak/-/keccak-3.0.4.tgz#edc09b89e633c0549da444432ecf062ffadee86d" @@ -8260,6 +8267,11 @@ luxon@^3.2.1: resolved "https://registry.yarnpkg.com/luxon/-/luxon-3.5.0.tgz#6b6f65c5cd1d61d1fd19dbf07ee87a50bf4b8e20" integrity sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ== +lz4js@0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/lz4js/-/lz4js-0.2.0.tgz#09f1a397cb2158f675146c3351dde85058cb322f" + integrity sha512-gY2Ia9Lm7Ep8qMiuGRhvUq0Q7qUereeldZPP1PMEJxPtEWHJLqw9pgX68oHajBH0nzJK4MaZEA/YNV3jT8u8Bg== + magic-sdk@^13.6.2: version "13.6.2" resolved "https://registry.yarnpkg.com/magic-sdk/-/magic-sdk-13.6.2.tgz#68766fd9d1805332d2a00e5da1bd30fce251a6ac" @@ -11357,12 +11369,7 @@ zod-to-json-schema@^3.23.0: resolved "https://registry.yarnpkg.com/zod-to-json-schema/-/zod-to-json-schema-3.24.1.tgz#f08c6725091aadabffa820ba8d50c7ab527f227a" integrity sha512-3h08nf3Vw3Wl3PK+q3ow/lIil81IT2Oa7YpQyUUDsEWbXveMesdfK1xBd2RhCkynwZndAxixji/7SYJJowr62w== -zod@3.23.8: - version "3.23.8" - resolved "https://registry.yarnpkg.com/zod/-/zod-3.23.8.tgz#e37b957b5d52079769fb8097099b592f0ef4067d" - integrity sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g== - -zod@^3.21.4, zod@^3.22.4, zod@^3.23.8: +zod@3.24.1, zod@^3.21.4, zod@^3.22.4, zod@^3.23.8: version "3.24.1" resolved "https://registry.yarnpkg.com/zod/-/zod-3.24.1.tgz#27445c912738c8ad1e9de1bea0359fa44d9d35ee" integrity sha512-muH7gBL9sI1nciMZV67X5fTKKBLtwpZ5VBp1vsOQzj1MhrBZ4wlVCm3gedKZWLp0Oyel8sIGfeiz54Su+OVT+A==