feat(otel): support for custom resource attributes via config#telemetry.resource and OTEL_RESOURCE_ATTRIBUTES env var

.changeset/lucky-cameras-shave.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/sdk": patch
+"trigger.dev": patch
+---
+
+feat: add ability to set custom resource properties through trigger.config.ts or via the OTEL_RESOURCE_ATTRIBUTES env var

apps/webapp/app/presenters/v3/SpanPresenter.server.ts

@@ -497,7 +497,18 @@ export class SpanPresenter extends BasePresenter {
       duration: span.duration,
       events: span.events,
       style: span.style,
-      properties: span.properties ? JSON.stringify(span.properties, null, 2) : undefined,
+      properties:
+        span.properties &&
+        typeof span.properties === "object" &&
+        Object.keys(span.properties).length > 0
+          ? JSON.stringify(span.properties, null, 2)
+          : undefined,
+      resourceProperties:
+        span.resourceProperties &&
+        typeof span.resourceProperties === "object" &&
+        Object.keys(span.resourceProperties).length > 0
+          ? JSON.stringify(span.resourceProperties, null, 2)
+          : undefined,
       entity: span.entity,
       metadata: span.metadata,
       triggeredRuns,

apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route.tsx

@@ -1071,6 +1071,17 @@ function SpanEntity({ span }: { span: Span }) {
             showOpenInModal
           />
         ) : null}
+        {span.resourceProperties !== undefined ? (
+          <CodeBlock
+            rowTitle="Resource properties"
+            code={span.resourceProperties}
+            maxLines={20}
+            showLineNumbers={false}
+            showCopyButton
+            showTextWrapping
+            showOpenInModal
+          />
+        ) : null}
       </div>
     );
   }

apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts

@@ -825,12 +825,16 @@ export async function resolveVariablesForEnvironment(
   runtimeEnvironment: RuntimeEnvironmentForEnvRepo,
   parentEnvironment?: RuntimeEnvironmentForEnvRepo
 ) {
-  const projectSecrets = await environmentVariablesRepository.getEnvironmentVariables(
+  let projectSecrets = await environmentVariablesRepository.getEnvironmentVariables(
     runtimeEnvironment.projectId,
     runtimeEnvironment.id,
     parentEnvironment?.id
   );
 
+  projectSecrets = renameVariables(projectSecrets, {
+    OTEL_RESOURCE_ATTRIBUTES: "CUSTOM_OTEL_RESOURCE_ATTRIBUTES",
+  });
+
   const overridableTriggerVariables = await resolveOverridableTriggerVariables(runtimeEnvironment);
 
   const builtInVariables =
@@ -853,6 +857,15 @@ export async function resolveVariablesForEnvironment(
   return result;
 }
 
+function renameVariables(variables: EnvironmentVariable[], renameMap: Record<string, string>) {
+  return variables.map((variable) => {
+    return {
+      ...variable,
+      key: renameMap[variable.key] ?? variable.key,
+    };
+  });
+}
+
 async function resolveOverridableTriggerVariables(
   runtimeEnvironment: RuntimeEnvironmentForEnvRepo
 ) {

apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts

@@ -184,7 +184,10 @@ export class ClickhouseEventRepository implements IEventRepository {
         message: event.message,
         kind: this.createEventToTaskEventV1InputKind(event),
         status: this.createEventToTaskEventV1InputStatus(event),
-        attributes: this.createEventToTaskEventV1InputAttributes(event.properties),
+        attributes: this.createEventToTaskEventV1InputAttributes(
+          event.properties,
+          event.resourceProperties
+        ),
         metadata: this.createEventToTaskEventV1InputMetadata(event),
         expires_at: convertDateToClickhouseDateTime(
           new Date(Date.now() + 365 * 24 * 60 * 60 * 1000) // 1 year
@@ -392,7 +395,24 @@ export class ClickhouseEventRepository implements IEventRepository {
     return "OK";
   }
 
-  private createEventToTaskEventV1InputAttributes(attributes: Attributes): Record<string, unknown> {
+  private createEventToTaskEventV1InputAttributes(
+    attributes: Attributes,
+    resourceAttributes?: Attributes
+  ): Record<string, unknown> {
+    if (!attributes && !resourceAttributes) {
+      return {};
+    }
+
+    return {
+      ...this.createAttributesToInputAttributes(attributes),
+      ...this.createAttributesToInputAttributes(resourceAttributes, "$resource"),
+    };
+  }
+
+  private createAttributesToInputAttributes(
+    attributes: Attributes | undefined,
+    key?: string
+  ): Record<string, unknown> {
     if (!attributes) {
       return {};
     }
@@ -406,6 +426,12 @@ export class ClickhouseEventRepository implements IEventRepository {
     const unflattenedAttributes = unflattenAttributes(publicAttributes);
 
     if (unflattenedAttributes && typeof unflattenedAttributes === "object") {
+      if (key) {
+        return {
+          [key]: unflattenedAttributes,
+        };
+      }
+
       return {
         ...unflattenedAttributes,
       };
@@ -1103,6 +1129,7 @@ export class ClickhouseEventRepository implements IEventRepository {
           events: [],
           style: {},
           properties: undefined,
+          resourceProperties: undefined,
           entity: {
             type: undefined,
             id: undefined,
@@ -1177,8 +1204,19 @@ export class ClickhouseEventRepository implements IEventRepository {
         }
       }
 
-      if (!span.properties && typeof record.attributes_text === "string") {
-        span.properties = this.#parseAttributes(record.attributes_text);
+      if (
+        (span.properties == null ||
+          (typeof span.properties === "object" && Object.keys(span.properties).length === 0)) &&
+        typeof record.attributes_text === "string"
+      ) {
+        const parsedAttributes = this.#parseAttributes(record.attributes_text);
+        const resourceAttributes = parsedAttributes["$resource"];
+
+        // Remove the $resource key from the attributes
+        delete parsedAttributes["$resource"];
+
+        span.properties = parsedAttributes;
+        span.resourceProperties = resourceAttributes as Record<string, unknown> | undefined;
       }
     }
 

apps/webapp/app/v3/eventRepository/eventRepository.types.ts

@@ -53,6 +53,7 @@ export type CreateEventInput = Omit<
   | "links"
 > & {
   properties: Attributes;
+  resourceProperties?: Attributes;
   metadata: Attributes | undefined;
   style: Attributes | undefined;
 };
@@ -209,6 +210,7 @@ export type SpanDetail = {
   events: SpanEvents; // Timeline events, SpanEvents component
   style: TaskEventStyle; // Icons, variants, accessories (RunIcon, SpanTitle)
   properties: Record<string, unknown> | string | number | boolean | null | undefined; // Displayed as JSON in span properties (CodeBlock)
+  resourceProperties?: Record<string, unknown> | string | number | boolean | null | undefined; // Displayed as JSON in span resource properties (CodeBlock)
 
   // ============================================================================
   // Entity & Relationships

apps/webapp/app/v3/otlpExporter.server.ts

@@ -204,6 +204,24 @@ function convertLogsToCreateableEvents(
 
   const resourceProperties = extractEventProperties(resourceAttributes);
 
+  const userDefinedResourceAttributes = truncateAttributes(
+    convertKeyValueItemsToMap(resourceAttributes ?? [], [], undefined, [
+      SemanticInternalAttributes.USAGE,
+      SemanticInternalAttributes.SPAN,
+      SemanticInternalAttributes.METADATA,
+      SemanticInternalAttributes.STYLE,
+      SemanticInternalAttributes.METRIC_EVENTS,
+      SemanticInternalAttributes.TRIGGER,
+      "process",
+      "sdk",
+      "service",
+      "ctx",
+      "cli",
+      "cloud",
+    ]),
+    spanAttributeValueLengthLimit
+  );
+
   const taskEventStore =
     extractStringAttribute(resourceAttributes, [SemanticInternalAttributes.TASK_EVENT_STORE]) ??
     env.EVENT_REPOSITORY_DEFAULT_STORE;
@@ -249,6 +267,7 @@ function convertLogsToCreateableEvents(
           status: logLevelToEventStatus(log.severityNumber),
           startTime: log.timeUnixNano,
           properties,
+          resourceProperties: userDefinedResourceAttributes,
           style: convertKeyValueItemsToMap(
             pickAttributes(log.attributes ?? [], SemanticInternalAttributes.STYLE),
             []
@@ -285,6 +304,24 @@ function convertSpansToCreateableEvents(
 
   const resourceProperties = extractEventProperties(resourceAttributes);
 
+  const userDefinedResourceAttributes = truncateAttributes(
+    convertKeyValueItemsToMap(resourceAttributes ?? [], [], undefined, [
+      SemanticInternalAttributes.USAGE,
+      SemanticInternalAttributes.SPAN,
+      SemanticInternalAttributes.METADATA,
+      SemanticInternalAttributes.STYLE,
+      SemanticInternalAttributes.METRIC_EVENTS,
+      SemanticInternalAttributes.TRIGGER,
+      "process",
+      "sdk",
+      "service",
+      "ctx",
+      "cli",
+      "cloud",
+    ]),
+    spanAttributeValueLengthLimit
+  );
+
   const taskEventStore =
     extractStringAttribute(resourceAttributes, [SemanticInternalAttributes.TASK_EVENT_STORE]) ??
     env.EVENT_REPOSITORY_DEFAULT_STORE;
@@ -336,6 +373,7 @@ function convertSpansToCreateableEvents(
           events: spanEventsToEventEvents(span.events ?? []),
           duration: span.endTimeUnixNano - span.startTimeUnixNano,
           properties,
+          resourceProperties: userDefinedResourceAttributes,
           style: convertKeyValueItemsToMap(
             pickAttributes(span.attributes ?? [], SemanticInternalAttributes.STYLE),
             []

packages/cli-v3/src/dev/devSupervisor.ts

@@ -455,9 +455,6 @@ class DevSupervisor implements WorkerRuntime {
       TRIGGER_API_URL: this.options.client.apiURL,
       TRIGGER_SECRET_KEY: this.options.client.accessToken!,
       OTEL_EXPORTER_OTLP_COMPRESSION: "none",
-      OTEL_RESOURCE_ATTRIBUTES: JSON.stringify({
-        [SemanticInternalAttributes.PROJECT_DIR]: this.options.config.workingDir,
-      }),
       OTEL_IMPORT_HOOK_INCLUDES,
     };
   }

packages/cli-v3/src/entryPoints/dev-run-worker.ts

@@ -209,6 +209,7 @@ async function doBootstrap() {
       logExporters: config.telemetry?.logExporters ?? [],
       diagLogLevel: (env.TRIGGER_OTEL_LOG_LEVEL as TracingDiagnosticLogLevel) ?? "none",
       forceFlushTimeoutMillis: 30_000,
+      resource: config.telemetry?.resource,
     });
 
     const otelTracer: Tracer = tracingSDK.getTracer("trigger-dev-worker", VERSION);

packages/cli-v3/src/entryPoints/managed-run-worker.ts

@@ -188,6 +188,7 @@ async function doBootstrap() {
       forceFlushTimeoutMillis: 30_000,
       exporters: config.telemetry?.exporters ?? [],
       logExporters: config.telemetry?.logExporters ?? [],
+      resource: config.telemetry?.resource,
     });
 
     const otelTracer: Tracer = tracingSDK.getTracer("trigger-dev-worker", VERSION);

packages/cli-v3/src/utilities/dotEnv.ts

@@ -2,7 +2,13 @@ import dotenv from "dotenv";
 import { resolve } from "node:path";
 import { env } from "std-env";
 
-const ENVVAR_FILES = [".env", ".env.development", ".env.local", ".env.development.local", "dev.vars"];
+const ENVVAR_FILES = [
+  ".env",
+  ".env.development",
+  ".env.local",
+  ".env.development.local",
+  "dev.vars",
+];
 
 export function resolveDotEnvVars(cwd?: string, envFile?: string) {
   const result: { [key: string]: string } = {};
@@ -23,6 +29,11 @@ export function resolveDotEnvVars(cwd?: string, envFile?: string) {
   delete result.TRIGGER_SECRET_KEY;
   delete result.OTEL_EXPORTER_OTLP_ENDPOINT;
 
+  if (result.OTEL_RESOURCE_ATTRIBUTES) {
+    result.CUSTOM_OTEL_RESOURCE_ATTRIBUTES = result.OTEL_RESOURCE_ATTRIBUTES;
+    delete result.OTEL_RESOURCE_ATTRIBUTES;
+  }
+
   return result;
 }
 

packages/core/src/v3/config.ts

@@ -12,6 +12,7 @@ import type {
 import type { LogLevel } from "./logger/taskLogger.js";
 import type { MachinePresetName } from "./schemas/common.js";
 import { LogRecordExporter } from "@opentelemetry/sdk-logs";
+import type { Resource } from "@opentelemetry/resources";
 
 export type CompatibilityFlag = "run_engine_v2";
 
@@ -107,6 +108,13 @@ export type TriggerConfig = {
      * @see https://trigger.dev/docs/config/config-file#exporters
      */
     logExporters?: Array<LogRecordExporter>;
+
+    /**
+     * Resource to use for OpenTelemetry. This is useful if you want to add custom resources to your tasks.
+     *
+     * @see https://trigger.dev/docs/config/config-file#resource
+     */
+    resource?: Resource;
   };
 
   /**