Simplified metrics_v1 value columns into a single column, improved how we are filtering out system metrics in dev, removed a few of the event loop lag metrics

ericallam · ericallam · commit ad382c088556 · 2026-02-16T15:23:25.000Z
diff --git a/apps/webapp/app/components/query/QueryEditor.tsx b/apps/webapp/app/components/query/QueryEditor.tsx
@@ -1175,9 +1175,9 @@ function QueryResultsCallouts({
     <div className="flex flex-col gap-2 px-2 pt-2">
       {hiddenColumns && hiddenColumns.length > 0 && (
         <Callout variant="warning" className="shrink-0 text-sm">
-          <code>SELECT *</code> doesn't return all columns because it's slow. The following columns
-          are not shown: <span className="font-mono text-xs">{hiddenColumns.join(", ")}</span>.
-          Specify them explicitly to include them.
+          <code>SELECT *</code> returns core columns only. To include{" "}
+          <span className="font-mono text-xs">{hiddenColumns.join(", ")}</span>, add them to your
+          SELECT explicitly.
         </Callout>
       )}
       {periodClipped && (
diff --git a/apps/webapp/app/v3/otlpExporter.server.ts b/apps/webapp/app/v3/otlpExporter.server.ts
@@ -505,11 +505,7 @@ function convertMetricsToClickhouseRows(
             metric_type: "gauge",
             metric_subject: resolved.machineId ?? "unknown",
             bucket_start: floorToTenSecondBucket(dp.timeUnixNano),
-            count: 0,
-            sum_value: 0,
-            max_value: value,
-            min_value: value,
-            last_value: value,
+            value,
             attributes: resolved.attributes,
           });
         }
@@ -530,11 +526,7 @@ function convertMetricsToClickhouseRows(
             metric_type: "sum",
             metric_subject: resolved.machineId ?? "unknown",
             bucket_start: floorToTenSecondBucket(dp.timeUnixNano),
-            count: 1,
-            sum_value: value,
-            max_value: value,
-            min_value: value,
-            last_value: value,
+            value,
             attributes: resolved.attributes,
           });
         }
@@ -546,8 +538,6 @@ function convertMetricsToClickhouseRows(
           const resolved = resolveDataPointContext(dp.attributes ?? [], resourceCtx);
           const count = Number(dp.count);
           const sum = dp.sum ?? 0;
-          const max = dp.max ?? 0;
-          const min = dp.min ?? 0;
 
           rows.push({
             organization_id: organizationId,
@@ -557,11 +547,7 @@ function convertMetricsToClickhouseRows(
             metric_type: "histogram",
             metric_subject: resolved.machineId ?? "unknown",
             bucket_start: floorToTenSecondBucket(dp.timeUnixNano),
-            count,
-            sum_value: sum,
-            max_value: max,
-            min_value: min,
-            last_value: count > 0 ? sum / count : 0,
+            value: count > 0 ? sum / count : 0,
             attributes: resolved.attributes,
           });
         }
diff --git a/apps/webapp/app/v3/querySchemas.ts b/apps/webapp/app/v3/querySchemas.ts
@@ -498,44 +498,21 @@ export const metricsSchema: TableSchema = {
         coreColumn: true,
       }),
     },
-    count: {
-      name: "count",
-      ...column("UInt64", {
-        description: "Number of data points in this bucket",
-        example: "6",
-        coreColumn: true,
-      }),
-    },
-    sum_value: {
-      name: "sum_value",
-      ...column("Float64", {
-        description: "Sum of values in this bucket",
-        example: "0.45",
-        coreColumn: true,
-      }),
-    },
-    max_value: {
-      name: "max_value",
+    value: {
+      name: "value",
       ...column("Float64", {
-        description: "Maximum value in this bucket",
-        example: "0.85",
+        description: "The metric value",
+        example: "0.75",
         coreColumn: true,
       }),
     },
-    min_value: {
-      name: "min_value",
-      ...column("Float64", {
-        description: "Minimum value in this bucket",
-        example: "0.12",
-        coreColumn: true,
-      }),
-    },
-    last_value: {
-      name: "last_value",
-      ...column("Float64", {
-        description: "Last recorded value in this bucket",
-        example: "0.42",
-        coreColumn: true,
+
+    // Attributes (JSON column for user-defined and system attributes)
+    attributes: {
+      name: "attributes",
+      ...column("JSON", {
+        description: "JSON attributes attached to the metric data point.",
+        example: '{"region": "us-east-1"}',
       }),
     },
 
diff --git a/apps/webapp/app/v3/services/aiQueryService.server.ts b/apps/webapp/app/v3/services/aiQueryService.server.ts
@@ -453,8 +453,8 @@ Only use explicit \`toStartOfHour\`/\`toStartOfDay\` etc. if the user specifical
 - Filter by metric name: WHERE metric_name = 'process.cpu.utilization'
 - Filter by run: WHERE run_id = 'run_abc123'
 - Filter by task: WHERE task_identifier = 'my-task'
-- Available metric names: process.cpu.utilization, process.cpu.time, process.memory.usage, system.memory.usage, system.memory.utilization, system.network.io, system.network.dropped, system.network.errors, nodejs.event_loop.utilization, nodejs.event_loop.delay.p50, nodejs.event_loop.delay.p99, nodejs.event_loop.delay.max, nodejs.heap.used, nodejs.heap.total
-- Use max_value or last_value for gauges (CPU utilization, memory usage), sum_value for counters (CPU time, network IO)
+- Available metric names: process.cpu.utilization, process.cpu.time, process.memory.usage, system.memory.usage, system.memory.utilization, system.network.io, system.network.dropped, system.network.errors, nodejs.event_loop.utilization, nodejs.event_loop.delay.p95, nodejs.event_loop.delay.max, nodejs.heap.used, nodejs.heap.total
+- Use \`value\` — the metric's observed value
 - Use prettyFormat(expr, 'bytes') to tell the UI to format values as bytes (e.g., "1.50 GiB") — keeps values numeric for charts
 - Use prettyFormat(expr, 'percent') for percentage values
 - prettyFormat does NOT change the SQL — it only adds a display hint
@@ -464,7 +464,7 @@ Only use explicit \`toStartOfHour\`/\`toStartOfDay\` etc. if the user specifical
 
 \`\`\`sql
 -- CPU utilization over time for a task
-SELECT timeBucket(), task_identifier, prettyFormat(avg(max_value), 'percent') AS avg_cpu
+SELECT timeBucket(), task_identifier, prettyFormat(avg(value), 'percent') AS avg_cpu
 FROM metrics
 WHERE metric_name = 'process.cpu.utilization'
 GROUP BY timeBucket, task_identifier
@@ -474,7 +474,7 @@ LIMIT 1000
 
 \`\`\`sql
 -- Peak memory usage per run
-SELECT run_id, task_identifier, prettyFormat(max(max_value), 'bytes') AS peak_memory
+SELECT run_id, task_identifier, prettyFormat(max(value), 'bytes') AS peak_memory
 FROM metrics
 WHERE metric_name = 'process.memory.usage'
 GROUP BY run_id, task_identifier
@@ -589,7 +589,7 @@ LIMIT 1000
 ### Common Metrics Patterns
 - Filter by metric: WHERE metric_name = 'process.cpu.utilization'
 - Available metric names: process.cpu.utilization, process.cpu.time, process.memory.usage, system.memory.usage, system.memory.utilization, system.network.io, system.network.dropped, system.network.errors, nodejs.event_loop.utilization, nodejs.event_loop.delay.p50, nodejs.event_loop.delay.p99, nodejs.event_loop.delay.max, nodejs.heap.used, nodejs.heap.total
-- Use max_value or last_value for gauges (CPU utilization, memory usage), sum_value for counters (CPU time, network IO)
+- Use \`value\` — the metric's observed value
 - Use prettyFormat(expr, 'bytes') for memory metrics (including nodejs.heap.*), prettyFormat(expr, 'percent') for CPU utilization
 - prettyFormat does NOT change the SQL — it only adds a display hint for the UI
 
diff --git a/apps/webapp/package.json b/apps/webapp/package.json
@@ -70,7 +70,7 @@
     "@opentelemetry/exporter-logs-otlp-http": "0.203.0",
     "@opentelemetry/exporter-metrics-otlp-proto": "0.203.0",
     "@opentelemetry/exporter-trace-otlp-http": "0.203.0",
-    "@opentelemetry/host-metrics": "^0.36.0",
+    "@opentelemetry/host-metrics": "^0.37.0",
     "@opentelemetry/instrumentation": "0.203.0",
     "@opentelemetry/instrumentation-aws-sdk": "^0.57.0",
     "@opentelemetry/instrumentation-express": "^0.52.0",
diff --git a/internal-packages/clickhouse/schema/017_create_metrics_v1.sql b/internal-packages/clickhouse/schema/017_create_metrics_v1.sql
@@ -8,11 +8,7 @@ CREATE TABLE IF NOT EXISTS trigger_dev.metrics_v1
   metric_type         LowCardinality(String),
   metric_subject      String CODEC(ZSTD(1)),
   bucket_start        DateTime CODEC(Delta(4), ZSTD(1)),
-  count               UInt64 DEFAULT 0 CODEC(ZSTD(1)),
-  sum_value           Float64 DEFAULT 0 CODEC(ZSTD(1)),
-  max_value           Float64 DEFAULT 0 CODEC(ZSTD(1)),
-  min_value           Float64 DEFAULT 0 CODEC(ZSTD(1)),
-  last_value          Float64 DEFAULT 0 CODEC(ZSTD(1)),
+  value               Float64 DEFAULT 0 CODEC(ZSTD(1)),
   attributes          JSON(
     `trigger.run_id` String,
     `trigger.task_slug` String,
diff --git a/internal-packages/clickhouse/src/metrics.ts b/internal-packages/clickhouse/src/metrics.ts
@@ -9,11 +9,7 @@ export const MetricsV1Input = z.object({
   metric_type: z.string(),
   metric_subject: z.string(),
   bucket_start: z.string(),
-  count: z.number(),
-  sum_value: z.number(),
-  max_value: z.number(),
-  min_value: z.number(),
-  last_value: z.number(),
+  value: z.number(),
   attributes: z.unknown(),
 });
 
diff --git a/packages/cli-v3/src/entryPoints/dev-run-worker.ts b/packages/cli-v3/src/entryPoints/dev-run-worker.ts
@@ -213,9 +213,8 @@ async function doBootstrap() {
       forceFlushTimeoutMillis: 30_000,
       resource: config.telemetry?.resource,
       hostMetrics: true,
+      hostMetricGroups: ["process.cpu", "process.memory"],
       nodejsRuntimeMetrics: true,
-      // Drop all system metrics from dev metrics export
-      droppedMetrics: ["system.*"],
     });
 
     const otelTracer: Tracer = tracingSDK.getTracer("trigger-dev-worker", VERSION);
diff --git a/packages/core/package.json b/packages/core/package.json
@@ -177,7 +177,7 @@
     "@opentelemetry/core": "2.0.1",
     "@opentelemetry/exporter-logs-otlp-http": "0.203.0",
     "@opentelemetry/exporter-metrics-otlp-http": "0.203.0",
-    "@opentelemetry/host-metrics": "^0.36.0",
+    "@opentelemetry/host-metrics": "^0.37.0",
     "@opentelemetry/exporter-trace-otlp-http": "0.203.0",
     "@opentelemetry/instrumentation": "0.203.0",
     "@opentelemetry/resources": "2.0.1",
diff --git a/packages/core/src/v3/otel/nodejsRuntimeMetrics.ts b/packages/core/src/v3/otel/nodejsRuntimeMetrics.ts
@@ -16,12 +16,8 @@ export function startNodejsRuntimeMetrics(meterProvider: MeterProvider) {
   const eld = monitorEventLoopDelay({ resolution: 20 });
   eld.enable();
 
-  const eldP50 = meter.createObservableGauge("nodejs.event_loop.delay.p50", {
-    description: "Median event loop delay",
-    unit: "s",
-  });
-  const eldP99 = meter.createObservableGauge("nodejs.event_loop.delay.p99", {
-    description: "p99 event loop delay",
+  const eldP95 = meter.createObservableGauge("nodejs.event_loop.delay.p95", {
+    description: "p95 event loop delay",
     unit: "s",
   });
   const eldMax = meter.createObservableGauge("nodejs.event_loop.delay.max", {
@@ -49,8 +45,7 @@ export function startNodejsRuntimeMetrics(meterProvider: MeterProvider) {
       obs.observe(eluGauge, diff.utilization);
 
       // Event loop delay (nanoseconds -> seconds)
-      obs.observe(eldP50, eld.percentile(50) / 1e9);
-      obs.observe(eldP99, eld.percentile(99) / 1e9);
+      obs.observe(eldP95, eld.percentile(95) / 1e9);
       obs.observe(eldMax, eld.max / 1e9);
       eld.reset();
 
@@ -59,6 +54,6 @@ export function startNodejsRuntimeMetrics(meterProvider: MeterProvider) {
       obs.observe(heapUsed, mem.heapUsed);
       obs.observe(heapTotal, mem.heapTotal);
     },
-    [eluGauge, eldP50, eldP99, eldMax, heapUsed, heapTotal]
+    [eluGauge, eldP95, eldMax, heapUsed, heapTotal]
   );
 }
diff --git a/packages/core/src/v3/otel/tracingSDK.ts b/packages/core/src/v3/otel/tracingSDK.ts
@@ -85,6 +85,8 @@ export type TracingSDKConfig = {
   diagLogLevel?: TracingDiagnosticLogLevel;
   resource?: Resource;
   hostMetrics?: boolean;
+  /** Limit host metrics collection to specific groups (e.g. ["process.cpu", "process.memory"]) */
+  hostMetricGroups?: string[];
   /** Enable Node.js runtime metrics (event loop utilization, heap usage, etc.) */
   nodejsRuntimeMetrics?: boolean;
   /** Metric instrument name patterns to drop (supports wildcards, e.g. "system.cpu.*") */
@@ -327,7 +329,10 @@ export class TracingSDK {
     metrics.setGlobalMeterProvider(meterProvider);
 
     if (config.hostMetrics) {
-      const hostMetrics = new HostMetrics({ meterProvider });
+      const hostMetrics = new HostMetrics({
+        meterProvider,
+        metricGroups: config.hostMetricGroups,
+      });
       hostMetrics.start();
     }
 
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
