implement semantic system filesystem and diskio metrics for nodejs and bun

Author: ericallam

internal-packages/clickhouse/schema/017_create_metrics_v1.sql

@@ -22,6 +22,11 @@ CREATE TABLE IF NOT EXISTS trigger_dev.metrics_v1
     `system.cpu.state` LowCardinality(String),
     `system.memory.state` LowCardinality(String),
     `system.device` String,
+    `system.filesystem.type` LowCardinality(String),
+    `system.filesystem.mountpoint` String,
+    `system.filesystem.mode` LowCardinality(String),
+    `system.filesystem.state` LowCardinality(String),
+    `disk.io.direction` LowCardinality(String),
     `process.cpu.state` LowCardinality(String),
     `network.io.direction` LowCardinality(String),
     max_dynamic_paths=8

packages/cli-v3/src/entryPoints/managed-run-worker.ts

@@ -193,6 +193,8 @@ async function doBootstrap() {
       resource: config.telemetry?.resource,
       hostMetrics: true,
       nodejsRuntimeMetrics: true,
+      filesystemMetrics: true,
+      diskIoMetrics: true,
     });
 
     const otelTracer: Tracer = tracingSDK.getTracer("trigger-dev-worker", VERSION);

packages/core/src/v3/otel/diskIoMetrics.ts

@@ -0,0 +1,92 @@
+import { type MeterProvider } from "@opentelemetry/sdk-metrics";
+import * as fs from "node:fs";
+import * as fsPromises from "node:fs/promises";
+
+const SECTOR_SIZE = 512;
+
+const FILTERED_DEVICE_PREFIXES = ["loop", "ram", "dm-"];
+
+type DiskStats = {
+  device: string;
+  readsCompleted: number;
+  sectorsRead: number;
+  writesCompleted: number;
+  sectorsWritten: number;
+};
+
+function parseProcDiskstats(content: string): DiskStats[] {
+  const entries: DiskStats[] = [];
+
+  for (const line of content.split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+
+    const fields = trimmed.split(/\s+/);
+    if (fields.length < 14) continue;
+
+    const device = fields[2]!;
+
+    if (FILTERED_DEVICE_PREFIXES.some((prefix) => device.startsWith(prefix))) {
+      continue;
+    }
+
+    entries.push({
+      device,
+      readsCompleted: parseInt(fields[3]!, 10),
+      sectorsRead: parseInt(fields[5]!, 10),
+      writesCompleted: parseInt(fields[7]!, 10),
+      sectorsWritten: parseInt(fields[9]!, 10),
+    });
+  }
+
+  return entries;
+}
+
+export function startDiskIoMetrics(meterProvider: MeterProvider) {
+  try {
+    fs.accessSync("/proc/diskstats", fs.constants.R_OK);
+  } catch {
+    return;
+  }
+
+  const meter = meterProvider.getMeter("system-disk", "1.0.0");
+
+  const ioCounter = meter.createObservableCounter("system.disk.io", {
+    description: "Disk I/O bytes read and written per device",
+    unit: "By",
+  });
+
+  const opsCounter = meter.createObservableCounter("system.disk.operations", {
+    description: "Disk read/write operation counts per device",
+    unit: "{operation}",
+  });
+
+  meter.addBatchObservableCallback(
+    async (obs) => {
+      try {
+        const content = await fsPromises.readFile("/proc/diskstats", "utf-8");
+        const stats = parseProcDiskstats(content);
+
+        for (const entry of stats) {
+          const readAttrs = {
+            "system.device": entry.device,
+            "disk.io.direction": "read",
+          };
+          const writeAttrs = {
+            "system.device": entry.device,
+            "disk.io.direction": "write",
+          };
+
+          obs.observe(ioCounter, entry.sectorsRead * SECTOR_SIZE, readAttrs);
+          obs.observe(ioCounter, entry.sectorsWritten * SECTOR_SIZE, writeAttrs);
+
+          obs.observe(opsCounter, entry.readsCompleted, readAttrs);
+          obs.observe(opsCounter, entry.writesCompleted, writeAttrs);
+        }
+      } catch {
+        // Skip entire cycle on failure
+      }
+    },
+    [ioCounter, opsCounter]
+  );
+}

packages/core/src/v3/otel/filesystemMetrics.ts

@@ -0,0 +1,134 @@
+import { type MeterProvider } from "@opentelemetry/sdk-metrics";
+import * as fs from "node:fs";
+import * as fsPromises from "node:fs/promises";
+
+const VIRTUAL_FS_TYPES = new Set([
+  "proc",
+  "sysfs",
+  "devpts",
+  "tmpfs",
+  "devtmpfs",
+  "cgroup",
+  "cgroup2",
+  "squashfs",
+  "autofs",
+  "debugfs",
+  "securityfs",
+  "pstore",
+  "bpf",
+  "tracefs",
+  "hugetlbfs",
+  "mqueue",
+  "fusectl",
+  "configfs",
+  "binfmt_misc",
+]);
+
+type MountEntry = {
+  device: string;
+  mountpoint: string;
+  fsType: string;
+  options: string;
+};
+
+function parseProcMounts(content: string): MountEntry[] {
+  const entries: MountEntry[] = [];
+
+  for (const line of content.split("\n")) {
+    if (!line.trim()) continue;
+
+    const parts = line.split(" ");
+    if (parts.length < 4) continue;
+
+    const fsType = parts[2]!;
+    if (VIRTUAL_FS_TYPES.has(fsType)) continue;
+
+    entries.push({
+      device: parts[0]!,
+      mountpoint: unescapeMountPath(parts[1]!),
+      fsType,
+      options: parts[3]!,
+    });
+  }
+
+  return entries;
+}
+
+function unescapeMountPath(path: string): string {
+  return path.replace(/\\040/g, " ").replace(/\\011/g, "\t");
+}
+
+export function startFilesystemMetrics(meterProvider: MeterProvider) {
+  try {
+    fs.accessSync("/proc/mounts", fs.constants.R_OK);
+  } catch {
+    return;
+  }
+
+  if (typeof fsPromises.statfs !== "function") {
+    return;
+  }
+
+  const meter = meterProvider.getMeter("system-filesystem", "1.0.0");
+
+  const usageCounter = meter.createObservableUpDownCounter("system.filesystem.usage", {
+    description: "Filesystem bytes used, free, and reserved per mountpoint",
+    unit: "By",
+  });
+
+  const utilizationGauge = meter.createObservableGauge("system.filesystem.utilization", {
+    description: "Fraction of filesystem space used (0-1)",
+    unit: "1",
+  });
+
+  meter.addBatchObservableCallback(
+    async (obs) => {
+      try {
+        const mountsContent = await fsPromises.readFile("/proc/mounts", "utf-8");
+        const mounts = parseProcMounts(mountsContent);
+
+        for (const mount of mounts) {
+          try {
+            const stats = await fsPromises.statfs(mount.mountpoint);
+            const bsize = stats.bsize;
+            const total = stats.blocks * bsize;
+            const free = stats.bavail * bsize;
+            const reserved = (stats.bfree - stats.bavail) * bsize;
+            const used = total - stats.bfree * bsize;
+
+            const mode = mount.options.startsWith("ro") ? "ro" : "rw";
+
+            const baseAttrs = {
+              "system.device": mount.device,
+              "system.filesystem.type": mount.fsType,
+              "system.filesystem.mountpoint": mount.mountpoint,
+              "system.filesystem.mode": mode,
+            };
+
+            obs.observe(usageCounter, used, {
+              ...baseAttrs,
+              "system.filesystem.state": "used",
+            });
+            obs.observe(usageCounter, free, {
+              ...baseAttrs,
+              "system.filesystem.state": "free",
+            });
+            obs.observe(usageCounter, reserved, {
+              ...baseAttrs,
+              "system.filesystem.state": "reserved",
+            });
+
+            if (total > 0) {
+              obs.observe(utilizationGauge, used / total, baseAttrs);
+            }
+          } catch {
+            // Skip this mount on statfs failure
+          }
+        }
+      } catch {
+        // Skip entire cycle on failure
+      }
+    },
+    [usageCounter, utilizationGauge]
+  );
+}

packages/core/src/v3/otel/tracingSDK.ts

@@ -63,6 +63,8 @@ import {
 import { traceContext } from "../trace-context-api.js";
 import { getEnvVar } from "../utils/getEnv.js";
 import { machineId } from "./machineId.js";
+import { startDiskIoMetrics } from "./diskIoMetrics.js";
+import { startFilesystemMetrics } from "./filesystemMetrics.js";
 import { startNodejsRuntimeMetrics } from "./nodejsRuntimeMetrics.js";
 
 export type TracingDiagnosticLogLevel =
@@ -89,6 +91,10 @@ export type TracingSDKConfig = {
   hostMetricGroups?: string[];
   /** Enable Node.js runtime metrics (event loop utilization, heap usage, etc.) */
   nodejsRuntimeMetrics?: boolean;
+  /** Enable filesystem metrics (Linux only, reads /proc/mounts + fs.statfs) */
+  filesystemMetrics?: boolean;
+  /** Enable disk I/O metrics (Linux only, reads /proc/diskstats) */
+  diskIoMetrics?: boolean;
   /** Metric instrument name patterns to drop (supports wildcards, e.g. "system.cpu.*") */
   droppedMetrics?: string[];
 };
@@ -340,6 +346,14 @@ export class TracingSDK {
       startNodejsRuntimeMetrics(meterProvider);
     }
 
+    if (config.filesystemMetrics) {
+      startFilesystemMetrics(meterProvider);
+    }
+
+    if (config.diskIoMetrics) {
+      startDiskIoMetrics(meterProvider);
+    }
+
     this.getLogger = loggerProvider.getLogger.bind(loggerProvider);
     this.getTracer = traceProvider.getTracer.bind(traceProvider);
   }