From ebee6f06e3f0e1d5ca4bd44e15f1d6093775ea47 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Fri, 9 Jan 2026 23:21:50 +1100 Subject: [PATCH] Add OTEL exporter example --- examples/features/otel-export/README.md | 103 +++++++ examples/features/otel-export/bun.lock | 119 ++++++++ .../export-jsonl-to-confident-otel.ts | 3 + .../otel-export/export-jsonl-to-otel.ts | 253 ++++++++++++++++++ examples/features/otel-export/package.json | 15 ++ .../features/otel-export/sample-results.jsonl | 2 + 6 files changed, 495 insertions(+) create mode 100644 examples/features/otel-export/README.md create mode 100644 examples/features/otel-export/bun.lock create mode 100644 examples/features/otel-export/export-jsonl-to-confident-otel.ts create mode 100644 examples/features/otel-export/export-jsonl-to-otel.ts create mode 100644 examples/features/otel-export/package.json create mode 100644 examples/features/otel-export/sample-results.jsonl diff --git a/examples/features/otel-export/README.md b/examples/features/otel-export/README.md new file mode 100644 index 0000000..91a7b74 --- /dev/null +++ b/examples/features/otel-export/README.md @@ -0,0 +1,103 @@ +# JSONL → OpenTelemetry export (Confident AI or Langfuse) + +This example shows how to take AgentV JSONL results (produced by `agentv eval --output-format jsonl`) and send them to a backend via OpenTelemetry (OTLP over HTTP). + +It is intentionally **standalone** and does not modify AgentV core. + +## Prerequisites + +- Bun +- Either: + - Confident AI API key, or + - Langfuse API keys + +## Install + +From this folder: + +```bash +bun install +``` + +## Configure + +This exporter supports `--backend confident` (default) or `--backend langfuse`. + +### Confident AI + +Set: + +- `CONFIDENT_API_KEY` (required) +- `OTEL_EXPORTER_OTLP_ENDPOINT` (optional, defaults to `https://otel.confident-ai.com`) + +Example: + +```bash +export CONFIDENT_API_KEY="confident_us_..." +export OTEL_EXPORTER_OTLP_ENDPOINT="https://otel.confident-ai.com" +``` + +Confident expects OTLP/HTTP traces at: + +- `https://otel.confident-ai.com/v1/traces` + +### Langfuse + +Set: + +- `LANGFUSE_PUBLIC_KEY` (required) +- `LANGFUSE_SECRET_KEY` (required) +- `OTEL_EXPORTER_OTLP_ENDPOINT` (optional, defaults to `https://cloud.langfuse.com/api/public/otel`) + +Example (Langfuse Cloud EU): + +```bash +export LANGFUSE_PUBLIC_KEY="pk-lf-..." +export LANGFUSE_SECRET_KEY="sk-lf-..." +export OTEL_EXPORTER_OTLP_ENDPOINT="https://cloud.langfuse.com/api/public/otel" +``` + +Example (Langfuse Cloud US): + +```bash +export LANGFUSE_PUBLIC_KEY="pk-lf-..." +export LANGFUSE_SECRET_KEY="sk-lf-..." +export OTEL_EXPORTER_OTLP_ENDPOINT="https://us.cloud.langfuse.com/api/public/otel" +``` + +Example (self-hosted Langfuse, v3.22.0+): + +```bash +export LANGFUSE_PUBLIC_KEY="pk-lf-..." +export LANGFUSE_SECRET_KEY="sk-lf-..." +export OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:3000/api/public/otel" +``` + +Note: Langfuse’s OTLP endpoint is HTTP/protobuf (not gRPC). + +## Run + +```bash +bun run export --in path/to/results.jsonl +``` + +Specify a backend: + +```bash +bun run export --backend confident --in path/to/results.jsonl +bun run export --backend langfuse --in path/to/results.jsonl +``` + +You can use the included sample file for a quick smoke test: + +```bash +bun run export --in ./sample-results.jsonl +``` + +What it exports: + +- One span per JSONL record (per eval case) +- Safe attributes derived from AgentV results (`eval_id`, `target`, `score`, `trace_summary` counts/metrics) +- Does **not** export prompts, tool inputs/outputs, or assistant text + +If you want to include additional data, extend the attribute builders in `export-jsonl-to-otel.ts`. diff --git a/examples/features/otel-export/bun.lock b/examples/features/otel-export/bun.lock new file mode 100644 index 0000000..a44a507 --- /dev/null +++ b/examples/features/otel-export/bun.lock @@ -0,0 +1,119 @@ +{ + "lockfileVersion": 1, + "configVersion": 1, + "workspaces": { + "": { + "name": "agentv-otel-confident-export-example", + "dependencies": { + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/exporter-trace-otlp-http": "^0.55.0", + "@opentelemetry/resources": "^1.28.0", + "@opentelemetry/sdk-trace-node": "^1.28.0", + "@opentelemetry/semantic-conventions": "^1.28.0", + }, + }, + }, + "packages": { + "@opentelemetry/api": ["@opentelemetry/api@1.9.0", "", {}, "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg=="], + + "@opentelemetry/api-logs": ["@opentelemetry/api-logs@0.55.0", "", { "dependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-3cpa+qI45VHYcA5c0bHM6VHo9gicv3p5mlLHNG3rLyjQU8b7e0st1rWtrUn3JbZ3DwwCfhKop4eQ9UuYlC6Pkg=="], + + "@opentelemetry/context-async-hooks": ["@opentelemetry/context-async-hooks@1.30.1", "", { "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-s5vvxXPVdjqS3kTLKMeBMvop9hbWkwzBpu+mUO2M7sZtlkyDJGwFe33wRKnbaYDo8ExRVBIIdwIGrqpxHuKttA=="], + + "@opentelemetry/core": ["@opentelemetry/core@1.28.0", "", { "dependencies": { "@opentelemetry/semantic-conventions": "1.27.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-ZLwRMV+fNDpVmF2WYUdBHlq0eOWtEaUJSusrzjGnBt7iSRvfjFE3RXYUZJrqou/wIDWV0DwQ5KIfYe9WXg9Xqw=="], + + "@opentelemetry/exporter-trace-otlp-http": ["@opentelemetry/exporter-trace-otlp-http@0.55.0", "", { "dependencies": { "@opentelemetry/core": "1.28.0", "@opentelemetry/otlp-exporter-base": "0.55.0", "@opentelemetry/otlp-transformer": "0.55.0", "@opentelemetry/resources": "1.28.0", "@opentelemetry/sdk-trace-base": "1.28.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-lMiNic63EVHpW+eChmLD2CieDmwQBFi72+LFbh8+5hY0ShrDGrsGP/zuT5MRh7M/vM/UZYO/2A/FYd7CMQGR7A=="], + + "@opentelemetry/otlp-exporter-base": ["@opentelemetry/otlp-exporter-base@0.55.0", "", { "dependencies": { "@opentelemetry/core": "1.28.0", "@opentelemetry/otlp-transformer": "0.55.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-iHQI0Zzq3h1T6xUJTVFwmFl5Dt5y1es+fl4kM+k5T/3YvmVyeYkSiF+wHCg6oKrlUAJfk+t55kaAu3sYmt7ZYA=="], + + "@opentelemetry/otlp-transformer": ["@opentelemetry/otlp-transformer@0.55.0", "", { "dependencies": { "@opentelemetry/api-logs": "0.55.0", "@opentelemetry/core": "1.28.0", "@opentelemetry/resources": "1.28.0", "@opentelemetry/sdk-logs": "0.55.0", "@opentelemetry/sdk-metrics": "1.28.0", "@opentelemetry/sdk-trace-base": "1.28.0", "protobufjs": "^7.3.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-kVqEfxtp6mSN2Dhpy0REo1ghP4PYhC1kMHQJ2qVlO99Pc+aigELjZDfg7/YKmL71gR6wVGIeJfiql/eXL7sQPA=="], + + "@opentelemetry/propagator-b3": ["@opentelemetry/propagator-b3@1.30.1", "", { "dependencies": { "@opentelemetry/core": "1.30.1" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-oATwWWDIJzybAZ4pO76ATN5N6FFbOA1otibAVlS8v90B4S1wClnhRUk7K+2CHAwN1JKYuj4jh/lpCEG5BAqFuQ=="], + + "@opentelemetry/propagator-jaeger": ["@opentelemetry/propagator-jaeger@1.30.1", "", { "dependencies": { "@opentelemetry/core": "1.30.1" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-Pj/BfnYEKIOImirH76M4hDaBSx6HyZ2CXUqk+Kj02m6BB80c/yo4BdWkn/1gDFfU+YPY+bPR2U0DKBfdxCKwmg=="], + + "@opentelemetry/resources": ["@opentelemetry/resources@1.30.1", "", { "dependencies": { "@opentelemetry/core": "1.30.1", "@opentelemetry/semantic-conventions": "1.28.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-5UxZqiAgLYGFjS4s9qm5mBVo433u+dSPUFWVWXmLAD4wB65oMCoXaJP1KJa9DIYYMeHu3z4BZcStG3LC593cWA=="], + + "@opentelemetry/sdk-logs": ["@opentelemetry/sdk-logs@0.55.0", "", { "dependencies": { "@opentelemetry/api-logs": "0.55.0", "@opentelemetry/core": "1.28.0", "@opentelemetry/resources": "1.28.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.4.0 <1.10.0" } }, "sha512-TSx+Yg/d48uWW6HtjS1AD5x6WPfLhDWLl/WxC7I2fMevaiBuKCuraxTB8MDXieCNnBI24bw9ytyXrDCswFfWgA=="], + + "@opentelemetry/sdk-metrics": ["@opentelemetry/sdk-metrics@1.28.0", "", { "dependencies": { "@opentelemetry/core": "1.28.0", "@opentelemetry/resources": "1.28.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-43tqMK/0BcKTyOvm15/WQ3HLr0Vu/ucAl/D84NO7iSlv6O4eOprxSHa3sUtmYkaZWHqdDJV0AHVz/R6u4JALVQ=="], + + "@opentelemetry/sdk-trace-base": ["@opentelemetry/sdk-trace-base@1.28.0", "", { "dependencies": { "@opentelemetry/core": "1.28.0", "@opentelemetry/resources": "1.28.0", "@opentelemetry/semantic-conventions": "1.27.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-ceUVWuCpIao7Y5xE02Xs3nQi0tOGmMea17ecBdwtCvdo9ekmO+ijc9RFDgfifMl7XCBf41zne/1POM3LqSTZDA=="], + + "@opentelemetry/sdk-trace-node": ["@opentelemetry/sdk-trace-node@1.30.1", "", { "dependencies": { "@opentelemetry/context-async-hooks": "1.30.1", "@opentelemetry/core": "1.30.1", "@opentelemetry/propagator-b3": "1.30.1", "@opentelemetry/propagator-jaeger": "1.30.1", "@opentelemetry/sdk-trace-base": "1.30.1", "semver": "^7.5.2" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-cBjYOINt1JxXdpw1e5MlHmFRc5fgj4GW/86vsKFxJCJ8AL4PdVtYH41gWwl4qd4uQjqEL1oJVrXkSy5cnduAnQ=="], + + "@opentelemetry/semantic-conventions": ["@opentelemetry/semantic-conventions@1.38.0", "", {}, "sha512-kocjix+/sSggfJhwXqClZ3i9Y/MI0fp7b+g7kCRm6psy2dsf8uApTRclwG18h8Avm7C9+fnt+O36PspJ/OzoWg=="], + + "@protobufjs/aspromise": ["@protobufjs/aspromise@1.1.2", "", {}, "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ=="], + + "@protobufjs/base64": ["@protobufjs/base64@1.1.2", "", {}, "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg=="], + + "@protobufjs/codegen": ["@protobufjs/codegen@2.0.4", "", {}, "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg=="], + + "@protobufjs/eventemitter": ["@protobufjs/eventemitter@1.1.0", "", {}, "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q=="], + + "@protobufjs/fetch": ["@protobufjs/fetch@1.1.0", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.1", "@protobufjs/inquire": "^1.1.0" } }, "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ=="], + + "@protobufjs/float": ["@protobufjs/float@1.0.2", "", {}, "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ=="], + + "@protobufjs/inquire": ["@protobufjs/inquire@1.1.0", "", {}, "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q=="], + + "@protobufjs/path": ["@protobufjs/path@1.1.2", "", {}, "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA=="], + + "@protobufjs/pool": ["@protobufjs/pool@1.1.0", "", {}, "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw=="], + + "@protobufjs/utf8": ["@protobufjs/utf8@1.1.0", "", {}, "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw=="], + + "@types/node": ["@types/node@25.0.3", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-W609buLVRVmeW693xKfzHeIV6nJGGz98uCPfeXI1ELMLXVeKYZ9m15fAMSaUPBHYLGFsVRcMmSCksQOrZV9BYA=="], + + "long": ["long@5.3.2", "", {}, "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA=="], + + "protobufjs": ["protobufjs@7.5.4", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.2", "@protobufjs/base64": "^1.1.2", "@protobufjs/codegen": "^2.0.4", "@protobufjs/eventemitter": "^1.1.0", "@protobufjs/fetch": "^1.1.0", "@protobufjs/float": "^1.0.2", "@protobufjs/inquire": "^1.1.0", "@protobufjs/path": "^1.1.2", "@protobufjs/pool": "^1.1.0", "@protobufjs/utf8": "^1.1.0", "@types/node": ">=13.7.0", "long": "^5.0.0" } }, "sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg=="], + + "semver": ["semver@7.7.3", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q=="], + + "undici-types": ["undici-types@7.16.0", "", {}, "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="], + + "@opentelemetry/core/@opentelemetry/semantic-conventions": ["@opentelemetry/semantic-conventions@1.27.0", "", {}, "sha512-sAay1RrB+ONOem0OZanAR1ZI/k7yDpnOQSQmTMuGImUQb2y8EbSaCJ94FQluM74xoU03vlb2d2U90hZluL6nQg=="], + + "@opentelemetry/exporter-trace-otlp-http/@opentelemetry/resources": ["@opentelemetry/resources@1.28.0", "", { "dependencies": { "@opentelemetry/core": "1.28.0", "@opentelemetry/semantic-conventions": "1.27.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-cIyXSVJjGeTICENN40YSvLDAq4Y2502hGK3iN7tfdynQLKWb3XWZQEkPc+eSx47kiy11YeFAlYkEfXwR1w8kfw=="], + + "@opentelemetry/otlp-transformer/@opentelemetry/resources": ["@opentelemetry/resources@1.28.0", "", { "dependencies": { "@opentelemetry/core": "1.28.0", "@opentelemetry/semantic-conventions": "1.27.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-cIyXSVJjGeTICENN40YSvLDAq4Y2502hGK3iN7tfdynQLKWb3XWZQEkPc+eSx47kiy11YeFAlYkEfXwR1w8kfw=="], + + "@opentelemetry/propagator-b3/@opentelemetry/core": ["@opentelemetry/core@1.30.1", "", { "dependencies": { "@opentelemetry/semantic-conventions": "1.28.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-OOCM2C/QIURhJMuKaekP3TRBxBKxG/TWWA0TL2J6nXUtDnuCtccy49LUJF8xPFXMX+0LMcxFpCo8M9cGY1W6rQ=="], + + "@opentelemetry/propagator-jaeger/@opentelemetry/core": ["@opentelemetry/core@1.30.1", "", { "dependencies": { "@opentelemetry/semantic-conventions": "1.28.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-OOCM2C/QIURhJMuKaekP3TRBxBKxG/TWWA0TL2J6nXUtDnuCtccy49LUJF8xPFXMX+0LMcxFpCo8M9cGY1W6rQ=="], + + "@opentelemetry/resources/@opentelemetry/core": ["@opentelemetry/core@1.30.1", "", { "dependencies": { "@opentelemetry/semantic-conventions": "1.28.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-OOCM2C/QIURhJMuKaekP3TRBxBKxG/TWWA0TL2J6nXUtDnuCtccy49LUJF8xPFXMX+0LMcxFpCo8M9cGY1W6rQ=="], + + "@opentelemetry/resources/@opentelemetry/semantic-conventions": ["@opentelemetry/semantic-conventions@1.28.0", "", {}, "sha512-lp4qAiMTD4sNWW4DbKLBkfiMZ4jbAboJIGOQr5DvciMRI494OapieI9qiODpOt0XBr1LjIDy1xAGAnVs5supTA=="], + + "@opentelemetry/sdk-logs/@opentelemetry/resources": ["@opentelemetry/resources@1.28.0", "", { "dependencies": { "@opentelemetry/core": "1.28.0", "@opentelemetry/semantic-conventions": "1.27.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-cIyXSVJjGeTICENN40YSvLDAq4Y2502hGK3iN7tfdynQLKWb3XWZQEkPc+eSx47kiy11YeFAlYkEfXwR1w8kfw=="], + + "@opentelemetry/sdk-metrics/@opentelemetry/resources": ["@opentelemetry/resources@1.28.0", "", { "dependencies": { "@opentelemetry/core": "1.28.0", "@opentelemetry/semantic-conventions": "1.27.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-cIyXSVJjGeTICENN40YSvLDAq4Y2502hGK3iN7tfdynQLKWb3XWZQEkPc+eSx47kiy11YeFAlYkEfXwR1w8kfw=="], + + "@opentelemetry/sdk-trace-base/@opentelemetry/resources": ["@opentelemetry/resources@1.28.0", "", { "dependencies": { "@opentelemetry/core": "1.28.0", "@opentelemetry/semantic-conventions": "1.27.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-cIyXSVJjGeTICENN40YSvLDAq4Y2502hGK3iN7tfdynQLKWb3XWZQEkPc+eSx47kiy11YeFAlYkEfXwR1w8kfw=="], + + "@opentelemetry/sdk-trace-base/@opentelemetry/semantic-conventions": ["@opentelemetry/semantic-conventions@1.27.0", "", {}, "sha512-sAay1RrB+ONOem0OZanAR1ZI/k7yDpnOQSQmTMuGImUQb2y8EbSaCJ94FQluM74xoU03vlb2d2U90hZluL6nQg=="], + + "@opentelemetry/sdk-trace-node/@opentelemetry/core": ["@opentelemetry/core@1.30.1", "", { "dependencies": { "@opentelemetry/semantic-conventions": "1.28.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-OOCM2C/QIURhJMuKaekP3TRBxBKxG/TWWA0TL2J6nXUtDnuCtccy49LUJF8xPFXMX+0LMcxFpCo8M9cGY1W6rQ=="], + + "@opentelemetry/sdk-trace-node/@opentelemetry/sdk-trace-base": ["@opentelemetry/sdk-trace-base@1.30.1", "", { "dependencies": { "@opentelemetry/core": "1.30.1", "@opentelemetry/resources": "1.30.1", "@opentelemetry/semantic-conventions": "1.28.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-jVPgBbH1gCy2Lb7X0AVQ8XAfgg0pJ4nvl8/IiQA6nxOsPvS+0zMJaFSs2ltXe0J6C8dqjcnpyqINDJmU30+uOg=="], + + "@opentelemetry/exporter-trace-otlp-http/@opentelemetry/resources/@opentelemetry/semantic-conventions": ["@opentelemetry/semantic-conventions@1.27.0", "", {}, "sha512-sAay1RrB+ONOem0OZanAR1ZI/k7yDpnOQSQmTMuGImUQb2y8EbSaCJ94FQluM74xoU03vlb2d2U90hZluL6nQg=="], + + "@opentelemetry/otlp-transformer/@opentelemetry/resources/@opentelemetry/semantic-conventions": ["@opentelemetry/semantic-conventions@1.27.0", "", {}, "sha512-sAay1RrB+ONOem0OZanAR1ZI/k7yDpnOQSQmTMuGImUQb2y8EbSaCJ94FQluM74xoU03vlb2d2U90hZluL6nQg=="], + + "@opentelemetry/propagator-b3/@opentelemetry/core/@opentelemetry/semantic-conventions": ["@opentelemetry/semantic-conventions@1.28.0", "", {}, "sha512-lp4qAiMTD4sNWW4DbKLBkfiMZ4jbAboJIGOQr5DvciMRI494OapieI9qiODpOt0XBr1LjIDy1xAGAnVs5supTA=="], + + "@opentelemetry/propagator-jaeger/@opentelemetry/core/@opentelemetry/semantic-conventions": ["@opentelemetry/semantic-conventions@1.28.0", "", {}, "sha512-lp4qAiMTD4sNWW4DbKLBkfiMZ4jbAboJIGOQr5DvciMRI494OapieI9qiODpOt0XBr1LjIDy1xAGAnVs5supTA=="], + + "@opentelemetry/sdk-logs/@opentelemetry/resources/@opentelemetry/semantic-conventions": ["@opentelemetry/semantic-conventions@1.27.0", "", {}, "sha512-sAay1RrB+ONOem0OZanAR1ZI/k7yDpnOQSQmTMuGImUQb2y8EbSaCJ94FQluM74xoU03vlb2d2U90hZluL6nQg=="], + + "@opentelemetry/sdk-metrics/@opentelemetry/resources/@opentelemetry/semantic-conventions": ["@opentelemetry/semantic-conventions@1.27.0", "", {}, "sha512-sAay1RrB+ONOem0OZanAR1ZI/k7yDpnOQSQmTMuGImUQb2y8EbSaCJ94FQluM74xoU03vlb2d2U90hZluL6nQg=="], + + "@opentelemetry/sdk-trace-node/@opentelemetry/core/@opentelemetry/semantic-conventions": ["@opentelemetry/semantic-conventions@1.28.0", "", {}, "sha512-lp4qAiMTD4sNWW4DbKLBkfiMZ4jbAboJIGOQr5DvciMRI494OapieI9qiODpOt0XBr1LjIDy1xAGAnVs5supTA=="], + + "@opentelemetry/sdk-trace-node/@opentelemetry/sdk-trace-base/@opentelemetry/semantic-conventions": ["@opentelemetry/semantic-conventions@1.28.0", "", {}, "sha512-lp4qAiMTD4sNWW4DbKLBkfiMZ4jbAboJIGOQr5DvciMRI494OapieI9qiODpOt0XBr1LjIDy1xAGAnVs5supTA=="], + } +} diff --git a/examples/features/otel-export/export-jsonl-to-confident-otel.ts b/examples/features/otel-export/export-jsonl-to-confident-otel.ts new file mode 100644 index 0000000..fe2dc4c --- /dev/null +++ b/examples/features/otel-export/export-jsonl-to-confident-otel.ts @@ -0,0 +1,3 @@ +// Back-compat entrypoint. +// Prefer running `bun run export --in ... [--backend confident|langfuse]` which executes `export-jsonl-to-otel.ts`. +import './export-jsonl-to-otel'; diff --git a/examples/features/otel-export/export-jsonl-to-otel.ts b/examples/features/otel-export/export-jsonl-to-otel.ts new file mode 100644 index 0000000..c4e939b --- /dev/null +++ b/examples/features/otel-export/export-jsonl-to-otel.ts @@ -0,0 +1,253 @@ +import { readFile } from 'node:fs/promises'; +import path from 'node:path'; + +import { trace } from '@opentelemetry/api'; +import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http'; +import { Resource } from '@opentelemetry/resources'; +import { BatchSpanProcessor, NodeTracerProvider } from '@opentelemetry/sdk-trace-node'; +import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'; + +type JsonObject = Record; + +type Backend = 'confident' | 'langfuse'; + +function getArgValue(flag: string): string | undefined { + const idx = process.argv.indexOf(flag); + if (idx === -1) return undefined; + const value = process.argv[idx + 1]; + if (!value || value.startsWith('-')) return undefined; + return value; +} + +function requireEnv(name: string): string { + const value = process.env[name]; + if (!value) { + throw new Error(`Missing required env var: ${name}`); + } + return value; +} + +function parseBackend(value: string | undefined): Backend { + if (!value) return 'confident'; + if (value === 'confident' || value === 'langfuse') return value; + throw new Error(`Invalid --backend value: ${value}. Expected: confident | langfuse`); +} + +function trimTrailingSlashes(value: string): string { + return value.replace(/\/+$/, ''); +} + +function ensureTracesSuffix(baseOrTracesUrl: string): string { + const trimmed = trimTrailingSlashes(baseOrTracesUrl); + if (trimmed.endsWith('/v1/traces')) return trimmed; + return `${trimmed}/v1/traces`; +} + +function toTracesUrl(backend: Backend): string { + const tracesEndpoint = process.env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT; + if (tracesEndpoint) return tracesEndpoint; + + if (backend === 'confident') { + const base = process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? 'https://otel.confident-ai.com'; + return ensureTracesSuffix(base); + } + + // Langfuse expects OTLP on /api/public/otel. + const base = process.env.OTEL_EXPORTER_OTLP_ENDPOINT ?? 'https://cloud.langfuse.com/api/public/otel'; + return ensureTracesSuffix(base); +} + +function parseJsonl(content: string): JsonObject[] { + const lines = content + .split('\n') + .map((line) => line.trim()) + .filter((line) => line.length > 0); + + const records: JsonObject[] = []; + for (const line of lines) { + records.push(JSON.parse(line) as JsonObject); + } + return records; +} + +function safeJsonStringify(value: unknown): string { + try { + return JSON.stringify(value); + } catch { + return JSON.stringify({ error: 'failed_to_stringify' }); + } +} + +function asNumber(value: unknown): number | undefined { + if (typeof value === 'number' && Number.isFinite(value)) return value; + return undefined; +} + +function asString(value: unknown): string | undefined { + if (typeof value === 'string') return value; + return undefined; +} + +function getTraceName(record: JsonObject): string { + const evalId = asString(record.eval_id) ?? 'unknown_eval_id'; + const dataset = asString(record.dataset); + return dataset ? `${dataset}/${evalId}` : evalId; +} + +function getSpanTimes(record: JsonObject): { startTime: number; endTime: number } { + const timestamp = asString(record.timestamp); + const endTimeMs = timestamp ? Date.parse(timestamp) : Date.now(); + const durationMs = asNumber((record.trace_summary as JsonObject | undefined)?.duration_ms); + + const startTimeMs = durationMs && durationMs >= 0 ? endTimeMs - durationMs : endTimeMs; + return { startTime: startTimeMs, endTime: endTimeMs }; +} + +function basicAuthHeaderValue(username: string, password: string): string { + // Both Node and Bun provide Buffer. + const encoded = Buffer.from(`${username}:${password}`, 'utf8').toString('base64'); + return `Basic ${encoded}`; +} + +function buildExporterHeaders(backend: Backend): Record { + if (backend === 'confident') { + const apiKey = requireEnv('CONFIDENT_API_KEY'); + return { 'x-confident-api-key': apiKey }; + } + + const explicitAuth = process.env.LANGFUSE_AUTH_STRING; + if (explicitAuth) { + return { Authorization: `Basic ${explicitAuth}` }; + } + + const publicKey = requireEnv('LANGFUSE_PUBLIC_KEY'); + const secretKey = requireEnv('LANGFUSE_SECRET_KEY'); + return { Authorization: basicAuthHeaderValue(publicKey, secretKey) }; +} + +function buildConfidentMetadata(record: JsonObject): JsonObject { + // AgentV CLI writes snake_case JSONL records. + // Keep this payload small and avoid prompts/tool I/O by default. + return { + eval_id: record.eval_id, + dataset: record.dataset, + conversation_id: record.conversation_id, + target: record.target, + score: record.score, + hits: record.hits, + misses: record.misses, + error: record.error, + trace_summary: record.trace_summary, + }; +} + +function buildLangfuseAttributes(record: JsonObject): Record { + const score = asNumber(record.score); + const hits = asNumber(record.hits); + const misses = asNumber(record.misses); + + const attrs: Record = { + // Trace-level mapping (Langfuse) + 'langfuse.trace.name': getTraceName(record), + + // Keep metadata queryable by setting explicit metadata keys. + 'langfuse.trace.metadata.eval_id': asString(record.eval_id) ?? 'unknown_eval_id', + }; + + const dataset = asString(record.dataset); + if (dataset) attrs['langfuse.trace.metadata.dataset'] = dataset; + + const conversationId = asString(record.conversation_id); + if (conversationId) attrs['langfuse.session.id'] = conversationId; + + const target = asString(record.target); + if (target) attrs['langfuse.trace.metadata.target'] = target; + + if (score !== undefined) attrs['langfuse.trace.metadata.score'] = String(score); + if (hits !== undefined) attrs['langfuse.trace.metadata.hits'] = String(hits); + if (misses !== undefined) attrs['langfuse.trace.metadata.misses'] = String(misses); + + const error = asString(record.error); + if (error) attrs['langfuse.trace.metadata.error'] = error; + + if (record.trace_summary !== undefined) { + attrs['langfuse.trace.metadata.trace_summary'] = safeJsonStringify(record.trace_summary); + } + + return attrs; +} + +function buildSpanAttributes(backend: Backend, record: JsonObject): Record { + if (backend === 'langfuse') { + return buildLangfuseAttributes(record); + } + + return { + // Confident trace-level name (shown in their UI) + 'confident.trace.name': getTraceName(record), + + // Minimal classification + 'confident.span.type': 'agent', + 'confident.span.name': 'agentv.eval_case', + + // Keep metadata small and JSON-stringified (Confident parses JSON strings) + 'confident.trace.metadata': safeJsonStringify(buildConfidentMetadata(record)), + }; +} + +async function main(): Promise { + const inPath = getArgValue('--in'); + if (!inPath) { + // Keep argument handling minimal. + // Example usage: bun run export --in path/to/results.jsonl + throw new Error('Missing required flag: --in '); + } + + const backend = parseBackend(getArgValue('--backend')); + const tracesUrl = toTracesUrl(backend); + const headers = buildExporterHeaders(backend); + + const serviceName = process.env.OTEL_SERVICE_NAME ?? 'agentv-jsonl-export'; + const resource = new Resource({ + [SemanticResourceAttributes.SERVICE_NAME]: serviceName, + }); + + const provider = new NodeTracerProvider({ resource }); + const exporter = new OTLPTraceExporter({ + url: tracesUrl, + headers, + }); + + provider.addSpanProcessor(new BatchSpanProcessor(exporter)); + provider.register(); + + try { + const tracer = trace.getTracer('agentv.jsonl.export'); + + const raw = await readFile(inPath, 'utf8'); + const records = parseJsonl(raw); + + for (const record of records) { + const { startTime, endTime } = getSpanTimes(record); + + const span = tracer.startSpan('agentv.eval_case', { + startTime, + attributes: buildSpanAttributes(backend, record), + }); + + span.end(endTime); + } + + await provider.forceFlush(); + + const resolved = path.resolve(inPath); + console.log(`Exported ${records.length} traces (${backend}) from ${resolved} -> ${tracesUrl}`); + } finally { + await provider.shutdown(); + } +} + +main().catch((err) => { + console.error(`Error: ${(err as Error).message}`); + process.exitCode = 1; +}); diff --git a/examples/features/otel-export/package.json b/examples/features/otel-export/package.json new file mode 100644 index 0000000..c6ba27f --- /dev/null +++ b/examples/features/otel-export/package.json @@ -0,0 +1,15 @@ +{ + "name": "agentv-otel-confident-export-example", + "private": true, + "type": "module", + "scripts": { + "export": "bun run ./export-jsonl-to-otel.ts" + }, + "dependencies": { + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/exporter-trace-otlp-http": "^0.55.0", + "@opentelemetry/resources": "^1.28.0", + "@opentelemetry/sdk-trace-node": "^1.28.0", + "@opentelemetry/semantic-conventions": "^1.28.0" + } +} diff --git a/examples/features/otel-export/sample-results.jsonl b/examples/features/otel-export/sample-results.jsonl new file mode 100644 index 0000000..2b67ccc --- /dev/null +++ b/examples/features/otel-export/sample-results.jsonl @@ -0,0 +1,2 @@ +{"timestamp":"2026-01-09T00:00:00.000Z","eval_id":"case-001","score":1,"hits":["meets_expected"],"misses":[],"candidate_answer":"OK","target":"mock","trace_summary":{"event_count":3,"tool_names":["grep","read"],"tool_calls_by_name":{"read":2,"grep":1},"error_count":0,"token_usage":{"input":120,"output":45},"cost_usd":0.0023,"duration_ms":850}} +{"timestamp":"2026-01-09T00:00:05.000Z","eval_id":"case-002","score":0,"hits":[],"misses":["missing_required_step"],"candidate_answer":"Error: tool failed","target":"mock","error":"Tool execution failed","trace_summary":{"event_count":1,"tool_names":["search"],"tool_calls_by_name":{"search":1},"error_count":1,"duration_ms":250}}