From dbebe1865adf147b6e098c0da9f9e49b231a9f3f Mon Sep 17 00:00:00 2001 From: SentienceDEV Date: Sun, 1 Feb 2026 11:40:56 -0800 Subject: [PATCH 1/2] improve tracing with automated upload screenshot if opt-in; auto step start --- src/agent-runtime.ts | 58 +++++++++++- src/tracing/tracer-factory.ts | 54 ++++++++++- src/tracing/tracer.ts | 87 +++++++++++++++++ tests/tracing/tracer.test.ts | 174 ++++++++++++++++++++++++++++++++++ 4 files changed, 369 insertions(+), 4 deletions(-) diff --git a/src/agent-runtime.ts b/src/agent-runtime.ts index d2edbd0f..a2d51f40 100644 --- a/src/agent-runtime.ts +++ b/src/agent-runtime.ts @@ -626,12 +626,25 @@ export class AgentRuntime { * Take a snapshot of the current page state. * * This updates lastSnapshot which is used as context for assertions. + * When emitTrace=true (default), automatically emits a 'snapshot' trace event + * with screenshot_base64 for Sentience Studio visualization. * * @param options - Options passed through to browser.snapshot() + * @param options.emitTrace - If true (default), emit a 'snapshot' trace event with screenshot. + * Set to false to disable automatic trace emission. * @returns Snapshot of current page state + * + * @example + * // Default: snapshot with auto-emit trace event + * const snapshot = await runtime.snapshot(); + * + * // Disable auto-emit for manual control + * const snapshot = await runtime.snapshot({ emitTrace: false }); + * // Later, manually emit if needed: + * tracer.emitSnapshot(snapshot, runtime.getStepId()); */ async snapshot(options?: Record): Promise { - const { _skipCaptchaHandling, ...snapshotOptions } = options || {}; + const { _skipCaptchaHandling, emitTrace = true, ...snapshotOptions } = options || {}; this.lastSnapshot = await this.browser.snapshot(this.page, snapshotOptions); if (this.lastSnapshot && !this.stepPreSnapshot) { this.stepPreSnapshot = this.lastSnapshot; @@ -640,9 +653,32 @@ export class AgentRuntime { if (!_skipCaptchaHandling) { await this.handleCaptchaIfNeeded(this.lastSnapshot, 'gateway'); } + + // Auto-emit snapshot trace event for Studio visualization + if (emitTrace && this.lastSnapshot && this.tracer) { + this.emitSnapshotTrace(this.lastSnapshot); + } + return this.lastSnapshot; } + /** + * Emit a snapshot trace event with screenshot for Studio visualization. + * + * This is called automatically by snapshot() when emitTrace=true. + */ + private emitSnapshotTrace(snapshot: Snapshot): void { + if (!this.tracer) { + return; + } + + try { + this.tracer.emitSnapshot(snapshot, this.stepId ?? undefined, this.stepIndex, 'jpeg'); + } catch { + // Best-effort: don't let trace emission errors break snapshot + } + } + /** * Evaluate JavaScript in the page context. */ @@ -1167,12 +1203,20 @@ export class AgentRuntime { * - Generates a new stepId * - Clears assertions from previous step * - Increments stepIndex (or uses provided value) + * - Emits step_start trace event (optional) * * @param goal - Description of what this step aims to achieve * @param stepIndex - Optional explicit step index (otherwise auto-increments) + * @param options - Optional settings: emitTrace (default true), preUrl * @returns Generated stepId in format 'step-N' where N is the step index */ - beginStep(goal: string, stepIndex?: number): string { + beginStep( + goal: string, + stepIndex?: number, + options?: { emitTrace?: boolean; preUrl?: string } + ): string { + const { emitTrace = true, preUrl } = options || {}; + // Clear previous step state this.assertionsThisStep = []; this.stepPreSnapshot = null; @@ -1190,6 +1234,16 @@ export class AgentRuntime { // Generate stepId in 'step-N' format for Studio compatibility this.stepId = `step-${this.stepIndex}`; + // Emit step_start trace event for Studio timeline display + if (emitTrace && this.tracer) { + try { + const url = preUrl || this.lastSnapshot?.url || this.page?.url?.() || ''; + this.tracer.emitStepStart(this.stepId, this.stepIndex, goal, 0, url); + } catch { + // Tracing must be non-fatal + } + } + return this.stepId; } diff --git a/src/tracing/tracer-factory.ts b/src/tracing/tracer-factory.ts index 36d598c7..044f4efa 100644 --- a/src/tracing/tracer-factory.ts +++ b/src/tracing/tracer-factory.ts @@ -19,6 +19,35 @@ import { Tracer } from './tracer'; import { CloudTraceSink, SentienceLogger } from './cloud-sink'; import { JsonlTraceSink } from './jsonl-sink'; +/** + * Helper to emit run_start event with available metadata + */ +function emitRunStart( + tracer: Tracer, + agentType?: string, + llmModel?: string, + goal?: string, + startUrl?: string +): void { + try { + const config: Record = {}; + if (goal) { + config.goal = goal; + } + if (startUrl) { + config.start_url = startUrl; + } + + tracer.emitRunStart( + agentType || 'SentienceAgent', + llmModel, + Object.keys(config).length > 0 ? config : undefined + ); + } catch { + // Tracing must be non-fatal + } +} + /** * Sentience API base URL (constant) */ @@ -198,6 +227,7 @@ function httpPost( * @param options.llmModel - LLM model used (e.g., "gpt-4-turbo", "claude-3-5-sonnet") * @param options.startUrl - Starting URL of the agent run (e.g., "https://amazon.com") * @param options.screenshotProcessor - Optional function to process screenshots before upload. Takes base64 string, returns processed base64 string. Useful for PII redaction or custom image processing. + * @param options.autoEmitRunStart - If true (default), automatically emit run_start event with provided metadata. This ensures traces have complete structure for Studio visualization. * @returns Tracer configured with appropriate sink * * @example @@ -213,6 +243,7 @@ function httpPost( * uploadTrace: true * }); * // Returns: Tracer with CloudTraceSink + * // run_start event is automatically emitted * * // With screenshot processor for PII redaction * const redactPII = (screenshot: string): string => { @@ -229,6 +260,10 @@ function httpPost( * const tracer = await createTracer({ apiKey: "sk_pro_xyz", runId: "demo", uploadTrace: false }); * // Returns: Tracer with JsonlTraceSink (local-only) * + * // Disable auto-emit for manual control + * const tracer = await createTracer({ runId: "demo", autoEmitRunStart: false }); + * tracer.emitRunStart("MyAgent", "gpt-4o"); // Manual emit + * * // Free tier user * const tracer = await createTracer({ runId: "demo" }); * // Returns: Tracer with JsonlTraceSink (local-only) @@ -250,6 +285,7 @@ export async function createTracer(options: { llmModel?: string; startUrl?: string; screenshotProcessor?: (screenshot: string) => string; + autoEmitRunStart?: boolean; }): Promise { const runId = options.runId || randomUUID(); const apiUrl = options.apiUrl || SENTIENCE_API_URL; @@ -303,11 +339,18 @@ export async function createTracer(options: { console.log('☁️ [Sentience] Cloud tracing enabled (Pro tier)'); // PRODUCTION FIX: Pass runId for persistent cache naming - return new Tracer( + const tracer = new Tracer( runId, new CloudTraceSink(uploadUrl, runId, options.apiKey, apiUrl, options.logger), options.screenshotProcessor ); + + // Auto-emit run_start for complete trace structure + if (options.autoEmitRunStart !== false) { + emitRunStart(tracer, options.agentType, options.llmModel, options.goal, options.startUrl); + } + + return tracer; } else if (response.status === 403) { console.log('⚠️ [Sentience] Cloud tracing requires Pro tier'); console.log(' Falling back to local-only tracing'); @@ -338,7 +381,14 @@ export async function createTracer(options: { const localPath = path.join(tracesDir, `${runId}.jsonl`); console.log(`💾 [Sentience] Local tracing: ${localPath}`); - return new Tracer(runId, new JsonlTraceSink(localPath), options.screenshotProcessor); + const tracer = new Tracer(runId, new JsonlTraceSink(localPath), options.screenshotProcessor); + + // Auto-emit run_start for complete trace structure + if (options.autoEmitRunStart !== false) { + emitRunStart(tracer, options.agentType, options.llmModel, options.goal, options.startUrl); + } + + return tracer; } /** diff --git a/src/tracing/tracer.ts b/src/tracing/tracer.ts index e6da064a..7990f24b 100644 --- a/src/tracing/tracer.ts +++ b/src/tracing/tracer.ts @@ -202,6 +202,93 @@ export class Tracer { this.emit('error', { step_id: stepId, error, attempt }, stepId); } + /** + * Emit snapshot event with screenshot for Studio visualization. + * + * This method builds and emits a 'snapshot' trace event that includes: + * - Page URL and element data + * - Screenshot (if present in snapshot) + * - Step correlation info + * + * Use this when you want screenshots to appear in the Sentience Studio timeline. + * + * @param snapshot - Snapshot object (must have 'screenshot' property for images) + * @param stepId - Step UUID (for correlating snapshot with a step) + * @param stepIndex - Step index (0-based) for Studio timeline ordering + * @param screenshotFormat - Format of screenshot ("jpeg" or "png", default: "jpeg") + * + * @example + * // After taking a snapshot with AgentRuntime + * const snapshot = await runtime.snapshot(); + * tracer.emitSnapshot(snapshot, runtime.getStepId(), runtime.getStepIndex()); + * + * // Or use auto-emit (default in AgentRuntime.snapshot()) + * const snapshot = await runtime.snapshot(); // Auto-emits snapshot event + */ + emitSnapshot( + snapshot: any, + stepId?: string, + stepIndex?: number, + screenshotFormat: string = 'jpeg' + ): void { + if (!snapshot) { + return; + } + + try { + // Build the snapshot event data + const data: TraceEventData = { + url: snapshot.url, + element_count: snapshot.elements?.length || 0, + timestamp: snapshot.timestamp, + }; + + // Include step_index if provided (required for UUID step_ids) + if (stepIndex !== undefined) { + data.step_index = stepIndex; + } + + // Include elements data (simplified for trace) + if (snapshot.elements && snapshot.elements.length > 0) { + // Normalize importance values to importance_score (0-1 range) + const importanceValues = snapshot.elements.map((el: any) => el.importance || 0); + const minImportance = Math.min(...importanceValues); + const maxImportance = Math.max(...importanceValues); + const importanceRange = maxImportance - minImportance; + + data.elements = snapshot.elements.map((el: any) => { + const importanceScore = + importanceRange > 0 ? (el.importance - minImportance) / importanceRange : 0.5; + return { + ...el, + importance_score: importanceScore, + }; + }); + } + + // Extract and add screenshot if present + const screenshotRaw = snapshot.screenshot; + if (screenshotRaw) { + // Extract base64 string from data URL if needed + // Format: "data:image/jpeg;base64,{base64_string}" + let screenshotBase64: string; + if (typeof screenshotRaw === 'string' && screenshotRaw.startsWith('data:image')) { + const commaIndex = screenshotRaw.indexOf(','); + screenshotBase64 = + commaIndex !== -1 ? screenshotRaw.slice(commaIndex + 1) : screenshotRaw; + } else { + screenshotBase64 = screenshotRaw; + } + data.screenshot_base64 = screenshotBase64; + data.screenshot_format = screenshotFormat; + } + + this.emit('snapshot', data, stepId); + } catch { + // Best-effort: don't let trace emission errors break the caller + } + } + /** * Automatically infer finalStatus from tracked step outcomes if not explicitly set. * This is called automatically in close() if finalStatus is still "unknown". diff --git a/tests/tracing/tracer.test.ts b/tests/tracing/tracer.test.ts index f4c7091a..c5c8ee74 100644 --- a/tests/tracing/tracer.test.ts +++ b/tests/tracing/tracer.test.ts @@ -684,4 +684,178 @@ describe('Tracer', () => { expect(tracer.getStats().final_status).toBe('success'); }); }); + + // ============================================================================ + // Tests for emitSnapshot() helper method + // ============================================================================ + + describe('emitSnapshot', () => { + interface MockSnapshot { + url: string; + screenshot?: string; + timestamp?: string; + elements: any[]; + } + + it('should emit snapshot event with basic data', async () => { + const sink = new JsonlTraceSink(testFile); + const tracer = new Tracer('test-run', sink); + + const snapshot: MockSnapshot = { + url: 'https://example.com', + timestamp: '2024-01-01T00:00:00.000Z', + elements: [], + }; + + tracer.emitSnapshot(snapshot, 'step-456', 1); + + await tracer.close(); + await new Promise(resolve => setTimeout(resolve, 100)); + + const content = await readFileWithRetry(testFile); + const event = JSON.parse(content.trim()) as TraceEvent; + + expect(event.type).toBe('snapshot'); + expect(event.step_id).toBe('step-456'); + expect(event.data.url).toBe('https://example.com'); + expect(event.data.step_index).toBe(1); + expect(event.data.element_count).toBe(0); + }); + + it('should include screenshot_base64 when screenshot is present', async () => { + const sink = new JsonlTraceSink(testFile); + const tracer = new Tracer('test-run', sink); + + const snapshot: MockSnapshot = { + url: 'https://example.com', + timestamp: '2024-01-01T00:00:00.000Z', + screenshot: 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJ', + elements: [], + }; + + tracer.emitSnapshot(snapshot, 'step-456'); + + await tracer.close(); + await new Promise(resolve => setTimeout(resolve, 100)); + + const content = await readFileWithRetry(testFile); + const event = JSON.parse(content.trim()) as TraceEvent; + + expect(event.type).toBe('snapshot'); + expect(event.data.screenshot_base64).toBe('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJ'); + expect(event.data.screenshot_format).toBe('jpeg'); + }); + + it('should extract base64 from data URL format', async () => { + const sink = new JsonlTraceSink(testFile); + const tracer = new Tracer('test-run', sink); + + const snapshot: MockSnapshot = { + url: 'https://example.com', + timestamp: '2024-01-01T00:00:00.000Z', + screenshot: 'data:image/jpeg;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJ', + elements: [], + }; + + tracer.emitSnapshot(snapshot, 'step-456'); + + await tracer.close(); + await new Promise(resolve => setTimeout(resolve, 100)); + + const content = await readFileWithRetry(testFile); + const event = JSON.parse(content.trim()) as TraceEvent; + + // Should extract just the base64 part (strip data URL prefix) + expect(event.data.screenshot_base64).toBe('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJ'); + expect(event.data.screenshot_format).toBe('jpeg'); + }); + + it('should work without stepId', async () => { + const sink = new JsonlTraceSink(testFile); + const tracer = new Tracer('test-run', sink); + + const snapshot: MockSnapshot = { + url: 'https://example.com', + timestamp: '2024-01-01T00:00:00.000Z', + elements: [], + }; + + tracer.emitSnapshot(snapshot); + + await tracer.close(); + await new Promise(resolve => setTimeout(resolve, 100)); + + const content = await readFileWithRetry(testFile); + const event = JSON.parse(content.trim()) as TraceEvent; + + expect(event.type).toBe('snapshot'); + expect(event.step_id).toBeUndefined(); + expect(event.data.url).toBe('https://example.com'); + }); + + it('should handle null snapshot gracefully', async () => { + const sink = new JsonlTraceSink(testFile); + const tracer = new Tracer('test-run', sink); + + tracer.emitSnapshot(null); + + await tracer.close(); + await new Promise(resolve => setTimeout(resolve, 100)); + + // Should not emit anything for null snapshot + const content = await readFileWithRetry(testFile); + expect(content.trim()).toBe(''); + }); + + it('should support custom screenshot format', async () => { + const sink = new JsonlTraceSink(testFile); + const tracer = new Tracer('test-run', sink); + + const snapshot: MockSnapshot = { + url: 'https://example.com', + timestamp: '2024-01-01T00:00:00.000Z', + screenshot: 'iVBORw0KGgo...', + elements: [], + }; + + tracer.emitSnapshot(snapshot, 'step-456', undefined, 'png'); + + await tracer.close(); + await new Promise(resolve => setTimeout(resolve, 100)); + + const content = await readFileWithRetry(testFile); + const event = JSON.parse(content.trim()) as TraceEvent; + + expect(event.data.screenshot_format).toBe('png'); + }); + + it('should include elements with normalized importance_score', async () => { + const sink = new JsonlTraceSink(testFile); + const tracer = new Tracer('test-run', sink); + + const snapshot: MockSnapshot = { + url: 'https://example.com', + timestamp: '2024-01-01T00:00:00.000Z', + elements: [ + { id: 1, role: 'button', importance: 100 }, + { id: 2, role: 'input', importance: 50 }, + { id: 3, role: 'link', importance: 0 }, + ], + }; + + tracer.emitSnapshot(snapshot, 'step-456'); + + await tracer.close(); + await new Promise(resolve => setTimeout(resolve, 100)); + + const content = await readFileWithRetry(testFile); + const event = JSON.parse(content.trim()) as TraceEvent; + + expect(event.data.elements).toHaveLength(3); + const elements = event.data.elements!; + expect(elements[0].importance_score).toBe(1.0); // max importance = 1.0 + expect(elements[1].importance_score).toBe(0.5); // mid = 0.5 + expect(elements[2].importance_score).toBe(0.0); // min = 0.0 + }); + }); }); From 21a97088e76ef2ae1f2496085413bcc2a151dd62 Mon Sep 17 00:00:00 2001 From: SentienceDEV Date: Sun, 1 Feb 2026 12:00:50 -0800 Subject: [PATCH 2/2] fix test expected --- tests/tracing/tracer-factory.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/tracing/tracer-factory.test.ts b/tests/tracing/tracer-factory.test.ts index fb836ab5..3392300d 100644 --- a/tests/tracing/tracer-factory.test.ts +++ b/tests/tracing/tracer-factory.test.ts @@ -305,6 +305,7 @@ describe('createTracer', () => { it('should work with agent workflow (Free tier)', async () => { const tracer = await createTracer({ runId: 'agent-test', + autoEmitRunStart: false, // Disable auto-emit since we're manually emitting }); tracer.emitRunStart('SentienceAgent', 'gpt-4');