Skip to content

Commit 5cfbb59

Browse files
ericallamericallam
committed
Create performance test harness for runs replication service
1 parent 839d5e8 commit 5cfbb59

File tree

14 files changed

+5412
-54
lines changed

14 files changed

+5412
-54
lines changed

‎.gitignore‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,11 @@ apps/**/public/build
5050
/playwright-report/
5151
/playwright/.cache/
5252

53+
# profiling outputs
54+
profiling-results/
55+
*.clinic-*
56+
*-v8.log
57+
5358
.cosine
5459
.trigger
5560
.tshy*

‎apps/webapp/package.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -221,6 +221,8 @@
221221
"zod-validation-error": "^1.5.0"
222222
},
223223
"devDependencies": {
224+
"@clinic/doctor": "^11.0.0",
225+
"@clinic/flame": "^13.0.0",
224226
"@internal/clickhouse": "workspace:*",
225227
"@internal/replication": "workspace:*",
226228
"@internal/testcontainers": "workspace:*",
@@ -263,6 +265,8 @@
263265
"@typescript-eslint/parser": "^5.59.6",
264266
"autoevals": "^0.0.130",
265267
"autoprefixer": "^10.4.13",
268+
"clinic": "^13.0.0",
269+
"commander": "^11.0.0",
266270
"css-loader": "^6.10.0",
267271
"datepicker": "link:@types/@react-aria/datepicker",
268272
"engine.io": "^6.5.4",

‎apps/webapp/scripts/profile-runs-replication.ts‎

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
#!/usr/bin/env tsx
2+
3+
import { program } from "commander";
4+
import path from "path";
5+
import fs from "fs/promises";
6+
import { RunsReplicationHarness } from "../test/performance/harness";
7+
import { getDefaultConfig, type HarnessConfig } from "../test/performance/config";
8+
9+
program
10+
.name("profile-runs-replication")
11+
.description("Profile RunsReplicationService performance and identify bottlenecks")
12+
.option("-c, --config <file>", "Config file path (JSON)")
13+
.option("-t, --throughput <number>", "Target throughput (records/sec)", "5000")
14+
.option("-d, --duration <number>", "Test duration per phase (seconds)", "60")
15+
.option("--mock-clickhouse", "Use mock ClickHouse (CPU-only profiling)")
16+
.option(
17+
"--profile <tool>",
18+
"Profiling tool: doctor, flame, both, none",
19+
"none"
20+
)
21+
.option("--output <dir>", "Output directory", "./profiling-results")
22+
.option("-v, --verbose", "Verbose logging")
23+
.parse();
24+
25+
async function loadConfig(options: any): Promise<HarnessConfig> {
26+
let config: HarnessConfig = getDefaultConfig() as HarnessConfig;
27+
28+
// Load from config file if provided
29+
if (options.config) {
30+
console.log(`Loading config from: ${options.config}`);
31+
const configFile = await fs.readFile(options.config, "utf-8");
32+
const fileConfig = JSON.parse(configFile);
33+
config = { ...config, ...fileConfig };
34+
}
35+
36+
// Override with CLI arguments
37+
if (options.throughput) {
38+
const throughput = parseInt(options.throughput, 10);
39+
config.producer.targetThroughput = throughput;
40+
41+
// Update all phases if no config file was provided
42+
if (!options.config) {
43+
config.phases = config.phases.map((phase) => ({
44+
...phase,
45+
targetThroughput: throughput,
46+
}));
47+
}
48+
}
49+
50+
if (options.duration) {
51+
const duration = parseInt(options.duration, 10);
52+
53+
// Update all phases if no config file was provided
54+
if (!options.config) {
55+
config.phases = config.phases.map((phase) => ({
56+
...phase,
57+
durationSec: duration,
58+
}));
59+
}
60+
}
61+
62+
if (options.mockClickhouse) {
63+
config.consumer.useMockClickhouse = true;
64+
}
65+
66+
if (options.profile) {
67+
config.profiling.enabled = options.profile !== "none";
68+
config.profiling.tool = options.profile;
69+
}
70+
71+
if (options.output) {
72+
config.profiling.outputDir = options.output;
73+
}
74+
75+
if (options.verbose) {
76+
config.output.verbose = true;
77+
}
78+
79+
// Ensure output directory exists
80+
const timestamp = new Date().toISOString().replace(/[:.]/g, "-").split("T")[0];
81+
const outputDir = path.join(config.profiling.outputDir, timestamp);
82+
config.profiling.outputDir = outputDir;
83+
config.output.metricsFile = path.join(outputDir, "metrics.json");
84+
85+
return config;
86+
}
87+
88+
function printConfig(config: HarnessConfig): void {
89+
console.log("\n" + "=".repeat(60));
90+
console.log("RunsReplicationService Performance Test Harness");
91+
console.log("=".repeat(60));
92+
console.log("\n📋 Configuration:");
93+
console.log(` Profiling DB: ${config.infrastructure.profilingDatabaseName}`);
94+
console.log(` Output Dir: ${config.profiling.outputDir}`);
95+
console.log(` Mock ClickHouse: ${config.consumer.useMockClickhouse ? "Yes (CPU-only)" : "No (full stack)"}`);
96+
console.log(` Profiling Tool: ${config.profiling.tool}`);
97+
console.log(` Verbose: ${config.output.verbose}`);
98+
99+
console.log("\n📊 Test Phases:");
100+
for (const phase of config.phases) {
101+
console.log(` - ${phase.name.padEnd(15)} ${phase.durationSec}s @ ${phase.targetThroughput} rec/sec`);
102+
}
103+
104+
console.log("\n⚙️ Producer Config:");
105+
console.log(` Insert/Update: ${(config.producer.insertUpdateRatio * 100).toFixed(0)}% inserts`);
106+
console.log(` Batch Size: ${config.producer.batchSize}`);
107+
console.log(` Payload Size: ${config.producer.payloadSizeKB} KB`);
108+
109+
console.log("\n⚙️ Consumer Config:");
110+
console.log(` Flush Batch Size: ${config.consumer.flushBatchSize}`);
111+
console.log(` Flush Interval: ${config.consumer.flushIntervalMs} ms`);
112+
console.log(` Max Concurrency: ${config.consumer.maxFlushConcurrency}`);
113+
114+
console.log("\n" + "=".repeat(60) + "\n");
115+
}
116+
117+
function printSummary(phases: any[]): void {
118+
console.log("\n" + "=".repeat(60));
119+
console.log("📈 Summary");
120+
console.log("=".repeat(60) + "\n");
121+
122+
for (const phase of phases) {
123+
console.log(`${phase.phase}:`);
124+
console.log(` Duration: ${(phase.durationMs / 1000).toFixed(1)}s`);
125+
console.log(` Producer Throughput: ${phase.producerThroughput.toFixed(1)} rec/sec`);
126+
console.log(` Consumer Throughput: ${phase.consumerThroughput.toFixed(1)} rec/sec`);
127+
console.log(` Event Loop Util: ${(phase.eventLoopUtilization * 100).toFixed(1)}%`);
128+
console.log(` Heap Used: ${phase.heapUsedMB.toFixed(1)} MB`);
129+
console.log(` Replication Lag P95: ${phase.replicationLagP95.toFixed(1)} ms`);
130+
console.log();
131+
}
132+
133+
console.log("=".repeat(60) + "\n");
134+
}
135+
136+
async function main() {
137+
const options = program.opts();
138+
const config = await loadConfig(options);
139+
140+
printConfig(config);
141+
142+
const harness = new RunsReplicationHarness(config);
143+
144+
try {
145+
await harness.setup();
146+
const phases = await harness.run();
147+
await harness.teardown();
148+
149+
// Export metrics
150+
await harness.exportMetrics(config.output.metricsFile);
151+
152+
// Print summary
153+
printSummary(phases);
154+
155+
console.log("\n✅ Profiling complete!");
156+
console.log(`📊 Results saved to: ${config.profiling.outputDir}\n`);
157+
158+
if (config.profiling.enabled && config.profiling.tool !== "none") {
159+
console.log("🔥 Profiling data:");
160+
console.log(` View flamegraph/analysis in: ${config.profiling.outputDir}\n`);
161+
}
162+
163+
process.exit(0);
164+
} catch (error) {
165+
console.error("\n❌ Profiling failed:");
166+
console.error(error);
167+
await harness.teardown().catch(() => {});
168+
process.exit(1);
169+
}
170+
}
171+
172+
main();

0 commit comments

Comments
 (0)