Skip to content

Commit c5447f1

Browse files
breardon2011breardon2011
committed
benchmark wip
1 parent 9de0d32 commit c5447f1

17 files changed

+2168
-35
lines changed

‎sandbox-sidecar/scripts/Dockerfile.runner‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
# Dockerfile.runner
2+
# Minimal Terraform runner image - similar to what you'd use in Kubernetes
3+
FROM ubuntu:22.04
4+
5+
ENV DEBIAN_FRONTEND=noninteractive
6+
7+
RUN apt-get update && apt-get install -y \
8+
curl \
9+
unzip \
10+
ca-certificates \
11+
git \
12+
&& rm -rf /var/lib/apt/lists/*
13+
14+
# Install Terraform (match the version we benchmark)
15+
ARG TF_VERSION=1.5.7
16+
RUN curl -fsSL https://releases.hashicorp.com/terraform/${TF_VERSION}/terraform_${TF_VERSION}_linux_amd64.zip -o /tmp/terraform.zip \
17+
&& unzip /tmp/terraform.zip -d /usr/local/bin \
18+
&& rm /tmp/terraform.zip \
19+
&& terraform version
20+
21+
WORKDIR /workspace
22+
23+
CMD ["bash"]
24+

‎sandbox-sidecar/scripts/debug-terraform-e2b.ts‎

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
// debug-terraform-e2b.ts
2+
//
3+
// Run a 10k-null Terraform apply inside an E2B sandbox with
4+
// *no* stdout/stderr callbacks, just final output.
5+
//
6+
// Usage:
7+
// cd sandbox-sidecar
8+
// npx tsx scripts/debug-terraform-e2b.ts
9+
//
10+
// This isolates whether the slowness is:
11+
// A) E2B's virtualization/environment
12+
// B) Our sidecar's SDK callback overhead
13+
// C) Our log handling code
14+
15+
import "dotenv/config";
16+
import { Sandbox } from "@e2b/code-interpreter";
17+
18+
// Use our pre-built template with terraform + providers cached
19+
const TEMPLATE_ID = "terraform-1-5-7--tpl-0-2-2";
20+
const WORK_DIR = "/home/user/benchmark";
21+
22+
// The benchmark Terraform config - 10k null resources
23+
const MAIN_TF = `
24+
# Benchmark: 10,000 Null Resources
25+
# Purpose: Test performance with large number of resources
26+
27+
terraform {
28+
required_providers {
29+
null = {
30+
source = "hashicorp/null"
31+
version = "~> 3.0"
32+
}
33+
}
34+
}
35+
36+
resource "null_resource" "massive" {
37+
count = 10000
38+
39+
triggers = {
40+
index = count.index
41+
}
42+
}
43+
`;
44+
45+
async function main() {
46+
const apiKey = process.env.E2B_API_KEY;
47+
if (!apiKey) {
48+
console.error("E2B_API_KEY environment variable is required");
49+
console.error("Set it in .env or export E2B_API_KEY=...");
50+
process.exit(1);
51+
}
52+
53+
console.log("=".repeat(60));
54+
console.log("E2B Terraform Performance Debug Script");
55+
console.log("=".repeat(60));
56+
console.log(`Template: ${TEMPLATE_ID}`);
57+
console.log(`Resources: 10,000 null_resource`);
58+
console.log("=".repeat(60));
59+
60+
console.log("\n[1/6] Creating E2B sandbox...");
61+
const startCreate = Date.now();
62+
63+
const sandbox = await Sandbox.create(TEMPLATE_ID, {
64+
apiKey,
65+
timeoutMs: 30 * 60 * 1000, // 30 minutes
66+
});
67+
68+
const createTime = Date.now() - startCreate;
69+
console.log(`Sandbox created: ${sandbox.sandboxId}`);
70+
console.log(`Creation time: ${createTime}ms`);
71+
72+
try {
73+
// 2) Create the benchmark directory and write main.tf
74+
console.log("\n[2/6] Creating benchmark terraform config...");
75+
await sandbox.commands.run(`mkdir -p ${WORK_DIR}`);
76+
await sandbox.files.write(`${WORK_DIR}/main.tf`, MAIN_TF);
77+
78+
let result = await sandbox.commands.run(`cat ${WORK_DIR}/main.tf`);
79+
console.log("Created main.tf:");
80+
console.log(result.stdout);
81+
82+
// 3) Run terraform init
83+
console.log("\n[3/6] Running terraform init...");
84+
const startInit = Date.now();
85+
result = await sandbox.commands.run(
86+
`cd ${WORK_DIR} && terraform init -input=false -no-color -plugin-dir=/usr/share/terraform/providers`,
87+
{ timeoutMs: 300000 }
88+
);
89+
const initTime = Date.now() - startInit;
90+
console.log(`Init time: ${initTime}ms`);
91+
console.log(result.stdout.slice(-500));
92+
if (result.exitCode !== 0) {
93+
console.error("terraform init failed:", result.stderr);
94+
return;
95+
}
96+
97+
// 4) TIME TEST: Run apply WITHOUT streaming callbacks
98+
console.log("\n[4/6] Running terraform apply (NO streaming callbacks)...");
99+
console.log("This will take a while. No output until complete.");
100+
console.log("Started at:", new Date().toISOString());
101+
102+
const startApply = Date.now();
103+
result = await sandbox.commands.run(
104+
`cd ${WORK_DIR} && terraform apply -auto-approve -input=false -no-color -parallelism=30`,
105+
{
106+
timeoutMs: 60 * 60 * 1000, // 1 hour
107+
// NO onStdout/onStderr callbacks!
108+
}
109+
);
110+
const applyTime = Date.now() - startApply;
111+
112+
console.log(`\nApply completed at: ${new Date().toISOString()}`);
113+
console.log(`Apply time: ${applyTime}ms (${(applyTime/1000).toFixed(1)}s) = ${(applyTime/60000).toFixed(2)} minutes`);
114+
console.log(`\nLast 500 chars of output:`);
115+
console.log(result.stdout.slice(-500));
116+
117+
if (result.exitCode !== 0) {
118+
console.error("terraform apply failed:", result.stderr);
119+
return;
120+
}
121+
122+
// 5) Destroy and re-apply with output redirected to file
123+
console.log("\n[5/6] Destroying resources for second test...");
124+
const startDestroy = Date.now();
125+
await sandbox.commands.run(
126+
`cd ${WORK_DIR} && terraform destroy -auto-approve -input=false -no-color -parallelism=30 > /tmp/destroy.log 2>&1`,
127+
{ timeoutMs: 60 * 60 * 1000 }
128+
);
129+
const destroyTime = Date.now() - startDestroy;
130+
console.log(`Destroy time: ${destroyTime}ms (${(destroyTime/1000).toFixed(1)}s)`);
131+
132+
// 6) Apply with file redirect - completely bypasses SDK stdout handling
133+
console.log("\n[6/6] Running apply with output redirected to file...");
134+
console.log("(This tests if SDK stdout collection has overhead)");
135+
const startApplyFile = Date.now();
136+
result = await sandbox.commands.run(
137+
`cd ${WORK_DIR} && terraform apply -auto-approve -input=false -no-color -parallelism=30 > /tmp/apply.log 2>&1; echo "EXIT_CODE=$?"`,
138+
{ timeoutMs: 60 * 60 * 1000 }
139+
);
140+
const applyFileTime = Date.now() - startApplyFile;
141+
142+
console.log(`Apply (file redirect) time: ${applyFileTime}ms (${(applyFileTime/1000).toFixed(1)}s) = ${(applyFileTime/60000).toFixed(2)} minutes`);
143+
144+
// Read last part of log
145+
result = await sandbox.commands.run(`tail -10 /tmp/apply.log`);
146+
console.log("Last 10 lines of apply.log:");
147+
console.log(result.stdout);
148+
149+
// Summary
150+
console.log("\n" + "=".repeat(60));
151+
console.log("SUMMARY");
152+
console.log("=".repeat(60));
153+
console.log(`Sandbox creation: ${createTime}ms (${(createTime/1000).toFixed(1)}s)`);
154+
console.log(`Terraform init: ${initTime}ms (${(initTime/1000).toFixed(1)}s)`);
155+
console.log(`Apply (SDK collects): ${applyTime}ms (${(applyTime/1000).toFixed(1)}s) = ${(applyTime/60000).toFixed(2)} min`);
156+
console.log(`Apply (file redirect): ${applyFileTime}ms (${(applyFileTime/1000).toFixed(1)}s) = ${(applyFileTime/60000).toFixed(2)} min`);
157+
console.log(`Destroy: ${destroyTime}ms (${(destroyTime/1000).toFixed(1)}s)`);
158+
console.log("=".repeat(60));
159+
160+
console.log("\nDIAGNOSIS:");
161+
if (applyTime > 600000) { // > 10 minutes
162+
console.log("❌ Apply took > 10 minutes.");
163+
console.log(" The E2B environment itself is slow.");
164+
console.log(" This is NOT a sidecar code issue - it's E2B's VM performance.");
165+
console.log(" Options:");
166+
console.log(" - Contact E2B about VM performance");
167+
console.log(" - Try a different sandbox provider");
168+
console.log(" - Accept this as the baseline for E2B");
169+
} else if (applyTime > 300000) { // > 5 minutes
170+
console.log("⚠️ Apply took 5-10 minutes.");
171+
console.log(" This is similar to Spacelift's performance.");
172+
console.log(" E2B is comparable but not faster.");
173+
} else if (applyTime > 60000) { // > 1 minute
174+
console.log("🟡 Apply took 1-5 minutes.");
175+
console.log(" E2B is reasonably fast.");
176+
console.log(" Our sidecar overhead might be adding to this.");
177+
} else {
178+
console.log("✅ Apply took < 1 minute!");
179+
console.log(" E2B is fast - any slowness is in our sidecar code.");
180+
}
181+
182+
const sdkOverhead = applyTime - applyFileTime;
183+
if (Math.abs(sdkOverhead) > 10000) { // > 10 second difference
184+
console.log(`\n📊 SDK stdout collection overhead: ${(sdkOverhead/1000).toFixed(1)}s`);
185+
}
186+
187+
} finally {
188+
console.log("\nKilling sandbox...");
189+
await sandbox.kill();
190+
console.log("Done.");
191+
}
192+
}
193+
194+
main().catch((err) => {
195+
console.error(err);
196+
process.exit(1);
197+
});
198+

‎sandbox-sidecar/scripts/debug-terraform-local.sh‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
#!/bin/bash
2+
#
3+
# debug-terraform-local.sh
4+
#
5+
# Run the same 10k null_resource benchmark locally to establish a baseline.
6+
# Compare this time against E2B to see the overhead.
7+
#
8+
# Usage:
9+
# cd sandbox-sidecar/scripts
10+
# chmod +x debug-terraform-local.sh
11+
# ./debug-terraform-local.sh
12+
#
13+
14+
set -e
15+
16+
WORK_DIR="/tmp/terraform-benchmark-$$"
17+
echo "============================================================"
18+
echo "Local Terraform Performance Benchmark"
19+
echo "============================================================"
20+
echo "Work directory: $WORK_DIR"
21+
echo "Terraform version: $(terraform version -json | jq -r '.terraform_version' 2>/dev/null || terraform version | head -1)"
22+
echo "============================================================"
23+
24+
# Create work directory
25+
mkdir -p "$WORK_DIR"
26+
cd "$WORK_DIR"
27+
28+
# Create the benchmark terraform config
29+
cat > main.tf << 'EOF'
30+
# Benchmark: 10,000 Null Resources
31+
# Purpose: Test performance with large number of resources
32+
33+
terraform {
34+
required_providers {
35+
null = {
36+
source = "hashicorp/null"
37+
version = "~> 3.0"
38+
}
39+
}
40+
}
41+
42+
resource "null_resource" "massive" {
43+
count = 10000
44+
45+
triggers = {
46+
index = count.index
47+
}
48+
}
49+
EOF
50+
51+
echo ""
52+
echo "[1/4] Created main.tf with 10,000 null_resources"
53+
cat main.tf
54+
echo ""
55+
56+
# Init
57+
echo "[2/4] Running terraform init..."
58+
INIT_START=$(date +%s%3N)
59+
terraform init -input=false -no-color > /dev/null 2>&1
60+
INIT_END=$(date +%s%3N)
61+
INIT_TIME=$((INIT_END - INIT_START))
62+
echo "Init time: ${INIT_TIME}ms"
63+
64+
# Apply
65+
echo ""
66+
echo "[3/4] Running terraform apply -parallelism=30..."
67+
echo "Started at: $(date)"
68+
APPLY_START=$(date +%s%3N)
69+
terraform apply -auto-approve -input=false -no-color -parallelism=30 > /tmp/apply-local.log 2>&1
70+
APPLY_END=$(date +%s%3N)
71+
APPLY_TIME=$((APPLY_END - APPLY_START))
72+
echo "Completed at: $(date)"
73+
echo ""
74+
echo "Last 10 lines of output:"
75+
tail -10 /tmp/apply-local.log
76+
echo ""
77+
78+
# Destroy
79+
echo "[4/4] Running terraform destroy..."
80+
DESTROY_START=$(date +%s%3N)
81+
terraform destroy -auto-approve -input=false -no-color -parallelism=30 > /dev/null 2>&1
82+
DESTROY_END=$(date +%s%3N)
83+
DESTROY_TIME=$((DESTROY_END - DESTROY_START))
84+
85+
# Cleanup
86+
cd /
87+
rm -rf "$WORK_DIR"
88+
89+
# Summary
90+
echo ""
91+
echo "============================================================"
92+
echo "SUMMARY - LOCAL MACHINE BASELINE"
93+
echo "============================================================"
94+
echo "Terraform init: ${INIT_TIME}ms ($((INIT_TIME / 1000))s)"
95+
echo "Terraform apply: ${APPLY_TIME}ms ($((APPLY_TIME / 1000))s) = $(echo "scale=2; $APPLY_TIME / 60000" | bc) minutes"
96+
echo "Terraform destroy: ${DESTROY_TIME}ms ($((DESTROY_TIME / 1000))s)"
97+
echo "============================================================"
98+
echo ""
99+
echo "Compare this to E2B results to see the overhead."
100+
echo ""
101+
102+
if [ $APPLY_TIME -lt 60000 ]; then
103+
echo "✅ Local apply took < 1 minute - this is the target for E2B"
104+
elif [ $APPLY_TIME -lt 120000 ]; then
105+
echo "🟡 Local apply took 1-2 minutes"
106+
else
107+
echo "⚠️ Local apply took > 2 minutes - your machine might be slow too"
108+
fi
109+

0 commit comments

Comments
 (0)