Skip to content

Commit f0c2017

Browse files
ammar-agentammar-agent
committed
🤖 feat: add robust Electron E2E tests for regression prevention
Add comprehensive E2E tests covering window lifecycle, IPC robustness, streaming edge cases, persistence, and error display. These tests target recent regression patterns including: - MockBrowserWindow.isDestroyed() issues - IPC send to destroyed window race conditions - Duplicate IPC handler registration on window recreate - Error state display and recovery New test files: - windowLifecycle.spec.ts: window operations, IPC stability - ipcRobustness.spec.ts: concurrent IPC calls, state corruption - streamEdgeCases.spec.ts: streaming during UI operations, errors - persistence.spec.ts: chat history, settings, mode persistence - errorDisplay.spec.ts: error messages, recovery flows Infrastructure changes: - Add error mock scenarios (rate limit, server, network errors) - Update stream timeline capture to handle stream-error events - CI matrix: Linux (comprehensive) + macOS (window lifecycle only) _Generated with `mux`_
1 parent 3b530f7 commit f0c2017

File tree

9 files changed

+689
-5
lines changed

9 files changed

+689
-5
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -148,9 +148,21 @@ jobs:
148148
run: make test-storybook
149149

150150
e2e-test:
151-
name: End-to-End Tests
152-
runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-16' || 'ubuntu-latest' }}
151+
name: E2E Tests (${{ matrix.os }})
153152
if: github.event.inputs.test_filter == ''
153+
strategy:
154+
fail-fast: false
155+
matrix:
156+
include:
157+
# Linux: comprehensive E2E tests
158+
- os: linux
159+
runner: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-16' || 'ubuntu-latest' }}
160+
test_scope: "all"
161+
# macOS: window lifecycle and platform-dependent tests only
162+
- os: macos
163+
runner: macos-latest
164+
test_scope: "window-lifecycle"
165+
runs-on: ${{ matrix.runner }}
154166
steps:
155167
- name: Checkout code
156168
uses: actions/checkout@v4
@@ -159,18 +171,24 @@ jobs:
159171

160172
- uses: ./.github/actions/setup-mux
161173

162-
- name: Install xvfb
174+
- name: Install xvfb (Linux)
175+
if: matrix.os == 'linux'
163176
run: |
164177
sudo apt-get update
165178
sudo apt-get install -y xvfb
166179
167180
- uses: ./.github/actions/setup-playwright
168181

169-
- name: Run e2e tests
182+
- name: Run comprehensive e2e tests (Linux)
183+
if: matrix.os == 'linux'
170184
run: xvfb-run -a make test-e2e
171185
env:
172186
ELECTRON_DISABLE_SANDBOX: 1
173187

188+
- name: Run window lifecycle e2e tests (macOS)
189+
if: matrix.os == 'macos'
190+
run: make test-e2e PLAYWRIGHT_ARGS="tests/e2e/scenarios/windowLifecycle.spec.ts"
191+
174192
docker-smoke-test:
175193
name: Docker Smoke Test
176194
runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-16' || 'ubuntu-latest' }}

‎src/node/services/mock/scenarios.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import * as review from "./scenarios/review";
33
import * as toolFlows from "./scenarios/toolFlows";
44
import * as slashCommands from "./scenarios/slashCommands";
55
import * as permissionModes from "./scenarios/permissionModes";
6+
import * as errorScenarios from "./scenarios/errorScenarios";
67
import type { ScenarioTurn } from "./scenarioTypes";
78

89
export const allScenarios: ScenarioTurn[] = [
@@ -11,4 +12,5 @@ export const allScenarios: ScenarioTurn[] = [
1112
...toolFlows.scenarios,
1213
...slashCommands.scenarios,
1314
...permissionModes.scenarios,
15+
...errorScenarios.scenarios,
1416
];

‎src/node/services/mock/scenarios/errorScenarios.ts‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
import type { ScenarioTurn } from "@/node/services/mock/scenarioTypes";
2+
import { STREAM_BASE_DELAY } from "@/node/services/mock/scenarioTypes";
3+
import { KNOWN_MODELS } from "@/common/constants/knownModels";
4+
5+
export const ERROR_PROMPTS = {
6+
TRIGGER_RATE_LIMIT: "Trigger rate limit error",
7+
TRIGGER_API_ERROR: "Trigger API error",
8+
TRIGGER_NETWORK_ERROR: "Trigger network error",
9+
} as const;
10+
11+
const rateLimitErrorTurn: ScenarioTurn = {
12+
user: {
13+
text: ERROR_PROMPTS.TRIGGER_RATE_LIMIT,
14+
thinkingLevel: "low",
15+
mode: "exec",
16+
},
17+
assistant: {
18+
messageId: "msg-error-ratelimit",
19+
events: [
20+
{
21+
kind: "stream-start",
22+
delay: 0,
23+
messageId: "msg-error-ratelimit",
24+
model: KNOWN_MODELS.GPT.id,
25+
},
26+
{
27+
kind: "stream-delta",
28+
delay: STREAM_BASE_DELAY,
29+
text: "Processing your request...",
30+
},
31+
{
32+
kind: "stream-error",
33+
delay: STREAM_BASE_DELAY * 2,
34+
error: "Rate limit exceeded. Please retry after 60 seconds.",
35+
errorType: "rate_limit",
36+
},
37+
],
38+
},
39+
};
40+
41+
const apiErrorTurn: ScenarioTurn = {
42+
user: {
43+
text: ERROR_PROMPTS.TRIGGER_API_ERROR,
44+
thinkingLevel: "low",
45+
mode: "exec",
46+
},
47+
assistant: {
48+
messageId: "msg-error-api",
49+
events: [
50+
{
51+
kind: "stream-start",
52+
delay: 0,
53+
messageId: "msg-error-api",
54+
model: KNOWN_MODELS.GPT.id,
55+
},
56+
{
57+
kind: "stream-error",
58+
delay: STREAM_BASE_DELAY,
59+
error: "Internal server error occurred while processing the request.",
60+
errorType: "server_error",
61+
},
62+
],
63+
},
64+
};
65+
66+
const networkErrorTurn: ScenarioTurn = {
67+
user: {
68+
text: ERROR_PROMPTS.TRIGGER_NETWORK_ERROR,
69+
thinkingLevel: "low",
70+
mode: "exec",
71+
},
72+
assistant: {
73+
messageId: "msg-error-network",
74+
events: [
75+
{
76+
kind: "stream-start",
77+
delay: 0,
78+
messageId: "msg-error-network",
79+
model: KNOWN_MODELS.GPT.id,
80+
},
81+
{
82+
kind: "stream-error",
83+
delay: STREAM_BASE_DELAY,
84+
error: "Network connection lost. Please check your internet connection.",
85+
errorType: "network",
86+
},
87+
],
88+
},
89+
};
90+
91+
export const scenarios: ScenarioTurn[] = [rateLimitErrorTurn, apiErrorTurn, networkErrorTurn];

‎tests/e2e/scenarios/errorDisplay.spec.ts‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
import { electronTest as test, electronExpect as expect } from "../electronTest";
2+
import { ERROR_PROMPTS } from "@/node/services/mock/scenarios/errorScenarios";
3+
import { LIST_PROGRAMMING_LANGUAGES } from "@/node/services/mock/scenarios/basicChat";
4+
5+
test.skip(
6+
({ browserName }) => browserName !== "chromium",
7+
"Electron scenario runs on chromium only"
8+
);
9+
10+
test.describe("error display", () => {
11+
test("rate limit error shows in transcript with message", async ({ ui, page }) => {
12+
await ui.projects.openFirstWorkspace();
13+
await ui.chat.setMode("Exec");
14+
15+
await ui.chat.captureStreamTimeline(async () => {
16+
await ui.chat.sendMessage(ERROR_PROMPTS.TRIGGER_RATE_LIMIT);
17+
});
18+
19+
// Error message should be visible in the transcript
20+
const transcript = page.getByRole("log", { name: "Conversation transcript" });
21+
await expect(transcript).toBeVisible();
22+
23+
// Should show the rate limit error text
24+
await expect(
25+
transcript.getByText("Rate limit exceeded. Please retry after 60 seconds.")
26+
).toBeVisible();
27+
});
28+
29+
test("server error shows in transcript", async ({ ui, page }) => {
30+
await ui.projects.openFirstWorkspace();
31+
await ui.chat.setMode("Exec");
32+
33+
await ui.chat.captureStreamTimeline(async () => {
34+
await ui.chat.sendMessage(ERROR_PROMPTS.TRIGGER_API_ERROR);
35+
});
36+
37+
const transcript = page.getByRole("log", { name: "Conversation transcript" });
38+
await expect(
39+
transcript.getByText("Internal server error occurred while processing the request.")
40+
).toBeVisible();
41+
});
42+
43+
test("network error shows in transcript", async ({ ui, page }) => {
44+
await ui.projects.openFirstWorkspace();
45+
await ui.chat.setMode("Exec");
46+
47+
await ui.chat.captureStreamTimeline(async () => {
48+
await ui.chat.sendMessage(ERROR_PROMPTS.TRIGGER_NETWORK_ERROR);
49+
});
50+
51+
const transcript = page.getByRole("log", { name: "Conversation transcript" });
52+
await expect(
53+
transcript.getByText("Network connection lost. Please check your internet connection.")
54+
).toBeVisible();
55+
});
56+
57+
test("app remains functional after error", async ({ ui, page }) => {
58+
await ui.projects.openFirstWorkspace();
59+
await ui.chat.setMode("Exec");
60+
61+
// Trigger an error
62+
await ui.chat.captureStreamTimeline(async () => {
63+
await ui.chat.sendMessage(ERROR_PROMPTS.TRIGGER_API_ERROR);
64+
});
65+
66+
// Verify app is still functional - can open settings
67+
await ui.settings.open();
68+
await ui.settings.expectOpen();
69+
await ui.settings.close();
70+
71+
// Chat input should still be usable
72+
const chatInput = page.getByRole("textbox", { name: /message/i });
73+
await expect(chatInput).toBeVisible();
74+
await expect(chatInput).toBeEnabled();
75+
});
76+
77+
test("multiple errors don't crash app", async ({ ui, page }) => {
78+
await ui.projects.openFirstWorkspace();
79+
await ui.chat.setMode("Exec");
80+
81+
// Trigger multiple errors in sequence
82+
await ui.chat.captureStreamTimeline(async () => {
83+
await ui.chat.sendMessage(ERROR_PROMPTS.TRIGGER_RATE_LIMIT);
84+
});
85+
86+
await ui.chat.captureStreamTimeline(async () => {
87+
await ui.chat.sendMessage(ERROR_PROMPTS.TRIGGER_API_ERROR);
88+
});
89+
90+
await ui.chat.captureStreamTimeline(async () => {
91+
await ui.chat.sendMessage(ERROR_PROMPTS.TRIGGER_NETWORK_ERROR);
92+
});
93+
94+
// App should still be responsive
95+
await expect(page.getByRole("navigation", { name: "Projects" })).toBeVisible();
96+
97+
// Can still send a normal message after errors
98+
await ui.chat.setMode("Plan");
99+
const timeline = await ui.chat.captureStreamTimeline(async () => {
100+
await ui.chat.sendMessage(LIST_PROGRAMMING_LANGUAGES);
101+
});
102+
103+
expect(timeline.events.some((e) => e.type === "stream-end")).toBe(true);
104+
});
105+
106+
test("error state clears on successful message", async ({ ui }) => {
107+
await ui.projects.openFirstWorkspace();
108+
await ui.chat.setMode("Exec");
109+
110+
// Trigger an error first
111+
const errorTimeline = await ui.chat.captureStreamTimeline(async () => {
112+
await ui.chat.sendMessage(ERROR_PROMPTS.TRIGGER_API_ERROR);
113+
});
114+
expect(errorTimeline.events.some((e) => e.type === "stream-error")).toBe(true);
115+
116+
// Send a successful message
117+
await ui.chat.setMode("Plan");
118+
const successTimeline = await ui.chat.captureStreamTimeline(async () => {
119+
await ui.chat.sendMessage(LIST_PROGRAMMING_LANGUAGES);
120+
});
121+
122+
// Should complete successfully
123+
expect(successTimeline.events.some((e) => e.type === "stream-end")).toBe(true);
124+
await ui.chat.expectTranscriptContains("Python");
125+
});
126+
});

‎tests/e2e/scenarios/ipcRobustness.spec.ts‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
import { electronTest as test, electronExpect as expect } from "../electronTest";
2+
import { LIST_PROGRAMMING_LANGUAGES } from "@/node/services/mock/scenarios/basicChat";
3+
4+
test.skip(
5+
({ browserName }) => browserName !== "chromium",
6+
"Electron scenario runs on chromium only"
7+
);
8+
9+
test.describe("IPC robustness", () => {
10+
test("IPC calls during stream don't race", async ({ ui, page }) => {
11+
await ui.projects.openFirstWorkspace();
12+
13+
// First, send a message and wait for it to complete
14+
const timeline = await ui.chat.captureStreamTimeline(async () => {
15+
await ui.chat.sendMessage(LIST_PROGRAMMING_LANGUAGES);
16+
});
17+
18+
expect(timeline.events.length).toBeGreaterThan(0);
19+
expect(timeline.events[timeline.events.length - 1]?.type).toBe("stream-end");
20+
21+
// Now trigger IPC calls by opening/closing settings rapidly
22+
await ui.settings.open();
23+
await ui.settings.close();
24+
await ui.settings.open();
25+
await ui.settings.close();
26+
27+
// Verify app is still responsive and transcript preserved
28+
await ui.chat.expectTranscriptContains("Python");
29+
});
30+
31+
test("concurrent IPC calls resolve correctly", async ({ ui, page }) => {
32+
await ui.projects.openFirstWorkspace();
33+
34+
// Trigger multiple UI operations that involve IPC
35+
await ui.settings.open();
36+
await ui.settings.selectSection("Providers");
37+
await ui.settings.selectSection("Models");
38+
await ui.settings.close();
39+
40+
// Immediately try to send a message
41+
const timeline = await ui.chat.captureStreamTimeline(async () => {
42+
await ui.chat.sendMessage(LIST_PROGRAMMING_LANGUAGES);
43+
});
44+
45+
expect(timeline.events.length).toBeGreaterThan(0);
46+
await ui.chat.expectTranscriptContains("JavaScript");
47+
});
48+
49+
test("rapid workspace interactions don't corrupt state", async ({ ui }) => {
50+
await ui.projects.openFirstWorkspace();
51+
52+
// Send a message
53+
await ui.chat.captureStreamTimeline(async () => {
54+
await ui.chat.sendMessage(LIST_PROGRAMMING_LANGUAGES);
55+
});
56+
57+
// Toggle mode rapidly - this exercises IPC without starting new streams
58+
await ui.chat.setMode("Exec");
59+
await ui.chat.setMode("Plan");
60+
61+
// Should still show previous messages after mode switching
62+
await ui.chat.expectTranscriptContains("Python");
63+
await ui.chat.expectTranscriptContains("JavaScript");
64+
await ui.chat.expectTranscriptContains("Rust");
65+
});
66+
67+
test("settings changes during chat don't cause errors", async ({ ui, page }) => {
68+
await ui.projects.openFirstWorkspace();
69+
70+
// Open settings
71+
await ui.settings.open();
72+
73+
// Navigate around settings
74+
await ui.settings.selectSection("Providers");
75+
await ui.settings.selectSection("General");
76+
77+
// Close settings
78+
await ui.settings.close();
79+
80+
// Send a message - should work normally
81+
const timeline = await ui.chat.captureStreamTimeline(async () => {
82+
await ui.chat.sendMessage(LIST_PROGRAMMING_LANGUAGES);
83+
});
84+
85+
expect(timeline.events.length).toBeGreaterThan(0);
86+
expect(timeline.events.some((e) => e.type === "stream-end")).toBe(true);
87+
});
88+
});

0 commit comments

Comments
 (0)