Fail an E2E test on anything the browser logs, in all three engines

e2e-tests/auth-refresh.spec.ts

@@ -1,4 +1,4 @@
-import { expect, test } from "@playwright/test";
+import { expect, test } from "./console-guard.ts";
 
 // Full-stack auth E2E: token timeout + silent re-mint ("stay signed in"). Runs against the
 // real Ory stack via e2e-tests/compose.auth.yml, where the session→JWT TTL is shortened to 8s and the

e2e-tests/console-guard.ts

@@ -0,0 +1,54 @@
+import { expect, test as base, type BrowserContext, type Page } from "@playwright/test";
+
+// The `test` every spec imports: it fails a test whose browser logged a console error or warning,
+// or threw, at any step — in whichever engine ran it. A zero-JS app has nothing to say in the
+// console, so anything there is a defect (a broken sub-resource, a rejected attribute, an engine
+// refusing a feature) that no assertion looks for.
+//
+// One module-level buffer is enough: a Playwright worker runs one test at a time, so the reset at
+// setup and the assertion at teardown bracket exactly the test in between.
+const problems: string[] = [];
+const allowed: RegExp[] = [];
+
+// The one message the stack itself provokes: the runner reaches `web`/`proxy` by container name over
+// plain http, and only a `localhost` origin is trustworthy without TLS — so Chromium drops the COOP
+// header the app sends and says so on every page. Over https, where a deployment serves, it applies.
+const EXPECTED = [/^console\.error: The Cross-Origin-Opener-Policy header has been ignored/];
+
+// Allow a message for the current test only, when the page under test provokes it on purpose.
+export function allowConsole(...patterns: RegExp[]): void {
+  allowed.push(...patterns);
+}
+
+function watch(page: Page): void {
+  page.on("console", (msg) => {
+    const type = msg.type();
+    if (type === "error" || type === "warning") problems.push(`console.${type}: ${msg.text()}`);
+  });
+  page.on("pageerror", (err) => problems.push(`pageerror: ${err.message}`));
+}
+
+// Every page of the context, however it is opened — `context.newPage()` fires this event too, so
+// watching the context is the whole job and a page must never be watched a second time on top.
+function watchContext(context: BrowserContext): BrowserContext {
+  context.on("page", watch);
+  return context;
+}
+
+// A spec that opens its own context — a page shared across a serial describe — goes through this.
+export function watchedPage(context: BrowserContext): Promise<Page> {
+  return watchContext(context).newPage();
+}
+
+export const test = base.extend<{ consoleGuard: void }>({
+  context: async ({ context }, use) => { await use(watchContext(context)); },
+  consoleGuard: [async ({}, use) => {
+    problems.length = 0;
+    allowed.length = 0;
+    await use();
+    const unexpected = problems.filter((p) => ![...EXPECTED, ...allowed].some((re) => re.test(p)));
+    expect(unexpected, "the browser logged nothing while this test ran").toEqual([]);
+  }, { auto: true }],
+});
+
+export { expect };

e2e-tests/devstack-login.spec.ts

@@ -1,4 +1,4 @@
-import { expect, test } from "@playwright/test";
+import { expect, test } from "./console-guard.ts";
 
 // Regression: the from-scratch dev experience the README/banner advertises must work. `docker compose
 // up`, open the printed login URL (http://localhost:3000), sign in as the seeded admin → you land on

e2e-tests/full-flow.spec.ts

@@ -1,4 +1,5 @@
-import { type Browser, type Page, expect, test } from "@playwright/test";
+import type { Browser, Page } from "@playwright/test";
+import { expect, test, watchedPage } from "./console-guard.ts";
 import { randomUUID } from "node:crypto";
 
 // Full browser E2E: the real Playwright UI against the live stack via the same-origin
@@ -27,7 +28,7 @@ async function loginPassword(page: Page): Promise<void> {
 // The themed Kratos page in another language: our own chrome, Kratos' own strings mapped by id, and
 // the card's own links keeping the choice (they are rendered by the flow body, not by the menu).
 test("the login page speaks the visitor's language, links included", async ({ browser }) => {
-  const page = await (await browser.newContext()).newPage();
+  const page = await watchedPage(await browser.newContext());
   await page.goto("/login?locale=sv-SE");
   await expect(page.locator("html")).toHaveAttribute("lang", "sv-SE");
   await expect(page.getByRole("heading", { name: "Logga in" })).toBeVisible();
@@ -43,7 +44,7 @@ test.describe.serial("authenticated admin journey", () => {
 
   test.beforeAll(async ({ browser: b }) => {
     browser = b;
-    page = await (await browser.newContext()).newPage();
+    page = await watchedPage(await browser.newContext());
     test.setTimeout(90_000);
     await loginPassword(page);
   });

e2e-tests/language.spec.ts

@@ -1,7 +1,6 @@
 import { readFileSync } from "node:fs";
 import { createPrivateKey, sign } from "node:crypto";
-import { mkdir } from "node:fs/promises";
-import { expect, test } from "@playwright/test";
+import { expect, test, watchedPage } from "./console-guard.ts";
 
 // Language switching in a real browser, Ory-free (the visual stack). Proves the whole path a
 // visitor takes: pick a language, read the page in it, and stay in it while clicking around —
@@ -9,7 +8,6 @@ import { expect, test } from "@playwright/test";
 
 const BASE_URL = process.env.BASE_URL ?? "http://localhost:3000";
 const SESSION_COOKIE = "plainpages_jwt";
-const SHOTS = "artifacts/screenshots";
 
 // Same trick as visual.spec.ts: sign a session JWT with the committed dev tokenizer key so the
 // gated pages render without standing up Ory.
@@ -38,8 +36,7 @@ test("the switcher changes language, and the choice survives clicking through th
   await expect(page).toHaveURL(/locale=sv-SE/);
   await expect(page.getByRole("heading", { name: "Startpanel" })).toBeVisible(); // the starter dashboard, in Swedish
   await expect(page.getByRole("link", { name: "Översikt", exact: true })).toBeVisible(); // the menu too
-  await mkdir(SHOTS, { recursive: true });
-  await page.screenshot({ fullPage: true, path: `${SHOTS}/live-05-swedish.png` });
+  await page.screenshot({ fullPage: true, path: `artifacts/screenshots/${test.info().project.name}/live-05-swedish.png` });
 
   // Clicking a menu item keeps Swedish — the host carries the choice onto the links it renders,
   // and the plugin's own page is translated from its own catalog. The section's own label comes
@@ -66,7 +63,7 @@ test("the switcher changes language, and the choice survives clicking through th
 
 test("a browser that asks for Swedish gets it without touching the URL", async ({ browser }) => {
   const context = await browser.newContext({ locale: "sv" }); // a browser set to Swedish, no region
-  const page = await context.newPage();
+  const page = await watchedPage(context);
   await page.goto(`${BASE_URL}/`);
 
   await expect(page.locator("html")).toHaveAttribute("lang", "sv-SE");

e2e-tests/oauth-login.spec.ts

@@ -1,4 +1,4 @@
-import { expect, test } from "@playwright/test";
+import { expect, test } from "./console-guard.ts";
 
 // Full-stack OAuth2 login + consent E2E: another app logs in *through* plainpages. Hydra
 // starts an authorization flow and hands the browser to web's /oauth2/login; web resolves it via

e2e-tests/playwright.config.ts

@@ -3,6 +3,9 @@ import { defineConfig, devices } from "@playwright/test";
 // Visual + functional checks against the live app (the `web` compose service, BASE_URL). Run via
 // e2e-tests/compose.visual.yml. Parallel per the project's E2E principle; deterministic colorScheme/viewport
 // so the rendered design is stable across runs.
+
+const ORY_FREE = /(visual|language)\.spec\.ts$/;
+
 export default defineConfig({
   testDir: ".",
   outputDir: "artifacts/test-output",
@@ -15,11 +18,14 @@ export default defineConfig({
     screenshot: "only-on-failure",
     viewport: { width: 1280, height: 800 },
   },
-  // CSS anchor positioning is the newest platform feature in the app and every popup menu rests on
-  // it, so the tests tagged @engines run in all three engines; the rest stay on chromium.
+  // The Ory-free suites run in all three engines: the console guard (console-guard.ts) only sees an
+  // engine's warnings when that engine renders the page, and the newest platform features in the app
+  // (popover, CSS anchor positioning, `:has()`) are exactly where engines disagree. They stay
+  // side-effect-free, so three parallel runs of them don't collide. The Ory-backed suites write to
+  // one shared backend and stay on chromium.
   projects: [
     { name: "chromium", use: { ...devices["Desktop Chrome"] } },
-    { name: "firefox", grep: /@engines/, use: { ...devices["Desktop Firefox"] } },
-    { name: "webkit", grep: /@engines/, use: { ...devices["Desktop Safari"] } },
+    { name: "firefox", testMatch: ORY_FREE, use: { ...devices["Desktop Firefox"] } },
+    { name: "webkit", testMatch: ORY_FREE, use: { ...devices["Desktop Safari"] } },
   ],
 });

e2e-tests/visual.spec.ts

@@ -1,14 +1,14 @@
 import { createPrivateKey, sign } from "node:crypto";
 import { readFileSync } from "node:fs";
-import { mkdir } from "node:fs/promises";
-import { expect, test, type Page } from "@playwright/test";
+import type { Page } from "@playwright/test";
+import { allowConsole, expect, test } from "./console-guard.ts";
 
-const SHOTS = "artifacts/screenshots";
 const BASE_URL = process.env.BASE_URL ?? "http://localhost:3000";
 const SESSION_COOKIE = "plainpages_jwt"; // src/auth/login.ts — web verifies it against the committed dev JWKS
 
+// Per engine: the three projects run this suite in parallel and would otherwise write one file.
 const shot = (page: Page, name: string): Promise<Buffer> =>
-  page.screenshot({ fullPage: true, path: `${SHOTS}/${name}.png` });
+  page.screenshot({ fullPage: true, path: `artifacts/screenshots/${test.info().project.name}/${name}.png` });
 
 // Sign a session JWT with the committed dev tokenizer key (bind-mounted at /repo/jwks.json), so the
 // gated dashboard renders for a "signed-in" user without standing up Ory — web verifies it
@@ -22,8 +22,6 @@ function devSession(permissions: string[] = []): string {
   return `${input}.${sign("SHA256", Buffer.from(input), { dsaEncoding: "ieee-p1363", key }).toString("base64url")}`;
 }
 
-test.beforeAll(async () => { await mkdir(SHOTS, { recursive: true }); });
-
 // The dashboard is gated: a page navigation needs a session. Plant one per test — a plain
 // member (no permissions) so the gated scheduling nav stays filtered out.
 test.beforeEach(async ({ context }) => {
@@ -74,7 +72,7 @@ test("theme switch flips the palette with no JavaScript", async ({ page }) => {
 // longer has to click the trigger again to get rid of one. Driven through the language picker; the
 // profile menu is the same block. Anchoring is asserted too — without `position-anchor` the panel
 // silently detaches and lands in the middle of the viewport.
-test("a popover menu sits on its trigger and closes on an outside click or Esc — no JavaScript @engines", async ({ page }) => {
+test("a popover menu sits on its trigger and closes on an outside click or Esc — no JavaScript", async ({ page }) => {
   await page.goto("/dashboard");
   const trigger = page.locator('button[aria-label="Language"]');
   const panel = page.locator('button[aria-label="Language"] + .menu-pop');
@@ -138,6 +136,7 @@ test("the public landing at / is ungated and links to sign in + register", async
 });
 
 test("unknown routes serve the 404 page (a real user-facing flow, covered end-to-end)", async ({ page }) => {
+  allowConsole(/status of 404/); // Chromium and WebKit log the status of the navigation under test
   const res = await page.goto("/no-such-page");
   expect(res?.status()).toBe(404);
   await expect(page.getByRole("heading", { name: "Page not found" })).toBeVisible();