fix(wasm): replace the module after an exception escapes a WASM call

README.md

@@ -171,6 +171,16 @@ pnpm run test
   pnpm run clean && pnpm run build && pnpm run test
   ```
 
+### Slow recovery checks (local only)
+
+Run these after changing the WASM recovery code in `templates/`. Both use the
+built `versions/<n>/wasm/` artifacts and are too slow for CI.
+
+```bash
+pnpm test:stress [13 18 ...]      # thousands of WASM crashes and >1 GiB inputs; memory must stay flat (a few minutes)
+pnpm test:mutations [13 18 ...]   # breaks each branch of the recovery code; the tests must catch every one (about 20 min)
+```
+
 ## Troubleshooting
 
 ### Common Issues

package.json

@@ -26,6 +26,8 @@
     "publish:versions": "node scripts/publish-versions.js",
     "update:versions-types": "node scripts/update-versions-types.js",
     "copy:templates": "node scripts/copy-templates.js",
+    "test:stress": "node scripts/test-stress.js",
+    "test:mutations": "node scripts/test-mutations.js",
     "build:parser": "pnpm --filter @pgsql/parser build",
     "build:parser:lts": "PARSER_BUILD_TYPE=lts pnpm --filter @pgsql/parser build",
     "build:parser:full": "PARSER_BUILD_TYPE=full pnpm --filter @pgsql/parser build",

scripts/test-mutations.js

@@ -0,0 +1,111 @@
+#!/usr/bin/env node
+// Checks that test/wasm-recovery.test.js covers every branch of the WASM
+// recovery code: each mutation below breaks one branch of the built
+// wasm/index.cjs, and the tests must fail (or hang) on every one of them.
+// Too slow for CI (about 20 s per mutation); run it after changing the
+// recovery code in templates/.
+//
+// usage: pnpm test:mutations [version ...] [--only <text>]   (defaults to 13 and 18)
+
+const { spawnSync } = require('child_process');
+const fs = require('fs');
+const path = require('path');
+
+const versionsDir = path.join(__dirname, '..', 'versions');
+const FINALLY = '        if (failures === failuresBefore) {';
+
+const replace = (from, to) => (source) => source.replace(from, to);
+const nth = (source, text, n, to) => {
+  let index = -1;
+  for (let i = 0; i <= n; i++) index = source.indexOf(text, index + 1);
+  return source.slice(0, index) + to + source.slice(index + text.length);
+};
+
+const MUTATIONS = [
+  ['callWasm: never marks the instance stale', replace('        stale = true;\n', '')],
+  ['callWasm: does not count the failure', replace('        failures++;\n', '')],
+  ['callWasm: never starts a fresh instance', replace('wasmModule = undefined;\n        instantiate();', 'wasmModule = undefined;\n')],
+  ['callWasm: drops the instance on the first failure', replace('if (++staleFailures >= MAX_STALE_FAILURES)', 'if (true)')],
+  ['callWasm: never drops a stale instance', replace('if (++staleFailures >= MAX_STALE_FAILURES)', 'if (false)')],
+  ['callWasm: cap off by one', replace('if (++staleFailures >= MAX_STALE_FAILURES)', 'if (++staleFailures > MAX_STALE_FAILURES)')],
+  ['callWasm: does not restore process.exitCode', replace('        if (proc)\n            proc.exitCode = exitCode;\n', '')],
+  ['callWasm: does not remember process.exitCode', replace('const exitCode = proc?.exitCode;', 'const exitCode = undefined;')],
+  ['callWasm: wraps every error', replace('if (error instanceof Error)\n            throw error;', 'if (false)\n            throw error;')],
+  ['callWasm: never wraps non-Error throws', replace('if (error instanceof Error)\n            throw error;', 'if (true)\n            throw error;')],
+  ['callWasm: no constructor-name fallback', replace('(error.message ?? error.constructor?.name)', '(error.message)')],
+  ['instantiate: does not reset staleFailures', replace('        staleFailures = 0;\n', '')],
+  ['instantiate: does not clear stale', replace('        stale = false;\n        staleFailures', '        staleFailures')],
+  ['instantiate: not single-flight', replace('    if (loading)\n        return loading;\n', '')],
+  ['instantiate: success never clears loading', replace('        staleFailures = 0;\n        loading = undefined;\n', '        staleFailures = 0;\n')],
+  ['instantiate: failure never clears loading', replace('    }, (error) => {\n        loading = undefined;\n', '    }, (error) => {\n')],
+  ['instantiate: swallows a failed load', replace('        loading = undefined;\n        throw error;\n', '        loading = undefined;\n')],
+  ['loadModule: ignores stale', replace('    while (!wasmModule || stale)\n        await instantiate();\n}', '    while (!wasmModule)\n        await instantiate();\n}')],
+  ['loadModule: waits only once', replace('    while (!wasmModule || stale)\n        await instantiate();\n}', '    if (!wasmModule || stale)\n        await instantiate();\n}')],
+  ['awaitInit: ignores stale', replace('        while (!wasmModule || stale)\n            await instantiate();', '        while (!wasmModule)\n            await instantiate();')],
+  ['awaitInit: waits only once', replace('        while (!wasmModule || stale)\n            await instantiate();', '        if (!wasmModule || stale)\n            await instantiate();')],
+];
+
+function functionAt(source, index) {
+  const pattern = /^(?:exports\.(\w+) = awaitInit\(async|function (\w+)\()/gm;
+  let name;
+  for (let match; (match = pattern.exec(source)) && match.index < index;) {
+    name = match[1] ? { name: match[1], async: true } : { name: match[2], async: false };
+  }
+  return name;
+}
+
+function mutationsFor(source) {
+  const list = [...MUTATIONS];
+  for (let n = 0, index = source.indexOf(FINALLY); index !== -1; n++, index = source.indexOf(FINALLY, index + 1)) {
+    const fn = functionAt(source, index);
+    list.push([`${fn.name} cleanup: frees after the call threw`, (s) => nth(s, FINALLY, n, '        if (true) {')]);
+    // An async call only starts on a non-stale instance, so this one can't change async behavior.
+    if (!fn.async) {
+      list.push([`${fn.name} cleanup: skips frees whenever stale`, (s) => nth(s, FINALLY, n, '        if (!stale) {')]);
+    }
+  }
+  return list;
+}
+
+const args = process.argv.slice(2);
+const onlyAt = args.indexOf('--only');
+const only = onlyAt === -1 ? undefined : args.splice(onlyAt, 2)[1];
+const versions = args.length ? args : ['13', '18'];
+
+let problems = 0;
+for (const version of versions) {
+  const dir = path.join(versionsDir, version);
+  const file = path.join(dir, 'wasm', 'index.cjs');
+  if (!fs.existsSync(file)) {
+    console.error(`v${version}: ${file} not found; build it first`);
+    problems++;
+    continue;
+  }
+  const original = fs.readFileSync(file, 'utf8');
+  const restore = () => fs.writeFileSync(file, original);
+  process.on('SIGINT', () => { restore(); process.exit(130); });
+  const mutations = mutationsFor(original).filter(([name]) => !only || name.includes(only));
+  let caught = 0;
+  try {
+    for (const [name, mutate] of mutations) {
+      const mutated = mutate(original);
+      if (mutated === original) {
+        console.log(`NOT APPLIED  v${version}  ${name}`);
+        problems++;
+        continue;
+      }
+      fs.writeFileSync(file, mutated);
+      const run = spawnSync(process.execPath, ['--test', 'test/wasm-recovery.test.js'], {
+        cwd: dir, encoding: 'utf8', timeout: 150000, stdio: ['ignore', 'pipe', 'ignore'],
+      });
+      const result = run.error?.code === 'ETIMEDOUT' ? 'caught (hangs)' : run.status !== 0 ? 'caught' : 'MISSED';
+      if (result === 'MISSED') problems++;
+      else caught++;
+      console.log(`${result.padEnd(14)} v${version}  ${name}`);
+    }
+  } finally {
+    restore();
+  }
+  console.log(`v${version}: ${caught}/${mutations.length} mutations caught\n`);
+}
+process.exit(problems ? 1 : 0);

scripts/test-stress.js

@@ -0,0 +1,104 @@
+#!/usr/bin/env node
+// Long-running recovery checks that are too slow for CI. Each scenario runs in
+// a fresh Node process with GC exposed and fails if WASM memory (V8 `external`)
+// grows, `SELECT 1` stops working, or process.exitCode changes.
+//
+// usage: pnpm test:stress [version ...]   (defaults to every built version)
+
+const { execFileSync } = require('child_process');
+const fs = require('fs');
+const path = require('path');
+
+const versionsDir = path.join(__dirname, '..', 'versions');
+const MAX_GROWTH = 32 * 1024 * 1024;
+
+const SCENARIOS = {
+  'async: 1,000 stack overflows': `
+    for (let i = 0; i < 1000; i++) {
+      await lib.parse(DEEP).catch(() => {});
+      check(i);
+    }`,
+  'sync: 1,000 stack overflows, awaiting loadModule() after each': `
+    for (let i = 0; i < 1000; i++) {
+      try { lib.parseSync(DEEP); } catch {}
+      await lib.loadModule();
+      check(i);
+    }`,
+  'sync: 200 stack overflows in one synchronous loop, 20 times': `
+    for (let i = 0; i < 20; i++) {
+      for (let j = 0; j < 200; j++) { try { lib.parseSync(DEEP); } catch {} }
+      await lib.loadModule();
+      check(i);
+    }`,
+  'async: 20 inputs over the 1 GiB WASM limit': `
+    for (let i = 0; i < 20; i++) {
+      await lib.parse(HUGE).catch(() => {});
+      check(i);
+    }`,
+  'control: 1,000 normal parses': `
+    for (let i = 0; i < 1000; i++) {
+      await lib.parse('SELECT 1' + '+1'.repeat(2000));
+      check(i);
+    }`,
+};
+
+function runScenario(indexPath, body) {
+  const script = `
+    const lib = require(${JSON.stringify(indexPath)});
+    const DEEP = 'SELECT 1' + '+1'.repeat(100000);
+    const HUGE = 'SELECT 1;\\n'.repeat(5000000);
+    let broken = null;
+    function check(i) {
+      try { lib.parseSync('SELECT 1'); } catch { if (broken === null) broken = i; }
+    }
+    (async () => {
+      await lib.loadModule();
+      lib.parseSync('SELECT 1');
+      gc(); gc();
+      const base = process.memoryUsage().external;
+      ${body}
+      await lib.loadModule();
+      await new Promise((resolve) => setTimeout(resolve, 0));
+      gc(); gc();
+      const grown = process.memoryUsage().external - base;
+      process.stdout.write('\\nresult=' + JSON.stringify({ grown, broken, exitCode: process.exitCode ?? null }));
+    })();`;
+  // libpg_query prints a memory dump to stdout on out-of-memory, so read the marked line.
+  const output = execFileSync(process.execPath, ['--expose-gc', '-e', script], {
+    encoding: 'utf8',
+    stdio: ['ignore', 'pipe', 'ignore'],
+    maxBuffer: 64 * 1024 * 1024,
+  });
+  return JSON.parse(output.match(/^result=(.*)$/m)[1]);
+}
+
+const requested = process.argv.slice(2);
+const versions = (requested.length ? requested : fs.readdirSync(versionsDir))
+  .filter((v) => fs.existsSync(path.join(versionsDir, v, 'wasm', 'index.cjs')));
+if (!versions.length) {
+  console.error('No built versions found; run `pnpm build` in versions/<n> first.');
+  process.exit(1);
+}
+
+let failed = 0;
+for (const version of versions) {
+  const indexPath = path.join(versionsDir, version, 'wasm', 'index.cjs');
+  for (const [name, body] of Object.entries(SCENARIOS)) {
+    const start = Date.now();
+    let problem;
+    let result;
+    try {
+      result = runScenario(indexPath, body);
+      if (result.grown >= MAX_GROWTH) problem = `WASM memory grew by ${(result.grown / 1048576).toFixed(1)} MB`;
+      else if (result.broken !== null) problem = `SELECT 1 failed from iteration ${result.broken}`;
+      else if (result.exitCode !== null) problem = `process.exitCode became ${result.exitCode}`;
+    } catch (error) {
+      problem = `process failed: ${error.message.split('\n')[0]}`;
+    }
+    const seconds = ((Date.now() - start) / 1000).toFixed(1);
+    const growth = result ? `${(result.grown / 1048576).toFixed(1)} MB` : '-';
+    console.log(`${problem ? 'FAIL' : 'ok  '}  v${version}  ${name}  (${seconds}s, memory ${growth})${problem ? `: ${problem}` : ''}`);
+    if (problem) failed++;
+  }
+}
+process.exit(failed ? 1 : 0);