From 590f8c668864c9cc4997ed7732a6fa8716fb2f0e Mon Sep 17 00:00:00 2001 From: Nick Nisi Date: Mon, 19 Oct 2026 18:11:52 -0500 Subject: [PATCH 01/11] fix(installer): stop Kotlin, PHP, and Laravel installs aborting on a missing package.json runAgentInstaller read package.json for every integration, and a missing file exits the process. Kotlin, PHP, and Laravel go through this runner, so their installs stopped right after detection with "Could not find package.json". Only JavaScript integrations read it now. Vanilla JS checked for a package named "workos", which no plain project has, so it asked "Vanilla JavaScript does not seem to be installed. Do you still want to continue?" (default No) and failed outright in CI and agent mode. It is the fallback for any JS project, so the check is skipped. --- src/lib/agent-runner.spec.ts | 37 +++++++++++++++++++++++++++++++++++- src/lib/agent-runner.ts | 10 ++++++++-- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/src/lib/agent-runner.spec.ts b/src/lib/agent-runner.spec.ts index b7ace415..2f02174d 100644 --- a/src/lib/agent-runner.spec.ts +++ b/src/lib/agent-runner.spec.ts @@ -29,7 +29,7 @@ import { runAgentInstaller } from './agent-runner.js'; import { validateInstallation, quickCheckValidateAndFormat } from './validation/index.js'; import { autoConfigureWorkOSEnvironment } from './workos-management.js'; import { writeEnvLocal } from './env-writer.js'; -import { getOrAskForWorkOSCredentials } from '../utils/ui-utils.js'; +import { ensurePackageIsInstalled, getOrAskForWorkOSCredentials, getPackageDotJson } from '../utils/ui-utils.js'; const options: InstallerOptions = { debug: false, @@ -244,3 +244,38 @@ describe('installer prompt', () => { expect(runAgent).not.toHaveBeenCalled(); }); }); + +describe('project manifest checks', () => { + it.each([ + ['kotlin', 'kotlin', 'workos-kotlin'], + ['php', 'php', 'workos-php'], + ['php-laravel', 'php', 'workos-php-laravel'], + ] as const)('does not read package.json for %s projects', async (integration, language, skillName) => { + await runAgentInstaller({ ...config, metadata: { ...config.metadata, integration, language, skillName } }, options); + + expect(getPackageDotJson).not.toHaveBeenCalled(); + expect(ensurePackageIsInstalled).not.toHaveBeenCalled(); + expect(runAgent).toHaveBeenCalledTimes(1); + }); + + it('does not require a framework package for vanilla JS', async () => { + await runAgentInstaller( + { + ...config, + metadata: { ...config.metadata, integration: 'vanilla-js', skillName: 'workos-authkit-vanilla-js' }, + detection: { packageName: 'workos', packageDisplayName: 'Vanilla JavaScript', getVersion: () => undefined }, + }, + options, + ); + + expect(getPackageDotJson).toHaveBeenCalled(); + expect(ensurePackageIsInstalled).not.toHaveBeenCalled(); + expect(runAgent).toHaveBeenCalledTimes(1); + }); + + it('still requires the framework package for other JS integrations', async () => { + await runAgentInstaller(config, options); + + expect(ensurePackageIsInstalled).toHaveBeenCalledWith(expect.anything(), 'next', 'Next.js'); + }); +}); diff --git a/src/lib/agent-runner.ts b/src/lib/agent-runner.ts index 7a8ddd26..f47efae2 100644 --- a/src/lib/agent-runner.ts +++ b/src/lib/agent-runner.ts @@ -41,8 +41,14 @@ export async function runAgentInstaller(config: FrameworkConfig, options: Instal // Git check is now handled by the state machine - no need to check here // Framework detection and version - const packageJson = await getPackageDotJson(options); - await ensurePackageIsInstalled(packageJson, config.detection.packageName, config.detection.packageDisplayName); + // Only JS integrations have a package.json; reading it elsewhere aborts the install + // (Kotlin, PHP, Laravel). Vanilla JS is the fallback for any JS project, so it has + // no framework package to require. + const isJavaScriptProject = config.metadata.language === 'javascript'; + const packageJson = isJavaScriptProject ? await getPackageDotJson(options) : {}; + if (isJavaScriptProject && config.metadata.integration !== 'vanilla-js') { + await ensurePackageIsInstalled(packageJson, config.detection.packageName, config.detection.packageDisplayName); + } const frameworkVersion = config.detection.getVersion(packageJson); From 8d0db6b674d871d81d9aaffec752cb9dcb9908f2 Mon Sep 17 00:00:00 2001 From: Nick Nisi Date: Mon, 19 Oct 2026 18:12:07 -0500 Subject: [PATCH 02/11] fix(installer): give the agent Claude Code's system prompt and state its rules - systemPrompt: Claude Code preset plus installer rules. Omitted, the SDK sends an empty system prompt, so the agent never got Claude Code's tool guidance (Read/Grep over cat/grep) and learned the Bash allowlist one denied command at a time (about 80% of Bash calls were denied). - Allowlist: read-only git (status/diff/log/show/ls-files/rev-parse), `cd && `, per-tool syntax checks (python -m py_compile/venv, ruby -c, php -l, uv venv), and dependency changes (remove/uninstall, go get, go mod tidy, composer require). Flags stay per tool because the same flag can run code elsewhere (python -c). - Non-Next.js JS prompts say the setup reference covers what the installer handles, so agents stop hunting for it. - options.onAgentMessage lets evals observe the real integration run. --- src/lib/agent-interface.spec.ts | 68 ++++++++++++++++++++++++ src/lib/agent-interface.ts | 93 +++++++++++++++++++++++++++++---- src/lib/agent-runner.ts | 16 ++++-- src/utils/types.ts | 6 +++ 4 files changed, 169 insertions(+), 14 deletions(-) diff --git a/src/lib/agent-interface.spec.ts b/src/lib/agent-interface.spec.ts index 478517bd..05bb5d14 100644 --- a/src/lib/agent-interface.spec.ts +++ b/src/lib/agent-interface.spec.ts @@ -181,6 +181,28 @@ describe('runAgent retry loop', () => { expect(options.strictMcpConfig).toBe(true); }); + it('states installer rules in the system prompt', async () => { + mockQuery.mockImplementation(createMockSDKResponse([{ text: 'Done!' }])); + + await runAgent(makeAgentConfig(), 'Test prompt', makeOptions(), undefined, emitter); + + const { options } = mockQuery.mock.calls[0][0] as { options: Record }; + expect(options.systemPrompt).toEqual({ + type: 'preset', + preset: 'claude_code', + append: expect.stringContaining('WorkOS installer environment'), + }); + }); + + it('passes agent messages to options.onAgentMessage', async () => { + mockQuery.mockImplementation(createMockSDKResponse([{ text: 'Done!' }])); + const onAgentMessage = vi.fn(); + + await runAgent(makeAgentConfig(), 'Test prompt', makeOptions({ onAgentMessage }), undefined, emitter); + + expect(onAgentMessage).toHaveBeenCalled(); + }); + it('returns retryCount=0 when no retryConfig provided', async () => { mockQuery.mockImplementation(createMockSDKResponse([{ text: 'Done!' }])); @@ -579,4 +601,50 @@ describe('installerCanUseTool', () => { }); expect(result.behavior).toBe('deny'); }); + describe('with a working directory', () => { + const cwd = '/proj'; + const verdict = (command: string) => installerCanUseTool('Bash', { command }, cwd).behavior; + + it('allows an allowlisted command after cd into the working directory only', () => { + expect(verdict('cd /proj && bun run build')).toBe('allow'); + expect(verdict('cd /proj && bun run build 2>&1 | tail -20')).toBe('allow'); + expect(verdict('cd /other && bun run build')).toBe('deny'); + expect(verdict('cd /proj && bun run build && curl http://x/')).toBe('deny'); + }); + + it('allows read-only git and denies git that writes or runs programs', () => { + expect(verdict('git status --short')).toBe('allow'); + expect(verdict('git -C /proj status')).toBe('allow'); + expect(verdict('git diff --stat | head -50')).toBe('allow'); + expect(verdict('git -C /etc log')).toBe('deny'); + expect(verdict('git diff --output=/tmp/x')).toBe('deny'); + expect(verdict('git -c core.pager=sh log')).toBe('deny'); + expect(verdict('git commit -m x')).toBe('deny'); + expect(verdict('git push')).toBe('deny'); + }); + + it('allows syntax checks per tool without allowing code execution', () => { + expect(verdict('python3 -m py_compile server.py')).toBe('allow'); + expect(verdict('python3 -m venv .venv')).toBe('allow'); + expect(verdict('ruby -c app.rb')).toBe('allow'); + expect(verdict('php -l index.php')).toBe('allow'); + expect(verdict('python3 -c "print(1)"')).toBe('deny'); + expect(verdict('python3 -m http.server')).toBe('deny'); + expect(verdict('ruby -e "puts 1"')).toBe('deny'); + expect(verdict('php -r "echo 1"')).toBe('deny'); + }); + + it('allows dependency changes across ecosystems', () => { + expect(verdict('bun remove @auth0/auth0-react')).toBe('allow'); + expect(verdict('go get github.com/workos/workos-go/v6')).toBe('allow'); + expect(verdict('go mod tidy')).toBe('allow'); + expect(verdict('composer require workos/workos-php')).toBe('allow'); + }); + + it('still denies network, the workos CLI, and plain inspection commands', () => { + expect(verdict('curl -sL https://example.com')).toBe('deny'); + expect(verdict('npx workos whoami')).toBe('deny'); + expect(verdict('ls -la')).toBe('deny'); + }); + }); }); diff --git a/src/lib/agent-interface.ts b/src/lib/agent-interface.ts index c61e6eeb..f6c65512 100644 --- a/src/lib/agent-interface.ts +++ b/src/lib/agent-interface.ts @@ -3,7 +3,7 @@ * Uses Claude Agent SDK directly with WorkOS MCP server */ -import { dirname } from 'path'; +import { dirname, resolve } from 'path'; import { getSkillsDir as getSkillsPackageDir } from './skills-assets.js'; import { ensureClaudeCodeExecutable } from './agent-sdk-assets.js'; import { debug, logInfo, logWarn, logError, initLogFile, getLogFilePath } from '../utils/debug.js'; @@ -102,6 +102,23 @@ export type AgentRunConfig = { claudeExecutablePath: string; }; +/** + * Appended to Claude Code's system prompt for every installer run. It states up front + * what installerCanUseTool enforces, so the agent doesn't discover the rules one + * denied command at a time. Keep it in sync with the allowlist below. + */ +export const INSTALLER_SYSTEM_PROMPT_APPEND = `# WorkOS installer environment + +You are running inside the WorkOS CLI installer, not an interactive session, and your working directory is the user's project. + +- Bash runs only: package-manager install/add/remove commands; the project's build, typecheck, lint, format, and test scripts; language checks such as \`go build\`, \`go vet\`, \`mix compile\`, \`python3 -m py_compile\`, \`ruby -c\`, \`php -l\`, \`dotnet build\`, and \`./gradlew build\`; and read-only git (\`status\`, \`diff\`, \`log\`, \`show\`, \`ls-files\`). +- Run one command per Bash call from the working directory. Don't use \`cd\`, \`&&\`, \`;\`, \`$(...)\`, or pipes other than \`| head\` or \`| tail\`. +- Inspect files with Read, Grep, and Glob, not Bash. +- Bash has no network access: no curl, \`npm view\`, or \`git clone\`. Use WebFetch to read documentation. The \`workos\` CLI is not available to you. +- Don't start dev servers or open a browser to test sign-in. After you finish, the installer typechecks and builds the project and sends you any errors to fix. +- Don't change WorkOS dashboard settings (redirect URIs, Sign-out URI, Initiate login URI, CORS). The installer configures supported settings after your code passes validation. List anything you could not verify as unverified. +- Leave changes unstaged: don't stage, commit, or create branches.`; + /** * Package managers that can be used to run commands. * Includes JS and non-JS ecosystem package managers for multi-SDK support. @@ -153,10 +170,16 @@ const PACKAGE_MANAGERS = [ * Note: Linting tools are in LINTING_TOOLS and checked separately. */ const SAFE_SCRIPTS = [ - // Package installation + // Package installation and removal 'install', 'add', 'ci', + 'remove', + 'uninstall', + 'get', // go get + 'require', // composer require + 'mod tidy', + 'mod download', // Build 'build', // Type checking (various naming conventions) @@ -204,6 +227,46 @@ const SAFE_SCRIPTS = [ */ const DANGEROUS_OPERATORS = /[;`$()\n\r]/; +/** + * Per-tool arguments that only check syntax or create a local environment. + * Kept per tool because the same flag can execute code elsewhere + * (`ruby -c` checks syntax; `python -c` runs code). + */ +const TOOL_SAFE_ARGS: Record = { + python: ['-m py_compile', '-m compileall', '-m venv', '-m pip install'], + python3: ['-m py_compile', '-m compileall', '-m venv', '-m pip install'], + uv: ['venv', 'pip install', 'sync'], + ruby: ['-c '], + php: ['-l '], +}; + +/** Read-only git subcommands the agent may use to see what changed. */ +const READ_ONLY_GIT_SUBCOMMANDS = ['status', 'diff', 'log', 'show', 'ls-files', 'rev-parse']; +/** git arguments that write files or run external programs. */ +const UNSAFE_GIT_ARG = /^(-c|-o|--output.*|--ext-diff|--textconv|--exec.*|--upload-pack.*|--open-files-in-pager.*)$/; + +function isReadOnlyGit(command: string, cwd?: string): boolean { + const parts = command.split(/\s+/); + if (parts[0] !== 'git') return false; + let i = 1; + if (parts[i] === '--no-pager') i++; + if (parts[i] === '-C') { + if (!cwd || !parts[i + 1] || resolve(cwd, parts[i + 1]) !== resolve(cwd)) return false; + i += 2; + } + if (!READ_ONLY_GIT_SUBCOMMANDS.includes(parts[i])) return false; + return !parts.slice(i + 1).some((arg) => UNSAFE_GIT_ARG.test(arg)); +} + +/** Strip `cd && ` so the rest is judged on its own. */ +function stripCdIntoWorkingDirectory(command: string, cwd?: string): string { + if (!cwd) return command; + const match = command.match(/^cd\s+(?:"([^"]+)"|'([^']+)'|(\S+))\s*&&\s*(\S.*)$/s); + if (!match) return command; + const dir = match[1] ?? match[2] ?? match[3]; + return resolve(cwd, dir) === resolve(cwd) ? match[4] : command; +} + /** * Check if command is an allowed package manager command. * Matches: [run|exec] [args...] @@ -226,7 +289,8 @@ function matchesAllowedPrefix(command: string): boolean { // Check if script starts with any safe script name or linting tool return ( SAFE_SCRIPTS.some((safe) => scriptPart.startsWith(safe)) || - LINTING_TOOLS.some((tool) => scriptPart.startsWith(tool)) + LINTING_TOOLS.some((tool) => scriptPart.startsWith(tool)) || + (TOOL_SAFE_ARGS[parts[0]] ?? []).some((arg) => parts.slice(1).join(' ').startsWith(arg)) ); } @@ -237,13 +301,20 @@ function matchesAllowedPrefix(command: string): boolean { * - Piping to tail/head for output limiting is allowed * - Stderr redirection (2>&1) is allowed */ -export function installerCanUseTool(toolName: string, input: Record): PermissionResult { +export function installerCanUseTool( + toolName: string, + input: Record, + workingDirectory?: string, +): PermissionResult { // Allow all non-Bash tools if (toolName !== 'Bash') { return { behavior: 'allow', updatedInput: input }; } - const command = (typeof input.command === 'string' ? input.command : '').trim(); + const command = stripCdIntoWorkingDirectory( + (typeof input.command === 'string' ? input.command : '').trim(), + workingDirectory, + ); // Block definitely dangerous operators: ; ` $ ( ) if (DANGEROUS_OPERATORS.test(command)) { @@ -283,7 +354,7 @@ export function installerCanUseTool(toolName: string, input: Record { logInfo('canUseTool called:', { toolName, input }); - const result = installerCanUseTool(toolName, input); + const result = installerCanUseTool(toolName, input, agentConfig.workingDirectory); logInfo('canUseTool result:', result); return Promise.resolve(result); }, tools: { type: 'preset', preset: 'claude_code' }, + // Without this the SDK sends an empty system prompt, so the agent never gets + // Claude Code's tool guidance (Read/Grep over cat/grep) or our installer rules. + systemPrompt: { type: 'preset', preset: 'claude_code', append: INSTALLER_SYSTEM_PROMPT_APPEND }, // Load no ~/.claude or project settings. Omitted, the SDK (since 0.2.89) loads them // all: the user's plugins, MCP servers, hooks, CLAUDE.md, and permission rules, // which apply before installerCanUseTool and would bypass the Bash gate. @@ -710,6 +784,7 @@ export async function runAgent( } try { onMessage?.(message); + options.onAgentMessage?.(message); } catch { /* non-critical */ } diff --git a/src/lib/agent-runner.ts b/src/lib/agent-runner.ts index f47efae2..07d4cb06 100644 --- a/src/lib/agent-runner.ts +++ b/src/lib/agent-runner.ts @@ -351,21 +351,27 @@ ${baseContent ? `## General Guidelines\n\n${baseContent}\n\n` : ''}## Integratio ${refContent} ${ - setupContent - ? `## Required Application Setup and Verification + baseContent && !setupContent + ? `The references above link to workos-authkit-setup.md. It covers dashboard configuration and browser checks that this installer handles itself, so you don't need to read it. + +` + : '' +}${ + setupContent + ? `## Required Application Setup and Verification ${setupContent} ## Installer execution boundary The setup reference above is already included in this prompt. Do not read a relative workos-authkit-setup.md from the app directory. -The agent's shell permissions do not allow WorkOS management commands. Do not run workos, install another CLI, use curl or SDK scripts to bypass that boundary, or attempt dashboard authentication. Implement and validate the app code only. The installer handles supported dashboard configuration outside the agent after code validation; unavailable configuration must remain explicitly unverified. +Don't install another CLI or use SDK scripts to change WorkOS settings or authenticate to the dashboard. Implement and validate the app code only. Create a dedicated /sign-in GET route in the App Router using getSignInUrl() from @workos-inc/authkit-nextjs and redirect(await getSignInUrl()) from next/navigation. Keep the OAuth callback using handleAuth() separate. The Initiate login URI is the app origin plus /sign-in, NEVER the callback URI. Read existing files before editing; do not replace an unrelated existing sign-in flow. If a page already serves /sign-in (including inside route groups), stop: this installer cannot place a route handler alongside that page. Never delete the page or create a conflicting route. Keep /sign-in public and follow the SDK README for PKCE cookie handling. Do not claim the full integration or browser flows are verified. Report code implementation separately from application configuration and browser testing. ` - : '' -} + : '' + } ${signInSection}Report your progress using [STATUS] prefixes. diff --git a/src/utils/types.ts b/src/utils/types.ts index 32b67733..f6addcad 100644 --- a/src/utils/types.ts +++ b/src/utils/types.ts @@ -119,6 +119,12 @@ export type InstallerOptions = { */ maxRetries?: number; + /** + * Observe every message from the agent session. Used by evals to record tool + * calls and latency while running the real integration flow. + */ + onAgentMessage?: (message: import('@anthropic-ai/claude-agent-sdk').SDKMessage) => void; + /** * Scaffold a new Next.js app when run in an empty directory. * Auto-enabled in headless mode; opt-in via --scaffold in interactive mode. From 69a64d8eeac233e8d5d14f3d45547ec4e0605390 Mon Sep 17 00:00:00 2001 From: Nick Nisi Date: Mon, 19 Oct 2026 18:12:19 -0500 Subject: [PATCH 03/11] test(evals): run the real installer integration and update Next.js fixtures The eval sent its own prompt ("invoke the workos-authkit- skill", a skill that no longer exists) and called runAgent directly, so it measured a different flow from the installer: agents searched for the skill, read reference files, and skipped production's validator. It now calls the integration's run() with installer options, observing messages through options.onAgentMessage and counting agent:retry corrections. The opt-in Fizzy fixture keeps its isolated-environment path. That surfaced two gaps the old path hid: four Next.js fixtures were on 14.2, below the installer's 15.3 minimum (now ^15.3 with React 19; partial-install moves authkit-nextjs from 0.15 to ^4), and the Kotlin/PHP/Laravel and vanilla-js aborts fixed in the previous commits. Eval runs set WORKOS_TELEMETRY=false. --- tests/evals/__tests__/agent-executor.spec.ts | 240 +++++++----------- tests/evals/agent-executor.ts | 64 +++++ tests/evals/index.ts | 2 + .../conflicting-middleware/package.json | 10 +- .../nextjs/example-auth0/package.json | 10 +- tests/fixtures/nextjs/example/package.json | 10 +- .../nextjs/partial-install/package.json | 12 +- 7 files changed, 181 insertions(+), 167 deletions(-) diff --git a/tests/evals/__tests__/agent-executor.spec.ts b/tests/evals/__tests__/agent-executor.spec.ts index fc3b8b66..f940b5b6 100644 --- a/tests/evals/__tests__/agent-executor.spec.ts +++ b/tests/evals/__tests__/agent-executor.spec.ts @@ -4,8 +4,9 @@ import { join } from 'node:path'; import { tmpdir } from 'node:os'; // Hoist mocks so they're available in vi.mock factories -const { mockRunAgent, mockConfig, mockCredentials } = vi.hoisted(() => ({ +const { mockRunAgent, mockIntegrationRun, mockConfig, mockCredentials } = vi.hoisted(() => ({ mockRunAgent: vi.fn(), + mockIntegrationRun: vi.fn(), mockConfig: { model: 'test-model', workos: { clientId: 'client_test', authkitDomain: 'test.workos.com', llmGatewayUrl: 'http://localhost:8000' }, @@ -34,6 +35,13 @@ vi.mock('../../../src/lib/agent-interface.js', () => ({ runAgent: mockRunAgent, })); +// The eval runs the real integration flow; the integration itself is mocked here. +vi.mock('../../../src/lib/registry.js', () => ({ + getRegistry: vi.fn(async () => ({ + get: (name: string) => (name === 'unknown' ? undefined : { run: mockIntegrationRun }), + })), +})); + vi.mock('../../../src/lib/agent-sdk-assets.js', () => ({ ensureClaudeCodeExecutable: vi.fn(async () => '/synthetic/claude'), })); @@ -92,193 +100,133 @@ describe('AgentExecutor', () => { // Create package.json so env writing works writeFileSync(join(testDir, 'package.json'), JSON.stringify({ name: 'test' })); mockRunAgent.mockReset(); + mockIntegrationRun.mockReset(); + mockConfig.model = 'test-model'; + delete process.env.EVAL_MODEL; }); afterEach(() => { rmSync(testDir, { recursive: true, force: true }); }); - it('calls production runAgent with correct AgentRunConfig', async () => { - mockRunAgent.mockResolvedValue({ retryCount: 0 }); + it('runs the real integration with installer options like production', async () => { + mockIntegrationRun.mockResolvedValue('done'); const executor = new AgentExecutor(testDir, 'nextjs'); - await executor.run(); - - expect(mockRunAgent).toHaveBeenCalledTimes(1); - - const [agentRunConfig] = mockRunAgent.mock.calls[0]; - expect(agentRunConfig.workingDirectory).toBe(testDir); - expect(agentRunConfig.model).toBe('test-model'); - // Must match production so every tool call goes through installerCanUseTool - expect(agentRunConfig.allowedTools).toEqual([]); - expect(agentRunConfig.mcpServers).toBeUndefined(); - // Direct mode — no gateway URL - expect(agentRunConfig.sdkEnv.ANTHROPIC_API_KEY).toBe('sk-ant-test'); - expect(agentRunConfig.sdkEnv.ANTHROPIC_BASE_URL).toBeUndefined(); - }); + const result = await executor.run(); - it('uses isolated Ruby environment and explicit callback without inheriting host secrets', async () => { - mockRunAgent.mockResolvedValue({}); - const executor = new AgentExecutor(testDir, 'ruby', { - environment: { HOME: '/synthetic/home', PATH: '/synthetic/bin' }, - redirectUri: 'http://app.fizzy.localhost:3006/auth/callback', + expect(result.success).toBe(true); + expect(mockIntegrationRun).toHaveBeenCalledTimes(1); + const [options] = mockIntegrationRun.mock.calls[0]; + expect(options).toMatchObject({ + installDir: testDir, + ci: true, + skipAuth: true, + direct: true, + apiKey: 'sk_test_key', + clientId: 'client_test_id', + noValidate: false, + maxRetries: 2, }); - await executor.run({ enabled: false, maxRetries: 0 }); - const [config, prompt] = mockRunAgent.mock.calls[0]; - expect(config.sdkEnv.HOME).toBe('/synthetic/home'); - expect(config.sdkEnv.PATH).toBe('/synthetic/bin'); - expect(config.sdkEnv).not.toHaveProperty('WORKOS_API_KEY'); - expect(prompt).toContain('configured in .env:'); - expect(prompt).not.toContain('configured in .env.local'); - expect(prompt).toContain('WORKOS_REDIRECT_URI=http://app.fizzy.localhost:3006/auth/callback'); - expect(readFileSync(join(testDir, '.env'), 'utf8')).toContain( - 'WORKOS_REDIRECT_URI=http://app.fizzy.localhost:3006/auth/callback', - ); + expect(typeof options.onAgentMessage).toBe('function'); + expect(options.emitter).toBeDefined(); + expect(mockRunAgent).not.toHaveBeenCalled(); }); - it('passes RetryConfig when correction is enabled', async () => { - mockRunAgent.mockResolvedValue({ retryCount: 0 }); + it('disables validation retries when correction is disabled', async () => { + mockIntegrationRun.mockResolvedValue('done'); - const executor = new AgentExecutor(testDir, 'nextjs'); - await executor.run({ enabled: true, maxRetries: 3 }); + await new AgentExecutor(testDir, 'nextjs').run({ enabled: false, maxRetries: 0 }); - const retryConfig = mockRunAgent.mock.calls[0][5]; // 6th arg - expect(retryConfig).toBeDefined(); - expect(retryConfig.maxRetries).toBe(3); - expect(typeof retryConfig.validateAndFormat).toBe('function'); + expect(mockIntegrationRun.mock.calls[0][0]).toMatchObject({ noValidate: true, maxRetries: 0 }); }); - it('passes no RetryConfig when correction is disabled', async () => { - mockRunAgent.mockResolvedValue({ retryCount: 0 }); - - const executor = new AgentExecutor(testDir, 'nextjs'); - await executor.run({ enabled: false, maxRetries: 0 }); - - const retryConfig = mockRunAgent.mock.calls[0][5]; - expect(retryConfig).toBeUndefined(); - }); - - it('passes InstallerOptions with skipAuth=true', async () => { - mockRunAgent.mockResolvedValue({ retryCount: 0 }); - - const executor = new AgentExecutor(testDir, 'nextjs'); - await executor.run(); - - const installerOptions = mockRunAgent.mock.calls[0][2]; // 3rd arg - expect(installerOptions.skipAuth).toBe(true); - expect(installerOptions.installDir).toBe(testDir); - }); - - it('passes onMessage callback as 7th argument', async () => { - mockRunAgent.mockResolvedValue({ retryCount: 0 }); - - const executor = new AgentExecutor(testDir, 'nextjs'); - await executor.run(); - - const onMessage = mockRunAgent.mock.calls[0][6]; // 7th arg - expect(typeof onMessage).toBe('function'); - }); - - it('maps retryCount=0 to correctionAttempts=0, selfCorrected=false', async () => { - mockRunAgent.mockResolvedValue({ retryCount: 0 }); - - const executor = new AgentExecutor(testDir, 'nextjs'); - const result = await executor.run(); - - expect(result.success).toBe(true); - expect(result.correctionAttempts).toBe(0); - expect(result.selfCorrected).toBe(false); - }); + it('counts agent:retry events as correction attempts', async () => { + mockIntegrationRun.mockImplementation(async (options) => { + options.emitter.emit('agent:retry', { attempt: 1, maxRetries: 2 }); + options.emitter.emit('agent:retry', { attempt: 2, maxRetries: 2 }); + return 'done'; + }); - it('maps retryCount>0 to selfCorrected=true on success', async () => { - mockRunAgent.mockResolvedValue({ retryCount: 2 }); - - const executor = new AgentExecutor(testDir, 'nextjs'); - const result = await executor.run(); + const result = await new AgentExecutor(testDir, 'nextjs').run(); expect(result.success).toBe(true); expect(result.correctionAttempts).toBe(2); expect(result.selfCorrected).toBe(true); }); - it('maps runAgent error result to failed AgentResult', async () => { - mockRunAgent.mockResolvedValue({ - error: 'EXECUTION_ERROR', - errorMessage: 'SDK crashed', - retryCount: 1, - }); + it('maps a thrown integration error to a failed result', async () => { + mockIntegrationRun.mockRejectedValue(new Error('Installation validation failed')); - const executor = new AgentExecutor(testDir, 'nextjs'); - const result = await executor.run(); + const result = await new AgentExecutor(testDir, 'nextjs').run(); expect(result.success).toBe(false); - expect(result.error).toBe('SDK crashed'); - expect(result.correctionAttempts).toBe(1); + expect(result.error).toBe('Installation validation failed'); expect(result.selfCorrected).toBe(false); }); - it('handles runAgent throwing an exception', async () => { - mockRunAgent.mockRejectedValue(new Error('Connection refused')); + it('records tool calls and text from agent messages', async () => { + mockIntegrationRun.mockImplementation(async (options) => { + options.onAgentMessage({ + type: 'assistant', + message: { + content: [ + { type: 'text', text: 'Installing AuthKit...' }, + { type: 'tool_use', name: 'Read', input: { file_path: 'package.json' } }, + ], + }, + }); + return 'done'; + }); - const executor = new AgentExecutor(testDir, 'nextjs'); - const result = await executor.run(); + const result = await new AgentExecutor(testDir, 'nextjs').run(); - expect(result.success).toBe(false); - expect(result.error).toBe('Connection refused'); - expect(result.correctionAttempts).toBe(0); + expect(result.output).toContain('Installing AuthKit...'); + expect(result.toolCalls).toEqual([expect.objectContaining({ tool: 'Read' })]); }); - it('writes env vars before calling runAgent', async () => { - mockRunAgent.mockResolvedValue({ retryCount: 0 }); + it('applies EVAL_MODEL to the installer config', async () => { + process.env.EVAL_MODEL = 'claude-opus-5-5'; + mockIntegrationRun.mockResolvedValue('done'); - const executor = new AgentExecutor(testDir, 'nextjs'); - await executor.run(); + await new AgentExecutor(testDir, 'nextjs').run(); - expect(writeEnvLocal).toHaveBeenCalledWith(testDir, { - WORKOS_API_KEY: 'sk_test_key', - WORKOS_CLIENT_ID: 'client_test_id', - }); + expect(mockConfig.model).toBe('claude-opus-5-5'); }); - it('onMessage callback collects text output from assistant messages', async () => { - mockRunAgent.mockResolvedValue({ retryCount: 0 }); - - const executor = new AgentExecutor(testDir, 'nextjs'); - await executor.run(); - - // Get the onMessage callback and simulate a message - const onMessage = mockRunAgent.mock.calls[0][6]; - onMessage({ - type: 'assistant', - message: { - content: [{ type: 'text', text: 'Installing AuthKit...' }], - }, + it('writes env vars before running the integration', async () => { + mockIntegrationRun.mockImplementation(async () => { + expect(writeEnvLocal).toHaveBeenCalledWith(testDir, { + WORKOS_API_KEY: 'sk_test_key', + WORKOS_CLIENT_ID: 'client_test_id', + }); + return 'done'; }); - // Run again to verify output is collected (can't check internal state, - // but we can verify it doesn't throw) - expect(onMessage).toBeDefined(); + const result = await new AgentExecutor(testDir, 'nextjs').run(); + expect(result.success).toBe(true); }); - it('builds prompt with correct skill name for framework', async () => { - mockRunAgent.mockResolvedValue({ retryCount: 0 }); - - const executor = new AgentExecutor(testDir, 'react-router'); - await executor.run(); - - const prompt = mockRunAgent.mock.calls[0][1]; // 2nd arg - expect(prompt).toContain('workos-authkit-react-router'); - expect(prompt).toContain('react-router'); + it('rejects a framework with no installer integration', async () => { + await expect(new AgentExecutor(testDir, 'unknown').run()).rejects.toThrow('No installer integration'); }); - it('defaults to correction enabled with maxRetries=2', async () => { - mockRunAgent.mockResolvedValue({ retryCount: 0 }); - - const executor = new AgentExecutor(testDir, 'nextjs'); - await executor.run(); // no retryConfig arg — uses default - - const retryConfig = mockRunAgent.mock.calls[0][5]; - expect(retryConfig).toBeDefined(); - expect(retryConfig.maxRetries).toBe(2); + it('uses isolated Ruby environment and explicit callback without inheriting host secrets', async () => { + mockRunAgent.mockResolvedValue({}); + const executor = new AgentExecutor(testDir, 'ruby', { + environment: { HOME: '/synthetic/home', PATH: '/synthetic/bin' }, + redirectUri: 'http://app.fizzy.localhost:3006/auth/callback', + }); + await executor.run({ enabled: false, maxRetries: 0 }); + const [config, prompt] = mockRunAgent.mock.calls[0]; + expect(config.sdkEnv.HOME).toBe('/synthetic/home'); + expect(config.sdkEnv.PATH).toBe('/synthetic/bin'); + expect(config.sdkEnv).not.toHaveProperty('WORKOS_API_KEY'); + expect(prompt).toContain('configured in .env:'); + expect(prompt).not.toContain('configured in .env.local'); + expect(prompt).toContain('WORKOS_REDIRECT_URI=http://app.fizzy.localhost:3006/auth/callback'); + expect(readFileSync(join(testDir, '.env'), 'utf8')).toContain( + 'WORKOS_REDIRECT_URI=http://app.fizzy.localhost:3006/auth/callback', + ); }); }); diff --git a/tests/evals/agent-executor.ts b/tests/evals/agent-executor.ts index d43a2806..b2ccf236 100644 --- a/tests/evals/agent-executor.ts +++ b/tests/evals/agent-executor.ts @@ -9,6 +9,8 @@ import { quickCheckValidateAndFormat } from '../../src/lib/validation/quick-chec import { runAgent, type AgentRunConfig, type RetryConfig } from '../../src/lib/agent-interface.js'; import { ensureClaudeCodeExecutable } from '../../src/lib/agent-sdk-assets.js'; import type { InstallerOptions } from '../../src/utils/types.js'; +import { getRegistry } from '../../src/lib/registry.js'; +import { createInstallerEventEmitter } from '../../src/lib/events.js'; import type { ToolCall, LatencyMetrics } from './types.js'; import type { SDKMessage } from '@anthropic-ai/claude-agent-sdk'; @@ -123,6 +125,11 @@ export class AgentExecutor { writeEnvFile(this.workDir, envVars); } + if (!this.options.environment) { + return this.runProductionIntegration(config, toolCalls, collectedOutput, label); + } + + // Isolated real-app fixtures (Fizzy) keep their own allowlisted environment and prompt. const skillName = SKILL_NAMES[this.framework]; const prompt = this.buildPrompt(skillName); @@ -205,6 +212,63 @@ export class AgentExecutor { } } + /** + * Run the framework's real installer integration, so the eval measures the same + * prompt, tool gate, and validation/retry loop that users get. + */ + private async runProductionIntegration( + config: AgentRetryConfig, + toolCalls: ToolCall[], + collectedOutput: string[], + label: string, + ): Promise { + const integration = (await getRegistry()).get(this.framework); + if (!integration) throw new Error(`No installer integration for framework: ${this.framework}`); + + if (process.env.EVAL_MODEL) getConfig().model = process.env.EVAL_MODEL; + + let correctionAttempts = 0; + const emitter = createInstallerEventEmitter(); + emitter.on('agent:retry', () => correctionAttempts++); + + const options: InstallerOptions = { + debug: this.options.verbose ?? false, + forceInstall: false, + installDir: this.workDir, + local: false, + ci: true, + skipAuth: true, + direct: true, + // Passing credentials marks configuration as caller-handled: no dashboard calls. + apiKey: this.credentials.workosApiKey, + clientId: this.credentials.workosClientId, + ...(this.options.redirectUri ? { redirectUri: this.options.redirectUri } : {}), + noValidate: !config.enabled, + maxRetries: config.maxRetries, + emitter, + onAgentMessage: (message) => this.trackMessage(message, toolCalls, collectedOutput, label), + }; + + const base = () => ({ + output: collectedOutput.join('\n'), + toolCalls, + latencyMetrics: this.latencyTracker.finish(), + correctionAttempts, + }); + + try { + await integration.run(options); + return { ...base(), success: true, selfCorrected: correctionAttempts > 0 }; + } catch (error) { + return { + ...base(), + success: false, + error: error instanceof Error ? error.message : String(error), + selfCorrected: false, + }; + } + } + private buildPrompt(skillName: string): string { return `You are integrating WorkOS AuthKit into this application. diff --git a/tests/evals/index.ts b/tests/evals/index.ts index 56f6d84a..22cd3ebd 100644 --- a/tests/evals/index.ts +++ b/tests/evals/index.ts @@ -8,6 +8,8 @@ import { diffRuns, printDiff } from './commands/diff.js'; import { listHistory, pruneHistory } from './commands/history.js'; async function main() { + // Eval runs call the real installer integrations; keep them out of installer telemetry. + process.env.WORKOS_TELEMETRY ??= 'false'; const options = parseArgs(process.argv.slice(2)); if (options.help) { diff --git a/tests/fixtures/nextjs/conflicting-middleware/package.json b/tests/fixtures/nextjs/conflicting-middleware/package.json index 40cd3b2e..4639d37e 100644 --- a/tests/fixtures/nextjs/conflicting-middleware/package.json +++ b/tests/fixtures/nextjs/conflicting-middleware/package.json @@ -8,14 +8,14 @@ "start": "next start" }, "dependencies": { - "next": "^14.2.0", - "react": "^18.3.0", - "react-dom": "^18.3.0" + "next": "^15.3.0", + "react": "^19.0.0", + "react-dom": "^19.0.0" }, "devDependencies": { "@types/node": "^20.0.0", - "@types/react": "^18.3.0", - "@types/react-dom": "^18.3.0", + "@types/react": "^19.0.0", + "@types/react-dom": "^19.0.0", "typescript": "^5.4.0" } } diff --git a/tests/fixtures/nextjs/example-auth0/package.json b/tests/fixtures/nextjs/example-auth0/package.json index 5d2912fc..5feff399 100644 --- a/tests/fixtures/nextjs/example-auth0/package.json +++ b/tests/fixtures/nextjs/example-auth0/package.json @@ -9,14 +9,14 @@ }, "dependencies": { "@auth0/nextjs-auth0": "^3.5.0", - "next": "^14.2.0", - "react": "^18.3.0", - "react-dom": "^18.3.0" + "next": "^15.3.0", + "react": "^19.0.0", + "react-dom": "^19.0.0" }, "devDependencies": { "@types/node": "^20.0.0", - "@types/react": "^18.3.0", - "@types/react-dom": "^18.3.0", + "@types/react": "^19.0.0", + "@types/react-dom": "^19.0.0", "typescript": "^5.4.0" } } diff --git a/tests/fixtures/nextjs/example/package.json b/tests/fixtures/nextjs/example/package.json index e5de786d..6f7a7a9b 100644 --- a/tests/fixtures/nextjs/example/package.json +++ b/tests/fixtures/nextjs/example/package.json @@ -8,14 +8,14 @@ "start": "next start" }, "dependencies": { - "next": "^14.2.0", - "react": "^18.3.0", - "react-dom": "^18.3.0" + "next": "^15.3.0", + "react": "^19.0.0", + "react-dom": "^19.0.0" }, "devDependencies": { "@types/node": "^20.0.0", - "@types/react": "^18.3.0", - "@types/react-dom": "^18.3.0", + "@types/react": "^19.0.0", + "@types/react-dom": "^19.0.0", "typescript": "^5.4.0" } } diff --git a/tests/fixtures/nextjs/partial-install/package.json b/tests/fixtures/nextjs/partial-install/package.json index e0332a1c..a913cc59 100644 --- a/tests/fixtures/nextjs/partial-install/package.json +++ b/tests/fixtures/nextjs/partial-install/package.json @@ -8,15 +8,15 @@ "start": "next start" }, "dependencies": { - "@workos-inc/authkit-nextjs": "^0.15.0", - "next": "^14.2.0", - "react": "^18.3.0", - "react-dom": "^18.3.0" + "@workos-inc/authkit-nextjs": "^4.0.0", + "next": "^15.3.0", + "react": "^19.0.0", + "react-dom": "^19.0.0" }, "devDependencies": { "@types/node": "^20.0.0", - "@types/react": "^18.3.0", - "@types/react-dom": "^18.3.0", + "@types/react": "^19.0.0", + "@types/react-dom": "^19.0.0", "typescript": "^5.4.0" } } From 4a1fd2d07f55d1941697c2eddfa5678d8c277c56 Mon Sep 17 00:00:00 2001 From: Nick Nisi Date: Mon, 19 Oct 2026 18:13:04 -0500 Subject: [PATCH 04/11] chore(deps): bump @workos/skills to 0.8.1 Brings workos/skills#55: the Next.js reference recommends authkitProxy and warns that a bare @workos-inc/node install pulls v11, outside the authkit-nextjs ^9 || ^10 peer range. --- bun.lock | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bun.lock b/bun.lock index 9ab1603d..5e9e274e 100644 --- a/bun.lock +++ b/bun.lock @@ -14,7 +14,7 @@ "@workos/emulate": "^0.14.0", "@workos/migrations": "^2.6.0", "@workos/openapi-spec": "^0.123.0", - "@workos/skills": "0.8.0", + "@workos/skills": "0.8.1", "chalk": "^6.0.1", "diff": "^9.0.0", "fast-glob": "^3.3.3", @@ -411,7 +411,7 @@ "@workos/openapi-spec": ["@workos/openapi-spec@0.123.0", "", {}, "sha512-ZsHblNwgpc53tLfNoZU3mRA+L/vO2xadWus6Y68XVfcp9cz9ufNax8qAeWxgspmc0zazsnwy2xb0+ZB0dyrEOw=="], - "@workos/skills": ["@workos/skills@0.8.0", "", { "dependencies": { "yaml": "^2.8.2" } }, "sha512-SNn1824L/q3wbVWLJQnxrsNwmAnYEREEwbgAtWwibjWTr0JH2uCbbpKIXdEbo2zbdBmxGvBnaCfR7jJHjTRn2A=="], + "@workos/skills": ["@workos/skills@0.8.1", "", { "dependencies": { "yaml": "^2.8.2" } }, "sha512-+b7WnFoHrtpMpzS60BApe7WQTzXl0oVf8WpWdpGQ88fMIa4ye9MGT/hxvC/Js2jm4es9HkCwHCFkVXUZO/YJPA=="], "accepts": ["accepts@2.0.0", "", { "dependencies": { "mime-types": "^3.0.0", "negotiator": "^1.0.0" } }, "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng=="], diff --git a/package.json b/package.json index 0d876714..569eacf0 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "@workos/emulate": "^0.14.0", "@workos/migrations": "^2.6.0", "@workos/openapi-spec": "^0.123.0", - "@workos/skills": "0.8.0", + "@workos/skills": "0.8.1", "chalk": "^6.0.1", "diff": "^9.0.0", "fast-glob": "^3.3.3", From de4b0c3a2b3db83d8c1b084b7a0e2bc07f3a3801 Mon Sep 17 00:00:00 2001 From: Nick Nisi Date: Wed, 21 Oct 2026 13:43:23 -0500 Subject: [PATCH 05/11] fix(installer): close cd-prefix bypass and only promise checks the runner performs - The Bash gate stripped `cd &&` by resolving the path, then checked only the remainder. `cd "/proj/$(id)/.." && git status` resolved to /proj and passed, while Bash still ran `$(id)`. Shell operators are now checked on the raw command first, and a cd target must be a plain path (no $, backticks, globs, quotes, or spaces) to be stripped. - The system prompt promised that the installer builds the project and returns errors. Only runners with validation do (runAgentInstaller); Python, Ruby, Go, Elixir, and .NET don't. The prompt now says which, based on whether this run has a retry config. - The setup-reference note no longer claims the installer runs browser checks; agents report sign-in flows as untested. --- src/lib/agent-interface.spec.ts | 37 +++++++++++++++++++------ src/lib/agent-interface.ts | 48 ++++++++++++++++++++++----------- src/lib/agent-runner.ts | 2 +- 3 files changed, 63 insertions(+), 24 deletions(-) diff --git a/src/lib/agent-interface.spec.ts b/src/lib/agent-interface.spec.ts index 05bb5d14..77122141 100644 --- a/src/lib/agent-interface.spec.ts +++ b/src/lib/agent-interface.spec.ts @@ -181,17 +181,30 @@ describe('runAgent retry loop', () => { expect(options.strictMcpConfig).toBe(true); }); - it('states installer rules in the system prompt', async () => { + const systemAppendFor = async (retryConfig?: { maxRetries: number; validateAndFormat: () => Promise }) => { + mockQuery.mockReset(); mockQuery.mockImplementation(createMockSDKResponse([{ text: 'Done!' }])); + await runAgent(makeAgentConfig(), 'Test prompt', makeOptions(), undefined, emitter, retryConfig); + const { options } = mockQuery.mock.calls[0][0] as { + options: { systemPrompt: { type: string; preset: string; append: string } }; + }; + expect(options.systemPrompt).toMatchObject({ type: 'preset', preset: 'claude_code' }); + return options.systemPrompt.append; + }; - await runAgent(makeAgentConfig(), 'Test prompt', makeOptions(), undefined, emitter); + it('states installer rules in the system prompt', async () => { + const append = await systemAppendFor(); + expect(append).toContain('WorkOS installer environment'); + expect(append).toContain('Inspect files with Read, Grep, and Glob'); + }); - const { options } = mockQuery.mock.calls[0][0] as { options: Record }; - expect(options.systemPrompt).toEqual({ - type: 'preset', - preset: 'claude_code', - append: expect.stringContaining('WorkOS installer environment'), - }); + it('only promises post-run build feedback when the runner validates', async () => { + const withoutValidation = await systemAppendFor(); + expect(withoutValidation).toContain('the installer does not check it afterwards'); + expect(withoutValidation).not.toContain('sends you any errors to fix'); + + const withValidation = await systemAppendFor({ maxRetries: 2, validateAndFormat: async () => null }); + expect(withValidation).toContain('sends you any errors to fix'); }); it('passes agent messages to options.onAgentMessage', async () => { @@ -612,6 +625,14 @@ describe('installerCanUseTool', () => { expect(verdict('cd /proj && bun run build && curl http://x/')).toBe('deny'); }); + it('denies shell code hidden in a cd path that resolves to the working directory', () => { + expect(verdict('cd "/proj/$(id)/.." && git status')).toBe('deny'); + expect(verdict('cd /proj/`id`/.. && git status')).toBe('deny'); + expect(verdict("cd '/proj/${HOME}/..' && bun run build")).toBe('deny'); + expect(verdict('cd /proj/*/.. && git status')).toBe('deny'); + expect(verdict('cd "/proj/x y/.." && git status')).toBe('deny'); + }); + it('allows read-only git and denies git that writes or runs programs', () => { expect(verdict('git status --short')).toBe('allow'); expect(verdict('git -C /proj status')).toBe('allow'); diff --git a/src/lib/agent-interface.ts b/src/lib/agent-interface.ts index f6c65512..8600a90e 100644 --- a/src/lib/agent-interface.ts +++ b/src/lib/agent-interface.ts @@ -105,9 +105,14 @@ export type AgentRunConfig = { /** * Appended to Claude Code's system prompt for every installer run. It states up front * what installerCanUseTool enforces, so the agent doesn't discover the rules one - * denied command at a time. Keep it in sync with the allowlist below. + * denied command at a time, and only promises post-run checks the runner performs. + * Keep it in sync with the allowlist below. */ -export const INSTALLER_SYSTEM_PROMPT_APPEND = `# WorkOS installer environment +export function buildInstallerSystemPromptAppend({ validatesAfterRun }: { validatesAfterRun: boolean }): string { + const afterRun = validatesAfterRun + ? 'After you finish, the installer typechecks and builds the project and sends you any errors to fix.' + : "Run the project's build or syntax check yourself before you finish; the installer does not check it afterwards."; + return `# WorkOS installer environment You are running inside the WorkOS CLI installer, not an interactive session, and your working directory is the user's project. @@ -115,9 +120,10 @@ You are running inside the WorkOS CLI installer, not an interactive session, and - Run one command per Bash call from the working directory. Don't use \`cd\`, \`&&\`, \`;\`, \`$(...)\`, or pipes other than \`| head\` or \`| tail\`. - Inspect files with Read, Grep, and Glob, not Bash. - Bash has no network access: no curl, \`npm view\`, or \`git clone\`. Use WebFetch to read documentation. The \`workos\` CLI is not available to you. -- Don't start dev servers or open a browser to test sign-in. After you finish, the installer typechecks and builds the project and sends you any errors to fix. -- Don't change WorkOS dashboard settings (redirect URIs, Sign-out URI, Initiate login URI, CORS). The installer configures supported settings after your code passes validation. List anything you could not verify as unverified. +- Don't start dev servers or open a browser to test sign-in; report sign-in flows as untested. ${afterRun} +- Don't change WorkOS dashboard settings (redirect URIs, Sign-out URI, Initiate login URI, CORS). The installer configures supported settings itself. List anything you could not verify as unverified. - Leave changes unstaged: don't stage, commit, or create branches.`; +} /** * Package managers that can be used to run commands. @@ -258,12 +264,20 @@ function isReadOnlyGit(command: string, cwd?: string): boolean { return !parts.slice(i + 1).some((arg) => UNSAFE_GIT_ARG.test(arg)); } -/** Strip `cd && ` so the rest is judged on its own. */ +/** Characters a `cd` target may contain; anything else could expand in the shell. */ +const PLAIN_PATH = /^[A-Za-z0-9_./@+,=:%-]+$/; + +/** + * Strip `cd && ` so the rest is judged on its own. Callers must + * check the unstripped command for DANGEROUS_OPERATORS first: the shell runs the + * original text, so a path like "/proj/$(id)/.." must never be resolved away. + */ function stripCdIntoWorkingDirectory(command: string, cwd?: string): string { if (!cwd) return command; const match = command.match(/^cd\s+(?:"([^"]+)"|'([^']+)'|(\S+))\s*&&\s*(\S.*)$/s); if (!match) return command; const dir = match[1] ?? match[2] ?? match[3]; + if (!PLAIN_PATH.test(dir)) return command; return resolve(cwd, dir) === resolve(cwd) ? match[4] : command; } @@ -311,19 +325,17 @@ export function installerCanUseTool( return { behavior: 'allow', updatedInput: input }; } - const command = stripCdIntoWorkingDirectory( - (typeof input.command === 'string' ? input.command : '').trim(), - workingDirectory, - ); + const rawCommand = (typeof input.command === 'string' ? input.command : '').trim(); - // Block definitely dangerous operators: ; ` $ ( ) - if (DANGEROUS_OPERATORS.test(command)) { - logWarn(`Denying bash command with dangerous operators: ${command}`); - debug(`Denying bash command with dangerous operators: ${command}`); + // Block definitely dangerous operators: ; ` $ ( ). Checked on the raw command, since + // that is what Bash runs, before any `cd &&` prefix is removed. + if (DANGEROUS_OPERATORS.test(rawCommand)) { + logWarn(`Denying bash command with dangerous operators: ${rawCommand}`); + debug(`Denying bash command with dangerous operators: ${rawCommand}`); analytics.capture(INSTALLER_INTERACTION_EVENT_NAME, { action: 'bash command denied', reason: 'dangerous operators', - command, + command: rawCommand, }); return { behavior: 'deny', @@ -331,6 +343,8 @@ export function installerCanUseTool( }; } + const command = stripCdIntoWorkingDirectory(rawCommand, workingDirectory); + // Normalize: remove safe stderr redirection (2>&1, 2>&2, etc.) const normalized = command.replace(/\s*\d*>&\d+\s*/g, ' ').trim(); @@ -750,7 +764,11 @@ export async function runAgent( tools: { type: 'preset', preset: 'claude_code' }, // Without this the SDK sends an empty system prompt, so the agent never gets // Claude Code's tool guidance (Read/Grep over cat/grep) or our installer rules. - systemPrompt: { type: 'preset', preset: 'claude_code', append: INSTALLER_SYSTEM_PROMPT_APPEND }, + systemPrompt: { + type: 'preset', + preset: 'claude_code', + append: buildInstallerSystemPromptAppend({ validatesAfterRun: Boolean(retryConfig && maxRetries > 0) }), + }, // Load no ~/.claude or project settings. Omitted, the SDK (since 0.2.89) loads them // all: the user's plugins, MCP servers, hooks, CLAUDE.md, and permission rules, // which apply before installerCanUseTool and would bypass the Bash gate. diff --git a/src/lib/agent-runner.ts b/src/lib/agent-runner.ts index 07d4cb06..424e024e 100644 --- a/src/lib/agent-runner.ts +++ b/src/lib/agent-runner.ts @@ -352,7 +352,7 @@ ${refContent} ${ baseContent && !setupContent - ? `The references above link to workos-authkit-setup.md. It covers dashboard configuration and browser checks that this installer handles itself, so you don't need to read it. + ? `The references above link to workos-authkit-setup.md. It covers dashboard configuration, which this installer handles itself, and browser checks, which you can't run here, so you don't need to read it. ` : '' From 873664a91dc36e9df91a6085d5e3af0a7a0a90ec Mon Sep 17 00:00:00 2001 From: Nick Nisi Date: Wed, 21 Oct 2026 13:43:24 -0500 Subject: [PATCH 06/11] refactor(debug): scope installer log files per async context initLogFile() set one module-level path, so concurrent installer runs in one process (evals) appended to whichever file was selected last. runWithLogFile(path, fn) gives a run its own log via AsyncLocalStorage; single runs keep the session log as before. --- src/utils/debug.spec.ts | 23 +++++++++++++++++++++++ src/utils/debug.ts | 27 +++++++++++++++++++++++---- 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/src/utils/debug.spec.ts b/src/utils/debug.spec.ts index bf57169c..3fda49c9 100644 --- a/src/utils/debug.spec.ts +++ b/src/utils/debug.spec.ts @@ -148,4 +148,27 @@ describe('debug logging', () => { const { getLogFilePath } = await import('./debug.js'); expect(getLogFilePath()).toBeNull(); }); + + it('keeps concurrent scoped runs in separate log files', async () => { + const { runWithLogFile, initLogFile, logInfo, getLogFilePath } = await import('./debug.js'); + const dir = mkdtempSync(join(tmpdir(), 'workos-scoped-log-')); + const a = join(dir, 'a', 'run.log'); + const b = join(dir, 'b', 'run.log'); + const run = (path: string, message: string) => + runWithLogFile(path, async () => { + initLogFile(); + await new Promise((resolve) => setTimeout(resolve, 5)); + logInfo(message); + return getLogFilePath(); + }); + + const [pathA, pathB] = await Promise.all([run(a, 'from-a'), run(b, 'from-b')]); + + expect([pathA, pathB]).toEqual([a, b]); + expect(readFileSync(a, 'utf8')).toContain('from-a'); + expect(readFileSync(a, 'utf8')).not.toContain('from-b'); + expect(readFileSync(b, 'utf8')).toContain('from-b'); + expect(readFileSync(b, 'utf8')).not.toContain('from-a'); + rmSync(dir, { recursive: true, force: true }); + }); }); diff --git a/src/utils/debug.ts b/src/utils/debug.ts index 9b313b00..a83c2744 100644 --- a/src/utils/debug.ts +++ b/src/utils/debug.ts @@ -1,5 +1,6 @@ import { appendFileSync, existsSync, mkdirSync, readdirSync, unlinkSync } from 'fs'; -import { join } from 'path'; +import { AsyncLocalStorage } from 'node:async_hooks'; +import { dirname, join } from 'path'; import { homedir } from 'os'; import chalk from 'chalk'; import { prepareMessage } from './logging.js'; @@ -9,6 +10,13 @@ import { isJsonMode } from './output.js'; let debugEnabled = false; let sessionLogPath: string | null = null; +/** A log file scoped to one async context, so concurrent runs (evals) don't share a path. */ +const scopedLogPath = new AsyncLocalStorage(); + +/** Run `fn` with all installer log lines written to `path` instead of the session log. */ +export function runWithLogFile(path: string, fn: () => T): T { + return scopedLogPath.run(path, fn); +} const LOG_DIR = join(homedir(), '.workos', 'logs'); const MAX_LOG_FILES = 10; @@ -45,6 +53,16 @@ function rotateLogFiles(): void { } export function initLogFile(): void { + const scoped = scopedLogPath.getStore(); + if (scoped) { + try { + mkdirSync(dirname(scoped), { recursive: true }); + appendFileSync(scoped, `WorkOS AuthKit Installer Run: ${new Date().toISOString()}\n`); + } catch { + /* logging must never break the run */ + } + return; + } try { rotateLogFiles(); const dir = ensureLogDir(); @@ -59,7 +77,7 @@ export function initLogFile(): void { } export function getLogFilePath(): string | null { - return sessionLogPath; + return scopedLogPath.getStore() ?? sessionLogPath; } function writeLog(level: 'INFO' | 'WARN' | 'ERROR', emoji: string, args: unknown[]): string { @@ -73,10 +91,11 @@ function writeLog(level: 'INFO' | 'WARN' | 'ERROR', emoji: string, args: unknown } // Write to log file - if (sessionLogPath) { + const logPath = scopedLogPath.getStore() ?? sessionLogPath; + if (logPath) { try { const timestamp = new Date().toISOString(); - appendFileSync(sessionLogPath, `[${timestamp}] ${emoji} ${level}: ${msg}\n`); + appendFileSync(logPath, `[${timestamp}] ${emoji} ${level}: ${msg}\n`); } catch { // Ignore write failures } From 7e1d20fdcfc323194f791f5cee9707828d6500f3 Mon Sep 17 00:00:00 2001 From: Nick Nisi Date: Wed, 21 Oct 2026 13:43:24 -0500 Subject: [PATCH 07/11] test(evals): fail scenarios when the installer fails and write the installer env - A blocking installer error (e.g. a Next.js result without /sign-in) made AgentExecutor return success: false, but ParallelRunner decided pass/fail from the grader alone. Installer failures now fail the scenario, with an "Installer completed" check and the error. - For JS frameworks the eval now calls configureInstallEnvironment, the same step the installer runs before the integration, so .env.local gets the framework's redirect URI key (NEXT_PUBLIC_WORKOS_REDIRECT_URI for Next.js) and client-side prefixed copies. - Each scenario writes its installer log to tests/eval-results/installer-logs/ instead of a shared file. --- tests/evals/__tests__/agent-executor.spec.ts | 18 +++++--- tests/evals/__tests__/parallel-runner.spec.ts | 43 +++++++++++++++++++ tests/evals/agent-executor.ts | 26 ++++++++++- tests/evals/parallel-runner.ts | 15 ++++++- 4 files changed, 94 insertions(+), 8 deletions(-) diff --git a/tests/evals/__tests__/agent-executor.spec.ts b/tests/evals/__tests__/agent-executor.spec.ts index f940b5b6..144f1aab 100644 --- a/tests/evals/__tests__/agent-executor.spec.ts +++ b/tests/evals/__tests__/agent-executor.spec.ts @@ -4,9 +4,10 @@ import { join } from 'node:path'; import { tmpdir } from 'node:os'; // Hoist mocks so they're available in vi.mock factories -const { mockRunAgent, mockIntegrationRun, mockConfig, mockCredentials } = vi.hoisted(() => ({ +const { mockRunAgent, mockIntegrationRun, mockConfigureEnv, mockConfig, mockCredentials } = vi.hoisted(() => ({ mockRunAgent: vi.fn(), mockIntegrationRun: vi.fn(), + mockConfigureEnv: vi.fn(async () => {}), mockConfig: { model: 'test-model', workos: { clientId: 'client_test', authkitDomain: 'test.workos.com', llmGatewayUrl: 'http://localhost:8000' }, @@ -35,6 +36,8 @@ vi.mock('../../../src/lib/agent-interface.js', () => ({ runAgent: mockRunAgent, })); +vi.mock('../../../src/lib/run-with-core.js', () => ({ configureInstallEnvironment: mockConfigureEnv })); + // The eval runs the real integration flow; the integration itself is mocked here. vi.mock('../../../src/lib/registry.js', () => ({ getRegistry: vi.fn(async () => ({ @@ -76,6 +79,7 @@ vi.mock('../../../src/utils/debug.js', () => ({ logError: vi.fn(), initLogFile: vi.fn(), getLogFilePath: vi.fn(() => null), + runWithLogFile: vi.fn((_path: string, fn: () => unknown) => fn()), })); vi.mock('../../../src/utils/analytics.js', () => ({ @@ -101,6 +105,7 @@ describe('AgentExecutor', () => { writeFileSync(join(testDir, 'package.json'), JSON.stringify({ name: 'test' })); mockRunAgent.mockReset(); mockIntegrationRun.mockReset(); + mockConfigureEnv.mockClear(); mockConfig.model = 'test-model'; delete process.env.EVAL_MODEL; }); @@ -194,17 +199,20 @@ describe('AgentExecutor', () => { expect(mockConfig.model).toBe('claude-opus-5-5'); }); - it('writes env vars before running the integration', async () => { + it('writes the installer env file before running the integration', async () => { mockIntegrationRun.mockImplementation(async () => { - expect(writeEnvLocal).toHaveBeenCalledWith(testDir, { - WORKOS_API_KEY: 'sk_test_key', - WORKOS_CLIENT_ID: 'client_test_id', + expect(mockConfigureEnv).toHaveBeenCalledWith({ + integration: 'nextjs', + credentials: { apiKey: 'sk_test_key', clientId: 'client_test_id' }, + options: expect.objectContaining({ installDir: testDir, ci: true }), }); return 'done'; }); const result = await new AgentExecutor(testDir, 'nextjs').run(); + expect(result.success).toBe(true); + expect(writeEnvLocal).not.toHaveBeenCalled(); }); it('rejects a framework with no installer integration', async () => { diff --git a/tests/evals/__tests__/parallel-runner.spec.ts b/tests/evals/__tests__/parallel-runner.spec.ts index 6e514fa3..52783296 100644 --- a/tests/evals/__tests__/parallel-runner.spec.ts +++ b/tests/evals/__tests__/parallel-runner.spec.ts @@ -1,8 +1,19 @@ import { afterEach, expect, it, vi } from 'vitest'; import { ParallelRunner } from '../parallel-runner.js'; import { evalEvents } from '../events.js'; +import { AgentExecutor } from '../agent-executor.js'; vi.mock('../agent-executor.js', () => ({ AgentExecutor: vi.fn() })); +vi.mock('../fixture-manager.js', () => ({ + FixtureManager: vi.fn(function FixtureManager() { + return { + setup: vi.fn(async () => '/tmp/fixture'), + cleanup: vi.fn(async () => {}), + getTempDir: () => '/tmp/fixture', + }; + }), +})); +vi.mock('../graders/collect-key-files.js', () => ({ collectKeyFiles: vi.fn(async () => undefined) })); afterEach(() => vi.restoreAllMocks()); @@ -19,3 +30,35 @@ it('completes a run without scheduling unused dashboard progress updates', async expect(interval).not.toHaveBeenCalled(); expect(completed).toHaveBeenCalledOnce(); }); + +it('fails a scenario when the installer fails even if the grader passes', async () => { + vi.spyOn(console, 'log').mockImplementation(() => {}); + vi.spyOn(process, 'on').mockReturnValue(process); + vi.mocked(AgentExecutor).mockImplementation(function () { + return { + run: vi.fn(async () => ({ + success: false, + error: 'Installation validation failed: missing /sign-in route', + output: '', + toolCalls: [], + correctionAttempts: 0, + selfCorrected: false, + })), + } as unknown as AgentExecutor; + }); + class PassingGrader { + async grade() { + return { passed: true, checks: [{ name: 'Build', passed: true }] }; + } + } + const runner = new ParallelRunner([{ framework: 'nextjs', state: 'example', grader: PassingGrader }], { + maxAttempts: 1, + concurrency: 1, + }); + + const [result] = await runner.run(); + + expect(result.passed).toBe(false); + expect(result.error).toContain('missing /sign-in route'); + expect(result.checks?.[0]).toMatchObject({ name: 'Installer completed', passed: false }); +}); diff --git a/tests/evals/agent-executor.ts b/tests/evals/agent-executor.ts index b2ccf236..d7891ccf 100644 --- a/tests/evals/agent-executor.ts +++ b/tests/evals/agent-executor.ts @@ -11,6 +11,8 @@ import { ensureClaudeCodeExecutable } from '../../src/lib/agent-sdk-assets.js'; import type { InstallerOptions } from '../../src/utils/types.js'; import { getRegistry } from '../../src/lib/registry.js'; import { createInstallerEventEmitter } from '../../src/lib/events.js'; +import { configureInstallEnvironment } from '../../src/lib/run-with-core.js'; +import { runWithLogFile } from '../../src/utils/debug.js'; import type { ToolCall, LatencyMetrics } from './types.js'; import type { SDKMessage } from '@anthropic-ai/claude-agent-sdk'; @@ -97,7 +99,23 @@ export class AgentExecutor { ...(this.options.redirectUri ? { WORKOS_REDIRECT_URI: this.options.redirectUri } : {}), }; - if (JS_FRAMEWORKS.includes(this.framework)) { + if (JS_FRAMEWORKS.includes(this.framework) && !this.options.environment) { + // Same env file the installer writes before running the integration, including + // the framework's redirect URI key and any client-side prefixed copies. + await configureInstallEnvironment({ + integration: this.framework, + credentials: { apiKey: this.credentials.workosApiKey, clientId: this.credentials.workosClientId }, + options: { + debug: false, + forceInstall: false, + installDir: this.workDir, + local: false, + ci: true, + skipAuth: true, + ...(this.options.redirectUri ? { redirectUri: this.options.redirectUri } : {}), + }, + }); + } else if (JS_FRAMEWORKS.includes(this.framework)) { writeEnvLocal(this.workDir, envVars); } else { // Git ignores cannot protect tracked files. Refuse rather than silently @@ -256,8 +274,12 @@ export class AgentExecutor { correctionAttempts, }); + // Scenarios run concurrently in one process; give each its own installer log. + const logName = `${(this.options.scenarioName ?? this.framework).replace(/[^a-z0-9]+/gi, '-')}-${Date.now()}.log`; + const logFile = join(process.cwd(), 'tests', 'eval-results', 'installer-logs', logName); + try { - await integration.run(options); + await runWithLogFile(logFile, () => integration.run(options)); return { ...base(), success: true, selfCorrected: correctionAttempts > 0 }; } catch (error) { return { diff --git a/tests/evals/parallel-runner.ts b/tests/evals/parallel-runner.ts index ac654bdf..ded9a3fb 100644 --- a/tests/evals/parallel-runner.ts +++ b/tests/evals/parallel-runner.ts @@ -122,7 +122,19 @@ export class ParallelRunner { lastToolCalls = agentResult.toolCalls; const grader = new scenario.grader(workDir); - const gradeResult = await grader.grade(); + const graded = await grader.grade(); + // The installer can reject a result the grader would accept (e.g. a missing + // Next.js /sign-in route fails its validation), so an installer failure fails + // the scenario too. + const gradeResult = agentResult.success + ? graded + : { + passed: false, + checks: [ + { name: 'Installer completed', passed: false, message: agentResult.error ?? 'Installer failed' }, + ...graded.checks, + ], + }; // Collect key files for quality grading (only on pass to avoid wasted effort) const keyFiles = @@ -133,6 +145,7 @@ export class ParallelRunner { passed: gradeResult.passed, duration: Date.now() - startTime, checks: gradeResult.checks, + ...(agentResult.success ? {} : { error: agentResult.error ?? 'Installer failed' }), agentOutput: agentResult.output, attempts: attempt, latencyMetrics: agentResult.latencyMetrics, From 03ce73797cfb892f2a9802b0ed89a5fe908b7b67 Mon Sep 17 00:00:00 2001 From: Nick Nisi Date: Wed, 21 Oct 2026 14:50:40 -0500 Subject: [PATCH 08/11] fix(installer): treat a timed-out build or typecheck as inconclusive The post-run quick check gave the build 60s, then killed it and sent the agent a bare "Build failed" with no output. On a busy machine a Next.js build passes 60s, so agents got a correction they could not act on and guessed at changes (in evals, switching middleware to the Node runtime). A check killed at its limit now logs a warning and does not trigger a correction. Limits are raised to 60s (typecheck) and 180s (build). --- src/lib/validation/quick-checks.spec.ts | 25 +++++++++++++++++ src/lib/validation/quick-checks.ts | 37 +++++++++++++++++++------ 2 files changed, 54 insertions(+), 8 deletions(-) diff --git a/src/lib/validation/quick-checks.spec.ts b/src/lib/validation/quick-checks.spec.ts index a36dd1ec..8184cfea 100644 --- a/src/lib/validation/quick-checks.spec.ts +++ b/src/lib/validation/quick-checks.spec.ts @@ -63,6 +63,31 @@ describe('runQuickChecks', () => { expect(result.agentRetryPrompt).toBeNull(); }); + it('treats a build killed at the time limit as inconclusive, not a failure', async () => { + const killed = () => { + const proc = new EventEmitter() as any; + proc.stdout = new EventEmitter(); + proc.stderr = new EventEmitter(); + setTimeout(() => proc.emit('close', null, 'SIGTERM'), 30); + return proc; + }; + mockSpawn.mockImplementationOnce(() => createMockProcess(0)).mockImplementationOnce(killed); + + const result = await runQuickChecks(testDir, { timeoutMs: 10 }); + + expect(result.passed).toBe(true); + expect(result.agentRetryPrompt).toBeNull(); + }); + + it('still fails a build that exits non-zero within the time limit', async () => { + mockSpawn.mockImplementationOnce(() => createMockProcess(0)).mockImplementationOnce(() => createMockProcess(1)); + + const result = await runQuickChecks(testDir, { timeoutMs: 10_000 }); + + expect(result.passed).toBe(false); + expect(result.results[1].phase).toBe('build'); + }); + it('short-circuits build when typecheck fails', async () => { const tsError = "src/middleware.ts(42,5): error TS2345: Argument of type 'string | undefined' is not assignable"; diff --git a/src/lib/validation/quick-checks.ts b/src/lib/validation/quick-checks.ts index 1e5da607..67968fb3 100644 --- a/src/lib/validation/quick-checks.ts +++ b/src/lib/validation/quick-checks.ts @@ -4,9 +4,12 @@ import { join } from 'path'; import type { QuickCheckResult, QuickChecksOutput, ValidationIssue } from './types.js'; import { detectBuildCommand, detectPackageManager, parseBuildErrors } from './build-validator.js'; import { SPAWN_OPTS } from '../../utils/platform.js'; +import { logWarn } from '../../utils/debug.js'; -const DEFAULT_TYPECHECK_TIMEOUT_MS = 30_000; -const DEFAULT_BUILD_TIMEOUT_MS = 60_000; +// Generous: a check killed at the limit is inconclusive, and a real Next.js build on a +// busy machine can take minutes. +const DEFAULT_TYPECHECK_TIMEOUT_MS = 60_000; +const DEFAULT_BUILD_TIMEOUT_MS = 180_000; /** * Run fast deterministic checks: typecheck first, then build. @@ -55,7 +58,7 @@ export async function runTypecheckValidation( return passResult('typecheck', startTime); } - const { exitCode, stdout, stderr } = await spawnCommand( + const { exitCode, stdout, stderr, timedOut } = await spawnCommand( typecheckCmd.command, typecheckCmd.args, projectDir, @@ -65,6 +68,11 @@ export async function runTypecheckValidation( if (exitCode === 0) { return passResult('typecheck', startTime); } + if (timedOut) { + // Killed at the limit: no evidence of an error, so don't ask the agent to fix one. + logWarn(`Typecheck timed out after ${timeoutMs}ms; treating as inconclusive`); + return passResult('typecheck', startTime); + } const output = stdout + stderr; const errors = parseTypecheckErrors(output); @@ -101,11 +109,21 @@ async function runBuildQuickCheck(projectDir: string, timeoutMs: number): Promis return passResult('build', startTime); } - const { exitCode, stdout, stderr } = await spawnCommand(buildCmd.command, buildCmd.args, projectDir, timeoutMs); + const { exitCode, stdout, stderr, timedOut } = await spawnCommand( + buildCmd.command, + buildCmd.args, + projectDir, + timeoutMs, + ); if (exitCode === 0) { return passResult('build', startTime); } + if (timedOut) { + // Killed at the limit: a bare "Build failed" sends the agent guessing at changes. + logWarn(`Build timed out after ${timeoutMs}ms; treating as inconclusive`); + return passResult('build', startTime); + } const output = stdout + stderr; const errors = parseBuildErrors(output); @@ -220,8 +238,9 @@ function spawnCommand( args: string[], cwd: string, timeoutMs: number, -): Promise<{ exitCode: number; stdout: string; stderr: string }> { +): Promise<{ exitCode: number; stdout: string; stderr: string; timedOut: boolean }> { return new Promise((resolve) => { + const startedAt = Date.now(); const proc = spawn(command, args, { cwd, timeout: timeoutMs, @@ -238,12 +257,14 @@ function spawnCommand( stderr += data.toString(); }); - proc.on('close', (code) => { - resolve({ exitCode: code ?? 1, stdout, stderr }); + proc.on('close', (code, signal) => { + // spawn's `timeout` kills the child with a signal once the limit passes. + const timedOut = code === null && signal !== null && Date.now() - startedAt >= timeoutMs; + resolve({ exitCode: code ?? 1, stdout, stderr, timedOut }); }); proc.on('error', () => { - resolve({ exitCode: 1, stdout, stderr }); + resolve({ exitCode: 1, stdout, stderr, timedOut: false }); }); }); } From 3f7cfa244a5ba5a4c4e052d3aacc98c61a124fee Mon Sep 17 00:00:00 2001 From: Nick Nisi Date: Wed, 21 Oct 2026 16:17:04 -0500 Subject: [PATCH 09/11] fix(installer): close quoted-flag bypass for read-only git The read-only git check tested whitespace-split tokens, but the shell removes quotes and backslashes before git sees them: `git diff "--output=/tmp/x"` passed and wrote the file, and a quoted --ext-diff slipped past its ban. Git commands containing quotes or backslashes are now denied, so the tokens match what runs. Allowed diff/show/log commands also run with --no-ext-diff --no-textconv, so no configured external diff or textconv program executes. --- src/lib/agent-interface.spec.ts | 17 +++++++++++++++++ src/lib/agent-interface.ts | 27 +++++++++++++++++++++++++-- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/src/lib/agent-interface.spec.ts b/src/lib/agent-interface.spec.ts index 77122141..2d15d5ef 100644 --- a/src/lib/agent-interface.spec.ts +++ b/src/lib/agent-interface.spec.ts @@ -644,6 +644,23 @@ describe('installerCanUseTool', () => { expect(verdict('git push')).toBe('deny'); }); + it('denies quoted or escaped git flags that the shell would unwrap', () => { + expect(verdict('git diff "--output=/tmp/x"')).toBe('deny'); + expect(verdict("git diff '--ext-diff'")).toBe('deny'); + expect(verdict('git diff --out\\put=/tmp/x')).toBe('deny'); + }); + + it('runs read-only git without external diff or textconv helpers', () => { + const command = (input: string) => + (installerCanUseTool('Bash', { command: input }, cwd) as { updatedInput: { command: string } }).updatedInput + .command; + expect(command('git diff --stat')).toBe('git diff --no-ext-diff --no-textconv --stat'); + expect(command('cd /proj && git -C /proj log -p | head -50')).toBe( + 'cd /proj && git -C /proj log --no-ext-diff --no-textconv -p | head -50', + ); + expect(command('git status --short')).toBe('git status --short'); + }); + it('allows syntax checks per tool without allowing code execution', () => { expect(verdict('python3 -m py_compile server.py')).toBe('allow'); expect(verdict('python3 -m venv .venv')).toBe('allow'); diff --git a/src/lib/agent-interface.ts b/src/lib/agent-interface.ts index 8600a90e..0b9219d8 100644 --- a/src/lib/agent-interface.ts +++ b/src/lib/agent-interface.ts @@ -252,6 +252,9 @@ const READ_ONLY_GIT_SUBCOMMANDS = ['status', 'diff', 'log', 'show', 'ls-files', const UNSAFE_GIT_ARG = /^(-c|-o|--output.*|--ext-diff|--textconv|--exec.*|--upload-pack.*|--open-files-in-pager.*)$/; function isReadOnlyGit(command: string, cwd?: string): boolean { + // Quotes and backslashes are removed by the shell, so `"--output=x"` or `--out\put` + // would slip past the token checks below. Without them, the split matches what runs. + if (/['"\\]/.test(command)) return false; const parts = command.split(/\s+/); if (parts[0] !== 'git') return false; let i = 1; @@ -264,6 +267,17 @@ function isReadOnlyGit(command: string, cwd?: string): boolean { return !parts.slice(i + 1).some((arg) => UNSAFE_GIT_ARG.test(arg)); } +/** + * Make an allowed read-only git command unable to run external diff or textconv + * programs, whatever the user's or repo's git config says. + */ +function withoutExternalGitHelpers(command: string): string { + return command.replace( + /\bgit((?:\s+--no-pager)?(?:\s+-C\s+\S+)?)\s+(diff|show|log)\b/, + 'git$1 $2 --no-ext-diff --no-textconv', + ); +} + /** Characters a `cd` target may contain; anything else could expand in the shell. */ const PLAIN_PATH = /^[A-Za-z0-9_./@+,=:%-]+$/; @@ -368,7 +382,11 @@ export function installerCanUseTool( }; } - if (matchesAllowedPrefix(baseCommand) || isReadOnlyGit(baseCommand, workingDirectory)) { + if (isReadOnlyGit(baseCommand, workingDirectory)) { + logInfo(`Allowing read-only git command with output limiter: ${command}`); + return { behavior: 'allow', updatedInput: { ...input, command: withoutExternalGitHelpers(rawCommand) } }; + } + if (matchesAllowedPrefix(baseCommand)) { logInfo(`Allowing bash command with output limiter: ${command}`); debug(`Allowing bash command with output limiter: ${command}`); return { behavior: 'allow', updatedInput: input }; @@ -391,7 +409,12 @@ export function installerCanUseTool( } // Check if command starts with any allowed prefix - if (matchesAllowedPrefix(normalized) || isReadOnlyGit(normalized, workingDirectory)) { + if (isReadOnlyGit(normalized, workingDirectory)) { + logInfo(`Allowing read-only git command: ${command}`); + return { behavior: 'allow', updatedInput: { ...input, command: withoutExternalGitHelpers(rawCommand) } }; + } + + if (matchesAllowedPrefix(normalized)) { logInfo(`Allowing bash command: ${command}`); debug(`Allowing bash command: ${command}`); return { behavior: 'allow', updatedInput: input }; From 9e178a3336036fa21f6022de58e93a9c31c1e435 Mon Sep 17 00:00:00 2001 From: Nick Nisi Date: Wed, 21 Oct 2026 16:17:05 -0500 Subject: [PATCH 10/11] fix(installer): write credentials for Kotlin, PHP, and Laravel installs With the package.json abort gone, these installs reached the agent with no credentials in the project: the installer's environment step skips non-JS apps, and runAgentInstaller skipped its own write when the caller supplied credentials. Non-JS integrations on this runner now always get WORKOS_API_KEY, WORKOS_CLIENT_ID, and WORKOS_REDIRECT_URI in .env (not .env.local; Laravel has a package.json for Vite), and the prompt names .env and no longer claims a generated cookie password. --- src/lib/agent-runner.spec.ts | 31 +++++++++++++++++++++++++++++-- src/lib/agent-runner.ts | 18 ++++++++++++++---- 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/src/lib/agent-runner.spec.ts b/src/lib/agent-runner.spec.ts index 2f02174d..8bf98d15 100644 --- a/src/lib/agent-runner.spec.ts +++ b/src/lib/agent-runner.spec.ts @@ -12,7 +12,7 @@ vi.mock('./validation/security-checks.js', () => ({ })); vi.mock('../steps/index.js', () => ({})); vi.mock('./workos-management.js', () => ({ autoConfigureWorkOSEnvironment: vi.fn() })); -vi.mock('./env-writer.js', () => ({ writeEnvLocal: vi.fn() })); +vi.mock('./env-writer.js', () => ({ writeEnvLocal: vi.fn(), writeEnvFile: vi.fn() })); vi.mock('../utils/ui-utils.js', () => ({ ensurePackageIsInstalled: vi.fn(), getOrAskForWorkOSCredentials: vi.fn(async () => ({ apiKey: 'test-key', clientId: 'client_test' })), @@ -28,7 +28,7 @@ import { initializeAgent, runAgent } from './agent-interface.js'; import { runAgentInstaller } from './agent-runner.js'; import { validateInstallation, quickCheckValidateAndFormat } from './validation/index.js'; import { autoConfigureWorkOSEnvironment } from './workos-management.js'; -import { writeEnvLocal } from './env-writer.js'; +import { writeEnvFile, writeEnvLocal } from './env-writer.js'; import { ensurePackageIsInstalled, getOrAskForWorkOSCredentials, getPackageDotJson } from '../utils/ui-utils.js'; const options: InstallerOptions = { @@ -258,6 +258,33 @@ describe('project manifest checks', () => { expect(runAgent).toHaveBeenCalledTimes(1); }); + it.each(['kotlin', 'php', 'php-laravel'] as const)( + 'writes %s credentials to .env even when the caller supplied them', + async (integration) => { + await runAgentInstaller( + { + ...config, + metadata: { ...config.metadata, integration, language: 'php', skillName: `workos-${integration}` }, + }, + { ...options, apiKey: 'sk_test', clientId: 'client_test' }, + ); + + expect(writeEnvFile).toHaveBeenCalledWith( + options.installDir, + expect.objectContaining({ + WORKOS_API_KEY: 'test-key', + WORKOS_CLIENT_ID: 'client_test', + WORKOS_REDIRECT_URI: expect.any(String), + }), + ); + expect(writeEnvLocal).not.toHaveBeenCalled(); + const prompt = vi.mocked(runAgent).mock.calls[0][1]; + expect(prompt).toContain('configured in .env:'); + expect(prompt).not.toContain('.env.local'); + expect(prompt).not.toContain('WORKOS_COOKIE_PASSWORD'); + }, + ); + it('does not require a framework package for vanilla JS', async () => { await runAgentInstaller( { diff --git a/src/lib/agent-runner.ts b/src/lib/agent-runner.ts index 424e024e..f77d5575 100644 --- a/src/lib/agent-runner.ts +++ b/src/lib/agent-runner.ts @@ -22,7 +22,7 @@ import { autoConfigureWorkOSEnvironment } from './workos-management.js'; import { assertSupportedNextJsRouter, assertNextjsSignInRouteAvailable } from '../integrations/nextjs/utils.js'; import { detectPort, resolveRedirectUri } from './port-detection.js'; import { buildSignInSection } from './sign-in-route.js'; -import { writeEnvLocal } from './env-writer.js'; +import { writeEnvFile, writeEnvLocal } from './env-writer.js'; /** * Universal agent-powered wizard runner. @@ -97,7 +97,16 @@ export async function runAgentInstaller(config: FrameworkConfig, options: Instal // Write environment variables to .env.local BEFORE agent runs // Skip if caller already handled this (prevents double-writing) - if (!callerHandledConfig) { + if (!isJavaScriptProject) { + // Non-JS projects use .env. The installer's environment step skips non-JS apps, so + // write credentials here even when the caller supplied them; otherwise the agent + // starts with none (Kotlin, PHP, Laravel). + writeEnvFile(options.installDir, { + ...(apiKey ? { WORKOS_API_KEY: apiKey } : {}), + WORKOS_CLIENT_ID: clientId, + WORKOS_REDIRECT_URI: resolveRedirectUri(config.metadata.integration, options), + }); + } else if (!callerHandledConfig) { const redirectUri = resolveRedirectUri(config.metadata.integration, options); // Next.js requires NEXT_PUBLIC_ prefix for client-side env vars @@ -329,7 +338,8 @@ async function buildIntegrationPrompt( ...(config.environment.requiresApiKey ? ['WORKOS_API_KEY'] : []), 'WORKOS_CLIENT_ID', config.metadata.integration === 'nextjs' ? 'NEXT_PUBLIC_WORKOS_REDIRECT_URI' : 'WORKOS_REDIRECT_URI', - 'WORKOS_COOKIE_PASSWORD', + // Only the .env.local writer generates a cookie password; non-JS SDKs configure their own. + ...(isJavaScript ? ['WORKOS_COOKIE_PASSWORD'] : []), ]; const envVarList = envVars.map((v) => `- ${v}`).join('\n'); const signInSection = buildSignInSection(config); @@ -343,7 +353,7 @@ async function buildIntegrationPrompt( ## Environment -The following environment variables have been configured in .env.local: +The following environment variables have been configured in ${config.metadata.language === 'javascript' ? '.env.local' : '.env'}: ${envVarList} ${baseContent ? `## General Guidelines\n\n${baseContent}\n\n` : ''}## Integration Instructions From 4bbb6512a1029a75284bd5cd22ae80077a5408e9 Mon Sep 17 00:00:00 2001 From: Nick Nisi Date: Wed, 21 Oct 2026 16:29:11 -0500 Subject: [PATCH 11/11] fix(installer): deny read-only git diff/show/log when helpers can't be disabled The read-only git check reads the command after removing `2>&1`, but the --no-ext-diff/--no-textconv rewrite matched the original text, so `git 2>&1 diff` was allowed unrewritten. The rewrite also matched the first "git diff|show|log" anywhere, so `git 2>&1 diff git log` rewrote the trailing path arguments instead of the real call. The rewrite is now anchored to the start of the command, and diff/show/log are denied when it can't apply, with a message saying to put options after the subcommand. --- src/lib/agent-interface.spec.ts | 8 +++++ src/lib/agent-interface.ts | 54 +++++++++++++++++++++++---------- 2 files changed, 46 insertions(+), 16 deletions(-) diff --git a/src/lib/agent-interface.spec.ts b/src/lib/agent-interface.spec.ts index 2d15d5ef..edbec4fb 100644 --- a/src/lib/agent-interface.spec.ts +++ b/src/lib/agent-interface.spec.ts @@ -661,6 +661,14 @@ describe('installerCanUseTool', () => { expect(command('git status --short')).toBe('git status --short'); }); + it('denies diff/show/log when the helper-disabling rewrite cannot apply', () => { + expect(verdict('git 2>&1 diff')).toBe('deny'); + expect(verdict('git 2>&1 log -p | head -20')).toBe('deny'); + expect(verdict('git 2>&1 diff git log')).toBe('deny'); + expect(verdict('git status 2>&1')).toBe('allow'); + expect(verdict('git diff --stat 2>&1 | tail -5')).toBe('allow'); + }); + it('allows syntax checks per tool without allowing code execution', () => { expect(verdict('python3 -m py_compile server.py')).toBe('allow'); expect(verdict('python3 -m venv .venv')).toBe('allow'); diff --git a/src/lib/agent-interface.ts b/src/lib/agent-interface.ts index 0b9219d8..0cb7720c 100644 --- a/src/lib/agent-interface.ts +++ b/src/lib/agent-interface.ts @@ -251,33 +251,59 @@ const READ_ONLY_GIT_SUBCOMMANDS = ['status', 'diff', 'log', 'show', 'ls-files', /** git arguments that write files or run external programs. */ const UNSAFE_GIT_ARG = /^(-c|-o|--output.*|--ext-diff|--textconv|--exec.*|--upload-pack.*|--open-files-in-pager.*)$/; -function isReadOnlyGit(command: string, cwd?: string): boolean { +/** The read-only git subcommand `command` runs, or null if it isn't an allowed read-only git call. */ +function readOnlyGitSubcommand(command: string, cwd?: string): string | null { // Quotes and backslashes are removed by the shell, so `"--output=x"` or `--out\put` // would slip past the token checks below. Without them, the split matches what runs. - if (/['"\\]/.test(command)) return false; + if (/['"\\]/.test(command)) return null; const parts = command.split(/\s+/); - if (parts[0] !== 'git') return false; + if (parts[0] !== 'git') return null; let i = 1; if (parts[i] === '--no-pager') i++; if (parts[i] === '-C') { - if (!cwd || !parts[i + 1] || resolve(cwd, parts[i + 1]) !== resolve(cwd)) return false; + if (!cwd || !parts[i + 1] || resolve(cwd, parts[i + 1]) !== resolve(cwd)) return null; i += 2; } - if (!READ_ONLY_GIT_SUBCOMMANDS.includes(parts[i])) return false; - return !parts.slice(i + 1).some((arg) => UNSAFE_GIT_ARG.test(arg)); + if (!READ_ONLY_GIT_SUBCOMMANDS.includes(parts[i])) return null; + return parts.slice(i + 1).some((arg) => UNSAFE_GIT_ARG.test(arg)) ? null : parts[i]; } +/** Subcommands that can run external diff or textconv helpers. */ +const GIT_DIFF_SUBCOMMANDS = ['diff', 'show', 'log']; + /** * Make an allowed read-only git command unable to run external diff or textconv * programs, whatever the user's or repo's git config says. */ function withoutExternalGitHelpers(command: string): string { + // Anchored: only the git call at the start (after an optional `cd &&`) is + // rewritten, so a later "git log" in the arguments can't satisfy the check. return command.replace( - /\bgit((?:\s+--no-pager)?(?:\s+-C\s+\S+)?)\s+(diff|show|log)\b/, - 'git$1 $2 --no-ext-diff --no-textconv', + /^((?:cd\s+\S+\s*&&\s*)?git(?:\s+--no-pager)?(?:\s+-C\s+\S+)?)\s+(diff|show|log)\b/, + '$1 $2 --no-ext-diff --no-textconv', ); } +/** + * Allow a read-only git command, or deny it when it could run external helpers and + * the helper-disabling rewrite can't be applied (e.g. `git 2>&1 diff`). + */ +function allowReadOnlyGit(subcommand: string, rawCommand: string, input: Record): PermissionResult { + if (!GIT_DIFF_SUBCOMMANDS.includes(subcommand)) { + logInfo(`Allowing read-only git command: ${rawCommand}`); + return { behavior: 'allow', updatedInput: input }; + } + const command = withoutExternalGitHelpers(rawCommand); + if (command === rawCommand) { + return { + behavior: 'deny', + message: `Bash command not allowed. Write read-only git as \`git ${subcommand} [args]\`, with options after the subcommand.`, + }; + } + logInfo(`Allowing read-only git command: ${command}`); + return { behavior: 'allow', updatedInput: { ...input, command } }; +} + /** Characters a `cd` target may contain; anything else could expand in the shell. */ const PLAIN_PATH = /^[A-Za-z0-9_./@+,=:%-]+$/; @@ -382,10 +408,8 @@ export function installerCanUseTool( }; } - if (isReadOnlyGit(baseCommand, workingDirectory)) { - logInfo(`Allowing read-only git command with output limiter: ${command}`); - return { behavior: 'allow', updatedInput: { ...input, command: withoutExternalGitHelpers(rawCommand) } }; - } + const gitSubcommand = readOnlyGitSubcommand(baseCommand, workingDirectory); + if (gitSubcommand) return allowReadOnlyGit(gitSubcommand, rawCommand, input); if (matchesAllowedPrefix(baseCommand)) { logInfo(`Allowing bash command with output limiter: ${command}`); debug(`Allowing bash command with output limiter: ${command}`); @@ -409,10 +433,8 @@ export function installerCanUseTool( } // Check if command starts with any allowed prefix - if (isReadOnlyGit(normalized, workingDirectory)) { - logInfo(`Allowing read-only git command: ${command}`); - return { behavior: 'allow', updatedInput: { ...input, command: withoutExternalGitHelpers(rawCommand) } }; - } + const gitSubcommand = readOnlyGitSubcommand(normalized, workingDirectory); + if (gitSubcommand) return allowReadOnlyGit(gitSubcommand, rawCommand, input); if (matchesAllowedPrefix(normalized)) { logInfo(`Allowing bash command: ${command}`);