diff --git a/parser/src/deno-libraries.ts b/parser/src/deno-libraries.ts index 15fdec18..f8bb62ed 100644 --- a/parser/src/deno-libraries.ts +++ b/parser/src/deno-libraries.ts @@ -87,6 +87,9 @@ function main(): number { const repo = path.resolve(process.argv[2] ?? '.'); const mirrors = new Set(); const packages = new Set(); + // Dependency specifiers that resolve to nothing: most often modules Deno has not + // downloaded (no `deno install` yet, or a cache the OS purged). + const unresolved = new Set(); for (const file of sourceFiles(repo)) { const context = denoContextForDir(path.dirname(file)); if (context === undefined) { @@ -101,6 +104,9 @@ function main(): number { for (const specifier of moduleSpecifiersOf(text, file)) { const resolved = resolveDenoSpecifier(specifier, file); if (resolved === undefined) { + if (!/^(\.{0,2}\/|[a-z][a-z0-9+.-]*:)/i.test(specifier) || /^(jsr|npm|https?):/.test(specifier)) { + unresolved.add(specifier); + } continue; } if (resolved.kind === 'JSR' || resolved.kind === 'REMOTE') { @@ -115,7 +121,7 @@ function main(): number { } // A mirror exists once something was copied into it; one that stayed empty stages nothing. const filled = [...mirrors].filter((m) => configIn(m) !== undefined).sort(); - process.stdout.write(`${JSON.stringify({ mirrors: filled, packages: [...packages].sort() })}\n`); + process.stdout.write(`${JSON.stringify({ mirrors: filled, packages: [...packages].sort(), unresolved: unresolved.size })}\n`); return 0; } diff --git a/parser/src/parsers/typescript/deno/deno-cache.ts b/parser/src/parsers/typescript/deno/deno-cache.ts index 10a3a17c..7d333d53 100644 --- a/parser/src/parsers/typescript/deno/deno-cache.ts +++ b/parser/src/parsers/typescript/deno/deno-cache.ts @@ -71,9 +71,16 @@ export function readRemote(url: string, root: DenoConfig): RemoteModule | undefi return { url, text, headers: {} }; } } - const cached = path.join(denoDir(), 'remote', parsed.protocol.replace(/:$/, ''), hostDir(parsed), - crypto.createHash('sha256').update(parsed.pathname + parsed.search).digest('hex')); - const raw = readText(cached); + const hostCache = path.join(denoDir(), 'remote', parsed.protocol.replace(/:$/, ''), hostDir(parsed)); + let cached = ''; + let raw: string | undefined; + for (const key of cacheKeysOf(parsed)) { + cached = path.join(hostCache, crypto.createHash('sha256').update(key).digest('hex')); + raw = readText(cached); + if (raw !== undefined) { + break; + } + } if (raw === undefined) { return undefined; } @@ -86,6 +93,17 @@ export function readRemote(url: string, root: DenoConfig): RemoteModule | undefi return { url, text: raw, headers: sidecar === undefined ? {} : headersOf(sidecar) }; } +/** + * The path and query Deno hashed for `url`. Deno's URL parser leaves a `^` in a + * path as written (`esm.sh/type-fest@^4.11.1`), where this one escapes it to + * `%5E`, so the unescaped spelling is tried as well. + */ +function cacheKeysOf(url: URL): string[] { + const key = url.pathname + url.search; + const unescaped = url.pathname.replace(/%5E/gi, '^') + url.search; + return unescaped === key ? [key] : [key, unescaped]; +} + function headersOf(json: string): Record { try { const meta = JSON.parse(json.trim()) as { headers?: Record }; @@ -122,6 +140,8 @@ function readText(file: string): string | undefined { export interface DenoLock { readonly jsr: ReadonlyMap; readonly npm: ReadonlyMap; + /** Remote URL to the URL it redirected to (`esm.sh/ky@1.7` to `esm.sh/ky@1.7.5`). */ + readonly redirects: ReadonlyMap; readonly text: string; } @@ -146,6 +166,7 @@ export function readLock(root: DenoConfig): DenoLock | undefined { result = { jsr: versionsByName(json.jsr ?? packages.jsr), npm: versionsByName(json.npm ?? packages.npm), + redirects: redirectsOf(json.redirects), text, }; } @@ -154,6 +175,18 @@ export function readLock(root: DenoConfig): DenoLock | undefined { return result; } +function redirectsOf(section: unknown): Map { + const out = new Map(); + if (section !== null && typeof section === 'object') { + for (const [from, to] of Object.entries(section as Record)) { + if (typeof to === 'string') { + out.set(from, to); + } + } + } + return out; +} + /** `{"@std/path@1.1.6": …, "zod@3.25.76_peer@1.0.0": …}` to name → versions. */ function versionsByName(section: unknown): Map { const out = new Map(); diff --git a/parser/src/parsers/typescript/deno/deno-config.ts b/parser/src/parsers/typescript/deno/deno-config.ts index 838e1e0c..d1c54492 100644 --- a/parser/src/parsers/typescript/deno/deno-config.ts +++ b/parser/src/parsers/typescript/deno/deno-config.ts @@ -50,6 +50,12 @@ export interface DenoConfig { readonly workspaceMembers: readonly string[]; /** `"vendor": true`: remote modules are kept under `/vendor`. */ readonly vendor: boolean; + /** + * `nodeModulesDir`: `none` (also `false`) serves npm packages from Deno's cache, `auto` + * (also `true`) installs them on use, `manual` expects an installed node_modules. + * `undefined` when the config does not say (Deno then expects node_modules beside a package.json). + */ + readonly nodeModulesDir: 'none' | 'auto' | 'manual' | undefined; /** `false` when the config turns the lockfile off; the path otherwise (it may not exist). */ readonly lockPath: string | false; readonly compilerOptions: Readonly>; @@ -188,6 +194,9 @@ export function readDenoConfig(configPath: string): DenoConfig | undefined { }, workspaceMembers: workspaceMembersOf(json.workspace, dir), vendor: json.vendor === true, + nodeModulesDir: json.nodeModulesDir === false || json.nodeModulesDir === 'none' ? 'none' + : json.nodeModulesDir === true || json.nodeModulesDir === 'auto' ? 'auto' + : json.nodeModulesDir === 'manual' ? 'manual' : undefined, lockPath: lockPathOf(json.lock, dir), compilerOptions: json.compilerOptions !== null && typeof json.compilerOptions === 'object' ? json.compilerOptions as Record diff --git a/parser/src/parsers/typescript/deno/deno-resolver.ts b/parser/src/parsers/typescript/deno/deno-resolver.ts index c5cf5aff..c3864cba 100644 --- a/parser/src/parsers/typescript/deno/deno-resolver.ts +++ b/parser/src/parsers/typescript/deno/deno-resolver.ts @@ -7,7 +7,7 @@ import { DenoConfig, DenoContext, DenoImportMap, denoContextFor } from '@/parser import { cachedNpmVersions, materialize, readLock, readRemote, RemoteModule, } from '@/parsers/typescript/deno/deno-cache'; -import { maxSatisfying } from '@/parsers/typescript/deno/deno-semver'; +import { maxSatisfying, satisfies } from '@/parsers/typescript/deno/deno-semver'; /** * Resolves a module specifier the way Deno does, for a file inside a Deno project. @@ -99,7 +99,73 @@ function resolveIn(specifier: string, containingFile: string, context: DenoConte if (target.startsWith('https:') || target.startsWith('http:')) { return resolveRemoteUrl(target, context.root); } - return memberExport(target, context); + return memberExport(target, context) ?? packageJsonDependency(target, containingFile, context); +} + +// ── 2b. a package.json dependency ─────────────────────────────────────────── + +/** + * A bare specifier no import map or member claims, named by the nearest + * package.json's dependencies: under `"nodeModulesDir": "none"` (or `auto`) Deno + * serves it from its npm cache (`vitest` in a member that declares it). Under + * `manual`, Deno's default beside a package.json, it expects node_modules and + * refuses otherwise, so this answers nothing and the compiler's resolution does. + */ +function packageJsonDependency(specifier: string, containingFile: string, context: DenoContext): + DenoResolution | undefined { + const mode = context.root.nodeModulesDir; + if (mode !== 'none' && mode !== 'auto') { + return undefined; + } + const m = /^((?:@[^/@]+\/)?[^/@]+)(\/.*)?$/.exec(specifier); + if (!m) { + return undefined; + } + const name = m[1]!; + const range = dependencyRange(name, path.dirname(containingFile), context.root.dir); + if (range === undefined) { + return undefined; + } + const installed = ts.resolveModuleName(specifier, containingFile, NPM_OPTIONS, ts.sys).resolvedModule; + if (installed !== undefined) { + return undefined; + } + const resolved = resolveNpm(`npm:${name}${range === '' ? '' : `@${range}`}${m[2] ?? ''}`, containingFile, context); + return resolved?.kind === 'NPM_CACHE' ? resolved : undefined; +} + +const DEPENDENCY_FIELDS = ['dependencies', 'devDependencies', 'peerDependencies', 'optionalDependencies']; + +/** `name`'s range in the nearest package.json at or above `dir` (up to `top`) that declares it; '' for a non-semver value. */ +function dependencyRange(name: string, dir: string, top: string): string | undefined { + for (let d = dir; ; d = path.dirname(d)) { + const manifest = readPackageJson(path.join(d, 'package.json')); + for (const field of DEPENDENCY_FIELDS) { + const value = (manifest?.[field] as Record | undefined)?.[name]; + if (typeof value === 'string') { + // `workspace:*` is a member (resolved above); `catalog:`, `file:` and tags name no range. + return /^[\d^~<>=*xX |.-]/.test(value) && !value.startsWith('workspace:') ? value.trim() : ''; + } + } + if (d === top || path.dirname(d) === d) { + return undefined; + } + } +} + +const packageJsons = new Map | null>(); + +function readPackageJson(file: string): Record | undefined { + let known = packageJsons.get(file); + if (known === undefined) { + try { + known = JSON.parse(fs.readFileSync(file, 'utf-8')) as Record; + } catch { + known = null; + } + packageJsons.set(file, known); + } + return known ?? undefined; } // ── 1. the import map ─────────────────────────────────────────────────────── @@ -273,28 +339,93 @@ function resolveNpm(target: string, containingFile: string, context: DenoContext const bare = name + (m[3] ?? ''); // A project with node_modules (`nodeModulesDir`, or a package.json) installs there. const installed = ts.resolveModuleName(bare, containingFile, NPM_OPTIONS, ts.sys).resolvedModule; + const lockedNpm = readLock(context.root)?.npm.get(name) ?? []; if (installed !== undefined && /[\\/]node_modules[\\/]/.test(installed.resolvedFileName)) { - return { - resolvedFileName: path.normalize(installed.resolvedFileName), - extension: installed.extension, - kind: 'NODE_MODULES', - packageName: installed.packageId?.name ?? name, - }; + // The version is checked only on the package itself: a package without types + // resolves to its `@types/` twin, whose version follows its own line. + const installedVersion = installed.packageId?.name === name ? installed.packageId.version : undefined; + if (range === '' || installedVersion === undefined || satisfies(installedVersion, range)) { + return { + resolvedFileName: path.normalize(installed.resolvedFileName), + extension: installed.extension, + kind: 'NODE_MODULES', + packageName: installed.packageId?.name ?? name, + }; + } + // The hoisted copy is another major (`npm:tailwindcss@^3` beside a hoisted 4): Deno + // keeps each version under node_modules/.deno/@/node_modules/. + const side = denoNodeModulesVersions(name, containingFile); + const version = maxSatisfying(lockedNpm.filter((v) => side.has(v)), range) ?? maxSatisfying(side.keys(), range); + const file = version === undefined ? undefined : resolveInPackage(bare, name, side.get(version)!); + if (file !== undefined) { + return { resolvedFileName: file.resolvedFileName, extension: file.extension, kind: 'NODE_MODULES', packageName: name }; + } } const cached = cachedNpmVersions(name); if (cached.size === 0) { return undefined; } - const locked = readLock(context.root)?.npm.get(name)?.filter((v) => cached.has(v)) ?? []; + const locked = lockedNpm.filter((v) => cached.has(v)); const version = maxSatisfying(locked, range) ?? maxSatisfying(cached.keys(), range); const packageDir = version === undefined ? undefined : cached.get(version); if (packageDir === undefined) { return undefined; } const file = resolveInPackage(bare, name, packageDir); - return file === undefined - ? undefined - : { resolvedFileName: file.resolvedFileName, extension: file.extension, kind: 'NPM_CACHE', packageName: name }; + if (file === undefined) { + return undefined; + } + // A package that ships only JavaScript is type-checked against its @types twin, as the compiler does. + const typed = DECLARATION.test(file.resolvedFileName) ? undefined : typesTwin(bare, name, context); + return { ...(typed ?? file), kind: 'NPM_CACHE', packageName: name }; +} + +const DECLARATION = /\.d\.[cm]?ts$/; + +function typesTwin(bare: string, name: string, context: DenoContext): + { resolvedFileName: string; extension: string } | undefined { + const twin = `@types/${name.startsWith('@') ? name.slice(1).replace('/', '__') : name}`; + const cached = cachedNpmVersions(twin); + if (cached.size === 0) { + return undefined; + } + const locked = (readLock(context.root)?.npm.get(twin) ?? []).filter((v) => cached.has(v)); + const version = maxSatisfying(locked, '') ?? maxSatisfying(cached.keys(), ''); + return version === undefined ? undefined + : resolveInPackage(twin + bare.slice(name.length), twin, cached.get(version)!); +} + +/** + * Each version of `name` under the nearest `node_modules/.deno`, with its package + * directory: `@scope/pkg` lives at `.deno/@scope+pkg@1.2.3[_peer]/node_modules/@scope/pkg`. + */ +function denoNodeModulesVersions(name: string, containingFile: string): Map { + const out = new Map(); + const prefix = name.replace('/', '+') + '@'; + for (let dir = path.dirname(containingFile); ; dir = path.dirname(dir)) { + const store = path.join(dir, 'node_modules', '.deno'); + let entries: string[] | undefined; + try { + entries = fs.readdirSync(store); + } catch { + entries = undefined; + } + if (entries !== undefined) { + for (const entry of entries) { + if (entry.startsWith(prefix)) { + const version = entry.slice(prefix.length).split('_')[0]!; + const packageDir = path.join(store, entry, 'node_modules', ...name.split('/')); + if (!out.has(version) && fs.existsSync(packageDir)) { + out.set(version, packageDir); + } + } + } + return out; + } + if (path.dirname(dir) === dir) { + return out; + } + } } /** @@ -403,8 +534,15 @@ function materializeWithImports(module: RemoteModule, root: DenoConfig): string } function followRedirects(url: string, root: DenoConfig): RemoteModule | undefined { + const redirects = readLock(root)?.redirects; let current = url; for (let hop = 0; hop < 10; hop++) { + // Deno 2 records a redirect in the lockfile and may keep no cache entry for it. + const locked = redirects?.get(current) ?? redirects?.get(current.replace(/%5E/gi, '^')); + if (locked !== undefined && locked !== current) { + current = locked; + continue; + } const module = readRemote(current, root); if (module === undefined) { return undefined; @@ -542,4 +680,5 @@ export function denoPackageNameOf(specifier: string, containingFile: string): st export function clearDenoResolutionCache(): void { resolutions.clear(); materializedUrls.clear(); + packageJsons.clear(); } diff --git a/parser/src/test/deno-tests.ts b/parser/src/test/deno-tests.ts index baf89a97..af46775d 100644 --- a/parser/src/test/deno-tests.ts +++ b/parser/src/test/deno-tests.ts @@ -46,10 +46,12 @@ function write(root: string, files: Record): string { } /** A module in the fake `$DENO_DIR` remote cache, keyed as Deno keys it. */ -function cacheRemote(denoDir: string, url: string, body: string, headers: Record = {}): void { +function cacheRemote(denoDir: string, url: string, body: string, headers: Record = {}, + key?: string): void { const u = new URL(url); + // `key`: the path Deno hashed, when it differs from this URL parser's spelling (`^` vs `%5E`). const file = path.join(denoDir, 'remote', u.protocol.replace(':', ''), u.hostname, - crypto.createHash('sha256').update(u.pathname + u.search).digest('hex')); + crypto.createHash('sha256').update(key ?? u.pathname + u.search).digest('hex')); fs.mkdirSync(path.dirname(file), { recursive: true }); fs.writeFileSync(file, `${body}\n// denoCacheMetadata=${JSON.stringify({ headers, url, time: 0 })}`); } @@ -327,6 +329,100 @@ const CHECKS: Check[] = [ : 'a relative import of a mirrored URL module was not copied'; }, }, + { + name: 'https: lockfile redirects and a caret in the URL', + proves: 'a redirect Deno recorded only in deno.lock is followed, and a URL with `^` finds the cache entry Deno hashed with the `^` as written', + rulesOut: 'partial-version CDN URLs (`esm.sh/ky@1.7`, `type-fest@^4`) left unresolved after `deno install`', + run: async (tmp) => { + const { denoDir, mirror } = freshEnvironment(tmp, 'remote-lock'); + cacheRemote(denoDir, 'https://esm.sh/ky@1.7.5', 'export default function ky() {}\n', + { 'content-type': 'application/typescript' }); + cacheRemote(denoDir, 'https://esm.sh/type-fest@^4.11.1', '', { location: 'https://esm.sh/type-fest@4.41.0' }, + '/type-fest@^4.11.1'); + cacheRemote(denoDir, 'https://esm.sh/type-fest@4.41.0', 'export type Simplify = T;\n', + { 'content-type': 'application/typescript' }); + const root = write(path.join(tmp, 'remote-lock'), { + 'deno.json': JSON.stringify({ imports: { ky: 'https://esm.sh/ky@1.7' } }), + 'deno.lock': JSON.stringify({ version: '5', redirects: { 'https://esm.sh/ky@1.7': 'https://esm.sh/ky@1.7.5' } }), + 'main.ts': '', + }); + return firstFailure([ + expectResolved(root, 'ky', 'main.ts', relativeTo(root, path.join(mirror, 'esm.sh/ky@1.7.5.ts')), 'REMOTE'), + expectResolved(root, 'https://esm.sh/type-fest@^4.11.1', 'main.ts', + relativeTo(root, path.join(mirror, 'esm.sh/type-fest@4.41.0.ts')), 'REMOTE'), + ]); + }, + }, + { + name: 'npm: the version range picks among node_modules/.deno copies', + proves: 'an npm: range the hoisted node_modules copy does not satisfy resolves into the matching node_modules/.deno/@ copy, and a package typed by its @types twin keeps the twin', + rulesOut: 'a member mapping `npm:pkg@^3` landing on the hoisted major 4, or a types-only twin rejected for its own version line', + run: async (tmp) => { + freshEnvironment(tmp, 'npm-store'); + const store = 'node_modules/.deno'; + const root = write(path.join(tmp, 'npm-store'), { + 'deno.json': JSON.stringify({ nodeModulesDir: 'manual', workspace: ['./v3'] }), + 'v3/deno.json': JSON.stringify({ name: '@x/v3', imports: { tw: 'npm:tw@^3.4.0', babel: 'npm:@babel/core@^7.28.0' } }), + 'v3/main.ts': '', + 'main.ts': '', + 'node_modules/tw/package.json': JSON.stringify({ name: 'tw', version: '4.1.0', types: './v4.d.ts' }), + 'node_modules/tw/v4.d.ts': '', + [`${store}/tw@4.1.0/node_modules/tw/package.json`]: JSON.stringify({ name: 'tw', version: '4.1.0', types: './v4.d.ts' }), + [`${store}/tw@4.1.0/node_modules/tw/v4.d.ts`]: '', + [`${store}/tw@3.4.18/node_modules/tw/package.json`]: JSON.stringify({ name: 'tw', version: '3.4.18', types: './v3.d.ts' }), + [`${store}/tw@3.4.18/node_modules/tw/v3.d.ts`]: '', + 'node_modules/@babel/core/package.json': JSON.stringify({ name: '@babel/core', version: '7.28.5', main: './lib/index.js' }), + 'node_modules/@babel/core/lib/index.js': '', + 'node_modules/@types/babel__core/package.json': JSON.stringify({ name: '@types/babel__core', version: '7.20.5', types: './index.d.ts' }), + 'node_modules/@types/babel__core/index.d.ts': '', + }); + return firstFailure([ + expectResolved(root, 'tw', 'v3/main.ts', `${store}/tw@3.4.18/node_modules/tw/v3.d.ts`, 'NODE_MODULES'), + expectResolved(root, 'npm:tw@^4', 'main.ts', 'node_modules/tw/v4.d.ts', 'NODE_MODULES'), + expectResolved(root, 'babel', 'v3/main.ts', 'node_modules/@types/babel__core/index.d.ts', 'NODE_MODULES'), + ]); + }, + }, + { + name: 'npm: a package.json dependency, and an @types twin', + proves: 'under "nodeModulesDir": "none" a bare specifier the nearest package.json declares resolves into Deno\'s npm cache, a JavaScript-only package resolves to its cached @types twin, and under "manual" (Deno\'s default beside a package.json) nothing is answered, as Deno refuses it', + rulesOut: 'Node-style members of a Deno workspace left unresolved, calls typed against a .js entry Deno never type-checks, or a cache answer where Deno demands node_modules', + run: async (tmp) => { + const { denoDir } = freshEnvironment(tmp, 'npm-pkgjson'); + const registry = path.join(denoDir, 'npm', 'registry.npmjs.org'); + write(path.join(registry, 'runner', '3.2.4'), { + 'package.json': JSON.stringify({ name: 'runner', version: '3.2.4', exports: { '.': { types: './index.d.ts' }, './config': { types: './config.d.ts' } } }), + 'index.d.ts': '', 'config.d.ts': '', + }); + write(path.join(registry, 'runner', '4.0.0'), { + 'package.json': JSON.stringify({ name: 'runner', version: '4.0.0', types: './index.d.ts' }), 'index.d.ts': '', + }); + write(path.join(registry, 'jsonly', '0.10.9'), { + 'package.json': JSON.stringify({ name: 'jsonly', version: '0.10.9', main: './main.js' }), 'main.js': '', + }); + write(path.join(registry, '@types', 'jsonly', '0.10.8'), { + 'package.json': JSON.stringify({ name: '@types/jsonly', version: '0.10.8', types: './index.d.ts' }), 'index.d.ts': '', + }); + const root = write(path.join(tmp, 'npm-pkgjson'), { + 'deno.json': JSON.stringify({ nodeModulesDir: 'none', workspace: ['./pkg'] }), + 'pkg/deno.json': JSON.stringify({ name: '@x/pkg' }), + 'manual/deno.json': '{}', + 'manual/package.json': JSON.stringify({ name: 'manual', dependencies: { runner: '^3.2.0' } }), + 'manual/main.ts': '', + 'pkg/package.json': JSON.stringify({ name: '@x/pkg', devDependencies: { runner: '^3.2.0', jsonly: 'catalog:' } }), + 'pkg/src/a.test.ts': '', + 'main.ts': '', + }); + return firstFailure([ + expectResolved(root, 'runner', 'pkg/src/a.test.ts', relativeTo(root, path.join(registry, 'runner', '3.2.4', 'index.d.ts')), 'NPM_CACHE'), + expectResolved(root, 'runner/config', 'pkg/src/a.test.ts', relativeTo(root, path.join(registry, 'runner', '3.2.4', 'config.d.ts'))), + expectResolved(root, 'jsonly', 'pkg/src/a.test.ts', relativeTo(root, path.join(registry, '@types', 'jsonly', '0.10.8', 'index.d.ts')), 'NPM_CACHE'), + // Not declared by the root's (absent) package.json: the compiler's own resolution keeps it. + expectResolved(root, 'runner', 'main.ts', ''), + expectResolved(root, 'runner', 'manual/main.ts', ''), + ]); + }, + }, { name: 'outside a Deno project nothing changes', proves: 'with no deno.json above a file the resolver answers nothing, so every other TypeScript project resolves as before', diff --git a/plugins/axiomcode/skills/axiomcode/scripts/ax_libs.py b/plugins/axiomcode/skills/axiomcode/scripts/ax_libs.py index 00082afd..8f01b85b 100644 --- a/plugins/axiomcode/skills/axiomcode/scripts/ax_libs.py +++ b/plugins/axiomcode/skills/axiomcode/scripts/ax_libs.py @@ -142,6 +142,9 @@ def deno_libraries(repo, engine): found = [d for d in roots.get('mirrors', []) + roots.get('packages', []) if os.path.isdir(d)] note(f"auto: {len(roots.get('mirrors', []))} Deno mirror(s) of jsr: and URL modules, " f"{len(roots.get('packages', []))} npm package(s) the Deno project imports") + if roots.get('unresolved'): + note(f"auto: {roots['unresolved']} Deno dependency import(s) resolve to nothing; if Deno has not downloaded " + f"them (or its cache was cleared), run `deno install` and index again") return found