diff --git a/CHANGELOG.md b/CHANGELOG.md index 0098a700a..02e017601 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,19 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +## [Unreleased] + +### Fixed +- One unresolvable Maven dependency no longer strips the jar paths from every other dependency of its build, which made reachability analysis call their vulnerabilities unreachable. +- Maven, Gradle and sbt dependencies whose jars cannot be fetched are reported, and reachability no longer treats them as resolved without code. sbt no longer passes them off as POM-only modules. +- Reachability scans that hit unresolved JVM dependencies now point out that ignoring them also needs `--reach-continue-on-install-errors`. +- Gradle dependencies that fail dependency verification are reported instead of passed on as healthy. +- Maven dependencies with missing or invalid POMs are reported in every module that uses them, `system`-scope jars reach reachability analysis, and Maven 4 failures show their real cause. +- Failed Gradle project dependencies are reported instead of silently dropping everything they pull in, and failures name the version that actually failed. +- A module excluded with `--exclude-paths` is still analysed as part of the projects that depend on it. +- A failing dependency no longer fails scopes or configurations that `excludeConfigs` leaves out, and a version-conflict loser no longer fails the build. +- With `--dynamic-sbom-inference`, an sbt build that fails before writing its results is reported as failed instead of reusing another build's results. + ## [1.10.0](https://github.com/SocketDev/socket-cli/releases/tag/v1.10.0) - 2026-10-22 ### Added diff --git a/src/commands/manifest/convert-gradle-to-facts.mts b/src/commands/manifest/convert-gradle-to-facts.mts index f9ab8a13d..3ed9a6329 100644 --- a/src/commands/manifest/convert-gradle-to-facts.mts +++ b/src/commands/manifest/convert-gradle-to-facts.mts @@ -1,5 +1,6 @@ import { runManifestFacts } from './run-manifest-facts.mts' +import type { RunManifestFactsOutcome } from './run-manifest-facts.mts' import type { BuildOptionHints } from './scripts/resolution-report-render.mts' import type { SidecarAccumulator } from './scripts/sidecar.mts' @@ -31,8 +32,8 @@ export async function convertGradleToFacts({ sidecarAcc?: SidecarAccumulator | undefined verbose: boolean withFiles?: boolean | undefined -}): Promise { - await runManifestFacts({ +}): Promise { + return await runManifestFacts({ bin, buildOpts: gradleOpts, cwd, diff --git a/src/commands/manifest/convert-maven-to-facts.mts b/src/commands/manifest/convert-maven-to-facts.mts index bb50bd417..ad436e6bc 100644 --- a/src/commands/manifest/convert-maven-to-facts.mts +++ b/src/commands/manifest/convert-maven-to-facts.mts @@ -1,5 +1,6 @@ import { runManifestFacts } from './run-manifest-facts.mts' +import type { RunManifestFactsOutcome } from './run-manifest-facts.mts' import type { BuildOptionHints } from './scripts/resolution-report-render.mts' import type { SidecarAccumulator } from './scripts/sidecar.mts' @@ -31,8 +32,8 @@ export async function convertMavenToFacts({ sidecarAcc?: SidecarAccumulator | undefined verbose: boolean withFiles?: boolean | undefined -}): Promise { - await runManifestFacts({ +}): Promise { + return await runManifestFacts({ bin, buildOpts: mavenOpts, cwd, diff --git a/src/commands/manifest/convert-sbt-to-facts.mts b/src/commands/manifest/convert-sbt-to-facts.mts index 999a4ccb5..b1a1d35e2 100644 --- a/src/commands/manifest/convert-sbt-to-facts.mts +++ b/src/commands/manifest/convert-sbt-to-facts.mts @@ -1,5 +1,6 @@ import { runManifestFacts } from './run-manifest-facts.mts' +import type { RunManifestFactsOutcome } from './run-manifest-facts.mts' import type { BuildOptionHints } from './scripts/resolution-report-render.mts' import type { SidecarAccumulator } from './scripts/sidecar.mts' @@ -35,8 +36,8 @@ export async function convertSbtToFacts({ tmpDir: string verbose: boolean withFiles?: boolean | undefined -}): Promise { - await runManifestFacts({ +}): Promise { + return await runManifestFacts({ bin, buildOpts: sbtOpts, cwd, diff --git a/src/commands/manifest/generate-recursive-manifests.mts b/src/commands/manifest/generate-recursive-manifests.mts index ce578f74b..3a0061f76 100644 --- a/src/commands/manifest/generate-recursive-manifests.mts +++ b/src/commands/manifest/generate-recursive-manifests.mts @@ -12,7 +12,10 @@ import { import { parseBuildToolOpts } from './parse-build-tool-opts.mts' import { runManifestFacts } from './run-manifest-facts.mts' import { resolveBuildToolBin } from './scripts/build-tool.mts' -import { DSI_SETUP_COMMAND } from './scripts/resolution-report-render.mts' +import { + DSI_SETUP_COMMAND, + REACH_CONTINUE_ON_INSTALL_ERRORS_FLAG, +} from './scripts/resolution-report-render.mts' import { SOCKET_JSON } from '../../constants.mts' import { withTmpDir } from '../../utils/fs.mts' import { @@ -179,6 +182,9 @@ async function runEcosystemCandidates({ excludeConfigs, excludePaths: excludePathsForRoot, hints: { + continueOnInstallErrorsFlag: withFiles + ? REACH_CONTINUE_ON_INSTALL_ERRORS_FLAG + : undefined, flags: optionFlags, setupCommand: DSI_SETUP_COMMAND, socketJson: displayPath(path.join(dir, SOCKET_JSON)), @@ -294,10 +300,10 @@ export async function generateRecursiveManifests({ // A shared global base across every sbt root in this run lets sbt's own // Scala-toolchain cache under /boot survive between invocations // instead of being reprovisioned per root (the plugin file is rewritten and - // records.tsv is fully overwritten - not appended - on every invocation, so - // reuse is safe). A caller-supplied sbtTmpDir is reused as-is (the caller - // owns its lifetime); otherwise one is allocated and cleaned up here, but - // only when there's an sbt root to benefit from sharing it at all. + // records.tsv is removed before every invocation, so reuse is safe). A + // caller-supplied sbtTmpDir is reused as-is (the caller owns its lifetime); + // otherwise one is allocated and cleaned up here, but only when there's an + // sbt root to benefit from sharing it at all. const outcomes = sbtTmpDir ? await runAll(sbtTmpDir) : candidatesByTool.get('sbt')?.length diff --git a/src/commands/manifest/generate_auto_manifest.mts b/src/commands/manifest/generate_auto_manifest.mts index 5803a065f..fc449d56c 100644 --- a/src/commands/manifest/generate_auto_manifest.mts +++ b/src/commands/manifest/generate_auto_manifest.mts @@ -11,6 +11,7 @@ import { convertSbtToMaven } from './convert_sbt_to_maven.mts' import { handleManifestConda } from './handle-manifest-conda.mts' import { parseBuildToolOpts } from './parse-build-tool-opts.mts' import { resolveBuildToolBin } from './scripts/build-tool.mts' +import { REACH_CONTINUE_ON_INSTALL_ERRORS_FLAG } from './scripts/resolution-report-render.mts' import { hasSidecarEntries, serializeSidecar } from './scripts/sidecar.mts' import { ENVIRONMENT_YML, @@ -21,6 +22,7 @@ import { InputError } from '../../utils/errors.mts' import { readOrDefaultSocketJson } from '../../utils/socket-json.mts' import type { GeneratableManifests } from './detect-manifest-actions.mts' +import type { RunManifestFactsOutcome } from './run-manifest-facts.mts' import type { BuildOptionHints } from './scripts/resolution-report-render.mts' import type { ResolvedPathsSidecar, @@ -41,12 +43,20 @@ export type GenerateAutoManifestResult = { resolvedPathsSidecar?: ResolvedPathsSidecar | undefined } -// Under --auto-manifest, a manifest generator that failed — raising the exit -// code above the value captured before it ran — aborts the whole run: a partial -// or empty SBOM silently under-reports dependencies. The generator has already -// logged the specifics. A tolerated resolution failure (ignoreUnresolved) warns -// without touching the exit code, so it passes through here and the run -// continues. +// A failed generator aborts --auto-manifest: a partial SBOM silently +// under-reports dependencies. Facts generators return null on failure; the +// others only change the exit code. +function abortFactsRunIfFailed( + ecosystem: string, + outcome: RunManifestFactsOutcome, +): void { + if (outcome === null) { + throw new InputError( + `Auto-manifest generation failed for the ${ecosystem} project; aborting (see the errors above).`, + ) + } +} + function abortManifestRunIfFailed( ecosystem: string, beforeExitCode: string | number | undefined, @@ -82,6 +92,12 @@ export async function generateAutoManifest({ }): Promise { const sockJson = readOrDefaultSocketJson(cwd) const generatedFiles: string[] = [] + const hints: BuildOptionHints = computeArtifactsSidecar + ? { + ...AUTO_MANIFEST_HINTS, + continueOnInstallErrorsFlag: REACH_CONTINUE_ON_INSTALL_ERRORS_FLAG, + } + : AUTO_MANIFEST_HINTS // Resolved paths across all JVM roots, serialized to one sidecar at the end. const sidecarAcc: SidecarAccumulator | undefined = computeArtifactsSidecar @@ -109,8 +125,7 @@ export async function generateAutoManifest({ // `defaults.manifest.sbt.facts: false` in socket.json. if (sockJson.defaults?.manifest?.sbt?.facts !== false) { logger.log('Detected a Scala sbt build, generating Socket facts...') - const beforeExitCode = process.exitCode - await convertSbtToFacts({ + const outcome = await convertSbtToFacts({ ...sbtArgs, excludeConfigs: sockJson.defaults?.manifest?.sbt?.excludeConfigs ?? '', excludePaths, @@ -119,12 +134,12 @@ export async function generateAutoManifest({ ), includeConfigs: sockJson.defaults?.manifest?.sbt?.includeConfigs ?? '', javaHome: sockJson.defaults?.manifest?.sbt?.javaHome ?? undefined, - hints: AUTO_MANIFEST_HINTS, + hints, sidecarAcc, tmpDir, withFiles: computeArtifactsSidecar, }) - abortManifestRunIfFailed('sbt', beforeExitCode) + abortFactsRunIfFailed('sbt', outcome) } else { logger.log('Detected a Scala sbt build, generating pom files with sbt...') const beforeExitCode = process.exitCode @@ -154,8 +169,7 @@ export async function generateAutoManifest({ logger.log( 'Detected a gradle build (Gradle, Kotlin, Scala), generating Socket facts...', ) - const beforeExitCode = process.exitCode - await convertGradleToFacts({ + const outcome = await convertGradleToFacts({ ...gradleArgs, excludeConfigs: sockJson.defaults?.manifest?.gradle?.excludeConfigs ?? '', @@ -166,11 +180,11 @@ export async function generateAutoManifest({ includeConfigs: sockJson.defaults?.manifest?.gradle?.includeConfigs ?? '', javaHome: sockJson.defaults?.manifest?.gradle?.javaHome ?? undefined, - hints: AUTO_MANIFEST_HINTS, + hints, sidecarAcc, withFiles: computeArtifactsSidecar, }) - abortManifestRunIfFailed('gradle', beforeExitCode) + abortFactsRunIfFailed('gradle', outcome) } else { logger.log( 'Detected a gradle build (Gradle, Kotlin, Scala), running default gradle generator...', @@ -183,8 +197,7 @@ export async function generateAutoManifest({ if (!sockJson?.defaults?.manifest?.maven?.disabled && detected.maven) { logger.log('Detected a Maven pom.xml build, generating Socket facts...') - const beforeExitCode = process.exitCode - await convertMavenToFacts({ + const outcome = await convertMavenToFacts({ // Configured bin wins; else prefer ./mvnw, else mvn on PATH. bin: sockJson.defaults?.manifest?.maven?.bin ?? @@ -200,12 +213,12 @@ export async function generateAutoManifest({ mavenOpts: parseBuildToolOpts( sockJson.defaults?.manifest?.maven?.mavenOpts ?? undefined, ), - hints: AUTO_MANIFEST_HINTS, + hints, sidecarAcc, verbose: Boolean(sockJson.defaults?.manifest?.maven?.verbose), withFiles: computeArtifactsSidecar, }) - abortManifestRunIfFailed('maven', beforeExitCode) + abortFactsRunIfFailed('maven', outcome) } if (!sockJson?.defaults?.manifest?.conda?.disabled && detected.conda) { diff --git a/src/commands/manifest/generate_auto_manifest.test.mts b/src/commands/manifest/generate_auto_manifest.test.mts index f9bd23135..47c5b8e9d 100644 --- a/src/commands/manifest/generate_auto_manifest.test.mts +++ b/src/commands/manifest/generate_auto_manifest.test.mts @@ -386,4 +386,18 @@ describe('generateAutoManifest — conda branch', () => { ).rejects.toThrow(/Auto-manifest generation failed for the conda project/) process.exitCode = undefined }) + + it('aborts on a failed facts generator even when the exit code was already set', async () => { + process.exitCode = 1 + vi.mocked(convertGradleToFacts).mockResolvedValueOnce(null) + await expect( + generateAutoManifest({ + cwd: '/tmp/repo', + detected: { ...baseDetected, gradle: true, count: 1 }, + outputKind: 'text', + verbose: false, + }), + ).rejects.toThrow(/Auto-manifest generation failed for the gradle project/) + process.exitCode = undefined + }) }) diff --git a/src/commands/manifest/run-manifest-facts.mts b/src/commands/manifest/run-manifest-facts.mts index b9598a44d..1a69346a4 100644 --- a/src/commands/manifest/run-manifest-facts.mts +++ b/src/commands/manifest/run-manifest-facts.mts @@ -179,7 +179,12 @@ export async function runManifestFacts({ report.failures, report.scannedConfigs, ecosystem, - { hints, ignoreUnresolved, unscannable: report.unscannable }, + { + hints, + ignoreUnresolved, + resolvedModules: report.resolvedModules, + unscannable: report.unscannable, + }, ) if (rendered.hasBlockingFailures) { diff --git a/src/commands/manifest/run-manifest-facts.test.mts b/src/commands/manifest/run-manifest-facts.test.mts index e3aadb200..45b45cf77 100644 --- a/src/commands/manifest/run-manifest-facts.test.mts +++ b/src/commands/manifest/run-manifest-facts.test.mts @@ -26,7 +26,12 @@ function okResult(buildRoot: string): ManifestRunResult { projects: [], }, factsFileName: 'pom.xml.socket.facts.json', - report: { failures: [], scannedConfigs: [], unscannable: [] }, + report: { + failures: [], + resolvedModules: new Set(), + scannedConfigs: [], + unscannable: [], + }, artifactPaths: { pathsById: new Map(), classpathByProject: new Map(), diff --git a/src/commands/manifest/scripts/assemble.mts b/src/commands/manifest/scripts/assemble.mts index cb53d3d9e..03d92bca6 100644 --- a/src/commands/manifest/scripts/assemble.mts +++ b/src/commands/manifest/scripts/assemble.mts @@ -8,6 +8,7 @@ import { type SocketFactsSbomMetadata, type SocketFactsSbomProject, } from './facts.mts' +import { moduleKey } from './resolution-report.mts' import type { ParsedRecords, RawCoord, RawProject } from './records.mts' import type { ResolutionReport } from './resolution-report.mts' @@ -33,6 +34,7 @@ type MergedNode = { // projectKey when this node is a build project, else empty. project: string targets: Set + unfetched: boolean } type SubprojectGraph = { @@ -52,6 +54,7 @@ type RootNode = { direct: boolean project: string targets: string[] + unfetched: boolean } export function assembleFacts( @@ -88,6 +91,7 @@ export function assembleFacts( artifactPaths: buildArtifactPaths( finalNodes, [...parsed.projects.values()], + [...parsed.excludedProjects.values()], graphs, fileExists, ), @@ -116,10 +120,12 @@ function buildPerRoot(parsed: ParsedRecords): Map { direct: false, project: n.project, targets: [], + unfetched: false, } nodes.set(id, node) } node.direct ||= n.direct + node.unfetched ||= n.unfetched node.targets.push(...n.targets) } for (const [p, c] of r.edges) { @@ -153,6 +159,7 @@ function mergeById(perRoot: Map): Map { direct: false, project: node.project, targets: new Set(), + unfetched: false, } finalNodes.set(coordId, fn) } @@ -162,6 +169,9 @@ function mergeById(perRoot: Map): Map { if (node.direct) { fn.direct = true } + if (node.unfetched) { + fn.unfetched = true + } for (const t of node.targets) { fn.targets.add(t) } @@ -402,21 +412,23 @@ function byProject( function buildArtifactPaths( finalNodes: Map, projects: RawProject[], + excludedProjects: RawProject[], graphs: Map, fileExists: (path: string) => boolean, ): ResolvedArtifactPaths { const pathsById: ResolvedArtifactPaths['pathsById'] = new Map() for (const [id, fn] of finalNodes) { if (!fn.project) { - pathsById.set(id, { - sources: [], - targets: [...fn.targets].filter(fileExists).sort(), - }) + const targets = [...fn.targets].filter(fileExists).sort() + // An artifact with any file that could not be fetched, or is gone from + // disk, has no targets: a partial list would claim it resolved whole. + const unresolved = fn.unfetched || targets.length < fn.targets.size + pathsById.set(id, unresolved ? { sources: [] } : { sources: [], targets }) } } // A project's own component shares its id, so this also covers dependency // edges onto a sibling project. - for (const p of projects) { + for (const p of [...projects, ...excludedProjects]) { pathsById.set(p.projectKey, { sources: [...new Set(p.sources)].filter(fileExists).sort(), targets: [...new Set(p.targets)].filter(fileExists).sort(), @@ -448,5 +460,20 @@ function buildReport(parsed: ParsedRecords): ResolutionReport { seenUnscannable.add(key) return true }) - return { failures, scannedConfigs: parsed.scannedConfigs, unscannable } + const resolvedModules = new Set() + if (failures.length) { + for (const r of parsed.roots.values()) { + for (const n of r.nodes.values()) { + resolvedModules.add( + moduleKey(r.projectKey, `${n.coord.group}:${n.coord.name}`), + ) + } + } + } + return { + failures, + resolvedModules, + scannedConfigs: parsed.scannedConfigs, + unscannable, + } } diff --git a/src/commands/manifest/scripts/assemble.test.mts b/src/commands/manifest/scripts/assemble.test.mts index 2167321a9..3652a5a96 100644 --- a/src/commands/manifest/scripts/assemble.test.mts +++ b/src/commands/manifest/scripts/assemble.test.mts @@ -115,6 +115,77 @@ describe('records → assemble → sidecar', () => { expect(bom?.targets).toEqual([]) expect(bom?.sources).toEqual([]) }) + it("carries an excluded module's paths into the kept project that depends on it", () => { + const records = [ + 'meta\tgradle\t8.10\t17', + 'project\t:app\tg\tapp\t1\tapp', + 'projectSrc\t:app\t/abs/app/src', + 'projectTgt\t:app\t/abs/app/classes', + 'excludedProject\t:lib', + 'projectSrc\t:lib\t/abs/lib/src', + 'projectTgt\t:lib\t/abs/lib/classes', + 'root\t0\t:app\tcompileClasspath\t1', + 'node\t0\t:lib\tg\tlib\t1\tjar\t\t1\t:lib', + ].join('\n') + const { artifactPaths, facts } = assembleFacts(parseRecords(records), { + fileExists: () => true, + }) + + expect(facts.projects?.map(p => p.id)).toEqual([':app']) + expect(artifactPaths.pathsById.get(':lib')).toEqual({ + sources: ['/abs/lib/src'], + targets: ['/abs/lib/classes'], + }) + expect([...artifactPaths.classpathByProject.keys()]).toEqual([':app']) + }) + it('states no targets for an artifact that failed to resolve or is gone from disk', () => { + const records = [ + 'meta\tmaven\t3.9.6\t17', + 'project\tg:app:1\tg\tapp\t1\t.', + 'root\t0\tg:app:1\tcompile\t1', + 'node\t0\tg:ok:jar:1\tg\tok\t1\tjar\t\t1', + 'node\t0\tg:nojar:jar:1\tg\tnojar\t1\tjar\t\t1', + 'node\t0\tg:nojar:test-jar:tests:1\tg\tnojar\t1\ttest-jar\ttests\t1', + 'node\t0\tg:gone:jar:1\tg\tgone\t1\tjar\t\t1', + 'node\t0\tg:bom:pom:1\tg\tbom\t1\tpom\t\t1', + 'node\t0\tg:half:jar:1\tg\thalf\t1\tjar\t\t1', + 'node\t0\tg:halfgone:jar:1\tg\thalfgone\t1\tjar\t\t1', + 'file\t0\tg:ok:jar:1\t/abs/ok.jar', + 'file\t0\tg:half:jar:1\t/abs/half-a.jar', + 'unfetched\t0\tg:half:jar:1', + 'file\t0\tg:halfgone:jar:1\t/abs/halfgone-a.jar', + 'file\t0\tg:halfgone:jar:1\t/abs/gone.jar', + 'file\t0\tg:gone:jar:1\t/abs/gone.jar', + 'file\t0\tg:nojar:test-jar:tests:1\t/abs/nojar-tests.jar', + 'unfetched\t0\tg:nojar:jar:1', + ].join('\n') + const { artifactPaths, facts } = assembleFacts(parseRecords(records), { + fileExists: p => p !== '/abs/gone.jar', + }) + const acc: SidecarAccumulator = new Map() + accumulateSidecar( + acc, + facts, + artifactPaths, + '/abs/pom.xml.socket.facts.json', + ) + const byId = new Map( + serializeSidecar(acc)['/abs/pom.xml.socket.facts.json']!.components.map( + c => [c.id, c], + ), + ) + + expect(byId.get('g:ok:jar:1')?.targets).toEqual(['/abs/ok.jar']) + expect(byId.get('g:nojar:jar:1')).not.toHaveProperty('targets') + expect(byId.get('g:nojar:test-jar:tests:1')?.targets).toEqual([ + '/abs/nojar-tests.jar', + ]) + expect(byId.get('g:gone:jar:1')).not.toHaveProperty('targets') + expect(byId.get('g:bom:pom:1')?.targets).toEqual([]) + // One of several files missing leaves the artifact incomplete. + expect(byId.get('g:half:jar:1')).not.toHaveProperty('targets') + expect(byId.get('g:halfgone:jar:1')).not.toHaveProperty('targets') + }) it('keeps a coordinate with divergent subtrees as one component with a dependency entry per subtree', () => { // :a and :b both depend on `lib`, which pulls a different `dep` version in // each subproject. diff --git a/src/commands/manifest/scripts/facts.mts b/src/commands/manifest/scripts/facts.mts index ca13ac151..6c48d539c 100644 --- a/src/commands/manifest/scripts/facts.mts +++ b/src/commands/manifest/scripts/facts.mts @@ -60,9 +60,10 @@ export type SocketFactsSbomProject = AnyPURL & { // Resolved on-disk paths for a --with-files run, keyed by component or project // id (a project and its own component share one). `targets` = classpath -// entries (jars / module output dirs); `sources` = module source roots. +// entries (jars / module output dirs), absent when the component's artifact +// did not resolve; `sources` = module source roots. export type ResolvedArtifactPaths = { - pathsById: Map + pathsById: Map // Component ids on each project's resolved classpath (union over its // configurations), keyed by project id. classpathByProject: Map diff --git a/src/commands/manifest/scripts/maven-extension/src/main/java/tech/coana/ext/CoanaFactsEventSpy.java b/src/commands/manifest/scripts/maven-extension/src/main/java/tech/coana/ext/CoanaFactsEventSpy.java new file mode 100644 index 000000000..089964c75 --- /dev/null +++ b/src/commands/manifest/scripts/maven-extension/src/main/java/tech/coana/ext/CoanaFactsEventSpy.java @@ -0,0 +1,31 @@ +package tech.coana.ext; + +import org.apache.maven.eventspy.AbstractEventSpy; +import org.eclipse.aether.RepositoryEvent; +import tech.coana.socket.CollectionObserver; + +import javax.inject.Named; +import javax.inject.Singleton; + +/** + * Relays repository events to the facts engine while it collects. Every Maven chains its event + * spies into the session's repository listener, which an extension cannot replace: the session is + * read-only since Maven 3.1 and immutable in Maven 4. + */ +@Named("coana-facts-events") +@Singleton +public class CoanaFactsEventSpy extends AbstractEventSpy { + + // Static: the container may hand the dispatcher and the participant different instances. + private static volatile CollectionObserver observer; + + static void observe(CollectionObserver o) { + observer = o; + } + + @Override + public void onEvent(Object event) { + CollectionObserver o = observer; + if (o != null && event instanceof RepositoryEvent) o.onEvent((RepositoryEvent) event); + } +} diff --git a/src/commands/manifest/scripts/maven-extension/src/main/java/tech/coana/ext/CoanaFactsLifecycleParticipant.java b/src/commands/manifest/scripts/maven-extension/src/main/java/tech/coana/ext/CoanaFactsLifecycleParticipant.java index d2fbf9217..3ff79ec57 100644 --- a/src/commands/manifest/scripts/maven-extension/src/main/java/tech/coana/ext/CoanaFactsLifecycleParticipant.java +++ b/src/commands/manifest/scripts/maven-extension/src/main/java/tech/coana/ext/CoanaFactsLifecycleParticipant.java @@ -5,8 +5,10 @@ import org.apache.maven.execution.MavenSession; import org.apache.maven.project.ProjectDependenciesResolver; import org.apache.maven.rtinfo.RuntimeInformation; +import org.eclipse.aether.RepositorySystem; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import tech.coana.socket.CollectionObserver; import tech.coana.socket.SocketFactsRecordsEngine; import javax.inject.Inject; @@ -28,12 +30,16 @@ public class CoanaFactsLifecycleParticipant extends AbstractMavenLifecyclePartic private static final Logger LOG = LoggerFactory.getLogger("coana"); private final ProjectDependenciesResolver dependenciesResolver; + private final RepositorySystem repositorySystem; private final RuntimeInformation runtimeInformation; @Inject public CoanaFactsLifecycleParticipant( - ProjectDependenciesResolver dependenciesResolver, RuntimeInformation runtimeInformation) { + ProjectDependenciesResolver dependenciesResolver, + RepositorySystem repositorySystem, + RuntimeInformation runtimeInformation) { this.dependenciesResolver = dependenciesResolver; + this.repositorySystem = repositorySystem; this.runtimeInformation = runtimeInformation; } @@ -54,11 +60,16 @@ public void afterSessionEnd(MavenSession session) throws MavenExecutionException opts.excludeConfigs = opt(session, "socket.excludeConfigs"); opts.excludePaths = opt(session, "socket.excludePaths"); File rootDir = new File(session.getExecutionRootDirectory()); + CollectionObserver observer = new CollectionObserver(); + CoanaFactsEventSpy.observe(observer); try { - new SocketFactsRecordsEngine(dependenciesResolver, runtimeInformation.getMavenVersion(), LOG) + new SocketFactsRecordsEngine( + dependenciesResolver, repositorySystem, runtimeInformation.getMavenVersion(), observer, LOG) .run(session, session.getProjects(), rootDir, opts); } catch (IOException exception) { throw new MavenExecutionException("Cannot write socket facts records", exception); + } finally { + CoanaFactsEventSpy.observe(null); } } diff --git a/src/commands/manifest/scripts/maven-extension/src/main/java/tech/coana/socket/CollectionObserver.java b/src/commands/manifest/scripts/maven-extension/src/main/java/tech/coana/socket/CollectionObserver.java new file mode 100644 index 000000000..f56dcd067 --- /dev/null +++ b/src/commands/manifest/scripts/maven-extension/src/main/java/tech/coana/socket/CollectionObserver.java @@ -0,0 +1,125 @@ +package tech.coana.socket; + +import org.eclipse.aether.RepositoryEvent; +import org.eclipse.aether.RequestTrace; +import org.eclipse.aether.collection.CollectStepData; +import org.eclipse.aether.graph.Dependency; +import org.eclipse.aether.graph.DependencyNode; + +import java.util.HashSet; +import java.util.IdentityHashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * What the repository events of dependency collection reveal beyond its result: the POMs Maven's + * default {@code ArtifactDescriptorPolicy} tolerated as missing or invalid (the node stays, its + * dependencies are lost, and no collection error is raised), build-wide, and the graph position of + * each collect step of the current module. + */ +public final class CollectionObserver { + + // CollectStepData is maven-resolver 1.8.1+ (Maven 3.9+); older resolvers trace no collect step. + private static final boolean COLLECT_STEPS = classExists("org.eclipse.aether.collection.CollectStepData"); + + // The resolver caches a tolerated POM build-wide, so only the first module to reach one sees its + // event; that module's detail is kept. + private final Map incompleteDescriptors = new LinkedHashMap<>(); + private final Set incompleteInModule = new HashSet<>(); + private final Map stepParents = new IdentityHashMap<>(); + + public synchronized void onEvent(RepositoryEvent event) { + if (event.getType() == RepositoryEvent.EventType.ARTIFACT_DESCRIPTOR_MISSING) { + incomplete(event, "is missing, no dependency information available"); + } else if (event.getType() == RepositoryEvent.EventType.ARTIFACT_DESCRIPTOR_INVALID) { + incomplete(event, "is invalid, transitive dependencies (if any) will not be available"); + } + // The depth-first collector hands each step its live node stack, so the parent is only right now. + if (COLLECT_STEPS) Steps.recordParent(event.getTrace(), stepParents); + } + + private void incomplete(RepositoryEvent event, String problem) { + String gav = SocketFactsRecordsEngine.gav(event.getArtifact()); + if (incompleteDescriptors.containsKey(gav) && !incompleteInModule.contains(gav)) return; + incompleteInModule.add(gav); + incompleteDescriptors.put( + gav, + "The POM for " + event.getArtifact() + " " + problem + ": " + + SocketFactsRecordsEngine.rootMessage(event.getException())); + } + + /** Starts the next module's collection. */ + synchronized void reset() { + incompleteInModule.clear(); + stepParents.clear(); + } + + /** GAV to failure detail of each POM the collection tolerated as missing or invalid. */ + synchronized Map incompleteDescriptors() { + return new LinkedHashMap<>(incompleteDescriptors); + } + + /** The node whose children the collect step behind {@code trace} was adding, if it was observed. */ + synchronized DependencyNode parentOf(RequestTrace trace) { + return COLLECT_STEPS ? Steps.parentOf(trace, stepParents) : null; + } + + /** The dependency, before management, that the collect step behind {@code trace} was adding. */ + synchronized Dependency dependencyOf(RequestTrace trace) { + return COLLECT_STEPS ? Steps.dependencyOf(trace) : null; + } + + private static boolean classExists(String name) { + try { + Class.forName(name, false, RequestTrace.class.getClassLoader()); + return true; + } catch (ClassNotFoundException | LinkageError e) { + return false; + } + } + + // Only loaded when CollectStepData exists. + private static final class Steps { + static void recordParent(RequestTrace trace, Map parents) { + RequestTrace at = stepTrace(trace); + if (at == null || parents.containsKey(at.getData())) return; + List path = ((CollectStepData) at.getData()).getPath(); + if (!path.isEmpty()) parents.put(at.getData(), path.get(path.size() - 1)); + } + + static DependencyNode parentOf(RequestTrace trace, Map parents) { + RequestTrace at = stepTrace(trace); + if (at == null) return null; + DependencyNode observed = parents.get(at.getData()); + if (observed != null) return observed; + // The depth-first collector hands every step its one live node stack, so an unobserved step's path + // is only trusted when it is checkably its own: the breadth-first collector nests each step's + // trace in its parent's step, whose dependency is then the path's last node. + RequestTrace enclosing = stepTrace(at.getParent()); + List path = ((CollectStepData) at.getData()).getPath(); + if (enclosing == null || path.isEmpty()) return null; + DependencyNode last = path.get(path.size() - 1); + Dependency parent = ((CollectStepData) enclosing.getData()).getNode(); + if (last.getDependency() == null + || !last.getDependency().getArtifact().getGroupId().equals(parent.getArtifact().getGroupId()) + || !last.getDependency().getArtifact().getArtifactId().equals(parent.getArtifact().getArtifactId())) { + return null; + } + return last; + } + + static Dependency dependencyOf(RequestTrace trace) { + RequestTrace at = stepTrace(trace); + return at == null ? null : ((CollectStepData) at.getData()).getNode(); + } + + private static RequestTrace stepTrace(RequestTrace trace) { + for (RequestTrace t = trace; t != null; t = t.getParent()) { + if (t.getData() instanceof CollectStepData) return t; + } + return null; + } + } +} diff --git a/src/commands/manifest/scripts/maven-extension/src/main/java/tech/coana/socket/SocketFactsRecordsEngine.java b/src/commands/manifest/scripts/maven-extension/src/main/java/tech/coana/socket/SocketFactsRecordsEngine.java index b343020e8..860a05ba4 100644 --- a/src/commands/manifest/scripts/maven-extension/src/main/java/tech/coana/socket/SocketFactsRecordsEngine.java +++ b/src/commands/manifest/scripts/maven-extension/src/main/java/tech/coana/socket/SocketFactsRecordsEngine.java @@ -1,5 +1,6 @@ package tech.coana.socket; +import org.apache.maven.RepositoryUtils; import org.apache.maven.execution.MavenSession; import org.apache.maven.model.Resource; import org.apache.maven.project.DefaultDependencyResolutionRequest; @@ -8,11 +9,17 @@ import org.apache.maven.project.DependencyResolutionResult; import org.apache.maven.project.MavenProject; import org.apache.maven.project.ProjectDependenciesResolver; +import org.eclipse.aether.RepositorySystem; +import org.eclipse.aether.RequestTrace; import org.eclipse.aether.artifact.Artifact; import org.eclipse.aether.graph.Dependency; import org.eclipse.aether.graph.DependencyFilter; import org.eclipse.aether.graph.DependencyNode; import org.eclipse.aether.resolution.ArtifactDescriptorException; +import org.eclipse.aether.resolution.ArtifactDescriptorRequest; +import org.eclipse.aether.resolution.ArtifactResult; +import org.eclipse.aether.resolution.DependencyRequest; +import org.eclipse.aether.resolution.VersionRangeRequest; import org.eclipse.aether.resolution.VersionRangeResolutionException; import org.slf4j.Logger; @@ -72,14 +79,30 @@ public static final class Options { private static final String NODE_DATA_CONFLICT_WINNER = "conflict.winner"; private final ProjectDependenciesResolver dependenciesResolver; + private final RepositorySystem repositorySystem; private final String mavenVersion; + private final CollectionObserver observer; private final Logger log; + // Maven 3.2.5's collector (aether 1.0.0) leaves a dependency whose POM failed out of the graph; from + // aether 1.0.2 (Maven 3.3) on it stays, minus its children. + private final boolean collectorDropsFailedNodes; + // Maven 3's JavaScopeDeriver and ClassicDependencyManager, which absentScope models. + private final boolean maven3ScopeRules; + // The resolver caches a failed POM build-wide, keyed by artifact alone, so only the first module to + // reach one sees the failure; every later module gets the node without children and no error. + // Keyed by GAV. + private final Map brokenDescriptors = new LinkedHashMap<>(); public SocketFactsRecordsEngine( - ProjectDependenciesResolver dependenciesResolver, String mavenVersion, Logger log) { + ProjectDependenciesResolver dependenciesResolver, RepositorySystem repositorySystem, String mavenVersion, + CollectionObserver observer, Logger log) { this.dependenciesResolver = dependenciesResolver; + this.repositorySystem = repositorySystem; this.mavenVersion = mavenVersion; + this.observer = observer; this.log = log; + this.collectorDropsFailedNodes = mavenVersion != null && mavenVersion.startsWith("3.2."); + this.maven3ScopeRules = mavenVersion == null || mavenVersion.startsWith("3."); } public void run(MavenSession session, List reactor, File rootDir, Options opts) @@ -108,11 +131,16 @@ public void run(MavenSession session, List reactor, File rootDir, // own "no POM in this directory" error surface instead of an NPE. if (module.getBasedir() == null) continue; String ws = SocketSupport.workspace(rootDir.toPath(), module.getBasedir().toPath()); - if (SocketSupport.isExcludedPath(ws, excludes)) continue; String key = projectKey(module); - rec(lines, "project", key, module.getGroupId(), module.getArtifactId(), module.getVersion(), ws); - File pom = module.getFile(); - if (pom != null && pom.isFile()) rec(lines, "projectBuild", key, SocketSupport.relativePath(rootDir.toPath(), pom.toPath())); + boolean excluded = SocketSupport.isExcludedPath(ws, excludes); + if (excluded) { + // Not analysed on its own, but a kept module depending on it is analysed with its code. + if (opts.withFiles) rec(lines, "excludedProject", key); + } else { + rec(lines, "project", key, module.getGroupId(), module.getArtifactId(), module.getVersion(), ws); + File pom = module.getFile(); + if (pom != null && pom.isFile()) rec(lines, "projectBuild", key, SocketSupport.relativePath(rootDir.toPath(), pom.toPath())); + } if (opts.withFiles) { for (String s : collectSources(module)) rec(lines, "projectSrc", key, s); for (String t : collectTargets(module)) rec(lines, "projectTgt", key, t); @@ -166,10 +194,12 @@ private void collectModule( String moduleCoord = module.getGroupId() + ":" + module.getArtifactId() + ":" + module.getVersion(); DependencyResolutionResult result; DependencyResolutionException thrown = null; + DependencyFilter resolutionFilter = materializationFilter(opts, passingScopes, reactorGavs, populateGavs); + observer.reset(); try { DependencyResolutionRequest request = new DefaultDependencyResolutionRequest(module, session.getRepositorySession()); - request.setResolutionFilter(materializationFilter(opts, passingScopes, reactorGavs, populateGavs)); + request.setResolutionFilter(resolutionFilter); result = dependenciesResolver.resolve(request); } catch (DependencyResolutionException e) { // Maven attaches the partial result — graph plus per-dependency errors — to the exception, so @@ -184,34 +214,76 @@ private void collectModule( } boolean reported = false; DependencyNode graph = result.getDependencyGraph(); - // Dependencies whose own resolution failed are reported, not emitted, as Gradle and sbt do. - Set unresolved = new HashSet<>(); - Map> scopesByGav = null; + List collectionFailures = new ArrayList<>(); for (Exception e : result.getCollectionErrors()) { - String failed = failedArtifact(e); + FailedRequest failed = failedRequest(e); if (failed == null) { unscannable.add(new Unscannable(rootMessage(e), moduleCoord)); } else { - // Maven keeps a dependency whose POM failed in the graph, minus its children (Maven 3.2 drops it), so - // the failure belongs to the scopes it has there. One absent from the graph has none. - if (scopesByGav == null) scopesByGav = scopesByGav(graph); - Set scopes = scopesByGav.getOrDefault(failed, Collections.emptySet()); - if (scopes.isEmpty()) failures.add(new Failure(failed, rootMessage(e), "", moduleCoord)); - for (String scope : scopes) { - if (passingScopes.contains(scope)) failures.add(new Failure(failed, rootMessage(e), scope, moduleCoord)); + collectionFailures.add(failed); + if (failed.descriptor) { + brokenDescriptors.put(gav(failed.artifact), failed); } - unresolved.add(failed); } log.warn("[socket-facts] could not build dependency graph for " + moduleCoord + ": " + rootMessage(e)); reported = true; } + Map incompleteDescriptors = observer.incompleteDescriptors(); + // Dependencies whose own resolution failed are reported, not emitted, as Gradle and sbt do. One + // whose POM Maven tolerated as missing or invalid is still emitted: its own artifact may be fine. + Set unresolved = new HashSet<>(); + Map> scopesByGav = + brokenDescriptors.isEmpty() && incompleteDescriptors.isEmpty() && collectionFailures.isEmpty() + ? Collections.>emptyMap() + : scopesByGav(graph); + for (Map.Entry e : brokenDescriptors.entrySet()) { + Set scopes = scopesByGav.get(e.getKey()); + if (scopes == null) continue; + unresolved.add(e.getKey()); + reported |= addFailures(failures, e.getKey(), e.getValue().detail, scopes, passingScopes, moduleCoord); + } + for (Map.Entry e : incompleteDescriptors.entrySet()) { + // A strict descriptor policy also throws for it: already reported above. + if (brokenDescriptors.containsKey(e.getKey())) continue; + Set scopes = scopesByGav.get(e.getKey()); + if (scopes != null) reported |= addFailures(failures, e.getKey(), e.getValue(), scopes, passingScopes, moduleCoord); + } + Set placed = new HashSet<>(); + Set onClasspath = null; + for (FailedRequest failed : collectionFailures) { + String coord = gav(failed.artifact); + if (scopesByGav.containsKey(coord)) continue; + if (onClasspath == null) onClasspath = onClasspath(graph); + String scope = absentScope(session, module, failed, onClasspath); + placed.add(coord); + if (scope == null) { + log.debug("[socket-facts] " + coord + " failed but sits on no classpath of " + moduleCoord); + } else if (scope.isEmpty() || passingScopes.contains(scope)) { + failures.add(new Failure(coord, failed.detail, scope, moduleCoord)); + } + } + if (collectorDropsFailedNodes) { + // An earlier module's cached failure leaves no trace in this graph; only a direct dependency + // can still be placed. + for (Map.Entry e : brokenDescriptors.entrySet()) { + if (scopesByGav.containsKey(e.getKey()) || placed.contains(e.getKey())) continue; + String scope = declaredScope(session, module, e.getValue().artifact); + if (scope != null && passingScopes.contains(scope)) { + failures.add(new Failure(e.getKey(), e.getValue().detail, scope, moduleCoord)); + reported = true; + } + } + } + // Maven throws on a collection error before it fetches any artifact, which would leave every + // healthy node of the partial graph without its file. + Set unfetched = new HashSet<>(); + if (!result.getCollectionErrors().isEmpty() && graph != null && opts.withFiles) { + materialize(session, graph, resolutionFilter, unresolved, moduleCoord, failures, unfetched); + } for (Dependency dep : result.getUnresolvedDependencies()) { Artifact artifact = dep.getArtifact(); - if (artifact == null) continue; - List errors = result.getResolutionErrors(dep); - failures.add(new Failure( - gav(artifact), rootMessage(errors.isEmpty() ? null : errors.get(0)), scopeOf(dep), moduleCoord)); - log.debug("[socket-facts] could not materialize " + artifact + " (" + scopeOf(dep) + ")"); + if (artifact == null || unresolved.contains(gav(artifact))) continue; + reportUnfetched(dep, result.getResolutionErrors(dep), moduleCoord, failures, unfetched); reported = true; } // Fail closed: a throw whose result named nothing must still surface, or an unresolved dependency @@ -233,6 +305,10 @@ private void collectModule( if (isProd(scopeOf(child.getDependency()))) directProdIds.add(id); } } + for (String id : unfetched) { + Node node = nodes.get(id); + if (node != null) node.unfetched = true; + } } /** @@ -264,10 +340,7 @@ public boolean accept(DependencyNode node, List parents) { Dependency dep = node == null ? null : node.getDependency(); Artifact artifact = dep == null ? null : dep.getArtifact(); if (artifact == null) return false; - String scope = scopeOf(dep); - // A system-scope artifact carries its systemPath on the model and has no repository to be - // fetched from; visit() reads the file straight off the node instead. - if ("system".equals(scope) || !passingScopes.contains(scope)) return false; + if (!passingScopes.contains(scopeOf(dep))) return false; String gav = gav(artifact); if (reactorGavs.contains(gav)) return false; return populateGavs == null || populateGavs.contains(gav); @@ -275,6 +348,47 @@ public boolean accept(DependencyNode node, List parents) { }; } + /** + * The artifact step Maven skipped, run on the partial graph the way the resolver's own + * {@code resolveDependencies} continues past a collection failure. The failed dependencies are left + * out: they are never emitted, and fetching them again would only repeat the failure. + */ + private void materialize( + MavenSession session, + DependencyNode graph, + final DependencyFilter resolutionFilter, + final Set unresolved, + String moduleCoord, + Set failures, + Set unfetched) { + DependencyFilter healthy = new DependencyFilter() { + @Override + public boolean accept(DependencyNode node, List parents) { + return resolutionFilter.accept(node, parents) && !unresolved.contains(gav(node.getArtifact())); + } + }; + List results; + try { + results = repositorySystem + .resolveDependencies(session.getRepositorySession(), new DependencyRequest(graph, healthy)) + .getArtifactResults(); + } catch (org.eclipse.aether.resolution.DependencyResolutionException e) { + results = e.getResult().getArtifactResults(); + } + for (ArtifactResult ar : results) { + if (ar.isResolved()) continue; + reportUnfetched(ar.getRequest().getDependencyNode().getDependency(), ar.getExceptions(), moduleCoord, failures, unfetched); + } + } + + private void reportUnfetched( + Dependency dep, List errors, String moduleCoord, Set failures, Set unfetched) { + String id = externalId(dep.getArtifact()); + failures.add(new Failure(id, rootMessage(errors.isEmpty() ? null : errors.get(0)), scopeOf(dep), moduleCoord)); + unfetched.add(id); + log.debug("[socket-facts] could not materialize " + dep.getArtifact() + " (" + scopeOf(dep) + ")"); + } + private String visit( DependencyNode dn, Set passingScopes, @@ -295,16 +409,12 @@ private String visit( String gav = gav(artifact); if (unresolved.contains(gav)) return null; boolean internal = reactorGavs.contains(gav); - // Maven's `type` rather than aether's file extension, so a test-jar keeps the coordId the - // assembler and the Gradle/SBT scripts already emit. String type = artifact.getProperty(ARTIFACT_PROPERTY_TYPE, artifact.getExtension()); String classifier = artifact.getClassifier(); - // Base version: a resolved remote snapshot's `version` is the timestamped build, which would - // put a coordinate in the records that no manifest ever names. String version = artifact.getBaseVersion(); String id = internal ? SocketSupport.bareId(artifact.getGroupId(), artifact.getArtifactId(), version) - : SocketSupport.coordId(artifact.getGroupId(), artifact.getArtifactId(), type, classifier, version); + : externalId(artifact); // One walk per node per module traversal: a shared subtree reached via another edge is already // fully recorded (node, children, resolved file), so hand back the id without re-descending. @@ -329,7 +439,18 @@ private String visit( return id; } - private static String gav(Artifact artifact) { + private static String externalId(Artifact artifact) { + // Maven's `type` rather than aether's file extension, so a test-jar keeps the coordId the + // assembler and the Gradle/SBT scripts already emit. Base version: a resolved remote snapshot's + // `version` is the timestamped build, which would put a coordinate in the records that no + // manifest ever names. + return SocketSupport.coordId( + artifact.getGroupId(), artifact.getArtifactId(), + artifact.getProperty(ARTIFACT_PROPERTY_TYPE, artifact.getExtension()), artifact.getClassifier(), + artifact.getBaseVersion()); + } + + static String gav(Artifact artifact) { return artifact.getGroupId() + ":" + artifact.getArtifactId() + ":" + artifact.getBaseVersion(); } @@ -405,6 +526,7 @@ private int emitRoot( if (nodeMap.containsKey(child)) rec(lines, "edge", rootId, n.id, child); } for (String f : n.files) rec(lines, "file", rootId, n.id, f); + if (n.unfetched) rec(lines, "unfetched", rootId, n.id); } return rootIdx + 1; } @@ -513,13 +635,17 @@ private void write(String recordsFile, List lines) throws IOException { log.info("[socket-facts] records written to: " + out.getAbsolutePath()); } - private static String rootMessage(Throwable t) { + static String rootMessage(Throwable t) { Throwable cur = t; String msg = null; int guard = 0; while (cur != null && guard++ < 12) { if (cur.getMessage() != null) msg = cur.getMessage(); - cur = cur.getCause(); + Throwable next = cur.getCause(); + // Maven 4 batches requests: BatchRequestException carries each failed request as a suppressed + // exception, not as its cause. + if (next == null && cur.getSuppressed().length > 0) next = cur.getSuppressed()[0]; + cur = next; } return (msg != null ? msg : "unknown resolution failure").trim(); } @@ -540,29 +666,134 @@ private static Map> scopesByGav(DependencyNode root) { return scopes; } - // The artifact a collection error is about, when the resolver names one. - private static String failedArtifact(Throwable t) { - Throwable cur = t; + // The request a collection error is about, when the resolver names one. + private static FailedRequest failedRequest(Exception e) { + Throwable cur = e; int guard = 0; while (cur != null && guard++ < 12) { - Artifact artifact = null; if (cur instanceof ArtifactDescriptorException) { - ArtifactDescriptorException e = (ArtifactDescriptorException) cur; - if (e.getResult() != null) artifact = e.getResult().getRequest().getArtifact(); + ArtifactDescriptorException d = (ArtifactDescriptorException) cur; + if (d.getResult() != null) { + ArtifactDescriptorRequest request = d.getResult().getRequest(); + return new FailedRequest(request.getArtifact(), request.getTrace(), true, rootMessage(e)); + } } else if (cur instanceof VersionRangeResolutionException) { - VersionRangeResolutionException e = (VersionRangeResolutionException) cur; - if (e.getResult() != null) artifact = e.getResult().getRequest().getArtifact(); + VersionRangeResolutionException r = (VersionRangeResolutionException) cur; + if (r.getResult() != null) { + VersionRangeRequest request = r.getResult().getRequest(); + return new FailedRequest(request.getArtifact(), request.getTrace(), false, rootMessage(e)); + } } - if (artifact != null) return gav(artifact); cur = cur.getCause(); } return null; } + private static boolean addFailures( + Set failures, String coord, String detail, Set scopes, Set passingScopes, + String moduleCoord) { + boolean added = false; + for (String scope : scopes) { + if (passingScopes.contains(scope)) added |= failures.add(new Failure(coord, detail, scope, moduleCoord)); + } + return added; + } + + // Null: on no classpath (from Maven 3.3 the graph keeps a failed POM's node, so an absent one lost a + // conflict). Empty: position unknown (no collect step before 3.9; Maven 4's scope rules not modelled). + private String absentScope( + MavenSession session, MavenProject module, FailedRequest failed, Set onClasspath) { + if (failed.descriptor && !collectorDropsFailedNodes) return null; + DependencyNode parent = observer.parentOf(failed.trace); + Dependency dependency = observer.dependencyOf(failed.trace); + if (parent != null && dependency != null) { + if (!onClasspath.contains(parent)) return null; + if (parent.getDependency() == null) return scopeOf(dependency); + if (!maven3ScopeRules) return ""; + String managed = managedScope(session, module, dependency.getArtifact()); + return derivedScope(parent.getDependency().getScope(), managed != null ? managed : scopeOf(dependency)); + } + String declared = declaredScope(session, module, failed.artifact); + return declared != null ? declared : ""; + } + + // ClassicDependencyManager: below the direct dependencies, only the root's dependencyManagement applies. + private static String managedScope(MavenSession session, MavenProject module, Artifact artifact) { + if (module.getDependencyManagement() == null) return null; + org.apache.maven.model.Dependency d = + modelDependency(session, module.getDependencyManagement().getDependencies(), artifact, false); + return d == null || d.getScope() == null || d.getScope().isEmpty() ? null : d.getScope(); + } + + // maven-resolver's JavaScopeDeriver. + private static String derivedScope(String parentScope, String childScope) { + if ("system".equals(childScope) || "test".equals(childScope)) return childScope; + if (parentScope == null || parentScope.isEmpty() || "compile".equals(parentScope)) return childScope; + if ("test".equals(parentScope) || "runtime".equals(parentScope)) return parentScope; + if ("system".equals(parentScope) || "provided".equals(parentScope)) return "provided"; + return "runtime"; + } + + // The module's own declaration of the artifact, with management applied. + private static String declaredScope(MavenSession session, MavenProject module, Artifact artifact) { + org.apache.maven.model.Dependency d = modelDependency(session, module.getDependencies(), artifact, true); + if (d == null) return null; + return d.getScope() == null || d.getScope().isEmpty() ? "compile" : d.getScope(); + } + + // The first of `deps` naming the artifact, compared as the resolver sees it: a `test-jar` type + // becomes classifier `tests`. With `matchVersion`, a ranged version matches whichever it resolved to. + private static org.apache.maven.model.Dependency modelDependency( + MavenSession session, List deps, Artifact artifact, boolean matchVersion) { + for (org.apache.maven.model.Dependency d : deps) { + Artifact candidate = + RepositoryUtils.toDependency(d, session.getRepositorySession().getArtifactTypeRegistry()).getArtifact(); + String version = candidate.getVersion(); + if (candidate.getGroupId().equals(artifact.getGroupId()) + && candidate.getArtifactId().equals(artifact.getArtifactId()) + && candidate.getClassifier().equals(artifact.getClassifier()) + && candidate.getExtension().equals(artifact.getExtension()) + && (!matchVersion + || version.equals(artifact.getVersion()) || version.startsWith("[") || version.startsWith("("))) { + return d; + } + } + return null; + } + + // The final graph's root and every node on its classpath, by identity. + private static Set onClasspath(DependencyNode root) { + Set seen = Collections.newSetFromMap(new IdentityHashMap<>()); + if (root == null) return seen; + Deque todo = new ArrayDeque<>(); + todo.push(root); + while (!todo.isEmpty()) { + DependencyNode dn = todo.pop(); + if (dn.getData().get(NODE_DATA_CONFLICT_WINNER) != null || !seen.add(dn)) continue; + todo.addAll(dn.getChildren()); + } + return seen; + } + + private static final class FailedRequest { + final Artifact artifact; + final RequestTrace trace; + // The artifact's POM could not be read; else no version satisfied its range. + final boolean descriptor; + final String detail; + + FailedRequest(Artifact artifact, RequestTrace trace, boolean descriptor, String detail) { + this.artifact = artifact; + this.trace = trace; + this.descriptor = descriptor; + this.detail = detail; + } + } + private static final class Failure { final String coord; final String detail; - // Empty when the failure belongs to no scope. + // Empty when the failure cannot be placed in a scope. final String config; final String project; @@ -623,6 +854,7 @@ private static final class Node { final String project; final TreeSet children = new TreeSet<>(); final TreeSet files = new TreeSet<>(); + boolean unfetched; Node(String id, String groupId, String artifactId, String type, String classifier, String version, String project) { this.id = id; diff --git a/src/commands/manifest/scripts/records.mts b/src/commands/manifest/scripts/records.mts index 428e614e3..2e99cd630 100644 --- a/src/commands/manifest/scripts/records.mts +++ b/src/commands/manifest/scripts/records.mts @@ -13,12 +13,14 @@ import type { ResolutionFailure, Unscannable } from './resolution-report.mts' // projectSrc projectKey path (--with-files only) // projectTgt projectKey path (--with-files only) // projectBuild projectKey path (build-root-relative; may be absent on disk) +// excludedProject projectKey (--with-files only; paths without a project) // root rootId projectKey config prod(0|1) // node rootId coordId group name version ext classifier direct(0|1) project // edge rootId parentCoordId childCoordId // file rootId coordId path (--with-files only) +// unfetched rootId coordId (--with-files only; requested artifact not fetched) // scanned config -// failure coord detail config projectKey (config empty: no configuration) +// failure coord detail config projectKey (config empty: not placeable in one) // unscannable config detail projectKey (config empty: whole project) // // `projectKey` is the tool's unique project identity (Maven: GAV; Gradle: @@ -45,6 +47,7 @@ export type RawNode = { project: string // --with-files only. targets: string[] + unfetched: boolean } export type RawRoot = { @@ -78,6 +81,9 @@ export type ParsedRecords = { // directory-addressed build. entry: string projects: Map + // Excluded from the scan: not projects of their own, but their paths serve + // the kept projects depending on them. + excludedProjects: Map roots: Map scannedConfigs: string[] failures: ResolutionFailure[] @@ -113,12 +119,14 @@ export function parseRecords(text: string): ParsedRecords { buildRoot: '', entry: '', projects: new Map(), + excludedProjects: new Map(), roots: new Map(), scannedConfigs: [], failures: [], unscannable: [], } const scanned = new Set() + const excluded = new Set() const root = (id: string): RawRoot => { let r = result.roots.get(id) @@ -188,6 +196,9 @@ export function parseRecords(text: string): ParsedRecords { project(f[1] ?? '').targets.push(f[2]) } break + case 'excludedProject': + excluded.add(f[1] ?? '') + break case 'projectBuild': if (f[2]) { project(f[1] ?? '').buildFiles.push(f[2]) @@ -215,6 +226,7 @@ export function parseRecords(text: string): ParsedRecords { direct: bool(f[8]), project: f[9] ?? '', targets: [], + unfetched: false, }) break } @@ -226,6 +238,13 @@ export function parseRecords(text: string): ParsedRecords { } break } + case 'unfetched': { + const node = root(f[1] ?? '').nodes.get(f[2] ?? '') + if (node) { + node.unfetched = true + } + break + } case 'file': { const node = root(f[1] ?? '').nodes.get(f[2] ?? '') if (node && f[3]) { @@ -261,6 +280,13 @@ export function parseRecords(text: string): ParsedRecords { break } } + for (const key of excluded) { + const p = result.projects.get(key) + if (p) { + result.projects.delete(key) + result.excludedProjects.set(key, p) + } + } result.scannedConfigs = [...scanned].sort() return result } diff --git a/src/commands/manifest/scripts/resolution-report-gradle.mts b/src/commands/manifest/scripts/resolution-report-gradle.mts index 4e71af729..c3503f74f 100644 --- a/src/commands/manifest/scripts/resolution-report-gradle.mts +++ b/src/commands/manifest/scripts/resolution-report-gradle.mts @@ -21,7 +21,11 @@ export function classifyGradleFailure(detail: string): FailureCategory { ) { return 'no-matching-variant' } - if (t.includes('cannot choose between')) { + // The artifact-level form: several transforms produce the requested variant. + if ( + t.includes('cannot choose between') || + t.includes('found multiple transformation chains') + ) { return 'variant-ambiguity' } if ( @@ -45,8 +49,8 @@ export function classifyGradleFailure(detail: string): FailureCategory { // Every kind blocks except variant-ambiguity: the module demonstrably exists // and is almost always captured via another configuration, so it's benign for -// the SBOM (non-blocking, count only). A module that resolves in NO config -// surfaces as not-found / no-matching-variant, which stay blocking. +// the SBOM (non-blocking, count only). The renderer blocks it anyway when the +// module resolved in no configuration of its project. export const GRADLE_DIALECT: ResolutionDialect = { label: 'Gradle', classify: classifyGradleFailure, @@ -89,6 +93,8 @@ export const GRADLE_DIALECT: ResolutionDialect = { }, { key: 'variant-ambiguity', + header: n => + ` Ambiguous variant — ${n} could not choose a variant, and the module resolved in no other configuration:`, blocking: false, notice: (n, depCount, configCount) => `Skipped ${depCount} dependency(ies) with ambiguous variant selection in ${configCount} configuration(s) — re-run with --verbose for ${n}'s messages.`, diff --git a/src/commands/manifest/scripts/resolution-report-maven.mts b/src/commands/manifest/scripts/resolution-report-maven.mts index 37cd7bceb..25f3800f5 100644 --- a/src/commands/manifest/scripts/resolution-report-maven.mts +++ b/src/commands/manifest/scripts/resolution-report-maven.mts @@ -19,7 +19,12 @@ export function classifyMavenFailure(detail: string): FailureCategory { t.includes('unauthorized') || t.includes('forbidden') || t.includes('peer not authenticated') || - t.includes('certpathbuilderexception') + t.includes('certpathbuilderexception') || + // Maven 3.8.1+ blocks plain-HTTP repositories behind its + // maven-default-http-blocker mirror. + t.includes('blocked mirror for repositories') || + // Offline (-o): no repository was asked, so the artifact may well exist. + t.includes('in offline mode') ) { return 'repository-or-network' } @@ -28,7 +33,10 @@ export function classifyMavenFailure(detail: string): FailureCategory { t.includes('failure to find') || t.includes('could not resolve') || t.includes('no versions available') || - t.includes('not found') + t.includes('not found') || + // maven-resolver's remote repository filters (prefixes, groupId): the + // repository is known not to serve that path or group. + t.includes('not allowed from') ) { return 'not-found' } @@ -36,7 +44,9 @@ export function classifyMavenFailure(detail: string): FailureCategory { if ( t.includes('failed to read artifact descriptor') || t.includes('invalid pom') || - t.includes('could not parse pom') + t.includes('could not parse pom') || + // Maven's warning for a POM its descriptor policy tolerated as invalid. + t.includes('is invalid, transitive dependencies') ) { return 'config-problem' } diff --git a/src/commands/manifest/scripts/resolution-report-render.mts b/src/commands/manifest/scripts/resolution-report-render.mts index 6e2efe3e9..916911a62 100644 --- a/src/commands/manifest/scripts/resolution-report-render.mts +++ b/src/commands/manifest/scripts/resolution-report-render.mts @@ -1,6 +1,7 @@ import { GRADLE_DIALECT } from './resolution-report-gradle.mts' import { SBT_DIALECT } from './resolution-report-ivy.mts' import { MAVEN_DIALECT } from './resolution-report-maven.mts' +import { moduleKey } from './resolution-report.mts' import constants from '../../../constants.mts' import { camelToKebab } from '../../../utils/strings.mts' @@ -43,6 +44,9 @@ export type BuildOption = 'excludeConfigs' | 'ignoreUnresolved' | 'verbose' // Where the invoking command lets the user set each build option, so a remedy // names something they can actually set. export type BuildOptionHints = { + // Reachability scans only: reachability cannot install a dependency that + // `ignoreUnresolved` lets through, so it must be told to continue too. + continueOnInstallErrorsFlag?: string | undefined // Taken as flags; every other option is set in `socketJson`. flags: ReadonlySet // Wizard that writes `socketJson`. @@ -53,6 +57,9 @@ export type BuildOptionHints = { export const DSI_SETUP_COMMAND = 'socket manifest setup --dynamic-sbom-inference' +export const REACH_CONTINUE_ON_INSTALL_ERRORS_FLAG = + '--reach-continue-on-install-errors' + // The single-build manifest commands take every option as a flag. export const FLAG_HINTS: BuildOptionHints = { flags: new Set([ @@ -79,19 +86,27 @@ const RESOLUTION_REPORT_ARTIFACT_LIMIT = 15 const RESOLUTION_REPORT_CONFIG_LIMIT = 20 // Drop a bare "group:name" when a versioned "group:name:v" of the same module -// is also present: the lenient resolver reports both forms for one failure. -function dedupCoords(coords: Iterable): string[] { - const set = new Set(coords) +// failed in the same projects: the lenient resolver reports both forms for one +// failure. +function dedupInfos }>( + infos: T[], +): T[] { const versioned = new Set() - for (const c of set) { - const p = c.split(':') + for (const i of infos) { + const p = i.coord.split(':') if (p.length >= 3) { - versioned.add(`${p[0]}:${p[1]}`) + for (const project of i.projects) { + versioned.add(moduleKey(project, i.coord)) + } } } - return [...set] - .filter(c => c.split(':').length >= 3 || !versioned.has(c)) - .sort() + return infos + .filter( + i => + i.coord.split(':').length >= 3 || + ![...i.projects].every(p => versioned.has(moduleKey(p, i.coord))), + ) + .sort((a, b) => (a.coord < b.coord ? -1 : a.coord > b.coord ? 1 : 0)) } function firstLine(s: string): string { @@ -122,6 +137,21 @@ function inProjects(projects: string | Iterable): string { return list.length ? ` (in ${fmtList(list, 3)})` : '' } +function installErrorsHint( + hints: BuildOptionHints, + ignored: boolean, +): string[] { + const flag = hints.continueOnInstallErrorsFlag + if (!flag) { + return [] + } + return [ + ignored + ? `Reachability analysis cannot install the ignored dependencies; pass ${flag} unless you already do.` + : `If you ignore unresolved dependencies, also pass ${flag}: reachability analysis cannot install them.`, + ] +} + // `excludableConfigs` is empty when excluding configurations cannot clear // every blocking failure. function proceedHint( @@ -164,9 +194,8 @@ function proceedHint( ] } -// Severity is per-kind; the exit-code decision lives in the caller. We do NOT -// cross-reference what resolved elsewhere: the failed selector carries no -// classifier/type, so relating a failed and a succeeded dep is unsound. +// What resolved elsewhere is consulted only by module (group:name): a failed +// selector carries no classifier/type to relate it to one artifact. export function renderResolutionReport( failures: ResolutionFailure[], scannedConfigs: string[], @@ -174,6 +203,7 @@ export function renderResolutionReport( opts: { hints?: BuildOptionHints | undefined ignoreUnresolved?: boolean | undefined + resolvedModules?: ReadonlySet | undefined unscannable?: Unscannable[] | undefined } = {}, ): RenderedResolutionReport { @@ -227,6 +257,20 @@ export function renderResolutionReport( } } const allInfos = [...byKey.values()] + // A non-blocking kind is only benign when its module resolved anyway: one + // that resolved in no configuration of its project is missing from the SBOM. + const resolved = opts.resolvedModules + const blockingInfos = new Set( + allInfos.filter( + info => + isBlocking(info.category) || + (!!resolved && + ![...info.projects].some(p => + resolved.has(moduleKey(p, info.coord)), + )), + ), + ) + const isInfoBlocking = (info: CoordInfo): boolean => blockingInfos.has(info) // A whole-config throw is classified by the same cause rules as a per-dep // failure: ambiguity stays lenient, every other cause is fail-closed. @@ -241,7 +285,7 @@ export function renderResolutionReport( const perDepBlockingConfigs = new Set() for (const info of allInfos) { - if (isBlocking(info.category)) { + if (isInfoBlocking(info)) { for (const c of info.configs) { perDepBlockingConfigs.add(c) } @@ -256,7 +300,7 @@ export function renderResolutionReport( // its project incomplete, so none can be reported as succeeded. const hasBlockingConfiglessFailure = blockingUnscannableProjects.length > 0 || - allInfos.some(i => i.configless && isBlocking(i.category)) + allInfos.some(i => i.configless && isInfoBlocking(i)) // An un-scannable config was attempted but resolved nothing, so it didn't succeed. const succeeded = hasBlockingConfiglessFailure ? [] @@ -264,16 +308,16 @@ export function renderResolutionReport( .filter(c => !blockingConfigs.has(c) && !unscannableConfigs.has(c)) .sort() - const groups = dialect.categories - .map(spec => ({ - spec, - infos: dedupCoords( - allInfos.filter(i => i.category === spec.key).map(i => i.coord), - ).map(c => byKey.get(keyOf(c, spec.key))!), - })) + const groups = dialect.categories.map(spec => ({ + spec, + infos: dedupInfos(allInfos.filter(i => i.category === spec.key)), + })) + const blockingGroups = groups + .map(g => ({ ...g, infos: g.infos.filter(isInfoBlocking) })) + .filter(g => g.infos.length) + const nonBlockingGroups = groups + .map(g => ({ ...g, infos: g.infos.filter(i => !isInfoBlocking(i)) })) .filter(g => g.infos.length) - const blockingGroups = groups.filter(g => g.spec.blocking) - const nonBlockingGroups = groups.filter(g => !g.spec.blocking) const blockingCount = blockingGroups.reduce((n, g) => n + g.infos.length, 0) const hasBlockingFailures = blockingCount > 0 || blockingUnscannable.length > 0 @@ -366,7 +410,10 @@ export function renderResolutionReport( tool, hasBlockingConfiglessFailure ? [] : blockingFailed, ), + ...installErrorsHint(hints, false), ) + } else if (hints.continueOnInstallErrorsFlag) { + out.push('', ...installErrorsHint(hints, true)) } out.push('') out.push( @@ -384,8 +431,8 @@ export function renderResolutionReport( const configCount = new Set(infos.flatMap(i => [...i.configs])).size notices.push(spec.notice(name, infos.length, configCount)) } - // A config-level throw whose cause classifies as variant ambiguity is surfaced, not failed — - // matching the deliberately-lenient per-dep variant-ambiguity policy. + // A config-level throw whose cause classifies as variant ambiguity is surfaced, not failed: many + // configurations are marked resolvable yet too underspecified to resolve. if (nonBlockingUnscannable.length) { const n = new Set(nonBlockingUnscannable.map(u => u.config)).size notices.push( @@ -443,6 +490,7 @@ export function renderResolutionErrorReport( opts: { hints?: BuildOptionHints | undefined ignoreUnresolved?: boolean | undefined + resolvedModules?: ReadonlySet | undefined unscannable?: Unscannable[] | undefined } = {}, ): RenderedResolutionReport { diff --git a/src/commands/manifest/scripts/resolution-report-render.test.mts b/src/commands/manifest/scripts/resolution-report-render.test.mts index 6dd05e42a..dc27cb950 100644 --- a/src/commands/manifest/scripts/resolution-report-render.test.mts +++ b/src/commands/manifest/scripts/resolution-report-render.test.mts @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest' import { DSI_SETUP_COMMAND, + REACH_CONTINUE_ON_INSTALL_ERRORS_FLAG, renderResolutionErrorReport, } from './resolution-report-render.mts' @@ -48,6 +49,21 @@ describe('resolution failure classification', () => { expect(r.nonBlockingNotice).toContain('ambiguous variant') }) + it('classifies an ambiguous Gradle artifact transform as variant ambiguity', () => { + const r = renderResolutionErrorReport( + [ + f( + 'com.example:amb:1.0', + 'Found multiple transformation chains that produce a variant of com.example:amb:1.0', + ), + ], + ['runtimeClasspath'], + 'gradle', + ) + expect(r.hasBlockingFailures).toBe(false) + expect(r.nonBlockingNotice).toContain('ambiguous variant') + }) + it("classifies Gradle's 'unable to find a matching variant' phrasing as blocking no-matching-variant", () => { const r = renderResolutionErrorReport( [ @@ -322,6 +338,26 @@ describe('resolution failure remedies', () => { expect(r.summary).not.toMatch(/--ignore-unresolved|--exclude-configs/) }) + it('tells a reachability scan to also continue on install errors', () => { + const r = renderResolutionErrorReport([mavenMiss], ['compile'], 'maven', { + hints: { + ...socketJsonOnly, + continueOnInstallErrorsFlag: REACH_CONTINUE_ON_INSTALL_ERRORS_FLAG, + }, + }) + expect(r.summary).toContain(' "ignoreUnresolved": true') + expect(r.summary).toContain( + 'If you ignore unresolved dependencies, also pass --reach-continue-on-install-errors: reachability analysis cannot install them.', + ) + }) + + it('does not mention install errors outside a reachability scan', () => { + const r = renderResolutionErrorReport([mavenMiss], ['compile'], 'maven', { + hints: socketJsonOnly, + }) + expect(r.summary).not.toContain('install') + }) + it("names the failing build root's own socket.json", () => { const r = renderResolutionErrorReport([mavenMiss], ['compile'], 'maven', { hints: { ...socketJsonOnly, socketJson: 'tools/codegen/socket.json' }, @@ -370,3 +406,114 @@ describe('resolution failure remedies', () => { expect(r.nonBlockingNotice).not.toContain('re-run with --verbose') }) }) + +describe('failures checked against what resolved', () => { + const ambiguous = f( + 'com.example:amb:1.0', + 'Cannot choose between the following variants of com.example:amb:1.0', + 'runtimeClasspath', + ':app', + ) + + it('keeps variant ambiguity lenient when the module resolved in its project', () => { + const r = renderResolutionErrorReport( + [ambiguous], + ['runtimeClasspath'], + 'gradle', + { resolvedModules: new Set([':app\tcom.example:amb']) }, + ) + expect(r.hasBlockingFailures).toBe(false) + expect(r.nonBlockingNotice).toContain('ambiguous variant selection') + }) + + it('blocks on variant ambiguity when the module resolved nowhere in its project', () => { + const r = renderResolutionErrorReport( + [ambiguous], + ['runtimeClasspath'], + 'gradle', + { resolvedModules: new Set([':lib\tcom.example:amb']) }, + ) + expect(r.hasBlockingFailures).toBe(true) + expect(r.summary).toContain('Ambiguous variant') + expect(r.summary).toContain(' - com.example:amb:1.0 (in `:app`)') + expect(r.nonBlockingNotice).toBe('') + }) + + it('keeps a bare module failure from another project than its versioned form', () => { + const r = renderResolutionErrorReport( + [ + f('com.example:x', 'Could not find com.example:x.', 'compile', ':a'), + f( + 'com.example:x:1.0', + 'Could not find com.example:x:1.0.', + 'compile', + ':b', + ), + ], + ['compile'], + 'gradle', + ) + expect(r.summary).toContain(' - com.example:x (in `:a`)') + expect(r.summary).toContain(' - com.example:x:1.0 (in `:b`)') + }) + + it('still reminds a reachability scan of install errors once failures are ignored', () => { + const r = renderResolutionErrorReport( + [f('com.example:missing:1.0', 'Could not find com.example:missing:1.0.')], + ['runtimeClasspath'], + 'gradle', + { + hints: { + continueOnInstallErrorsFlag: REACH_CONTINUE_ON_INSTALL_ERRORS_FLAG, + flags: new Set(), + setupCommand: 'socket manifest setup', + socketJson: 'socket.json', + }, + ignoreUnresolved: true, + }, + ) + expect(r.summary).toContain('Ignored 1 unresolved dependency(ies)') + expect(r.summary).toContain( + 'Reachability analysis cannot install the ignored dependencies; pass --reach-continue-on-install-errors unless you already do.', + ) + }) +}) + +const mavenReport = (detail: string) => + renderResolutionErrorReport( + [f('com.example:x:jar:1.0', detail, 'compile')], + ['compile'], + 'maven', + ) + +describe('Maven failure classification', () => { + it('classifies a repository blocked by the HTTP blocker as a repository error', () => { + const r = mavenReport( + 'Blocked mirror for repositories: [insecure (http://repo.example.invalid/maven, default, releases+snapshots)]', + ) + expect(r.hasBlockingFailures).toBe(true) + expect(r.summary).toContain('Repository or network error') + }) + + it('classifies offline mode as a repository error, not a miss', () => { + const r = mavenReport( + 'The POM for com.example:x:jar:1.0 is missing, no dependency information available: Cannot access central (https://repo.maven.apache.org/maven2) in offline mode', + ) + expect(r.summary).toContain('Repository or network error') + expect(r.summary).not.toContain('Not found in any repository') + }) + + it('classifies a POM Maven tolerated as invalid as a POM problem', () => { + const r = mavenReport( + 'The POM for com.example:x:jar:1.0 is invalid, transitive dependencies (if any) will not be available: 1 problem was encountered while building the effective model for com.example:x:1.0', + ) + expect(r.summary).toContain('POM/descriptor problem') + }) + + it('classifies a path the repository prefix filter rules out as not found', () => { + const r = mavenReport( + 'prefixes: Path com/example/x/1.0/x-1.0.pom NOT allowed from central', + ) + expect(r.summary).toContain('Not found in any repository') + }) +}) diff --git a/src/commands/manifest/scripts/resolution-report.mts b/src/commands/manifest/scripts/resolution-report.mts index 3a280195a..37244f828 100644 --- a/src/commands/manifest/scripts/resolution-report.mts +++ b/src/commands/manifest/scripts/resolution-report.mts @@ -3,8 +3,8 @@ export type ResolutionFailure = { // Build tool's own failure message (deepest cause; may be multi-line). // Classified on; first line shown by default, whole thing at --verbose. detail: string - // Empty when the failed module is absent from the tool's resolved graph, so - // no configuration holds it. + // Empty when the failure cannot be placed in a configuration; it then counts + // against every configuration of its project. config: string // projectKey of the subproject being resolved. project: string @@ -18,8 +18,16 @@ export type Unscannable = { project: string } +// A module (`group:name`, from any longer coordinate) within one project. +export function moduleKey(project: string, coord: string): string { + return `${project}\t${coord.split(':').slice(0, 2).join(':')}` +} + export type ResolutionReport = { failures: ResolutionFailure[] + // moduleKey of every module a project resolved in some configuration; empty + // when nothing failed. + resolvedModules: ReadonlySet scannedConfigs: string[] unscannable: Unscannable[] } diff --git a/src/commands/manifest/scripts/run.mts b/src/commands/manifest/scripts/run.mts index 5eaf7d25d..bb3ad29a2 100644 --- a/src/commands/manifest/scripts/run.mts +++ b/src/commands/manifest/scripts/run.mts @@ -299,6 +299,9 @@ async function invokeSbtIn( ): Promise { await writeSbtPlugin(globalBase, pluginSourceFilename, pluginDestFilename) const recordsFile = path.join(globalBase, 'records.tsv') + // The global base is shared across sbt roots: a build that dies before + // writing must not be read as the previous root's records. + await fs.rm(recordsFile, { force: true }) const bin = resolveBuildToolBin('sbt', opts.projectDir, opts.bin) // Fresh per-run global base (not ~/.sbt): sbt executes everything under // plugins/, so a shared path is a code-injection surface. BSP off for this diff --git a/src/commands/manifest/scripts/run.test.mts b/src/commands/manifest/scripts/run.test.mts index 3d27007b6..1caf1becb 100644 --- a/src/commands/manifest/scripts/run.test.mts +++ b/src/commands/manifest/scripts/run.test.mts @@ -1,8 +1,12 @@ +import { promises as fs } from 'node:fs' +import os from 'node:os' +import path from 'node:path' + import { describe, expect, it, vi } from 'vitest' import { spawn } from '@socketsecurity/registry/lib/spawn' -import { enumerateWorkspaces } from './run.mts' +import { enumerateWorkspaces, runManifestScript } from './run.mts' vi.mock('@socketsecurity/registry/lib/spawn', () => ({ spawn: vi.fn(async () => ({ code: 0, stderr: '', stdout: '' })), @@ -25,3 +29,39 @@ describe('build tool output', () => { }, ) }) + +describe('shared sbt global base', () => { + it("does not read the previous root's records when a build dies before writing", async () => { + const base = await fs.mkdtemp(path.join(os.tmpdir(), 'sbt-shared-base-')) + try { + const recordsArg = (args: readonly string[]) => + args + .find(a => a.startsWith('-Dsocket.recordsFile='))! + .slice('-Dsocket.recordsFile='.length) + vi.mocked(spawn).mockImplementationOnce((async ( + _bin: string, + args: readonly string[], + ) => { + await fs.writeFile( + recordsArg(args), + 'buildRoot\t/a\nroot\t0\troot\tcompile\t1\nnode\t0\tx:y:jar:1\tx\ty\t1\tjar\t\t1\t\n', + ) + return { code: 0, stderr: '', stdout: '' } + }) as any) + vi.mocked(spawn).mockImplementationOnce((async () => ({ + code: 1, + stderr: 'error: not found: value broken', + stdout: '', + })) as any) + const opts = { bin: 'sbt', stdio: 'pipe' as const, tmpDir: base } + const a = await runManifestScript('sbt', { ...opts, projectDir: base }) + const b = await runManifestScript('sbt', { ...opts, projectDir: base }) + expect(a.facts.components).toHaveLength(1) + expect(b.code).toBe(1) + expect(b.buildRoot).toBeUndefined() + expect(b.facts.components).toHaveLength(0) + } finally { + await fs.rm(base, { force: true, recursive: true }) + } + }) +}) diff --git a/src/commands/manifest/scripts/sidecar.mts b/src/commands/manifest/scripts/sidecar.mts index fb821d8c1..a19b54212 100644 --- a/src/commands/manifest/scripts/sidecar.mts +++ b/src/commands/manifest/scripts/sidecar.mts @@ -53,15 +53,17 @@ export type SidecarAccumulator = Map< { projects: SidecarProjectEntry[]; components: SidecarComponentEntry[] } > -// `[]` means resolved and found nothing (e.g. a pom/BOM with no artifact). +// `[]` means resolved and found nothing (e.g. a pom/BOM with no artifact); no +// `targets` means the artifact did not resolve. function attachPaths( entry: T, artifactPaths: ResolvedArtifactPaths, -): T & { targets: string[]; sources: string[] } { +): T & { targets?: string[]; sources: string[] } { const paths = artifactPaths.pathsById.get(entry.id) + const targets = paths ? paths.targets : [] return { ...entry, - targets: [...(paths?.targets ?? [])], + ...(targets ? { targets: [...targets] } : {}), sources: [...(paths?.sources ?? [])], } } diff --git a/src/commands/manifest/scripts/socket-facts.init.gradle b/src/commands/manifest/scripts/socket-facts.init.gradle index e7b64340d..28adb8f2d 100644 --- a/src/commands/manifest/scripts/socket-facts.init.gradle +++ b/src/commands/manifest/scripts/socket-facts.init.gradle @@ -53,7 +53,6 @@ gradle.ext.socketFactsState = [ // Whole configs whose resolution threw (vs a single unresolved dep in `failures`). unscannable : Collections.synchronizedList([]), scannedConfigs : Collections.synchronizedSet([] as Set), - projectKeys : Collections.synchronizedSet([] as Set), // Only populated with `-Psocket.withFiles=true`: reading `artifact.file` downloads the artifact. paths : Collections.synchronizedMap([:]), perSub : Collections.synchronizedMap([:]), @@ -63,10 +62,12 @@ gradle.ext.socketFactsState = [ projectArtifactExt : Collections.synchronizedMap([:]), // Project path -> "group:name", to tell this build's projects from an included build's. projectGaByPath : Collections.synchronizedMap([:]), + // Project path -> "group:name:version", the coordinate a failed project dependency is reported by. + projectGavByPath : Collections.synchronizedMap([:]), ] -// Capture every project's (group:name) before collectors run so they can filter intra-project -// deps without an ordering dependency on other subprojects. +// Capture every project's identity before collectors run so they can recognize intra-project deps +// without an ordering dependency on other subprojects. gradle.projectsEvaluated { g -> def rootPath = g.rootProject.projectDir.toPath() // '/'-normalized so values are stable across OSes and portable to another machine. @@ -75,7 +76,6 @@ gradle.projectsEvaluated { g -> r.isEmpty() ? '.' : r } g.rootProject.allprojects.each { p -> - g.socketFactsState.projectKeys.add("${p.group ?: ''}:${p.name}") // The module's real artifact extension (jar for java; '' for a project that builds no archive, // e.g. a pure aggregator). Used to stamp intra-project deps that resolve without an artifact. def artExt = '' @@ -90,12 +90,11 @@ gradle.projectsEvaluated { g -> } catch (Exception ignore) {} g.socketFactsState.projectArtifactExt[p.path] = artExt g.socketFactsState.projectGaByPath[p.path] = "${p.group ?: ''}:${p.name}".toString() + g.socketFactsState.projectGavByPath[p.path] = "${p.group ?: ''}:${p.name}:${p.version ?: ''}".toString() // A wholly excluded subproject emits no project record and its configs are never resolved (see the - // collector). projectKeys/projectArtifactExt above stay populated so a KEPT project depending on it - // still recognizes it as a first-party module. - if (g.ext.socketIsExcluded.call(rel(p.projectDir))) { - return - } + // collector). The project maps above stay populated so a KEPT project depending on it + // still recognizes it as a first-party module, and its paths are kept so it is analysed with its code. + def excluded = g.ext.socketIsExcluded.call(rel(p.projectDir)) // Guarded: projects without the java `sourceSets` convention (pure aggregators, AGP-only // Android modules) contribute empty lists rather than failing. // ALL source sets are emitted (not just main/test) so custom ones (integrationTest, @@ -133,6 +132,7 @@ gradle.projectsEvaluated { g -> sources: (sources as List).sort(), targets: (targets as List).sort(), buildFiles: buildFiles, + excluded: excluded, ]) } } @@ -157,13 +157,15 @@ allprojects { project -> // visit/upsertNode closures capture them); `failures`/`scannedConfigs` stay shared. def nodes = [:] def directIds = [] as Set + def configName = '' // "group:name:version" -> path of the build project this config resolved it to. def projectByGav = [:] def failures = state.failures def scannedConfigs = state.scannedConfigs - def projectKeys = state.projectKeys def projectArtifactExt = state.projectArtifactExt def paths = [:] + // Ids of nodes whose requested artifact yielded no file. + def unfetched = [] as Set def withFiles = gradle.socketProp.call(project, 'socket.withFiles')?.toString()?.toLowerCase() == 'true' // Optional `-Psocket.populateFilesFor=` (newline-delimited GAVs): scope `--with-files` // materialization to those GAVs. This run walks EVERY resolvable config (not told the SBOM's @@ -203,10 +205,6 @@ allprojects { project -> [coord.groupId, coord.artifactId, coord.ext, coord.classifier, coord.version].findAll { it }.join(':') } - def isIntraProject = { String group, String name -> - projectKeys.contains("${group ?: ''}:${name}") - } - // A first-party module dep can resolve with no published artifact (classes-dir variant, or // variant-ambiguous moduleArtifacts), leaving it ext-less. Stamp the module's real artifact // extension so it keeps a full, stable coordinate instead of an ext-less one. @@ -234,6 +232,24 @@ allprojects { project -> id } + def causeMessages = { problem -> + def out = [] + def t = problem + int guard = 0 + while (t != null && guard++ < 12) { out << t.message; t = t.cause } + out + } + + // The deepest cause's full message: the TS report classifies it, so nothing is categorized here. + def rawDetail = { problem -> + def messages = causeMessages(problem).findAll { it } + return (messages ? messages[-1] : 'unknown resolution failure').toString().trim() + } + + def recordFailure = { String coord, String detail -> + failures << [coord: coord, detail: detail, config: configName, project: project.path] + } + // Walk a resolved dependency and its transitive closure. We never touch `artifact.file` here — // that forces Gradle to download the file (catastrophic on builds declaring distribution // archives as deps); `artifact.extension`/`artifact.classifier` read already-resolved metadata. @@ -251,15 +267,23 @@ allprojects { project -> // `moduleArtifacts` forces ARTIFACT-level variant selection, which can throw on a // variant-ambiguous dep even when the dependency GRAPH resolves fine. Guard per-dep: on - // failure emit at GAV level and keep walking so one ambiguous artifact can't sink the config. + // failure keep walking so one ambiguous artifact can't sink the config. + def gav = "${dep.moduleGroup ?: ''}:${dep.moduleName}:${dep.moduleVersion ?: ''}".toString() + def shouldMaterialize = withFiles && (populateGavs == null || populateGavs.contains(gav)) + def projPath = projectByGav[gav] def artifacts + def artifactsError = null try { artifacts = dep.moduleArtifacts } catch (Exception e) { artifacts = [] as Set + artifactsError = e } - def projPath = projectByGav["${dep.moduleGroup ?: ''}:${dep.moduleName}:${dep.moduleVersion ?: ''}".toString()] - if (artifacts.isEmpty()) { + if (artifactsError != null && projPath == null) { + // No artifact was selected, so there is no coordinate to emit: report the module and + // bubble its children up (below), as for a skipped first-party module. + recordFailure(gav, rawDetail(artifactsError)) + } else if (artifacts.isEmpty()) { def ext = effExt(projPath, '') // Skip a no-artifact first-party module (aggregator / build root): it builds no archive, so // it's fully described by `projects` and is never a real artifact dependency. @@ -273,17 +297,6 @@ allprojects { project -> ], isProd, projPath) } } else { - // Build the GAV scope key only when a scope is set: the no-scope path never reads it and - // this is the hottest line under --parallel. - def shouldMaterialize - if (!withFiles) { - shouldMaterialize = false - } else if (populateGavs == null) { - shouldMaterialize = true - } else { - def gav = "${dep.moduleGroup ?: ''}:${dep.moduleName}:${dep.moduleVersion ?: ''}".toString() - shouldMaterialize = populateGavs.contains(gav) - } artifacts.each { a -> // Directory variants (java-classes-directory etc.) carry no extension; for a first-party // module fall back to its real artifact ext (never artifact.type, a Gradle variant attr). @@ -310,7 +323,13 @@ allprojects { project -> set << path } } catch (Exception e) { - println "[socket-facts] could not materialize ${aid}: ${e.message?.readLines()?.first()}" + println "[socket-facts] could not materialize ${aid}: ${rawDetail(e).readLines().find()}" + // A requested artifact that yields no file is a failure: without one, the dependency + // would read as resolved with nothing to analyse. + if (projPath == null) { + recordFailure(gav, rawDetail(e)) + unfetched << aid + } } } } @@ -320,9 +339,9 @@ allprojects { project -> dep.children.each { child -> childIds.addAll(visit(child, isProd, cache)) } - // A skipped no-artifact first-party module produced no node, so bubble its resolved children - // up to the parent (return childIds) — otherwise the module's transitives orphan (attached to - // nothing, not bubbled). The module itself is described by `projects`. + // A skipped no-artifact first-party module, or a module whose artifacts could not be + // selected, produced no node, so bubble its resolved children up to the parent (return + // childIds); otherwise the module's transitives would be attached to nothing. if (producedIds.isEmpty()) { cache[dep] = childIds return childIds @@ -408,13 +427,137 @@ allprojects { project -> return true } - // Deepest cause's message, in FULL — Coana classifies/templates it, so we don't categorize here. - def rawDetail = { problem -> - def deepest = null - def t = problem - int guard = 0 - while (t != null && guard++ < 12) { if (t.message) { deepest = t.message }; t = t.cause } - return (deepest ?: 'unknown resolution failure').toString().trim() + // A failed edge is reported by the selector Gradle ATTEMPTED (after forces and substitutions). + // A project of this build is named by its own coordinate, as its resolved node would be. + def selectorCoord = { sel -> + if (sel instanceof org.gradle.api.artifacts.component.ModuleComponentSelector) { + return "${sel.group}:${sel.module}${sel.version ? ':' + sel.version : ''}".toString() + } + if (sel instanceof org.gradle.api.artifacts.component.ProjectComponentSelector) { + def build = sel.metaClass.respondsTo(sel, 'getBuildPath') ? sel.buildPath : + sel.metaClass.respondsTo(sel, 'getBuildName') ? sel.buildName : ':' + def gav = build == ':' ? state.projectGavByPath[sel.projectPath] : null + if (gav != null) { + return gav + } + } + sel.displayName.toString() + } + + // Gradle verifies only artifact-set reads, never the walk's `ResolvedArtifact.file`, and fails + // them with every fatal failure so far; only STRICT throws, from 6.2, with the root build's metadata. + def rootBuild = gradle + while (rootBuild.parent != null) { + rootBuild = rootBuild.parent + } + def startParameter = gradle.startParameter + def verifyArtifacts = withFiles && + new File(rootBuild.rootProject.rootDir, 'gradle/verification-metadata.xml').isFile() && + startParameter.metaClass.respondsTo(startParameter, 'getDependencyVerificationMode') && + startParameter.dependencyVerificationMode.toString() == 'STRICT' + def gavOf = { Map c -> "${c.groupId}:${c.artifactId}:${c.version}".toString() } + def verificationMessage = { Throwable e -> + causeMessages(e).find { it?.startsWith('Dependency verification failed') } + } + // Old Gradle splits a module into one ResolvedDependency per target configuration, of which only + // some carry its artifacts: an artifact-less one is the same component, so fold it into them. + def foldArtifactless = { + if (!nodes.values().any { n -> !n.coord.ext && n.project == null }) { + return + } + def withExt = [:] + nodes.each { id, n -> + if (n.coord.ext) { + withExt.get(gavOf(n.coord), []) << id + } + } + def folds = [:] + nodes.each { id, n -> + def siblings = (!n.coord.ext && n.project == null) ? withExt[gavOf(n.coord)] : null + if (siblings) { + folds[id] = siblings + } + } + if (folds.isEmpty()) { + return + } + def replaced = { Collection ids -> ids.collectMany { c -> folds[c] ?: [c] } } + folds.each { id, siblings -> + def folded = nodes.remove(id) + siblings.each { sid -> + nodes[sid].children.addAll(replaced(folded.children).findAll { it != sid }) + if (folded.prod) { + nodes[sid].prod = true + } + } + } + nodes.each { nid, n -> + if (n.children.any { folds.containsKey(it) }) { + def children = replaced(n.children).findAll { it != nid } + n.children.clear() + n.children.addAll(children) + } + } + def direct = replaced(directIds) + directIds.clear() + directIds.addAll(direct) + } + + // Drops the files of this config's artifacts that failed verification and reports them. + def checkVerification = { cfg -> + def external = paths.keySet().findAll { id -> nodes[id].project == null } + if (external.isEmpty()) { + return + } + def fetched = external.collect { id -> gavOf(nodes[id].coord) } as Set + def message = null + try { + cfg.incoming.artifactView { + lenient(true) + componentFilter { id -> + id instanceof org.gradle.api.artifacts.component.ModuleComponentIdentifier && + fetched.contains("${id.group}:${id.module}:${id.version}".toString()) + } + }.artifacts.each { it.file } + } catch (Exception e) { + message = verificationMessage(e) + if (message == null) { + // Any other failure concerns files the walk already read without it. + println "[socket-facts] could not check artifact verification for ${cfg.name}: ${rawDetail(e).readLines().find()}" + } + } + if (message == null) { + return + } + // The report names each failed artifact by its identifier's display name, + // " (::)", whose file part is `-[-].` + // rather than the cached file's own name. + def failed = [:] + (message =~ /(\S+) \(([^\s():]+):([^\s():]+):([^\s()]+)\)/).each { m -> + failed.get("${m[2]}:${m[3]}:${m[4]}".toString(), [] as Set) << m[1] + } + if (failed.isEmpty()) { + throw new GradleException(message) + } + failed.each { gav, fileNames -> + def ids = external.findAll { id -> gavOf(nodes[id].coord) == gav } + // A failed metadata file (pom, module) names none of the module's artifacts: all are suspect. + // A failed artifact this config did not fetch leaves it untouched. + def metadataOnly = fileNames.every { it.endsWith('.pom') || it.endsWith('.module') } + def matched = ids.findAll { id -> + def c = nodes[id].coord + def named = "${c.artifactId}-${c.version}${c.classifier ? '-' + c.classifier : ''}${c.ext ? '.' + c.ext : ''}" + fileNames.contains(named.toString()) || paths[id].any { p -> fileNames.contains(new File(p).name) } + } + def hit = matched ?: (metadataOnly ? ids : []) + if (!hit.isEmpty()) { + recordFailure(gav, message) + hit.each { id -> + paths.remove(id) + unfetched << id + } + } + } } targetConfigs.each { cfg -> @@ -426,10 +569,13 @@ allprojects { project -> // Fresh tree for THIS resolution root — captured by visit/upsertNode, so no union with siblings. nodes = [:] directIds = [] as Set + configName = cfg.name paths = [:] + unfetched = [] as Set projectByGav = [:] // Per-config try/catch: AGP-style configs can fail variant ambiguity from an init-script // context lacking the consumer attributes AGP sets internally. + def walked = false try { def lenient = cfg.resolvedConfiguration.lenientConfiguration // The resolution graph names a project dependency by project path; within one graph a @@ -453,20 +599,25 @@ allprojects { project -> firstLevel.each { dep -> directIds.addAll(visit(dep, isProd, cache)) } + foldArtifactless() // Unresolved deps drive the root aggregator's abort/warn decision but are NOT emitted as // nodes — their selector-only coordinates would surface as half-formed phantom artifacts. - lenient.unresolvedModuleDependencies.each { dep -> - def sel = dep.selector - if (isIntraProject(sel.group, sel.name)) { - return + // hasError() covers every unresolved dependency, project ones included. + if (cfg.resolvedConfiguration.hasError()) { + def reported = [] as Set + cfg.incoming.resolutionResult.allDependencies.each { d -> + if (d instanceof org.gradle.api.artifacts.result.UnresolvedDependencyResult) { + def coord = selectorCoord(d.attempted ?: d.requested) + if (reported.add(coord)) { + recordFailure(coord, rawDetail(d.failure)) + } + } } - failures << [ - coord : "${sel.group}:${sel.name}${sel.version ? ':' + sel.version : ''}".toString(), - detail: rawDetail(dep.problem), - config: cfg.name, - project: project.path, - ] } + if (verifyArtifacts) { + checkVerification(cfg) + } + walked = true } catch (Exception e) { // A config-LEVEL resolution failure (a resolutionStrategy/beforeResolve rule that throws, // an artifact-transform failure, a broken repository, ...) rather than a per-dep failure. @@ -474,11 +625,11 @@ allprojects { project -> state.unscannable << [config: cfg.name, detail: rawDetail(e), project: project.path] } // Stash this resolution root's tree; the key only has to be unique per root. Skip empty - // configs — they'd contribute nothing. - if (!nodes.isEmpty()) { + // configs, which contribute nothing, and a partial walk, which can orphan components. + if (walked && !nodes.isEmpty()) { synchronized (state.perSub) { state.perSub["${project.path}::${cfg.name}".toString()] = - [nodes: nodes, direct: directIds, paths: paths, projectPath: project.path, config: cfg.name, prod: isProd] + [nodes: nodes, direct: directIds, paths: paths, unfetched: unfetched, projectPath: project.path, config: cfg.name, prod: isProd] } } } @@ -518,8 +669,12 @@ rootProject { rp -> def projectsInfo synchronized (state.projectsInfo) { projectsInfo = new ArrayList(state.projectsInfo) } projectsInfo.each { pi -> - rec(['project', pi.path, pi.group ?: '', pi.name, pi.version ?: '', pi.dir]) - pi.buildFiles.each { f -> rec(['projectBuild', pi.path, f]) } + if (pi.excluded) { + if (withFilesProjects) rec(['excludedProject', pi.path]) + } else { + rec(['project', pi.path, pi.group ?: '', pi.name, pi.version ?: '', pi.dir]) + pi.buildFiles.each { f -> rec(['projectBuild', pi.path, f]) } + } if (withFilesProjects) { pi.sources.each { s -> rec(['projectSrc', pi.path, s]) } pi.targets.each { t -> rec(['projectTgt', pi.path, t]) } @@ -541,6 +696,7 @@ rootProject { rp -> node.children.each { childId -> rec(['edge', rootId, coordId, childId]) } def fs = tree.paths[coordId] if (fs) { (fs as List).sort().each { p -> rec(['file', rootId, coordId, p]) } } + if (tree.unfetched.contains(coordId)) rec(['unfetched', rootId, coordId]) } } diff --git a/src/commands/manifest/scripts/socket-facts.plugin.scala b/src/commands/manifest/scripts/socket-facts.plugin.scala index 2b455276d..758d5352e 100644 --- a/src/commands/manifest/scripts/socket-facts.plugin.scala +++ b/src/commands/manifest/scripts/socket-facts.plugin.scala @@ -63,10 +63,26 @@ object SocketFactsPlugin extends AutoPlugin { // Real artifact ext per build module, so an ext-less inter-project dep gets its true coordinate. val moduleExts = buildModuleExts(allRefs, extracted) + // Every project: `update` also runs each upstream project's update, which would fail strictly. + lazy val lenientState = extracted.append(allRefs.map(r => updateConfiguration.in(r) ~= (c => withMissingOk(c))), st) + val pomsByCache = mutable.HashMap.empty[Option[File], PomPackaging] + allRefs.foreach { ref => if (!isExcludedRef(ref)) { - runUpdateResilient(updateTaskName, ref, extracted, st, matcher, failures, unscannable).foreach { case (report, unresolved) => - foldReport(report, unresolved, ref, extracted, matcher, scannedConfigs, withFiles, populateScope, moduleExts).foreach { + // Built-in lm-coursier (sbt 1.3+) and the legacy sbt-coursier plugin name their caches differently. + val legacyCoursierCache = extracted.getOpt(SettingKey[File]("coursierCache").in(ref)) + val csrCache = extracted.getOpt(SettingKey[File]("csrCacheDirectory").in(ref)).orElse(legacyCoursierCache) + val poms = pomsByCache.getOrElseUpdate(csrCache, new PomPackaging(csrCache)) + // sbt 0.13 has no `useCoursier`: it resolves with Ivy unless the legacy plugin replaces it. + val ivy = extracted.getOpt(SettingKey[Boolean]("useCoursier").in(ref)) match { + case Some(useCoursier) => !useCoursier + case None => legacyCoursierCache.isEmpty + } + val resolver = new Resolver(ivy, poms) + runUpdateResilient(updateTaskName, ref, extracted, st, lenientState, matcher, failures, unscannable, resolver) + .foreach { case (report, unresolved, ivyUnfetched, updateFailed) => + foldReport(report, unresolved, ref, extracted, matcher, scannedConfigs, withFiles, populateScope, moduleExts, + resolver, ivyUnfetched, updateFailed, failures).foreach { case (rootKey, tree) => perSub(rootKey) = tree } } @@ -101,21 +117,24 @@ object SocketFactsPlugin extends AutoPlugin { rec("buildRoot", rootCanonPath.toString) // One `project` record per build module (sources/targets only with --with-files). Excluded - // subprojects are omitted (they were also skipped during resolution above). + // subprojects get none (they were also skipped during resolution above), but keep their paths so + // a kept project depending on one is analysed with its code. allRefs.foreach { ref => - if (!isExcludedRef(ref)) { - val mid = rootIdOf(extracted, ref) - val ver = if (mid.revision == null) "" else mid.revision + val mid = rootIdOf(extracted, ref) + val ver = if (mid.revision == null) "" else mid.revision + if (isExcludedRef(ref)) { + if (withFiles) rec("excludedProject", ref.project) + } else { val baseDir = extracted.get(baseDirectory.in(ref)) rec("project", ref.project, mid.organization, mid.name, ver, relOf(baseDir)) (buildFilesByRef.getOrElse(ref, Nil) ++ baseBuildSbt(baseDir, rootCanonPath).map(relOf)).distinct.sorted .foreach(f => rec("projectBuild", ref.project, f)) - if (withFiles) { - moduleDirs.get(mid.organization + ":" + mid.name + ":" + ver).foreach { - case (sources, targets) => - sources.foreach(s => rec("projectSrc", ref.project, s)) - targets.foreach(t => rec("projectTgt", ref.project, t)) - } + } + if (withFiles) { + moduleDirs.get(mid.organization + ":" + mid.name + ":" + ver).foreach { + case (sources, targets) => + sources.foreach(s => rec("projectSrc", ref.project, s)) + targets.foreach(t => rec("projectTgt", ref.project, t)) } } } @@ -135,6 +154,7 @@ object SocketFactsPlugin extends AutoPlugin { node.project) node.children.foreach(ch => rec("edge", rootId, coordId, ch)) node.targets.foreach(p => rec("file", rootId, coordId, p)) + if (node.unfetched) rec("unfetched", rootId, coordId) } } @@ -194,17 +214,19 @@ object SocketFactsPlugin extends AutoPlugin { }.toMap } - // When a failure names modules, a lenient (missingOk) retry keeps everything else and, where the - // resolver still places a failed module, its configurations. Failed modules are reported, not emitted. + // A failure naming no module is an artifact that could not be fetched, which only the lenient + // report's missing artifacts reveal; with neither, the project is unscannable. private def runUpdateResilient( taskName: String, ref: ProjectRef, extracted: Extracted, state: State, + lenient: => State, matcher: String => Boolean, failures: mutable.LinkedHashSet[Failure], - unscannable: mutable.LinkedHashSet[Unscannable] - ): Option[(UpdateReport, Set[String])] = { + unscannable: mutable.LinkedHashSet[Unscannable], + resolver: Resolver + ): Option[(UpdateReport, Set[String], IvyUnfetched, Boolean)] = { val key = TaskKey[UpdateReport](taskName) def runOn(st: State): Either[(Seq[ModuleID], String), UpdateReport] = @@ -215,14 +237,18 @@ object SocketFactsPlugin extends AutoPlugin { } runOn(state) match { - case Right(rep) => Some((rep, Set.empty[String])) - case Left((failed, detail)) if failed.isEmpty => - unscannable += Unscannable(detail, ref.project) - None + case Right(rep) => Some((rep, Set.empty[String], Map.empty: IvyUnfetched, false)) case Left((failed, detail)) => - val lenient = extracted.append(Seq(updateConfiguration.in(ref) ~= (c => withMissingOk(c))), state) - runOn(lenient) match { - case Right(rep) => + val lenientRun = runOn(lenient).right.map { rep => + (rep, if (resolver.ivy) ivyUnfetchedOf(rep, extracted.get(target.in(ref))) else Map.empty: IvyUnfetched) + } + lenientRun match { + case Right((rep, ivyUnfetched)) + if failed.isEmpty && ivyUnfetched.isEmpty && + !rep.configurations.exists(_.modules.exists(m => !m.evicted && resolver.missingOf(m).nonEmpty)) => + unscannable += Unscannable(detail, ref.project) + None + case Right((rep, ivyUnfetched)) => val confsByGav = mutable.HashMap.empty[String, mutable.LinkedHashSet[String]] rep.configurations.foreach { cr => cr.modules.foreach { r => @@ -236,7 +262,7 @@ object SocketFactsPlugin extends AutoPlugin { case Some(confs) => confs.filter(matcher).foreach(c => failures += Failure(coordOf(m), detailFor(m, detail), c, ref.project)) } } - Some((rep, failed.map(gavKey).toSet)) + Some((rep, failed.map(gavKey).toSet, ivyUnfetched, failed.isEmpty)) case Left(_) => // Even a lenient update failed: the whole project is unresolved, but the modules sbt // named are still reported. @@ -247,6 +273,77 @@ object SocketFactsPlugin extends AutoPlugin { } } + // config -> "org:name:rev" -> artifacts Ivy failed to download. + private type IvyUnfetched = Map[String, Map[String, Seq[IvyArtifact]]] + + private final case class IvyArtifact(ext: String, classifier: String, detail: String) + + // sbt 0.13's UpdateReport omits artifacts Ivy failed to download; Ivy's own report per configuration, + // `/reports/--.xml`, keeps them. + private def ivyUnfetchedOf(rep: UpdateReport, projectTarget: File): IvyUnfetched = { + val descriptor = Option(rep.cachedDescriptor).filter(_.isFile) + val bound = projectTarget.getCanonicalFile.toPath + def reportsDirs(f: File): Stream[File] = + Option(f.getParentFile).filter(_.getCanonicalFile.toPath.startsWith(bound)).toStream + .flatMap(p => new File(p, "reports") #:: reportsDirs(p)) + (for { + d <- descriptor.toSeq + id <- ivyResolveId(d).toSeq + cr <- rep.configurations + conf = confName(cr) + report <- reportsDirs(d).map(new File(_, id + "-" + conf + ".xml")).find(_.isFile).toSeq + arts = ivyFailedArtifacts(report) + if arts.nonEmpty + } yield conf -> arts).toMap + } + + private def parseXml(f: File): org.w3c.dom.Document = { + val dbf = javax.xml.parsers.DocumentBuilderFactory.newInstance() + dbf.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true) + dbf.newDocumentBuilder().parse(f) + } + + // Ivy's ResolveOptions.getDefaultResolveId: `-` of the resolved descriptor. + private def ivyResolveId(descriptor: File): Option[String] = + try { + val info = parseXml(descriptor).getElementsByTagName("info") + if (info.getLength == 0) None + else { + val e = info.item(0).asInstanceOf[org.w3c.dom.Element] + Some(e.getAttribute("organisation") + "-" + e.getAttribute("module")) + } + } catch { case _: Exception => None } + + private def ivyFailedArtifacts(report: File): Map[String, Seq[IvyArtifact]] = + try { + val modules = parseXml(report).getElementsByTagName("module") + val out = for { + mi <- 0 until modules.getLength + module = modules.item(mi).asInstanceOf[org.w3c.dom.Element] + revisions = module.getElementsByTagName("revision") + ri <- 0 until revisions.getLength + revision = revisions.item(ri).asInstanceOf[org.w3c.dom.Element] + if revision.getAttribute("evicted").isEmpty + artifacts = revision.getElementsByTagName("artifact") + ai <- 0 until artifacts.getLength + a = artifacts.item(ai).asInstanceOf[org.w3c.dom.Element] + if a.getAttribute("status") == "failed" + } yield { + val (org, name, rev) = (module.getAttribute("organisation"), module.getAttribute("name"), revision.getAttribute("name")) + val classifier = a.getAttribute("extra-classifier") + val file = a.getAttribute("name") + (if (classifier.isEmpty) "" else "-" + classifier) + "." + a.getAttribute("ext") + // An absent artifact is "missing artifact" (empty under sbt 0.13's Ivy); one the resolver chain + // failed itself has "null"; anything else is a transfer failure's message. + val details = a.getAttribute("details") + val cause = + if (details.isEmpty || details == "missing artifact") "not found" + else if (details == "null") "download failed" + else "download failed (" + details + ")" + (org + ":" + name + ":" + rev) -> IvyArtifact(a.getAttribute("ext"), classifier, cause + ": " + org + "#" + name + ";" + rev + "!" + file) + } + out.groupBy(_._1).map { case (gav, pairs) => gav -> pairs.map(_._2).distinct } + } catch { case _: Exception => Map.empty } + // UpdateConfiguration has `withMissingOk` on sbt 1.x; on 0.13 its only Boolean field is missingOk. private def withMissingOk[T](c: T): T = { val cls = c.getClass @@ -307,7 +404,13 @@ object SocketFactsPlugin extends AutoPlugin { scannedConfigs: mutable.LinkedHashSet[String], withFiles: Boolean, populateScope: Option[Set[String]], - moduleExts: Map[String, String] + moduleExts: Map[String, String], + resolver: Resolver, + ivyUnfetched: IvyUnfetched, + // sbt's own update failed on artifacts alone, so one it could not fetch is a failure even + // without files. + updateFailed: Boolean, + failures: mutable.LinkedHashSet[Failure] ): mutable.LinkedHashMap[String, RootTree] = { val perRoot = mutable.LinkedHashMap.empty[String, RootTree] val rootKey = moduleKey(rootIdOf(extracted, ref)) @@ -386,10 +489,18 @@ object SocketFactsPlugin extends AutoPlugin { cr.modules.foreach { m => if (emittable(m)) { val ids = midToIds.getOrElseUpdate(moduleKey(m.module), mutable.LinkedHashSet.empty[String]) - variantsOf(m).foreach { case (coord, fileOpt) => - val node = nodes.getOrElseUpdate(coord.id, new Node(coord)) - ids += coord.id - if (withFiles && inScope(m.module)) fileOpt.foreach(f => node.targets += f.getAbsolutePath) + val ivyMissing = + if (ivyUnfetched.isEmpty) Nil + else ivyUnfetched.getOrElse(cfg, Map.empty[String, Seq[IvyArtifact]]).getOrElse(gavKey(m.module), Nil) + val recordFiles = withFiles && inScope(m.module) + variantsOf(m, resolver, ivyMissing).foreach { v => + val node = nodes.getOrElseUpdate(v.coord.id, new Node(v.coord)) + ids += v.coord.id + if (recordFiles) v.file.foreach(f => node.targets += f.getAbsolutePath) + v.unfetched.foreach { url => + if (recordFiles) node.unfetched = true + if (recordFiles || updateFailed) failures += Failure(coordOf(m.module), url, cfg, ref.project) + } } } } @@ -419,22 +530,85 @@ object SocketFactsPlugin extends AutoPlugin { perRoot } - private def variantsOf(m: ModuleReport): Seq[(Coord, Option[File])] = { + private def variantsOf(m: ModuleReport, resolver: Resolver, ivyMissing: Seq[IvyArtifact]): Seq[Variant] = { val mid = m.module // `mid.revision` is normally set for a resolved module, but normalize null (Coord.id calls // `.nonEmpty`, which NPEs on null) to match the other call sites (gavKey, module records). val ver = if (mid.revision == null) "" else mid.revision - val arts = m.artifacts - if (arts == null || arts.isEmpty) - // No resolved artifact (e.g. a pom-only module): leave it ext-less. - Seq((Coord(mid.organization, mid.name, ver, "", ""), None)) - else - arts.toList.map { - case (art, file) => - (Coord(mid.organization, mid.name, ver, extOf(art), classifierOf(art)), Option(file)) + def coordFor(art: Artifact) = Coord(mid.organization, mid.name, ver, extOf(art), classifierOf(art)) + val fetched = Option(m.artifacts).toList.flatten.map { case (art, file) => Variant(coordFor(art), Option(file), None) } + val missing = resolver.missingOf(m).map { art => + Variant(coordFor(art), None, Some("not found: " + art.url.map(_.toString).getOrElse(coordFor(art).id))) + } ++ ivyMissing.map(a => Variant(Coord(mid.organization, mid.name, ver, a.ext, a.classifier), None, Some(a.detail))) + if (fetched.isEmpty && missing.isEmpty) + // No artifact expected (a pom-only module): leave it ext-less. + Seq(Variant(Coord(mid.organization, mid.name, ver, "", ""), None, None)) + else fetched ++ missing + } + + // Artifacts the resolver expected and did not fetch. Under Ivy they come from its resolution report + // instead (see ivyUnfetchedOf). + private final class Resolver(val ivy: Boolean, poms: PomPackaging) { + def missingOf(m: ModuleReport): List[Artifact] = + if (ivy) Nil else Option(m.missingArtifacts).toList.flatten.filterNot(poms.isUnexpectedDefault) + } + + // Coursier requests the default jar of every Maven module and skips it silently when absent unless + // the POM declares a jar packaging. Only the module's own POM tells whether its artifact is a jar + // (packaging defaults to jar, so a parent's never applies); an unreadable POM counts as one. + private final class PomPackaging(csrCacheDir: Option[File]) { + private val memo = mutable.HashMap.empty[String, Boolean] + + def isUnexpectedDefault(a: Artifact): Boolean = { + val ext = extOf(a) + a.classifier.isEmpty && ext == "jar" && a.url.map(_.toString).exists { url => + url.endsWith("." + ext) && memo.getOrElseUpdate(url, { + val pomUrl = url.dropRight(ext.length) + "pom" + coursierLocalFiles(pomUrl, csrCacheDir).find(_.isFile).flatMap(pomPackaging).exists(p => !JarPackagings(p)) + }) } + } } + // coursier.maven.MavenAttributes.typeExtensions (2.1.24, which adds ejb): the packagings whose + // artifact is a `.jar`. Any other packaging (pom, aar, war, ...) publishes its own artifact. + private val JarPackagings = Set("eclipse-plugin", "maven-plugin", "hk2-jar", "orbit", "scala-jar", "jar", "bundle", + "doc", "src", "test-jar", "ejb-client", "ejb") + + // coursier.paths.CachePath.localFile. A repository with credentials is cached under `user@host`, + // and the update report does not carry the user. + private def coursierLocalFiles(url: String, csrCacheDir: Option[File]): Seq[File] = + if (url.startsWith("file:/")) + try Seq(new File(new java.net.URI(url))) + catch { case _: Exception => Nil } + else { + val i = url.indexOf(':') + val protocol = if (i < 0) "" else url.substring(0, i) + val rest = if (i < 0) "" else url.substring(i + 1).dropWhile(_ == '/') + csrCacheDir.toSeq.flatMap { dir => + val protocolDir = new File(dir, coursierEscape(protocol)) + val host = coursierEscape(rest.takeWhile(_ != '/')) + val path = coursierEscape(rest.dropWhile(_ != '/').drop(1)) + val withUser = Option(protocolDir.listFiles).toSeq.flatten.filter(_.getName.endsWith("%40" + host)) + (new File(protocolDir, host) +: withUser).map(new File(_, path)) + } + } + + private def coursierEscape(s: String): String = + s.flatMap { ch => + if (ch > 128 || " %$&+,:;=?@<>#".indexOf(ch) >= 0) "%" + "%02X".format(ch.toInt) else ch.toString + } + + // Maven's default packaging is jar. + private def pomPackaging(pom: File): Option[String] = + try { + val kids = parseXml(pom).getDocumentElement.getChildNodes + val declared = (0 until kids.getLength).map(kids.item).collectFirst { + case e: org.w3c.dom.Element if e.getNodeName.split(':').last == "packaging" => e.getTextContent.trim + } + Some(declared.filter(_.nonEmpty).getOrElse("jar")) + } catch { case _: Exception => None } + // ---- config selection --------------------------------------------------- // With no includes the default is ALL configurations (captures build/tooling deps). @@ -618,8 +792,13 @@ object SocketFactsPlugin extends AutoPlugin { var project = "" // External artifact's resolved jar(s); --with-files only. val targets = mutable.TreeSet.empty[String] + // An expected artifact was not fetched; --with-files only. + var unfetched = false } + // `unfetched`: why the expected artifact could not be fetched. + private final case class Variant(coord: Coord, file: Option[File], unfetched: Option[String]) + private final class RootTree( val projectKey: String, val config: String, diff --git a/src/commands/manifest/scripts/test/gradle-compat/.gitignore b/src/commands/manifest/scripts/test/gradle-compat/.gitignore index 45d17de21..08d008ad2 100644 --- a/src/commands/manifest/scripts/test/gradle-compat/.gitignore +++ b/src/commands/manifest/scripts/test/gradle-compat/.gitignore @@ -6,3 +6,9 @@ project/.socket.facts.json project/.gradle/ project/build/ .populate-for.txt +failures/localrepo/ +failures/records.tsv +failures/.gradle/ +failures/build/ +failures/lib/build/ +failures/app/build/ diff --git a/src/commands/manifest/scripts/test/gradle-compat/failures/app/build.gradle b/src/commands/manifest/scripts/test/gradle-compat/failures/app/build.gradle new file mode 100644 index 000000000..6dc323bca --- /dev/null +++ b/src/commands/manifest/scripts/test/gradle-compat/failures/app/build.gradle @@ -0,0 +1,8 @@ +// Kept out of the root project: on some old Gradle, a project dependency on a missing configuration +// fails resolving the declaring project's other configurations too. +configurations { + projectFail +} +dependencies { + projectFail project(path: ':lib', configuration: 'nonexistent') +} diff --git a/src/commands/manifest/scripts/test/gradle-compat/failures/build.gradle b/src/commands/manifest/scripts/test/gradle-compat/failures/build.gradle new file mode 100644 index 000000000..f3103d021 --- /dev/null +++ b/src/commands/manifest/scripts/test/gradle-compat/failures/build.gradle @@ -0,0 +1,57 @@ +// Hermetic fixture for ../smoke-test-failures.sh: each configuration fails to resolve in one way, +// against a generated local Maven repo (see ../../make-stub-repo.sh). +allprojects { + repositories { + maven { url rootProject.file('localrepo').toURI() } + } +} + +configurations { + forced + shadowing + verified + walkAborted + quiet +} +configurations.forced.resolutionStrategy.force 'demo:c:9.9' + +dependencies { + forced 'demo:a:1.0' + // Shares the group:name of project :lib, but is an external module. + shadowing 'demo:lib:7.7' + verified 'demo:a:1.0' + verified 'demo:tampered:1.0' + walkAborted 'demo:a:1.0' + walkAborted 'demo:boom:1.0' + quiet 'demo:quiet:1.0' +} + +// Artifact transforms and dependency verification exist only on newer Gradle. +if (org.gradle.util.GradleVersion.current() >= org.gradle.util.GradleVersion.version('6.2')) { + apply from: 'transforms.gradle' +} + +// No build throws from the walk's own reads on demand, so make two of them throw for one module +// each: boom's children, and quiet's file with an empty message. +tasks.matching { it.name == 'socketFactsCollect' }.all { t -> + t.doFirst { + def boom = configurations.walkAborted.resolvedConfiguration.firstLevelModuleDependencies.find { it.moduleName == 'boom' } + def depMeta = boom.getClass().metaClass + def children = depMeta.getMetaMethod('getChildren', [] as Object[]) + depMeta.getChildren = { -> + if (delegate.moduleName == 'boom') { + throw new IllegalStateException('walk aborted by the fixture') + } + children.invoke(delegate) + } + def quiet = configurations.quiet.resolvedConfiguration.resolvedArtifacts.find { it.name == 'quiet' } + def artifactMeta = quiet.getClass().metaClass + def file = artifactMeta.getMetaMethod('getFile', [] as Object[]) + artifactMeta.getFile = { -> + if (delegate.name == 'quiet') { + throw new RuntimeException('') + } + file.invoke(delegate) + } + } +} diff --git a/src/commands/manifest/scripts/test/gradle-compat/failures/gradle/verification-metadata.xml b/src/commands/manifest/scripts/test/gradle-compat/failures/gradle/verification-metadata.xml new file mode 100644 index 000000000..6b71080b3 --- /dev/null +++ b/src/commands/manifest/scripts/test/gradle-compat/failures/gradle/verification-metadata.xml @@ -0,0 +1,18 @@ + + + + + false + false + + + + + + + + + + + + diff --git a/src/commands/manifest/scripts/test/gradle-compat/failures/lib/build.gradle b/src/commands/manifest/scripts/test/gradle-compat/failures/lib/build.gradle new file mode 100644 index 000000000..82fbf7fc9 --- /dev/null +++ b/src/commands/manifest/scripts/test/gradle-compat/failures/lib/build.gradle @@ -0,0 +1,3 @@ +apply plugin: 'java' +group = 'demo' +version = '3.0' diff --git a/src/commands/manifest/scripts/test/gradle-compat/failures/settings.gradle b/src/commands/manifest/scripts/test/gradle-compat/failures/settings.gradle new file mode 100644 index 000000000..53f191a18 --- /dev/null +++ b/src/commands/manifest/scripts/test/gradle-compat/failures/settings.gradle @@ -0,0 +1,2 @@ +rootProject.name = 'gradle-compat-failures' +include 'lib', 'app' diff --git a/src/commands/manifest/scripts/test/gradle-compat/failures/transforms.gradle b/src/commands/manifest/scripts/test/gradle-compat/failures/transforms.gradle new file mode 100644 index 000000000..e688dc60e --- /dev/null +++ b/src/commands/manifest/scripts/test/gradle-compat/failures/transforms.gradle @@ -0,0 +1,25 @@ +// Two transforms producing the same artifact type, so selecting a `foo` artifact is ambiguous even +// though the graph resolves. +abstract class IdentityOne implements org.gradle.api.artifacts.transform.TransformAction { + @org.gradle.api.artifacts.transform.InputArtifact + abstract Provider getInputArtifact() + void transform(org.gradle.api.artifacts.transform.TransformOutputs outputs) { outputs.file(inputArtifact.get().asFile) } +} +abstract class IdentityTwo implements org.gradle.api.artifacts.transform.TransformAction { + @org.gradle.api.artifacts.transform.InputArtifact + abstract Provider getInputArtifact() + void transform(org.gradle.api.artifacts.transform.TransformOutputs outputs) { outputs.file(inputArtifact.get().asFile) } +} +def artifactType = Attribute.of('artifactType', String) +dependencies { + registerTransform(IdentityOne) { from.attribute(artifactType, 'jar'); to.attribute(artifactType, 'foo') } + registerTransform(IdentityTwo) { from.attribute(artifactType, 'jar'); to.attribute(artifactType, 'foo') } +} +configurations { + ambiguous { + attributes { attribute(artifactType, 'foo') } + } +} +dependencies { + ambiguous 'demo:a:1.0' +} diff --git a/src/commands/manifest/scripts/test/gradle-compat/smoke-test-failures.sh b/src/commands/manifest/scripts/test/gradle-compat/smoke-test-failures.sh new file mode 100644 index 000000000..4e29a7426 --- /dev/null +++ b/src/commands/manifest/scripts/test/gradle-compat/smoke-test-failures.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# Run socket-facts.init.gradle against the `failures` project, whose configurations each fail to +# resolve in one way, and assert every failure is reported and no partial or fabricated component +# is emitted: +# - a forced version that doesn't exist is reported as that version, not the one requested; +# - a failed project dependency, and an external module sharing a project's group:name, are reported; +# - an artifact failing dependency verification (Gradle 6.2+) is reported and has no file; +# - a module whose artifact can't be selected is reported with no component, with or without +# --with-files; +# - a walk that throws midway leaves no resolution root, only an unscannable configuration; +# - a file that can't be read with an empty message is reported like any other. +# +# Usage: smoke-test-failures.sh /path/to/gradle +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +GRADLE="${1:?usage: smoke-test-failures.sh }" +INIT="$HERE/../../socket-facts.init.gradle" +PROJECT="$HERE/failures" +RECORDS="$PROJECT/records.tsv" +# shellcheck source=SCRIPTDIR/../compat-cache.sh +. "$HERE/../compat-cache.sh" +GUH="$SOCKET_COMPAT_CACHE/gradle-home" + +bash "$HERE/../make-stub-repo.sh" "$PROJECT/localrepo" \ + 'demo:a:1.0+demo:c:1.0' 'demo:c:1.0' 'demo:boom:1.0+demo:c:1.0' 'demo:quiet:1.0' 'demo:tampered:1.0' + +run() { + rm -rf "$GUH" "$RECORDS" "$PROJECT/.gradle" "$PROJECT/build" "$PROJECT/lib/build" "$PROJECT/app/build" + ( cd "$PROJECT" && "$GRADLE" --no-daemon --offline -g "$GUH" \ + --init-script "$INIT" -Psocket.recordsFile="$RECORDS" "$@" socketFacts -q ) > /dev/null +} + +echo "+ $("$GRADLE" --version 2>/dev/null | sed -n 's/^Gradle //p' | head -1) (failures)" +for files in true false; do + run -Psocket.withFiles="$files" + python3 - "$RECORDS" "$files" <<'PY' +import sys +rows = [l.rstrip('\n').split('\t') for l in open(sys.argv[1]) if l.strip()] +with_files = sys.argv[2] == 'true' +version = next(r[2] for r in rows if r[0] == 'meta') +modern = tuple(int(p) for p in version.split('.')[:2]) >= (6, 2) +roots = {r[1]: r[3] for r in rows if r[0] == 'root'} +nodes = {(roots[r[1]], r[2]): r for r in rows if r[0] == 'node'} +files = {(roots[r[1]], r[2]) for r in rows if r[0] == 'file'} +unfetched = {(roots[r[1]], r[2]) for r in rows if r[0] == 'unfetched'} +failures = {(r[3], r[1]) for r in rows if r[0] == 'failure'} +unscannable = {r[1] for r in rows if r[0] == 'unscannable'} +errors = [] + +def expect(cond, msg): + if not cond: + errors.append(msg) + +expect(('forced', 'demo:c:9.9') in failures, f"forced: missing failure for the forced demo:c:9.9: {sorted(failures)}") +expect(('forced', 'demo:c:1.0') not in failures, "forced: failure names the requested demo:c:1.0, not the attempted 9.9") +# Old Gradle fails the whole configuration instead of the one dependency. +expect(('projectFail', 'demo:lib:3.0') in failures or 'projectFail' in unscannable, + f"projectFail: failed project dependency not reported: {sorted(failures)}") +expect(('shadowing', 'demo:lib:7.7') in failures, f"shadowing: external module sharing :lib's group:name not reported: {sorted(failures)}") +expect(not [n for n in nodes.values() if not n[6]], f"ext-less component emitted: {[n[2] for n in nodes.values() if not n[6]]}") +expect('walkAborted' in unscannable, f"walkAborted: walk that threw not reported unscannable: {sorted(unscannable)}") +expect('walkAborted' not in roots.values(), "walkAborted: partial tree of a walk that threw was emitted") +if modern: + # Only some Gradle versions select legacy-API artifacts through the transforms; the others return + # the untransformed jar. Either way, a module is emitted with an artifact or reported, never both. + ambiguous = [k for k in nodes if k[0] == 'ambiguous'] + reported = ('ambiguous', 'demo:a:1.0') in failures + expect(reported != bool(ambiguous), f"ambiguous: components {ambiguous}, reported={reported}") + +tampered = ('verified', 'demo:tampered:jar:1.0') +if with_files: + expect(('quiet', 'demo:quiet:1.0') in failures, f"quiet: unreadable file not reported: {sorted(failures)}") + expect(('quiet', 'demo:quiet:jar:1.0') in unfetched, f"quiet: unreadable file not marked unfetched: {sorted(unfetched)}") + expect('quiet' not in unscannable, "quiet: unreadable file sank the whole configuration") + expect(('verified', 'demo:a:jar:1.0') in files, f"verified: trusted demo:a has no file: {sorted(files)}") + if modern: + expect(('verified', 'demo:tampered:1.0') in failures, f"verified: verification failure not reported: {sorted(failures)}") + expect(tampered in unfetched, f"verified: tampered jar not marked unfetched: {sorted(unfetched)}") + expect(tampered not in files, "verified: tampered jar emitted as a file") + else: + expect(tampered in files, "verified: no verification before Gradle 6.2, so the jar must materialize") +else: + expect(not files and not unfetched, "file or unfetched record emitted without --with-files") + +if errors: + print("FAIL:") + for e in errors: print(" -", e) + sys.exit(1) +print(f"PASS (failures, withFiles={sys.argv[2]}): every failure reported, no partial or fabricated component") +PY + python3 "$HERE/../assert-reachability.py" "$RECORDS" +done diff --git a/src/commands/manifest/scripts/test/gradle-compat/smoke-test.sh b/src/commands/manifest/scripts/test/gradle-compat/smoke-test.sh index 7ec20c7e1..9fb864b51 100755 --- a/src/commands/manifest/scripts/test/gradle-compat/smoke-test.sh +++ b/src/commands/manifest/scripts/test/gradle-compat/smoke-test.sh @@ -9,7 +9,8 @@ # - every resolved dependency gets an on-disk jar `file` record under -Psocket.withFiles (incl. the # production dep reached via the legacy `compile` config on old Gradle); # - every component is reachable from a direct dependency of its own root (no orphaned components); -# - -Psocket.populateFilesFor scopes materialization to a single GAV. +# - -Psocket.populateFilesFor scopes materialization to a single GAV; +# - a jar that cannot be fetched is reported as a failure, and the other jars still materialize. # # Usage: smoke-test.sh /path/to/gradle set -euo pipefail @@ -23,11 +24,15 @@ RECORDS="$PROJECT/records.tsv" GUH="$SOCKET_COMPAT_CACHE/gradle-home" # isolated Gradle user home -> hermetic, no global init scripts bash "$HERE/../make-stub-repo.sh" "$PROJECT/localrepo" 'demo.lib:foo:1.0' 'demo.test:bar:1.0' -rm -rf "$GUH" "$RECORDS" "$PROJECT/.gradle" "$PROJECT/build" + +run() { + rm -rf "$GUH" "$RECORDS" "$PROJECT/.gradle" "$PROJECT/build" + ( cd "$PROJECT" && "$GRADLE" --no-daemon --offline -g "$GUH" \ + --init-script "$INIT" -Psocket.withFiles=true -Psocket.recordsFile="$RECORDS" "$@" socketFacts -q ) +} echo "+ $("$GRADLE" --version 2>/dev/null | sed -n 's/^Gradle //p' | head -1)" -( cd "$PROJECT" && "$GRADLE" --no-daemon --offline -g "$GUH" \ - --init-script "$INIT" -Psocket.withFiles=true -Psocket.recordsFile="$RECORDS" socketFacts -q ) +run python3 - "$RECORDS" 'full' <<'PY' import sys @@ -72,9 +77,7 @@ python3 "$HERE/../assert-reachability.py" "$RECORDS" # Second run: scope --with-files to a single GAV and assert ONLY that artifact is materialized. SCOPE="$HERE/.populate-for.txt" printf 'demo.lib:foo:1.0\n' > "$SCOPE" -rm -rf "$GUH" "$RECORDS" "$PROJECT/.gradle" "$PROJECT/build" -( cd "$PROJECT" && "$GRADLE" --no-daemon --offline -g "$GUH" \ - --init-script "$INIT" -Psocket.withFiles=true -Psocket.populateFilesFor="$SCOPE" -Psocket.recordsFile="$RECORDS" socketFacts -q ) +run -Psocket.populateFilesFor="$SCOPE" rm -f "$SCOPE" python3 - "$RECORDS" 'scoped' <<'PY' @@ -96,3 +99,28 @@ print("PASS (populateFilesFor scoping): foo materialized, bar skipped") PY python3 "$HERE/../assert-reachability.py" "$RECORDS" + +# Third run: bar's jar is gone from the repository, so it must be reported rather than emitted as +# resolved with no file. +find "$PROJECT/localrepo/demo/test/bar" -name '*.jar' -delete +run > /dev/null + +python3 - "$RECORDS" <<'PY' +import sys +rows = [l.rstrip('\n').split('\t') for l in open(sys.argv[1]) if l.strip()] +files = {r[2] for r in rows if r[0] == 'file'} +failures = {r[1] for r in rows if r[0] == 'failure'} +unfetched = {r[2] for r in rows if r[0] == 'unfetched'} +errors = [] +if 'demo.lib:foo:jar:1.0' not in files: + errors.append(f"missing jar run: foo must still be materialized: {sorted(files)}") +if 'demo.test:bar:1.0' not in failures: + errors.append(f"missing jar run: bar's jar cannot be fetched, so it must be reported: {sorted(failures)}") +if unfetched != {'demo.test:bar:jar:1.0'}: + errors.append(f"missing jar run: only bar's jar is unfetched: {sorted(unfetched)}") +if errors: + print("FAIL:") + for e in errors: print(" -", e) + sys.exit(1) +print("PASS (missing jar): bar reported, foo materialized") +PY diff --git a/src/commands/manifest/scripts/test/maven-compat/.gitignore b/src/commands/manifest/scripts/test/maven-compat/.gitignore index 2923ea429..9867fa6a1 100644 --- a/src/commands/manifest/scripts/test/maven-compat/.gitignore +++ b/src/commands/manifest/scripts/test/maven-compat/.gitignore @@ -13,3 +13,6 @@ duplicate-failure/target/ duplicate-failure/*/target/ graph-failure/records.tsv graph-failure/target/ +descriptor-failure/records.tsv +descriptor-failure/target/ +descriptor-failure/*/target/ diff --git a/src/commands/manifest/scripts/test/maven-compat/descriptor-failure/a/pom.xml b/src/commands/manifest/scripts/test/maven-compat/descriptor-failure/a/pom.xml new file mode 100644 index 000000000..e2be25e7e --- /dev/null +++ b/src/commands/manifest/scripts/test/maven-compat/descriptor-failure/a/pom.xml @@ -0,0 +1,61 @@ + + + 4.0.0 + + demo + descriptor-failure + 1.0 + + a + + + + demo.descfail + broken + 1.0 + + + demo.descfail + nopom + 1.0 + + + demo.descfail + nothing + 1.0 + + + demo.descfail + sys + 1.0 + system + ${descfail.sysDir}/sys.jar + + + demo.descfail + sysmissing + 1.0 + system + ${descfail.sysDir}/missing.jar + + + demo.descfail + testparent + 1.0 + test + + + demo.descfail + conflict + 2.0 + + + demo.descfail + loserparent + 1.0 + + + diff --git a/src/commands/manifest/scripts/test/maven-compat/descriptor-failure/b/pom.xml b/src/commands/manifest/scripts/test/maven-compat/descriptor-failure/b/pom.xml new file mode 100644 index 000000000..9c2a7b8b2 --- /dev/null +++ b/src/commands/manifest/scripts/test/maven-compat/descriptor-failure/b/pom.xml @@ -0,0 +1,23 @@ + + + 4.0.0 + + demo + descriptor-failure + 1.0 + + b + + + demo.descfail + ok + 1.0 + + + demo.descfail + broken + 1.0 + test + + + diff --git a/src/commands/manifest/scripts/test/maven-compat/descriptor-failure/pom.xml b/src/commands/manifest/scripts/test/maven-compat/descriptor-failure/pom.xml new file mode 100644 index 000000000..b9ced32fc --- /dev/null +++ b/src/commands/manifest/scripts/test/maven-compat/descriptor-failure/pom.xml @@ -0,0 +1,13 @@ + + + 4.0.0 + demo + descriptor-failure + 1.0 + pom + + + b + a + + diff --git a/src/commands/manifest/scripts/test/maven-compat/graph-failure/pom.xml b/src/commands/manifest/scripts/test/maven-compat/graph-failure/pom.xml index aab459112..b93299123 100644 --- a/src/commands/manifest/scripts/test/maven-compat/graph-failure/pom.xml +++ b/src/commands/manifest/scripts/test/maven-compat/graph-failure/pom.xml @@ -9,15 +9,20 @@ 8 UTF-8 - + demo.graphfail ok 1.0 + + demo.graphfail + nojar + 1.0 + demo.graphfail lib diff --git a/src/commands/manifest/scripts/test/maven-compat/smoke-test-descriptor-failure.sh b/src/commands/manifest/scripts/test/maven-compat/smoke-test-descriptor-failure.sh new file mode 100755 index 000000000..ae7badcee --- /dev/null +++ b/src/commands/manifest/scripts/test/maven-compat/smoke-test-descriptor-failure.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +# Regression guard for failures Maven's resolver hides from a plain resolution result: +# - A POM that failed once is cached build-wide, so a later module gets the dependency without an +# error. It must still fail there, in that module's scope (broken: test in b, compile in a). +# - A missing POM is tolerated by Maven's default descriptor policy, losing the dependency's own +# dependencies. It must fail; the dependency stays emitted with its jar (nopom), and one whose jar +# is missing too gets no file (nothing). +# - A system-scope jar gets its file; a missing systemPath fails, under the full coordinate. +# - A failure absent from the graph fails in the scope it would have had: rangedchild under the +# test-scoped testparent is test; before Maven 3.9 and on Maven 4 the position is unknown (no scope). +# - A version-conflict loser (conflict 1.0) is on no classpath and must not fail; Maven 3.2 drops +# a failed POM's node, so there it cannot be told apart and has no scope. +# +# Runs offline: the POMs are seeded straight into the local repository. +# +# Usage: smoke-test-descriptor-failure.sh +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +MVN="${1:?usage: smoke-test-descriptor-failure.sh }" +JAR="${2:?usage: smoke-test-descriptor-failure.sh }" +PROJECT="$HERE/descriptor-failure" +RECORDS="$PROJECT/records.tsv" +# shellcheck source=SCRIPTDIR/../compat-cache.sh +. "$HERE/../compat-cache.sh" +M2="$SOCKET_COMPAT_CACHE/m2" +REPO="$M2/demo/descfail" +SYS_DIR="$SOCKET_COMPAT_CACHE/descfail-sys" + +pom() { + # $1 artifactId, $2 version, $3 extra POM body. + mkdir -p "$REPO/$1/$2" + cat > "$REPO/$1/$2/$1-$2.pom" < + 4.0.0 + demo.descfail + $1 + $2 + $3 + +POM +} + +jar_for() { + mkdir -p "$REPO/$1/$2/empty" + ( cd "$REPO/$1/$2" && jar cf "$1-$2.jar" -C empty . && rm -r empty ) +} + +dep() { + echo "demo.descfail$1$2" +} + +rm -rf "$REPO" "$SYS_DIR" +pom ok 1.0 '' +jar_for ok 1.0 +pom broken 1.0 'demo.descfailabsent-parent1.0' +jar_for broken 1.0 +jar_for nopom 1.0 +pom testparent 1.0 "$(dep rangedchild '[5.0,6.0)')" +jar_for testparent 1.0 +pom conflict 2.0 '' +jar_for conflict 2.0 +pom conflict 1.0 'demo.descfailabsent-parent1.0' +pom loserparent 1.0 "$(dep conflict 1.0)" +jar_for loserparent 1.0 +mkdir -p "$SYS_DIR/empty" +( cd "$SYS_DIR" && jar cf sys.jar -C empty . && rm -r empty ) + +run() { + rm -f "$RECORDS" + ( cd "$PROJECT" && "$MVN" --batch-mode -q -o \ + "-Dmaven.ext.class.path=$JAR" \ + -Dcoana.task=socket-facts \ + "-Dmaven.repo.local=$M2" \ + "-Ddescfail.sysDir=$SYS_DIR" \ + "-Dsocket.recordsFile=$RECORDS" \ + -Dsocket.withFiles=true \ + "$@" \ + validate ) +} + +# $1: whether test scope is scanned. +check() { + python3 - "$RECORDS" "$1" "$STEPS" "$KEEPS_FAILED" <<'PY' +import sys +with_test, steps, keeps_failed = (a == 'true' for a in sys.argv[2:5]) +a, b = 'demo:a:1.0', 'demo:b:1.0' +expected = { + ('demo.descfail:broken:1.0', 'compile', a), + ('demo.descfail:nopom:1.0', 'compile', a), + ('demo.descfail:nothing:1.0', 'compile', a), + ('demo.descfail:nothing:jar:1.0', 'compile', a), + ('demo.descfail:sysmissing:jar:1.0', 'system', a), +} +if not keeps_failed: + expected.add(('demo.descfail:conflict:1.0', '', a)) +if with_test: + expected.add(('demo.descfail:broken:1.0', 'test', b)) +if with_test or not steps: + expected.add(('demo.descfail:rangedchild:[5.0,6.0)', 'test' if steps else '', a)) +rows = [l.rstrip('\n').split('\t') for l in open(sys.argv[1]) if l.strip()] +failures = [r[1:] for r in rows if r[0] == 'failure'] +actual = {(f[0], f[2], f[3]) for f in failures} +nodes = {r[2] for r in rows if r[0] == 'node'} +files = {r[2] for r in rows if r[0] == 'file'} +unfetched = {r[2] for r in rows if r[0] == 'unfetched'} +unscannable = [r[1:] for r in rows if r[0] == 'unscannable'] +errors = [] +if actual != expected: + errors.append(f"failures differ: missing={sorted(expected - actual)} unexpected={sorted(actual - expected)}") +for f in failures: + if 'One or more requests failed' in f[1]: + errors.append(f"{f[0]} reports the batch wrapper, not its cause: {f[1]!r}") +if any(n.startswith('demo.descfail:broken:') for n in nodes): + errors.append(f"broken's POM failed, so it must not be emitted; nodes={sorted(nodes)}") +for coord in ('demo.descfail:ok:jar:1.0', 'demo.descfail:nopom:jar:1.0', 'demo.descfail:sys:jar:1.0'): + if coord not in files: + errors.append(f"{coord} must be emitted with its jar; files={sorted(files)}") +if unfetched != {'demo.descfail:nothing:jar:1.0', 'demo.descfail:sysmissing:jar:1.0'}: + errors.append(f"only nothing's and sysmissing's jars are unfetched; unfetched={sorted(unfetched)}") +if any(f.startswith(('demo.descfail:nothing:', 'demo.descfail:sysmissing:')) for f in files): + errors.append(f"a jar that is missing must get no file; files={sorted(files)}") +if unscannable: + errors.append(f"every failure names its artifact, so nothing is unscannable: {unscannable}") +if errors: + print("FAIL:") + for e in errors: + print(" -", e) + sys.exit(1) +print(f"PASS (descriptor failure): {len(actual)} failure(s)") +PY +} + +VERSION="$("$MVN" -v 2>/dev/null | head -1)" +echo "+ $VERSION (descriptor failure)" +case "$VERSION" in + *"Maven 3.2."*) STEPS=false; KEEPS_FAILED=false ;; + *"Maven 3.9."*) STEPS=true; KEEPS_FAILED=true ;; + *) STEPS=false; KEEPS_FAILED=true ;; +esac + +run +check true +run -Dsocket.excludeConfigs=test +check false diff --git a/src/commands/manifest/scripts/test/maven-compat/smoke-test-graph-failure.sh b/src/commands/manifest/scripts/test/maven-compat/smoke-test-graph-failure.sh index f100536a8..f9c827402 100755 --- a/src/commands/manifest/scripts/test/maven-compat/smoke-test-graph-failure.sh +++ b/src/commands/manifest/scripts/test/maven-compat/smoke-test-graph-failure.sh @@ -1,9 +1,10 @@ #!/usr/bin/env bash # Regression guard: a dependency that fails while Maven collects the graph must be reported as that -# dependency, in the module it was collected for, and never emitted as a node. Maven keeps a -# dependency whose POM failed in the graph (Maven 3.2's resolver leaves it out instead), so it fails -# in its own scope, and an excluded scope drops it; a dependency the graph lacks (an unsatisfiable -# range) has no scope. The module itself must never be reported as the failure. +# dependency, in the module it was collected for, and never emitted as a node. It fails in its own +# scope, and an excluded scope drops it; one the graph lacks (an unsatisfiable range, or a failed POM +# on Maven 3.2) fails in the scope it is declared with. The module itself must never be reported as +# the failure. Under --with-files, every healthy dependency must still get its file, and a jar that +# cannot be fetched is reported under its full coordinate, with the cause rather than a wrapper. # # Runs offline: the POMs are seeded straight into the local repository, so their missing parent and # the absent versions of ranged can only fail to resolve. @@ -34,15 +35,18 @@ for lib in lib testlib; do POM done -mkdir -p "$M2/demo/graphfail/ok/1.0" -cat > "$M2/demo/graphfail/ok/1.0/ok-1.0.pom" <<'POM' +for lib in ok nojar; do + mkdir -p "$M2/demo/graphfail/$lib/1.0" + cat > "$M2/demo/graphfail/$lib/1.0/$lib-1.0.pom" < 4.0.0 demo.graphfail - ok + $lib 1.0 POM +done +( cd "$M2/demo/graphfail/ok/1.0" && mkdir -p empty && jar cf ok-1.0.jar -C empty . && rm -r empty ) run() { rm -f "$RECORDS" @@ -55,15 +59,14 @@ run() { validate ) } -# $1: expected scope per coord (null: excluded) when the failed dependency stays in the graph. +# $1: expected scope per coord (null: excluded). check() { - python3 - "$RECORDS" "$1" "$KEEPS_FAILED" <<'PY' + python3 - "$RECORDS" "$1" <<'PY' import json, sys def read(path): return [l.rstrip('\n').split('\t') for l in open(path) if l.strip()] rows = read(sys.argv[1]) expected = json.loads(sys.argv[2]) -keeps_failed = sys.argv[3] == 'true' emitted = {f"{r[3]}:{r[4]}:{r[5]}" for r in rows if r[0] == 'node'} errors = [] if 'demo.graphfail:ok:1.0' not in emitted: @@ -76,8 +79,6 @@ for coord, scope in expected.items(): found = [f for f in failures if f[0] == coord] if coord in emitted: errors.append(f"{coord} failed to resolve, so it must not be emitted as a node") - if not keeps_failed: - scope = '' if scope is None: if found: errors.append(f"{coord} is in an excluded scope and must not be reported, got {found}") @@ -100,13 +101,38 @@ print(f"PASS (graph failure): {expected}") PY } +check_files() { + python3 - "$RECORDS" <<'PY' +import sys +rows = [l.rstrip('\n').split('\t') for l in open(sys.argv[1]) if l.strip()] +files = {r[2] for r in rows if r[0] == 'file'} +failures = {(r[1], r[3]) for r in rows if r[0] == 'failure'} +details = {r[2] for r in rows if r[0] == 'failure' and r[1] == 'demo.graphfail:nojar:jar:1.0'} +unfetched = {r[2] for r in rows if r[0] == 'unfetched'} +errors = [] +if 'demo.graphfail:ok:jar:1.0' not in files: + errors.append(f"ok resolved, so its jar must be materialized despite the collection errors; files={sorted(files)}") +if ('demo.graphfail:nojar:jar:1.0', 'compile') not in failures: + errors.append(f"nojar's jar cannot be fetched, so it must be reported; failures={sorted(failures)}") +if any('One or more requests failed' in d for d in details): + errors.append(f"nojar's failure must give the cause, not the batch wrapper; details={sorted(details)}") +if unfetched != {'demo.graphfail:nojar:jar:1.0'}: + errors.append(f"only nojar's jar is unfetched; unfetched={sorted(unfetched)}") +if errors: + print("FAIL:") + for e in errors: + print(" -", e) + sys.exit(1) +print("PASS (graph failure, with files)") +PY +} + VERSION="$("$MVN" -v 2>/dev/null | head -1)" echo "+ $VERSION (graph failure)" -case "$VERSION" in - *"Maven 3.2."*) KEEPS_FAILED=false ;; - *) KEEPS_FAILED=true ;; -esac run -check '{"demo.graphfail:lib:1.0": "compile", "demo.graphfail:testlib:1.0": "test", "demo.graphfail:ranged:[1.0,2.0)": ""}' +check '{"demo.graphfail:lib:1.0": "compile", "demo.graphfail:testlib:1.0": "test", "demo.graphfail:ranged:[1.0,2.0)": "compile"}' run -Dsocket.excludeConfigs=test -check '{"demo.graphfail:lib:1.0": "compile", "demo.graphfail:testlib:1.0": null, "demo.graphfail:ranged:[1.0,2.0)": ""}' +check '{"demo.graphfail:lib:1.0": "compile", "demo.graphfail:testlib:1.0": null, "demo.graphfail:ranged:[1.0,2.0)": "compile"}' +run -Dsocket.withFiles=true +check '{"demo.graphfail:lib:1.0": "compile", "demo.graphfail:testlib:1.0": "test", "demo.graphfail:ranged:[1.0,2.0)": "compile"}' +check_files diff --git a/src/commands/manifest/scripts/test/run-compat.sh b/src/commands/manifest/scripts/test/run-compat.sh index 475e19988..fb7b668ff 100755 --- a/src/commands/manifest/scripts/test/run-compat.sh +++ b/src/commands/manifest/scripts/test/run-compat.sh @@ -23,12 +23,14 @@ TOOL="${1:-all}" CACHE="$SOCKET_COMPAT_CACHE" # Same matrix as the former CI workflow. Rows: " [scala]". -GRADLE_MATRIX=("1.12 8" "2.14.1 8" "3.3 8" "8.10.2 17" "9.2.1 21") +# 6.2 is the oldest Gradle with dependency verification. +GRADLE_MATRIX=("1.12 8" "2.14.1 8" "3.3 8" "6.2.2 8" "8.10.2 17" "9.2.1 21") # The Maven rows span the range the extension claims to support (see maven-extension/pom.xml): # 3.2.5 is the oldest, 4.x the newest. Resolution goes through Maven's own # ProjectDependenciesResolver, so both ends are load-bearing, not decorative. MAVEN_MATRIX=("3.2.5 8" "3.6.3 11" "3.8.8 17" "3.9.9 17" "4.0.0-rc-6 21") -SBT_MATRIX=("0.13.18 8 2.10.7" "1.4.9 11 2.12.20" "1.6.2 17 2.12.20" "1.9.9 17 2.12.20") +# The `ivy` row runs sbt 1.x with `useCoursier := false`, the other resolver path; failure suite only. +SBT_MATRIX=("0.13.18 8 2.10.7" "1.4.9 11 2.12.20" "1.6.2 17 2.12.20" "1.9.9 17 2.12.20" "1.9.9 17 2.12.20 ivy") # Select a JDK for the given Java major: use $JDK if set, else current java. use_jdk() { @@ -62,6 +64,7 @@ run_gradle() { fi bash "$HERE/gradle-compat/smoke-test.sh" "$dir/bin/gradle" bash "$HERE/gradle-compat/smoke-test-workspaces.sh" "$dir/bin/gradle" + bash "$HERE/gradle-compat/smoke-test-failures.sh" "$dir/bin/gradle" done } @@ -89,6 +92,7 @@ run_maven() { bash "$HERE/maven-compat/smoke-test-repo-inheritance.sh" "$dir/bin/mvn" "$jar" bash "$HERE/maven-compat/smoke-test-duplicate-failure.sh" "$dir/bin/mvn" "$jar" bash "$HERE/maven-compat/smoke-test-graph-failure.sh" "$dir/bin/mvn" "$jar" + bash "$HERE/maven-compat/smoke-test-descriptor-failure.sh" "$dir/bin/mvn" "$jar" done } @@ -100,12 +104,14 @@ run_sbt() { for row in "${SBT_MATRIX[@]}"; do # shellcheck disable=SC2086 set -- $row - local ver="$1" java="$2" scala="$3" - echo "== sbt $ver / scala $scala (wants JDK $java) ==" + local ver="$1" java="$2" scala="$3" resolver="${4:-}" + echo "== sbt $ver / scala $scala${resolver:+ / $resolver} (wants JDK $java) ==" use_jdk "$java" - bash "$HERE/sbt-compat/smoke-test.sh" "$ver" "$scala" - bash "$HERE/sbt-compat/smoke-test-workspaces.sh" "$ver" "$scala" - bash "$HERE/sbt-compat/smoke-test-failure.sh" "$ver" "$scala" + if [ -z "$resolver" ]; then + bash "$HERE/sbt-compat/smoke-test.sh" "$ver" "$scala" + bash "$HERE/sbt-compat/smoke-test-workspaces.sh" "$ver" "$scala" + fi + bash "$HERE/sbt-compat/smoke-test-failure.sh" "$ver" "$scala" ${resolver:+"$resolver"} done } diff --git a/src/commands/manifest/scripts/test/sbt-compat/.gitignore b/src/commands/manifest/scripts/test/sbt-compat/.gitignore index 77a47cca2..90cf81ffc 100644 --- a/src/commands/manifest/scripts/test/sbt-compat/.gitignore +++ b/src/commands/manifest/scripts/test/sbt-compat/.gitignore @@ -12,3 +12,8 @@ failure/target/ failure/project/ failure/scala-version.sbt failure/records.tsv +artifact-failure/target/ +artifact-failure/*/target/ +artifact-failure/project/ +artifact-failure/scala-version.sbt +artifact-failure/records.tsv diff --git a/src/commands/manifest/scripts/test/sbt-compat/artifact-failure/build.sbt b/src/commands/manifest/scripts/test/sbt-compat/artifact-failure/build.sbt new file mode 100644 index 000000000..e7d024e3f --- /dev/null +++ b/src/commands/manifest/scripts/test/sbt-compat/artifact-failure/build.sbt @@ -0,0 +1,20 @@ +// Artifact-failure fixture for ../../../socket-facts.plugin.scala: every module resolves, but ok has no +// tests-classifier jar, so the only failure is an artifact that names no module. app depends on core, +// whose update sbt also runs for app. Reuses ../failure/localrepo, generated by ../smoke-test-failure.sh. +organization in ThisBuild := "demo" +version in ThisBuild := "0.1.0" + +lazy val stubs = Seq( + resolvers := Seq( + "Socket Stubs" at s"file://${(baseDirectory in ThisBuild).value.getAbsolutePath}/../failure/localrepo" + ) ++ resolvers.value +) + +lazy val core = (project in file("core")) + .settings(stubs) + .settings( + libraryDependencies += "demo.fail" % "ok" % "1.0", + libraryDependencies += "demo.fail" % "ok" % "1.0" classifier "tests" + ) + +lazy val app = (project in file("app")).settings(stubs).dependsOn(core) diff --git a/src/commands/manifest/scripts/test/sbt-compat/failure/build.sbt b/src/commands/manifest/scripts/test/sbt-compat/failure/build.sbt index a9b6ac99c..6cbac47a1 100644 --- a/src/commands/manifest/scripts/test/sbt-compat/failure/build.sbt +++ b/src/commands/manifest/scripts/test/sbt-compat/failure/build.sbt @@ -1,6 +1,8 @@ // Failure fixture for ../../../socket-facts.plugin.scala: ok resolves; lib and testlib name a parent POM -// that does not exist; ghost does not exist at all. Generated into `localrepo` by -// ../smoke-test-failure.sh. Uses the `in ThisBuild` form so it parses on sbt 0.13 and 1.x. +// that does not exist; ghost does not exist at all; nojar has a POM but no jar; pomonly is +// `pom`; pkwar is a war with no jar; ok has no tests-classifier jar. Generated +// into `localrepo` by ../smoke-test-failure.sh. Uses the `in ThisBuild` form so it parses on sbt 0.13 +// and 1.x. organization in ThisBuild := "demo" version in ThisBuild := "0.1.0" @@ -15,5 +17,9 @@ lazy val root = (project in file(".")) libraryDependencies += "demo.fail" % "ok" % "1.0", libraryDependencies += "demo.fail" % "lib" % "1.0", libraryDependencies += "demo.fail" % "testlib" % "1.0" % Test, - libraryDependencies += "demo.fail" % "ghost" % "1.0" + libraryDependencies += "demo.fail" % "ghost" % "1.0", + libraryDependencies += "demo.fail" % "nojar" % "1.0", + libraryDependencies += "demo.fail" % "pomonly" % "1.0", + libraryDependencies += "demo.fail" % "pkwar" % "1.0", + libraryDependencies += "demo.fail" % "ok" % "1.0" % Test classifier "tests" ) diff --git a/src/commands/manifest/scripts/test/sbt-compat/smoke-test-failure.sh b/src/commands/manifest/scripts/test/sbt-compat/smoke-test-failure.sh index b0110cd11..6a6fa2b7f 100755 --- a/src/commands/manifest/scripts/test/sbt-compat/smoke-test-failure.sh +++ b/src/commands/manifest/scripts/test/sbt-compat/smoke-test-failure.sh @@ -5,11 +5,22 @@ # the declared dependencies whose parent POM is missing (lib, testlib), coursier (sbt 1.3+) names the # missing parent POM itself; both name ghost. Each failure's detail is its own module's message. # -# Usage: smoke-test-failure.sh +# A second run with files: a jar that cannot be fetched (nojar; lib/testlib's under coursier; ok's +# tests classifier, test-scoped) must be reported and marked `unfetched` under its full coordinate, +# in the configurations that requested it, while pomonly stays an artifactless module and pkwar keeps +# its war. Coursier skips a missing jar silently and lists one for every Maven module whatever its +# packaging; Ivy's own resolution report names what it failed to download. +# +# Then ../artifact-failure, whose only failure is ok's missing tests jar, without and with files: an +# artifact failure names no module, yet sbt's update fails on it, so it must be reported in both runs +# and must not make core or its dependent app unscannable. +# +# Usage: smoke-test-failure.sh [coursier|ivy] (ivy: sbt 1.x only) set -euo pipefail HERE="$(cd "$(dirname "$0")" && pwd)" SBT_VERSION="${1:?usage: smoke-test-failure.sh }" SCALA_VERSION="${2:?usage: smoke-test-failure.sh }" +RESOLVER="${3:-coursier}" PLUGIN="$HERE/../../socket-facts.plugin.scala" PROJECT="$HERE/failure" RECORDS="$PROJECT/records.tsv" @@ -34,6 +45,34 @@ for lib in lib testlib; do POM done +mkdir -p "$REPO/demo/fail/nojar/1.0" "$REPO/demo/fail/pomonly/1.0" "$REPO/demo/fail/pkwar/1.0" +cat > "$REPO/demo/fail/pkwar/1.0/pkwar-1.0.pom" <<'POM' + + 4.0.0 + demo.fail + pkwar + 1.0 + war + +POM +cp "$REPO/demo/fail/ok/1.0/ok-1.0.jar" "$REPO/demo/fail/pkwar/1.0/pkwar-1.0.war" +cat > "$REPO/demo/fail/nojar/1.0/nojar-1.0.pom" <<'POM' + + 4.0.0 + demo.fail + nojar + 1.0 + +POM +cat > "$REPO/demo/fail/pomonly/1.0/pomonly-1.0.pom" <<'POM' + + 4.0.0 + demo.fail + pomonly + 1.0 + pom + +POM rm -rf "$IVY/cache/demo.fail" GB="$(mktemp -d)/global-base" @@ -43,12 +82,20 @@ cp "$PLUGIN" "$GB/plugins/SocketFactsPlugin.scala" mkdir -p "$PROJECT/project" echo "sbt.version=$SBT_VERSION" > "$PROJECT/project/build.properties" echo "scalaVersion in ThisBuild := \"$SCALA_VERSION\"" > "$PROJECT/scala-version.sbt" +if [ "$RESOLVER" = ivy ]; then + echo 'useCoursier in ThisBuild := false' >> "$PROJECT/scala-version.sbt" +fi rm -rf "$RECORDS" "$PROJECT/target" "$PROJECT/project/target" +run() { + local dir="$1" + shift + ( cd "$dir" && sbt -Dsbt.global.base="$GB" -Dsbt.server.autostart=false \ + -Dsbt.ivy.home="$IVY" "$@" --batch socketFacts ) +} + echo "+ sbt $SBT_VERSION (scala $SCALA_VERSION) (failure)" -( cd "$PROJECT" && sbt -Dsbt.global.base="$GB" -Dsbt.server.autostart=false \ - -Dsbt.ivy.home="$IVY" \ - -Dsocket.recordsFile="$RECORDS" --batch socketFacts ) +run "$PROJECT" -Dsocket.recordsFile="$RECORDS" python3 - "$RECORDS" <<'PY' import sys @@ -84,3 +131,112 @@ if errors: sys.exit(1) print(f"PASS (failure): reported {sorted(failed)}; ok still emitted") PY + +rm -rf "$RECORDS" "$PROJECT/target" "$PROJECT/project/target" +echo "+ sbt $SBT_VERSION (scala $SCALA_VERSION) (failure, with files)" +run "$PROJECT" -Dsocket.withFiles=true -Dsocket.recordsFile="$RECORDS" + +python3 - "$RECORDS" <<'PY' +import sys +rows = [l.rstrip('\n').split('\t') for l in open(sys.argv[1]) if l.strip()] +nodes = {r[2]: r for r in rows if r[0] == 'node'} +unfetched = {r[2] for r in rows if r[0] == 'unfetched'} +failed = {r[1] for r in rows if r[0] == 'failure'} +config_of = {r[1]: r[3] for r in rows if r[0] == 'root'} +unfetched_configs = {(r[2], config_of[r[1]]) for r in rows if r[0] == 'unfetched'} +files = {r[2] for r in rows if r[0] == 'file'} +errors = [] + +if 'demo.fail:pomonly:1.0' not in nodes: + errors.append(f"pomonly is pom-packaged, so it must be emitted without an artifact; nodes={sorted(nodes)}") +if any('pomonly' in u for u in unfetched) or 'demo.fail:pomonly:1.0' in failed: + errors.append(f"pomonly has no jar to fetch; unfetched={sorted(unfetched)} failures={sorted(failed)}") +for name in ('nojar', 'ok'): + if f'demo.fail:{name}:1.0' not in failed: + errors.append(f"{name}'s jar cannot be fetched, so it must be reported; failures={sorted(failed)}") +if 'demo.fail:nojar:jar:1.0' not in unfetched: + errors.append(f"nojar must be emitted as an unfetched jar; unfetched={sorted(unfetched)}") +if 'demo.fail:ok:jar:1.0' not in files: + errors.append("ok's own jar resolved, so its missing tests jar must not cost it its file") +tests_configs = {c for (u, c) in unfetched_configs if u == 'demo.fail:ok:jar:tests:1.0'} +if not tests_configs or any('test' not in c.lower() for c in tests_configs): + errors.append(f"ok's tests jar is requested only in Test, so only test configs may miss it: {sorted(tests_configs)}") +for name in ('nojar', 'lib', 'testlib'): + if f'demo.fail:{name}:1.0' in nodes: + errors.append(f"{name} has a jar-packaged POM, so it must not look artifactless") + if f'demo.fail:{name}:jar:1.0' in nodes and f'demo.fail:{name}:jar:1.0' not in unfetched: + errors.append(f"{name} is emitted without its jar, so it must be marked unfetched; unfetched={sorted(unfetched)}") +if 'demo.fail:pkwar:war:1.0' not in files: + errors.append(f"pkwar's war resolved, so it must keep its file; nodes={sorted(n for n in nodes if 'pkwar' in n)}") +if any('pkwar' in f for f in failed): + errors.append(f"pkwar's packaging is war, so a missing jar is expected; failures={sorted(failed)}") +details = [r[2] for r in rows if r[0] == 'failure' and r[1] == 'demo.fail:nojar:1.0'] +if len(set(details)) != 1 or 'not found' not in details[0]: + errors.append(f"nojar's jar is absent, so it must be reported once as not found: {sorted(set(details))}") +stray = unfetched - {f'demo.fail:{n}:jar:1.0' for n in ('nojar', 'lib', 'testlib')} - {'demo.fail:ok:jar:tests:1.0'} +if stray: + errors.append(f"only the jars that are missing may be unfetched: {sorted(stray)}") + +if errors: + print("FAIL:") + for e in errors: + print(" -", e) + sys.exit(1) +print(f"PASS (failure, with files): unfetched {sorted(unfetched)}; pomonly artifactless") +PY + +ARTIFACT="$HERE/artifact-failure" +mkdir -p "$ARTIFACT/project" +echo "sbt.version=$SBT_VERSION" > "$ARTIFACT/project/build.properties" +cp "$PROJECT/scala-version.sbt" "$ARTIFACT/scala-version.sbt" +rm -rf "$ARTIFACT/records.tsv" "$ARTIFACT/target" "$ARTIFACT"/*/target "$ARTIFACT/project/target" +echo "+ sbt $SBT_VERSION (scala $SCALA_VERSION) (artifact failure)" +run "$ARTIFACT" -Dsocket.recordsFile="$ARTIFACT/records.tsv" + +python3 - "$ARTIFACT/records.tsv" <<'PY' +import sys +rows = [l.rstrip('\n').split('\t') for l in open(sys.argv[1]) if l.strip()] +unscannable = [r[1:] for r in rows if r[0] == 'unscannable'] +failed = {r[1] for r in rows if r[0] == 'failure'} +errors = [] +if unscannable: + errors.append(f"a missing artifact names no module, but must not make a project unscannable: {unscannable}") +if failed != {'demo.fail:ok:1.0'}: + errors.append(f"sbt's update fails on ok's missing tests jar, so it must be reported without files too; failures={sorted(failed)}") +if errors: + print("FAIL:") + for e in errors: + print(" -", e) + sys.exit(1) +print("PASS (artifact failure): ok's missing tests jar reported without files") +PY + +rm -rf "$ARTIFACT/records.tsv" +echo "+ sbt $SBT_VERSION (scala $SCALA_VERSION) (artifact failure, with files)" +run "$ARTIFACT" -Dsocket.withFiles=true -Dsocket.recordsFile="$ARTIFACT/records.tsv" + +python3 - "$ARTIFACT/records.tsv" <<'PY' +import sys +rows = [l.rstrip('\n').split('\t') for l in open(sys.argv[1]) if l.strip()] +unscannable = [r[1:] for r in rows if r[0] == 'unscannable'] +project_of = {r[1]: r[2] for r in rows if r[0] == 'root'} +unfetched_by = {} +for r in rows: + if r[0] == 'unfetched': + unfetched_by.setdefault(project_of[r[1]], set()).add(r[2]) +failed = {r[1] for r in rows if r[0] == 'failure'} +errors = [] +if unscannable: + errors.append(f"a missing artifact names no module, but must not make a project unscannable: {unscannable}") +for p in ('core', 'app'): + if unfetched_by.get(p) != {'demo.fail:ok:jar:tests:1.0'}: + errors.append(f"{p} reaches ok's missing tests jar, so only it may be unfetched: {sorted(unfetched_by.get(p, ()))}") +if failed != {'demo.fail:ok:1.0'}: + errors.append(f"only ok's tests jar is missing; failures={sorted(failed)}") +if errors: + print("FAIL:") + for e in errors: + print(" -", e) + sys.exit(1) +print("PASS (artifact failure, with files): ok's tests jar unfetched in core and app; nothing unscannable") +PY