diff --git a/.changeset/esrp-publish.md b/.changeset/esrp-publish.md new file mode 100644 index 000000000000..ca1cc77f02c3 --- /dev/null +++ b/.changeset/esrp-publish.md @@ -0,0 +1,5 @@ +--- +"@typescript/vfs": patch +--- + +Switch to new release pipeline diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index e7259cd6c95c..8ffcb13d0115 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -102,12 +102,23 @@ jobs: # git checkout origin/v2 -- pnpm changeset status --since=origin/v2 + release-pipeline: + name: release pipeline + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: 'lts/*' + - run: node build/azure-pipelines/pack-changed-packages.mjs --check-stage-capacity + required: runs-on: ubuntu-latest if: ${{ always() }} needs: - tests - changesets + - release-pipeline steps: - name: Check required jobs diff --git a/.github/workflows/publish-packages.yml b/.github/workflows/publish-packages.yml index 52df2ce4538b..a4f1545c2470 100644 --- a/.github/workflows/publish-packages.yml +++ b/.github/workflows/publish-packages.yml @@ -65,6 +65,5 @@ jobs: - uses: changesets/action@ae32849d5ba541f9ae29e40e22a623bc13562f51 # v2.1.2 with: github-token: ${{ steps.app-token.outputs.token }} - publish-script: pnpm ci:publish - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }} + # Packages are published to npm by build/azure-pipelines/release.yml. + publish-script: pnpm changeset git-tag diff --git a/build/azure-pipelines/pack-changed-packages.mjs b/build/azure-pipelines/pack-changed-packages.mjs new file mode 100644 index 000000000000..05ca67738351 --- /dev/null +++ b/build/azure-pipelines/pack-changed-packages.mjs @@ -0,0 +1,177 @@ +import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; + +const root = process.cwd(); +const releasePipelinePath = path.join(root, "build/azure-pipelines/release.yml"); +const checkStageCapacity = process.argv.includes("--check-stage-capacity"); +const outDir = process.argv.find(arg => !arg.startsWith("--") && arg !== process.argv[0] && arg !== process.argv[1]) || process.env.BUILD_ARTIFACTSTAGINGDIRECTORY; + +function readJson(file) { + return JSON.parse(fs.readFileSync(file, "utf8")); +} + +function collectWorkspacePackages() { + const packages = new Map(); + for (const dir of fs.readdirSync(path.join(root, "packages"))) { + const packageJsonPath = path.join(root, "packages", dir, "package.json"); + if (!fs.existsSync(packageJsonPath)) continue; + + const packageJson = readJson(packageJsonPath); + if (!packageJson.name || packageJson.private) continue; + + packages.set(packageJson.name, { + dir: path.relative(root, path.dirname(packageJsonPath)), + packageJson, + }); + } + return packages; +} + +function dependenciesOnPackedPackages(pkg, packedNames) { + const dependencies = new Set(); + for (const section of ["dependencies", "peerDependencies", "optionalDependencies"]) { + for (const dependency of Object.keys(pkg.packageJson[section] || {})) { + if (packedNames.has(dependency)) dependencies.add(dependency); + } + } + return dependencies; +} + +function computeDepths(packages, packageNames) { + const packageNameSet = new Set(packageNames); + const memo = new Map(); + const visiting = new Set(); + + function depth(name) { + if (memo.has(name)) return memo.get(name); + if (visiting.has(name)) throw new Error(`Workspace dependency cycle involving ${name}`); + + visiting.add(name); + const deps = dependenciesOnPackedPackages(packages.get(name), packageNameSet); + const value = deps.size === 0 ? 0 : 1 + Math.max(...[...deps].map(depth)); + visiting.delete(name); + memo.set(name, value); + return value; + } + + for (const name of packageNames) depth(name); + return memo; +} + +function validateReleasePipelineStageCapacity(packages) { + const allPackageNames = [...packages.keys()]; + const maxRequiredDepth = Math.max(0, ...computeDepths(packages, allPackageNames).values()); + const releaseYaml = fs.readFileSync(releasePipelinePath, "utf8"); + const stageNumbers = [...releaseYaml.matchAll(/^\s+-\s+index:\s+(\d+)\s*$/gm)] + .map(match => Number(match[1])); + const maxAvailableStage = Math.max(-1, ...stageNumbers); + + if (maxRequiredDepth !== maxAvailableStage) { + throw new Error(`Workspace publish graph requires stages 0-${maxRequiredDepth}, but ${releasePipelinePath} defines stages 0-${maxAvailableStage}`); + } + + console.log(`Workspace publish graph requires stages 0-${maxRequiredDepth}; release pipeline defines stages 0-${maxAvailableStage}.`); + return maxAvailableStage + 1; +} + +const packages = collectWorkspacePackages(); +const maxStages = validateReleasePipelineStageCapacity(packages); + +if (checkStageCapacity) { + process.exit(0); +} + +if (!outDir) { + throw new Error("Usage: node build/azure-pipelines/pack-changed-packages.mjs "); +} + +// PRs are always squash merged, so a version bump from the "Version Packages" +// PR is always in the tip commit of its own push. +function readPreviousPackageJson(pkg) { + try { + return JSON.parse(execFileSync("git", ["show", `HEAD~1:${pkg.dir}/package.json`], { encoding: "utf8" })); + } + catch { + return undefined; + } +} + +function versionChanged(pkg) { + const previous = readPreviousPackageJson(pkg); + return previous?.version !== pkg.packageJson.version; +} + +function findChangedPackageNames() { + const releaseBranch = `refs/heads/${readJson(path.join(root, ".changeset/config.json")).baseBranch}`; + const sourceBranch = process.env.BUILD_SOURCEBRANCH; + if (sourceBranch !== undefined && sourceBranch !== releaseBranch) { + console.log(`Building ${sourceBranch}, not ${releaseBranch}; not staging any packages.`); + return []; + } + + return [...packages.values()] + .filter(versionChanged) + .map(pkg => pkg.packageJson.name); +} + +const changedPackageNames = findChangedPackageNames(); + +let packed = []; +if (changedPackageNames.length) { + const packedDir = path.join(outDir, "packed"); + fs.mkdirSync(packedDir, { recursive: true }); + + const filters = changedPackageNames.flatMap(name => ["--filter", name]); + const output = execFileSync("pnpm", ["-r", ...filters, "pack", "--pack-destination", packedDir, "--json"], { + encoding: "utf8", + stdio: ["ignore", "pipe", "inherit"], + }); + const parsed = output.trim() ? JSON.parse(output) : []; + packed = Array.isArray(parsed) ? parsed : [parsed]; + + for (const packedPackage of packed) { + packedPackage.filename = path.join(packedDir, path.basename(packedPackage.filename)); + } +} + +packed = packed.filter(pkg => packages.has(pkg.name)); +const packedNames = packed.map(pkg => pkg.name); +const packedByName = new Map(packed.map(pkg => [pkg.name, pkg])); +const depths = computeDepths(packages, packedNames); +const manifest = { stages: Array.from({ length: maxStages }, () => []) }; +for (let i = 0; i < maxStages; i++) { + fs.mkdirSync(path.join(outDir, `stage-${i}`), { recursive: true }); +} + +for (const name of packedNames) { + const depth = depths.get(name); + if (depth >= maxStages) { + throw new Error(`${name} requires publish stage ${depth}, but max stage is ${maxStages - 1}`); + } + + const packedPackage = packedByName.get(name); + const filename = path.basename(packedPackage.filename); + const stageDir = path.join(outDir, `stage-${depth}`); + fs.mkdirSync(stageDir, { recursive: true }); + fs.copyFileSync(packedPackage.filename, path.join(stageDir, filename)); + manifest.stages[depth].push({ + name, + version: packedPackage.version, + filename, + }); +} + +fs.rmSync(path.join(outDir, "packed"), { recursive: true, force: true }); + +for (let i = 0; i < maxStages; i++) { + console.log(`Stage ${i}: ${manifest.stages[i].map(pkg => `${pkg.name}@${pkg.version}`).join(", ") || "(empty)"}`); + console.log(`##vso[task.setvariable variable=PublishStage${i};isOutput=true]${manifest.stages[i].length > 0}`); +} + +const shouldPublish = packedNames.length > 0; +console.log(`##vso[task.setvariable variable=ShouldPublish]${shouldPublish}`); +console.log(`##vso[task.setvariable variable=ShouldPublish;isOutput=true]${shouldPublish}`); +if (shouldPublish && process.env.BUILD_BUILDNUMBER) { + console.log(`##vso[build.updatebuildnumber]${process.env.BUILD_BUILDNUMBER}-release`); +} diff --git a/build/azure-pipelines/release.yml b/build/azure-pipelines/release.yml new file mode 100644 index 000000000000..02402249d00b --- /dev/null +++ b/build/azure-pipelines/release.yml @@ -0,0 +1,144 @@ +name: $(date:yyyyMMdd).$(rev:r) + +trigger: + branches: + include: + - v2 +pr: none + +variables: + TeamName: TypeScript + Codeql.Enabled: true + ReleaseOwners: "jabaile@microsoft.com" + ReleaseApprovers: "alexchi@microsoft.com" + NpmArtifactName: npm-packages + +parameters: + - name: publishStages + type: object + default: + - index: 0 + - index: 1 + +resources: + repositories: + - repository: MicroBuildTemplate + type: git + name: 1ESPipelineTemplates/MicroBuildTemplate + ref: refs/tags/release + +extends: + template: azure-pipelines/MicroBuild.1ES.Official.Publish.yml@MicroBuildTemplate + parameters: + pool: + name: AzurePipelines-EO + image: 1ESPT-Ubuntu22.04 + os: linux + sdl: + sourceAnalysisPool: VSEng-MicroBuildVSStable + binskim: + scanOutputDirectoryOnly: true + settings: + networkIsolationPolicy: Permissive,CFSClean + stages: + - stage: Build + displayName: Build + jobs: + - job: Build + displayName: Build and pack changed packages + templateContext: + mb: + # This pipeline uses the publish template so the package publish + # stages can run. Do not inject publish authorization into the + # normal build job. + publish: + enabled: false + signing: + enabled: false + sdl: + codeSignValidation: + enabled: false + outputParentDirectory: "$(Build.ArtifactStagingDirectory)" + outputs: + - output: pipelineArtifact + condition: and(succeeded(), eq(variables['ShouldPublish'], 'true')) + targetPath: "$(Build.ArtifactStagingDirectory)/$(NpmArtifactName)" + artifactName: $(NpmArtifactName) + steps: + - checkout: self + fetchDepth: 2 + + - task: UseNode@1 + inputs: + version: "22.x" + displayName: "Install Node.js" + + - script: | + set -euo pipefail + cat > .npmrc << 'EOF' + registry=https://pkgs.dev.azure.com/devdiv/devdiv/_packaging/devdiv_PublicPackages/npm/registry/ + EOF + displayName: "Set up .npmrc" + + - task: npmAuthenticate@0 + inputs: + workingFile: .npmrc + displayName: "Authenticate npm" + + - script: | + set -euo pipefail + # Global installs ignore the project .npmrc, so point npm at it + # explicitly to install from the authenticated feed. + npm install -g --userconfig "$PWD/.npmrc" "$(node -p "require('./package.json').packageManager.split('+')[0]")" + pnpm --version + displayName: "Install package manager" + + - script: pnpm install + displayName: "Install dependencies" + + # The site's bootstrap fetches from registry.npmjs.org, which network + # isolation blocks, and no published package needs it. + - script: pnpm --filter "./packages/*" --filter "!typescriptlang-org" run bootstrap + displayName: "Bootstrap packages" + + - script: pnpm build + displayName: "Build packages" + + - script: node build/azure-pipelines/pack-changed-packages.mjs "$(Build.ArtifactStagingDirectory)/$(NpmArtifactName)" + name: packChangedPackages + displayName: "Pack changed packages" + + - stage: Publish + displayName: Publish to npmjs + dependsOn: Build + condition: and(succeeded(), eq(dependencies.Build.outputs['Build.packChangedPackages.ShouldPublish'], 'true')) + jobs: + - job: Publish + displayName: Publish npm packages + variables: + - ${{ each publishStage in parameters.publishStages }}: + - name: PublishStage${{ publishStage.index }} + value: $[stageDependencies.Build.Build.outputs['packChangedPackages.PublishStage${{ publishStage.index }}']] + templateContext: + type: releaseJob + isProduction: true + inputs: + - input: pipelineArtifact + artifactName: $(NpmArtifactName) + targetPath: $(Pipeline.Workspace)/$(NpmArtifactName) + steps: + - checkout: none + + # ESRP succeeds for versions that are already on npm, so rerunning + # this job after a partial failure is safe. + - ${{ each publishStage in parameters.publishStages }}: + - template: MicroBuild.Publish.yml@MicroBuildTemplate + parameters: + condition: and(succeeded(), eq(variables['PublishStage${{ publishStage.index }}'], 'true')) + intent: "PackageDistribution" + contentType: "npm" + contentSource: "Folder" + folderLocation: "$(Pipeline.Workspace)/$(NpmArtifactName)/stage-${{ publishStage.index }}" + waitForReleaseCompletion: true + owners: "$(ReleaseOwners)" + approvers: "$(ReleaseApprovers)" diff --git a/package.json b/package.json index 200cf3cd8336..1ec3211ab872 100644 --- a/package.json +++ b/package.json @@ -66,7 +66,6 @@ ] }, "scripts": { - "ci:publish": "pnpm changeset git-tag && pnpm publish -r", "bootstrap": "pnpm -r run bootstrap && BOOTSTRAPPING=true pnpm -r run build", "start": "concurrently -p \"[{name}]\" -n \"BUILD,SITE\" -c \"bgBlue.bold,bgMagenta.bold\" \"node watcher.js\" \"pnpm run --filter=typescriptlang-org start\"", "build": "pnpm run --filter=!typescriptlang-org build",