Add new ESRP + changesets publish system

.changeset/esrp-publish.md

@@ -0,0 +1,5 @@
+---
+"@typescript/vfs": patch
+---
+
+Switch to new release pipeline

.github/workflows/CI.yml

@@ -102,12 +102,23 @@ jobs:
         # git checkout origin/v2 -- <ignored files>
         pnpm changeset status --since=origin/v2
 
+  release-pipeline:
+    name: release pipeline
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+    - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
+      with:
+        node-version: 'lts/*'
+    - run: node build/azure-pipelines/pack-changed-packages.mjs --check-stage-capacity
+
   required:
     runs-on: ubuntu-latest
     if: ${{ always() }}
     needs:
       - tests
       - changesets
+      - release-pipeline
 
     steps:
       - name: Check required jobs

.github/workflows/publish-packages.yml

@@ -65,6 +65,5 @@ jobs:
       - uses: changesets/action@ae32849d5ba541f9ae29e40e22a623bc13562f51 # v2.1.2
         with:
           github-token: ${{ steps.app-token.outputs.token }}
-          publish-script: pnpm ci:publish
-        env:
-          NODE_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }}
+          # Packages are published to npm by build/azure-pipelines/release.yml.
+          publish-script: pnpm changeset git-tag

build/azure-pipelines/pack-changed-packages.mjs

@@ -0,0 +1,177 @@
+import { execFileSync } from "node:child_process";
+import fs from "node:fs";
+import path from "node:path";
+
+const root = process.cwd();
+const releasePipelinePath = path.join(root, "build/azure-pipelines/release.yml");
+const checkStageCapacity = process.argv.includes("--check-stage-capacity");
+const outDir = process.argv.find(arg => !arg.startsWith("--") && arg !== process.argv[0] && arg !== process.argv[1]) || process.env.BUILD_ARTIFACTSTAGINGDIRECTORY;
+
+function readJson(file) {
+  return JSON.parse(fs.readFileSync(file, "utf8"));
+}
+
+function collectWorkspacePackages() {
+  const packages = new Map();
+  for (const dir of fs.readdirSync(path.join(root, "packages"))) {
+    const packageJsonPath = path.join(root, "packages", dir, "package.json");
+    if (!fs.existsSync(packageJsonPath)) continue;
+
+    const packageJson = readJson(packageJsonPath);
+    if (!packageJson.name || packageJson.private) continue;
+
+    packages.set(packageJson.name, {
+      dir: path.relative(root, path.dirname(packageJsonPath)),
+      packageJson,
+    });
+  }
+  return packages;
+}
+
+function dependenciesOnPackedPackages(pkg, packedNames) {
+  const dependencies = new Set();
+  for (const section of ["dependencies", "peerDependencies", "optionalDependencies"]) {
+    for (const dependency of Object.keys(pkg.packageJson[section] || {})) {
+      if (packedNames.has(dependency)) dependencies.add(dependency);
+    }
+  }
+  return dependencies;
+}
+
+function computeDepths(packages, packageNames) {
+  const packageNameSet = new Set(packageNames);
+  const memo = new Map();
+  const visiting = new Set();
+
+  function depth(name) {
+    if (memo.has(name)) return memo.get(name);
+    if (visiting.has(name)) throw new Error(`Workspace dependency cycle involving ${name}`);
+
+    visiting.add(name);
+    const deps = dependenciesOnPackedPackages(packages.get(name), packageNameSet);
+    const value = deps.size === 0 ? 0 : 1 + Math.max(...[...deps].map(depth));
+    visiting.delete(name);
+    memo.set(name, value);
+    return value;
+  }
+
+  for (const name of packageNames) depth(name);
+  return memo;
+}
+
+function validateReleasePipelineStageCapacity(packages) {
+  const allPackageNames = [...packages.keys()];
+  const maxRequiredDepth = Math.max(0, ...computeDepths(packages, allPackageNames).values());
+  const releaseYaml = fs.readFileSync(releasePipelinePath, "utf8");
+  const stageNumbers = [...releaseYaml.matchAll(/^\s+-\s+index:\s+(\d+)\s*$/gm)]
+    .map(match => Number(match[1]));
+  const maxAvailableStage = Math.max(-1, ...stageNumbers);
+
+  if (maxRequiredDepth !== maxAvailableStage) {
+    throw new Error(`Workspace publish graph requires stages 0-${maxRequiredDepth}, but ${releasePipelinePath} defines stages 0-${maxAvailableStage}`);
+  }
+
+  console.log(`Workspace publish graph requires stages 0-${maxRequiredDepth}; release pipeline defines stages 0-${maxAvailableStage}.`);
+  return maxAvailableStage + 1;
+}
+
+const packages = collectWorkspacePackages();
+const maxStages = validateReleasePipelineStageCapacity(packages);
+
+if (checkStageCapacity) {
+  process.exit(0);
+}
+
+if (!outDir) {
+  throw new Error("Usage: node build/azure-pipelines/pack-changed-packages.mjs <artifact-staging-directory>");
+}
+
+// PRs are always squash merged, so a version bump from the "Version Packages"
+// PR is always in the tip commit of its own push.
+function readPreviousPackageJson(pkg) {
+  try {
+    return JSON.parse(execFileSync("git", ["show", `HEAD~1:${pkg.dir}/package.json`], { encoding: "utf8" }));
+  }
+  catch {
+    return undefined;
+  }
+}
+
+function versionChanged(pkg) {
+  const previous = readPreviousPackageJson(pkg);
+  return previous?.version !== pkg.packageJson.version;
+}
+
+function findChangedPackageNames() {
+  const releaseBranch = `refs/heads/${readJson(path.join(root, ".changeset/config.json")).baseBranch}`;
+  const sourceBranch = process.env.BUILD_SOURCEBRANCH;
+  if (sourceBranch !== undefined && sourceBranch !== releaseBranch) {
+    console.log(`Building ${sourceBranch}, not ${releaseBranch}; not staging any packages.`);
+    return [];
+  }
+
+  return [...packages.values()]
+    .filter(versionChanged)
+    .map(pkg => pkg.packageJson.name);
+}
+
+const changedPackageNames = findChangedPackageNames();
+
+let packed = [];
+if (changedPackageNames.length) {
+  const packedDir = path.join(outDir, "packed");
+  fs.mkdirSync(packedDir, { recursive: true });
+
+  const filters = changedPackageNames.flatMap(name => ["--filter", name]);
+  const output = execFileSync("pnpm", ["-r", ...filters, "pack", "--pack-destination", packedDir, "--json"], {
+    encoding: "utf8",
+    stdio: ["ignore", "pipe", "inherit"],
+  });
+  const parsed = output.trim() ? JSON.parse(output) : [];
+  packed = Array.isArray(parsed) ? parsed : [parsed];
+
+  for (const packedPackage of packed) {
+    packedPackage.filename = path.join(packedDir, path.basename(packedPackage.filename));
+  }
+}
+
+packed = packed.filter(pkg => packages.has(pkg.name));
+const packedNames = packed.map(pkg => pkg.name);
+const packedByName = new Map(packed.map(pkg => [pkg.name, pkg]));
+const depths = computeDepths(packages, packedNames);
+const manifest = { stages: Array.from({ length: maxStages }, () => []) };
+for (let i = 0; i < maxStages; i++) {
+  fs.mkdirSync(path.join(outDir, `stage-${i}`), { recursive: true });
+}
+
+for (const name of packedNames) {
+  const depth = depths.get(name);
+  if (depth >= maxStages) {
+    throw new Error(`${name} requires publish stage ${depth}, but max stage is ${maxStages - 1}`);
+  }
+
+  const packedPackage = packedByName.get(name);
+  const filename = path.basename(packedPackage.filename);
+  const stageDir = path.join(outDir, `stage-${depth}`);
+  fs.mkdirSync(stageDir, { recursive: true });
+  fs.copyFileSync(packedPackage.filename, path.join(stageDir, filename));
+  manifest.stages[depth].push({
+    name,
+    version: packedPackage.version,
+    filename,
+  });
+}
+
+fs.rmSync(path.join(outDir, "packed"), { recursive: true, force: true });
+
+for (let i = 0; i < maxStages; i++) {
+  console.log(`Stage ${i}: ${manifest.stages[i].map(pkg => `${pkg.name}@${pkg.version}`).join(", ") || "(empty)"}`);
+  console.log(`##vso[task.setvariable variable=PublishStage${i};isOutput=true]${manifest.stages[i].length > 0}`);
+}
+
+const shouldPublish = packedNames.length > 0;
+console.log(`##vso[task.setvariable variable=ShouldPublish]${shouldPublish}`);
+console.log(`##vso[task.setvariable variable=ShouldPublish;isOutput=true]${shouldPublish}`);
+if (shouldPublish && process.env.BUILD_BUILDNUMBER) {
+  console.log(`##vso[build.updatebuildnumber]${process.env.BUILD_BUILDNUMBER}-release`);
+}

build/azure-pipelines/release.yml

@@ -0,0 +1,144 @@
+name: $(date:yyyyMMdd).$(rev:r)
+
+trigger:
+  branches:
+    include:
+      - v2
+pr: none
+
+variables:
+  TeamName: TypeScript
+  Codeql.Enabled: true
+  ReleaseOwners: "[email protected]"
+  ReleaseApprovers: "[email protected]"
+  NpmArtifactName: npm-packages
+
+parameters:
+  - name: publishStages
+    type: object
+    default:
+      - index: 0
+      - index: 1
+
+resources:
+  repositories:
+    - repository: MicroBuildTemplate
+      type: git
+      name: 1ESPipelineTemplates/MicroBuildTemplate
+      ref: refs/tags/release
+
+extends:
+  template: azure-pipelines/MicroBuild.1ES.Official.Publish.yml@MicroBuildTemplate
+  parameters:
+    pool:
+      name: AzurePipelines-EO
+      image: 1ESPT-Ubuntu22.04
+      os: linux
+    sdl:
+      sourceAnalysisPool: VSEng-MicroBuildVSStable
+      binskim:
+        scanOutputDirectoryOnly: true
+    settings:
+      networkIsolationPolicy: Permissive,CFSClean
+    stages:
+      - stage: Build
+        displayName: Build
+        jobs:
+          - job: Build
+            displayName: Build and pack changed packages
+            templateContext:
+              mb:
+                # This pipeline uses the publish template so the package publish
+                # stages can run. Do not inject publish authorization into the
+                # normal build job.
+                publish:
+                  enabled: false
+                signing:
+                  enabled: false
+              sdl:
+                codeSignValidation:
+                  enabled: false
+              outputParentDirectory: "$(Build.ArtifactStagingDirectory)"
+              outputs:
+                - output: pipelineArtifact
+                  condition: and(succeeded(), eq(variables['ShouldPublish'], 'true'))
+                  targetPath: "$(Build.ArtifactStagingDirectory)/$(NpmArtifactName)"
+                  artifactName: $(NpmArtifactName)
+            steps:
+              - checkout: self
+                fetchDepth: 2
+
+              - task: UseNode@1
+                inputs:
+                  version: "22.x"
+                displayName: "Install Node.js"
+
+              - script: |
+                  set -euo pipefail
+                  cat > .npmrc << 'EOF'
+                  registry=https://pkgs.dev.azure.com/devdiv/devdiv/_packaging/devdiv_PublicPackages/npm/registry/
+                  EOF
+                displayName: "Set up .npmrc"
+
+              - task: npmAuthenticate@0
+                inputs:
+                  workingFile: .npmrc
+                displayName: "Authenticate npm"
+
+              - script: |
+                  set -euo pipefail
+                  # Global installs ignore the project .npmrc, so point npm at it
+                  # explicitly to install from the authenticated feed.
+                  npm install -g --userconfig "$PWD/.npmrc" "$(node -p "require('./package.json').packageManager.split('+')[0]")"
+                  pnpm --version
+                displayName: "Install package manager"
+
+              - script: pnpm install
+                displayName: "Install dependencies"
+
+              # The site's bootstrap fetches from registry.npmjs.org, which network
+              # isolation blocks, and no published package needs it.
+              - script: pnpm --filter "./packages/*" --filter "!typescriptlang-org" run bootstrap
+                displayName: "Bootstrap packages"
+
+              - script: pnpm build
+                displayName: "Build packages"
+
+              - script: node build/azure-pipelines/pack-changed-packages.mjs "$(Build.ArtifactStagingDirectory)/$(NpmArtifactName)"
+                name: packChangedPackages
+                displayName: "Pack changed packages"
+
+      - stage: Publish
+        displayName: Publish to npmjs
+        dependsOn: Build
+        condition: and(succeeded(), eq(dependencies.Build.outputs['Build.packChangedPackages.ShouldPublish'], 'true'))
+        jobs:
+          - job: Publish
+            displayName: Publish npm packages
+            variables:
+              - ${{ each publishStage in parameters.publishStages }}:
+                - name: PublishStage${{ publishStage.index }}
+                  value: $[stageDependencies.Build.Build.outputs['packChangedPackages.PublishStage${{ publishStage.index }}']]
+            templateContext:
+              type: releaseJob
+              isProduction: true
+              inputs:
+                - input: pipelineArtifact
+                  artifactName: $(NpmArtifactName)
+                  targetPath: $(Pipeline.Workspace)/$(NpmArtifactName)
+            steps:
+              - checkout: none
+
+              # ESRP succeeds for versions that are already on npm, so rerunning
+              # this job after a partial failure is safe.
+              - ${{ each publishStage in parameters.publishStages }}:
+                - template: MicroBuild.Publish.yml@MicroBuildTemplate
+                  parameters:
+                    condition: and(succeeded(), eq(variables['PublishStage${{ publishStage.index }}'], 'true'))
+                    intent: "PackageDistribution"
+                    contentType: "npm"
+                    contentSource: "Folder"
+                    folderLocation: "$(Pipeline.Workspace)/$(NpmArtifactName)/stage-${{ publishStage.index }}"
+                    waitForReleaseCompletion: true
+                    owners: "$(ReleaseOwners)"
+                    approvers: "$(ReleaseApprovers)"

package.json

@@ -66,7 +66,6 @@
     ]
   },
   "scripts": {
-    "ci:publish": "pnpm changeset git-tag && pnpm publish -r",
     "bootstrap": "pnpm -r run bootstrap && BOOTSTRAPPING=true pnpm -r run build",
     "start": "concurrently -p \"[{name}]\" -n \"BUILD,SITE\" -c \"bgBlue.bold,bgMagenta.bold\" \"node watcher.js\" \"pnpm run --filter=typescriptlang-org start\"",
     "build": "pnpm run --filter=!typescriptlang-org build",