Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions docs/ncu-ci.md
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,8 @@ workload=$(ncu-ci workload) || exit 0

`ncu-ci rate <type>` calculate the success rate for CI jobs in the last 100 runs per [CI Health History](https://github.com/nodejs/reliability#ci-health-history), where `<type>` can be either `pr` for `node-test-pull-request` or `commit` for `node-test-commit`. See `ncu-ci rate --help` for more.

Builds targeting the `nodejs-private` organization are excluded from the counts.

Examples:

```sh
Expand Down Expand Up @@ -109,6 +111,9 @@ node on git:main ❯ ncu-ci rate commit

`ncu-ci walk <type>` walks CI and displays failures, where `<type>` can be either `pr` for `node-test-pull-request` or `commit` for `node-test-commit`. See `ncu-ci walk --help` for more.

Builds targeting the `nodejs-private` organization are excluded from the health
summary and failure report.

Example:
```sh
node on git:main ❯ ncu-ci walk commit
Expand Down
7 changes: 5 additions & 2 deletions lib/ci/ci_utils.js
Original file line number Diff line number Diff line change
Expand Up @@ -66,13 +66,16 @@ function filterBuild(builds, type) {
export async function listBuilds(cli, request, type, since) {
// assert(type === COMMIT || type === PR)
const { jobName } = CI_TYPES.get(type);
const tree = 'builds[url,result,timestamp]';
const tree = 'builds[url,result,timestamp,actions[parameters[name,value]]]';
const url = `https://${CI_DOMAIN}/job/${jobName}/api/json?tree=${qs.escape(tree)}`;

cli.startSpinner(`Querying ${url}`);

const result = await request.json(url);
let builds = result.builds;
let builds = result.builds.filter(build => !build.actions?.some(action =>
action.parameters?.some(({ name, value }) =>
(name === 'TARGET_GITHUB_ORG' || name === 'GITHUB_ORG') &&
typeof value === 'string' && value.toLowerCase() === 'nodejs-private')));
if (since) {
builds = builds.filter(build => build.timestamp > since);
}
Expand Down
21 changes: 12 additions & 9 deletions lib/ci/failure_aggregator.js
Original file line number Diff line number Diff line change
Expand Up @@ -93,15 +93,18 @@ export class FailureAggregator {
aggregates = this.aggregates = this.aggregate();
}

const last = parseJobFromURL(this.failures[0].upstream);
const first = parseJobFromURL(
this.failures[this.failures.length - 1].upstream
);
const jobName = CI_TYPES.get(first.type).jobName;
let output = 'Failures in ';
output += `[${jobName}/${first.jobid}](${first.link}) to `;
output += `[${jobName}/${last.jobid}](${last.link}) `;
output += 'that failed 2 or more PRs\n';
let output = 'No failures to report\n';
if (this.failures.length > 0) {
const last = parseJobFromURL(this.failures[0].upstream);
const first = parseJobFromURL(
this.failures[this.failures.length - 1].upstream
);
const jobName = CI_TYPES.get(first.type).jobName;
output = 'Failures in ';
output += `[${jobName}/${first.jobid}](${first.link}) to `;
output += `[${jobName}/${last.jobid}](${last.link}) `;
output += 'that failed 2 or more PRs\n';
}
output += '(Generated with `ncu-ci ';
output += `${process.argv.slice(2).join(' ')}\`)\n\n`;

Expand Down
12 changes: 12 additions & 0 deletions test/unit/ci_failure_aggregator.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,18 @@ function failure(prid, jobid, targetBranch, reason = 'Example failure') {
const health = { formatAsMarkdown: () => 'Health summary\n' };

describe('Failure aggregation target branches', () => {
it('reports health when no failures remain', () => {
const cli = new TestCLI();
const aggregator = new FailureAggregator(cli, [health]);
assert.deepEqual(Object.keys(aggregator.aggregate()), []);
const markdown = aggregator.formatAsMarkdown();
assert.ok(markdown.startsWith('No failures to report\n'));
assert.ok(markdown.includes('Health summary\n'));
assert.equal(markdown.includes('Failed PR'), false);
aggregator.display();
assert.deepEqual(cli._calls.table, []);
});

it('preserves branches across runs without changing PR counts or CI ordering', () => {
const cli = new TestCLI();
const failures = [
Expand Down
96 changes: 96 additions & 0 deletions test/unit/ci_list_builds.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
import assert from 'node:assert/strict';
import { describe, it } from 'node:test';

import { listBuilds } from '../../lib/ci/ci_utils.js';
import { CI_TYPES, CI_TYPES_KEYS } from '../../lib/ci/ci_type_parser.js';
import TestCLI from '../fixtures/test_cli.js';

const { PR, COMMIT } = CI_TYPES_KEYS;
const results = ['SUCCESS', 'FAILURE', 'ABORTED', 'UNSTABLE', null];

function build(type, jobid, result, owner, timestamp = 200) {
const { jobName } = CI_TYPES.get(type);
return {
url: `https://ci.nodejs.org/job/${jobName}/${jobid}/`,
result,
timestamp,
...(owner === undefined
? {}
: {
actions: [{}, {
parameters: [{
name: type === PR ? 'TARGET_GITHUB_ORG' : 'GITHUB_ORG',
value: owner
}]
}]
})
};
}

function requestFor(builds) {
return {
async json(url) {
assert.equal(new URL(url).searchParams.get('tree'),
'builds[url,result,timestamp,actions[parameters[name,value]]]');
return { builds };
}
};
}

function jobids(builds) {
return Object.fromEntries(Object.entries(builds).map(([key, value]) =>
[key, key === 'count' ? value : value.map(build => build.jobid)]));
}

describe('Reliability build listings', () => {
for (const type of [PR, COMMIT]) {
it(`excludes private ${type} builds from every result and the total`, async() => {
const builds = results.flatMap((result, index) => [
build(type, index + 1, result, 'nodejs'),
build(type, index + 101, result, index % 2 ? 'NODEJS-PRIVATE' : 'nodejs-private')
]);
builds.push(
build(type, 6, 'FAILURE', 'panva'),
build(type, 7, 'FAILURE', 'nodejs-private-extra'),
build(type, 8, 'FAILURE'),
{ ...build(type, 9, 'FAILURE'), actions: [{}] },
{
...build(type, 10, 'FAILURE'), actions: [{
parameters: [
{ name: 'BRANCH', value: 'nodejs-private' }
]
}]
}
);

assert.deepEqual(jobids(await listBuilds(new TestCLI(), requestFor(builds), type)), {
success: [1],
failed: [2, 6, 7, 8, 9, 10],
aborted: [3],
pending: [5],
unstable: [4],
count: 10
});
});

it(`retains the since cutoff while excluding private ${type} builds`, async() => {
const builds = [
build(type, 1, 'FAILURE', 'nodejs', 100),
build(type, 2, 'FAILURE', 'nodejs', 150),
build(type, 3, 'FAILURE', 'nodejs', 200),
build(type, 4, 'FAILURE', 'nodejs-private', 300)
];
const filtered = await listBuilds(new TestCLI(), requestFor(builds), type, new Date(150));
assert.deepEqual(filtered.failed.map(build => build.jobid), [3]);
assert.equal(filtered.count, 1);
});

it(`returns empty result buckets when all ${type} builds are private`, async() => {
const builds = results.map((result, index) =>
build(type, index + 1, result, 'nodejs-private'));
assert.deepEqual(jobids(await listBuilds(new TestCLI(), requestFor(builds), type)), {
success: [], failed: [], aborted: [], pending: [], unstable: [], count: 0
});
});
}
});
Loading