From 5ad69d67c51e48336f7dede81ebc065187992a2f Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 28 Oct 2026 14:43:33 +0530 Subject: [PATCH 1/3] tools: scripts to evaluate contributions Count PRs/issues/reviews in apache/cloudstack* repos and name mentions on lists.apache.org Signed-off-by: Abhishek Kumar --- tools/contribution/README.md | 117 +++++++ tools/contribution/apache_list_mentions.py | 292 ++++++++++++++++++ .../contribution/github_org_contributions.py | 258 ++++++++++++++++ 3 files changed, 667 insertions(+) create mode 100644 tools/contribution/README.md create mode 100644 tools/contribution/apache_list_mentions.py create mode 100755 tools/contribution/github_org_contributions.py diff --git a/tools/contribution/README.md b/tools/contribution/README.md new file mode 100644 index 000000000000..d476751993a3 --- /dev/null +++ b/tools/contribution/README.md @@ -0,0 +1,117 @@ + + +# Contribution Evaluation Scripts + +Helper scripts to gather activity numbers for a set of contributors, e.g. as +input when discussing committer or PMC nominations. The numbers are only a +rough signal; read the actual contributions before drawing conclusions. + +| Script | Data source | What it counts | +|---|---|---| +| `github_org_contributions.py` | GitHub GraphQL API | PRs opened, issues opened and PR reviews per user, in CloudStack repos (by default) of an org | +| `apache_list_mentions.py` | lists.apache.org mbox archives | How often each name (or alias) appears in mailing list message bodies | + +## Requirements + +- Python 3.7+ +- [`requests`](https://pypi.org/project/requests/): `pip install requests` + +## github_org_contributions.py + +Requires a GitHub token in `GITHUB_TOKEN`. A classic token with no scopes (or a +fine-grained token with public read-only access) is enough for public repos. +If the GitHub CLI is logged in, `export GITHUB_TOKEN=$(gh auth token)` works too. + +```bash +export GITHUB_TOKEN= +python3 github_org_contributions.py --org apache --users octocat monalisa +python3 github_org_contributions.py --org apache --users octocat \ + --from 2023-01-01 --to 2025-12-31 --csv contributions.csv +``` + +| Option | Description | +|---|---| +| `--org` | GitHub organization login (required) | +| `--users` | One or more GitHub usernames, space separated (required) | +| `--repo-pattern` | Regex matched (case-insensitively) against repo names; only matching repos are counted. Default: `^cloudstack`. Use `.` for all org repos | +| `--from` | Start, `YYYY-MM-DD` or ISO 8601. Default: one year before `--to` | +| `--to` | End, `YYYY-MM-DD` (inclusive) or ISO 8601. Default: now (UTC) | +| `--csv` | Also write the table to this CSV file | + +Notes: + +- Only repos owned by `--org` whose name matches `--repo-pattern` are counted. + The default `^cloudstack` covers `apache/cloudstack`, `cloudstack-documentation`, + `cloudstack-www`, `cloudstack-cloudmonkey`, `cloudstack-go`, + `cloudstack-terraform-provider` and the other `cloudstack-*` repos. The `repos` + column lists the repos that contributed to the counts. +- Dates without a time are in UTC; `--from` starts at the beginning of that day + and `--to` includes the whole day. +- GitHub limits a query to one year, so longer windows are split into yearly + chunks and summed. +- The counts are the ones GitHub uses for the contribution graph. Contributions + to private repos the token cannot see are not counted. +- Exit status is 1 if any user could not be looked up; the error is shown in the + `error` column. + +## apache_list_mentions.py + +Counts how often each name (or alias) appears in the bodies of a list's +messages over the last N months, including the current month. + +```bash +python3 apache_list_mentions.py --list dev --domain cloudstack.apache.org \ + --months 12 --names names.txt +``` + +| Option | Description | +|---|---| +| `--list` | List name, e.g. `dev`, `users` (required) | +| `--domain` | List domain, e.g. `cloudstack.apache.org` (required) | +| `--names` | Names file, see below (required) | +| `--months` | Number of months back, including the current one. Default: 12 | +| `--cache-dir` | Where downloaded archives are kept. Default: `~/.cache/apache-list-mentions` | +| `--include-quotes` | Also count lines starting with `>` (quoted replies) | +| `--include-signature` | Also count text after a `-- ` signature separator | + +Names file format, one person per line, results are printed in the same order: + +```text +# Lines starting with # are comments +Alex Roe +Jane Doe: Jane, J. Doe +John Smith: John, JS | john@example.org +``` + +- The part before `:` is the name shown in the output; it is always matched + as well as the comma separated aliases after it. +- Anything after `|` (e.g. email addresses) is currently ignored. +- Matching is case-insensitive, on whole words, and any whitespace may separate + the words of a name. + +Notes: + +- The counts lean towards overcounting rather than undercounting: every mention + in a message body counts, including a person's sign-offs in their own + messages, and short or common aliases match wherever they appear. +- Archives of past months are downloaded once and reused from the cache. The + current month is downloaded again on every run. +- The cache holds full copies of the list archives, so its default location is + outside the source tree to keep them from being committed by accident. diff --git a/tools/contribution/apache_list_mentions.py b/tools/contribution/apache_list_mentions.py new file mode 100644 index 000000000000..83eef32039a6 --- /dev/null +++ b/tools/contribution/apache_list_mentions.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Apache mailing list name-mention counter (cached mbox downloads). + +What it does: +- Downloads monthly mbox archives from lists.apache.org ONLY if not already cached locally + (the current, still-incomplete month is always re-downloaded) +- Scans the email bodies (optionally excluding quoted lines and signatures) +- Counts how many times EACH provided name (and its aliases) is mentioned +- Prints counts in the SAME ORDER as your names file (no top-N / no ranking) + +Names file format (names.txt): + # comments allowed + Canonical Name + Canonical Name: alias1, alias2 + Canonical Name: alias1, alias2 | email1, email2 (emails optional; currently not used for mention counting) + +Example: + Jane Doe: Jane, JD + John Smith: John, J. Smith + +Usage: + python3 apache_list_mentions.py --list dev --domain cloudstack.apache.org --months 12 --names names.txt +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import email +import mailbox +import re +import sys +from email.utils import parseaddr +from pathlib import Path +from typing import Dict, Iterable, List, Tuple + +import requests + +MBOX_API = "https://lists.apache.org/api/mbox.lua?list={list}&domain={domain}&d={ym}" +DEFAULT_CACHE_DIR = Path.home() / ".cache" / "apache-list-mentions" + +# Heuristics for cleaning bodies before counting mentions +QUOTE_LINE_RE = re.compile(r"^\s*>") # drop quoted lines (reply history) +SIG_CUTOFF_RE = re.compile(r"^\s*--\s*$") # signature separator + +def utc_now() -> dt.datetime: + return dt.datetime.now(dt.timezone.utc) + +def month_starts(end_utc: dt.datetime, months: int) -> Iterable[dt.date]: + """ + Yield month starts for the last N months ending at end_utc month (inclusive). + """ + y, m = end_utc.year, end_utc.month + for i in range(months): + mm = m - i + yy = y + while mm <= 0: + mm += 12 + yy -= 1 + yield dt.date(yy, mm, 1) + +def download_mbox(list_name: str, domain: str, ym: str, out_dir: Path) -> Path: + """ + Download a single YYYY-MM mbox archive from lists.apache.org and save to out_dir. + """ + url = MBOX_API.format(list=list_name, domain=domain, ym=ym) + out_dir.mkdir(parents=True, exist_ok=True) + out_path = out_dir / f"{list_name}@{domain}-{ym}.mbox" + + headers = {"User-Agent": "apache-list-mentions/1.0"} + r = requests.get(url, headers=headers, timeout=120) + + if r.status_code == 404 or not r.content.strip(): + raise FileNotFoundError(f"No archive found for {ym}") + + r.raise_for_status() + # Write via a temp file so an interrupted download is never reused as a cached archive + tmp_path = out_path.with_suffix(".mbox.part") + tmp_path.write_bytes(r.content) + tmp_path.replace(out_path) + return out_path + +def strip_html(html: str) -> str: + """ + Very simple HTML->text stripping (good enough for mailing list bodies). + """ + html = re.sub(r"(?is)<(script|style).*?>.*?", " ", html) + html = re.sub(r"(?is)<.*?>", " ", html) + html = re.sub(r" | ", " ", html) + html = re.sub(r"&", "&", html) + html = re.sub(r"<", "<", html) + html = re.sub(r">", ">", html) + return html + +def extract_text(msg: email.message.Message) -> str: + """ + Prefer text/plain parts; fallback to text/html stripped; ignore attachments. + """ + if msg.is_multipart(): + plains: List[str] = [] + htmls: List[str] = [] + for part in msg.walk(): + disp = (part.get("Content-Disposition") or "").lower() + if "attachment" in disp: + continue + + ctype = part.get_content_type() + try: + raw = part.get_payload(decode=True) + if raw is None: + continue + text = raw.decode(part.get_content_charset() or "utf-8", "replace") + except Exception: + continue + + if ctype == "text/plain": + plains.append(text) + elif ctype == "text/html": + htmls.append(text) + + if plains: + return "\n".join(plains) + if htmls: + return strip_html("\n".join(htmls)) + return "" + + # non-multipart + ctype = msg.get_content_type() + try: + raw = msg.get_payload(decode=True) + if raw is None: + return "" + text = raw.decode(msg.get_content_charset() or "utf-8", "replace") + except Exception: + return "" + + return strip_html(text) if ctype == "text/html" else text + +def normalize_text(text: str, drop_quotes: bool, drop_signature: bool) -> str: + """ + Remove quoted lines and/or signatures to reduce false mention counts. + """ + lines = text.splitlines() + out: List[str] = [] + for ln in lines: + if drop_signature and SIG_CUTOFF_RE.match(ln): + break + if drop_quotes and QUOTE_LINE_RE.match(ln): + continue + out.append(ln) + return "\n".join(out) + +def parse_names_file(path: Path) -> Tuple[List[str], Dict[str, List[str]]]: + """ + Parse names file while preserving order. + + Supports: + Canonical Name + Canonical Name: alias1, alias2 + Canonical Name: alias1, alias2 | email1, email2 (emails ignored in this script) + + Returns: + ordered_canonicals: list of canonical names in file order + aliases_map: canonical -> [canonical + aliases] + """ + ordered: List[str] = [] + aliases_map: Dict[str, List[str]] = {} + + for raw in path.read_text(encoding="utf-8").splitlines(): + ln = raw.strip() + if not ln or ln.startswith("#"): + continue + + left = ln.split("|", 1)[0].strip() + + if ":" in left: + canon, aliases = left.split(":", 1) + canon = canon.strip() + als = [a.strip() for a in aliases.split(",") if a.strip()] + variants = [canon] + [a for a in als if a.lower() != canon.lower()] + else: + canon = left + variants = [canon] + + ordered.append(canon) + aliases_map[canon] = variants + + return ordered, aliases_map + +def build_mention_regex(aliases_map: Dict[str, List[str]]) -> Dict[str, re.Pattern]: + """ + Build case-insensitive regex per canonical name that matches any alias as a whole token. + """ + out: Dict[str, re.Pattern] = {} + for canon, variants in aliases_map.items(): + pats: List[str] = [] + for v in variants: + v = v.strip() + if not v: + continue + v_esc = re.escape(v) + v_esc = re.sub(r"\\\s+", r"\\s+", v_esc) # flexible whitespace for multi-word names + pats.append(rf"(? int: + ap = argparse.ArgumentParser(description="Count mentions of provided names in Apache mailing list archives.") + ap.add_argument("--list", required=True, help="List name, e.g. dev, users") + ap.add_argument("--domain", required=True, help="List domain, e.g. cloudstack.apache.org") + ap.add_argument("--months", type=int, default=12, help="How many months back (default: 12)") + ap.add_argument("--cache-dir", default=str(DEFAULT_CACHE_DIR), + help=f"Cache directory for downloaded mbox files (default: {DEFAULT_CACHE_DIR})") + ap.add_argument("--names", required=True, help="Path to names file (canonical names + aliases)") + ap.add_argument("--include-quotes", action="store_true", help="Include quoted lines in counting (default: exclude)") + ap.add_argument("--include-signature", action="store_true", help="Include signature in counting (default: exclude)") + args = ap.parse_args() + + cache_dir = Path(args.cache_dir).expanduser() + end = utc_now() + current_ym = f"{end.year:04d}-{end.month:02d}" + + ordered_names, aliases_map = parse_names_file(Path(args.names)) + mention_rx = build_mention_regex(aliases_map) + + mentions = {canon: 0 for canon in ordered_names} + + downloaded = 0 + reused = 0 + missing = 0 + total_msgs = 0 + + for d in month_starts(end, args.months): + ym = f"{d.year:04d}-{d.month:02d}" + mbox_path = cache_dir / f"{args.list}@{args.domain}-{ym}.mbox" + + try: + # The current month is still receiving mail, so never trust its cached copy + if mbox_path.exists() and ym != current_ym: + reused += 1 + else: + mbox_path = download_mbox(args.list, args.domain, ym, cache_dir) + downloaded += 1 + + mbox = mailbox.mbox(mbox_path) + for msg in mbox: + total_msgs += 1 + body = extract_text(msg) + body = normalize_text( + body, + drop_quotes=not args.include_quotes, + drop_signature=not args.include_signature, + ) + for canon in ordered_names: + mentions[canon] += len(mention_rx[canon].findall(body)) + + except FileNotFoundError: + missing += 1 + except Exception as e: + print(f"WARN: {ym}: {e}", file=sys.stderr) + + print(f"List: {args.list}@{args.domain}") + print(f"Months scanned: {args.months} (missing months: {missing})") + print(f"Archives reused: {reused}, downloaded this run: {downloaded}") + print(f"Total messages parsed: {total_msgs}") + print("") + print("Name mentions (in file order):") + width = max((len(n) for n in ordered_names), default=10) + for canon in ordered_names: + print(f"{canon:<{width}} : {mentions.get(canon, 0)}") + + return 0 + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/contribution/github_org_contributions.py b/tools/contribution/github_org_contributions.py new file mode 100755 index 000000000000..516c5b32248b --- /dev/null +++ b/tools/contribution/github_org_contributions.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +GitHub org activity counters for a set of users (GraphQL). + +Counts (scoped to org-owned repos whose name matches --repo-pattern, default ^cloudstack): +- Pull requests created (opened) +- Issues created (opened) +- Pull request reviews + +Time window: +- If --from is NOT provided → defaults to the year before --to +- Windows longer than a year are split into yearly chunks (GitHub limitation) and summed +- A date-only --to (YYYY-MM-DD) includes that whole day + +Auth: + export GITHUB_TOKEN=ghp_... + +Examples: + python3 github_org_contributions.py --org apache --users octocat + python3 github_org_contributions.py --org apache --users octocat monalisa --from 2025-02-01 --to 2026-01-31 + +See README.md in this directory for details and caveats. +""" + +import argparse +import csv +import datetime as dt +import os +import re +import sys +from typing import Any, Dict, Iterable, List, Tuple + +import requests + +GQL_ENDPOINT = "https://api.github.com/graphql" +# GitHub's upper limit for the *ContributionsByRepository fields +MAX_REPOSITORIES = 100 + +COUNTERS = { + "prs_created": "pullRequestContributionsByRepository", + "issues_created": "issueContributionsByRepository", + "pr_reviews": "pullRequestReviewContributionsByRepository", +} + + +def utc_now() -> dt.datetime: + return dt.datetime.now(dt.timezone.utc) + + +def iso_to_datetime(s: str, end_of_day: bool = False) -> dt.datetime: + if len(s) == 10: # YYYY-MM-DD + d = dt.datetime.fromisoformat(s).replace(tzinfo=dt.timezone.utc) + return d + dt.timedelta(days=1) if end_of_day else d + if s.endswith(("Z", "z")): # fromisoformat() only accepts "Z" from Python 3.11 + s = s[:-1] + "+00:00" + d = dt.datetime.fromisoformat(s) + if d.tzinfo is None: + d = d.replace(tzinfo=dt.timezone.utc) + return d.astimezone(dt.timezone.utc) + + +def shift_years(d: dt.datetime, years: int) -> dt.datetime: + try: + return d.replace(year=d.year + years) + except ValueError: # Feb 29 + return d.replace(year=d.year + years, day=28) + + +def year_chunks(from_dt: dt.datetime, to_dt: dt.datetime) -> Iterable[Tuple[dt.datetime, dt.datetime]]: + """ + Split [from_dt, to_dt] into consecutive windows of at most one year. + """ + start = from_dt + while start < to_dt: + end = min(shift_years(start, 1), to_dt) + yield start, end + start = end + + +def graphql(token: str, query: str, variables: Dict[str, Any]) -> Dict[str, Any]: + headers = { + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.github+json", + } + r = requests.post(GQL_ENDPOINT, json={"query": query, "variables": variables}, headers=headers, timeout=60) + r.raise_for_status() + payload = r.json() + if payload.get("errors"): + msgs = "; ".join(e.get("message", "Unknown error") for e in payload["errors"]) + raise RuntimeError(f"GraphQL error(s): {msgs}") + return payload["data"] + + +ORG_ID_QUERY = """ +query($org: String!) { + organization(login: $org) { id login } +} +""" + +USER_CONTRIB_QUERY = """ +query($login: String!, $orgId: ID!, $from: DateTime!, $to: DateTime!, $maxRepos: Int!) { + user(login: $login) { + login + contributionsCollection(organizationID: $orgId, from: $from, to: $to) { + pullRequestContributionsByRepository(maxRepositories: $maxRepos) { + repository { name owner { login } } + contributions { totalCount } + } + issueContributionsByRepository(maxRepositories: $maxRepos) { + repository { name owner { login } } + contributions { totalCount } + } + pullRequestReviewContributionsByRepository(maxRepositories: $maxRepos) { + repository { name owner { login } } + contributions { totalCount } + } + } + } +} +""" + + +def count_user(token: str, login: str, org_id: str, org_login: str, repo_rx: re.Pattern, + from_dt: dt.datetime, to_dt: dt.datetime) -> Dict[str, Any]: + """ + Sum the user's contributions to matching org repos over all yearly chunks of the window. + """ + counts = {k: 0 for k in COUNTERS} + repos = set() + user_login = login + for start, end in year_chunks(from_dt, to_dt): + data = graphql(token, USER_CONTRIB_QUERY, { + "login": login, + "orgId": org_id, + "from": start.isoformat(), + "to": end.isoformat(), + "maxRepos": MAX_REPOSITORIES, + }) + user = data.get("user") + if not user: + raise RuntimeError("User not found") + user_login = user["login"] + + cc = user["contributionsCollection"] + for key, field in COUNTERS.items(): + entries = cc[field] + if len(entries) >= MAX_REPOSITORIES: + print(f"WARN: {login}: {field} hit the {MAX_REPOSITORIES} repository limit for " + f"{start.date()} -> {end.date()}; counts may be incomplete", file=sys.stderr) + for e in entries: + repo = e["repository"] + if repo["owner"]["login"].lower() != org_login.lower() or not repo_rx.search(repo["name"]): + continue + counts[key] += e["contributions"]["totalCount"] + repos.add(repo["name"]) + + return {"user": user_login, **counts, "repos": " ".join(sorted(repos)), "error": ""} + + +def main() -> int: + ap = argparse.ArgumentParser(description="Count PRs, issues, and PR reviews for GitHub users within an org.") + ap.add_argument("--org", required=True, help="GitHub organization login (e.g. apache)") + ap.add_argument("--users", nargs="+", required=True, help="GitHub usernames") + ap.add_argument("--repo-pattern", default="^cloudstack", + help="Regex matched against repo names; only matching repos are counted (default: ^cloudstack). " + "Use '.' to count all org repos") + ap.add_argument("--from", dest="from_", default=None, + help="Start date (YYYY-MM-DD or ISO). Default: one year before --to") + ap.add_argument("--to", default=None, + help="End date (YYYY-MM-DD, inclusive, or ISO). Default: now (UTC)") + ap.add_argument("--csv", default=None, help="Optional CSV output path") + args = ap.parse_args() + + token = os.environ.get("GITHUB_TOKEN") + if not token: + print("ERROR: GITHUB_TOKEN is not set", file=sys.stderr) + return 2 + + try: + repo_rx = re.compile(args.repo_pattern, re.IGNORECASE) + except re.error as e: + print(f"ERROR: invalid --repo-pattern: {e}", file=sys.stderr) + return 2 + + try: + to_dt = iso_to_datetime(args.to, end_of_day=True) if args.to else utc_now() + from_dt = iso_to_datetime(args.from_) if args.from_ else shift_years(to_dt, -1) + except ValueError as e: + print(f"ERROR: invalid date: {e}", file=sys.stderr) + return 2 + + if to_dt <= from_dt: + print("ERROR: --to must be after --from", file=sys.stderr) + return 2 + + try: + org = graphql(token, ORG_ID_QUERY, {"org": args.org}).get("organization") + except Exception as e: + print(f"ERROR: failed to look up organization '{args.org}': {e}", file=sys.stderr) + return 2 + if not org: + print(f"ERROR: Organization '{args.org}' not found or inaccessible", file=sys.stderr) + return 2 + + rows: List[Dict[str, Any]] = [] + + for u in args.users: + try: + rows.append(count_user(token, u, org["id"], org["login"], repo_rx, from_dt, to_dt)) + except Exception as e: + rows.append({"user": u, **{k: None for k in COUNTERS}, "repos": "", "error": str(e)}) + + headers = ["user", *COUNTERS, "repos", "error"] + + # Print table + colw = {h: len(h) for h in headers} + for r in rows: + for h in headers: + colw[h] = max(colw[h], len("" if r[h] is None else str(r[h]))) + + def fmt(r): + return " ".join(("" if r[h] is None else str(r[h])).ljust(colw[h]) for h in headers).rstrip() + + print(f"Org: {org['login']} (repos matching: {args.repo_pattern})") + print(f"Window (UTC): {from_dt.isoformat()} -> {to_dt.isoformat()}") + print(fmt({h: h for h in headers})) + print("-" * (sum(colw.values()) + 2 * (len(headers) - 1))) + for r in rows: + print(fmt(r)) + + if args.csv: + with open(args.csv, "w", newline="", encoding="utf-8") as f: + w = csv.DictWriter(f, fieldnames=headers) + w.writeheader() + w.writerows(rows) + print(f"\nWrote CSV: {args.csv}") + + return 1 if any(r["error"] for r in rows) else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From a096cf31580a8d7626893c55a281243d7fd0cb81 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 28 Oct 2026 17:21:42 +0530 Subject: [PATCH 2/3] address copilot comments Signed-off-by: Abhishek Kumar --- tools/contribution/README.md | 2 +- tools/contribution/apache_list_mentions.py | 28 +++++++++++++------ .../contribution/github_org_contributions.py | 4 +-- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/tools/contribution/README.md b/tools/contribution/README.md index d476751993a3..e1bf4e0f9cc6 100644 --- a/tools/contribution/README.md +++ b/tools/contribution/README.md @@ -30,7 +30,7 @@ rough signal; read the actual contributions before drawing conclusions. ## Requirements -- Python 3.7+ +- Python 3.8+ - [`requests`](https://pypi.org/project/requests/): `pip install requests` ## github_org_contributions.py diff --git a/tools/contribution/apache_list_mentions.py b/tools/contribution/apache_list_mentions.py index 83eef32039a6..1d1079d395a2 100644 --- a/tools/contribution/apache_list_mentions.py +++ b/tools/contribution/apache_list_mentions.py @@ -47,7 +47,6 @@ import mailbox import re import sys -from email.utils import parseaddr from pathlib import Path from typing import Dict, Iterable, List, Tuple @@ -85,15 +84,26 @@ def download_mbox(list_name: str, domain: str, ym: str, out_dir: Path) -> Path: out_path = out_dir / f"{list_name}@{domain}-{ym}.mbox" headers = {"User-Agent": "apache-list-mentions/1.0"} - r = requests.get(url, headers=headers, timeout=120) - - if r.status_code == 404 or not r.content.strip(): + with requests.get(url, headers=headers, timeout=120, stream=True) as r: + if r.status_code == 404: + raise FileNotFoundError(f"No archive found for {ym}") + r.raise_for_status() + + # Stream to a temp file so an interrupted download is never reused as a cached archive + tmp_path = out_path.with_suffix(".mbox.part") + has_content = False + try: + with tmp_path.open("wb") as f: + for chunk in r.iter_content(chunk_size=1024 * 1024): + has_content = has_content or bool(chunk.strip()) + f.write(chunk) + except BaseException: + tmp_path.unlink(missing_ok=True) + raise + + if not has_content: + tmp_path.unlink(missing_ok=True) raise FileNotFoundError(f"No archive found for {ym}") - - r.raise_for_status() - # Write via a temp file so an interrupted download is never reused as a cached archive - tmp_path = out_path.with_suffix(".mbox.part") - tmp_path.write_bytes(r.content) tmp_path.replace(out_path) return out_path diff --git a/tools/contribution/github_org_contributions.py b/tools/contribution/github_org_contributions.py index 516c5b32248b..5c4c44d6c394 100755 --- a/tools/contribution/github_org_contributions.py +++ b/tools/contribution/github_org_contributions.py @@ -84,7 +84,7 @@ def shift_years(d: dt.datetime, years: int) -> dt.datetime: def year_chunks(from_dt: dt.datetime, to_dt: dt.datetime) -> Iterable[Tuple[dt.datetime, dt.datetime]]: """ - Split [from_dt, to_dt] into consecutive windows of at most one year. + Split [from_dt, to_dt) into consecutive end-exclusive windows of at most one year. """ start = from_dt while start < to_dt: @@ -154,7 +154,7 @@ def count_user(token: str, login: str, org_id: str, org_login: str, repo_rx: re. }) user = data.get("user") if not user: - raise RuntimeError("User not found") + raise RuntimeError(f"User not found: {login}") user_login = user["login"] cc = user["contributionsCollection"] From c20c84d1fab005fc1f4487bf28d86b60282b471b Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 28 Oct 2026 17:54:36 +0530 Subject: [PATCH 3/3] fix Signed-off-by: Abhishek Kumar --- tools/contribution/README.md | 8 ++++---- tools/contribution/apache_list_mentions.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/contribution/README.md b/tools/contribution/README.md index e1bf4e0f9cc6..ffa9fea6c851 100644 --- a/tools/contribution/README.md +++ b/tools/contribution/README.md @@ -24,7 +24,7 @@ input when discussing committer or PMC nominations. The numbers are only a rough signal; read the actual contributions before drawing conclusions. | Script | Data source | What it counts | -|---|---|---| +| --- | --- | --- | | `github_org_contributions.py` | GitHub GraphQL API | PRs opened, issues opened and PR reviews per user, in CloudStack repos (by default) of an org | | `apache_list_mentions.py` | lists.apache.org mbox archives | How often each name (or alias) appears in mailing list message bodies | @@ -47,7 +47,7 @@ python3 github_org_contributions.py --org apache --users octocat \ ``` | Option | Description | -|---|---| +| --- | --- | | `--org` | GitHub organization login (required) | | `--users` | One or more GitHub usernames, space separated (required) | | `--repo-pattern` | Regex matched (case-insensitively) against repo names; only matching repos are counted. Default: `^cloudstack`. Use `.` for all org repos | @@ -82,14 +82,14 @@ python3 apache_list_mentions.py --list dev --domain cloudstack.apache.org \ ``` | Option | Description | -|---|---| +| --- | --- | | `--list` | List name, e.g. `dev`, `users` (required) | | `--domain` | List domain, e.g. `cloudstack.apache.org` (required) | | `--names` | Names file, see below (required) | | `--months` | Number of months back, including the current one. Default: 12 | | `--cache-dir` | Where downloaded archives are kept. Default: `~/.cache/apache-list-mentions` | | `--include-quotes` | Also count lines starting with `>` (quoted replies) | -| `--include-signature` | Also count text after a `-- ` signature separator | +| `--include-signature` | Also count text after the signature separator (a line containing only `--`) | Names file format, one person per line, results are printed in the same order: diff --git a/tools/contribution/apache_list_mentions.py b/tools/contribution/apache_list_mentions.py index 1d1079d395a2..6f1e8d380a80 100644 --- a/tools/contribution/apache_list_mentions.py +++ b/tools/contribution/apache_list_mentions.py @@ -203,8 +203,8 @@ def parse_names_file(path: Path) -> Tuple[List[str], Dict[str, List[str]]]: if ":" in left: canon, aliases = left.split(":", 1) canon = canon.strip() - als = [a.strip() for a in aliases.split(",") if a.strip()] - variants = [canon] + [a for a in als if a.lower() != canon.lower()] + alias_list = [a.strip() for a in aliases.split(",") if a.strip()] + variants = [canon] + [a for a in alias_list if a.lower() != canon.lower()] else: canon = left variants = [canon]