fix(holdings): reject broken fresh holdings with the shared holdings guard

scripts/update-data.test.ts

@@ -6,7 +6,7 @@ import { join } from 'node:path';
 import { pathToFileURL } from 'node:url';
 import {
   CONTROL_NAMES, HISTORY_RANGE_PATTERN, RETURNS_BASIS_DERIVED, RETURNS_BASIS_OFFICIAL,
-  annualizedToTotal, aumSource, cleanHoldingTicker, deriveMetrics, dividendYieldBasisFromKind, formatDate, hasConfiguredFilters, inferDistributionFrequency, installSystemCa,
+  annualizedToTotal, aumSource, cleanHoldingTicker, deriveMetrics, dividendYieldBasisFromKind, formatDate, hasConfiguredFilters, holdingsGuard, inferDistributionFrequency, installSystemCa,
   isCertError, isinFromCusip, looksLikeProductPage, mergeDistributionRecords, normalizeHoldingName, nportUrlFor, numberOrNull, parseAumRange,
   parseCatalogMarkdown, parseChart, parseFundListJson, parseEdgarAtomFilings, parseFundTickerMap, parseNport, parseOfficialDistributions,
   parseProductPageSummary, parseRange, parseRanges, performanceAsOfDate, priceReturns, readConfig, readKnownRows, resolveControls,
@@ -258,6 +258,21 @@ describe('parsing', () => {
     expect(parseEdgarAtomFilings(atom)[0].url).toContain('0000000000-26-000001.txt');
   });
 
+  test('holdings guard: header-only, renamed weight column, broken weights, a bad weight sum and a halved sheet are rejected', () => {
+    const headers = ['Name', 'Ticker', 'Weight'];
+    const rows = [{ Name: 'A', Ticker: 'A', Weight: '60%' }, { Name: 'B', Ticker: 'B', Weight: '40%' }];
+    expect(holdingsGuard(headers, rows, 2)).toBeNull();
+    expect(holdingsGuard(headers, [['A', 'A', '60'], ['B', 'B', '40']], null)).toBeNull();
+    expect(holdingsGuard([], [], 0)).toBeNull(); // nothing published before: an empty sheet is honest
+    expect(holdingsGuard(headers, [], 505)).toContain('no holdings rows');
+    expect(holdingsGuard(['Name', 'Ticker', 'Wt'], rows, 2)).toContain('no weight column');
+    expect(holdingsGuard(headers, rows.map((row) => ({ ...row, Weight: '6\u200B0' })), 2)).toContain('numeric weight');
+    const allocation = [{ Name: 'Cash&Collateral', Ticker: '-', Weight: '100' }, { Name: 'Equities', Ticker: '-', Weight: '476' }];
+    expect(holdingsGuard(headers, allocation, 2)).toContain('outside 50..150');
+    expect(holdingsGuard(headers, allocation, 2, ['sum'])).toBeNull();
+    expect(holdingsGuard(headers, rows, 5)).toContain('less than half');
+  });
+
   test('Yahoo chart: close, adjusted close and dividends', () => {
     const parsed = parseChart({ chart: { result: [{ timestamp: [Date.UTC(2025, 0, 2) / 1000, Date.UTC(2025, 0, 3) / 1000], indicators: { quote: [{ close: [10, 11], volume: [100, 200] }], adjclose: [{ adjclose: [9, 10] }] }, events: { dividends: { '1735776000': { amount: 0.25 } } }, meta: { regularMarketPrice: 11, exchangeName: 'NYSEArca' } }] } });
     expect(parsed.days[1]).toMatchObject({ date: '2025-01-03', close: 11, adjClose: 10, volume: 200 });
@@ -409,7 +424,7 @@ function secResponse(url: string, nportAsOf: string): Response | null {
   if (url.includes('company_tickers_mf')) return new Response(JSON.stringify({ fields: ['cik', 'seriesId', 'classId', 'symbol'], data: [['1350487', 'S000043966', 'C000136444', 'AAA']] }));
   if (url.includes('company_tickers')) return new Response('{}');
   if (url.includes('browse-edgar')) return new Response('<feed><entry><filing-type>NPORT-P</filing-type><accession-number>0000940400-26-000001</accession-number><filing-date>2026-07-30</filing-date></entry></feed>');
-  if (url.includes('/Archives/edgar/')) return new Response(`<edgarSubmission><genInfo><regCik>0001350487</regCik><repPdDate>${nportAsOf}</repPdDate><seriesName>Fund AAA</seriesName><seriesId>S000043966</seriesId></genInfo><fundInfo><netAssets>1000000</netAssets></fundInfo><invstOrSecs><invstOrSec><name>NPORT CO ${nportAsOf}</name><cusip>999999999</cusip><pctVal>5</pctVal><valUSD>50</valUSD><balance>1</balance><assetCat>EC</assetCat></invstOrSec></invstOrSecs></edgarSubmission>`);
+  if (url.includes('/Archives/edgar/')) return new Response(`<edgarSubmission><genInfo><regCik>0001350487</regCik><repPdDate>${nportAsOf}</repPdDate><seriesName>Fund AAA</seriesName><seriesId>S000043966</seriesId></genInfo><fundInfo><netAssets>1000000</netAssets></fundInfo><invstOrSecs><invstOrSec><name>NPORT CO ${nportAsOf}</name><cusip>999999999</cusip><pctVal>100</pctVal><valUSD>50</valUSD><balance>1</balance><assetCat>EC</assetCat></invstOrSec></invstOrSecs></edgarSubmission>`);
   return new Response('nope', { status: 404 });
 }
 
@@ -696,6 +711,27 @@ describe('pipeline', () => {
     expect(s.index().funds.map((f: any) => f.ticker)).toContain('CCC');
   });
 
+  test('fresh holdings that fail the guard keep the fund exactly as published and report it failed', async () => {
+    // BBB has no N-PORT series in the SEC world, so its holdings come from the product-page top holdings table
+    const s = new Scenario(['BBB']);
+    const base = s.handler;
+    const serve = (...rows: string[]) => {
+      const page = `${productPage('BBB')}\n\n### Holdings\n\n#### As of 9/15/2026\n\n| Name | Weight (%) |\n| --- | --- |\n${rows.join('\n')}\n\nView all holdings`;
+      s.handler = (url, init) => (url.startsWith('https://r.jina.ai/') && !url.endsWith('/us/products') ? new Response(page) : base(url, init));
+    };
+    serve('| Alpha Corp | 40.00 |', '| Beta Corp | 30.00 |', '| Gamma Corp | 20.00 |', '| Delta Corp | 10.00 |');
+    await s.run({ EDGAR_FALLBACK: 'true' });
+    expect(s.meta('BBB').holdings.totalRows).toBe(4);
+    s.backdate();
+    const before = s.files().map((path) => readFileSync(path, 'utf8'));
+    serve('| Alpha Corp | 90.00 |'); // one row of four left: a truncated page, not a new portfolio
+    process.exitCode = 0;
+    await s.run({ EDGAR_FALLBACK: 'true', TICKERS: 'BBB' });
+    expect(s.touched()).toEqual([]);
+    expect(s.files().map((path) => readFileSync(path, 'utf8'))).toEqual(before);
+    expect(process.exitCode).toBe(1);
+  });
+
   test('N-PORT freshness: a filing older than the published holdings never replaces them, a newer one does', async () => {
     const s = new Scenario(['AAA']);
     await s.run({ EDGAR_FALLBACK: 'true' });

scripts/update-data.ts

@@ -232,6 +232,10 @@ export function setApiRoot(root: URL): void {
 }
 
 const HOLDINGS_HEADERS = ['Name', 'Ticker', 'Identifier', 'Weight', 'Market Value', 'Shares Held', 'Asset Category'];
+/** Funds whose holdings legitimately fail a holdingsGuard check (geared, futures, look-through, top-N lists): ticker, or '*' for every fund -> relaxed checks. */
+const HOLDINGS_GUARD_RELAX: Record<string, string[]> = {
+  WSPC: ['sum'], // no N-PORT filing yet: the product page lists only its top 10 holdings (about 47% of the fund)
+};
 const MONTHS = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
 const TRUTHY = new Set(['1', 'true', 'yes', 'y', 'on']);
 const AUM_BOUNDS = { nano: [0, 10_000_000], micro: [10_000_000, 300_000_000], small: [300_000_000, 2_000_000_000], mid: [2_000_000_000, 10_000_000_000], large: [10_000_000_000, undefined] } as const;
@@ -1815,6 +1819,7 @@ async function processFund(fund: CatalogFund, config: UpdaterConfig, previous: J
   let holdingsAsOf: string | null = null;
   let holdingsSource = 'not available from current public SEC filing';
   let nport: ParsedNport | null = null;
+  let holdingsFresh = false; // false while holdingsRows is the previous run's sheet
   if (config.edgarFallback) {
     try {
       const filing = await resolveNportFiling(fund, config);
@@ -1827,6 +1832,7 @@ async function processFund(fund: CatalogFund, config: UpdaterConfig, previous: J
         if (seriesMatches && fresher && parsed.holdings.length) {
           const names = await loadCompanyTickerTable(config);
           holdingsRows = fillNportTickers(parsed.holdings, names);
+          holdingsFresh = true;
           holdingsHeaders = holdingsRows.some((row) => 'Coupon' in row || 'Maturity' in row) ? [...HOLDINGS_HEADERS, 'Coupon', 'Maturity'] : HOLDINGS_HEADERS;
           holdingsAsOf = parsed.repPdDate || null;
           nport = parsed;
@@ -1843,6 +1849,7 @@ async function processFund(fund: CatalogFund, config: UpdaterConfig, previous: J
     const top = parseTopHoldingsMarkdown(productPageMarkdown, fund.ticker, companyTickerMap || new Map());
     if (top.rows.length && (!previousMeta?.holdings?.totalRows || /^WisdomTree product page/.test(String(previousMeta.holdings.source || '')))) {
       holdingsRows = top.rows;
+      holdingsFresh = true;
       holdingsHeaders = top.headers;
       holdingsAsOf = top.asOfDate;
       holdingsSource = 'WisdomTree product page top holdings (issuer fallback; full N-PORT filing unavailable)';
@@ -1909,6 +1916,11 @@ async function processFund(fund: CatalogFund, config: UpdaterConfig, previous: J
     return { __skipped: true, ticker: fund.ticker, __skipReasons: returnFilterReasons };
   }
 
+  // Fresh holdings that look broken (no weight column, non-numeric weights, a bad weight sum, half the rows gone) never replace
+  // the published ones: the fund fails before anything of it is written, so it stays exactly as published.
+  const holdingsProblem = holdingsFresh ? holdingsGuard(holdingsHeaders, holdingsRows, numberOrNull(previousMeta?.holdings?.totalRows), HOLDINGS_GUARD_RELAX[fund.ticker] ?? HOLDINGS_GUARD_RELAX['*'] ?? []) : null;
+  if (holdingsProblem) throw new Error(`holdings guard: ${holdingsProblem}; ${previousMeta ? 'kept the previous complete data' : 'no data published for this fund yet'}`);
+
   const history = historyRows(days, previousRows);
   const historyAsOf = derived.asOfDate || previousMeta?.history?.asOf || null;
   const holdingPages = await writePages(fundDir, fund.ticker, 'holdings', holdingsHeaders, holdingsRows, config.holdingsPageSize, holdingsAsOf, holdingsSource);
@@ -2495,3 +2507,49 @@ if ((import.meta as { main?: boolean }).main) {
   if (process.argv.some((arg) => ['-h', '--help', 'help'].includes(arg))) console.log(USAGE.trim());
   else await main().catch((error) => { console.error(error instanceof Error ? error.stack : String(error)); process.exitCode = 1; });
 }
+
+// <holdings-guard> shared block of .claude/tools/etfs/etf-std/holdings-guard.block.ts, applied by apply-holdings-guard.ts: never edit it in one repo
+/**
+ * Sanity check of a fund's new holdings before they replace the published ones. Returns why they must be rejected
+ * (the caller then keeps the fund's previous state and reports it as failed), or null when they may be written.
+ * An empty sheet passes only when nothing was published before. Otherwise a Name and a weight column (Weight,
+ * Weight (%) or Market Weight) are required, >= 80% of the rows carry a numeric weight, the weights add up to
+ * 50..150 and the sheet keeps >= 50% of the previously published rows. `relax` switches off the checks a fund
+ * legitimately fails: 'numeric' and 'sum' (geared, inverse, futures, look-through, top-N lists) and 'count'.
+ * Rows are arrays in header order or objects keyed by the header text.
+ */
+export function holdingsGuard(headers: ReadonlyArray<string>, rows: ReadonlyArray<unknown>, previousRows: number | null | undefined, relax: ReadonlyArray<string> = []): string | null {
+  const previous = Number(previousRows) > 0 ? Number(previousRows) : 0;
+  if (!rows.length) return previous ? `no holdings rows (${previous} published)` : null;
+  const names = headers.map((header) => String(header).trim().toLowerCase());
+  const nameAt = names.indexOf('name');
+  let weightAt = names.indexOf('weight');
+  if (weightAt < 0) weightAt = names.indexOf('weight (%)');
+  if (weightAt < 0) weightAt = names.indexOf('market weight');
+  if (nameAt < 0 || weightAt < 0) return `no ${nameAt < 0 ? 'Name' : 'weight'} column (headers: ${headers.join(', ') || 'none'})`;
+  let numeric = 0;
+  let sum = 0;
+  for (let i = 0; i < rows.length; i++) {
+    const row: any = rows[i];
+    const weight = holdingsGuardNumber(Array.isArray(row) ? row[weightAt] : row && typeof row === 'object' ? row[headers[weightAt]] : undefined);
+    if (weight !== null) {
+      numeric++;
+      sum += weight;
+    }
+  }
+  if (relax.indexOf('numeric') < 0 && numeric < rows.length * 0.8) return `only ${numeric} of ${rows.length} rows have a numeric weight`;
+  if (relax.indexOf('sum') < 0 && (sum < 50 || sum > 150)) return `weights add up to ${sum.toFixed(2)}, outside 50..150`;
+  if (relax.indexOf('count') < 0 && previous && rows.length < previous * 0.5) return `${rows.length} rows, less than half of the ${previous} published`;
+  return null;
+}
+
+/** A weight cell as a number: %, $, commas and spaces are ignored, (x) is negative; null when it is no plain number (a zero-width character inside counts as broken). */
+function holdingsGuardNumber(value: unknown): number | null {
+  if (typeof value === 'number') return Number.isFinite(value) ? value : null;
+  let text = String(value === null || value === undefined ? '' : value).replace(/[ \t ,%$]/g, '').replace(/−/g, '-');
+  const negative = /^\(.*\)$/.test(text);
+  if (negative) text = text.slice(1, -1);
+  if (!/^[-+]?(\d+\.?\d*|\.\d+)(e[-+]?\d+)?$/i.test(text)) return null;
+  return negative ? -Number(text) : Number(text);
+}
+// </holdings-guard>