From f2811f16affa0200731f25daff41528513d4d5f8 Mon Sep 17 00:00:00 2001 From: Maksim Kostromin Date: Sat, 24 Oct 2026 23:50:03 -0400 Subject: [PATCH] fix(holdings): reject broken fresh holdings with the shared holdings guard Fresh N-PORT or product-page holdings that fail holdingsGuard (no weight column, non-numeric weights, weights outside 50..150, less than half the published rows) fail the fund before anything of it is written, so it stays exactly as published. WSPC is relaxed for the sum check: its product page lists only the top 10 holdings. --- scripts/update-data.test.ts | 40 +++++++++++++++++++++++-- scripts/update-data.ts | 58 +++++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 2 deletions(-) diff --git a/scripts/update-data.test.ts b/scripts/update-data.test.ts index f907881..3d3883f 100644 --- a/scripts/update-data.test.ts +++ b/scripts/update-data.test.ts @@ -6,7 +6,7 @@ import { join } from 'node:path'; import { pathToFileURL } from 'node:url'; import { CONTROL_NAMES, HISTORY_RANGE_PATTERN, RETURNS_BASIS_DERIVED, RETURNS_BASIS_OFFICIAL, - annualizedToTotal, aumSource, cleanHoldingTicker, deriveMetrics, dividendYieldBasisFromKind, formatDate, hasConfiguredFilters, inferDistributionFrequency, installSystemCa, + annualizedToTotal, aumSource, cleanHoldingTicker, deriveMetrics, dividendYieldBasisFromKind, formatDate, hasConfiguredFilters, holdingsGuard, inferDistributionFrequency, installSystemCa, isCertError, isinFromCusip, looksLikeProductPage, mergeDistributionRecords, normalizeHoldingName, nportUrlFor, numberOrNull, parseAumRange, parseCatalogMarkdown, parseChart, parseFundListJson, parseEdgarAtomFilings, parseFundTickerMap, parseNport, parseOfficialDistributions, parseProductPageSummary, parseRange, parseRanges, performanceAsOfDate, priceReturns, readConfig, readKnownRows, resolveControls, @@ -258,6 +258,21 @@ describe('parsing', () => { expect(parseEdgarAtomFilings(atom)[0].url).toContain('0000000000-26-000001.txt'); }); + test('holdings guard: header-only, renamed weight column, broken weights, a bad weight sum and a halved sheet are rejected', () => { + const headers = ['Name', 'Ticker', 'Weight']; + const rows = [{ Name: 'A', Ticker: 'A', Weight: '60%' }, { Name: 'B', Ticker: 'B', Weight: '40%' }]; + expect(holdingsGuard(headers, rows, 2)).toBeNull(); + expect(holdingsGuard(headers, [['A', 'A', '60'], ['B', 'B', '40']], null)).toBeNull(); + expect(holdingsGuard([], [], 0)).toBeNull(); // nothing published before: an empty sheet is honest + expect(holdingsGuard(headers, [], 505)).toContain('no holdings rows'); + expect(holdingsGuard(['Name', 'Ticker', 'Wt'], rows, 2)).toContain('no weight column'); + expect(holdingsGuard(headers, rows.map((row) => ({ ...row, Weight: '6\u200B0' })), 2)).toContain('numeric weight'); + const allocation = [{ Name: 'Cash&Collateral', Ticker: '-', Weight: '100' }, { Name: 'Equities', Ticker: '-', Weight: '476' }]; + expect(holdingsGuard(headers, allocation, 2)).toContain('outside 50..150'); + expect(holdingsGuard(headers, allocation, 2, ['sum'])).toBeNull(); + expect(holdingsGuard(headers, rows, 5)).toContain('less than half'); + }); + test('Yahoo chart: close, adjusted close and dividends', () => { const parsed = parseChart({ chart: { result: [{ timestamp: [Date.UTC(2025, 0, 2) / 1000, Date.UTC(2025, 0, 3) / 1000], indicators: { quote: [{ close: [10, 11], volume: [100, 200] }], adjclose: [{ adjclose: [9, 10] }] }, events: { dividends: { '1735776000': { amount: 0.25 } } }, meta: { regularMarketPrice: 11, exchangeName: 'NYSEArca' } }] } }); expect(parsed.days[1]).toMatchObject({ date: '2025-01-03', close: 11, adjClose: 10, volume: 200 }); @@ -409,7 +424,7 @@ function secResponse(url: string, nportAsOf: string): Response | null { if (url.includes('company_tickers_mf')) return new Response(JSON.stringify({ fields: ['cik', 'seriesId', 'classId', 'symbol'], data: [['1350487', 'S000043966', 'C000136444', 'AAA']] })); if (url.includes('company_tickers')) return new Response('{}'); if (url.includes('browse-edgar')) return new Response('NPORT-P0000940400-26-0000012026-07-30'); - if (url.includes('/Archives/edgar/')) return new Response(`0001350487${nportAsOf}Fund AAAS0000439661000000NPORT CO ${nportAsOf}9999999995501EC`); + if (url.includes('/Archives/edgar/')) return new Response(`0001350487${nportAsOf}Fund AAAS0000439661000000NPORT CO ${nportAsOf}999999999100501EC`); return new Response('nope', { status: 404 }); } @@ -696,6 +711,27 @@ describe('pipeline', () => { expect(s.index().funds.map((f: any) => f.ticker)).toContain('CCC'); }); + test('fresh holdings that fail the guard keep the fund exactly as published and report it failed', async () => { + // BBB has no N-PORT series in the SEC world, so its holdings come from the product-page top holdings table + const s = new Scenario(['BBB']); + const base = s.handler; + const serve = (...rows: string[]) => { + const page = `${productPage('BBB')}\n\n### Holdings\n\n#### As of 9/15/2026\n\n| Name | Weight (%) |\n| --- | --- |\n${rows.join('\n')}\n\nView all holdings`; + s.handler = (url, init) => (url.startsWith('https://r.jina.ai/') && !url.endsWith('/us/products') ? new Response(page) : base(url, init)); + }; + serve('| Alpha Corp | 40.00 |', '| Beta Corp | 30.00 |', '| Gamma Corp | 20.00 |', '| Delta Corp | 10.00 |'); + await s.run({ EDGAR_FALLBACK: 'true' }); + expect(s.meta('BBB').holdings.totalRows).toBe(4); + s.backdate(); + const before = s.files().map((path) => readFileSync(path, 'utf8')); + serve('| Alpha Corp | 90.00 |'); // one row of four left: a truncated page, not a new portfolio + process.exitCode = 0; + await s.run({ EDGAR_FALLBACK: 'true', TICKERS: 'BBB' }); + expect(s.touched()).toEqual([]); + expect(s.files().map((path) => readFileSync(path, 'utf8'))).toEqual(before); + expect(process.exitCode).toBe(1); + }); + test('N-PORT freshness: a filing older than the published holdings never replaces them, a newer one does', async () => { const s = new Scenario(['AAA']); await s.run({ EDGAR_FALLBACK: 'true' }); diff --git a/scripts/update-data.ts b/scripts/update-data.ts index 7433379..db2d01d 100755 --- a/scripts/update-data.ts +++ b/scripts/update-data.ts @@ -232,6 +232,10 @@ export function setApiRoot(root: URL): void { } const HOLDINGS_HEADERS = ['Name', 'Ticker', 'Identifier', 'Weight', 'Market Value', 'Shares Held', 'Asset Category']; +/** Funds whose holdings legitimately fail a holdingsGuard check (geared, futures, look-through, top-N lists): ticker, or '*' for every fund -> relaxed checks. */ +const HOLDINGS_GUARD_RELAX: Record = { + WSPC: ['sum'], // no N-PORT filing yet: the product page lists only its top 10 holdings (about 47% of the fund) +}; const MONTHS = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']; const TRUTHY = new Set(['1', 'true', 'yes', 'y', 'on']); const AUM_BOUNDS = { nano: [0, 10_000_000], micro: [10_000_000, 300_000_000], small: [300_000_000, 2_000_000_000], mid: [2_000_000_000, 10_000_000_000], large: [10_000_000_000, undefined] } as const; @@ -1815,6 +1819,7 @@ async function processFund(fund: CatalogFund, config: UpdaterConfig, previous: J let holdingsAsOf: string | null = null; let holdingsSource = 'not available from current public SEC filing'; let nport: ParsedNport | null = null; + let holdingsFresh = false; // false while holdingsRows is the previous run's sheet if (config.edgarFallback) { try { const filing = await resolveNportFiling(fund, config); @@ -1827,6 +1832,7 @@ async function processFund(fund: CatalogFund, config: UpdaterConfig, previous: J if (seriesMatches && fresher && parsed.holdings.length) { const names = await loadCompanyTickerTable(config); holdingsRows = fillNportTickers(parsed.holdings, names); + holdingsFresh = true; holdingsHeaders = holdingsRows.some((row) => 'Coupon' in row || 'Maturity' in row) ? [...HOLDINGS_HEADERS, 'Coupon', 'Maturity'] : HOLDINGS_HEADERS; holdingsAsOf = parsed.repPdDate || null; nport = parsed; @@ -1843,6 +1849,7 @@ async function processFund(fund: CatalogFund, config: UpdaterConfig, previous: J const top = parseTopHoldingsMarkdown(productPageMarkdown, fund.ticker, companyTickerMap || new Map()); if (top.rows.length && (!previousMeta?.holdings?.totalRows || /^WisdomTree product page/.test(String(previousMeta.holdings.source || '')))) { holdingsRows = top.rows; + holdingsFresh = true; holdingsHeaders = top.headers; holdingsAsOf = top.asOfDate; holdingsSource = 'WisdomTree product page top holdings (issuer fallback; full N-PORT filing unavailable)'; @@ -1909,6 +1916,11 @@ async function processFund(fund: CatalogFund, config: UpdaterConfig, previous: J return { __skipped: true, ticker: fund.ticker, __skipReasons: returnFilterReasons }; } + // Fresh holdings that look broken (no weight column, non-numeric weights, a bad weight sum, half the rows gone) never replace + // the published ones: the fund fails before anything of it is written, so it stays exactly as published. + const holdingsProblem = holdingsFresh ? holdingsGuard(holdingsHeaders, holdingsRows, numberOrNull(previousMeta?.holdings?.totalRows), HOLDINGS_GUARD_RELAX[fund.ticker] ?? HOLDINGS_GUARD_RELAX['*'] ?? []) : null; + if (holdingsProblem) throw new Error(`holdings guard: ${holdingsProblem}; ${previousMeta ? 'kept the previous complete data' : 'no data published for this fund yet'}`); + const history = historyRows(days, previousRows); const historyAsOf = derived.asOfDate || previousMeta?.history?.asOf || null; const holdingPages = await writePages(fundDir, fund.ticker, 'holdings', holdingsHeaders, holdingsRows, config.holdingsPageSize, holdingsAsOf, holdingsSource); @@ -2495,3 +2507,49 @@ if ((import.meta as { main?: boolean }).main) { if (process.argv.some((arg) => ['-h', '--help', 'help'].includes(arg))) console.log(USAGE.trim()); else await main().catch((error) => { console.error(error instanceof Error ? error.stack : String(error)); process.exitCode = 1; }); } + +// shared block of .claude/tools/etfs/etf-std/holdings-guard.block.ts, applied by apply-holdings-guard.ts: never edit it in one repo +/** + * Sanity check of a fund's new holdings before they replace the published ones. Returns why they must be rejected + * (the caller then keeps the fund's previous state and reports it as failed), or null when they may be written. + * An empty sheet passes only when nothing was published before. Otherwise a Name and a weight column (Weight, + * Weight (%) or Market Weight) are required, >= 80% of the rows carry a numeric weight, the weights add up to + * 50..150 and the sheet keeps >= 50% of the previously published rows. `relax` switches off the checks a fund + * legitimately fails: 'numeric' and 'sum' (geared, inverse, futures, look-through, top-N lists) and 'count'. + * Rows are arrays in header order or objects keyed by the header text. + */ +export function holdingsGuard(headers: ReadonlyArray, rows: ReadonlyArray, previousRows: number | null | undefined, relax: ReadonlyArray = []): string | null { + const previous = Number(previousRows) > 0 ? Number(previousRows) : 0; + if (!rows.length) return previous ? `no holdings rows (${previous} published)` : null; + const names = headers.map((header) => String(header).trim().toLowerCase()); + const nameAt = names.indexOf('name'); + let weightAt = names.indexOf('weight'); + if (weightAt < 0) weightAt = names.indexOf('weight (%)'); + if (weightAt < 0) weightAt = names.indexOf('market weight'); + if (nameAt < 0 || weightAt < 0) return `no ${nameAt < 0 ? 'Name' : 'weight'} column (headers: ${headers.join(', ') || 'none'})`; + let numeric = 0; + let sum = 0; + for (let i = 0; i < rows.length; i++) { + const row: any = rows[i]; + const weight = holdingsGuardNumber(Array.isArray(row) ? row[weightAt] : row && typeof row === 'object' ? row[headers[weightAt]] : undefined); + if (weight !== null) { + numeric++; + sum += weight; + } + } + if (relax.indexOf('numeric') < 0 && numeric < rows.length * 0.8) return `only ${numeric} of ${rows.length} rows have a numeric weight`; + if (relax.indexOf('sum') < 0 && (sum < 50 || sum > 150)) return `weights add up to ${sum.toFixed(2)}, outside 50..150`; + if (relax.indexOf('count') < 0 && previous && rows.length < previous * 0.5) return `${rows.length} rows, less than half of the ${previous} published`; + return null; +} + +/** A weight cell as a number: %, $, commas and spaces are ignored, (x) is negative; null when it is no plain number (a zero-width character inside counts as broken). */ +function holdingsGuardNumber(value: unknown): number | null { + if (typeof value === 'number') return Number.isFinite(value) ? value : null; + let text = String(value === null || value === undefined ? '' : value).replace(/[ \t ,%$]/g, '').replace(/−/g, '-'); + const negative = /^\(.*\)$/.test(text); + if (negative) text = text.slice(1, -1); + if (!/^[-+]?(\d+\.?\d*|\.\d+)(e[-+]?\d+)?$/i.test(text)) return null; + return negative ? -Number(text) : Number(text); +} +//