From 75bce1e0d58b69e9b5008726d238c68fddcb1317 Mon Sep 17 00:00:00 2001 From: Maksim Kostromin Date: Sun, 25 Oct 2026 23:15:29 -0400 Subject: [PATCH] fix(holdings): keep junk sheets and non-numeric weights out of the feed - product-page top holdings: a weight that is no number stays empty, never "0", so the holdings guard's numeric check can fire (open-items etfs Q4) - a published sheet that fails the guard on its own is not republished: WSLS and WTAL lose their allocation tables (weight sums 581 and 600) and the product-page source label until a valid source exists (Q5) --- api/wisdomtree/funds/WSLS/holdings/001.json | 323 ------------ api/wisdomtree/funds/WSLS/meta.json | 10 +- api/wisdomtree/funds/WTAL/holdings/001.json | 530 -------------------- api/wisdomtree/funds/WTAL/meta.json | 10 +- api/wisdomtree/index.json | 8 +- scripts/update-data.test.ts | 14 + scripts/update-data.ts | 20 +- 7 files changed, 41 insertions(+), 874 deletions(-) delete mode 100644 api/wisdomtree/funds/WSLS/holdings/001.json delete mode 100644 api/wisdomtree/funds/WTAL/holdings/001.json diff --git a/api/wisdomtree/funds/WSLS/holdings/001.json b/api/wisdomtree/funds/WSLS/holdings/001.json deleted file mode 100644 index 7648123..0000000 --- a/api/wisdomtree/funds/WSLS/holdings/001.json +++ /dev/null @@ -1,323 +0,0 @@ -{ - "ticker": "WSLS", - "page": 1, - "pageSize": 250, - "totalRows": 34, - "headers": [ - "Name", - "Ticker", - "Identifier", - "Weight", - "Market Value", - "Shares Held", - "Asset Category" - ], - "rows": [ - { - "Name": "Group", - "Ticker": "-", - "Identifier": "-", - "Weight": "0", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Cash&Collateral", - "Ticker": "-", - "Identifier": "-", - "Weight": "100", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Treasuries", - "Ticker": "-", - "Identifier": "-", - "Weight": "90.68", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Cash", - "Ticker": "-", - "Identifier": "-", - "Weight": "9.32", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Equities", - "Ticker": "-", - "Identifier": "-", - "Weight": "100.72", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Financial Services", - "Ticker": "-", - "Identifier": "-", - "Weight": "15.8", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Materials", - "Ticker": "-", - "Identifier": "-", - "Weight": "12.32", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Health Care Equipment & Services", - "Ticker": "-", - "Identifier": "-", - "Weight": "0", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Pharmaceuticals Biotechnology & Life Sciences", - "Ticker": "-", - "Identifier": "-", - "Weight": "5.96", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Utilities", - "Ticker": "-", - "Identifier": "-", - "Weight": "5.9", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Energy", - "Ticker": "-", - "Identifier": "-", - "Weight": "6.75", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Consumer Services", - "Ticker": "-", - "Identifier": "-", - "Weight": "3.88", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Equity Real Estate Investment Trusts (REITs)", - "Ticker": "-", - "Identifier": "-", - "Weight": "1.04", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Insurance", - "Ticker": "-", - "Identifier": "-", - "Weight": "0", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Commercial & Professional Services", - "Ticker": "-", - "Identifier": "-", - "Weight": "5.23", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Food Beverage & Tobacco", - "Ticker": "-", - "Identifier": "-", - "Weight": "0", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Banks", - "Ticker": "-", - "Identifier": "-", - "Weight": "4", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Capital Goods", - "Ticker": "-", - "Identifier": "-", - "Weight": "11.41", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Food & Staples Retailing", - "Ticker": "-", - "Identifier": "-", - "Weight": "2.34", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Automobiles & Components", - "Ticker": "-", - "Identifier": "-", - "Weight": "2.79", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Technology Hardware & Equipment", - "Ticker": "-", - "Identifier": "-", - "Weight": "3.37", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Consumer Durables & Apparel", - "Ticker": "-", - "Identifier": "-", - "Weight": "4.86", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Transportation", - "Ticker": "-", - "Identifier": "-", - "Weight": "2.02", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Telecommunication Services", - "Ticker": "-", - "Identifier": "-", - "Weight": "0", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Other", - "Ticker": "-", - "Identifier": "-", - "Weight": "0", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Commercial & Professional Services", - "Ticker": "-", - "Identifier": "-", - "Weight": "0", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Media & Entertainment", - "Ticker": "-", - "Identifier": "-", - "Weight": "1.89", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Real Estate Management & Development", - "Ticker": "-", - "Identifier": "-", - "Weight": "0", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Semiconductors & Semiconductor Equipment", - "Ticker": "-", - "Identifier": "-", - "Weight": "1.88", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Household & Personal Products", - "Ticker": "-", - "Identifier": "-", - "Weight": "0.59", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Software & Services", - "Ticker": "-", - "Identifier": "-", - "Weight": "6.03", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Retailing", - "Ticker": "-", - "Identifier": "-", - "Weight": "2.65", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Equity Futures", - "Ticker": "-", - "Identifier": "-", - "Weight": "90", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "U.S. Equity", - "Ticker": "-", - "Identifier": "-", - "Weight": "90", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - } - ] -} diff --git a/api/wisdomtree/funds/WSLS/meta.json b/api/wisdomtree/funds/WSLS/meta.json index db71363..634de36 100644 --- a/api/wisdomtree/funds/WSLS/meta.json +++ b/api/wisdomtree/funds/WSLS/meta.json @@ -11,7 +11,7 @@ "holdingsDownload": null, "pricesDownload": null, "yahooChart": "https://query1.finance.yahoo.com/v8/finance/chart/WSLS", - "holdingsSource": "WisdomTree product page top holdings (issuer fallback; full N-PORT filing unavailable)", + "holdingsSource": "not available from current public SEC filing", "historySource": "Yahoo Finance public chart API (adjusted close)", "distributionsSource": "not available", "provider": "WisdomTree U.S. product catalog + official WisdomTree product page + SEC EDGAR Form N-PORT-P + Yahoo Finance public chart API" @@ -135,14 +135,12 @@ "rows": [] }, "holdings": { - "pages": [ - "holdings/001.json" - ], + "pages": [], "pageSize": 250, - "totalRows": 34, + "totalRows": 0, "asOfDate": null, "asOf": "—", - "source": "WisdomTree product page top holdings (issuer fallback; full N-PORT filing unavailable)" + "source": "not available from current public SEC filing" }, "history": { "pages": [ diff --git a/api/wisdomtree/funds/WTAL/holdings/001.json b/api/wisdomtree/funds/WTAL/holdings/001.json deleted file mode 100644 index 60cf455..0000000 --- a/api/wisdomtree/funds/WTAL/holdings/001.json +++ /dev/null @@ -1,530 +0,0 @@ -{ - "ticker": "WTAL", - "page": 1, - "pageSize": 250, - "totalRows": 57, - "headers": [ - "Name", - "Ticker", - "Identifier", - "Weight", - "Market Value", - "Shares Held", - "Asset Category" - ], - "rows": [ - { - "Name": "Group", - "Ticker": "-", - "Identifier": "-", - "Weight": "0", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Cash&Collateral", - "Ticker": "-", - "Identifier": "-", - "Weight": "100", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Treasuries", - "Ticker": "-", - "Identifier": "-", - "Weight": "97.96", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Cash", - "Ticker": "-", - "Identifier": "-", - "Weight": "2.04", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "International Equities", - "Ticker": "-", - "Identifier": "-", - "Weight": "112.95", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Capital Goods", - "Ticker": "-", - "Identifier": "-", - "Weight": "6.84", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Banks", - "Ticker": "-", - "Identifier": "-", - "Weight": "13.4", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Materials", - "Ticker": "-", - "Identifier": "-", - "Weight": "22.29", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Commercial & Professional Services", - "Ticker": "-", - "Identifier": "-", - "Weight": "0", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Health Care Equipment & Services", - "Ticker": "-", - "Identifier": "-", - "Weight": "1.02", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Utilities", - "Ticker": "-", - "Identifier": "-", - "Weight": "3.06", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Transportation", - "Ticker": "-", - "Identifier": "-", - "Weight": "6.4", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Semiconductors & Semiconductor Equipment", - "Ticker": "-", - "Identifier": "-", - "Weight": "4.66", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Consumer Services", - "Ticker": "-", - "Identifier": "-", - "Weight": "3.58", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Automobiles & Components", - "Ticker": "-", - "Identifier": "-", - "Weight": "3.35", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Energy", - "Ticker": "-", - "Identifier": "-", - "Weight": "8.36", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Food Beverage & Tobacco", - "Ticker": "-", - "Identifier": "-", - "Weight": "0.89", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Financial Services", - "Ticker": "-", - "Identifier": "-", - "Weight": "7.33", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Software & Services", - "Ticker": "-", - "Identifier": "-", - "Weight": "2.42", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Real Estate Management & Development", - "Ticker": "-", - "Identifier": "-", - "Weight": "0.88", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Food & Staples Retailing", - "Ticker": "-", - "Identifier": "-", - "Weight": "1.41", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Household & Personal Products", - "Ticker": "-", - "Identifier": "-", - "Weight": "0.79", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Other", - "Ticker": "-", - "Identifier": "-", - "Weight": "0", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Media & Entertainment", - "Ticker": "-", - "Identifier": "-", - "Weight": "0", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Equity Real Estate Investment Trusts (REITs)", - "Ticker": "-", - "Identifier": "-", - "Weight": "2.08", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Telecommunication Services", - "Ticker": "-", - "Identifier": "-", - "Weight": "2.89", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Technology Hardware & Equipment", - "Ticker": "-", - "Identifier": "-", - "Weight": "1.46", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Consumer Durables & Apparel", - "Ticker": "-", - "Identifier": "-", - "Weight": "2.95", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Insurance", - "Ticker": "-", - "Identifier": "-", - "Weight": "7.35", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Retailing", - "Ticker": "-", - "Identifier": "-", - "Weight": "3.24", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Pharmaceuticals Biotechnology & Life Sciences", - "Ticker": "-", - "Identifier": "-", - "Weight": "6.28", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "U.S. Equities", - "Ticker": "-", - "Identifier": "-", - "Weight": "86.9", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Equity Real Estate Investment Trusts (REITs)", - "Ticker": "-", - "Identifier": "-", - "Weight": "0", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Health Care Equipment & Services", - "Ticker": "-", - "Identifier": "-", - "Weight": "1.45", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Materials", - "Ticker": "-", - "Identifier": "-", - "Weight": "9.88", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Financial Services", - "Ticker": "-", - "Identifier": "-", - "Weight": "12.8", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Pharmaceuticals Biotechnology & Life Sciences", - "Ticker": "-", - "Identifier": "-", - "Weight": "2.63", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Energy", - "Ticker": "-", - "Identifier": "-", - "Weight": "7.11", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Insurance", - "Ticker": "-", - "Identifier": "-", - "Weight": "0", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Consumer Services", - "Ticker": "-", - "Identifier": "-", - "Weight": "5.24", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Utilities", - "Ticker": "-", - "Identifier": "-", - "Weight": "5.04", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Capital Goods", - "Ticker": "-", - "Identifier": "-", - "Weight": "11.22", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Media & Entertainment", - "Ticker": "-", - "Identifier": "-", - "Weight": "0.69", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Consumer Durables & Apparel", - "Ticker": "-", - "Identifier": "-", - "Weight": "3.62", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Food Beverage & Tobacco", - "Ticker": "-", - "Identifier": "-", - "Weight": "0.99", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Commercial & Professional Services", - "Ticker": "-", - "Identifier": "-", - "Weight": "3.7", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Transportation", - "Ticker": "-", - "Identifier": "-", - "Weight": "2.54", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Automobiles & Components", - "Ticker": "-", - "Identifier": "-", - "Weight": "2.23", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Food & Staples Retailing", - "Ticker": "-", - "Identifier": "-", - "Weight": "1.99", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Telecommunication Services", - "Ticker": "-", - "Identifier": "-", - "Weight": "0", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Banks", - "Ticker": "-", - "Identifier": "-", - "Weight": "2.25", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Technology Hardware & Equipment", - "Ticker": "-", - "Identifier": "-", - "Weight": "2.34", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Retailing", - "Ticker": "-", - "Identifier": "-", - "Weight": "3.22", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Household & Personal Products", - "Ticker": "-", - "Identifier": "-", - "Weight": "1.49", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Semiconductors & Semiconductor Equipment", - "Ticker": "-", - "Identifier": "-", - "Weight": "1.86", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Software & Services", - "Ticker": "-", - "Identifier": "-", - "Weight": "4.17", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - }, - { - "Name": "Other", - "Ticker": "-", - "Identifier": "-", - "Weight": "0.44", - "Market Value": "-", - "Shares Held": "-", - "Asset Category": "-" - } - ] -} diff --git a/api/wisdomtree/funds/WTAL/meta.json b/api/wisdomtree/funds/WTAL/meta.json index 4110715..f7a6644 100644 --- a/api/wisdomtree/funds/WTAL/meta.json +++ b/api/wisdomtree/funds/WTAL/meta.json @@ -11,7 +11,7 @@ "holdingsDownload": null, "pricesDownload": null, "yahooChart": "https://query1.finance.yahoo.com/v8/finance/chart/WTAL", - "holdingsSource": "WisdomTree product page top holdings (issuer fallback; full N-PORT filing unavailable)", + "holdingsSource": "not available from current public SEC filing", "historySource": "Yahoo Finance public chart API (adjusted close)", "distributionsSource": "not available", "provider": "WisdomTree U.S. product catalog + official WisdomTree product page + SEC EDGAR Form N-PORT-P + Yahoo Finance public chart API" @@ -135,14 +135,12 @@ "rows": [] }, "holdings": { - "pages": [ - "holdings/001.json" - ], + "pages": [], "pageSize": 250, - "totalRows": 57, + "totalRows": 0, "asOfDate": null, "asOf": "—", - "source": "WisdomTree product page top holdings (issuer fallback; full N-PORT filing unavailable)" + "source": "not available from current public SEC filing" }, "history": { "pages": [ diff --git a/api/wisdomtree/index.json b/api/wisdomtree/index.json index b9d8c4b..9b37b85 100644 --- a/api/wisdomtree/index.json +++ b/api/wisdomtree/index.json @@ -1,5 +1,5 @@ { - "generatedAt": "2026-10-26T02:17:34Z", + "generatedAt": "2026-10-26T03:15:20Z", "source": { "provider": "WisdomTree Asset Management, Inc. (U.S.-listed ETFs)", "market": "us", @@ -14,7 +14,7 @@ }, "counts": { "funds": 96, - "holdings": 39334, + "holdings": 39243, "history": 252132 }, "funds": [ @@ -10215,7 +10215,7 @@ "returnsBasis": "official WisdomTree product-page Market Price Returns (month-end table); gaps filled with estimates derived from Yahoo Finance adjusted closes", "performanceAsOf": null }, - "holdings": 34, + "holdings": 0, "history": 10 }, { @@ -10563,7 +10563,7 @@ "returnsBasis": "official WisdomTree product-page Market Price Returns (month-end table); gaps filled with estimates derived from Yahoo Finance adjusted closes", "performanceAsOf": null }, - "holdings": 57, + "holdings": 0, "history": 11 }, { diff --git a/scripts/update-data.test.ts b/scripts/update-data.test.ts index 1fcadfc..88a3223 100644 --- a/scripts/update-data.test.ts +++ b/scripts/update-data.test.ts @@ -271,6 +271,10 @@ describe('parsing', () => { // a list right before the next heading ends there; a page without a holdings section is no rejection expect(parseTopHoldingsMarkdown('### Holdings\n\n| Name | Weight |\n| --- | --- |\n| A Corp | 5 |\n### Recent Distributions\n\n| Name | Weight |\n| --- | --- |\n| B Corp | 6 |', 'AAA').rows.map((row) => row.Name)).toEqual(['A Corp']); expect(parseTopHoldingsMarkdown(usfrPage, 'USFR')).toMatchObject({ rows: [], rejected: null }); + // a weight that is no number stays empty, never "0", so the guard's numeric check fires + const broken = parseTopHoldingsMarkdown(holdingsSection('| Name | Weight (%) |', '| A Corp | n/a |', '| B Corp | — |', '| C Corp | 5.00 |'), 'AAA'); + expect(broken.rows.map((row) => row.Weight)).toEqual(['', '', '5']); + expect(holdingsGuard(broken.headers, broken.rows, null, ['sum'])).toContain('numeric weight'); }); test('holdings guard: header-only, renamed weight column, broken weights, a bad weight sum and a halved sheet are rejected', () => { @@ -759,6 +763,16 @@ describe('pipeline', () => { process.exitCode = 0; await s.run({ EDGAR_FALLBACK: 'true', TICKERS: 'BBB' }); expect([s.meta('BBB').holdings, s.meta('BBB').returns.monthEnd.ytd, process.exitCode]).toEqual([published, 9.99, 0]); + // a published sheet that is itself junk (the old parser published the allocation summary, WSLS and WTAL) is dropped, not republished + const page = join(s.dir, 'api/wisdomtree/funds/BBB/holdings/001.json'); + const sheet = JSON.parse(readFileSync(page, 'utf8')); + writeFileSync(page, JSON.stringify({ ...sheet, rows: [{ ...sheet.rows[0], Name: 'Cash&Collateral', Weight: '100' }, { ...sheet.rows[1], Name: 'Equities', Weight: '476' }] })); + await s.run({ EDGAR_FALLBACK: 'true', TICKERS: 'BBB' }); + expect([s.meta('BBB').holdings, s.meta('BBB').source.holdingsSource, existsSync(page), process.exitCode]) + .toEqual([{ pages: [], pageSize: published.pageSize, totalRows: 0, asOfDate: null, asOf: '—', source: 'not available from current public SEC filing' }, 'not available from current public SEC filing', false, 0]); + s.backdate(); + await s.run({ EDGAR_FALLBACK: 'true', TICKERS: 'BBB' }); + expect(s.touched()).toEqual([]); }); test('N-PORT freshness: a filing older than the published holdings never replaces them, a newer one does', async () => { diff --git a/scripts/update-data.ts b/scripts/update-data.ts index 6fbe301..f2cb74a 100755 --- a/scripts/update-data.ts +++ b/scripts/update-data.ts @@ -1189,7 +1189,8 @@ export function parseTopHoldingsMarkdown(markdown: string, ticker: string, compa const weight = numberOrNull(cells[weightAt]); sum += weight ?? 0; const resolvedTicker = companyNames.get(normalizeHoldingName(name)) || companyNames.get(normalizeHoldingNameCore(name)) || '-'; - rows.push({ Name: name, Ticker: resolvedTicker || '-', Identifier: '-', Weight: String(weight ?? 0), 'Market Value': '-', 'Shares Held': '-', 'Asset Category': '-' }); + // a weight that is no number stays empty (never "0"), so the holdings guard can reject the list + rows.push({ Name: name, Ticker: resolvedTicker || '-', Identifier: '-', Weight: weight === null ? '' : String(weight), 'Market Value': '-', 'Shares Held': '-', 'Asset Category': '-' }); } if (sum > TOP_HOLDINGS_MAX_SUM && !TOP_HOLDINGS_LEVERAGED.has(ticker)) return none(`top holdings add up to ${round(sum, 2)}, more than ${TOP_HOLDINGS_MAX_SUM}`); return { headers: HOLDINGS_HEADERS, rows, asOfDate, rejected: null }; @@ -1881,10 +1882,19 @@ async function processFund(fund: CatalogFund, config: UpdaterConfig, previous: J } if (!holdingsRows.length) { - holdingsRows = await readPreviousSheet(fund.ticker, 'holdings'); - holdingsHeaders = (await readPreviousHeaders(fund.ticker, 'holdings')) || holdingsHeaders; - holdingsAsOf = previousMeta?.holdings?.asOfDate || null; - holdingsSource = previousMeta?.holdings?.source || holdingsSource; + const previousSheet = await readPreviousSheet(fund.ticker, 'holdings'); + const previousHeaders = (await readPreviousHeaders(fund.ticker, 'holdings')) || holdingsHeaders; + // A published sheet that fails the guard on its own is junk, not holdings (WSLS and WTAL published their allocation + // summary, weights adding up to 581 and 600): it is not republished, the fund has no holdings until a valid source exists. + const junk = previousSheet.length ? holdingsGuard(previousHeaders, previousSheet, null, HOLDINGS_GUARD_RELAX[fund.ticker] ?? HOLDINGS_GUARD_RELAX['*'] ?? []) : null; + if (junk) { + console.warn(`[ ${'holdings'.padEnd(9)}] ${fund.ticker}: published holdings dropped (${junk}); none until a valid source exists`); + } else { + holdingsRows = previousSheet; + holdingsHeaders = previousHeaders; + holdingsAsOf = previousMeta?.holdings?.asOfDate || null; + holdingsSource = previousMeta?.holdings?.source || holdingsSource; + } } let chart: ParsedChart | null = null;