diff --git a/README.md b/README.md index 5ae83ac..33c03e4 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ Each fund carries a derived `metrics` object that powers the catalog columns sha - `ytd` / `tr1y` - official YTD and 1-year returns -> *YTD Return*, *TR 1Y* - `cagr3y` / `cagr5y` / `cagr10y` - published annualized 3Y/5Y/10Y figures -> *CAGR 3Y/5Y/10Y* - `tr3y` / `tr5y` / `tr10y` - cumulative 3Y/5Y/10Y figures `(1 + CAGR)^n - 1` -> *TR 3Y/5Y/10Y* -- `siAnn` - since-inception annualized -> *SI Ann.*; derived only for funds with at least one year of history +- `siAnn` - since-inception annualized -> *SI Ann.*; `null` below one year between inception and the returns date (WisdomTree prints the cumulative figure for younger funds), derived only from a Yahoo series that starts at the first trade date - `dividendYield` - 12-month trailing yield or indicated yield (latest distribution x frequency / price); a catalog 0.00% is the provider's published value and is kept as published (`yields.dividendYieldKind` says so) - `dividendYieldBasis` - code of the definition behind `dividendYield`, `null` exactly when `dividendYield` is `null`; it travels with the yield (a retained yield keeps its code): diff --git a/scripts/update-data.test.ts b/scripts/update-data.test.ts index f49e63c..4e72f16 100644 --- a/scripts/update-data.test.ts +++ b/scripts/update-data.test.ts @@ -7,10 +7,10 @@ import { pathToFileURL } from 'node:url'; import { CLOSED_FUNDS, CONTROL_NAMES, HISTORY_RANGE_PATTERN, RETURNS_BASIS_DERIVED, RETURNS_BASIS_OFFICIAL, annualizedToTotal, aumSource, cleanHoldingTicker, closedFund, deriveMetrics, dividendYieldBasisFromKind, formatDate, hasConfiguredFilters, holdingsGuard, inferDistributionFrequency, installSystemCa, - isCertError, isinFromCusip, looksLikeProductPage, mergeDistributionRecords, normalizeHoldingName, nportUrlFor, numberOrNull, parseAumRange, + isCertError, isinFromCusip, looksLikeProductPage, mergeDistributionRecords, mergeOfficialReturns, normalizeHoldingName, nportUrlFor, numberOrNull, parseAumRange, parseCatalogMarkdown, parseChart, parseFundListJson, parseEdgarAtomFilings, parseFundTickerMap, parseNport, parseOfficialDistributions, parseProductPageSummary, parseRange, parseTopHoldingsMarkdown, parseRanges, performanceAsOfDate, priceReturns, readConfig, readKnownRows, resolveControls, - rowFromMeta, runUpdate, runtimeControls, setApiRoot, toIsoDate, writeIndex, yahooChartQuery, + rowFromMeta, runUpdate, runtimeControls, setApiRoot, toIsoDate, withoutYoungSinceInception, writeIndex, yahooChartQuery, } from './update-data'; // --------------------------------------------------------------------------- @@ -328,22 +328,22 @@ describe('metrics', () => { const day = (date: string, close: number, adjClose = close) => ({ date, close, adjClose, volume: 1 }); test('returns use adjusted closes, annualized values convert to cumulative', () => { - const result = priceReturns([day('2024-01-02', 100), day('2025-01-02', 110, 115), day('2026-01-02', 121, 132.25)], new Date('2026-01-03T00:00:00Z')); + const result = priceReturns([day('2024-01-02', 100), day('2025-01-02', 110, 115), day('2026-01-02', 121, 132.25)]); expect(result.yr1).toBe(32.25); expect(annualizedToTotal(10, 3)).toBe(33.1); }); test('siAnn needs at least one year of history (null, never 0), a flat young fund keeps null horizons', () => { expect(priceReturns([day('2026-06-01', 100), day('2026-09-25', 110)]).siAnn).toBeNull(); - expect(priceReturns([day('2024-09-25', 100), day('2026-09-25', 121)]).siAnn).toBeCloseTo(10, 0); - const young = priceReturns([day('2026-06-01', 100), day('2026-09-25', 110)], new Date('2026-09-26T00:00:00Z')); + expect(priceReturns([day('2024-09-25', 100), day('2026-09-25', 121)], '2024-09-25').siAnn).toBeCloseTo(10, 0); + const young = priceReturns([day('2026-06-01', 100), day('2026-09-25', 110)]); expect([young.yr1, young.cagr3y, young.cagr5y, young.cagr10y]).toEqual([null, null, null, null]); }); test('metrics end with returnsBasis then performanceAsOf, official and derived bases travel with their own date', () => { const days = [day('2025-01-02', 100), day('2026-01-02', 110)]; const fund = { dividendYield: null, secYield: null } as any; - const derived = priceReturns(days, new Date('2026-01-03T00:00:00Z')); + const derived = priceReturns(days); const estimated = deriveMetrics(derived, fund, [], { paymentsPerYear: null }, 110); expect([estimated.returnsBasis, estimated.performanceAsOf]).toEqual([RETURNS_BASIS_DERIVED, '2026-01-02']); expect(Object.keys(estimated).slice(-2)).toEqual(['returnsBasis', 'performanceAsOf']); @@ -354,7 +354,7 @@ describe('metrics', () => { }); test('dividendYieldBasis maps each yield source to a code and is null with a null yield', () => { - const derived = priceReturns([day('2025-01-02', 100), day('2026-01-02', 110)], new Date('2026-01-03T00:00:00Z')); + const derived = priceReturns([day('2025-01-02', 100), day('2026-01-02', 110)]); const dividends = [0, 1, 2].map((month) => ({ epoch: Date.UTC(2026, month, 15) / 1000, amount: 0.1 })); const basis = (fund: any, rows = dividends) => deriveMetrics(derived, { secYield: null, ...fund }, rows, { paymentsPerYear: rows.length ? 12 : null }, 100).dividendYieldBasis; expect(basis({ dividendYield: 4.72 })).toBe('official-trailing-12m'); @@ -377,6 +377,33 @@ describe('metrics 2', () => { expect([none.metrics.dividendYield, none.metrics.dividendYieldBasis]).toEqual([null, null]); }); + test('derived siAnn only from a series that starts at inception and spans a year, the 1-month window clamps to a shorter month', () => { + const day = (date: string, close: number) => ({ date, close, adjClose: close, volume: 1 }); + const closes = [day('2021-09-30', 100), day('2024-09-30', 150), day('2026-09-30', 200)]; + expect(priceReturns(closes, '2021-09-28').siAnn).toBeCloseTo(14.87, 1); + // A HISTORY_RANGE window that starts years after inception is not since inception (was cagr5y-like 14.87) + expect(priceReturns(closes.slice(1), '2021-09-28').siAnn).toBeNull(); + expect(priceReturns(closes, null).siAnn).toBeNull(); + // Oct 31 minus one month is Sep 30, not "Sep 31" = Oct 1 (that anchor gave -45) + expect(priceReturns([day('2026-09-30', 100), day('2026-10-01', 200), day('2026-10-31', 110)], '2026-09-30').mo1).toBe(10); + }); + + test('official since inception is null below one year between inception and the table date (main, NAV and index rows)', () => { + const monthEnd = parseProductPageSummary(usfrPage).officialReturns.monthEnd; + expect([monthEnd?.siAnn, monthEnd?.navReturns?.siAnn, monthEnd?.indexReturns?.siAnn]).toEqual([1.99, 2, 2.17]); + // NTSD: launched 2026-03-19, the 2026-09-30 table carried the cumulative 20.18 as annualized + const young = withoutYoungSinceInception(monthEnd, '2026-03-19'); + expect([young?.siAnn, young?.navReturns?.siAnn, young?.indexReturns?.siAnn, young?.yr1, young?.ytd]).toEqual([null, null, null, 4, 2.59]); + expect(withoutYoungSinceInception(monthEnd, '2025-08-31')?.siAnn).toBe(1.99); + expect(withoutYoungSinceInception(monthEnd, '2014-12-11')?.siAnn).toBe(1.99); + expect(withoutYoungSinceInception(monthEnd, null)?.siAnn).toBe(1.99); + // and the gap is not filled with a Yahoo estimate from a later close (WQTM published -9.59 as of 2026-09-30) + const yahoo = { ...priceReturns([{ date: '2025-10-09', close: 100, adjClose: 100, volume: 1 }, { date: '2026-10-23', close: 90, adjClose: 90, volume: 1 }], '2025-10-09'), asOfDate: '2026-10-23' }; + expect(yahoo.siAnn).not.toBeNull(); + expect(mergeOfficialReturns(yahoo, withoutYoungSinceInception(monthEnd, '2025-10-09'), '2025-10-09').siAnn).toBeNull(); + expect(mergeOfficialReturns(yahoo, { ...monthEnd!, siAnn: null }, '2014-12-11').siAnn).toBe(yahoo.siAnn); + }); + test('distribution frequency is inferred like in every sibling updater', () => { const monthly = [0, 1, 2, 3].map((month) => ({ epoch: Date.UTC(2026, month, 15) / 1000, amount: 0.1 })); expect(inferDistributionFrequency(monthly)).toEqual({ frequency: 'Monthly', paymentsPerYear: 12 }); diff --git a/scripts/update-data.ts b/scripts/update-data.ts index 31d1d49..4a79e7c 100755 --- a/scripts/update-data.ts +++ b/scripts/update-data.ts @@ -1418,12 +1418,43 @@ function annualized(start: number | null, end: number | null, years: number): nu return round(((end / start) ** (1 / years) - 1) * 100, 2); } -/** Since-inception annualized return; null under one year of history (a few weeks annualized is noise). */ -function annualizedSince(first: ChartDay, end: number, date: Date): number | null { +/** Days between two ISO dates (UTC), null when either is not an ISO date. */ +function daysBetween(from: string | null | undefined, to: string | null | undefined): number | null { + const a = toIsoDate(from), b = toIsoDate(to); + if (!/^\d{4}-\d{2}-\d{2}$/.test(a) || !/^\d{4}-\d{2}-\d{2}$/.test(b)) return null; + return (Date.parse(`${b}T00:00:00Z`) - Date.parse(`${a}T00:00:00Z`)) / 86_400_000; +} + +/** + * Since-inception annualized return derived from closes. It is a since-INCEPTION figure only when the series starts at + * inception (first close within 10 days of the fund's first trade date, the official inception as fallback), so a + * HISTORY_RANGE window never turns cagr5y into siAnn; null under one year (a few months annualized is noise) or when + * the inception is unknown. + */ +function annualizedSince(first: ChartDay, end: number, date: Date, inception: string | null): number | null { + const gap = daysBetween(inception, first.date); + if (gap === null || gap > 10) return null; const years = (date.getTime() - new Date(`${first.date}T00:00:00Z`).getTime()) / (365.25 * 86_400_000); return years < 1 ? null : annualized(first.adjClose, end, years); } +/** + * WisdomTree publishes "Since Inception" CUMULATIVE for a fund younger than one year (NTSD, launched 2026-03-19: 20.18 as of + * 2026-09-30) in the Average Annual row. Annualized since-inception exists only from one year on, so below 365 days + * between inception and the table date it is null (main, NAV and index rows); an unknown inception keeps the figure. + */ +export function withoutYoungSinceInception(returns: OfficialProductReturns | null, inception: string | null): OfficialProductReturns | null { + if (!returns) return returns; + const age = daysBetween(inception, returns.asOfDate); + if (age === null || age >= 365) return returns; + return { + ...returns, + siAnn: null, + navReturns: returns.navReturns ? { ...returns.navReturns, siAnn: null } : null, + indexReturns: returns.indexReturns ? { ...returns.indexReturns, siAnn: null } : null, + }; +} + function anchor(days: ChartDay[], target: Date): ChartDay | null { let found: ChartDay | null = null; for (const day of days) { @@ -1433,7 +1464,8 @@ function anchor(days: ChartDay[], target: Date): ChartDay | null { return found; } -export function priceReturns(days: ChartDay[], now = new Date()): PriceReturns { +/** `inception`: the fund's first trade date (Yahoo) or official inception, ISO; siAnn stays null without it. */ +export function priceReturns(days: ChartDay[], inception: string | null = null): PriceReturns { const ordered = [...days].sort((a, b) => a.date.localeCompare(b.date)); const last = ordered[ordered.length - 1]; if (!last) return { ...EMPTY_PRICE_RETURNS }; @@ -1442,7 +1474,9 @@ export function priceReturns(days: ChartDay[], now = new Date()): PriceReturns { const yearStart = new Date(Date.UTC(date.getUTCFullYear(), 0, 1)); const quarterStartMonth = Math.floor(date.getUTCMonth() / 3) * 3; const quarterStart = new Date(Date.UTC(date.getUTCFullYear(), quarterStartMonth, 1)); - const monthStart = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth() - 1, date.getUTCDate())); + // Same day one month back, clamped to the end of a shorter month (Oct 31 -> Sep 30, never "Sep 31" = Oct 1). + const lastDayOfPreviousMonth = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), 0)).getUTCDate(); + const monthStart = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth() - 1, Math.min(date.getUTCDate(), lastDayOfPreviousMonth))); const start = (d: ChartDay | null) => d?.adjClose ?? null; const end = last.adjClose; return { @@ -1454,7 +1488,7 @@ export function priceReturns(days: ChartDay[], now = new Date()): PriceReturns { cagr3y: annualized(start(anchor(ordered, target(3))), end, 3), cagr5y: annualized(start(anchor(ordered, target(5))), end, 5), cagr10y: annualized(start(anchor(ordered, target(10))), end, 10), - siAnn: ordered.length > 1 ? annualizedSince(ordered[0], end, date) : null, + siAnn: ordered.length > 1 ? annualizedSince(ordered[0], end, date, inception) : null, }; } @@ -1539,8 +1573,15 @@ export function historyRows(days: ChartDay[], published: JsonRecord[] = []): Jso return days.map((day) => ({ Date: formatDate(day.date), Close: String(day.close), 'Adj Close': keepPublishedAdjClose(day.adjClose, publishedAdj.get(day.date)), Volume: String(day.volume) })); } -function mergeOfficialReturns(derived: PriceReturns, official: OfficialProductReturns | null): PriceReturns { +/** + * Official tenors win, gaps are filled from the Yahoo estimate. siAnn is never filled for a fund younger than one year at the + * official table date: the official figure was nulled for that reason, and a Yahoo estimate dated weeks later would sit under + * the official date (WQTM: -9.59 to 2026-10-23 published as of 2026-09-30). + */ +export function mergeOfficialReturns(derived: PriceReturns, official: OfficialProductReturns | null, inception: string | null = null): PriceReturns { if (!official) return derived; + const age = daysBetween(inception, official.asOfDate); + const youngAtTableDate = age !== null && age < 365; return { ...derived, asOfDate: official.asOfDate || derived.asOfDate, @@ -1551,7 +1592,7 @@ function mergeOfficialReturns(derived: PriceReturns, official: OfficialProductRe cagr3y: official.cagr3y ?? derived.cagr3y, cagr5y: official.cagr5y ?? derived.cagr5y, cagr10y: official.cagr10y ?? derived.cagr10y, - siAnn: official.siAnn ?? derived.siAnn, + siAnn: official.siAnn ?? (youngAtTableDate ? null : derived.siAnn), }; } @@ -1809,6 +1850,10 @@ async function processFund(fund: CatalogFund, config: UpdaterConfig, previous: J productPageMarkdown = page.text; productVia = page.via; productSummary = parseProductPageSummary(productPageMarkdown); + productSummary.officialReturns = { + monthEnd: withoutYoungSinceInception(productSummary.officialReturns.monthEnd, fund.inception), + quarterEnd: withoutYoungSinceInception(productSummary.officialReturns.quarterEnd, fund.inception), + }; fresh = true; if (productSummary.name) fund.name = productSummary.name; if (productSummary.cusip) fund.cusip = productSummary.cusip; @@ -1933,9 +1978,10 @@ async function processFund(fund: CatalogFund, config: UpdaterConfig, previous: J const distributionEvents = distributionRecordEvents(distributionRecords); const frequency = retainedDistributions ? { frequency: String(retainedDistributions.frequency || '—'), paymentsPerYear: numberOrNull(retainedDistributions.paymentsPerYear) } : inferDistributionFrequency(distributionEvents); const latest = distributionEvents[distributionEvents.length - 1] || null; - const derived = priceReturns(days); + const firstTrade = chart?.firstTradeDate ? new Date(chart.firstTradeDate * 1000).toISOString().slice(0, 10) : null; + const derived = priceReturns(days, firstTrade ?? fund.inception); const officialReturns = returnsFailed ? null : productSummary?.officialReturns.monthEnd || null; - const effectiveReturns = retainedReturns ? monthEndToPriceReturns(retainedReturns.monthEnd) : mergeOfficialReturns(derived, officialReturns); + const effectiveReturns = retainedReturns ? monthEndToPriceReturns(retainedReturns.monthEnd) : mergeOfficialReturns(derived, officialReturns, fund.inception); const officialBasis = Boolean(officialReturns) || retainedReturns !== null; // NAV, market price and premium/discount travel together: from the page, or all retained when the page failed.