fix(metrics): since-inception never annualized below one year or from a history window

README.md

@@ -64,7 +64,7 @@ Each fund carries a derived `metrics` object that powers the catalog columns sha
 - `ytd` / `tr1y` - official YTD and 1-year returns -> *YTD Return*, *TR 1Y*
 - `cagr3y` / `cagr5y` / `cagr10y` - published annualized 3Y/5Y/10Y figures -> *CAGR 3Y/5Y/10Y*
 - `tr3y` / `tr5y` / `tr10y` - cumulative 3Y/5Y/10Y figures `(1 + CAGR)^n - 1` -> *TR 3Y/5Y/10Y*
-- `siAnn` - since-inception annualized -> *SI Ann.*; derived only for funds with at least one year of history
+- `siAnn` - since-inception annualized -> *SI Ann.*; `null` below one year between inception and the returns date (WisdomTree prints the cumulative figure for younger funds), derived only from a Yahoo series that starts at the first trade date
 - `dividendYield` - 12-month trailing yield or indicated yield (latest distribution x frequency / price); a catalog 0.00% is the provider's published value and is kept as published (`yields.dividendYieldKind` says so)
 - `dividendYieldBasis` - code of the definition behind `dividendYield`, `null` exactly when `dividendYield` is `null`; it travels with the yield (a retained yield keeps its code):
 

scripts/update-data.test.ts

@@ -7,10 +7,10 @@ import { pathToFileURL } from 'node:url';
 import {
   CLOSED_FUNDS, CONTROL_NAMES, HISTORY_RANGE_PATTERN, RETURNS_BASIS_DERIVED, RETURNS_BASIS_OFFICIAL,
   annualizedToTotal, aumSource, cleanHoldingTicker, closedFund, deriveMetrics, dividendYieldBasisFromKind, formatDate, hasConfiguredFilters, holdingsGuard, inferDistributionFrequency, installSystemCa,
-  isCertError, isinFromCusip, looksLikeProductPage, mergeDistributionRecords, normalizeHoldingName, nportUrlFor, numberOrNull, parseAumRange,
+  isCertError, isinFromCusip, looksLikeProductPage, mergeDistributionRecords, mergeOfficialReturns, normalizeHoldingName, nportUrlFor, numberOrNull, parseAumRange,
   parseCatalogMarkdown, parseChart, parseFundListJson, parseEdgarAtomFilings, parseFundTickerMap, parseNport, parseOfficialDistributions,
   parseProductPageSummary, parseRange, parseTopHoldingsMarkdown, parseRanges, performanceAsOfDate, priceReturns, readConfig, readKnownRows, resolveControls,
-  rowFromMeta, runUpdate, runtimeControls, setApiRoot, toIsoDate, writeIndex, yahooChartQuery,
+  rowFromMeta, runUpdate, runtimeControls, setApiRoot, toIsoDate, withoutYoungSinceInception, writeIndex, yahooChartQuery,
 } from './update-data';
 
 // ---------------------------------------------------------------------------
@@ -328,22 +328,22 @@ describe('metrics', () => {
   const day = (date: string, close: number, adjClose = close) => ({ date, close, adjClose, volume: 1 });
 
   test('returns use adjusted closes, annualized values convert to cumulative', () => {
-    const result = priceReturns([day('2024-01-02', 100), day('2025-01-02', 110, 115), day('2026-01-02', 121, 132.25)], new Date('2026-01-03T00:00:00Z'));
+    const result = priceReturns([day('2024-01-02', 100), day('2025-01-02', 110, 115), day('2026-01-02', 121, 132.25)]);
     expect(result.yr1).toBe(32.25);
     expect(annualizedToTotal(10, 3)).toBe(33.1);
   });
 
   test('siAnn needs at least one year of history (null, never 0), a flat young fund keeps null horizons', () => {
     expect(priceReturns([day('2026-06-01', 100), day('2026-09-25', 110)]).siAnn).toBeNull();
-    expect(priceReturns([day('2024-09-25', 100), day('2026-09-25', 121)]).siAnn).toBeCloseTo(10, 0);
-    const young = priceReturns([day('2026-06-01', 100), day('2026-09-25', 110)], new Date('2026-09-26T00:00:00Z'));
+    expect(priceReturns([day('2024-09-25', 100), day('2026-09-25', 121)], '2024-09-25').siAnn).toBeCloseTo(10, 0);
+    const young = priceReturns([day('2026-06-01', 100), day('2026-09-25', 110)]);
     expect([young.yr1, young.cagr3y, young.cagr5y, young.cagr10y]).toEqual([null, null, null, null]);
   });
 
   test('metrics end with returnsBasis then performanceAsOf, official and derived bases travel with their own date', () => {
     const days = [day('2025-01-02', 100), day('2026-01-02', 110)];
     const fund = { dividendYield: null, secYield: null } as any;
-    const derived = priceReturns(days, new Date('2026-01-03T00:00:00Z'));
+    const derived = priceReturns(days);
     const estimated = deriveMetrics(derived, fund, [], { paymentsPerYear: null }, 110);
     expect([estimated.returnsBasis, estimated.performanceAsOf]).toEqual([RETURNS_BASIS_DERIVED, '2026-01-02']);
     expect(Object.keys(estimated).slice(-2)).toEqual(['returnsBasis', 'performanceAsOf']);
@@ -354,7 +354,7 @@ describe('metrics', () => {
   });
 
   test('dividendYieldBasis maps each yield source to a code and is null with a null yield', () => {
-    const derived = priceReturns([day('2025-01-02', 100), day('2026-01-02', 110)], new Date('2026-01-03T00:00:00Z'));
+    const derived = priceReturns([day('2025-01-02', 100), day('2026-01-02', 110)]);
     const dividends = [0, 1, 2].map((month) => ({ epoch: Date.UTC(2026, month, 15) / 1000, amount: 0.1 }));
     const basis = (fund: any, rows = dividends) => deriveMetrics(derived, { secYield: null, ...fund }, rows, { paymentsPerYear: rows.length ? 12 : null }, 100).dividendYieldBasis;
     expect(basis({ dividendYield: 4.72 })).toBe('official-trailing-12m');
@@ -377,6 +377,33 @@ describe('metrics 2', () => {
     expect([none.metrics.dividendYield, none.metrics.dividendYieldBasis]).toEqual([null, null]);
   });
 
+  test('derived siAnn only from a series that starts at inception and spans a year, the 1-month window clamps to a shorter month', () => {
+    const day = (date: string, close: number) => ({ date, close, adjClose: close, volume: 1 });
+    const closes = [day('2021-09-30', 100), day('2024-09-30', 150), day('2026-09-30', 200)];
+    expect(priceReturns(closes, '2021-09-28').siAnn).toBeCloseTo(14.87, 1);
+    // A HISTORY_RANGE window that starts years after inception is not since inception (was cagr5y-like 14.87)
+    expect(priceReturns(closes.slice(1), '2021-09-28').siAnn).toBeNull();
+    expect(priceReturns(closes, null).siAnn).toBeNull();
+    // Oct 31 minus one month is Sep 30, not "Sep 31" = Oct 1 (that anchor gave -45)
+    expect(priceReturns([day('2026-09-30', 100), day('2026-10-01', 200), day('2026-10-31', 110)], '2026-09-30').mo1).toBe(10);
+  });
+
+  test('official since inception is null below one year between inception and the table date (main, NAV and index rows)', () => {
+    const monthEnd = parseProductPageSummary(usfrPage).officialReturns.monthEnd;
+    expect([monthEnd?.siAnn, monthEnd?.navReturns?.siAnn, monthEnd?.indexReturns?.siAnn]).toEqual([1.99, 2, 2.17]);
+    // NTSD: launched 2026-03-19, the 2026-09-30 table carried the cumulative 20.18 as annualized
+    const young = withoutYoungSinceInception(monthEnd, '2026-03-19');
+    expect([young?.siAnn, young?.navReturns?.siAnn, young?.indexReturns?.siAnn, young?.yr1, young?.ytd]).toEqual([null, null, null, 4, 2.59]);
+    expect(withoutYoungSinceInception(monthEnd, '2025-08-31')?.siAnn).toBe(1.99);
+    expect(withoutYoungSinceInception(monthEnd, '2014-12-11')?.siAnn).toBe(1.99);
+    expect(withoutYoungSinceInception(monthEnd, null)?.siAnn).toBe(1.99);
+    // and the gap is not filled with a Yahoo estimate from a later close (WQTM published -9.59 as of 2026-09-30)
+    const yahoo = { ...priceReturns([{ date: '2025-10-09', close: 100, adjClose: 100, volume: 1 }, { date: '2026-10-23', close: 90, adjClose: 90, volume: 1 }], '2025-10-09'), asOfDate: '2026-10-23' };
+    expect(yahoo.siAnn).not.toBeNull();
+    expect(mergeOfficialReturns(yahoo, withoutYoungSinceInception(monthEnd, '2025-10-09'), '2025-10-09').siAnn).toBeNull();
+    expect(mergeOfficialReturns(yahoo, { ...monthEnd!, siAnn: null }, '2014-12-11').siAnn).toBe(yahoo.siAnn);
+  });
+
   test('distribution frequency is inferred like in every sibling updater', () => {
     const monthly = [0, 1, 2, 3].map((month) => ({ epoch: Date.UTC(2026, month, 15) / 1000, amount: 0.1 }));
     expect(inferDistributionFrequency(monthly)).toEqual({ frequency: 'Monthly', paymentsPerYear: 12 });

scripts/update-data.ts

@@ -1418,12 +1418,43 @@ function annualized(start: number | null, end: number | null, years: number): nu
   return round(((end / start) ** (1 / years) - 1) * 100, 2);
 }
 
-/** Since-inception annualized return; null under one year of history (a few weeks annualized is noise). */
-function annualizedSince(first: ChartDay, end: number, date: Date): number | null {
+/** Days between two ISO dates (UTC), null when either is not an ISO date. */
+function daysBetween(from: string | null | undefined, to: string | null | undefined): number | null {
+  const a = toIsoDate(from), b = toIsoDate(to);
+  if (!/^\d{4}-\d{2}-\d{2}$/.test(a) || !/^\d{4}-\d{2}-\d{2}$/.test(b)) return null;
+  return (Date.parse(`${b}T00:00:00Z`) - Date.parse(`${a}T00:00:00Z`)) / 86_400_000;
+}
+
+/**
+ * Since-inception annualized return derived from closes. It is a since-INCEPTION figure only when the series starts at
+ * inception (first close within 10 days of the fund's first trade date, the official inception as fallback), so a
+ * HISTORY_RANGE window never turns cagr5y into siAnn; null under one year (a few months annualized is noise) or when
+ * the inception is unknown.
+ */
+function annualizedSince(first: ChartDay, end: number, date: Date, inception: string | null): number | null {
+  const gap = daysBetween(inception, first.date);
+  if (gap === null || gap > 10) return null;
   const years = (date.getTime() - new Date(`${first.date}T00:00:00Z`).getTime()) / (365.25 * 86_400_000);
   return years < 1 ? null : annualized(first.adjClose, end, years);
 }
 
+/**
+ * WisdomTree publishes "Since Inception" CUMULATIVE for a fund younger than one year (NTSD, launched 2026-03-19: 20.18 as of
+ * 2026-09-30) in the Average Annual row. Annualized since-inception exists only from one year on, so below 365 days
+ * between inception and the table date it is null (main, NAV and index rows); an unknown inception keeps the figure.
+ */
+export function withoutYoungSinceInception(returns: OfficialProductReturns | null, inception: string | null): OfficialProductReturns | null {
+  if (!returns) return returns;
+  const age = daysBetween(inception, returns.asOfDate);
+  if (age === null || age >= 365) return returns;
+  return {
+    ...returns,
+    siAnn: null,
+    navReturns: returns.navReturns ? { ...returns.navReturns, siAnn: null } : null,
+    indexReturns: returns.indexReturns ? { ...returns.indexReturns, siAnn: null } : null,
+  };
+}
+
 function anchor(days: ChartDay[], target: Date): ChartDay | null {
   let found: ChartDay | null = null;
   for (const day of days) {
@@ -1433,7 +1464,8 @@ function anchor(days: ChartDay[], target: Date): ChartDay | null {
   return found;
 }
 
-export function priceReturns(days: ChartDay[], now = new Date()): PriceReturns {
+/** `inception`: the fund's first trade date (Yahoo) or official inception, ISO; siAnn stays null without it. */
+export function priceReturns(days: ChartDay[], inception: string | null = null): PriceReturns {
   const ordered = [...days].sort((a, b) => a.date.localeCompare(b.date));
   const last = ordered[ordered.length - 1];
   if (!last) return { ...EMPTY_PRICE_RETURNS };
@@ -1442,7 +1474,9 @@ export function priceReturns(days: ChartDay[], now = new Date()): PriceReturns {
   const yearStart = new Date(Date.UTC(date.getUTCFullYear(), 0, 1));
   const quarterStartMonth = Math.floor(date.getUTCMonth() / 3) * 3;
   const quarterStart = new Date(Date.UTC(date.getUTCFullYear(), quarterStartMonth, 1));
-  const monthStart = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth() - 1, date.getUTCDate()));
+  // Same day one month back, clamped to the end of a shorter month (Oct 31 -> Sep 30, never "Sep 31" = Oct 1).
+  const lastDayOfPreviousMonth = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), 0)).getUTCDate();
+  const monthStart = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth() - 1, Math.min(date.getUTCDate(), lastDayOfPreviousMonth)));
   const start = (d: ChartDay | null) => d?.adjClose ?? null;
   const end = last.adjClose;
   return {
@@ -1454,7 +1488,7 @@ export function priceReturns(days: ChartDay[], now = new Date()): PriceReturns {
     cagr3y: annualized(start(anchor(ordered, target(3))), end, 3),
     cagr5y: annualized(start(anchor(ordered, target(5))), end, 5),
     cagr10y: annualized(start(anchor(ordered, target(10))), end, 10),
-    siAnn: ordered.length > 1 ? annualizedSince(ordered[0], end, date) : null,
+    siAnn: ordered.length > 1 ? annualizedSince(ordered[0], end, date, inception) : null,
   };
 }
 
@@ -1539,8 +1573,15 @@ export function historyRows(days: ChartDay[], published: JsonRecord[] = []): Jso
   return days.map((day) => ({ Date: formatDate(day.date), Close: String(day.close), 'Adj Close': keepPublishedAdjClose(day.adjClose, publishedAdj.get(day.date)), Volume: String(day.volume) }));
 }
 
-function mergeOfficialReturns(derived: PriceReturns, official: OfficialProductReturns | null): PriceReturns {
+/**
+ * Official tenors win, gaps are filled from the Yahoo estimate. siAnn is never filled for a fund younger than one year at the
+ * official table date: the official figure was nulled for that reason, and a Yahoo estimate dated weeks later would sit under
+ * the official date (WQTM: -9.59 to 2026-10-23 published as of 2026-09-30).
+ */
+export function mergeOfficialReturns(derived: PriceReturns, official: OfficialProductReturns | null, inception: string | null = null): PriceReturns {
   if (!official) return derived;
+  const age = daysBetween(inception, official.asOfDate);
+  const youngAtTableDate = age !== null && age < 365;
   return {
     ...derived,
     asOfDate: official.asOfDate || derived.asOfDate,
@@ -1551,7 +1592,7 @@ function mergeOfficialReturns(derived: PriceReturns, official: OfficialProductRe
     cagr3y: official.cagr3y ?? derived.cagr3y,
     cagr5y: official.cagr5y ?? derived.cagr5y,
     cagr10y: official.cagr10y ?? derived.cagr10y,
-    siAnn: official.siAnn ?? derived.siAnn,
+    siAnn: official.siAnn ?? (youngAtTableDate ? null : derived.siAnn),
   };
 }
 
@@ -1809,6 +1850,10 @@ async function processFund(fund: CatalogFund, config: UpdaterConfig, previous: J
       productPageMarkdown = page.text;
       productVia = page.via;
       productSummary = parseProductPageSummary(productPageMarkdown);
+      productSummary.officialReturns = {
+        monthEnd: withoutYoungSinceInception(productSummary.officialReturns.monthEnd, fund.inception),
+        quarterEnd: withoutYoungSinceInception(productSummary.officialReturns.quarterEnd, fund.inception),
+      };
       fresh = true;
       if (productSummary.name) fund.name = productSummary.name;
       if (productSummary.cusip) fund.cusip = productSummary.cusip;
@@ -1933,9 +1978,10 @@ async function processFund(fund: CatalogFund, config: UpdaterConfig, previous: J
   const distributionEvents = distributionRecordEvents(distributionRecords);
   const frequency = retainedDistributions ? { frequency: String(retainedDistributions.frequency || '—'), paymentsPerYear: numberOrNull(retainedDistributions.paymentsPerYear) } : inferDistributionFrequency(distributionEvents);
   const latest = distributionEvents[distributionEvents.length - 1] || null;
-  const derived = priceReturns(days);
+  const firstTrade = chart?.firstTradeDate ? new Date(chart.firstTradeDate * 1000).toISOString().slice(0, 10) : null;
+  const derived = priceReturns(days, firstTrade ?? fund.inception);
   const officialReturns = returnsFailed ? null : productSummary?.officialReturns.monthEnd || null;
-  const effectiveReturns = retainedReturns ? monthEndToPriceReturns(retainedReturns.monthEnd) : mergeOfficialReturns(derived, officialReturns);
+  const effectiveReturns = retainedReturns ? monthEndToPriceReturns(retainedReturns.monthEnd) : mergeOfficialReturns(derived, officialReturns, fund.inception);
   const officialBasis = Boolean(officialReturns) || retainedReturns !== null;
 
   // NAV, market price and premium/discount travel together: from the page, or all retained when the page failed.