diff --git a/bench/CONTRACT.md b/bench/CONTRACT.md index 3cebf9b..7ff31d7 100644 --- a/bench/CONTRACT.md +++ b/bench/CONTRACT.md @@ -63,6 +63,13 @@ column-by-column; an unknown `kind` or a missing dimension field fails the inges `compression_size` records include `uncompressed_bytes`, the Arrow memory size after the source Parquet file is decoded. The column is nullable so historical rows remain valid. +`string-bench` shares the two compression tables with `compress-bench`. Its records use the +dataset `strings/`, the string encoder (`onpair-12`, `fsst`, ...) as `dataset_variant`, +and `vortex-file-compressed` as `format`; `encode` is the Vortex file write and `decode` the scan. +Its `uncompressed_bytes` is the column's canonical Arrow `Utf8View` size. The site lists these +rows in the `String Encoding` groups and compares each encoder with `fsst`, so the prefix and the +`fsst` label are part of the contract. + `random_access_time` records include `open_mode`. The value is `cached` for a reused accessor or `reopen` when each timed take includes a new accessor. Historical rows use `cached`. The `reopen` mode does not clear the OS page cache. diff --git a/bench/web/components/SummaryCard.tsx b/bench/web/components/SummaryCard.tsx index c5b8f57..d2e2571 100644 --- a/bench/web/components/SummaryCard.tsx +++ b/bench/web/components/SummaryCard.tsx @@ -131,7 +131,7 @@ export function SummaryCard({ summary }: { summary?: Summary | undefined }) {
- Vs Parquet + Vs {summary.baseline ?? 'Parquet'} Min Mean Max diff --git a/bench/web/lib/benchmark-docs.test.ts b/bench/web/lib/benchmark-docs.test.ts index 51a8fbd..a70bd9e 100644 --- a/bench/web/lib/benchmark-docs.test.ts +++ b/bench/web/lib/benchmark-docs.test.ts @@ -42,6 +42,12 @@ describe('groupDocPath', () => { expect(groupDocPath({ k: 'CompressionTimeGroup' })).toBe('benchmarks/compress-bench/README.md'); // Two measurement kinds of one benchmark, so one doc. expect(groupDocPath({ k: 'CompressionSizeGroup' })).toBe('benchmarks/compress-bench/README.md'); + expect(groupDocPath({ k: 'CompressionTimeGroup', suite: 'strings' })).toBe( + 'benchmarks/string-bench/README.md', + ); + expect(groupDocPath({ k: 'CompressionSizeGroup', suite: 'strings' })).toBe( + 'benchmarks/string-bench/README.md', + ); expect(groupDocPath({ k: 'RandomAccessGroup', storage: 'nvme' })).toBe( 'benchmarks/random-access-bench/README.md', ); diff --git a/bench/web/lib/benchmark-docs.ts b/bench/web/lib/benchmark-docs.ts index 22ef256..096d339 100644 --- a/bench/web/lib/benchmark-docs.ts +++ b/bench/web/lib/benchmark-docs.ts @@ -52,6 +52,9 @@ const QUERY_DOCS: Readonly> = { /** The compression benchmark, whose two measurement kinds share one doc. */ const COMPRESS_BENCH_DOC = 'benchmarks/compress-bench/README.md'; +/** The string encoding benchmark, which shares the compression tables. */ +const STRING_BENCH_DOC = 'benchmarks/string-bench/README.md'; + /** The random-access benchmark. */ const RANDOM_ACCESS_BENCH_DOC = 'benchmarks/random-access-bench/README.md'; @@ -79,7 +82,7 @@ export function groupDocPath(key: GroupKey): string | null { switch (key.k) { case 'CompressionTimeGroup': case 'CompressionSizeGroup': - return COMPRESS_BENCH_DOC; + return key.suite === 'strings' ? STRING_BENCH_DOC : COMPRESS_BENCH_DOC; case 'RandomAccessGroup': return RANDOM_ACCESS_BENCH_DOC; case 'QueryGroup': diff --git a/bench/web/lib/compression-suite.ts b/bench/web/lib/compression-suite.ts new file mode 100644 index 0000000..ae26aea --- /dev/null +++ b/bench/web/lib/compression-suite.ts @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +/** + * Which benchmark suite wrote a compression row: `compress-bench`, or `string-bench`, + * which shares the compression tables under the `strings/` dataset prefix with the + * string encoder in `dataset_variant`. + */ +export type CompressionSuite = 'compress' | 'strings'; + +/** Dataset prefix `string-bench` gives every row it writes. */ +export const STRING_DATASET_PREFIX = 'strings/'; + +/** The `LIKE` pattern matching every `string-bench` dataset. */ +export const STRING_DATASET_PATTERN = `${STRING_DATASET_PREFIX}%`; + +export function compressionSuite(dataset: string): CompressionSuite { + return dataset.startsWith(STRING_DATASET_PREFIX) ? 'strings' : 'compress'; +} diff --git a/bench/web/lib/descriptions.ts b/bench/web/lib/descriptions.ts index d97a3af..db581ac 100644 --- a/bench/web/lib/descriptions.ts +++ b/bench/web/lib/descriptions.ts @@ -51,6 +51,10 @@ function staticDescription(name: string): string | null { 'Compares compressed file sizes and compression ratios across different encoding ' + 'strategies' ); + case 'String Encoding': + return 'Measures Vortex file write and scan throughput for each string encoder'; + case 'String Encoding Size': + return 'Compares Vortex file sizes for each string encoder'; case 'Clickbench': return ( "ClickHouse's analytical benchmark suite testing real-world query patterns on web " + diff --git a/bench/web/lib/groups.test.ts b/bench/web/lib/groups.test.ts index f4fc293..8e48510 100644 --- a/bench/web/lib/groups.test.ts +++ b/bench/web/lib/groups.test.ts @@ -527,6 +527,96 @@ describe.skipIf(!databaseAvailable)('summary math fidelity (testcontainers Postg ).toBeNull(); }); + it('splits string-bench rows into their own groups, summaries, and chart bundles', async () => { + const sha = 'a'.repeat(40); + await insertCommit(sha, '2026-04-23T12:00:00Z'); + await insertCompTimePair(sha, 'encode', 100, 200); + await insertCompSizePair(sha, 100, 200); + // `string-bench` writes `strings/` with the encoder in `dataset_variant`. + for (const [encoder, encodeNs, decodeNs, bytes] of [ + ['fsst', 400, 100, 300], + ['onpair-12', 200, 50, 200], + ] as const) { + await getPool().query( + `INSERT INTO compression_times + (measurement_id, commit_sha, dataset, dataset_variant, format, op, + value_ns, all_runtimes_ns) + VALUES ($1, $2, 'strings/fineweb/text', $3, 'vortex-file-compressed', 'encode', $5, + '{1}'::bigint[]), + ($4, $2, 'strings/fineweb/text', $3, 'vortex-file-compressed', 'decode', $6, + '{1}'::bigint[])`, + [nextId(), sha, encoder, nextId(), encodeNs, decodeNs], + ); + await getPool().query( + `INSERT INTO compression_sizes + (measurement_id, commit_sha, dataset, dataset_variant, format, + value_bytes, uncompressed_bytes) + VALUES ($1, $2, 'strings/fineweb/text', $3, 'vortex-file-compressed', $4, 1200)`, + [nextId(), sha, encoder, bytes], + ); + } + + const groups = await collectGroups(); + expect(groups.map((group) => group.name)).toEqual([ + 'Compression', + 'Compression Size', + 'String Encoding', + 'String Encoding Size', + ]); + const byName = new Map(groups.map((group) => [group.name, group])); + expect(groupKeyFromSlug(byName.get('Compression')!.slug)).toEqual({ + k: 'CompressionTimeGroup', + }); + expect(byName.get('Compression')!.charts.map((chart) => chart.name)).toEqual(['tpch-lineitem']); + expect(byName.get('Compression')!.summary).toMatchObject({ + type: 'compression', + rankings: [ + { name: 'vortex-file-compressed', operation: 'encode', ratio: 2 }, + { name: 'parquet', operation: 'encode', ratio: 1 }, + ], + }); + expect(groupKeyFromSlug(byName.get('String Encoding')!.slug)).toEqual({ + k: 'CompressionTimeGroup', + suite: 'strings', + }); + expect(byName.get('String Encoding')!.charts).toEqual( + ['fsst', 'onpair-12'].map((encoder) => ({ + name: `fineweb/text/${encoder}`, + slug: chartKeyToSlug({ + k: 'CompressionTime', + dataset: 'strings/fineweb/text', + dataset_variant: encoder, + }), + })), + ); + expect(byName.get('String Encoding')!.summary).toMatchObject({ + type: 'compression', + rankings: [ + { name: 'onpair-12', operation: 'encode', ratio: 2, throughputGbS: 6 }, + { name: 'fsst', operation: 'encode', ratio: 1, throughputGbS: 3 }, + { name: 'onpair-12', operation: 'decode', ratio: 2, throughputGbS: 24 }, + { name: 'fsst', operation: 'decode', ratio: 1, throughputGbS: 12 }, + ], + }); + expect(byName.get('String Encoding Size')!.summary).toMatchObject({ + type: 'compressionSize', + baseline: 'FSST', + rankings: [ + { name: 'onpair-12', ratio: 2 / 3, compressionRatio: 6 }, + { name: 'fsst', ratio: 1, compressionRatio: 4 }, + ], + }); + + const bundle = await collectGroupCharts( + { k: 'CompressionSizeGroup', suite: 'strings' }, + parseCommitWindow(null), + ); + expect(bundle?.charts.map((chart) => chart.name)).toEqual([ + 'fineweb/text/fsst', + 'fineweb/text/onpair-12', + ]); + }); + it('ranks random access over every chart, not the alphabetically first one', async () => { // The bug this pins: the old summary walked the group's chart links and // published the first populated chart's raw times under the group-wide diff --git a/bench/web/lib/queries.ts b/bench/web/lib/queries.ts index 85030c6..116c8bd 100644 --- a/bench/web/lib/queries.ts +++ b/bench/web/lib/queries.ts @@ -36,6 +36,11 @@ import type { PoolClient } from 'pg'; import { BRANCH_COMMITS, factOnReadBranch } from './branch'; import { getPool } from './db'; import { randomAccessStorage } from './random-access'; +import { + compressionSuite, + type CompressionSuite, + STRING_DATASET_PREFIX, +} from './compression-suite'; import { groupDescription } from './descriptions'; import type { FilterUniverse } from './chart-format'; import { type ChartKind, compareCodeUnits, FAMILIES, type GroupKind } from './families'; @@ -710,9 +715,13 @@ function chartInGroup(group: GroupKey, key: ChartKey): boolean { key.storage === group.storage ); case 'CompressionTimeGroup': - return key.k === 'CompressionTime'; + return ( + key.k === 'CompressionTime' && compressionSuite(key.dataset) === (group.suite ?? 'compress') + ); case 'CompressionSizeGroup': - return key.k === 'CompressionSize'; + return ( + key.k === 'CompressionSize' && compressionSuite(key.dataset) === (group.suite ?? 'compress') + ); case 'RandomAccessGroup': return key.k === 'RandomAccess' && randomAccessStorage(key.dataset) === group.storage; case 'VectorSearchGroup': @@ -1148,6 +1157,8 @@ export interface GroupChartsResponse { const GROUP_ORDER: readonly string[] = [ 'Compression', 'Compression Size', + 'String Encoding', + 'String Encoding Size', 'Random Access (NVMe)', 'Random Access (S3)', 'Clickbench', @@ -1421,7 +1432,50 @@ async function collectQueryGroups(): Promise { type DatasetVariantRow = { dataset: string; dataset_variant: string | null }; -/** The single `Compression` group, or `[]` if the fact table is empty. */ +/** Group display names per compression suite and family. */ +const COMPRESSION_GROUP_NAMES: Record< + 'CompressionTimeGroup' | 'CompressionSizeGroup', + Record +> = { + CompressionTimeGroup: { compress: 'Compression', strings: 'String Encoding' }, + CompressionSizeGroup: { compress: 'Compression Size', strings: 'String Encoding Size' }, +}; + +/** + * Split one compression table's `(dataset, dataset_variant)` rows into the `compress-bench` + * group and the `string-bench` group, dropping a group with no rows. String charts drop the + * shared `strings/` prefix from their names. + */ +function compressionGroups( + rows: readonly DatasetVariantRow[], + groupKind: 'CompressionTimeGroup' | 'CompressionSizeGroup', +): Group[] { + const chartKind = groupKind === 'CompressionTimeGroup' ? 'CompressionTime' : 'CompressionSize'; + const suites: CompressionSuite[] = ['compress', 'strings']; + return suites.flatMap((suite) => { + const charts: ChartLink[] = rows + .filter((row) => compressionSuite(row.dataset) === suite) + .map((row) => { + const dataset = + suite === 'strings' ? row.dataset.slice(STRING_DATASET_PREFIX.length) : row.dataset; + return { + name: row.dataset_variant !== null ? `${dataset}/${row.dataset_variant}` : dataset, + slug: chartKeyToSlug({ + k: chartKind, + dataset: row.dataset, + dataset_variant: row.dataset_variant, + }), + }; + }); + if (charts.length === 0) { + return []; + } + const key: GroupKey = suite === 'strings' ? { k: groupKind, suite } : { k: groupKind }; + return [{ name: COMPRESSION_GROUP_NAMES[groupKind][suite], slug: groupKeyToSlug(key), charts }]; + }); +} + +/** The `Compression` and `String Encoding` groups, or `[]` if the fact table is empty. */ async function collectCompressionTimeGroup(): Promise { const text = ` SELECT dataset, dataset_variant @@ -1430,21 +1484,10 @@ async function collectCompressionTimeGroup(): Promise { ORDER BY dataset, dataset_variant NULLS FIRST `; const rows = (await getPool().query(text)).rows; - if (rows.length === 0) { - return []; - } - const charts: ChartLink[] = rows.map((row) => ({ - name: row.dataset_variant !== null ? `${row.dataset}/${row.dataset_variant}` : row.dataset, - slug: chartKeyToSlug({ - k: 'CompressionTime', - dataset: row.dataset, - dataset_variant: row.dataset_variant, - }), - })); - return [{ name: 'Compression', slug: groupKeyToSlug({ k: 'CompressionTimeGroup' }), charts }]; + return compressionGroups(rows, 'CompressionTimeGroup'); } -/** The single `Compression Size` group, or `[]` if the fact table is empty. */ +/** The `Compression Size` and `String Encoding Size` groups, or `[]` if the fact table is empty. */ async function collectCompressionSizeGroup(): Promise { const text = ` SELECT dataset, dataset_variant @@ -1453,20 +1496,7 @@ async function collectCompressionSizeGroup(): Promise { ORDER BY dataset, dataset_variant NULLS FIRST `; const rows = (await getPool().query(text)).rows; - if (rows.length === 0) { - return []; - } - const charts: ChartLink[] = rows.map((row) => ({ - name: row.dataset_variant !== null ? `${row.dataset}/${row.dataset_variant}` : row.dataset, - slug: chartKeyToSlug({ - k: 'CompressionSize', - dataset: row.dataset, - dataset_variant: row.dataset_variant, - }), - })); - return [ - { name: 'Compression Size', slug: groupKeyToSlug({ k: 'CompressionSizeGroup' }), charts }, - ]; + return compressionGroups(rows, 'CompressionSizeGroup'); } /** One random-access group per storage backend, omitting empty sections. */ diff --git a/bench/web/lib/slug.test.ts b/bench/web/lib/slug.test.ts index cd79076..977a011 100644 --- a/bench/web/lib/slug.test.ts +++ b/bench/web/lib/slug.test.ts @@ -31,6 +31,8 @@ const GROUP_KEYS: GroupKey[] = [ { k: 'QueryGroup', dataset: 'tpch', dataset_variant: null, scale_factor: '1', storage: 'nvme' }, { k: 'CompressionTimeGroup' }, { k: 'CompressionSizeGroup' }, + { k: 'CompressionTimeGroup', suite: 'strings' }, + { k: 'CompressionSizeGroup', suite: 'strings' }, { k: 'RandomAccessGroup', storage: 'nvme' }, { k: 'RandomAccessGroup', storage: 's3' }, { k: 'VectorSearchGroup', dataset: 'cohere', layout: 'partitioned' }, @@ -84,6 +86,19 @@ describe('group slug round-trip', () => { expect(chartPrefix).not.toBe(groupPrefix); }); + it('keeps the compress-bench group slugs from before the string-bench split', () => { + expect(groupKeyToSlug({ k: 'CompressionTimeGroup' })).toBe( + `ctg.${Buffer.from('{"k":"CompressionTimeGroup"}').toString('base64url')}`, + ); + }); + + it.each(['compress', '', null, 1])('rejects invalid compression group suite %s', (suite) => { + const slug = + 'ctg.' + + Buffer.from(JSON.stringify({ k: 'CompressionTimeGroup', suite })).toString('base64url'); + expect(() => groupKeyFromSlug(slug)).toThrow(); + }); + it('resolves legacy random-access group slugs to NVMe', () => { const legacy = `rag.${Buffer.from('{"k":"RandomAccessGroup"}').toString('base64url')}`; expect(groupKeyFromSlug(legacy)).toEqual({ k: 'RandomAccessGroup', storage: 'nvme' }); diff --git a/bench/web/lib/slug.ts b/bench/web/lib/slug.ts index ea1178f..6373b6f 100644 --- a/bench/web/lib/slug.ts +++ b/bench/web/lib/slug.ts @@ -43,8 +43,9 @@ export type GroupKey = scale_factor: string | null; storage: string; } - | { k: 'CompressionTimeGroup' } - | { k: 'CompressionSizeGroup' } + // `suite` is absent for the `compress-bench` group, so its slug predates the split. + | { k: 'CompressionTimeGroup'; suite?: 'strings' } + | { k: 'CompressionSizeGroup'; suite?: 'strings' } | { k: 'RandomAccessGroup'; storage: RandomAccessStorage } | { k: 'VectorSearchGroup'; dataset: string; layout: string }; @@ -240,7 +241,7 @@ export function groupKeyToSlug(key: GroupKey): string { break; case 'CompressionTimeGroup': case 'CompressionSizeGroup': - ordered = { k: key.k }; + ordered = key.suite === undefined ? { k: key.k } : { k: key.k, suite: key.suite }; break; } return `${prefix}.${encodePayload(JSON.stringify(ordered))}`; @@ -277,7 +278,14 @@ export function groupKeyFromSlug(slug: string): GroupKey { return { k: kind, storage }; } case 'CompressionTimeGroup': - case 'CompressionSizeGroup': - return { k: kind }; + case 'CompressionSizeGroup': { + if (obj.suite === undefined) { + return { k: kind }; + } + if (obj.suite !== 'strings') { + throw new Error('compression group suite must be strings when present'); + } + return { k: kind, suite: 'strings' }; + } } } diff --git a/bench/web/lib/summary.test.ts b/bench/web/lib/summary.test.ts index d92f3a6..1e6a8a3 100644 --- a/bench/web/lib/summary.test.ts +++ b/bench/web/lib/summary.test.ts @@ -27,17 +27,17 @@ describe('compression summaries', () => { { format: 'vortex-file-compressed', valueBytes: 1, - parquetBytes: 4, + baselineBytes: 4, uncompressedBytes: 16, }, { format: 'vortex-file-compressed', valueBytes: 900, - parquetBytes: 100, + baselineBytes: 100, uncompressedBytes: 3_600, }, - { format: 'parquet', valueBytes: 4, parquetBytes: 4, uncompressedBytes: 16 }, - { format: 'parquet', valueBytes: 100, parquetBytes: 100, uncompressedBytes: null }, + { format: 'parquet', valueBytes: 4, baselineBytes: 4, uncompressedBytes: 16 }, + { format: 'parquet', valueBytes: 100, baselineBytes: 100, uncompressedBytes: null }, ], }); @@ -72,21 +72,21 @@ describe('compression summaries', () => { format: 'vortex-file-compressed', op: 'encode', valueNs: 100, - parquetNs: 200, + baselineNs: 200, uncompressedBytes: 1_000, }, { format: 'vortex-file-compressed', op: 'encode', valueNs: 300, - parquetNs: 300, + baselineNs: 300, uncompressedBytes: 2_000, }, { format: 'vortex-file-compressed', op: 'encode', valueNs: 500, - parquetNs: 250, + baselineNs: 250, uncompressedBytes: null, }, ], @@ -108,19 +108,19 @@ describe('compression summaries', () => { { format: 'vortex-file-compressed', valueBytes: 200, - parquetBytes: 100, + baselineBytes: 100, uncompressedBytes: 2_000, }, { format: 'parquet', valueBytes: 100, - parquetBytes: 100, + baselineBytes: 100, uncompressedBytes: 500, }, { format: 'lance', valueBytes: 300, - parquetBytes: 100, + baselineBytes: 100, uncompressedBytes: null, }, ], @@ -138,6 +138,52 @@ describe('compression summaries', () => { ]); }); + it('compares string encoders with FSST at the newest string-bench commit', async () => { + query.mockResolvedValueOnce({ + rows: [ + { format: 'fsst', op: 'encode', valueNs: 400, baselineNs: 400, uncompressedBytes: 1_200 }, + { + format: 'onpair-12', + op: 'encode', + valueNs: 200, + baselineNs: 400, + uncompressedBytes: 1_200, + }, + ], + }); + query.mockResolvedValueOnce({ + rows: [ + { format: 'fsst', valueBytes: 300, baselineBytes: 300, uncompressedBytes: 1_200 }, + { format: 'onpair-12', valueBytes: 200, baselineBytes: 300, uncompressedBytes: 1_200 }, + ], + }); + + const time = await collectGroupSummary({ k: 'CompressionTimeGroup', suite: 'strings' }); + const size = await collectGroupSummary({ k: 'CompressionSizeGroup', suite: 'strings' }); + + expect(query.mock.calls.map((call) => call[1])).toEqual([ + ['strings/%', 'fsst'], + ['strings/%', 'fsst'], + ]); + if (time === null || time.type !== 'compression') { + throw new Error('expected a compression summary'); + } + expect(time.rankings).toEqual([ + { name: 'onpair-12', operation: 'encode', ratio: 2, throughputGbS: 6 }, + { name: 'fsst', operation: 'encode', ratio: 1, throughputGbS: 3 }, + ]); + expect(time.explanation).toBe( + 'Geomean throughput ratio to FSST | Aggregate throughput (higher is better)', + ); + if (size === null || size.type !== 'compressionSize') { + throw new Error('expected a compressionSize summary'); + } + expect(size.baseline).toBe('FSST'); + expect(size.rankings.map((ranking) => ranking.name)).toEqual(['onpair-12', 'fsst']); + expect(size.rankings[0]?.ratio).toBeCloseTo(2 / 3, 6); + expect(size.rankings[0]?.compressionRatio).toBeCloseTo(6, 6); + }); + it('applies extensible snapshot policies to timings and sizes', async () => { query.mockResolvedValue({ rows: [] }); diff --git a/bench/web/lib/summary.ts b/bench/web/lib/summary.ts index 1cd6671..c5e683c 100644 --- a/bench/web/lib/summary.ts +++ b/bench/web/lib/summary.ts @@ -41,6 +41,7 @@ import { getPool } from './db'; import { randomAccessStorage, type RandomAccessStorage } from './random-access'; import { compareCodeUnits } from './families'; import type { GroupKey } from './slug'; +import { STRING_DATASET_PATTERN } from './compression-suite'; /** * Freshness buckets for compression-time and compression-size summaries. @@ -56,6 +57,9 @@ const ARROW_IPC_FORMAT = 'arrow-ipc'; const COMPRESSION_SNAPSHOT_ANCHOR = 'vortex-file-compressed'; const COMPRESSION_BASELINE = 'parquet'; +/** The `string-bench` encoder every other string encoder is compared with. */ +const STRING_ENCODING_BASELINE = 'fsst'; + function compressionSummaryQueryParams(): [string[], string[], string, string] { return [ [...COMPRESSION_SHARED_SNAPSHOT_FORMATS, ...COMPRESSION_INDEPENDENT_SNAPSHOT_FORMATS], @@ -154,6 +158,8 @@ export type Summary = title: string; rankings: CompressionSizeRanking[]; explanation: string; + /** What `ratio` compares sizes with, when not Parquet. */ + baseline?: string; } | { type: 'queryBenchmark'; @@ -178,9 +184,11 @@ export function collectGroupSummary(key: GroupKey): Promise { case 'QueryGroup': return collectQuerySummary(key.dataset, key.dataset_variant, key.scale_factor, key.storage); case 'CompressionTimeGroup': - return collectCompressionSummary(); + return key.suite === 'strings' ? collectStringEncodingSummary() : collectCompressionSummary(); case 'CompressionSizeGroup': - return collectCompressionSizeSummary(); + return key.suite === 'strings' + ? collectStringEncodingSizeSummary() + : collectCompressionSizeSummary(); case 'RandomAccessGroup': return collectRandomAccessSummary(key.storage); case 'VectorSearchGroup': @@ -516,7 +524,32 @@ async function collectVectorSearchSummary( async function collectCompressionSummary(): Promise { // Both geomeans use the latest encode timestamp. If no encode pair exists, // they use the latest decode timestamp. SQL preserves sub-second precision. - const rows = await compressionSamples(); + const rankings = compressionRankings(await compressionSamples()); + if (rankings.length === 0) { + return null; + } + return { + type: 'compression', + title: 'Write Throughput', + rankings, + explanation: 'Geomean throughput ratio to Parquet | Aggregate throughput (higher is better)', + }; +} + +/** One timing paired with the baseline's timing for the same dataset, commit and operation. */ +interface CompressionTimeSample { + format: string; + op: string; + valueNs: number; + baselineNs: number; + uncompressedBytes: number | null; +} + +/** + * Rank each `(operation, format)` by its geomean throughput ratio to the baseline, with its + * aggregate logical throughput, encode before decode and best first. + */ +function compressionRankings(rows: readonly CompressionTimeSample[]): CompressionRanking[] { const grouped = new Map< string, { @@ -539,7 +572,7 @@ async function collectCompressionSummary(): Promise { totalBytes: 0, totalNs: 0, }; - aggregate.ratios.push(row.parquetNs / row.valueNs); + aggregate.ratios.push(row.baselineNs / row.valueNs); if ( row.uncompressedBytes !== null && row.uncompressedBytes > 0 && @@ -575,15 +608,7 @@ async function collectCompressionSummary(): Promise { } return b.ratio - a.ratio || compareCodeUnits(a.name, b.name); }); - if (rankings.length === 0) { - return null; - } - return { - type: 'compression', - title: 'Write Throughput', - rankings, - explanation: 'Geomean throughput ratio to Parquet | Aggregate throughput (higher is better)', - }; + return rankings; } /** @@ -594,15 +619,7 @@ async function collectCompressionSummary(): Promise { * The JSON field lookup returns `NULL` before migration 009 adds the column, so web deployment * does not depend on migration deployment order. */ -async function compressionSamples(): Promise< - Array<{ - format: string; - op: string; - valueNs: number; - parquetNs: number; - uncompressedBytes: number | null; - }> -> { +async function compressionSamples(): Promise { const text = ` WITH pairs AS ( SELECT t.format AS format, @@ -664,7 +681,7 @@ async function compressionSamples(): Promise< SELECT selected.format AS format, selected.op AS op, selected.value_ns AS "valueNs", - selected.parquet_ns AS "parquetNs", + selected.parquet_ns AS "baselineNs", uncompressed.uncompressed_bytes AS "uncompressedBytes" FROM selected LEFT JOIN latest_uncompressed_sizes uncompressed @@ -673,29 +690,46 @@ async function compressionSamples(): Promise< ORDER BY selected.op, selected.format, selected.dataset, selected.dataset_variant NULLS FIRST `; - return ( - await getPool().query<{ - format: string; - op: string; - valueNs: number; - parquetNs: number; - uncompressedBytes: number | null; - }>(text, compressionSummaryQueryParams()) - ).rows; + return (await getPool().query(text, compressionSummaryQueryParams())).rows; } async function collectCompressionSizeSummary(): Promise { - const rows = await compressionSizeSamples(); + const rankings = compressionSizeRankings(await compressionSizeSamples()); + if (rankings.length === 0) { + return null; + } + return { + type: 'compressionSize', + title: 'Compression Size Summary', + rankings, + explanation: + 'Geometric means of compressed sizes versus Arrow (higher is better) and versus Parquet-zstd (lower is better)', + }; +} + +/** One size paired with the baseline's size for the same dataset and commit. */ +interface CompressionSizeSample { + format: string; + valueBytes: number; + baselineBytes: number; + uncompressedBytes: number | null; +} + +/** + * Rank each format by its size ratios to the baseline and its compression ratio to Arrow. The + * order is by compression ratio once every format has one, and by baseline ratio until then. + */ +function compressionSizeRankings(rows: readonly CompressionSizeSample[]): CompressionSizeRanking[] { const grouped = new Map(); for (const row of rows) { const ratios = grouped.get(row.format) ?? { sizeRatios: [], compressionRatios: [] }; if ( row.valueBytes > 0 && Number.isFinite(row.valueBytes) && - row.parquetBytes > 0 && - Number.isFinite(row.parquetBytes) + row.baselineBytes > 0 && + Number.isFinite(row.baselineBytes) ) { - ratios.sizeRatios.push(row.valueBytes / row.parquetBytes); + ratios.sizeRatios.push(row.valueBytes / row.baselineBytes); } if ( row.uncompressedBytes !== null && @@ -737,16 +771,7 @@ async function collectCompressionSizeSummary(): Promise { compareCodeUnits(a.name, b.name) ); }); - if (rankings.length === 0) { - return null; - } - return { - type: 'compressionSize', - title: 'Compression Size Summary', - rankings, - explanation: - 'Geometric means of compressed sizes versus Arrow (higher is better) and versus Parquet-zstd (lower is better)', - }; + return rankings; } /** @@ -759,14 +784,7 @@ async function collectCompressionSizeSummary(): Promise { * deploy that includes this query. A JSON field lookup would tolerate an older schema, but it * serializes whole rows and timed out in production, so it was dropped once 009 was applied there. */ -async function compressionSizeSamples(): Promise< - Array<{ - format: string; - valueBytes: number; - parquetBytes: number; - uncompressedBytes: number | null; - }> -> { +async function compressionSizeSamples(): Promise { const text = ` WITH pairs AS ( SELECT s.format AS format, @@ -837,7 +855,7 @@ async function compressionSizeSamples(): Promise< ) SELECT selected.format AS format, selected.value_bytes AS "valueBytes", - selected.parquet_bytes AS "parquetBytes", + selected.parquet_bytes AS "baselineBytes", COALESCE(uncompressed.uncompressed_bytes, selected.uncompressed_bytes) AS "uncompressedBytes" FROM selected_with_arrow_ipc selected @@ -846,14 +864,117 @@ async function compressionSizeSamples(): Promise< AND uncompressed.dataset_variant IS NOT DISTINCT FROM selected.dataset_variant ORDER BY selected.format, selected.dataset, selected.dataset_variant NULLS FIRST `; - return ( - await getPool().query<{ - format: string; - valueBytes: number; - parquetBytes: number; - uncompressedBytes: number | null; - }>(text, compressionSizeSummaryQueryParams()) + return (await getPool().query(text, compressionSizeSummaryQueryParams())) + .rows; +} + +/** + * The `String Encoding` rollup: each string encoder's write (`encode`) and scan (`decode`) + * throughput against FSST's, on the columns both measured at the newest `string-bench` commit. + * The suite runs every encoder in one job, so one commit is a complete snapshot. + */ +async function collectStringEncodingSummary(): Promise { + const text = ` + WITH latest AS ( + SELECT t.commit_sha + FROM compression_times t + JOIN ${BRANCH_COMMITS} c USING (commit_sha) + WHERE t.dataset LIKE $1 + ORDER BY c.timestamp DESC, t.commit_sha DESC + LIMIT 1 + ) + SELECT t.dataset_variant AS format, + t.op AS op, + t.value_ns::float8 AS "valueNs", + b.value_ns::float8 AS "baselineNs", + s.uncompressed_bytes::float8 AS "uncompressedBytes" + FROM compression_times t + JOIN latest USING (commit_sha) + JOIN compression_times b + ON b.commit_sha = t.commit_sha + AND b.dataset = t.dataset + AND b.format = t.format + AND b.op = t.op + AND b.dataset_variant = $2 + LEFT JOIN compression_sizes s + ON s.commit_sha = t.commit_sha + AND s.dataset = t.dataset + AND s.dataset_variant = t.dataset_variant + AND s.format = t.format + WHERE t.dataset LIKE $1 + AND t.dataset_variant IS NOT NULL + AND t.op IN ('encode', 'decode') + AND t.value_ns > 0 + AND b.value_ns > 0 + ORDER BY t.op, t.dataset_variant, t.dataset + `; + const rows = ( + await getPool().query(text, [ + STRING_DATASET_PATTERN, + STRING_ENCODING_BASELINE, + ]) + ).rows; + const rankings = compressionRankings(rows); + if (rankings.length === 0) { + return null; + } + return { + type: 'compression', + title: 'Write Throughput', + rankings, + explanation: 'Geomean throughput ratio to FSST | Aggregate throughput (higher is better)', + }; +} + +/** + * The `String Encoding Size` rollup: each string encoder's Vortex file size against FSST's and + * against the column's canonical size, at the newest `string-bench` commit. + */ +async function collectStringEncodingSizeSummary(): Promise { + const text = ` + WITH latest AS ( + SELECT s.commit_sha + FROM compression_sizes s + JOIN ${BRANCH_COMMITS} c USING (commit_sha) + WHERE s.dataset LIKE $1 + ORDER BY c.timestamp DESC, s.commit_sha DESC + LIMIT 1 + ) + SELECT s.dataset_variant AS format, + s.value_bytes::float8 AS "valueBytes", + b.value_bytes::float8 AS "baselineBytes", + s.uncompressed_bytes::float8 AS "uncompressedBytes" + FROM compression_sizes s + JOIN latest USING (commit_sha) + JOIN compression_sizes b + ON b.commit_sha = s.commit_sha + AND b.dataset = s.dataset + AND b.format = s.format + AND b.dataset_variant = $2 + WHERE s.dataset LIKE $1 + AND s.dataset_variant IS NOT NULL + AND s.value_bytes > 0 + AND b.value_bytes > 0 + ORDER BY s.dataset_variant, s.dataset + `; + const rows = ( + await getPool().query(text, [ + STRING_DATASET_PATTERN, + STRING_ENCODING_BASELINE, + ]) ).rows; + const rankings = compressionSizeRankings(rows); + if (rankings.length === 0) { + return null; + } + return { + type: 'compressionSize', + title: 'String Encoding Size Summary', + rankings, + explanation: + 'Geometric means of file sizes versus Arrow (higher is better) and versus FSST (lower is better)', + baseline: 'FSST', + }; } async function collectQuerySummary(