diff --git a/apps/media-server/src/__tests__/lib/media-routes-real-world.integration.test.ts b/apps/media-server/src/__tests__/lib/media-routes-real-world.integration.test.ts index c471a1d1f1..3ef76eb181 100644 --- a/apps/media-server/src/__tests__/lib/media-routes-real-world.integration.test.ts +++ b/apps/media-server/src/__tests__/lib/media-routes-real-world.integration.test.ts @@ -909,10 +909,10 @@ describe("media routes real-world integration tests", () => { let hashSettled = false; const hashing = spyOn( recordingVerification, - "hashRecordingFile", + "hashRecordingUpload", ).mockImplementation( - (_path, signal) => - new Promise((_resolve, reject) => { + (_path, _upload, signal) => + new Promise<{ sha256: string }>((_resolve, reject) => { signal?.addEventListener("abort", () => { hashAborted = true; setTimeout(() => { @@ -948,12 +948,12 @@ describe("media routes real-world integration tests", () => { // Hashing now overlaps the decode check, so whichever check reads the changed bytes first refuses it. test("refuses a local output changed while it is decoded and hashed", async () => { - const originalHash = recordingVerification.hashRecordingFile; + const originalHash = recordingVerification.hashRecordingUpload; const hashing = spyOn( recordingVerification, - "hashRecordingFile", - ).mockImplementation(async (path, signal) => { - const digest = await originalHash(path, signal); + "hashRecordingUpload", + ).mockImplementation(async (path, upload, signal) => { + const digest = await originalHash(path, upload, signal); const bytes = await readFile(path); bytes[bytes.length - 1] ^= 1; await writeFile(path, bytes); diff --git a/apps/media-server/src/__tests__/lib/recording-verification.integration.test.ts b/apps/media-server/src/__tests__/lib/recording-verification.integration.test.ts index 3da0952f53..433d57a174 100644 --- a/apps/media-server/src/__tests__/lib/recording-verification.integration.test.ts +++ b/apps/media-server/src/__tests__/lib/recording-verification.integration.test.ts @@ -26,6 +26,7 @@ import { } from "../../lib/recording-timing"; import { decodePreverifiedRun, + hashRecordingUpload, inspectRecordingSources, isRetryableRecordingVerificationError, type PreverifiedVideoRun, @@ -34,6 +35,7 @@ import { verifyRemoteRecordingBytes, verifyRemuxedRecording, } from "../../lib/recording-verification"; +import { getActiveSubprocessCount } from "../../lib/subprocess"; const FIXTURES = join(import.meta.dir, "..", "fixtures"); let directory: string; @@ -2974,6 +2976,107 @@ describe("chunked packet-proven verification", () => { } }); + test("a packet mismatch wins over the fragment decode it runs beside", async () => { + const recording = await segmentedRecording("chunk-mismatch", 15); + const bytes = await readFile(recording.output); + const mdat = bytes.indexOf(Buffer.from("mdat")); + if (mdat < 0) throw new Error("Fixture has no media data"); + bytes[mdat + 64] ^= 0xff; + await writeFile(recording.output, bytes); + const active = getActiveSubprocessCount(); + process.env.MEDIA_SERVER_VERIFY_DECODE_CHUNKS = "3"; + try { + await expect( + verifyRemuxedRecording( + recording.combined, + silent, + recording.output, + { requireAudio: false }, + { init: recording.init, segments: recording.segments }, + ), + ).rejects.toThrow( + /video does not preserve the source \(first difference at packet \d+: data differs\)/, + ); + expect(getActiveSubprocessCount()).toBe(active); + } finally { + delete process.env.MEDIA_SERVER_VERIFY_DECODE_CHUNKS; + } + }); + + test("decodes the fragments while the output is still being written", async () => { + const recording = await segmentedRecording("chunk-early", 15); + process.env.MEDIA_SERVER_VERIFY_DECODE_CHUNKS = "3"; + try { + const expected = await verifyRemuxedRecording( + recording.combined, + silent, + recording.output, + { requireAudio: false }, + { init: recording.init, segments: recording.segments }, + ); + const pending = join(directory, "chunk-early-pending.mp4"); + let written!: () => void; + const outputReady = new Promise((resolve) => { + written = resolve; + }); + const early = verifyRemuxedRecording( + recording.combined, + silent, + pending, + { requireAudio: false }, + { init: recording.init, segments: recording.segments }, + [], + outputReady, + ); + const warn = spyOn(console, "warn"); + let result: Awaited; + try { + await new Promise((resolve) => setTimeout(resolve, 200)); + await muxMediaTracksToMp4(recording.combined, silent, pending); + written(); + result = await early; + expect( + warn.mock.calls.some((call) => + String(call[0]).includes("recording_chunked_decode_fallback"), + ), + ).toBe(false); + } finally { + warn.mockRestore(); + } + expect(result.sourcePreserved).toBe(true); + expect(result.video.frameCount).toBe(expected.video.frameCount); + expect(result.video.duration).toBe(expected.video.duration); + expect(Boolean(result.audio)).toBe(Boolean(expected.audio)); + } finally { + delete process.env.MEDIA_SERVER_VERIFY_DECODE_CHUNKS; + } + }); + + test("a failed output write fails verification as itself and stops the decoders", async () => { + const recording = await segmentedRecording("chunk-write-failed", 15); + const active = getActiveSubprocessCount(); + process.env.MEDIA_SERVER_VERIFY_DECODE_CHUNKS = "3"; + try { + const failure = new Error("Remux failed"); + await expect( + verifyRemuxedRecording( + recording.combined, + silent, + join(directory, "chunk-write-failed-missing.mp4"), + { requireAudio: false }, + { init: recording.init, segments: recording.segments }, + [], + new Promise((_resolve, reject) => + setTimeout(() => reject(failure), 100), + ), + ), + ).rejects.toBe(failure); + expect(getActiveSubprocessCount()).toBe(active); + } finally { + delete process.env.MEDIA_SERVER_VERIFY_DECODE_CHUNKS; + } + }); + async function preverify( recording: { init: string; segments: string[] }, from: number, @@ -3153,6 +3256,45 @@ describe("chunked packet-proven verification", () => { }); }); +describe("upload content tags", () => { + const md5 = (bytes: Uint8Array) => createHash("md5").update(bytes).digest(); + const multipartTag = (bytes: Uint8Array, partSize: number) => { + const parts: Buffer[] = []; + for (let start = 0; start < bytes.length; start += partSize) + parts.push(md5(bytes.subarray(start, start + partSize))); + return `"${createHash("md5").update(Buffer.concat(parts)).digest("hex")}-${parts.length}"`; + }; + + test.each([ + ["uneven", 3 * 1024 * 1024 + 12_345], + ["part aligned", 2 * 1024 * 1024], + ["one partial part", 777], + ])("matches the S3 ETags of a %s file", async (_, size) => { + const bytes = new Uint8Array(size).map( + (_, index) => (index * 13 + 5) % 253, + ); + const work = await mkdtemp(join(tmpdir(), "upload-tag-")); + try { + const path = join(work, "result.mp4"); + await writeFile(path, bytes); + const sha256 = createHash("sha256").update(bytes).digest("hex"); + expect(await hashRecordingUpload(path, {})).toEqual({ + sha256, + contentIdentity: `"${md5(bytes).toString("hex")}"`, + }); + expect( + await hashRecordingUpload(path, { partSize: 1024 * 1024 }), + ).toEqual({ + sha256, + contentIdentity: multipartTag(bytes, 1024 * 1024), + }); + expect(await hashRecordingUpload(path, undefined)).toEqual({ sha256 }); + } finally { + await rm(work, { recursive: true, force: true }); + } + }); +}); + describe("ranged remote byte verification", () => { const payload = new Uint8Array(3 * 1024 * 1024 + 12_345).map( (_, index) => (index * 31 + 7) % 251, @@ -3245,6 +3387,99 @@ describe("ranged remote byte verification", () => { } }); + test("trusts an ETag that is the uploaded file's own content tag", async () => { + const contentIdentity = `"${createHash("md5").update(payload).digest("hex")}"`; + const requests: string[] = []; + const server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch(request) { + requests.push(request.headers.get("range") ?? "full"); + if (request.headers.get("if-match") !== contentIdentity) + return new Response("Object changed", { status: 412 }); + return new Response(payload.slice(0, 1), { + status: 206, + headers: { + ETag: contentIdentity, + "Content-Range": `bytes 0-0/${payload.length}`, + }, + }); + }, + }); + try { + const result = await verifyRemoteRecordingBytes( + `http://127.0.0.1:${server.port}/recording.mp4`, + { + expectedObjectIdentity: contentIdentity, + expectedSha256: sha256, + expectedFileSize: payload.length, + uploadedContentIdentity: contentIdentity, + }, + ); + expect(result).toEqual({ + objectIdentity: contentIdentity, + fileSize: payload.length, + remoteSha256: sha256, + }); + expect(requests).toEqual(["bytes=0-0"]); + } finally { + await server.stop(true); + } + }); + + test("re-reads every byte when the ETag is not the uploaded content tag", async () => { + const { server, ranges } = rangedServer(); + try { + const result = await verifyRemoteRecordingBytes( + `http://127.0.0.1:${server.port}/recording.mp4`, + { + expectedObjectIdentity: identity, + expectedSha256: sha256, + expectedFileSize: payload.length, + uploadedContentIdentity: `"${createHash("md5").update(payload).digest("hex")}"`, + }, + ); + expect(result.remoteSha256).toBe(sha256); + expect(ranges).toHaveLength(4); + } finally { + await server.stop(true); + } + }); + + test("rejects a content-tagged object whose size differs", async () => { + const contentIdentity = `"${createHash("md5").update(payload).digest("hex")}"`; + const server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch() { + return new Response(payload.slice(0, 1), { + status: 206, + headers: { + ETag: contentIdentity, + "Content-Range": `bytes 0-0/${payload.length + 1}`, + }, + }); + }, + }); + try { + await expect( + verifyRemoteRecordingBytes( + `http://127.0.0.1:${server.port}/recording.mp4`, + { + expectedObjectIdentity: contentIdentity, + expectedSha256: sha256, + expectedFileSize: payload.length, + uploadedContentIdentity: contentIdentity, + }, + ), + ).rejects.toThrow( + "Uploaded recording size does not match the muxed file", + ); + } finally { + await server.stop(true); + } + }); + test("rejects a range served from a different object", async () => { const { server } = rangedServer((start, end) => start === 2 * 1024 * 1024 diff --git a/apps/media-server/src/lib/recording-packet-proof.ts b/apps/media-server/src/lib/recording-packet-proof.ts index b933b7f11e..192fb5804e 100644 --- a/apps/media-server/src/lib/recording-packet-proof.ts +++ b/apps/media-server/src/lib/recording-packet-proof.ts @@ -22,7 +22,7 @@ interface PacketStream { const STREAM_FIELDS = "index,start_pts,codec_type,codec_name,profile,level,codec_tag_string,width,height,sample_aspect_ratio,pix_fmt,color_range,color_space,color_transfer,color_primaries,chroma_location,field_order,refs,sample_fmt,sample_rate,channels,channel_layout,time_base,extradata_hash"; -async function runInspector( +export async function runInspector( command: string, args: string[], signal: AbortSignal, @@ -245,7 +245,11 @@ class PacketLog { this.pts[this.length] = seconds(pts); this.dts[this.length] = seconds(dts); this.sizes[this.length] = Number(size); - this.hashes[this.length] = Number.parseInt(dataHash.slice(7, 15), 16); + const digest = dataHash.indexOf(":") + 1; + this.hashes[this.length] = Number.parseInt( + dataHash.slice(digest, digest + 8), + 16, + ); this.keyframes[this.length] = (flags?.includes("K") ? 1 : 0) | (flags?.includes("D") ? 2 : 0); this.length++; @@ -516,6 +520,13 @@ async function readTrack( ); hash.update("\n"); }; + // Source and output packets are hashed by the same ffprobe and only compared with each other, to catch a + // remux that changed a packet; FFmpeg's SHA-256 made this read the slowest part of verifying a long video. + // Audio keeps SHA-256 because its tail is checked against a hash computed outside ffprobe. + const packetHash = + kind === "video" + ? { name: "murmur3", pattern: /^murmur3:[a-f0-9]{32}$/ } + : { name: "sha256", pattern: /^SHA256:[a-f0-9]{64}$/ }; await probe( path, [ @@ -523,7 +534,7 @@ async function readTrack( selection, "-show_packets", "-show_data_hash", - "sha256", + packetHash.name, "-show_entries", "packet=stream_index,pts,dts,duration,size,flags,data_hash:packet_side_data", "-of", @@ -547,7 +558,7 @@ async function readTrack( if ( Number(fields.stream_index) !== index || !/^[1-9]\d*$/.test(fields.size ?? "") || - !/^SHA256:[a-f0-9]{64}$/.test(fields.data_hash ?? "") + !packetHash.pattern.test(fields.data_hash ?? "") ) throw new Error("Recording packet content is incomplete"); if (pendingPacket) commitPacket(pendingPacket, fields.dts); diff --git a/apps/media-server/src/lib/recording-verification.ts b/apps/media-server/src/lib/recording-verification.ts index 540d5a08b7..887083a27d 100644 --- a/apps/media-server/src/lib/recording-verification.ts +++ b/apps/media-server/src/lib/recording-verification.ts @@ -6,6 +6,8 @@ import { availableParallelism } from "node:os"; import { dirname, isAbsolute, join } from "node:path"; import { Readable } from "node:stream"; import { setTimeout as sleep } from "node:timers/promises"; +import { fileURLToPath } from "node:url"; +import { z } from "zod"; import { decodeThreadsFromEnv, PROCESS_TIMEOUT_MS } from "./media-common"; import { fetchMedia, @@ -15,6 +17,7 @@ import { import { proveRecordingPackets, RecordingPacketMismatchError, + runInspector, } from "./recording-packet-proof"; import { RecordingTimingError, @@ -22,6 +25,7 @@ import { readRecordingVideoTiming, } from "./recording-timing"; import { registerSubprocess, unregisterSubprocess } from "./subprocess"; +import { digestUploadFile } from "./upload-digest"; const MAX_LOCAL_VERIFICATION_MS = 2 * 60 * 60_000; const DECODER_STALL_MS = 5 * 60_000; @@ -119,6 +123,8 @@ export interface RemoteRecordingBytesOptions { expectedFileSize: number; abortSignal?: AbortSignal; timeoutMs?: number; + // The ETag storage would return for the local file this server uploaded (see hashRecordingUpload). + uploadedContentIdentity?: string; } export interface RemoteRecordingBytesResult { @@ -1017,9 +1023,18 @@ async function decodePacketProvenChunks( source: RecordingVideoChunkSource, chunkCount: number, options: RecordingVerificationOptions, - packetProof: Awaited>, + hasAudio: boolean, + packetProofRead: Promise>>, preverified: PreverifiedVideoRun[] = [], + outputReady?: Promise, ): Promise { + let knownVideoPackets: number | undefined; + packetProofRead.then( + (proof) => { + knownVideoPackets = proof.videoPackets; + }, + () => {}, + ); const cached = [...preverified]; let covered = cached.reduce((total, run) => total + run.segments, 0); if (covered > source.segments.length) @@ -1069,6 +1084,15 @@ async function decodePacketProvenChunks( let failure: Error | undefined; let decodedFrames = 0; let lastAdvance = performance.now(); + // Time spent waiting for the output to be written is not a stalled decoder. + let outputPending = outputReady !== undefined; + outputReady?.then( + () => { + outputPending = false; + lastAdvance = performance.now(); + }, + () => {}, + ); let lastReport = 0; const stopAll = () => { for (const decoder of decoders) { @@ -1094,7 +1118,7 @@ async function decodePacketProvenChunks( stopAll(); }, options.timeoutMs ?? MAX_LOCAL_VERIFICATION_MS); const stall = setInterval(() => { - if (performance.now() - lastAdvance > DECODER_STALL_MS) { + if (!outputPending && performance.now() - lastAdvance > DECODER_STALL_MS) { failure ??= new RecordingVerificationError( "Recording decoder stopped making progress", true, @@ -1109,7 +1133,7 @@ async function decodePacketProvenChunks( lastReport = performance.now(); options.onProgress?.({ frames: decodedFrames, - totalFrames: packetProof.videoPackets, + totalFrames: knownVideoPackets, }); } }; @@ -1125,7 +1149,7 @@ async function decodePacketProvenChunks( ); const jobs = [ ...chunkPaths.map((path) => videoChunkDecodeArgs(path, threads)), - ...(packetProof.hasAudio + ...(hasAudio ? [ [ ...base, @@ -1171,6 +1195,8 @@ async function decodePacketProvenChunks( } }; for (const [index, args] of jobs.entries()) { + // Fragment chunks are source files; only the audio decode reads the output. + if (index === chunkPaths.length) await outputReady; options.abortSignal?.throwIfAborted(); const decoder = registerSubprocess(await spawnRecordingDecoder(args, 0)); decoders.push(decoder); @@ -1195,6 +1221,9 @@ async function decodePacketProvenChunks( const exits = await Promise.all(decoders.map((decoder) => decoder.exited)); await Promise.all(readers); if (failure) throw failure; + const packetProof = await packetProofRead; + if (packetProof.hasAudio !== hasAudio) + throw new ChunkedDecodeUnavailable("audio track presence differs"); const stderr = await Promise.all(errors); for (const [index, decoder] of decoders.entries()) { if (exits[index] !== 0 || stderr[index].error || decoder.signalCode) @@ -1282,23 +1311,62 @@ export async function verifyRemuxedRecording( >, videoChunks?: RecordingVideoChunkSource, preverified: PreverifiedVideoRun[] = [], + // Settles once `outputPath` is written; the fragment decode can start before it. + outputReady?: Promise, ): Promise { + const outputWritten = outputReady + ? outputReady.then(() => undefined) + : Promise.resolve(); + outputWritten.catch(() => {}); const signal = AbortSignal.any([ ...(options.abortSignal ? [options.abortSignal] : []), AbortSignal.timeout(options.timeoutMs ?? MAX_LOCAL_VERIFICATION_MS), ]); options = { ...options, abortSignal: signal }; + const proofRead = outputWritten + .then(() => proveRecordingPackets(videoPath, audioPath, outputPath, signal)) + .then((proof) => { + if (proof.outputTiming.terminalPacketCount !== 1) + throw new Error( + "Tied terminal samples require decoded source evidence", + ); + return proof; + }); + proofRead.catch(() => {}); + const decodeOptions = { + ...options, + timeoutMs: options.timeoutMs ?? MAX_LOCAL_VERIFICATION_MS, + }; + const chunkCount = verifyDecodeChunks(); + // The fragment decode only needs the packet proof for its final checks, so both run at once; nothing the + // decode finds counts unless the proof passes, and a failed proof stops the decoders. + const chunkAbort = new AbortController(); + const chunked = + videoChunks && + (preverified.length > 0 || + (chunkCount > 1 && videoChunks.segments.length >= chunkCount * 2)) + ? decodePacketProvenChunks( + outputPath, + videoChunks, + chunkCount, + { + ...decodeOptions, + abortSignal: AbortSignal.any([signal, chunkAbort.signal]), + }, + Boolean(audioPath), + proofRead, + preverified, + outputWritten, + ) + : undefined; + chunked?.catch(() => {}); let proof: Awaited>; try { - proof = await proveRecordingPackets( - videoPath, - audioPath, - outputPath, - signal, - ); - if (proof.outputTiming.terminalPacketCount !== 1) - throw new Error("Tied terminal samples require decoded source evidence"); + proof = await proofRead; } catch (error) { + chunkAbort.abort(); + await chunked?.catch(() => {}); + await outputWritten; signal.throwIfAborted(); if (error instanceof RecordingPacketMismatchError) throw error; const sourceEvidence = await inspectRecordingSources( @@ -1314,26 +1382,10 @@ export async function verifyRemuxedRecording( throw new Error("Recording source preservation was not verified"); return { ...result, sourcePreserved: true }; } - const decodeOptions = { - ...options, - timeoutMs: options.timeoutMs ?? MAX_LOCAL_VERIFICATION_MS, - }; - const chunkCount = verifyDecodeChunks(); let result: RecordingVerificationResult | undefined; - if ( - videoChunks && - (preverified.length > 0 || - (chunkCount > 1 && videoChunks.segments.length >= chunkCount * 2)) - ) { + if (chunked) { try { - result = await decodePacketProvenChunks( - outputPath, - videoChunks, - chunkCount, - decodeOptions, - proof, - preverified, - ); + result = await chunked; } catch (error) { signal.throwIfAborted(); if ( @@ -2229,6 +2281,14 @@ export async function verifyRemoteRecordingBytes( if (before.fileSize !== options.expectedFileSize) { throw new Error("Uploaded recording size does not match the muxed file"); } + if ( + options.uploadedContentIdentity !== undefined && + options.uploadedContentIdentity === before.objectIdentity + ) { + if (signal.aborted) + throw new Error("Recording verification was cancelled"); + return { ...before, remoteSha256: options.expectedSha256 }; + } const remoteSha256 = await hashRemoteRecording( input, before.objectIdentity, @@ -2271,9 +2331,64 @@ export async function hashRecordingFile( if (!isAbsolute(input) || !(await lstat(input)).isFile()) { throw new Error("Recording hashing requires a local regular file"); } - const hash = createHash("sha256"); - for await (const chunk of createReadStream(input, { signal: abortSignal })) { - hash.update(chunk); + return (await digestUploadFile(input, "none", abortSignal)).sha256; +} + +const uploadDigestSchema = z.object({ + sha256: z.string().regex(/^[a-f0-9]{64}$/), + contentIdentity: z + .string() + .regex(/^"[a-f0-9]{32}(-[1-9]\d*)?"$/) + .optional(), +}); + +// Hashes the output for its SHA-256 and the ETag storage should return for it. This runs beside the +// verification decode, so it runs in its own process instead of taking turns with the decoder output on +// this thread. +export async function hashRecordingUpload( + input: string, + upload: { partSize?: number } | undefined, + abortSignal?: AbortSignal, +): Promise<{ sha256: string; contentIdentity?: string }> { + if (!isAbsolute(input) || !(await lstat(input)).isFile()) { + throw new Error("Recording hashing requires a local regular file"); } - return hash.digest("hex"); + const partSize = + upload?.partSize !== undefined && + Number.isSafeInteger(upload.partSize) && + upload.partSize > 0 + ? upload.partSize + : undefined; + const mode = !upload + ? "none" + : partSize === undefined + ? "whole" + : String(partSize); + let output = ""; + const result = await runInspector( + process.execPath, + [ + fileURLToPath(new URL("./upload-digest.ts", import.meta.url)), + input, + mode, + ], + abortSignal ?? new AbortController().signal, + (line) => { + output += line; + if (output.length > 4096) + throw new Error("Recording hash output exceeds its bound"); + }, + ); + const parsed = uploadDigestSchema.safeParse( + (() => { + try { + return JSON.parse(output); + } catch { + return undefined; + } + })(), + ); + if (result.code !== 0 || !parsed.success) + throw new Error("Recording hashing failed"); + return parsed.data; } diff --git a/apps/media-server/src/lib/upload-digest.ts b/apps/media-server/src/lib/upload-digest.ts new file mode 100644 index 0000000000..aa3f329c9e --- /dev/null +++ b/apps/media-server/src/lib/upload-digest.ts @@ -0,0 +1,67 @@ +import { createHash } from "node:crypto"; +import { createReadStream } from "node:fs"; + +export type UploadDigestMode = "none" | "whole" | { partSize: number }; + +// S3 answers a PUT with the MD5 of the bytes it stored, and a multipart upload with the MD5 of its parts' MD5s +// plus the part count. Computing the same tag from the local file lets an upload whose ETag matches it skip +// reading the whole object back; any other ETag (SSE-KMS, Drive, other providers) keeps the full re-read. +export async function digestUploadFile( + path: string, + mode: UploadDigestMode, + signal?: AbortSignal, +): Promise<{ sha256: string; contentIdentity?: string }> { + const sha256 = createHash("sha256"); + const whole = mode === "whole" ? createHash("md5") : null; + const partSize = typeof mode === "object" ? mode.partSize : undefined; + const partDigests: Buffer[] = []; + let part = partSize !== undefined ? createHash("md5") : null; + let partBytes = 0; + for await (const chunk of createReadStream(path, { signal })) { + const bytes = chunk as Buffer; + sha256.update(bytes); + whole?.update(bytes); + if (!part || partSize === undefined) continue; + let offset = 0; + while (offset < bytes.length) { + const take = Math.min(partSize - partBytes, bytes.length - offset); + part.update(bytes.subarray(offset, offset + take)); + partBytes += take; + offset += take; + if (partBytes === partSize) { + partDigests.push(part.digest()); + part = createHash("md5"); + partBytes = 0; + } + } + } + if (part && partBytes > 0) partDigests.push(part.digest()); + let contentIdentity: string | undefined; + if (whole) contentIdentity = `"${whole.digest("hex")}"`; + else if (partDigests.length > 0) + contentIdentity = `"${createHash("md5").update(Buffer.concat(partDigests)).digest("hex")}-${partDigests.length}"`; + return { sha256: sha256.digest("hex"), contentIdentity }; +} + +export function parseUploadDigestMode(value: string | undefined) { + if (value === "none" || value === "whole") return value; + const partSize = Number(value); + if (!Number.isSafeInteger(partSize) || partSize <= 0) + throw new Error("Invalid upload digest mode"); + return { partSize }; +} + +if (import.meta.main) { + try { + const path = process.argv[2]; + if (!path) throw new Error("Missing upload digest path"); + console.log( + JSON.stringify( + await digestUploadFile(path, parseUploadDigestMode(process.argv[3])), + ), + ); + } catch { + console.log(JSON.stringify({ error: "unavailable" })); + process.exitCode = 1; + } +} diff --git a/apps/media-server/src/routes/video.ts b/apps/media-server/src/routes/video.ts index a7513a7b2e..672077278e 100644 --- a/apps/media-server/src/routes/video.ts +++ b/apps/media-server/src/routes/video.ts @@ -97,6 +97,7 @@ import { RecordingTimingError } from "../lib/recording-timing"; import { decodePreverifiedRun, hashRecordingFile, + hashRecordingUpload, isRetryableRecordingVerificationError, type PreverifiedVideoRun, verifyDecodeChunks, @@ -3265,7 +3266,13 @@ async function muxSegmentsAsync( let outputUploadStarted = false; let pendingOutputUpload: PendingStorageUpload | undefined; let pendingOutputHash: - | { promise: Promise; abort: AbortController } + | { + promise: Promise<{ sha256: string; contentIdentity?: string }>; + abort: AbortController; + } + | undefined; + let pendingVerification: + | { promise: Promise; abort: AbortController } | undefined; let processingTimeout: ReturnType | undefined; let errorCode: RecordingErrorCode = "processing-unavailable"; @@ -3274,6 +3281,7 @@ async function muxSegmentsAsync( : undefined; const startedAt = Date.now(); let previewAssets: Promise | undefined; + let previewUploads: Promise | undefined; const previewAbort = new AbortController(); try { @@ -3434,7 +3442,7 @@ async function muxSegmentsAsync( errorCode = "output-invalid"; updateJob(jobId, { progress: 65, message: "Combining video and audio..." }); sendCurrentJobWebhook(jobId); - await withJobHeartbeat(jobId, () => + const muxed = withJobHeartbeat(jobId, () => withMuxMemoryGuard(abortController, () => muxMediaTracksToMp4( combinedVideoPath, @@ -3448,6 +3456,42 @@ async function muxSegmentsAsync( errorCode = classifySourceError(error); throw error; }); + // The fragment decode reads only the downloaded fragments, so it runs while the remux writes the output; + // the packet proof and audio decode wait for the output, and a failed remux still fails as one. + const verificationAbort = new AbortController(); + let outputWritten = false; + const verification = withJobHeartbeat(jobId, () => + withMuxMemoryGuard(abortController, () => + verifyRemuxedRecording( + combinedVideoPath, + combinedAudioPath, + resultPath, + { + requireAudio: requiredAudio, + abortSignal: AbortSignal.any([ + abortController.signal, + verificationAbort.signal, + ]), + onProgress: ({ frames, totalFrames }) => { + if (!outputWritten) return; + updateJob(jobId, { + progress: totalFrames + ? 70 + Math.min(4, Math.floor((5 * frames) / totalFrames)) + : 70, + message: `Checking recording: ${frames.toLocaleString("en-US")} frames verified...`, + }); + }, + }, + videoChunks, + preverified, + muxed, + ), + ), + ); + verification.catch(() => {}); + pendingVerification = { promise: verification, abort: verificationAbort }; + await muxed; + outputWritten = true; const beforeDecode = await lstat(resultPath, { bigint: true }); if (!beforeDecode.isFile()) throw new Error("Recording verification requires a local regular file"); @@ -3460,8 +3504,11 @@ async function muxSegmentsAsync( abortController.signal, ); const hashAbort = new AbortController(); - const outputHash = hashRecordingFile( + const outputHash = hashRecordingUpload( resultPath, + outputUpload.type === "multipart" + ? { partSize: Math.floor(outputUpload.partSize) } + : {}, AbortSignal.any([abortController.signal, hashAbort.signal]), ); outputHash.catch(() => {}); @@ -3472,29 +3519,9 @@ async function muxSegmentsAsync( }); sendCurrentJobWebhook(jobId); const verificationStartedAt = Date.now(); - const localVerified = await withJobHeartbeat(jobId, () => - withMuxMemoryGuard(abortController, () => - verifyRemuxedRecording( - combinedVideoPath, - combinedAudioPath, - resultPath, - { - requireAudio: requiredAudio, - abortSignal: abortController.signal, - onProgress: ({ frames, totalFrames }) => { - updateJob(jobId, { - progress: totalFrames - ? 70 + Math.min(4, Math.floor((5 * frames) / totalFrames)) - : 70, - message: `Checking recording: ${frames.toLocaleString("en-US")} frames verified...`, - }); - }, - }, - videoChunks, - preverified, - ), - ), - ); + const outputProbe = probeVideoFile(resultPath); + outputProbe.catch(() => {}); + const localVerified = await verification; if (videoChunks) await rm(videoDir, { recursive: true, force: true }); logVideoEvent("video_mux_verification_complete", { jobId, @@ -3516,7 +3543,8 @@ async function muxSegmentsAsync( message: "Verifying the recording file...", }); sendCurrentJobWebhook(jobId); - const outputSha256 = await withJobHeartbeat(jobId, () => outputHash); + const { sha256: outputSha256, contentIdentity: outputContentIdentity } = + await withJobHeartbeat(jobId, () => outputHash); const afterHash = await lstat(resultPath, { bigint: true }); if ( !afterHash.isFile() || @@ -3550,10 +3578,24 @@ async function muxSegmentsAsync( AbortSignal.any([abortController.signal, previewAbort.signal]), videoId, ); - - let metadata = await withJobHeartbeat(jobId, () => - probeVideoFile(resultPath), + // Previews go to this attempt's own keys, which nothing reads before the recording is published, so + // they upload while the recording uploads; the job still only completes once both are done. + const previewSignal = AbortSignal.any([ + abortController.signal, + previewAbort.signal, + ]); + previewUploads = previewAssets.then((assets) => + uploadPreviewAssets( + assets, + thumbnailPresignedUrl, + previewGifPresignedUrl, + previewSignal, + videoId, + ), ); + previewUploads.catch(() => {}); + + let metadata = await withJobHeartbeat(jobId, () => outputProbe); if ( !Number.isFinite(metadata.duration) || metadata.duration <= 0 || @@ -3607,6 +3649,7 @@ async function muxSegmentsAsync( expectedSha256: outputSha256, expectedFileSize: metadata.fileSize, expectedObjectIdentity: uploadObjectIdentity, + uploadedContentIdentity: outputContentIdentity, abortSignal: abortController.signal, }), ); @@ -3667,38 +3710,8 @@ async function muxSegmentsAsync( const uploadedAt = Date.now(); await withJobHeartbeat(jobId, async () => { - const assets = await previewAssets; + await previewUploads; abortController.signal.throwIfAborted(); - await Promise.all([ - thumbnailPresignedUrl && assets?.thumbnail - ? uploadToS3( - assets.thumbnail, - thumbnailPresignedUrl, - "image/jpeg", - abortController.signal, - ).catch((thumbErr) => { - abortController.signal.throwIfAborted(); - console.warn( - `[mux-segments] Thumbnail generation failed for ${videoId}:`, - thumbErr, - ); - }) - : undefined, - previewGifPresignedUrl && assets?.previewGif - ? uploadFileToS3( - assets.previewGif.path, - previewGifPresignedUrl, - "image/gif", - abortController.signal, - ).catch((previewErr) => { - abortController.signal.throwIfAborted(); - console.warn( - "[mux-segments] Preview GIF generation failed:", - previewErr, - ); - }) - : undefined, - ]); }); abortController.signal.throwIfAborted(); @@ -3749,9 +3762,13 @@ async function muxSegmentsAsync( error: logReason(error, MUX_FAILURE_LOG_LENGTH), resources: getSystemResources(), }); - // Stop a hash still reading the output before the work directory is removed. + // Stop a hash or decode still reading the work directory before it is removed. pendingOutputHash?.abort.abort(); - await pendingOutputHash?.promise.catch(() => {}); + pendingVerification?.abort.abort(); + await Promise.all([ + pendingOutputHash?.promise.catch(() => {}), + pendingVerification?.promise.catch(() => {}), + ]); if (!outputUploadStarted) { // Parts still in flight could land after the abort, so let them settle first. await pendingOutputUpload?.cancel(); @@ -3778,8 +3795,9 @@ async function muxSegmentsAsync( sendCurrentJobWebhook(jobId); } finally { if (processingTimeout) clearTimeout(processingTimeout); - // Previews still rendering here belong to a job that failed before using them. + // Previews still rendering or uploading here belong to a job that failed before using them. previewAbort.abort(); + await previewUploads?.catch(() => {}); await (await previewAssets)?.previewGif?.cleanup(); await rm(workDir, { recursive: true, force: true }).catch(() => {}); releaseWorkDir(); @@ -3791,6 +3809,47 @@ interface PreviewAssets { previewGif: TempFileHandle | null; } +// A failed preview upload only costs the asset; an abort still fails the job. +async function uploadPreviewAssets( + assets: PreviewAssets, + thumbnailPresignedUrl: string | undefined, + previewGifPresignedUrl: string | undefined, + signal: AbortSignal, + videoId: string, +): Promise { + signal.throwIfAborted(); + await Promise.all([ + thumbnailPresignedUrl && assets.thumbnail + ? uploadToS3( + assets.thumbnail, + thumbnailPresignedUrl, + "image/jpeg", + signal, + ).catch((thumbErr) => { + signal.throwIfAborted(); + console.warn( + `[mux-segments] Thumbnail generation failed for ${videoId}:`, + thumbErr, + ); + }) + : undefined, + previewGifPresignedUrl && assets.previewGif + ? uploadFileToS3( + assets.previewGif.path, + previewGifPresignedUrl, + "image/gif", + signal, + ).catch((previewErr) => { + signal.throwIfAborted(); + console.warn( + "[mux-segments] Preview GIF generation failed:", + previewErr, + ); + }) + : undefined, + ]); +} + // Never rejects: a failed preview only costs the asset, and the caller checks for abort. async function renderPreviewAssets( resultPath: string, diff --git a/apps/web/__tests__/unit/desktop-recording-source.test.ts b/apps/web/__tests__/unit/desktop-recording-source.test.ts index 3e61ecb97d..5e3e6732e2 100644 --- a/apps/web/__tests__/unit/desktop-recording-source.test.ts +++ b/apps/web/__tests__/unit/desktop-recording-source.test.ts @@ -2717,6 +2717,24 @@ describe("listed source commit on Cap's bucket", () => { ).toBeGreaterThan(0); }); + it("lists the snapshot once before verifying and once to verify it", async () => { + const storage = listedStorage(); + seedRecording(storage, 20, 10); + const source = await commitDesktopRecordingSource(recording(), generation); + const listedPrefixes = storage.bucket.listObjects.mock.calls.map( + ([input]) => input.prefix, + ); + expect( + listedPrefixes.filter((prefix) => prefix?.endsWith("/segments/video/")), + ).toHaveLength(2); + expect( + listedPrefixes.filter((prefix) => prefix?.endsWith("/segments/audio/")), + ).toHaveLength(2); + const shape = inventoryShape(storage, source.inventoryKey); + expect(shape).toHaveLength(32); + expect(shape.every((entry) => entry.identityMatchesCopy)).toBe(true); + }); + it("proves checkpoint writes by their ETag instead of reading them back", async () => { const storage = listedStorage(); seedRecording(storage, 20, 10); diff --git a/apps/web/__tests__/unit/early-segment-playback.test.ts b/apps/web/__tests__/unit/early-segment-playback.test.ts index 81b13b5ca3..7942898168 100644 --- a/apps/web/__tests__/unit/early-segment-playback.test.ts +++ b/apps/web/__tests__/unit/early-segment-playback.test.ts @@ -95,25 +95,15 @@ describe("Early segment tracks", () => { }); }); -describe("Early instant playback flag", () => { +describe("Early instant playback", () => { afterEach(() => vi.unstubAllEnvs()); - it("is off unless configured", () => { + it("is on unless explicitly disabled", () => { vi.stubEnv("CAP_EARLY_INSTANT_PLAYBACK", ""); - expect(isEarlyInstantPlaybackEnabled("owner")).toBe(false); - vi.stubEnv("CAP_EARLY_INSTANT_PLAYBACK", "true"); - expect(isEarlyInstantPlaybackEnabled("owner")).toBe(false); - }); - - it("can be enabled for everyone", () => { - vi.stubEnv("CAP_EARLY_INSTANT_PLAYBACK", " enabled "); - expect(isEarlyInstantPlaybackEnabled("owner")).toBe(true); - }); - - it("can be rolled out to listed owners", () => { - vi.stubEnv("CAP_EARLY_INSTANT_PLAYBACK", "first, owner"); - expect(isEarlyInstantPlaybackEnabled("owner")).toBe(true); - expect(isEarlyInstantPlaybackEnabled("other")).toBe(false); - expect(isEarlyInstantPlaybackEnabled("own")).toBe(false); + expect(isEarlyInstantPlaybackEnabled()).toBe(true); + vi.stubEnv("CAP_EARLY_INSTANT_PLAYBACK", "owner-id"); + expect(isEarlyInstantPlaybackEnabled()).toBe(true); + vi.stubEnv("CAP_EARLY_INSTANT_PLAYBACK", " disabled "); + expect(isEarlyInstantPlaybackEnabled()).toBe(false); }); }); diff --git a/apps/web/app/s/[videoId]/page.tsx b/apps/web/app/s/[videoId]/page.tsx index 5475a68401..a0f57a16f9 100644 --- a/apps/web/app/s/[videoId]/page.tsx +++ b/apps/web/app/s/[videoId]/page.tsx @@ -527,7 +527,7 @@ async function AuthorizedContent({ const earlySegmentPlayback = recordingStopped && video.source?.type === "desktopSegments" && - isEarlyInstantPlaybackEnabled(video.owner.id); + isEarlyInstantPlaybackEnabled(); // Everything below is an independent round trip (DB or storage); each is // started here and awaited together further down, so the page pays for the diff --git a/apps/web/lib/desktop-recording-source.ts b/apps/web/lib/desktop-recording-source.ts index ebf74763a0..18784118e4 100644 --- a/apps/web/lib/desktop-recording-source.ts +++ b/apps/web/lib/desktop-recording-source.ts @@ -1594,6 +1594,7 @@ async function advanceSourceSnapshot( if (!context.listed) return advanceSourceSnapshotOnce(context, checkpoint, verification); const deadline = Date.now() + LISTED_ADVANCE_BUDGET_MS; + const step: ListedStep = {}; let current = checkpoint; for (;;) { const result = await advanceSourceSnapshotOnce( @@ -1601,6 +1602,7 @@ async function advanceSourceSnapshot( current, verification, deadline, + step, ); if ("source" in result) return { source: result.source }; current = result.checkpoint; @@ -1636,11 +1638,23 @@ function assertReceiptMatchesPlan( invalidSource("Recording copy receipt does not match its source plan"); } +// Nothing in the plan, enumerate or copy phases of one step writes a copy before the copy phase does, so they +// share one listing; the only other writer, segment preparation, at worst leaves a copy unlisted and it is +// copied again. Verification always lists afresh: it is what proves the originals did not change meanwhile. +type ListedStep = { listing?: Promise }; + +function stepListing(context: SourceContext, step: ListedStep | undefined) { + if (!step) return listSnapshot(context, true); + step.listing ??= listSnapshot(context, true); + return step.listing; +} + async function advanceSourceSnapshotOnce( context: SourceContext, checkpoint: DesktopRecordingSourceCheckpoint, verification?: RecordingVerification, deadline?: number, + step?: ListedStep, ): Promise< | { checkpoint: DesktopRecordingSourceCheckpoint; listed?: boolean } | { source: DesktopRecordingSource } @@ -1653,7 +1667,7 @@ async function advanceSourceSnapshotOnce( const [plan, listing] = await Promise.all([ createSourcePlan(context, verification), context.listed && segmented && deadline !== undefined - ? listSnapshot(context, false) + ? stepListing(context, step) : null, ]); return { @@ -1678,7 +1692,7 @@ async function advanceSourceSnapshotOnce( if (checkpoint.phase === "enumerate") { if (treeCount(checkpoint.planRoots) !== checkpoint.cursor) invalidSource("Recording source plan cursor is incomplete"); - const listing = listed ? await listSnapshot(context, false) : null; + const listing = listed ? await stepListing(context, step) : null; if (listing && deadline !== undefined) { let cursor = checkpoint.cursor; let planRoots = checkpoint.planRoots; @@ -1761,9 +1775,7 @@ async function advanceSourceSnapshotOnce( "Recording copy receipts do not match their checkpoint cursor", ); const listing = - listed && !checkpoint.multipart - ? await listSnapshot(context, true) - : null; + listed && !checkpoint.multipart ? await stepListing(context, step) : null; if (listing && deadline !== undefined) { let cursor = checkpoint.cursor; let receiptRoots = checkpoint.receiptRoots; diff --git a/apps/web/lib/early-instant-playback.ts b/apps/web/lib/early-instant-playback.ts index 71a7b8da2b..b0d47f528d 100644 --- a/apps/web/lib/early-instant-playback.ts +++ b/apps/web/lib/early-instant-playback.ts @@ -1,9 +1,3 @@ -export function isEarlyInstantPlaybackEnabled(ownerId: string): boolean { - const setting = process.env.CAP_EARLY_INSTANT_PLAYBACK?.trim(); - if (!setting) return false; - if (setting === "enabled") return true; - return setting - .split(",") - .map((id) => id.trim()) - .includes(ownerId); +export function isEarlyInstantPlaybackEnabled(): boolean { + return process.env.CAP_EARLY_INSTANT_PLAYBACK?.trim() !== "disabled"; } diff --git a/apps/web/workflows/finalize-desktop-recording.ts b/apps/web/workflows/finalize-desktop-recording.ts index a9d575a8e1..cbf8a6fffa 100644 --- a/apps/web/workflows/finalize-desktop-recording.ts +++ b/apps/web/workflows/finalize-desktop-recording.ts @@ -655,6 +655,11 @@ async function dispatchDesktopRecordingJob( if (!env.MEDIA_SERVER_URL || !env.MEDIA_SERVER_WEBHOOK_SECRET) { throw new Error("Recording content verification is not configured"); } + // Listing the runs decoded during the recording needs nothing below, so it overlaps the inventory read. + const preverifiedRunUrls = + current.source.kind === "mp4" + ? Promise.resolve([]) + : buildPreverifiedRunUrls(video); const urls = await buildDesktopRecordingSourceUrls( video, current.source, @@ -743,9 +748,9 @@ async function dispatchDesktopRecordingJob( ); } path = "/video/mux-segments"; - const [output, preverifiedRunUrls] = await Promise.all([ + const [output, runUrls] = await Promise.all([ buildDesktopSegmentsOutput({ video, attempt, sourceBytes }), - buildPreverifiedRunUrls(video), + preverifiedRunUrls, ]); body = { audioLevels: true, @@ -753,7 +758,7 @@ async function dispatchDesktopRecordingJob( ...urls, ...output, manifestSha256: current.source.manifestSha256, - ...(preverifiedRunUrls.length > 0 ? { preverifiedRunUrls } : {}), + ...(runUrls.length > 0 ? { preverifiedRunUrls: runUrls } : {}), }; } try {