diff --git a/.gitignore b/.gitignore index 61dc9d2fc..33cf7a9d2 100644 --- a/.gitignore +++ b/.gitignore @@ -49,6 +49,7 @@ playwright-report/ # Vitest browser mode screenshots __screenshots__/ +.vitest-attachments/ # shell files /shell.sh diff --git a/src/lib/exporter/audioEncoder.test.ts b/src/lib/exporter/audioEncoder.test.ts index 0fb3708c5..13f29af10 100644 --- a/src/lib/exporter/audioEncoder.test.ts +++ b/src/lib/exporter/audioEncoder.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import { AudioProcessor, downmixPlanarChannelsForExport } from "./audioEncoder"; +import { AudioProcessor, downmixPlanarChannelsForExport, mixPlanarSources } from "./audioEncoder"; describe("AudioProcessor.selectSupportedExportCodec", () => { afterEach(() => { @@ -70,3 +70,95 @@ describe("downmixPlanarChannelsForExport", () => { expect(Array.from(stereo)).toEqual([0.25, -0.5, 0.25, -0.5]); }); }); + +describe("mixPlanarSources", () => { + it("sums overlapping stereo sources sample-aligned", () => { + const system = { + planes: [new Float32Array([0.1, 0.2, 0.3]), new Float32Array([0.1, 0.2, 0.3])], + startFrame: 0, + }; + const mic = { + planes: [new Float32Array([0.2, 0.2, 0.2]), new Float32Array([0.4, 0.4, 0.4])], + startFrame: 0, + }; + + const mixed = mixPlanarSources([system, mic], 2, 3); + + expect(mixed).toHaveLength(2); + expect(Array.from(mixed[0])).toEqual([ + expect.closeTo(0.3, 5), + expect.closeTo(0.4, 5), + expect.closeTo(0.5, 5), + ]); + expect(Array.from(mixed[1])).toEqual([ + expect.closeTo(0.5, 5), + expect.closeTo(0.6, 5), + expect.closeTo(0.7, 5), + ]); + }); + + it("recovers a track when the other is silent (the #108 case)", () => { + // System audio captured silence; the microphone track holds the real signal. + const silentSystem = { + planes: [new Float32Array([0, 0, 0]), new Float32Array([0, 0, 0])], + startFrame: 0, + }; + const mic = { + planes: [new Float32Array([0.5, -0.5, 0.25]), new Float32Array([0.5, -0.5, 0.25])], + startFrame: 0, + }; + + const mixed = mixPlanarSources([silentSystem, mic], 2, 3); + + expect(Array.from(mixed[0])).toEqual([ + expect.closeTo(0.5, 5), + expect.closeTo(-0.5, 5), + expect.closeTo(0.25, 5), + ]); + }); + + it("aligns each source at its startFrame offset", () => { + const early = { planes: [new Float32Array([1, 1])], startFrame: 0 }; + // Mic packets often start slightly after the video (source start_time > 0). + const late = { planes: [new Float32Array([0.5, 0.5])], startFrame: 2 }; + + const mixed = mixPlanarSources([early, late], 1, 4); + + expect(Array.from(mixed[0])).toEqual([ + expect.closeTo(1, 5), + expect.closeTo(1, 5), + expect.closeTo(0.5, 5), + expect.closeTo(0.5, 5), + ]); + }); + + it("upmixes a mono source to stereo before mixing", () => { + const mono = { planes: [new Float32Array([0.3, 0.3])], startFrame: 0 }; + const stereo = { + planes: [new Float32Array([0.1, 0.1]), new Float32Array([0.2, 0.2])], + startFrame: 0, + }; + + const mixed = mixPlanarSources([mono, stereo], 2, 2); + + expect(Array.from(mixed[0])).toEqual([expect.closeTo(0.4, 5), expect.closeTo(0.4, 5)]); + expect(Array.from(mixed[1])).toEqual([expect.closeTo(0.5, 5), expect.closeTo(0.5, 5)]); + }); + + it("clamps the summed signal to [-1, 1] to avoid overflow", () => { + const a = { planes: [new Float32Array([0.8, -0.8])], startFrame: 0 }; + const b = { planes: [new Float32Array([0.8, -0.8])], startFrame: 0 }; + + const mixed = mixPlanarSources([a, b], 1, 2); + + expect(Array.from(mixed[0])).toEqual([1, -1]); + }); + + it("ignores samples that fall beyond the mixed length", () => { + const source = { planes: [new Float32Array([0.5, 0.5, 0.5])], startFrame: 1 }; + + const mixed = mixPlanarSources([source], 1, 2); + + expect(Array.from(mixed[0])).toEqual([expect.closeTo(0, 5), expect.closeTo(0.5, 5)]); + }); +}); diff --git a/src/lib/exporter/audioEncoder.ts b/src/lib/exporter/audioEncoder.ts index b97aea1fd..9e7e85cae 100644 --- a/src/lib/exporter/audioEncoder.ts +++ b/src/lib/exporter/audioEncoder.ts @@ -1,4 +1,4 @@ -import { WebDemuxer } from "web-demuxer"; +import { AVMediaType, type WebAVStream, WebDemuxer } from "web-demuxer"; import { MAX_NATIVE_PLAYBACK_RATE, type SpeedRegion, @@ -133,6 +133,68 @@ function getStereoDownmixWeights(sourceChannels: number) { }; } +/** One decoded audio source to be mixed: its per-channel PCM and where it starts on the timeline. */ +export interface PlanarAudioSource { + /** Per-channel planar samples (`channels[c][frame]`). */ + planes: Float32Array[]; + /** Timeline sample offset of this source's first sample (from the stream's start_time). */ + startFrame: number; +} + +/** + * Mixes several decoded audio sources into a single planar buffer. + * + * Native macOS recordings write system audio and the microphone as two separate + * tracks in the same file. The exporter historically decoded only the one track + * FFmpeg's `av_find_best_stream` selects — which, when both tracks are marked + * default, is the first (system) track. If nothing was playing that track is + * silent, so the mic was dropped from the export (issue #108). Mixing every audio + * source together mirrors the browser recorder, which already blends system + mic + * into one track before encoding. + * + * Each source is first downmixed to `targetChannels`, then summed sample-aligned at + * its `startFrame`. The result is clamped to [-1, 1] so a loud overlap can't wrap or + * overflow the encoder. Sources of differing lengths and offsets are supported; + * samples past `totalFrames` are dropped. + */ +export function mixPlanarSources( + sources: PlanarAudioSource[], + targetChannels: number, + totalFrames: number, +): Float32Array[] { + const mixed = Array.from({ length: targetChannels }, () => new Float32Array(totalFrames)); + + for (const source of sources) { + const frameCount = source.planes[0]?.length ?? 0; + if (frameCount === 0) continue; + + // Reuse the export downmix so mono→stereo, surround→stereo, etc. match the + // single-track path exactly. Output is planar-concatenated: [ch0…, ch1…]. + const downmixed = downmixPlanarChannelsForExport(source.planes, targetChannels); + + for (let channel = 0; channel < targetChannels; channel++) { + const target = mixed[channel]; + const base = channel * frameCount; + for (let frame = 0; frame < frameCount; frame++) { + const outFrame = source.startFrame + frame; + if (outFrame < 0) continue; + if (outFrame >= totalFrames) break; + target[outFrame] += downmixed[base + frame]; + } + } + } + + for (const channel of mixed) { + for (let frame = 0; frame < channel.length; frame++) { + const sample = channel[frame]; + if (sample > 1) channel[frame] = 1; + else if (sample < -1) channel[frame] = -1; + } + } + + return mixed; +} + export function downmixPlanarChannelsForExport( sourcePlanes: Float32Array[], targetChannels: number, @@ -226,6 +288,269 @@ export class AudioProcessor { ); } + /** All audio streams in the source, in container order. Empty on failure. */ + private static async listAudioStreams(demuxer: WebDemuxer): Promise { + try { + const info = await demuxer.getMediaInfo(); + return info.streams.filter((s) => s.codec_type_string === "audio"); + } catch { + return []; + } + } + + /** + * Fully decodes one specific audio stream to contiguous planar PCM. Unlike + * `demuxer.read("audio")`/`getDecoderConfig("audio")` — which resolve to the + * single stream `av_find_best_stream` picks — this targets a stream by its + * container index, so each track of a multi-track recording can be decoded + * independently and then mixed. Returns null when the stream can't be decoded. + */ + private async decodeAudioStreamToPlanes( + demuxer: WebDemuxer, + stream: WebAVStream, + readEndSec: number | undefined, + ): Promise<(PlanarAudioSource & { sampleRate: number; numberOfChannels: number }) | null> { + const config: AudioDecoderConfig = { + codec: stream.codec_string || "", + sampleRate: stream.sample_rate, + numberOfChannels: stream.channels, + description: stream.extradata && stream.extradata.length > 0 ? stream.extradata : undefined, + }; + if (!config.codec || !config.sampleRate || !config.numberOfChannels) return null; + + const support = await AudioDecoder.isConfigSupported(config); + if (!support.supported) { + console.warn( + `[AudioProcessor] Audio codec not supported for stream ${stream.index}:`, + config.codec, + ); + return null; + } + + const decoded: AudioData[] = []; + const decoder = new AudioDecoder({ + output: (data: AudioData) => decoded.push(data), + error: (e: DOMException) => console.error("[AudioProcessor] Decode error:", e), + }); + decoder.configure(config); + + const safeReadEnd = + typeof readEndSec === "number" && Number.isFinite(readEndSec) ? Math.max(0, readEndSec) : 0; + const packetStream = demuxer.readAVPacket( + 0, + safeReadEnd, + AVMediaType.AVMEDIA_TYPE_AUDIO, + stream.index, + ); + const reader = packetStream.getReader(); + try { + while (!this.cancelled) { + const { done, value: packet } = await reader.read(); + if (done || !packet) break; + decoder.decode( + new EncodedAudioChunk({ + type: packet.keyframe === 1 ? "key" : "delta", + timestamp: Math.round(packet.timestamp * 1_000_000), + duration: Math.round(packet.duration * 1_000_000), + data: packet.data, + }), + ); + while (decoder.decodeQueueSize > DECODE_BACKPRESSURE_LIMIT && !this.cancelled) { + await new Promise((resolve) => setTimeout(resolve, 1)); + } + } + } finally { + try { + await reader.cancel(); + } catch { + /* reader already closed */ + } + } + + if (decoder.state === "configured") { + if (!this.cancelled) await decoder.flush(); + decoder.close(); + } + + if (this.cancelled || decoded.length === 0) { + for (const d of decoded) d.close(); + return null; + } + + // Decode order is normally monotonic; sort defensively so timeline placement + // stays correct even if the decoder emits out of order. + decoded.sort((a, b) => a.timestamp - b.timestamp); + const sampleRate = decoded[0].sampleRate; + const numberOfChannels = decoded[0].numberOfChannels; + // Absolute timeline position of this stream's first sample (mic tracks often + // start slightly after the video, e.g. start_time ~0.17s), so mixing keeps + // every track locked to the same source-time origin the video uses. + const startFrame = Math.max(0, Math.round((decoded[0].timestamp / 1_000_000) * sampleRate)); + + let totalFrames = 0; + for (const d of decoded) { + const frameOffset = + Math.round((d.timestamp / 1_000_000) * sampleRate) - startFrame + d.numberOfFrames; + if (frameOffset > totalFrames) totalFrames = frameOffset; + } + + const planes = Array.from({ length: numberOfChannels }, () => new Float32Array(totalFrames)); + for (const d of decoded) { + const at = Math.max(0, Math.round((d.timestamp / 1_000_000) * sampleRate) - startFrame); + const room = totalFrames - at; + if (room > 0) { + const take = Math.min(d.numberOfFrames, room); + for (let c = 0; c < numberOfChannels; c++) { + const temp = new Float32Array(d.numberOfFrames); + d.copyTo(temp, { format: "f32-planar", planeIndex: c }); + planes[c].set(temp.subarray(0, take), at); + } + } + d.close(); + } + + return { planes, startFrame, sampleRate, numberOfChannels }; + } + + /** + * Decodes every audio stream and mixes them into one planar timeline (see + * {@link mixPlanarSources}). Returns null when there is at most one audio stream + * (callers keep the existing single-stream fast path) or when the streams can't + * be mixed (e.g. mismatched sample rates — never the case for native captures, + * which are all 48 kHz). + */ + private async decodeMixedAudioPlanes( + demuxer: WebDemuxer, + streams: WebAVStream[], + readEndSec: number | undefined, + outputChannels: number, + ): Promise<{ planes: Float32Array[]; sampleRate: number; numberOfChannels: number } | null> { + if (streams.length < 2) return null; + + const sources: Array = []; + for (const stream of streams) { + if (this.cancelled) return null; + const decoded = await this.decodeAudioStreamToPlanes(demuxer, stream, readEndSec); + if (decoded) sources.push(decoded); + } + + if (sources.length === 0) return null; + // A single decodable stream can't be "mixed" — fall back to the normal path. + if (sources.length === 1) return null; + + const sampleRate = sources[0].sampleRate; + if (sources.some((s) => s.sampleRate !== sampleRate)) { + console.warn("[AudioProcessor] Audio streams differ in sample rate; skipping mix"); + return null; + } + + const channels = Math.min(2, Math.max(1, outputChannels || 2)); + let totalFrames = 0; + for (const s of sources) { + const end = s.startFrame + (s.planes[0]?.length ?? 0); + if (end > totalFrames) totalFrames = end; + } + if (totalFrames === 0) return null; + + const planes = mixPlanarSources( + sources.map((s) => ({ planes: s.planes, startFrame: s.startFrame })), + channels, + totalFrames, + ); + console.log( + `[AudioProcessor] Mixed ${sources.length} audio tracks (${totalFrames} frames @ ${sampleRate}Hz)`, + ); + return { planes, sampleRate, numberOfChannels: channels }; + } + + /** + * Encodes a pre-mixed planar timeline into the muxer, applying the same trim + * skipping + timestamp remap the single-stream trim-only path uses. Blocks are a + * single AAC frame wide so trim boundaries land at the same granularity. + */ + private async encodeMixedPlanarTimeline( + mixed: { planes: Float32Array[]; sampleRate: number; numberOfChannels: number }, + muxer: VideoMuxer, + sortedTrims: TrimRegion[], + exportCodec: ExportAudioCodec, + ): Promise { + const { planes, sampleRate } = mixed; + const totalFrames = planes[0]?.length ?? 0; + if (totalFrames === 0) return; + + const outputSampleRate = exportCodec.sampleRate || sampleRate; + const outputChannels = exportCodec.numberOfChannels || mixed.numberOfChannels; + + const encodeConfig: AudioEncoderConfig = { + codec: exportCodec.encoderCodec, + sampleRate: outputSampleRate, + numberOfChannels: outputChannels, + bitrate: AUDIO_BITRATE, + }; + const encodeSupport = await AudioEncoder.isConfigSupported(encodeConfig); + if (!encodeSupport.supported) { + console.warn(`[AudioProcessor] ${exportCodec.label} encoding not supported, skipping audio`); + return; + } + + const encodedChunks: { chunk: EncodedAudioChunk; meta?: EncodedAudioChunkMetadata }[] = []; + const encoder = new AudioEncoder({ + output: (chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata) => { + encodedChunks.push({ chunk, meta }); + }, + error: (e: DOMException) => console.error("[AudioProcessor] Encode error:", e), + }); + encoder.configure(encodeConfig); + + // One AAC frame per block so trim skipping matches the packet granularity of + // the single-stream path. + const BLOCK_FRAMES = 1024; + for (let start = 0; start < totalFrames && !this.cancelled; start += BLOCK_FRAMES) { + const count = Math.min(BLOCK_FRAMES, totalFrames - start); + const timestampMs = (start / sampleRate) * 1000; + if (this.isInTrimRegion(timestampMs, sortedTrims)) continue; + + const trimOffsetMs = this.computeTrimOffset(timestampMs, sortedTrims); + const adjustedTimestampUs = Math.max( + 0, + Math.round((start / sampleRate - trimOffsetMs / 1000) * 1_000_000), + ); + + const buffer = new Float32Array(count * outputChannels); + for (let c = 0; c < outputChannels; c++) { + const srcPlane = planes[Math.min(c, planes.length - 1)]; + buffer.set(srcPlane.subarray(start, start + count), c * count); + } + + const audioData = new AudioData({ + format: "f32-planar", + sampleRate: outputSampleRate, + numberOfFrames: count, + numberOfChannels: outputChannels, + timestamp: adjustedTimestampUs, + data: buffer.buffer as ArrayBuffer, + }); + encoder.encode(audioData); + audioData.close(); + + while (encoder.encodeQueueSize > ENCODE_BACKPRESSURE_LIMIT && !this.cancelled) { + await new Promise((resolve) => setTimeout(resolve, 1)); + } + } + + if (encoder.state === "configured") { + await encoder.flush(); + encoder.close(); + } + + for (const { chunk, meta } of encodedChunks) { + if (this.cancelled) break; + await muxer.addAudioChunk(chunk, meta); + } + + console.log(`[AudioProcessor] Encoded mixed audio timeline: ${encodedChunks.length} chunks`); + } + /** * Two modes: no speed regions uses the fast WebCodecs trim-only pipeline; speed * regions use the pitch-preserving rendered timeline pipeline. @@ -254,7 +579,12 @@ export class AudioProcessor { if (sortedSpeedRegions.length > 0) { const segments = buildSpeedSegments(validatedDurationSec, sortedTrims, sortedSpeedRegions); - if (maxTimelineSpeed(segments) > MAX_NATIVE_PLAYBACK_RATE) { + // The real-time capture path plays the file through an