Detect recording glitches during ingest and emit glitches.json manifest

src/L0-pure/types/index.ts

@@ -712,6 +712,37 @@ export interface SilenceRemovalResult {
   wasEdited: boolean;
 }
 
+export type RecordingGlitchType =
+  | 'freeze-frame'
+  | 'audio-dropout'
+  | 'freeze-with-audio-dropout';
+
+export type RecordingGlitchAction = 'auto-trim' | 'review';
+
+export type RecordingGlitchConfidence = 'medium' | 'high';
+
+export interface RecordingGlitch {
+  type: RecordingGlitchType;
+  start: number;
+  end: number;
+  duration: number;
+  action: RecordingGlitchAction;
+  confidence: RecordingGlitchConfidence;
+  detectors: string[];
+}
+
+export interface RecordingGlitchManifest {
+  generatedAt: string;
+  videoPath: string;
+  thresholds: {
+    freezeMinDuration: number;
+    audioMinDuration: number;
+    audioNoiseThreshold: string;
+    autoTrimMaxDuration: number;
+  };
+  glitches: RecordingGlitch[];
+}
+
 // ============================================================================
 // AGENT RESULT (Copilot SDK)
 // ============================================================================

src/L2-clients/ffmpeg/glitchDetection.ts

@@ -0,0 +1,161 @@
+import { createFFmpeg } from './ffmpeg.js'
+import { detectSilence, type SilenceRegion } from './silenceDetection.js'
+import logger from '../../L1-infra/logger/configLogger.js'
+import type { RecordingGlitch, RecordingGlitchManifest } from '../../L0-pure/types/index.js'
+
+interface FreezeRegion {
+  start: number
+  end: number
+  duration: number
+}
+
+export interface RecordingGlitchDetectionOptions {
+  freezeMinDuration?: number
+  freezeNoiseTolerance?: string
+  audioMinDuration?: number
+  audioNoiseThreshold?: string
+  autoTrimMaxDuration?: number
+  correlationGap?: number
+}
+
+const DEFAULT_OPTIONS: Required<RecordingGlitchDetectionOptions> = {
+  freezeMinDuration: 0.08,
+  freezeNoiseTolerance: '0.001',
+  audioMinDuration: 0.08,
+  audioNoiseThreshold: '-45dB',
+  autoTrimMaxDuration: 0.5,
+  correlationGap: 0.15,
+}
+
+async function detectFreezeFrames(
+  videoPath: string,
+  minDuration: number,
+  noiseTolerance: string,
+): Promise<FreezeRegion[]> {
+  logger.info(`Detecting freeze frames in: ${videoPath} (min=${minDuration}s, noise=${noiseTolerance})`)
+
+  return new Promise<FreezeRegion[]>((resolve, reject) => {
+    const regions: FreezeRegion[] = []
+    let pendingStart: number | null = null
+    let stderr = ''
+
+    createFFmpeg(videoPath)
+      .videoFilters(`freezedetect=n=${noiseTolerance}:d=${minDuration}`)
+      .format('null')
+      .output('-')
+      .on('stderr', (line: string) => {
+        stderr += line + '\n'
+      })
+      .on('end', () => {
+        for (const line of stderr.split('\n')) {
+          const startMatch = line.match(/freeze_start:\s*([\d.]+)/)
+          if (startMatch) {
+            pendingStart = parseFloat(startMatch[1])
+          }
+
+          const endMatch = line.match(/freeze_end:\s*([\d.]+)\s*\|\s*freeze_duration:\s*([\d.]+)/)
+          if (endMatch) {
+            const end = parseFloat(endMatch[1])
+            const duration = parseFloat(endMatch[2])
+            const start = pendingStart ?? Math.max(0, end - duration)
+            regions.push({ start, end, duration })
+            pendingStart = null
+          }
+        }
+
+        const validRegions = regions.filter(region => region.end > region.start)
+        logger.info(`Detected ${validRegions.length} freeze-frame regions`)
+        resolve(validRegions)
+      })
+      .on('error', (err) => {
+        logger.error(`Freeze detection failed: ${err.message}`)
+        reject(new Error(`Freeze detection failed: ${err.message}`))
+      })
+      .run()
+  })
+}
+
+function overlapsWithGap(
+  a: { start: number; end: number },
+  b: { start: number; end: number },
+  gap: number,
+): boolean {
+  return a.start <= b.end + gap && b.start <= a.end + gap
+}
+
+function sortByStart<T extends { start: number }>(regions: T[]): T[] {
+  return [...regions].sort((a, b) => a.start - b.start)
+}
+
+export async function detectRecordingGlitches(
+  videoPath: string,
+  options: RecordingGlitchDetectionOptions = {},
+): Promise<RecordingGlitchManifest> {
+  const resolved = { ...DEFAULT_OPTIONS, ...options }
+  const [freezeRegions, audioRegions] = await Promise.all([
+    detectFreezeFrames(videoPath, resolved.freezeMinDuration, resolved.freezeNoiseTolerance),
+    detectSilence(videoPath, resolved.audioMinDuration, resolved.audioNoiseThreshold),
+  ])
+
+  const glitches: RecordingGlitch[] = []
+  const matchedAudio = new Set<number>()
+
+  for (const freeze of freezeRegions) {
+    const overlappingAudio = audioRegions
+      .map((region, index) => ({ region, index }))
+      .filter(({ region }) => overlapsWithGap(freeze, region, resolved.correlationGap))
+
+    if (overlappingAudio.length > 0) {
+      for (const { index } of overlappingAudio) matchedAudio.add(index)
+      const mergedStart = Math.min(freeze.start, ...overlappingAudio.map(({ region }) => region.start))
+      const mergedEnd = Math.max(freeze.end, ...overlappingAudio.map(({ region }) => region.end))
+      const duration = mergedEnd - mergedStart
+
+      glitches.push({
+        type: 'freeze-with-audio-dropout',
+        start: mergedStart,
+        end: mergedEnd,
+        duration,
+        action: duration <= resolved.autoTrimMaxDuration ? 'auto-trim' : 'review',
+        confidence: 'high',
+        detectors: ['freezedetect', 'silencedetect'],
+      })
+      continue
+    }
+
+    glitches.push({
+      type: 'freeze-frame',
+      start: freeze.start,
+      end: freeze.end,
+      duration: freeze.duration,
+      action: 'review',
+      confidence: 'medium',
+      detectors: ['freezedetect'],
+    })
+  }
+
+  audioRegions.forEach((region: SilenceRegion, index: number) => {
+    if (matchedAudio.has(index)) return
+    glitches.push({
+      type: 'audio-dropout',
+      start: region.start,
+      end: region.end,
+      duration: region.duration,
+      action: 'review',
+      confidence: 'medium',
+      detectors: ['silencedetect'],
+    })
+  })
+
+  return {
+    generatedAt: new Date().toISOString(),
+    videoPath,
+    thresholds: {
+      freezeMinDuration: resolved.freezeMinDuration,
+      audioMinDuration: resolved.audioMinDuration,
+      audioNoiseThreshold: resolved.audioNoiseThreshold,
+      autoTrimMaxDuration: resolved.autoTrimMaxDuration,
+    },
+    glitches: sortByStart(glitches),
+  }
+}

src/L3-services/videoOperations/videoOperations.ts

@@ -4,6 +4,7 @@ import { extractClip as _extractClip, extractCompositeClip as _extractCompositeC
 import { singlePassEdit as _singlePassEdit, singlePassEditAndCaption as _singlePassEditAndCaption } from '../../L2-clients/ffmpeg/singlePassEdit.js'
 import { burnCaptions as _burnCaptions } from '../../L2-clients/ffmpeg/captionBurning.js'
 import { detectSilence as _detectSilence } from '../../L2-clients/ffmpeg/silenceDetection.js'
+import { detectRecordingGlitches as _detectRecordingGlitches } from '../../L2-clients/ffmpeg/glitchDetection.js'
 import { captureFrame as _captureFrame } from '../../L2-clients/ffmpeg/frameCapture.js'
 import { generatePlatformVariants as _generatePlatformVariants } from '../../L2-clients/ffmpeg/aspectRatio.js'
 import { detectWebcamRegion as _detectWebcamRegion, getVideoResolution as _getVideoResolution } from '../../L2-clients/ffmpeg/faceDetection.js'
@@ -13,6 +14,7 @@ import { transcodeToMp4 as _transcodeToMp4 } from '../../L2-clients/ffmpeg/trans
 // Re-export types (exempt from layer rules)
 export type { KeepSegment } from '../../L2-clients/ffmpeg/singlePassEdit.js'
 export type { SilenceRegion } from '../../L2-clients/ffmpeg/silenceDetection.js'
+export type { RecordingGlitchDetectionOptions } from '../../L2-clients/ffmpeg/glitchDetection.js'
 export type { Platform } from '../../L2-clients/ffmpeg/aspectRatio.js'
 
 // Video information
@@ -69,6 +71,10 @@ export function detectSilence(...args: Parameters<typeof _detectSilence>): Retur
   return _detectSilence(...args)
 }
 
+export function detectRecordingGlitches(...args: Parameters<typeof _detectRecordingGlitches>): ReturnType<typeof _detectRecordingGlitches> {
+  return _detectRecordingGlitches(...args)
+}
+
 // Frame capture
 export function captureFrame(...args: Parameters<typeof _captureFrame>): ReturnType<typeof _captureFrame> {
   return _captureFrame(...args)

src/L4-agents/videoServiceBridge.ts

@@ -14,6 +14,7 @@ export {
   compositeOverlays,
   getVideoResolution,
   detectWebcamRegion,
+  detectRecordingGlitches,
   burnCaptions,
   singlePassEditAndCaption,
   transcodeToMp4,

src/L5-assets/MainVideoAsset.ts

@@ -32,7 +32,7 @@ import {
 } from '../L1-infra/fileSystem/fileSystem.js'
 import { slugify } from '../L0-pure/text/text.js'
 import { generateSRT, generateVTT, generateStyledASS } from '../L0-pure/captions/captionGenerator.js'
-import { ffprobe, burnCaptions, transcodeToMp4, applyIntroOutro } from '../L4-agents/videoServiceBridge.js'
+import { ffprobe, burnCaptions, transcodeToMp4, applyIntroOutro, detectRecordingGlitches } from '../L4-agents/videoServiceBridge.js'
 import { transcribeVideo, analyzeVideoClipDirection } from '../L4-agents/analysisServiceBridge.js'
 import { removeDeadSilence } from '../L4-agents/SilenceRemovalAgent.js'
 import { generateShorts } from '../L4-agents/ShortsAgent.js'
@@ -209,6 +209,11 @@ export class MainVideoAsset extends VideoAsset {
     return join(this.videoDir, 'transcript-edited.json')
   }
 
+  /** Path to detected recording glitches manifest */
+  get glitchesPath(): string {
+    return join(this.videoDir, 'glitches.json')
+  }
+
   // ── Static Factory Methods ─────────────────────────────────────────────────
 
   /**
@@ -259,6 +264,7 @@ export class MainVideoAsset extends VideoAsset {
         'editorial-direction.md',
         'cost-report.md',
         'layout.json',
+        'glitches.json',
       ]
       for (const pattern of stalePatterns) {
         await removeFile(join(videoDir, pattern))
@@ -343,6 +349,14 @@ export class MainVideoAsset extends VideoAsset {
       logger.warn(`Metadata extraction failed: ${err instanceof Error ? err.message : String(err)}`)
     }
 
+    try {
+      const glitches = await detectRecordingGlitches(destPath)
+      await writeJsonFile(asset.glitchesPath, glitches)
+      logger.info(`Saved ${glitches.glitches.length} detected glitches to ${asset.glitchesPath}`)
+    } catch (err) {
+      logger.warn(`Glitch detection failed: ${err instanceof Error ? err.message : String(err)}`)
+    }
+
     return asset
   }
 

src/__tests__/unit/L2-clients/ffmpegTools.test.ts

@@ -30,6 +30,7 @@ const {
     audioFrequency: vi.fn().mockReturnThis(),
     audioChannels: vi.fn().mockReturnThis(),
     audioFilters: vi.fn().mockReturnThis(),
+    videoFilters: vi.fn().mockReturnThis(),
     noVideo: vi.fn().mockReturnThis(),
     format: vi.fn().mockReturnThis(),
     frames: vi.fn().mockReturnThis(),
@@ -110,6 +111,7 @@ import { extractClip, extractCompositeClip, extractCompositeClipWithTransitions
 import { burnCaptions } from '../../../L2-clients/ffmpeg/captionBurning.js';
 import { extractAudio, splitAudioIntoChunks } from '../../../L2-clients/ffmpeg/audioExtraction.js';
 import { detectSilence } from '../../../L2-clients/ffmpeg/silenceDetection.js';
+import { detectRecordingGlitches } from '../../../L2-clients/ffmpeg/glitchDetection.js';
 import { captureFrame, captureFrames } from '../../../L2-clients/ffmpeg/frameCapture.js';
 
 // ── Helpers ────────────────────────────────────────────────────────────────
@@ -505,6 +507,78 @@ describe('silenceDetection', () => {
   });
 });
 
+describe('glitchDetection', () => {
+  it('merges overlapping freeze and audio dropout regions into high-confidence glitches', async () => {
+    const stderrLines = [
+      '[freezedetect @ 0x1234] freeze_start: 5.0',
+      '[freezedetect @ 0x1234] freeze_end: 5.2 | freeze_duration: 0.2',
+      '[silencedetect @ 0x1234] silence_start: 5.05',
+      '[silencedetect @ 0x1234] silence_end: 5.18 | silence_duration: 0.13',
+    ];
+
+    mockFfmpegInstance.on.mockImplementation(function (this: any, event: string, cb: Function) {
+      if (event === 'stderr') {
+        setTimeout(() => { for (const line of stderrLines) cb(line); }, 0);
+      }
+      if (event === 'end') setTimeout(() => cb(), 10);
+      return this;
+    });
+
+    const result = await detectRecordingGlitches('/video.mp4');
+
+    expect(result.glitches).toHaveLength(1);
+    expect(result.glitches[0]).toMatchObject({
+      type: 'freeze-with-audio-dropout',
+      start: 5,
+      end: 5.2,
+      action: 'auto-trim',
+      confidence: 'high',
+      detectors: ['freezedetect', 'silencedetect'],
+    });
+    expect(result.glitches[0].duration).toBeCloseTo(0.2, 5);
+    expect(mockFfmpegInstance.videoFilters).toHaveBeenCalledWith('freezedetect=n=0.001:d=0.08');
+    expect(mockFfmpegInstance.audioFilters).toHaveBeenCalledWith('silencedetect=noise=-45dB:d=0.08');
+  });
+
+  it('keeps unmatched glitches as review items', async () => {
+    mockFfmpegInstance.on.mockImplementation(function (this: any, event: string, cb: Function) {
+      if (event === 'stderr') {
+        setTimeout(() => {
+          cb('[freezedetect @ 0x1234] freeze_start: 12.0');
+          cb('[freezedetect @ 0x1234] freeze_end: 12.8 | freeze_duration: 0.8');
+          cb('[silencedetect @ 0x1234] silence_start: 20.0');
+          cb('[silencedetect @ 0x1234] silence_end: 20.2 | silence_duration: 0.2');
+        }, 0);
+      }
+      if (event === 'end') setTimeout(() => cb(), 10);
+      return this;
+    });
+
+    const result = await detectRecordingGlitches('/video.mp4');
+
+    expect(result.glitches).toEqual([
+      {
+        type: 'freeze-frame',
+        start: 12,
+        end: 12.8,
+        duration: 0.8,
+        action: 'review',
+        confidence: 'medium',
+        detectors: ['freezedetect'],
+      },
+      {
+        type: 'audio-dropout',
+        start: 20,
+        end: 20.2,
+        duration: 0.2,
+        action: 'review',
+        confidence: 'medium',
+        detectors: ['silencedetect'],
+      },
+    ]);
+  });
+});
+
 // ═══════════════════════════════════════════════════════════════════════════
 // 5. frameCapture.ts
 // ═══════════════════════════════════════════════════════════════════════════