diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 852250ec4..37084f0a9 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -1052,6 +1052,7 @@ function readNativeWindowsEncoderSelection(output: string) { try { return JSON.parse(lastLine) as { video?: string; + frameTransport?: "gpu-zero-copy" | "cpu-readback"; preferSoftwareEncoder?: boolean; }; } catch { @@ -1767,6 +1768,7 @@ export function registerIpcHandlers( path: outputPath, helperPath, videoEncoderSelection: encoderSelection?.video ?? null, + videoFrameTransport: encoderSelection?.frameTransport ?? null, }; } catch (error) { console.error("Failed to start native Windows recording:", error); diff --git a/electron/native/README.md b/electron/native/README.md index 79399d46b..f70012e73 100644 --- a/electron/native/README.md +++ b/electron/native/README.md @@ -84,9 +84,9 @@ Current V2 JSON shape: The current helper implementation supports display/window video capture, system audio loopback, selected-microphone capture, Media Foundation webcam capture, and a DirectShow webcam fallback for virtual cameras that are not exposed through Media Foundation. Webcam frames are currently composed into the primary MP4 as a bottom-right picture-in-picture overlay. Browser `deviceId` values do not always map to Media Foundation symbolic links or WASAPI endpoint IDs, so the renderer passes both browser IDs and user-visible device names. For microphones, the helper tries the requested WASAPI endpoint ID first, then resolves an active capture endpoint by `microphoneDeviceName`, then falls back to the default endpoint. For webcams, Electron resolves a matching DirectShow filter CLSID for the selected label; the helper uses Media Foundation first, then that exact DirectShow filter when the requested camera is absent from Media Foundation. -Encoder selection: by default the helper keeps the existing sink-writer path first. If that path fails while setting up H.264, it retries with the Microsoft software H.264 encoder (`mfh264enc.dll`). The key of this retry is registering that encoder locally in the helper process via `MFTRegisterLocalByCLSID`, which makes a software H.264 encoder available even when the machine's hardware encoders are missing or broken; hardware transforms are disabled for the retry only as a secondary guard so the sink writer prefers the locally registered software encoder, not as the fallback mechanism itself. Set `preferSoftwareEncoder: true` in the helper JSON, or set `OPENSCREEN_WGC_PREFER_SOFTWARE_ENCODER=true` before launching Electron, to force the software path from the first attempt. +Encoder selection: by default the helper explicitly enables Media Foundation hardware transforms, supplies the capture D3D11 device through `MF_SINK_WRITER_D3D_MANAGER`, and passes DXGI texture-backed samples to the sink writer. A four-texture GPU ring keeps each texture alive until `IMFSinkWriter::PlaceMarker` confirms that Media Foundation consumed it. This avoids the former per-frame staging-texture `Map` and BGRA memory copy. If the GPU-backed writer cannot be created, or the DXGI frame path fails while recording, the helper keeps recording through the CPU readback path. If H.264 setup itself fails, it retries with the Microsoft software H.264 encoder (`mfh264enc.dll`). The key of this retry is registering that encoder locally in the helper process via `MFTRegisterLocalByCLSID`, which makes a software H.264 encoder available even when the machine's hardware encoders are missing or broken; hardware transforms are disabled for the retry only as a secondary guard so the sink writer prefers the locally registered software encoder, not as the fallback mechanism itself. Set `preferSoftwareEncoder: true` in the helper JSON, or set `OPENSCREEN_WGC_PREFER_SOFTWARE_ENCODER=true` before launching Electron, to force the software path from the first attempt. -The helper reports the outcome through the `encoder-selection` stdout event (`video` is `default`, `software-preferred`, or `software-fallback`). When the app sees `software-fallback` — the default encoder failed and the helper switched on its own — it shows a small dismissible notice in the recording HUD with a "Don't show again" option, because software encoding can raise CPU usage. An explicit `software-preferred` selection shows no notice, and the event stays available for diagnostics either way. +The helper reports the outcome through the `encoder-selection` stdout event. `video` is `hardware` when the selected H.264 transform exposes the Media Foundation hardware URL attribute, `software-default` when the hardware-enabled path selected a software encoder, `software-preferred` or `software-fallback` for the explicit software paths, and `default` only when the sink writer does not expose enough information to classify the transform. `frameTransport` is `gpu-zero-copy` when video frames stay in D3D11/DXGI memory, or `cpu-readback` when the compatibility path is active. The final `encoder-summary` event includes the effective frame transport, GPU/CPU frame counts, elapsed time, and helper process CPU time. The app shows its dismissible software-encoder notice for `software-default` and `software-fallback`, because CPU usage can be higher. An explicit `software-preferred` selection shows no notice, and the event stays available for diagnostics either way. Encoder diagnostic on final sink-writer failure: when the final `MFCreateSinkWriterFromURL` attempt fails, the helper logs the registered H.264 video encoder MFT count (via `MFTEnumEx`), the registered AAC encoder count when audio was requested, and the hex HRESULT. If no H.264 encoder is registered, it additionally emits the four-bullet actionable error (missing Media Feature Pack / GPU driver registration / empty `HKLM:\SOFTWARE\Microsoft\Windows Media Foundation\Transforms` / reboot). If an H.264 encoder IS registered but the sink writer still failed, it logs a hint pointing at invalid output path, missing MP4 mux, or GPU driver incompatibility. There is still no fail-fast pre-flight gate because `MFTEnumEx` and `MFCreateSinkWriterFromURL` can disagree about which H.264 encoders are available in non-interactive / Session 0 contexts. @@ -96,6 +96,7 @@ Smoke-test the helper with: npm run test:wgc-helper:win npm run test:wgc-helper:win -- --software-encoder npm run test:wgc-helper:win -- --software-fallback +npm run test:wgc-helper:win -- --gpu-frame-fallback npm run test:wgc-window:win npm run test:wgc-audio:win npm run test:wgc-mic:win @@ -103,6 +104,8 @@ npm run test:wgc-mixed-audio:win npm run test:wgc-webcam:win ``` +Set `OPENSCREEN_WGC_TEST_FPS=60` to change the smoke-test frame rate, and set `OPENSCREEN_WGC_TEST_REQUIRE_GPU_ZERO_COPY=true` to fail the test unless a hardware H.264 encoder consumes every screen frame through the DXGI path. `OPENSCREEN_WGC_TEST_DISPLAY_X`, `OPENSCREEN_WGC_TEST_DISPLAY_Y`, `OPENSCREEN_WGC_TEST_DISPLAY_WIDTH`, and `OPENSCREEN_WGC_TEST_DISPLAY_HEIGHT` can select a monitor by its physical Win32 bounds for multi-monitor coverage. + `--software-encoder` keeps testing the explicit `software-preferred` path with `preferSoftwareEncoder: true`. `--software-fallback` keeps `preferSoftwareEncoder: false` and sets diff --git a/electron/native/wgc-capture/src/main.cpp b/electron/native/wgc-capture/src/main.cpp index 30a3069d6..ad92c7697 100644 --- a/electron/native/wgc-capture/src/main.cpp +++ b/electron/native/wgc-capture/src/main.cpp @@ -83,6 +83,29 @@ struct CaptureControl { } }; +uint64_t currentProcessCpuTimeHns() { + FILETIME creationTime{}; + FILETIME exitTime{}; + FILETIME kernelTime{}; + FILETIME userTime{}; + if (!GetProcessTimes( + GetCurrentProcess(), + &creationTime, + &exitTime, + &kernelTime, + &userTime)) { + return 0; + } + + ULARGE_INTEGER kernel{}; + kernel.LowPart = kernelTime.dwLowDateTime; + kernel.HighPart = kernelTime.dwHighDateTime; + ULARGE_INTEGER user{}; + user.LowPart = userTime.dwLowDateTime; + user.HighPart = userTime.dwHighDateTime; + return kernel.QuadPart + user.QuadPart; +} + std::wstring utf8ToWide(const std::string& value) { if (value.empty()) { return {}; @@ -404,6 +427,14 @@ int main(int argc, char* argv[]) { const bool injectDefaultSinkWriterFailureOnce = injectDefaultSinkWriterFailureLength == 1 && injectDefaultSinkWriterFailure[0] == '1'; + char injectGpuFrameFailure[2]{}; + const DWORD injectGpuFrameFailureLength = GetEnvironmentVariableA( + "OPENSCREEN_WGC_TEST_INJECT_GPU_FRAME_FAILURE_ONCE", + injectGpuFrameFailure, + static_cast(sizeof(injectGpuFrameFailure))); + const bool injectGpuFrameFailureOnce = + injectGpuFrameFailureLength == 1 && + injectGpuFrameFailure[0] == '1'; std::cout << "{\"event\":\"ready\",\"schemaVersion\":2}" << std::endl; @@ -435,9 +466,9 @@ int main(int argc, char* argv[]) { return 1; } - // WGC owns the captured texture size. Encoding must use that exact size - // until a dedicated GPU scaling pass is introduced; CopyResource requires - // matching resource dimensions. + // H.264 requires even dimensions. WGC window textures can be odd-sized, so + // keep the largest even rectangle and crop at most one pixel on the right + // or bottom during the GPU copy below. int width = session.captureWidth(); int height = session.captureHeight(); width = (std::max(2, width) / 2) * 2; @@ -516,6 +547,8 @@ int main(int argc, char* argv[]) { MFEncoderOptions encoderOptions{}; encoderOptions.preferSoftwareEncoder = config.preferSoftwareEncoder; encoderOptions.injectDefaultSinkWriterFailureOnce = injectDefaultSinkWriterFailureOnce; + encoderOptions.injectGpuFrameFailureOnce = injectGpuFrameFailureOnce; + encoderOptions.allowGpuFrameTransport = !config.webcamEnabled || writeSeparateWebcam; MFEncoder encoder; if (!encoder.initialize( @@ -533,6 +566,8 @@ int main(int argc, char* argv[]) { } std::cout << "{\"event\":\"encoder-selection\",\"schemaVersion\":2,\"video\":\"" << encoder.videoEncoderSelection() + << "\",\"frameTransport\":\"" + << encoder.videoFrameTransport() << "\",\"preferSoftwareEncoder\":" << (config.preferSoftwareEncoder ? "true" : "false") << "}" << std::endl; @@ -540,6 +575,8 @@ int main(int argc, char* argv[]) { if (writeSeparateWebcam) { MFEncoderOptions webcamEncoderOptions = encoderOptions; webcamEncoderOptions.injectDefaultSinkWriterFailureOnce = false; + webcamEncoderOptions.injectGpuFrameFailureOnce = false; + webcamEncoderOptions.allowGpuFrameTransport = false; const int webcamPixels = std::max(1, webcamCapture.width()) * std::max(1, webcamCapture.height()); const int webcamBitrate = webcamPixels >= 1280 * 720 ? 8'000'000 : 4'000'000; if (!webcamEncoder.initialize( @@ -579,6 +616,8 @@ int main(int argc, char* argv[]) { if (!latestFrameTexture) { D3D11_TEXTURE2D_DESC desc{}; texture->GetDesc(&desc); + desc.Width = static_cast(width); + desc.Height = static_cast(height); desc.BindFlags = 0; desc.CPUAccessFlags = 0; desc.MiscFlags = 0; @@ -590,7 +629,23 @@ int main(int argc, char* argv[]) { } } - session.context()->CopyResource(latestFrameTexture.Get(), texture); + const D3D11_BOX sourceBox{ + 0, + 0, + 0, + static_cast(width), + static_cast(height), + 1, + }; + session.context()->CopySubresourceRegion( + latestFrameTexture.Get(), + 0, + 0, + 0, + 0, + texture, + 0, + &sourceBox); latestFrameTimestampHns = timestampHns; if (!firstFrameWritten.exchange(true)) { control.cv.notify_all(); @@ -834,6 +889,8 @@ int main(int argc, char* argv[]) { if (audioMixer) { audioMixer->beginTimeline(); } + const auto recordingWallStart = std::chrono::steady_clock::now(); + const uint64_t recordingCpuStartHns = currentProcessCpuTimeHns(); startVideoWriter(); std::cout << "{\"event\":\"recording-started\",\"schemaVersion\":2}" << std::endl; @@ -869,10 +926,14 @@ int main(int argc, char* argv[]) { logStopStep("wgc-session-close"); { std::scoped_lock lock(mutex); - encoder.finalize(); + if (!encoder.finalize()) { + encodeFailed = true; + } logStopStep("encoder-finalize"); if (writeSeparateWebcam) { - webcamEncoder.finalize(); + if (!webcamEncoder.finalize()) { + encodeFailed = true; + } logStopStep("webcam-encoder-finalize"); } } @@ -886,6 +947,23 @@ int main(int argc, char* argv[]) { return 1; } + const auto recordingElapsedMs = std::chrono::duration_cast( + std::chrono::steady_clock::now() - recordingWallStart).count(); + const uint64_t recordingCpuEndHns = currentProcessCpuTimeHns(); + const uint64_t recordingCpuTimeMs = recordingCpuEndHns >= recordingCpuStartHns + ? (recordingCpuEndHns - recordingCpuStartHns) / 10'000ULL + : 0; + + std::cout << "{\"event\":\"encoder-summary\",\"schemaVersion\":2,\"video\":\"" + << encoder.videoEncoderSelection() + << "\",\"frameTransport\":\"" + << encoder.videoFrameTransport() + << "\",\"gpuFrames\":" << encoder.gpuFramesWritten() + << ",\"cpuFrames\":" << encoder.cpuFramesWritten() + << ",\"elapsedMs\":" << recordingElapsedMs + << ",\"processCpuTimeMs\":" << recordingCpuTimeMs + << "}" << std::endl; + std::cout << "{\"event\":\"recording-stopped\",\"schemaVersion\":2,\"screenPath\":\"" << jsonEscape(config.outputPath) << "\""; if (writeSeparateWebcam) { diff --git a/electron/native/wgc-capture/src/mf_encoder.cpp b/electron/native/wgc-capture/src/mf_encoder.cpp index 42b708910..7699c190a 100644 --- a/electron/native/wgc-capture/src/mf_encoder.cpp +++ b/electron/native/wgc-capture/src/mf_encoder.cpp @@ -4,11 +4,17 @@ #include #include +#include #include +#include #include +#include +#include #include #include +#include +#include namespace { @@ -22,6 +28,54 @@ bool succeeded(HRESULT hr, const char* label) { return false; } +class SinkWriterCallback final : public Microsoft::WRL::RuntimeClass< + Microsoft::WRL::RuntimeClassFlags, + IMFSinkWriterCallback> { +public: + STDMETHODIMP OnFinalize(HRESULT status) override { + { + std::scoped_lock lock(mutex_); + finalizeStatus_ = status; + finalized_ = true; + } + condition_.notify_all(); + return S_OK; + } + + STDMETHODIMP OnMarker(DWORD, LPVOID context) override { + const auto marker = reinterpret_cast(context); + { + std::scoped_lock lock(mutex_); + completedMarker_ = std::max(completedMarker_, marker); + } + condition_.notify_all(); + return S_OK; + } + + bool waitForMarker(uintptr_t marker, std::chrono::milliseconds timeout) { + std::unique_lock lock(mutex_); + return condition_.wait_for(lock, timeout, [&] { + return completedMarker_ >= marker || finalized_; + }) && completedMarker_ >= marker; + } + + bool waitForFinalize(std::chrono::milliseconds timeout, HRESULT& status) { + std::unique_lock lock(mutex_); + if (!condition_.wait_for(lock, timeout, [&] { return finalized_; })) { + return false; + } + status = finalizeStatus_; + return true; + } + +private: + std::mutex mutex_; + std::condition_variable condition_; + uintptr_t completedMarker_ = 0; + bool finalized_ = false; + HRESULT finalizeStatus_ = E_PENDING; +}; + // Count how many Media Foundation Transforms are registered for a given // (category, output subtype) pair. Caller does not need the activations // themselves; we just want to know whether at least one is registered so we @@ -133,7 +187,9 @@ void logMissingH264EncoderError() { enum class SinkWriterCreateStage { SoftwareEncoderRegistration, CreateAttributes, - DisableHardwareTransforms, + ConfigureHardwareTransforms, + ConfigureD3DManager, + ConfigureAsyncCallback, CreateSinkWriter, }; @@ -181,6 +237,8 @@ HRESULT createSinkWriterFromUrl( bool forceSoftwareEncoder, bool injectDefaultSinkWriterFailureOnce, bool& injectedDefaultSinkWriterFailure, + IMFDXGIDeviceManager* d3dManager, + IMFSinkWriterCallback* asyncCallback, Microsoft::WRL::ComPtr& sinkWriter, SinkWriterCreateStage& failedStage) { // Default to the sink-writer creation step; the software-path steps below @@ -204,22 +262,49 @@ HRESULT createSinkWriterFromUrl( return registerHr; } - HRESULT hr = MFCreateAttributes(&attributes, 1); + } + + HRESULT hr = MFCreateAttributes(&attributes, 3); + if (FAILED(hr)) { + std::cerr << "ERROR: MFCreateAttributes(sink writer) failed (hr=0x" + << std::hex << hr << std::dec << ")" << std::endl; + failedStage = SinkWriterCreateStage::CreateAttributes; + return hr; + } + + if (d3dManager != nullptr) { + hr = attributes->SetUnknown(MF_SINK_WRITER_D3D_MANAGER, d3dManager); if (FAILED(hr)) { - std::cerr << "ERROR: MFCreateAttributes(sink writer) failed (hr=0x" + std::cerr << "ERROR: Set MF_SINK_WRITER_D3D_MANAGER failed (hr=0x" << std::hex << hr << std::dec << ")" << std::endl; - failedStage = SinkWriterCreateStage::CreateAttributes; + failedStage = SinkWriterCreateStage::ConfigureD3DManager; return hr; } - hr = attributes->SetUINT32(MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, FALSE); + } + + if (asyncCallback != nullptr) { + hr = attributes->SetUnknown(MF_SINK_WRITER_ASYNC_CALLBACK, asyncCallback); if (FAILED(hr)) { - std::cerr << "ERROR: Set MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS failed (hr=0x" + std::cerr << "ERROR: Set MF_SINK_WRITER_ASYNC_CALLBACK failed (hr=0x" << std::hex << hr << std::dec << ")" << std::endl; - failedStage = SinkWriterCreateStage::DisableHardwareTransforms; + failedStage = SinkWriterCreateStage::ConfigureAsyncCallback; return hr; } } + // Sink writers do not use hardware encoders by default. Make the normal + // path explicitly hardware-enabled and the software path explicitly + // hardware-disabled. + hr = attributes->SetUINT32( + MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, + forceSoftwareEncoder ? FALSE : TRUE); + if (FAILED(hr)) { + std::cerr << "ERROR: Set MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS failed (hr=0x" + << std::hex << hr << std::dec << ")" << std::endl; + failedStage = SinkWriterCreateStage::ConfigureHardwareTransforms; + return hr; + } + failedStage = SinkWriterCreateStage::CreateSinkWriter; if ( !forceSoftwareEncoder && @@ -242,6 +327,63 @@ HRESULT createSinkWriterFromUrl( return sinkWriterHr; } +const char* detectDefaultVideoEncoderSelection( + IMFSinkWriter* sinkWriter, + DWORD videoStreamIndex) { + if (sinkWriter == nullptr) { + return kVideoEncoderSelectionDefault; + } + + Microsoft::WRL::ComPtr sinkWriterEx; + const HRESULT queryHr = sinkWriter->QueryInterface(IID_PPV_ARGS(&sinkWriterEx)); + if (FAILED(queryHr)) { + std::cerr + << "WARNING: IMFSinkWriterEx is unavailable; the active H.264 encoder type is unknown " + << "(hr=0x" << std::hex << queryHr << std::dec << ")." + << std::endl; + return kVideoEncoderSelectionDefault; + } + + for (DWORD transformIndex = 0;; transformIndex += 1) { + GUID category = GUID_NULL; + Microsoft::WRL::ComPtr transform; + const HRESULT transformHr = sinkWriterEx->GetTransformForStream( + videoStreamIndex, + transformIndex, + &category, + &transform); + if (FAILED(transformHr)) { + break; + } + if (!IsEqualGUID(category, MFT_CATEGORY_VIDEO_ENCODER) || !transform) { + continue; + } + + Microsoft::WRL::ComPtr transformAttributes; + if (SUCCEEDED(transform->GetAttributes(&transformAttributes)) && transformAttributes) { + UINT32 hardwareUrlLength = 0; + if (SUCCEEDED(transformAttributes->GetStringLength( + MFT_ENUM_HARDWARE_URL_Attribute, + &hardwareUrlLength))) { + std::cerr << "INFO: Media Foundation selected a hardware H.264 encoder." + << std::endl; + return kVideoEncoderSelectionHardware; + } + } + + std::cerr << "WARNING: Media Foundation selected a software H.264 encoder " + << "even though hardware transforms were enabled." + << std::endl; + return kVideoEncoderSelectionSoftwareDefault; + } + + std::cerr + << "WARNING: The sink writer did not expose its H.264 encoder transform; " + << "the active encoder type is unknown." + << std::endl; + return kVideoEncoderSelectionDefault; +} + void logSinkWriterCreateFailure(HRESULT sinkWriterHr, const AudioInputFormat* audioFormat) { const UINT32 h264EncoderCount = countRegisteredH264VideoEncoders(); const UINT32 aacEncoderCount = (audioFormat != nullptr) @@ -324,6 +466,23 @@ void compositeWebcam(BYTE* destination, int width, int height, const BgraFrameVi } // namespace +struct MFEncoder::GpuFramePipeline { + struct Slot { + Microsoft::WRL::ComPtr texture; + uintptr_t completionMarker = 0; + }; + + Microsoft::WRL::ComPtr deviceManager; + Microsoft::WRL::ComPtr callback; + std::vector slots; + size_t nextSlot = 0; + uintptr_t nextMarker = 1; + bool enabled = false; + bool asyncWriter = false; +}; + +MFEncoder::MFEncoder() = default; + MFEncoder::~MFEncoder() { finalize(); } @@ -332,6 +491,91 @@ const char* MFEncoder::videoEncoderSelection() const { return videoEncoderSelection_; } +const char* MFEncoder::videoFrameTransport() const { + return videoFrameTransport_; +} + +uint64_t MFEncoder::gpuFramesWritten() const { + return gpuFramesWritten_; +} + +uint64_t MFEncoder::cpuFramesWritten() const { + return cpuFramesWritten_; +} + +bool MFEncoder::initializeGpuFramePipeline() { + constexpr size_t kTextureRingSize = 4; + + if (!device_ || !context_) { + std::cerr << "WARNING: GPU frame transport is unavailable because the D3D11 device is missing." + << std::endl; + return false; + } + + auto pipeline = std::make_unique(); + pipeline->callback = Microsoft::WRL::Make(); + if (!pipeline->callback) { + std::cerr << "WARNING: Failed to allocate the asynchronous sink-writer callback." + << std::endl; + return false; + } + + UINT resetToken = 0; + HRESULT hr = MFCreateDXGIDeviceManager(&resetToken, &pipeline->deviceManager); + if (FAILED(hr)) { + std::cerr << "WARNING: MFCreateDXGIDeviceManager failed (hr=0x" + << std::hex << hr << std::dec << ")." << std::endl; + return false; + } + hr = pipeline->deviceManager->ResetDevice(device_.Get(), resetToken); + if (FAILED(hr)) { + std::cerr << "WARNING: IMFDXGIDeviceManager::ResetDevice failed (hr=0x" + << std::hex << hr << std::dec << ")." << std::endl; + return false; + } + + D3D11_TEXTURE2D_DESC desc{}; + desc.Width = static_cast(width_); + desc.Height = static_cast(height_); + desc.MipLevels = 1; + desc.ArraySize = 1; + desc.Format = DXGI_FORMAT_B8G8R8A8_UNORM; + desc.SampleDesc.Count = 1; + desc.Usage = D3D11_USAGE_DEFAULT; + desc.BindFlags = 0; + desc.CPUAccessFlags = 0; + desc.MiscFlags = 0; + + pipeline->slots.resize(kTextureRingSize); + for (auto& slot : pipeline->slots) { + hr = device_->CreateTexture2D(&desc, nullptr, &slot.texture); + if (FAILED(hr)) { + std::cerr << "WARNING: CreateTexture2D(GPU encoder ring) failed (hr=0x" + << std::hex << hr << std::dec << ")." << std::endl; + return false; + } + } + + pipeline->enabled = true; + pipeline->asyncWriter = true; + gpuFramePipeline_ = std::move(pipeline); + videoFrameTransport_ = kVideoFrameTransportGpuZeroCopy; + return true; +} + +void MFEncoder::disableGpuFrameTransport(const char* reason, HRESULT hr) { + if (!gpuFramePipeline_ || !gpuFramePipeline_->enabled) { + videoFrameTransport_ = kVideoFrameTransportCpuReadback; + return; + } + + gpuFramePipeline_->enabled = false; + videoFrameTransport_ = kVideoFrameTransportCpuReadback; + std::cerr << "WARNING: frame-path-fallback from=gpu-zero-copy to=cpu-readback reason=" + << (reason ? reason : "unknown") << " hr=0x" + << std::hex << hr << std::dec << std::endl; +} + bool MFEncoder::initialize( const std::wstring& outputPath, int width, @@ -347,7 +591,17 @@ bool MFEncoder::initialize( fps_ = std::max(1, fps); device_ = device; context_ = context; + stagingTexture_.Reset(); + gpuFramePipeline_.reset(); + firstTimestampHns_ = -1; + lastTimestampHns_ = -1; + finalized_ = false; videoEncoderSelection_ = kVideoEncoderSelectionDefault; + videoFrameTransport_ = kVideoFrameTransportCpuReadback; + gpuFramesWritten_ = 0; + cpuFramesWritten_ = 0; + injectGpuFrameFailureOnce_ = options.injectGpuFrameFailureOnce; + injectedGpuFrameFailure_ = false; if (!succeeded(MFStartup(MF_VERSION), "MFStartup")) { return false; @@ -381,24 +635,33 @@ bool MFEncoder::initialize( auto resetSinkWriterAttempt = [&]() { sinkWriter_.Reset(); + gpuFramePipeline_.reset(); videoStreamIndex_ = 0; audioStreamIndex_ = 0; hasAudioStream_ = false; videoEncoderSelection_ = kVideoEncoderSelectionDefault; + videoFrameTransport_ = kVideoFrameTransportCpuReadback; }; auto configureSinkWriterAttempt = [&, audioFormat]( bool forceSoftwareEncoder, const char* selection, - bool logCreateFailure) { + bool logCreateFailure, + bool enableGpuFrameTransport) { resetSinkWriterAttempt(); + if (enableGpuFrameTransport && !initializeGpuFramePipeline()) { + return false; + } + SinkWriterCreateStage failedStage = SinkWriterCreateStage::CreateSinkWriter; const HRESULT sinkWriterHr = createSinkWriterFromUrl( outputPath, forceSoftwareEncoder, options.injectDefaultSinkWriterFailureOnce, injectedDefaultSinkWriterFailure, + gpuFramePipeline_ ? gpuFramePipeline_->deviceManager.Get() : nullptr, + gpuFramePipeline_ ? gpuFramePipeline_->callback.Get() : nullptr, sinkWriter_, failedStage); if (FAILED(sinkWriterHr)) { @@ -434,7 +697,14 @@ bool MFEncoder::initialize( return false; } - videoEncoderSelection_ = selection; + videoEncoderSelection_ = forceSoftwareEncoder + ? selection + : detectDefaultVideoEncoderSelection(sinkWriter_.Get(), videoStreamIndex_); + if ( + enableGpuFrameTransport && + videoEncoderSelection_ != kVideoEncoderSelectionHardware) { + disableGpuFrameTransport("non-hardware-encoder", S_OK); + } return true; }; @@ -442,10 +712,33 @@ bool MFEncoder::initialize( return configureSinkWriterAttempt( true, kVideoEncoderSelectionSoftwarePreferred, - true); + true, + false); + } + + if ( + options.allowGpuFrameTransport && + !options.injectDefaultSinkWriterFailureOnce && + configureSinkWriterAttempt( + false, + kVideoEncoderSelectionDefault, + false, + true)) { + return true; + } + + if (options.allowGpuFrameTransport && !options.injectDefaultSinkWriterFailureOnce) { + std::cerr + << "WARNING: GPU-backed Media Foundation setup failed; " + << "retrying the hardware encoder with CPU frame transport." + << std::endl; } - if (configureSinkWriterAttempt(false, kVideoEncoderSelectionDefault, false)) { + if (configureSinkWriterAttempt( + false, + kVideoEncoderSelectionDefault, + false, + false)) { return true; } @@ -456,7 +749,8 @@ bool MFEncoder::initialize( return configureSinkWriterAttempt( true, kVideoEncoderSelectionSoftwareFallback, - true); + true, + false); } bool MFEncoder::configureAudioStream(const AudioInputFormat& audioFormat) { @@ -603,23 +897,141 @@ bool MFEncoder::copyBgraFrameToBuffer(const BgraFrameView& frame, BYTE* destinat return true; } -bool MFEncoder::writeFrame(ID3D11Texture2D* texture, int64_t timestampHns, const BgraFrameView* webcamFrame) { - std::scoped_lock writerLock(writerMutex_); - if (!sinkWriter_ || finalized_) { +bool MFEncoder::writeGpuFrame( + ID3D11Texture2D* texture, + int64_t sampleTime, + int64_t sampleDuration, + bool& retryWithCpu) { + constexpr auto kSlotWaitTimeout = std::chrono::seconds(2); + retryWithCpu = false; + + if ( + texture == nullptr || + !gpuFramePipeline_ || + !gpuFramePipeline_->enabled || + gpuFramePipeline_->slots.empty()) { + retryWithCpu = true; return false; } - if (firstTimestampHns_ < 0) { - firstTimestampHns_ = timestampHns; + auto& pipeline = *gpuFramePipeline_; + if (injectGpuFrameFailureOnce_ && !injectedGpuFrameFailure_) { + injectedGpuFrameFailure_ = true; + std::cerr + << "TEST-ONLY: Injected GPU frame transport failure; injection consumed exactly once." + << std::endl; + disableGpuFrameTransport("test-injected-gpu-frame-failure", E_FAIL); + retryWithCpu = true; + return false; } - int64_t sampleTime = timestampHns - firstTimestampHns_; - if (sampleTime <= lastTimestampHns_) { - sampleTime = lastTimestampHns_ + (10'000'000LL / fps_); + auto& slot = pipeline.slots[pipeline.nextSlot]; + if ( + slot.completionMarker != 0 && + !pipeline.callback->waitForMarker(slot.completionMarker, kSlotWaitTimeout)) { + disableGpuFrameTransport("texture-ring-timeout", HRESULT_FROM_WIN32(WAIT_TIMEOUT)); + retryWithCpu = true; + return false; } - const int64_t sampleDuration = 10'000'000LL / fps_; - lastTimestampHns_ = sampleTime; + + D3D11_TEXTURE2D_DESC sourceDesc{}; + texture->GetDesc(&sourceDesc); + if ( + sourceDesc.Width != static_cast(width_) || + sourceDesc.Height != static_cast(height_) || + sourceDesc.Format != DXGI_FORMAT_B8G8R8A8_UNORM) { + disableGpuFrameTransport("unexpected-source-texture", E_INVALIDARG); + retryWithCpu = true; + return false; + } + + context_->CopyResource(slot.texture.Get(), texture); + Microsoft::WRL::ComPtr buffer; + HRESULT hr = MFCreateDXGISurfaceBuffer( + __uuidof(ID3D11Texture2D), + slot.texture.Get(), + 0, + FALSE, + &buffer); + if (FAILED(hr)) { + disableGpuFrameTransport("MFCreateDXGISurfaceBuffer", hr); + retryWithCpu = true; + return false; + } + + Microsoft::WRL::ComPtr buffer2d; + DWORD contiguousLength = 0; + hr = buffer.As(&buffer2d); + if (SUCCEEDED(hr)) { + hr = buffer2d->GetContiguousLength(&contiguousLength); + } + if (SUCCEEDED(hr)) { + hr = buffer->SetCurrentLength(contiguousLength); + } + if (FAILED(hr) || contiguousLength == 0) { + disableGpuFrameTransport("IMF2DBuffer::GetContiguousLength", FAILED(hr) ? hr : E_UNEXPECTED); + retryWithCpu = true; + return false; + } + + Microsoft::WRL::ComPtr sample; + hr = MFCreateSample(&sample); + if (FAILED(hr)) { + disableGpuFrameTransport("MFCreateSample", hr); + retryWithCpu = true; + return false; + } + hr = sample->AddBuffer(buffer.Get()); + if (FAILED(hr)) { + disableGpuFrameTransport("IMFSample::AddBuffer", hr); + retryWithCpu = true; + return false; + } + hr = sample->SetSampleTime(sampleTime); + if (FAILED(hr)) { + disableGpuFrameTransport("IMFSample::SetSampleTime", hr); + retryWithCpu = true; + return false; + } + hr = sample->SetSampleDuration(sampleDuration); + if (FAILED(hr)) { + disableGpuFrameTransport("IMFSample::SetSampleDuration", hr); + retryWithCpu = true; + return false; + } + + hr = sinkWriter_->WriteSample(videoStreamIndex_, sample.Get()); + if (FAILED(hr)) { + std::cerr << "ERROR: WriteSample(DXGI) failed (hr=0x" + << std::hex << hr << std::dec << ")" << std::endl; + return false; + } + + const uintptr_t marker = pipeline.nextMarker++; + hr = sinkWriter_->PlaceMarker( + videoStreamIndex_, + reinterpret_cast(marker)); + if (FAILED(hr)) { + // The frame was already accepted. Keep every ring texture alive until + // Finalize, but stop reusing them and send subsequent frames through + // the CPU path. + gpuFramesWritten_ += 1; + disableGpuFrameTransport("IMFSinkWriter::PlaceMarker", hr); + return true; + } + + slot.completionMarker = marker; + pipeline.nextSlot = (pipeline.nextSlot + 1) % pipeline.slots.size(); + gpuFramesWritten_ += 1; + return true; +} + +bool MFEncoder::writeCpuFrame( + ID3D11Texture2D* texture, + int64_t sampleTime, + int64_t sampleDuration, + const BgraFrameView* webcamFrame) { Microsoft::WRL::ComPtr buffer; const DWORD frameBytes = static_cast(width_ * height_ * 4); if (!succeeded(MFCreateMemoryBuffer(frameBytes, &buffer), "MFCreateMemoryBuffer")) { @@ -648,7 +1060,41 @@ bool MFEncoder::writeFrame(ID3D11Texture2D* texture, int64_t timestampHns, const sample->SetSampleTime(sampleTime); sample->SetSampleDuration(sampleDuration); - return succeeded(sinkWriter_->WriteSample(videoStreamIndex_, sample.Get()), "WriteSample"); + if (!succeeded(sinkWriter_->WriteSample(videoStreamIndex_, sample.Get()), "WriteSample")) { + return false; + } + cpuFramesWritten_ += 1; + return true; +} + +bool MFEncoder::writeFrame(ID3D11Texture2D* texture, int64_t timestampHns, const BgraFrameView* webcamFrame) { + std::scoped_lock writerLock(writerMutex_); + if (!sinkWriter_ || finalized_ || texture == nullptr) { + return false; + } + + if (firstTimestampHns_ < 0) { + firstTimestampHns_ = timestampHns; + } + + int64_t sampleTime = timestampHns - firstTimestampHns_; + if (sampleTime <= lastTimestampHns_) { + sampleTime = lastTimestampHns_ + (10'000'000LL / fps_); + } + const int64_t sampleDuration = 10'000'000LL / fps_; + lastTimestampHns_ = sampleTime; + + if (webcamFrame == nullptr && gpuFramePipeline_ && gpuFramePipeline_->enabled) { + bool retryWithCpu = false; + if (writeGpuFrame(texture, sampleTime, sampleDuration, retryWithCpu)) { + return true; + } + if (!retryWithCpu) { + return false; + } + } + + return writeCpuFrame(texture, sampleTime, sampleDuration, webcamFrame); } bool MFEncoder::writeBgraFrame(const BgraFrameView& frame, int64_t timestampHns) { @@ -696,7 +1142,11 @@ bool MFEncoder::writeBgraFrame(const BgraFrameView& frame, int64_t timestampHns) sample->SetSampleTime(sampleTime); sample->SetSampleDuration(sampleDuration); - return succeeded(sinkWriter_->WriteSample(videoStreamIndex_, sample.Get()), "WriteSample(webcam)"); + if (!succeeded(sinkWriter_->WriteSample(videoStreamIndex_, sample.Get()), "WriteSample(webcam)")) { + return false; + } + cpuFramesWritten_ += 1; + return true; } bool MFEncoder::writeAudio(const BYTE* data, DWORD byteCount, int64_t timestampHns, int64_t durationHns) { @@ -749,9 +1199,30 @@ bool MFEncoder::finalize() { finalized_ = true; bool ok = true; if (sinkWriter_) { - ok = succeeded(sinkWriter_->Finalize(), "SinkWriter::Finalize"); + const HRESULT finalizeHr = sinkWriter_->Finalize(); + ok = succeeded(finalizeHr, "SinkWriter::Finalize"); + if ( + SUCCEEDED(finalizeHr) && + gpuFramePipeline_ && + gpuFramePipeline_->asyncWriter && + gpuFramePipeline_->callback) { + HRESULT callbackStatus = E_PENDING; + if (!gpuFramePipeline_->callback->waitForFinalize( + std::chrono::seconds(30), + callbackStatus)) { + std::cerr << "ERROR: Timed out waiting for asynchronous sink-writer finalization." + << std::endl; + ok = false; + } else if (!succeeded(callbackStatus, "SinkWriter::OnFinalize")) { + ok = false; + } + } sinkWriter_.Reset(); } + std::cerr << "INFO: frame-path-summary transport=" << videoFrameTransport_ + << " gpuFrames=" << gpuFramesWritten_ + << " cpuFrames=" << cpuFramesWritten_ << std::endl; + gpuFramePipeline_.reset(); stagingTexture_.Reset(); context_.Reset(); device_.Reset(); diff --git a/electron/native/wgc-capture/src/mf_encoder.h b/electron/native/wgc-capture/src/mf_encoder.h index 8f5787481..f6598714a 100644 --- a/electron/native/wgc-capture/src/mf_encoder.h +++ b/electron/native/wgc-capture/src/mf_encoder.h @@ -8,6 +8,7 @@ #include #include +#include #include #include @@ -29,15 +30,21 @@ struct AudioInputFormat { struct MFEncoderOptions { bool preferSoftwareEncoder = false; bool injectDefaultSinkWriterFailureOnce = false; + bool injectGpuFrameFailureOnce = false; + bool allowGpuFrameTransport = true; }; constexpr const char* kVideoEncoderSelectionDefault = "default"; +constexpr const char* kVideoEncoderSelectionHardware = "hardware"; +constexpr const char* kVideoEncoderSelectionSoftwareDefault = "software-default"; constexpr const char* kVideoEncoderSelectionSoftwarePreferred = "software-preferred"; constexpr const char* kVideoEncoderSelectionSoftwareFallback = "software-fallback"; +constexpr const char* kVideoFrameTransportGpuZeroCopy = "gpu-zero-copy"; +constexpr const char* kVideoFrameTransportCpuReadback = "cpu-readback"; class MFEncoder { public: - MFEncoder() = default; + MFEncoder(); ~MFEncoder(); MFEncoder(const MFEncoder&) = delete; @@ -58,8 +65,25 @@ class MFEncoder { bool writeAudio(const BYTE* data, DWORD byteCount, int64_t timestampHns, int64_t durationHns); bool finalize(); const char* videoEncoderSelection() const; + const char* videoFrameTransport() const; + uint64_t gpuFramesWritten() const; + uint64_t cpuFramesWritten() const; private: + struct GpuFramePipeline; + + bool initializeGpuFramePipeline(); + bool writeGpuFrame( + ID3D11Texture2D* texture, + int64_t sampleTime, + int64_t sampleDuration, + bool& retryWithCpu); + bool writeCpuFrame( + ID3D11Texture2D* texture, + int64_t sampleTime, + int64_t sampleDuration, + const BgraFrameView* webcamFrame); + void disableGpuFrameTransport(const char* reason, HRESULT hr); bool ensureStagingTexture(ID3D11Texture2D* texture); bool copyFrameToBuffer( ID3D11Texture2D* texture, @@ -73,6 +97,7 @@ class MFEncoder { Microsoft::WRL::ComPtr device_; Microsoft::WRL::ComPtr context_; Microsoft::WRL::ComPtr stagingTexture_; + std::unique_ptr gpuFramePipeline_; std::mutex writerMutex_; DWORD videoStreamIndex_ = 0; DWORD audioStreamIndex_ = 0; @@ -84,4 +109,9 @@ class MFEncoder { int64_t lastTimestampHns_ = -1; bool finalized_ = false; const char* videoEncoderSelection_ = kVideoEncoderSelectionDefault; + const char* videoFrameTransport_ = kVideoFrameTransportCpuReadback; + uint64_t gpuFramesWritten_ = 0; + uint64_t cpuFramesWritten_ = 0; + bool injectGpuFrameFailureOnce_ = false; + bool injectedGpuFrameFailure_ = false; }; diff --git a/scripts/test-windows-wgc-helper.mjs b/scripts/test-windows-wgc-helper.mjs index c6c69441e..8b274ed27 100644 --- a/scripts/test-windows-wgc-helper.mjs +++ b/scripts/test-windows-wgc-helper.mjs @@ -12,6 +12,12 @@ const HELPER_PATH = path.join(ROOT, "electron", "native", "bin", "win32-x64", "wgc-capture.exe"); const DURATION_MS = Number(process.env.OPENSCREEN_WGC_TEST_DURATION_MS ?? 5000); +const CAPTURE_FPS = Number(process.env.OPENSCREEN_WGC_TEST_FPS ?? 30); +const REQUIRE_GPU_ZERO_COPY = process.env.OPENSCREEN_WGC_TEST_REQUIRE_GPU_ZERO_COPY === "true"; +const DISPLAY_X = Number(process.env.OPENSCREEN_WGC_TEST_DISPLAY_X ?? 0); +const DISPLAY_Y = Number(process.env.OPENSCREEN_WGC_TEST_DISPLAY_Y ?? 0); +const DISPLAY_WIDTH = Number(process.env.OPENSCREEN_WGC_TEST_DISPLAY_WIDTH ?? 1920); +const DISPLAY_HEIGHT = Number(process.env.OPENSCREEN_WGC_TEST_DISPLAY_HEIGHT ?? 1080); const WITH_SYSTEM_AUDIO = process.env.OPENSCREEN_WGC_TEST_SYSTEM_AUDIO === "true" || process.argv.includes("--system-audio"); @@ -32,21 +38,52 @@ const WITH_SOFTWARE_ENCODER = const WITH_SOFTWARE_FALLBACK = process.env.OPENSCREEN_WGC_TEST_SOFTWARE_FALLBACK === "true" || process.argv.includes("--software-fallback"); +const WITH_GPU_FRAME_FALLBACK = + process.env.OPENSCREEN_WGC_TEST_GPU_FRAME_FALLBACK === "true" || + process.argv.includes("--gpu-frame-fallback"); const INJECT_DEFAULT_SINK_WRITER_FAILURE_ENV = "OPENSCREEN_WGC_TEST_INJECT_DEFAULT_SINK_WRITER_FAILURE_ONCE"; +const INJECT_GPU_FRAME_FAILURE_ENV = "OPENSCREEN_WGC_TEST_INJECT_GPU_FRAME_FAILURE_ONCE"; const INJECTION_MARKER = "TEST-ONLY: Injected default MFCreateSinkWriterFromURL failure"; +const GPU_FRAME_INJECTION_MARKER = "TEST-ONLY: Injected GPU frame transport failure"; if (WITH_SOFTWARE_ENCODER && WITH_SOFTWARE_FALLBACK) { throw new Error("--software-encoder and --software-fallback are mutually exclusive"); } +if (WITH_GPU_FRAME_FALLBACK && (WITH_SOFTWARE_ENCODER || WITH_SOFTWARE_FALLBACK)) { + throw new Error("--gpu-frame-fallback cannot be combined with a software encoder test"); +} +if (!Number.isInteger(CAPTURE_FPS) || CAPTURE_FPS < 1 || CAPTURE_FPS > 240) { + throw new Error(`OPENSCREEN_WGC_TEST_FPS must be an integer from 1 to 240, got ${CAPTURE_FPS}`); +} +if ( + ![DISPLAY_X, DISPLAY_Y, DISPLAY_WIDTH, DISPLAY_HEIGHT].every(Number.isInteger) || + DISPLAY_WIDTH <= 0 || + DISPLAY_HEIGHT <= 0 +) { + throw new Error("WGC test display bounds must be integers with a positive width and height"); +} +if (REQUIRE_GPU_ZERO_COPY && (WITH_SOFTWARE_ENCODER || WITH_SOFTWARE_FALLBACK)) { + throw new Error("GPU zero-copy cannot be required with a software encoder test"); +} +if (REQUIRE_GPU_ZERO_COPY && WITH_GPU_FRAME_FALLBACK) { + throw new Error("GPU zero-copy cannot be required while forcing a GPU frame fallback"); +} -function runHelper(config, { injectDefaultSinkWriterFailure = false } = {}) { +function runHelper( + config, + { injectDefaultSinkWriterFailure = false, injectGpuFrameFailure = false } = {}, +) { return new Promise((resolve, reject) => { const env = { ...process.env }; delete env[INJECT_DEFAULT_SINK_WRITER_FAILURE_ENV]; + delete env[INJECT_GPU_FRAME_FAILURE_ENV]; if (injectDefaultSinkWriterFailure) { env[INJECT_DEFAULT_SINK_WRITER_FAILURE_ENV] = "1"; } + if (injectGpuFrameFailure) { + env[INJECT_GPU_FRAME_FAILURE_ENV] = "1"; + } const child = spawn(HELPER_PATH, [JSON.stringify(config)], { env, stdio: ["pipe", "pipe", "pipe"], @@ -261,13 +298,13 @@ const config = { sourceType: fixtureWindow ? "window" : "display", sourceId: fixtureWindow ? fixtureWindow.sourceId : "screen:0:0", displayId: 0, - fps: 30, + fps: CAPTURE_FPS, videoWidth: 1280, videoHeight: 720, - displayX: 0, - displayY: 0, - displayW: 1920, - displayH: 1080, + displayX: DISPLAY_X, + displayY: DISPLAY_Y, + displayW: DISPLAY_WIDTH, + displayH: DISPLAY_HEIGHT, hasDisplayBounds: true, captureSystemAudio: WITH_SYSTEM_AUDIO, captureMic: WITH_MICROPHONE, @@ -294,6 +331,7 @@ let result; try { result = await runHelper(config, { injectDefaultSinkWriterFailure: WITH_SOFTWARE_FALLBACK, + injectGpuFrameFailure: WITH_GPU_FRAME_FALLBACK, }); } finally { if (fixtureWindow) { @@ -341,6 +379,10 @@ const encoderSelectionLine = result.stdout .split(/\r?\n/) .find((line) => line.includes('"event":"encoder-selection"')); const encoderSelection = encoderSelectionLine ? JSON.parse(encoderSelectionLine) : null; +const encoderSummaryLine = result.stdout + .split(/\r?\n/) + .find((line) => line.includes('"event":"encoder-summary"')); +const encoderSummary = encoderSummaryLine ? JSON.parse(encoderSummaryLine) : null; const nativeWebcamDiagnostics = result.stderr .split(/\r?\n/) .filter((line) => line.includes("Native webcam candidate")); @@ -386,23 +428,80 @@ if ( `WGC helper did not apply requested cursor capture mode (${CAPTURE_CURSOR}): ${result.stdout}`, ); } -const expectedEncoderSelection = WITH_SOFTWARE_FALLBACK - ? "software-fallback" +const expectedEncoderSelections = WITH_SOFTWARE_FALLBACK + ? ["software-fallback"] : WITH_SOFTWARE_ENCODER - ? "software-preferred" - : "default"; + ? ["software-preferred"] + : ["hardware", "software-default", "default"]; if ( - encoderSelection?.video !== expectedEncoderSelection || + !expectedEncoderSelections.includes(encoderSelection?.video) || encoderSelection.preferSoftwareEncoder !== WITH_SOFTWARE_ENCODER ) { throw new Error( - `WGC helper encoder selection was ${JSON.stringify(encoderSelection)}, expected ${expectedEncoderSelection} with preferSoftwareEncoder=${WITH_SOFTWARE_ENCODER}: ${result.stdout}`, + `WGC helper encoder selection was ${JSON.stringify(encoderSelection)}, expected one of ${expectedEncoderSelections.join(", ")} with preferSoftwareEncoder=${WITH_SOFTWARE_ENCODER}: ${result.stdout}`, + ); +} +if (!["gpu-zero-copy", "cpu-readback"].includes(encoderSelection?.frameTransport)) { + throw new Error( + `WGC helper reported an invalid frame transport: ${JSON.stringify(encoderSelection)}`, + ); +} +if ( + !encoderSummary || + !["gpu-zero-copy", "cpu-readback"].includes(encoderSummary.frameTransport) || + !Number.isInteger(encoderSummary.gpuFrames) || + !Number.isInteger(encoderSummary.cpuFrames) || + !Number.isInteger(encoderSummary.elapsedMs) || + !Number.isInteger(encoderSummary.processCpuTimeMs) || + encoderSummary.elapsedMs <= 0 || + encoderSummary.processCpuTimeMs < 0 || + encoderSummary.gpuFrames + encoderSummary.cpuFrames <= 0 +) { + throw new Error(`WGC helper encoder summary is missing or invalid: ${result.stdout}`); +} +if ( + (WITH_SOFTWARE_ENCODER || WITH_SOFTWARE_FALLBACK) && + (encoderSelection.frameTransport !== "cpu-readback" || + encoderSummary.frameTransport !== "cpu-readback" || + encoderSummary.gpuFrames !== 0 || + encoderSummary.cpuFrames <= 0) +) { + throw new Error( + `Software encoder test unexpectedly used GPU frame transport: selection=${JSON.stringify(encoderSelection)} summary=${JSON.stringify(encoderSummary)}`, ); } +if ( + WITH_GPU_FRAME_FALLBACK && + (encoderSelection.video !== "hardware" || + encoderSelection.frameTransport !== "gpu-zero-copy" || + encoderSummary.frameTransport !== "cpu-readback" || + encoderSummary.gpuFrames !== 0 || + encoderSummary.cpuFrames <= 0) +) { + throw new Error( + `Forced GPU frame fallback did not switch cleanly to CPU transport: selection=${JSON.stringify(encoderSelection)} summary=${JSON.stringify(encoderSummary)}`, + ); +} +if (REQUIRE_GPU_ZERO_COPY) { + if ( + encoderSelection.video !== "hardware" || + encoderSelection.frameTransport !== "gpu-zero-copy" || + encoderSummary.frameTransport !== "gpu-zero-copy" || + encoderSummary.gpuFrames <= 0 || + encoderSummary.cpuFrames !== 0 + ) { + throw new Error( + `WGC helper did not remain on GPU zero-copy: selection=${JSON.stringify(encoderSelection)} summary=${JSON.stringify(encoderSummary)}\n${result.stdout}\n${result.stderr}`, + ); + } +} const combinedHelperOutput = `${result.stdout}\n${result.stderr}`; const helperDiagnosticLines = combinedHelperOutput.split(/\r?\n/).filter(Boolean); const injectionLines = helperDiagnosticLines.filter((line) => line.includes(INJECTION_MARKER)); +const gpuFrameInjectionLines = helperDiagnosticLines.filter((line) => + line.includes(GPU_FRAME_INJECTION_MARKER), +); const fallbackDiagnosticPatterns = [ INJECTION_MARKER, "WARNING: Default MFCreateSinkWriterFromURL failed (hr=0x80070003)", @@ -437,6 +536,26 @@ if (WITH_SOFTWARE_FALLBACK) { `WGC helper unexpectedly injected a default sink-writer failure: ${combinedHelperOutput}`, ); } +if (WITH_GPU_FRAME_FALLBACK) { + if ( + gpuFrameInjectionLines.length !== 1 || + !gpuFrameInjectionLines[0].includes("consumed exactly once") || + !helperDiagnosticLines.some((line) => + line.includes( + "frame-path-fallback from=gpu-zero-copy to=cpu-readback reason=test-injected-gpu-frame-failure", + ), + ) + ) { + throw new Error( + `Expected one GPU frame transport injection and fallback diagnostic: ${combinedHelperOutput}`, + ); + } +} else if (gpuFrameInjectionLines.length !== 0) { + throw new Error(`WGC helper unexpectedly injected a GPU frame failure: ${combinedHelperOutput}`); +} +if (REQUIRE_GPU_ZERO_COPY && combinedHelperOutput.includes("frame-path-fallback")) { + throw new Error(`WGC helper fell back from GPU zero-copy: ${combinedHelperOutput}`); +} if ((WITH_SYSTEM_AUDIO || WITH_MICROPHONE) && !hasAudio) { throw new Error(`WGC helper output has no audio stream: ${outputPath}`); } @@ -462,6 +581,9 @@ console.log( index: stream.index, codecType: stream.codec_type, codecName: stream.codec_name, + width: stream.width, + height: stream.height, + averageFrameRate: stream.avg_frame_rate, duration: stream.duration, })), webcamStreams: webcamStreams.map((stream) => ({ @@ -474,6 +596,7 @@ console.log( })), cursorCapture, encoderSelection, + encoderSummary, selectedMicrophoneDeviceName: audioFormat?.microphoneDeviceName, selectedWebcamDeviceName: webcamFormat?.deviceName, nativeMicrophoneDiagnostics, diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index af95b4722..7ac84cb6b 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -895,10 +895,10 @@ export function useScreenRecorder(): UseScreenRecorderReturn { throw new Error(result.error ?? "Native Windows capture failed."); } - // Tell the user when the helper silently switched away from the default - // GPU encoder; an explicit software-preferred selection needs no notice. + // Tell the user whenever the normal hardware-enabled path ended up on + // software; an explicit software-preferred selection needs no notice. setSoftwareEncoderFallbackNoticeVisible( - result.videoEncoderSelection === "software-fallback" && + ["software-default", "software-fallback"].includes(result.videoEncoderSelection ?? "") && !loadUserPreferences().hideSoftwareEncoderFallbackNotice, ); diff --git a/src/lib/nativeWindowsRecording.ts b/src/lib/nativeWindowsRecording.ts index 313eff9b5..415b9f2ab 100644 --- a/src/lib/nativeWindowsRecording.ts +++ b/src/lib/nativeWindowsRecording.ts @@ -45,8 +45,10 @@ export type NativeWindowsRecordingStartResult = { path?: string; helperPath?: string; error?: string; - /** Helper-reported encoder selection: "default", "software-preferred", or "software-fallback". */ + /** Helper-reported encoder selection: hardware, software-default/preferred/fallback, or default when detection is unavailable. */ videoEncoderSelection?: string | null; + /** Helper-reported frame transport for Windows capture diagnostics. */ + videoFrameTransport?: "gpu-zero-copy" | "cpu-readback" | null; }; export function parseWindowHandleFromSourceId(sourceId?: string | null) {