From dfe18c4b88a8803e46476f602e835424bf720842 Mon Sep 17 00:00:00 2001 From: John Wallace Date: Mon, 14 Sep 2026 19:51:00 -0400 Subject: [PATCH 1/8] Record continuous participant ISOs with muxed 1080p video and audio --- CLAUDE.md | 25 +- CMakeLists.txt | 17 +- docs/CORE_PLUGIN_FUNCTIONALITY.md | 57 +- ...so-participant-av-validation-2026-09-14.md | 77 ++ src/iso-av-mux.h | 114 ++ src/iso-d3d-conformer.h | 204 +++ src/iso-frame-conform.h | 47 + src/iso-provider-policy.h | 17 + src/iso-track-writer.cpp | 322 +++++ src/iso-track-writer.h | 58 + src/zoom-iso-panel.cpp | 62 +- src/zoom-iso-recorder.cpp | 1161 ++++------------- src/zoom-iso-recorder.h | 133 +- tests/iso-provider-policy-test.cpp | 28 + tests/iso-track-writer-test.cpp | 95 ++ tests/verify-iso-av.py | 56 + 16 files changed, 1349 insertions(+), 1124 deletions(-) create mode 100644 docs/iso-participant-av-validation-2026-09-14.md create mode 100644 src/iso-av-mux.h create mode 100644 src/iso-d3d-conformer.h create mode 100644 src/iso-frame-conform.h create mode 100644 src/iso-provider-policy.h create mode 100644 src/iso-track-writer.cpp create mode 100644 src/iso-track-writer.h create mode 100644 tests/iso-provider-policy-test.cpp create mode 100644 tests/iso-track-writer-test.cpp create mode 100644 tests/verify-iso-av.py diff --git a/CLAUDE.md b/CLAUDE.md index c2f85d7..36d954b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -227,21 +227,16 @@ Every one of these is documented at length where it lives; the list is the map. interval rather than zeroing on fire -- zeroing discards the remainder that pushed a tick over threshold, which at 60 fps lands every 7 frames (~117 ms, ~8.6 Hz) instead of the documented 10 Hz. -- **ISO recording timing** (`src/iso-video-pacer.h`, `src/iso-audio-gap-fill.h`): - raw video has no per-frame timestamps and ffmpeg cannot be trusted to - invent correct ones from a byte stream — `-use_wallclock_as_timestamps` - is confirmed (via `ffprobe -show_frames`, 2026-08-21) to have **no - effect** on this project's ffmpeg build's rawvideo demuxer, despite - looking like the textbook fix. `record_video_frame()` is called 1:1 with - Zoom's real, fluctuating (10-60fps) per-source delivery, so it must pace - itself to a fixed cadence (duplicate to backfill a stall, drop to shed a - burst) BEFORE the pipe — see `iso_video_frames_due()`. Audio has the - mirror-image problem for a different reason: Zoom only calls back audio - for someone currently talking, so `record_audio_frame()` must backfill - silence across every gap (`iso_audio_silence_frames()`) or the WAV - shrinks by every silent stretch. Both anchor to the same - `os_gettime_ns()` clock so video and audio stay in sync with each other, - not just individually correct. +- **ISO participant A/V recording** (`src/iso-track-writer.cpp`, + `src/iso-av-mux.h`): one writer per Zoom participant ID owns a continuous + 1920x1080/30 H.264 + 48 kHz stereo AAC MP4 until Stop. Source UUIDs and input + dimensions MUST NOT key writer lifetime. Workers scale/letterbox and resample, + hold video/fill silence against the same monotonic clock, and mux timestamped + raw media to one FFmpeg pipe. The input is full-range BT.709; preserve its + colour metadata. B-frames are disabled to avoid fragmented-MP4 startup A/V + offset. Keep startup video history while workers catch up: shrinking the + input queue on the first tick loses early frames. Validate decoded flash/tone + timing with `tests/verify-iso-av.py`, not only frame counts or process success. - **Colour range is normalised, never re-declared** (`src/i420-range-expand.h`, applied in `engine/src/engine-video.cpp`'s `onRawDataFrameReceived`): the engine requests `VideoRawdataColorspace_BT709_F` and the plugin declares diff --git a/CMakeLists.txt b/CMakeLists.txt index d56d371..ce1ec46 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -285,6 +285,7 @@ if(COREVIDEO_BUILD_PLUGIN) src/zoom-diagnostics-dialog.cpp src/zoom-output-profile.cpp src/zoom-iso-recorder.cpp + src/iso-track-writer.cpp src/iso-ffmpeg-pipe.cpp src/zoom-iso-panel.cpp src/zoom-dock.cpp @@ -317,7 +318,7 @@ if(COREVIDEO_BUILD_PLUGIN) if(WIN32) # winmm: talkback-cue.cpp's PlaySound(SND_MEMORY | SND_ASYNC) call -- # see that file's header comment for why it was chosen over WASAPI. - target_link_libraries(obs-zoom-plugin PRIVATE crypt32 winmm) + target_link_libraries(obs-zoom-plugin PRIVATE crypt32 winmm d3d11 d3dcompiler) add_executable(CoreVideoOAuthCallback src/oauth-callback-helper.cpp) target_link_libraries(CoreVideoOAuthCallback PRIVATE ws2_32 shell32 ole32) install(TARGETS CoreVideoOAuthCallback @@ -651,9 +652,23 @@ if(BUILD_TESTING) target_include_directories(CoreVideoIsoRecordingTest PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src") + if(COREVIDEO_BUILD_PLUGIN AND WIN32) + add_executable(CoreVideoIsoTrackWriterTest tests/iso-track-writer-test.cpp + src/iso-track-writer.cpp src/iso-ffmpeg-pipe.cpp) + target_include_directories(CoreVideoIsoTrackWriterTest PRIVATE src) + target_link_libraries(CoreVideoIsoTrackWriterTest PRIVATE OBS::libobs) + if(WIN32) + target_compile_definitions(CoreVideoIsoTrackWriterTest PRIVATE NOMINMAX WIN32_LEAN_AND_MEAN) + target_link_libraries(CoreVideoIsoTrackWriterTest PRIVATE d3d11 d3dcompiler) + endif() + endif() + add_executable(CoreVideoIsoEncoderPlanTest tests/iso-encoder-plan-test.cpp ) + add_executable(CoreVideoIsoProviderPolicyTest tests/iso-provider-policy-test.cpp) + target_include_directories(CoreVideoIsoProviderPolicyTest PRIVATE src) + add_test(NAME CoreVideoIsoProviderPolicy COMMAND CoreVideoIsoProviderPolicyTest) target_include_directories(CoreVideoIsoEncoderPlanTest PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/src" ) diff --git a/docs/CORE_PLUGIN_FUNCTIONALITY.md b/docs/CORE_PLUGIN_FUNCTIONALITY.md index 6444168..ebd651b 100644 --- a/docs/CORE_PLUGIN_FUNCTIONALITY.md +++ b/docs/CORE_PLUGIN_FUNCTIONALITY.md @@ -580,15 +580,16 @@ There are no Companion actions for talkback yet. ## Auto ISO Recording -![CoreVideo ISO recording flow](images/iso-recording-flow.svg) - -ISO recording is controlled by the OBS plugin, not the engine. When enabled, +ISO recording is controlled by the OBS plugin. On macOS, configure and test a working FFmpeg executable in **Zoom ISO Recorder** first; the package does not bundle FFmpeg. -CoreVideo records one video file and one PCM WAV audio file per active source -segment. A new segment starts when the resolved participant or source resolution -changes. +CoreVideo records one continuous MP4 per Zoom participant ID during Record/Stop. +Each file contains H.264 video at 1920x1080, 30 fps and AAC stereo audio at 48 kHz. +Incoming video is scaled to fit with black bars when needed. Zoom resolution +changes, duplicate OBS sources, camera gaps, and source reassignment do not +restart that participant's encoder or create another file. A new Zoom ID after +a rejoin is treated as a new participant; display names are not identity keys. Requirements: @@ -614,9 +615,9 @@ dock. The panel provides: - **Start ISO Recording** and **Stop ISO Recording** buttons. - Live status showing idle/recording and active session count. - Active session table with source, participant, resolution, video frame count, - audio chunk count, current video/audio file paths, and FFmpeg error details. + audio chunk count, combined file path, and FFmpeg error details. - Recently completed sessions remain in the table after stop so operators can - confirm completed MP4/WAV outputs before opening the folder. + confirm completed MP4 outputs before opening the folder. The panel uses the same `ZoomIsoRecorder` backend as the TCP and OSC APIs. It persists the output folder, FFmpeg path, and program-recording toggle in OBS @@ -651,33 +652,25 @@ OSC equivalents: Output files are written as: -- `*.mp4` for encoded I420 video through FFmpeg using the selected H.264 - encoder -- `*.wav` for matching PCM audio -- `*.ffmpeg.log` beside them, holding FFmpeg's own account of the session. Read - this first when a file is missing or truncated. +- `*.mp4` containing H.264 video and AAC audio for one participant. +- `*.mp4.ffmpeg.log` containing encoder diagnostics. ### Timing -Both files are paced to real elapsed time against the same clock, so each is -individually accurate and the two stay in sync with each other. - -That is not free, and it is worth knowing why. Raw video carries no per-frame -timestamps, and Zoom's per-source delivery fluctuates between roughly 10 and -60 fps with conditions outside CoreVideo's control - so frames are paced to a -fixed cadence before they reach FFmpeg, duplicating the held frame to backfill a -stall and dropping excess from a burst. Audio has the mirror-image problem for a -different reason: Zoom only calls audio back for someone currently making sound, -so silence is backfilled across every gap or the WAV shrinks by the total silent -duration. Both were wrong before v0.1.42/v0.1.43 - a source averaging 15 fps -recorded under a declared 30 fps finished in about half the real duration, and an -over-eager first gap-fill briefly doubled every WAV. If you see either symptom, -check your version first. - -The hardware encoder fallback chain is NVENC -> QSV -> AMF -> libx264, and it -walks the whole chain: a source demoted off NVENC on a machine with no working -QSV or AMF runtime now reaches libx264, the tier with no hardware dependency to -fail on. +A worker per participant uses a monotonic clock to emit 30 video frames and +48,000 audio samples per second. It holds the last picture during video gaps +(or black before the first frame) and inserts silence during audio gaps. Input +resolution and audio format changes are conformed on the worker without +restarting the output. Known fixed participants start at Record; participants +first encountered later have a `start_offset_ms` in status. + +Timestamped video and PCM audio travel over one internal Matroska pipe to +FFmpeg, which writes a single fragmented MP4. There are no separate temporary +media files and no merge step at Stop. Producer buffers and encoder queues are +bounded. An encoder failure is reported on that participant; it does not start +an automatic replacement file. Hardware availability is tested by encoding an +actual frame before recording starts. Automatic placement uses working NVENC, +QSV, AMF, then libx264, accounting for OBS's NVENC sessions. When `record_program` is true, CoreVideo also starts the normal OBS program recording and stops it when ISO recording stops, but only if CoreVideo started diff --git a/docs/iso-participant-av-validation-2026-09-14.md b/docs/iso-participant-av-validation-2026-09-14.md new file mode 100644 index 0000000..790e7f2 --- /dev/null +++ b/docs/iso-participant-av-validation-2026-09-14.md @@ -0,0 +1,77 @@ +# Continuous participant ISO recording validation + +Development build: `v0.1.47-dev`. This change has not been installed into the +running OBS instance or published as a release. + +## Change + +The previous recorder opened separate video MP4 and audio WAV files and keyed +video lifetime to source/resolution changes. The replacement owns one writer +per Zoom participant ID for the Record/Stop interval. Duplicate source UUIDs +share that writer. Input size changes only reconfigure the scaler. + +Every participant MP4 contains 1920x1080/30 H.264 and 48 kHz stereo AAC. +Aspect ratio is preserved with black bars. A worker per participant performs +scaling, resampling and timestamped A/V transport over one FFmpeg stdin pipe. +Video gaps hold the picture; audio gaps contain silence. A participant with no +media callbacks still gets black video and silence. No merge step or automatic +replacement segment is created. Diagnostic text logs remain beside the MP4. + +The Windows GPU scaler and aspect-fit geometry were adapted from CoreVideo +Pro's `D3DIsoFrameConformer.h` and `IsoFrameConform.h`. This recorder uses planar +I420 rather than Pro's NV12 output, and preserves the engine's full-range +BT.709 pixels. Other platforms use the OBS video scaler. + +## Completed checks + +- Full Windows plugin/engine build succeeded; 68 CTest regressions passed. +- Real worker fixtures: two simultaneous libx264 participant encoders and eight + simultaneous NVENC participant encoders on this RTX 4090 host. Each run also + had a silent/no-video libx264 writer and a deliberately invalid encoder process. +- Every participant produced one MP4 with both tracks, 180 decoded video frames + over six seconds, fixed 1920x1080/30 video, stereo 48 kHz AAC, and no decode errors. +- Inputs changed between 640x360, 1280x720, 1920x1080, and 640x480, followed by a + camera-off second and recovery. Decoded pixels verified pillarboxing, recovery + to full canvas, and separate U/V colors through scaling and native-size input. +- Input audio changed from 32 kHz mono to 48 kHz stereo, with callbacks withheld + outside test tones. Clustered callback timestamps verified that PCM placement + never rewinds or overwrites a previously received batch. +- Decoded flashes/tones stayed within 20 ms on both encoders; all six 100 ms + audio tones retained their duration. No late audio samples were reported. +- No-input tracks decoded as continuous black and silent AAC. The failed encoder + reported an error without stopping the healthy recordings. +- Provider-policy regression covers duplicate source suppression, monotonic + timestamps, alternate takeover after a gap, and source removal. + +Validation found and corrected startup history loss, fragmented MP4 startup +offset with B-frames, input color metadata, planar chroma conversion, and audio +rewinding on bunched arrival timestamps. + +## Reproduce + +Build `CoreVideoIsoTrackWriterTest` with the Windows plugin dependencies. Run +from an empty directory, with OBS's runtime DLL directory on PATH: + +```powershell +& $writerTest $ffmpeg h264_nvenc 8 +python tests/verify-iso-av.py $ffmpegBin $fixtureDirectory +``` + +Run the verifier from the repository root. Use `libx264 2` for the CPU fixture. +The test refuses to overwrite existing MP4s. The verifier checks decoded media, +not only FFmpeg exit status. Local final results are in `build-iso/av-burst-cpu` +and `build-iso/av-burst-nvenc`; regression results are in `build-iso/av-ctest.log`. + +## Remaining release validation + +- Live Zoom soak, including repeated resolution changes, mute/camera toggles, + duplicate OBS sources, source reassignment/removal, Stop, and a new Record run. +- Live lip-sync check, sustained resource usage at the intended participant + count, and playback/import in the production editor. +- Hosted macOS/Linux builds and platform recording checks; local validation + above is Windows only. + +The exposed identity is Zoom's numeric participant ID. A rejoin that receives a +new ID creates a new participant file; matching by display name would risk +combining different people. Known fixed participants share the Record epoch; +late participants expose their offset through `start_offset_ms` status. diff --git a/src/iso-av-mux.h b/src/iso-av-mux.h new file mode 100644 index 0000000..fdc34da --- /dev/null +++ b/src/iso-av-mux.h @@ -0,0 +1,114 @@ +#pragma once + +// Streaming Matroska transport to FFmpeg over one pipe. One fixed I420 track +// and one PCM S16LE stereo track, both on a microsecond clock. This is an +// internal transport; FFmpeg writes the user's single H.264/AAC MP4. +// Codec mapping: https://www.matroska.org/technical/codec_specs.html +#include +#include +#include + +namespace iso_av +{ +using Bytes = std::vector; +inline void be(Bytes &out, uint64_t value, int count) +{ + for (int i = count - 1; i >= 0; --i) + out.push_back(uint8_t(value >> (i * 8))); +} +inline void element(Bytes &out, uint32_t id, const Bytes &value) +{ + int n = id > 0xffffff ? 4 : id > 0xffff ? 3 : id > 0xff ? 2 : 1; + be(out, id, n); + // Fixed eight-byte EBML size, valid for every element in this transport. + be(out, (uint64_t(1) << 56) | value.size(), 8); + out.insert(out.end(), value.begin(), value.end()); +} +inline void number(Bytes &out, uint32_t id, uint64_t value) +{ + Bytes b; + be(b, value, 8); + element(out, id, b); +} +inline void string(Bytes &out, uint32_t id, const char *value) +{ + element(out, id, Bytes(value, value + std::strlen(value))); +} +inline Bytes header() +{ + Bytes out, ebml, info, tracks, video, audio, v, a; + number(ebml, 0x4286, 1); + number(ebml, 0x42f7, 1); + number(ebml, 0x42f2, 4); + number(ebml, 0x42f3, 8); + string(ebml, 0x4282, "matroska"); + number(ebml, 0x4287, 4); + number(ebml, 0x4285, 2); + element(out, 0x1a45dfa3, ebml); + be(out, 0x18538067, 4); + be(out, 0x01ffffffffffffffULL, 8); // unknown segment size + number(info, 0x2ad7b1, 1000); // one tick = one microsecond + string(info, 0x4d80, "CoreVideo"); + string(info, 0x5741, "CoreVideo"); + element(out, 0x1549a966, info); + number(video, 0xd7, 1); + number(video, 0x73c5, 1); + number(video, 0x83, 1); + string(video, 0x86, "V_MS/VFW/FOURCC"); + Bytes bitmap(40, 0); + const auto le = [&](size_t at, uint32_t value) { + for (int i = 0; i < 4; ++i) + bitmap[at + i] = uint8_t(value >> (8 * i)); + }; + le(0, 40); + le(4, 1920); + le(8, 1080); + bitmap[12] = 1; + bitmap[14] = 12; + le(16, 0x30323449); + le(20, 1920 * 1080 * 3 / 2); // I420 BITMAPINFOHEADER + element(video, 0x63a2, bitmap); + number(video, 0x23e383, 33333333); + number(v, 0xb0, 1920); + number(v, 0xba, 1080); + // The engine normalizes Zoom pixels to full-range BT.709 before delivery. + Bytes colour; + number(colour, 0x55b1, 1); + number(colour, 0x55b9, 2); + number(colour, 0x55ba, 1); + number(colour, 0x55bb, 1); + element(v, 0x55b0, colour); + element(video, 0xe0, v); + element(tracks, 0xae, video); + number(audio, 0xd7, 2); + number(audio, 0x73c5, 2); + number(audio, 0x83, 2); + string(audio, 0x86, "A_PCM/INT/LIT"); + double rate = 48000; + uint64_t bits; + std::memcpy(&bits, &rate, 8); + Bytes frequency; + be(frequency, bits, 8); + element(a, 0xb5, frequency); + number(a, 0x9f, 2); + number(a, 0x6264, 16); + element(audio, 0xe1, a); + element(tracks, 0xae, audio); + element(out, 0x1654ae6b, tracks); + return out; +} +inline Bytes frame(uint64_t index, const Bytes &video, const Bytes &audio) +{ + Bytes cluster, block; + number(cluster, 0xe7, index * 1000000ULL / 30); + block = {0x81, 0, 0, 0x80}; + block.insert(block.end(), video.begin(), video.end()); + element(cluster, 0xa3, block); + block = {0x82, 0, 0, 0x80}; + block.insert(block.end(), audio.begin(), audio.end()); + element(cluster, 0xa3, block); + Bytes out; + element(out, 0x1f43b675, cluster); + return out; +} +} // namespace iso_av diff --git a/src/iso-d3d-conformer.h b/src/iso-d3d-conformer.h new file mode 100644 index 0000000..018232d --- /dev/null +++ b/src/iso-d3d-conformer.h @@ -0,0 +1,204 @@ +#pragma once + +#include "iso-frame-conform.h" +#include +#include +#include +#include + +namespace corevideo::modules +{ + +// Adapted from CoreVideo Pro's D3DIsoFrameConformer. This transport needs planar +// I420 (Pro's writer takes NV12), with full-range black. Owned by one ISO worker; +// no immediate context is shared with the live compositor. +class D3DIsoFrameConformer +{ + public: + explicit D3DIsoFrameConformer(bool warpForTest = false) : warpForTest_(warpForTest) {} + bool prepare(std::string &error) { return device_ || initialize(error); } + + bool convert(const std::vector &input, int sw, int sh, int dw, int dh, + std::vector &output, std::string &error) + { + if (sw < 2 || sh < 2 || dw < 2 || dh < 2 || sw > 8192 || sh > 8192 || dw > 8192 || + dh > 8192 || ((sw | sh | dw | dh) & 1) || input.size() < size_t(sw) * sh * 3 / 2) { + error = "invalid I420 conformance dimensions or payload"; + return false; + } + if (!device_ && !initialize(error)) + return false; + const UINT inputBytes = UINT(size_t(sw) * sh * 3 / 2); + const UINT outputBytes = UINT(size_t(dw) * dh * 3 / 2); + if ((inputBytes + 3u) / 4u * 4u != inputCapacity_ || + (outputBytes + 3u) / 4u * 4u != outputCapacity_) { + if (!allocate(inputBytes, outputBytes, error)) + return false; + } + D3D11_MAPPED_SUBRESOURCE mapped{}; + if (FAILED(context_->Map(input_.Get(), 0, D3D11_MAP_WRITE_DISCARD, 0, &mapped))) { + error = "map ISO GPU upload"; + return false; + } + std::memcpy(mapped.pData, input.data(), inputBytes); + if (inputCapacity_ > inputBytes) + std::memset(static_cast(mapped.pData) + inputBytes, 0, + inputCapacity_ - inputBytes); + context_->Unmap(input_.Get(), 0); + const auto fit = isoFitRect(sw, sh, dw, dh); + const UINT groups = (outputCapacity_ / 4u + 255u) / 256u; + const UINT groupsX = std::min(groups, 65535u); + const UINT constants[12] = {UINT(sw), UINT(sh), UINT(dw), UINT(dh), + UINT(fit.x), UINT(fit.y), UINT(fit.width), UINT(fit.height), + inputBytes, outputBytes, groupsX * 256u, 0}; + context_->UpdateSubresource(constants_.Get(), 0, nullptr, constants, 0, 0); + ID3D11ShaderResourceView *srv = inputView_.Get(); + ID3D11UnorderedAccessView *uav = outputView_.Get(); + ID3D11Buffer *cb = constants_.Get(); + context_->CSSetShader(shader_.Get(), nullptr, 0); + context_->CSSetShaderResources(0, 1, &srv); + context_->CSSetUnorderedAccessViews(0, 1, &uav, nullptr); + context_->CSSetConstantBuffers(0, 1, &cb); + context_->Dispatch(groupsX, (groups + groupsX - 1) / groupsX, 1); + srv = nullptr; + uav = nullptr; + context_->CSSetShaderResources(0, 1, &srv); + context_->CSSetUnorderedAccessViews(0, 1, &uav, nullptr); + context_->CopyResource(readback_.Get(), output_.Get()); + if (FAILED(context_->Map(readback_.Get(), 0, D3D11_MAP_READ, 0, &mapped))) { + error = "map ISO GPU readback"; + return false; + } + output.resize(outputBytes); + std::memcpy(output.data(), mapped.pData, outputBytes); + context_->Unmap(readback_.Get(), 0); + return true; + } + + private: + template using Ptr = Microsoft::WRL::ComPtr; + bool initialize(std::string &error) + { + D3D_FEATURE_LEVEL level{}; + const D3D_FEATURE_LEVEL requested[] = {D3D_FEATURE_LEVEL_11_0}; + if (FAILED(D3D11CreateDevice( + nullptr, warpForTest_ ? D3D_DRIVER_TYPE_WARP : D3D_DRIVER_TYPE_HARDWARE, nullptr, 0, + requested, 1, D3D11_SDK_VERSION, &device_, &level, &context_))) { + error = "create ISO GPU conformance device"; + return false; + } + // Four output bytes per invocation avoid sub-word UAV write races. All + // box bounds, rounding, chroma alignment, and bar levels match the oracle. + static constexpr char source[] = R"( +ByteAddressBuffer src : register(t0); +RWByteAddressBuffer dst : register(u0); +cbuffer Parameters : register(b0) { uint4 dims; uint4 fit; uint4 sizes; }; +uint byteAt(uint i) { return (src.Load(i & ~3u) >> ((i & 3u) * 8u)) & 255u; } +uint box(uint offset, uint pw, uint ph, uint x, uint y, uint rw, uint rh) { + uint x0=x*pw/rw, x1=min(pw,max(x0+1,(x+1)*pw/rw)); + uint y0=y*ph/rh, y1=min(ph,max(y0+1,(y+1)*ph/rh)); + uint total=0, count=(x1-x0)*(y1-y0); + for(uint sy=y0;sy=sizes.y) return 0; + uint luma=dims.z*dims.w; + if(i=fit.x+fit.z || y=fit.y+fit.w) return 0; + return box(0,dims.x,dims.y,x-fit.x,y-fit.y,fit.z,fit.w); + } + uint cl=luma/4,j=i-luma,plane=j/cl,p=j%cl,x=p%(dims.z/2),y=p/(dims.z/2); + if(x=(fit.x+fit.z)/2 || y=(fit.y+fit.w)/2) return 128; + uint sl=dims.x*dims.y; + return box(sl+plane*(sl/4),dims.x/2,dims.y/2,x-fit.x/2,y-fit.y/2,fit.z/2,fit.w/2); +} +[numthreads(256,1,1)] void main(uint3 tid : SV_DispatchThreadID) { + uint at=(tid.x+tid.y*sizes.z)*4; + if(at>=sizes.y) return; + dst.Store(at,pixel(at)|(pixel(at+1)<<8)|(pixel(at+2)<<16)|(pixel(at+3)<<24)); +})"; + Ptr code, diagnostics; + const HRESULT compiled = D3DCompile( + source, sizeof(source) - 1, "IsoBoxConform", nullptr, nullptr, "main", "cs_5_0", + D3DCOMPILE_ENABLE_STRICTNESS | D3DCOMPILE_OPTIMIZATION_LEVEL3, 0, &code, &diagnostics); + if (FAILED(compiled) || + FAILED(device_->CreateComputeShader(code->GetBufferPointer(), code->GetBufferSize(), + nullptr, &shader_))) { + error = "compile ISO GPU box filter"; + device_.Reset(); + context_.Reset(); + return false; + } + D3D11_BUFFER_DESC desc{}; + desc.ByteWidth = 48; + desc.Usage = D3D11_USAGE_DEFAULT; + desc.BindFlags = D3D11_BIND_CONSTANT_BUFFER; + if (FAILED(device_->CreateBuffer(&desc, nullptr, &constants_))) { + error = "create ISO GPU constants"; + device_.Reset(); + context_.Reset(); + return false; + } + return true; + } + bool allocate(UINT inBytes, UINT outBytes, std::string &error) + { + inputView_.Reset(); + outputView_.Reset(); + input_.Reset(); + output_.Reset(); + readback_.Reset(); + inputCapacity_ = outputCapacity_ = 0; + D3D11_BUFFER_DESC desc{}; + desc.ByteWidth = (inBytes + 3u) / 4u * 4u; + desc.Usage = D3D11_USAGE_DYNAMIC; + desc.BindFlags = D3D11_BIND_SHADER_RESOURCE; + desc.CPUAccessFlags = D3D11_CPU_ACCESS_WRITE; + desc.MiscFlags = D3D11_RESOURCE_MISC_BUFFER_ALLOW_RAW_VIEWS; + HRESULT hr = device_->CreateBuffer(&desc, nullptr, &input_); + D3D11_SHADER_RESOURCE_VIEW_DESC srv{}; + srv.Format = DXGI_FORMAT_R32_TYPELESS; + srv.ViewDimension = D3D11_SRV_DIMENSION_BUFFEREX; + srv.BufferEx.NumElements = desc.ByteWidth / 4; + srv.BufferEx.Flags = D3D11_BUFFEREX_SRV_FLAG_RAW; + if (SUCCEEDED(hr)) + hr = device_->CreateShaderResourceView(input_.Get(), &srv, &inputView_); + desc.ByteWidth = (outBytes + 3u) / 4u * 4u; + desc.Usage = D3D11_USAGE_DEFAULT; + desc.BindFlags = D3D11_BIND_UNORDERED_ACCESS; + desc.CPUAccessFlags = 0; + if (SUCCEEDED(hr)) + hr = device_->CreateBuffer(&desc, nullptr, &output_); + D3D11_UNORDERED_ACCESS_VIEW_DESC uav{}; + uav.Format = DXGI_FORMAT_R32_TYPELESS; + uav.ViewDimension = D3D11_UAV_DIMENSION_BUFFER; + uav.Buffer.NumElements = desc.ByteWidth / 4; + uav.Buffer.Flags = D3D11_BUFFER_UAV_FLAG_RAW; + if (SUCCEEDED(hr)) + hr = device_->CreateUnorderedAccessView(output_.Get(), &uav, &outputView_); + desc.Usage = D3D11_USAGE_STAGING; + desc.BindFlags = 0; + desc.CPUAccessFlags = D3D11_CPU_ACCESS_READ; + desc.MiscFlags = 0; + if (SUCCEEDED(hr)) + hr = device_->CreateBuffer(&desc, nullptr, &readback_); + if (FAILED(hr)) { + error = "allocate ISO GPU conformance buffers"; + return false; + } + inputCapacity_ = (inBytes + 3u) / 4u * 4u; + outputCapacity_ = (outBytes + 3u) / 4u * 4u; + return true; + } + bool warpForTest_ = false; + UINT inputCapacity_ = 0, outputCapacity_ = 0; + Ptr device_; + Ptr context_; + Ptr shader_; + Ptr input_, output_, readback_, constants_; + Ptr inputView_; + Ptr outputView_; +}; +} // namespace corevideo::modules diff --git a/src/iso-frame-conform.h b/src/iso-frame-conform.h new file mode 100644 index 0000000..52e2ec8 --- /dev/null +++ b/src/iso-frame-conform.h @@ -0,0 +1,47 @@ +#pragma once + +// Aspect-fit geometry adapted from CoreVideo Pro IsoFrameConform.h. +// ISO output is always 1920x1080; chroma-aligned bars preserve aspect ratio. +#include +#include +#include +#include + +namespace corevideo::modules +{ + +struct IsoFitRect { + int x = 0; + int y = 0; + int width = 0; + int height = 0; +}; + +// Largest centred rect of the source's aspect that fits the destination, with +// every edge EVEN: I420 and NV12 carry chroma at half resolution, so an odd +// origin or extent splits a chroma sample across the letterbox edge and fringes +// it. A degenerate input returns an empty rect rather than dividing by zero. +[[nodiscard]] inline IsoFitRect isoFitRect(int srcW, int srcH, int dstW, int dstH) +{ + if (srcW <= 0 || srcH <= 0 || dstW <= 0 || dstH <= 0) { + return {}; + } + const auto even = [](int value) { return value & ~1; }; + // Compare aspects in integer cross-products: no float, no rounding drift. + const long long srcAspect = static_cast(srcW) * dstH; + const long long dstAspect = static_cast(dstW) * srcH; + int width = dstW; + int height = dstH; + if (srcAspect > dstAspect) { + // Source is wider: full width, bars top and bottom. + height = static_cast((static_cast(dstW) * srcH + srcW / 2) / srcW); + } else if (srcAspect < dstAspect) { + // Source is taller: full height, bars left and right. + width = static_cast((static_cast(dstH) * srcW + srcH / 2) / srcH); + } + width = std::max(2, std::min(even(width), even(dstW))); + height = std::max(2, std::min(even(height), even(dstH))); + return {even((dstW - width) / 2), even((dstH - height) / 2), width, height}; +} + +} // namespace corevideo::modules diff --git a/src/iso-provider-policy.h b/src/iso-provider-policy.h new file mode 100644 index 0000000..8c4a324 --- /dev/null +++ b/src/iso-provider-policy.h @@ -0,0 +1,17 @@ +#pragma once +#include +#include + +// One provider per participant stream prevents duplicate OBS sources from +// doubling audio. An alternate may take over after a 250 ms delivery gap. +inline bool iso_take_provider(std::string &owner, uint64_t &last, const std::string &source, + uint64_t now) +{ + if (now <= last) + return false; + if (!owner.empty() && owner != source && now - last < 250000000ULL) + return false; + owner = source; + last = now; + return true; +} diff --git a/src/iso-track-writer.cpp b/src/iso-track-writer.cpp new file mode 100644 index 0000000..e4179a4 --- /dev/null +++ b/src/iso-track-writer.cpp @@ -0,0 +1,322 @@ +#include "iso-track-writer.h" +#include "iso-av-mux.h" +#include "iso-frame-conform.h" +#ifdef _WIN32 +#include "iso-d3d-conformer.h" +#else +#include +#endif +#include +#include +#include +#include +#include + +IsoTrackWriter::IsoTrackWriter(Config config) + : m_config(std::move(config)), m_thread([this] { run(); }) +{ +} +IsoTrackWriter::~IsoTrackWriter() +{ + { + std::lock_guard lock(m_mutex); + m_abort = true; + } + m_cv.notify_all(); + if (m_thread.joinable()) + m_thread.join(); +} +void IsoTrackWriter::fail(const std::string &error) +{ + std::lock_guard lock(m_mutex); + if (m_status.error.empty()) + m_status.error = error; +} +IsoTrackWriter::Status IsoTrackWriter::status() const +{ + std::lock_guard lock(m_mutex); + return m_status; +} +void IsoTrackWriter::close(uint64_t end_ns) +{ + { + std::lock_guard lock(m_mutex); + m_closing = true; + m_end_ns = end_ns; + } + m_cv.notify_all(); +} +bool IsoTrackWriter::wait_finished(int timeout_ms) +{ + std::unique_lock lock(m_mutex); + return m_cv.wait_for(lock, std::chrono::milliseconds(timeout_ms), + [&] { return m_status.done; }); +} +bool IsoTrackWriter::video(uint32_t w, uint32_t h, const uint8_t *y, const uint8_t *u, + const uint8_t *v, uint32_t sy, uint32_t suv, uint64_t ns) +{ + if (!y || !u || !v || w < 2 || h < 2 || w > 4096 || h > 2160 || ((w | h) & 1) || sy < w || + suv < w / 2) + return false; + Video frame{w, h, ns, std::vector(size_t(w) * h * 3 / 2)}; + auto *dst = frame.pixels.data(); + for (uint32_t row = 0; row < h; ++row) { + std::memcpy(dst, y + size_t(row) * sy, w); + dst += w; + } + for (auto *plane : {u, v}) + for (uint32_t row = 0; row < h / 2; ++row) { + std::memcpy(dst, plane + size_t(row) * suv, w / 2); + dst += w / 2; + } + std::lock_guard lock(m_mutex); + if (m_closing || m_status.done || !m_status.error.empty()) + return false; + // Coalesce delivery bursts; the independent output clock never loses time. + // Keep one second through GPU/encoder startup and delivery bursts. Shrinking + // this after the first output tick discards still-pending startup frames. + const size_t limit = 32; + while (m_video.size() >= limit) { + m_video.pop_front(); + ++m_status.coalesced_video; + } + m_video.push_back(std::move(frame)); + m_cv.notify_all(); + return true; +} +bool IsoTrackWriter::audio(const uint8_t *pcm, uint32_t bytes, uint32_t rate, uint16_t channels, + uint64_t ns) +{ + if (!pcm || !bytes || (channels != 1 && channels != 2) || rate < 8000 || rate > 192000 || + bytes % (channels * 2) || bytes > rate * channels * 2 / 2) + return false; + std::lock_guard lock(m_mutex); + if (m_closing || m_status.done || !m_status.error.empty()) + return false; + if (m_audio.size() >= 256) { + m_status.error = "ISO audio input backlog exceeded its bound"; + return false; + } + m_audio.push_back({rate, channels, ns, {pcm, pcm + bytes}}); + m_cv.notify_all(); + return true; +} +void IsoTrackWriter::run() +{ + IsoFfmpegPipe pipe; + audio_resampler_t *resampler = nullptr; +#ifndef _WIN32 + video_scaler_t *scaler = nullptr; +#endif + try { + std::vector args = { + "-hide_banner", "-loglevel", "warning", "-n", "-f", "matroska", "-i", + "pipe:0", "-map", "0:v:0", "-map", "0:a:0", "-c:v", m_config.encoder}; + if (m_config.encoder == "libx264") + args.insert(args.end(), {"-preset", "veryfast", "-crf", "18", "-threads", "2"}); + else + args.insert(args.end(), {"-b:v", "12M", "-maxrate", "20M", "-bufsize", "24M"}); + args.insert(args.end(), {"-r", "30", + "-fps_mode", "cfr", + "-g", "60", + "-bf", "0", + "-color_range", "pc", + "-colorspace", "bt709", + "-c:a", "aac", + "-b:a", "192k", + "-ar", "48000", + "-ac", "2", + "-movflags", "+frag_keyframe+empty_moov", + m_config.path}); + std::string error; + if (!pipe.start(m_config.ffmpeg, args, m_config.log_path, IsoFfmpegPipe::kMaximumPixelBytes, + &error)) + throw std::runtime_error(error); + if (!pipe.try_queue(iso_av::header())) + throw std::runtime_error("ISO transport header rejected"); + constexpr uint64_t latency = 150000000ULL; + constexpr size_t ringFrames = 48000 * 4; + std::vector ring(ringFrames * 2, 0); + uint64_t index = 0, audioEnd = 0; + uint32_t rate = 0; + uint16_t channels = 0; + std::vector picture(1920 * 1080 * 3 / 2, 128); + std::fill_n(picture.data(), 1920 * 1080, 0); +#ifdef _WIN32 + corevideo::modules::D3DIsoFrameConformer conformer; +#else + uint32_t oldW = 0, oldH = 0; +#endif + for (;;) { + Video newest{}; + std::deque