Fix selective recording in Simple output mode

obs-studio-server/source/osn-simple-recording.cpp

@@ -304,7 +304,7 @@ static void UpdateRecordingSettings_crf(enum osn::RecQuality quality, osn::Simpl
 	if (!settings)
 		return;
 	if (obs_get_multiple_rendering()) {
-		obs_encoder_set_video_mix(recording->videoEncoder, obs_video_mix_get(recording->GetCanvas(), OBS_STREAMING_VIDEO_RENDERING));
+		obs_encoder_set_video_mix(recording->videoEncoder, obs_video_mix_get(recording->GetCanvas(), OBS_RECORDING_VIDEO_RENDERING));
 	} else {
 		obs_encoder_set_video_mix(recording->videoEncoder, obs_video_mix_get(recording->GetCanvas(), OBS_MAIN_VIDEO_RENDERING));
 	}

tests/osn-tests/src/test_media_probe.ts

@@ -0,0 +1,41 @@
+import 'mocha';
+import { expect } from 'chai';
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { getVideoFrameRgb } from '../util/media_probe';
+
+describe('Recorded RGB frame decoding', function () {
+    this.timeout(15000);
+    let directory: string;
+
+    before(() => {
+        directory = fs.mkdtempSync(path.join(os.tmpdir(), 'osn-media-probe-'));
+    });
+
+    after(() => {
+        if (directory) fs.rmSync(directory, { recursive: true, force: true });
+    });
+
+    function createFrame(width: number, height: number, pixels = Buffer.alloc(width * height * 3, 255)): string {
+        const file = path.join(directory, `${width}x${height}.ppm`);
+        fs.writeFileSync(file, Buffer.concat([Buffer.from(`P6\n${width} ${height}\n255\n`), pixels]));
+        return file;
+    }
+
+    it('preserves RGB pixels, including whitespace and hash bytes at the start of the raster', () => {
+        const pixels = Buffer.from([10, 13, 32, 35, 0, 255, 9, 11, 12, 255, 0, 1]);
+        expect(getVideoFrameRgb(createFrame(2, 2, pixels), 0, 2, 2)).to.deep.equal(pixels);
+    });
+
+    for (const [width, height] of [[180, 320], [640, 90]]) {
+        it(`rejects a ${width}x${height} frame when 320x180 is requested, despite equal pixel counts`, () => {
+            const file = createFrame(width, height);
+            expect(() => getVideoFrameRgb(file, 0, 320, 180)).to.throw(`decoded ${width}x${height}`);
+        });
+    }
+
+    it('rejects a seek that produces no frame', () => {
+        expect(() => getVideoFrameRgb(createFrame(2, 2), 5, 2, 2)).to.throw();
+    });
+});

tests/osn-tests/src/test_osn_selective_recording.ts

@@ -0,0 +1,196 @@
+import 'mocha';
+import { expect } from 'chai';
+import { randomUUID } from 'crypto';
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import * as osn from '../osn';
+import { OBSHandler } from '../util/obs_handler';
+import { deleteConfigFiles, sleep } from '../util/general';
+import { EOBSOutputSignal, EOBSOutputType } from '../util/obs_enums';
+import { getVideoFrameRgb } from '../util/media_probe';
+import { expectRegionColor, RGB } from '../util/media_assertions';
+
+const testName = 'osn-selective-recording';
+const width = 320;
+const height = 180;
+const background: RGB = [0, 255, 0];
+const markers = [
+    { name: 'stream-only', x: 16, y: 16, color: 0xffffffff, rgb: [255, 255, 255] as RGB, stream: true, record: false },
+    { name: 'recording-only', x: 104, y: 16, color: 0xff0000ff, rgb: [255, 0, 0] as RGB, stream: false, record: true },
+    { name: 'both', x: 192, y: 16, color: 0xffff0000, rgb: [0, 0, 255] as RGB, stream: true, record: true },
+    { name: 'neither', x: 16, y: 104, color: 0xff00ffff, rgb: [255, 255, 0] as RGB, stream: false, record: false },
+];
+
+describe(testName, function () {
+    this.timeout(30000);
+    let obs: OBSHandler;
+    let video: osn.IVideo;
+    let scene: osn.IScene;
+    let directory: string;
+    let previousMultipleRendering: boolean;
+    const inputs: osn.IInput[] = [];
+    const items: osn.ISceneItem[] = [];
+    // Retain timed-out outputs until runtime shutdown rather than freeing active encoders.
+    const pendingOutputs: (() => void)[] = [];
+
+    before(() => {
+        deleteConfigFiles();
+        obs = new OBSHandler(testName, false);
+        previousMultipleRendering = osn.Global.multipleRendering;
+        directory = fs.mkdtempSync(path.join(os.tmpdir(), `${testName}-`));
+        video = osn.VideoFactory.create();
+        video.video = {
+            fpsNum: 30, fpsDen: 1, baseWidth: width, baseHeight: height,
+            outputWidth: width, outputHeight: height, outputFormat: osn.EVideoFormat.NV12,
+            colorspace: osn.EColorSpace.CS709, range: osn.ERangeType.Partial,
+            scaleType: osn.EScaleType.Bilinear, fpsType: osn.EFPSType.Fractional,
+        };
+        osn.AudioTrackFactory.setAtIndex(osn.AudioTrackFactory.create(160, testName), 1);
+        scene = osn.SceneFactory.create(testName);
+        function addBlock(name: string, x: number, y: number, color: number,
+            stream: boolean, record: boolean, blockWidth = 64, blockHeight = 64) {
+            const input = osn.InputFactory.create('color_source', `${testName}-${name}`, {
+                width: blockWidth, height: blockHeight, color,
+            });
+            inputs.push(input);
+            const item = scene.add(input);
+            items.push(item);
+            item.video = video;
+            item.position = { x, y };
+            item.visible = true;
+            item.streamVisible = stream;
+            item.recordingVisible = record;
+        }
+        addBlock('background', 0, 0, 0xff00ff00, true, true, width, height);
+        markers.forEach(marker => addBlock(marker.name, marker.x, marker.y, marker.color, marker.stream, marker.record));
+        osn.Global.setOutputSource(0, scene);
+    });
+
+    beforeEach(() => {
+        expect(pendingOutputs.length).to.equal(0, 'A preceding output failed to stop safely');
+    });
+
+    after(() => {
+        try {
+            if (pendingOutputs.length === 0 && obs) {
+                osn.Global.multipleRendering = previousMultipleRendering;
+                osn.Global.setOutputSource(0, null);
+                items.forEach(item => item.remove());
+                if (scene) scene.release();
+                inputs.forEach(input => input.release());
+                if (video) video.destroy();
+            }
+        } finally {
+            if (obs) obs.shutdown();
+            // Factory destruction also releases client-side signal workers after disconnect.
+            pendingOutputs.forEach(destroy => destroy());
+            if (directory) fs.rmSync(directory, { recursive: true, force: true });
+        }
+    });
+
+    async function record(quality?: osn.ERecordingQuality): Promise<string> {
+        const recording = quality === undefined ? osn.AdvancedRecordingFactory.create() : osn.SimpleRecordingFactory.create();
+        let encoder: osn.IVideoEncoder;
+        let audio: osn.IAudioEncoder;
+        let started = false;
+        let captureActive = false;
+        const signals: osn.EOutputSignal[] = [];
+        const name = `${testName}-${randomUUID()}`;
+        const hasSignal = (name: EOBSOutputSignal) => signals.some(signal =>
+            signal.type === EOBSOutputType.Recording && signal.signal === name);
+
+        async function waitFor(names: EOBSOutputSignal[], rejectFailure = true) {
+            const deadline = Date.now() + 10000;
+            while (Date.now() < deadline) {
+                const failure = signals.find(signal =>
+                    (signal.signal === EOBSOutputSignal.Stop && signal.code !== osn.EOutputCode.Success) ||
+                    signal.signal === EOBSOutputSignal.WriteError);
+                if (rejectFailure && failure) throw new Error(`Recording failed: ${JSON.stringify(failure)}`);
+                if (names.every(hasSignal) && (!names.includes(EOBSOutputSignal.Stop) || !captureActive)) return;
+                await sleep(25);
+            }
+            throw new Error(`Recording timed out waiting for ${names.join(', ')}: ${JSON.stringify(signals)}`);
+        }
+
+        function destroyOutput() {
+            if (quality === undefined) osn.AdvancedRecordingFactory.destroy(recording as osn.IAdvancedRecording);
+            else osn.SimpleRecordingFactory.destroy(recording as osn.ISimpleRecording);
+        }
+
+        try {
+            encoder = osn.VideoEncoderFactory.create('obs_x264', name, {
+                rate_control: 'CRF', crf: 23, preset: 'ultrafast', keyint_sec: 1,
+            });
+            recording.video = video;
+            recording.videoEncoder = encoder;
+            recording.path = directory;
+            recording.format = osn.ERecordingFormat.MP4;
+            recording.fileFormat = name;
+            if (quality === undefined) {
+                (recording as osn.IAdvancedRecording).useStreamEncoders = false;
+                (recording as osn.IAdvancedRecording).mixer = 1;
+            } else {
+                (recording as osn.ISimpleRecording).quality = quality;
+                audio = osn.AudioEncoderFactory.create('ffmpeg_aac', `${name}-audio`);
+                (recording as osn.ISimpleRecording).audioEncoder = audio;
+            }
+            recording.signalHandler = signal => {
+                signals.push(signal);
+                if (signal.signal === EOBSOutputSignal.Activate) captureActive = true;
+                if (signal.signal === EOBSOutputSignal.Deactivate) captureActive = false;
+            };
+            recording.start();
+            started = true;
+            await waitFor([EOBSOutputSignal.Start]);
+            await sleep(2000);
+            recording.stop();
+            await waitFor([EOBSOutputSignal.Stop, EOBSOutputSignal.Wrote]);
+            started = false;
+            return recording.lastFile();
+        } finally {
+            try {
+                if (started) {
+                    if (!hasSignal(EOBSOutputSignal.Stop)) recording.stop(true);
+                    await waitFor([EOBSOutputSignal.Stop], false);
+                    started = false;
+                }
+            } finally {
+                if (started) pendingOutputs.push(destroyOutput);
+                else {
+                    destroyOutput();
+                    if (audio) audio.release();
+                    if (encoder) encoder.release();
+                }
+            }
+        }
+    }
+
+    async function checkRecording(selective: boolean, quality?: osn.ERecordingQuality) {
+        // Set the rendering mode before output/encoder creation. No replay or streaming is involved.
+        osn.Global.multipleRendering = selective;
+        const file = await record(quality);
+        for (const time of [0.5, 1, 1.5]) {
+            const frame = { data: getVideoFrameRgb(file, time, width, height), width, height };
+            // Sample inside the blocks, away from compression and chroma edges.
+            expectRegionColor(frame, { x: 288, y: 144 }, background, `Background control at ${time}s`);
+            markers.forEach(marker => expectRegionColor(frame, { x: marker.x + 24, y: marker.y + 24 },
+                selective && !marker.record ? background : marker.rgb, `${marker.name} at ${time}s`));
+        }
+    }
+
+    for (const [name, quality] of [
+        ['HighQuality', osn.ERecordingQuality.HighQuality],
+        ['HigherQuality', osn.ERecordingQuality.HigherQuality],
+    ] as [string, osn.ERecordingQuality][]) {
+        for (const selective of [false, true]) {
+            it(`Simple ${name}: ${selective ? 'honors recording visibility' : 'ignores selective masks when disabled'}`, async () => {
+                await checkRecording(selective, quality);
+            });
+        }
+    }
+
+    it('Advanced: honors recording visibility with a dedicated encoder', async () => {
+        await checkRecording(true);
+    });
+});

tests/osn-tests/util/README.md

@@ -11,6 +11,36 @@ expected results in the test.
 | `RtmpTestServer` (`rtmp-test-server.ts`) | Owns the loopback listener, connections and per-attempt media captures. Independent of OSN. |
 | `expectAudioTracks`, `expectVideoFrames` (`rtmp-assertions.ts`) | Assert received media. They neither start outputs nor wait for packets. |
 | `media_probe.ts` | Uses ffprobe/ffmpeg for recorded-file inspection. |
+| `expectRegionColor` (`media_assertions.ts`) | Asserts the mean RGB color of a region in a decoded frame. No runtime or output ownership. |
+
+## Selective recording
+
+`src/test_osn_selective_recording.ts` records generated color sources without a
+provider account, streaming output or replay buffer. It checks decoded RGB frames
+for stream-only, recording-only, both-visible and neither-visible scene items.
+An always-visible background also rejects empty or black recordings. Both Simple
+quality presets, disabled selective recording and an Advanced dedicated encoder
+are covered. The suite is discovered by the existing integration/CI scripts.
+
+`getVideoFrameRgb()` in `media_probe.ts` decodes one unscaled frame and rejects
+missing frames or unexpected dimensions. FFmpeg emits binary PPM so the decoded
+width and height are checked independently, followed by the RGB payload length.
+`src/test_media_probe.ts` covers equal-pixel-count dimension mismatches, exact RGB
+bytes and missing frames without starting OBS. FFmpeg lookup honors `FFMPEG_PATH`, then
+checks the OSN package root and macOS `Frameworks` directory, then PATH. Color
+checks sample inside the blocks and tolerate compression differences.
+
+`expectRegionColor()` accepts `{ data, width, height }` for an RGB24 frame and a
+region `{ x, y, width?, height? }` measured from its top-left pixel. Region size
+defaults to 8x8 and the per-channel tolerance defaults to 25; both are configurable.
+It rejects invalid frame dimensions, buffer lengths, region bounds and tolerances.
+Keep expected colors and the choice of interior regions in each test.
+
+After building and installing OSN, run both suites with:
+
+```text
+yarn electron-mocha -t 30000 -r ts-node/register tests/osn-tests/src/test_osn_selective_recording.ts tests/osn-tests/src/test_media_probe.ts
+```
 
 ## Output setup and lifetime
 

tests/osn-tests/util/media_assertions.ts

@@ -0,0 +1,57 @@
+import { expect } from 'chai';
+
+export type RGB = [number, number, number];
+
+export interface IRgbFrame {
+    data: Buffer;
+    width: number;
+    height: number;
+}
+
+export interface IRgbRegion {
+    x: number;
+    y: number;
+    width?: number;
+    height?: number;
+}
+
+/**
+ * Asserts the rounded mean color of a region in an unscaled RGB24 frame.
+ * Region coordinates start at the top-left pixel and default to an 8x8 patch.
+ * Tolerance applies independently to each RGB channel and defaults to 25.
+ * Throws RangeError for invalid dimensions, buffer length, region bounds or tolerance;
+ * throws a Chai AssertionError when the mean color is outside the tolerance.
+ */
+export function expectRegionColor(frame: IRgbFrame, region: IRgbRegion, expected: RGB,
+    description: string, tolerance = 25): void {
+    const { data, width: frameWidth, height: frameHeight } = frame;
+    const { x, y, width = 8, height = 8 } = region;
+    if (!Number.isSafeInteger(frameWidth) || frameWidth <= 0 ||
+        !Number.isSafeInteger(frameHeight) || frameHeight <= 0) {
+        throw new RangeError(`${description}: frame dimensions must be positive integers`);
+    }
+    const expectedBytes = frameWidth * frameHeight * 3;
+    if (data.length !== expectedBytes) {
+        throw new RangeError(`${description}: expected ${expectedBytes} RGB24 bytes, got ${data.length}`);
+    }
+    if (!Number.isSafeInteger(x) || x < 0 || !Number.isSafeInteger(y) || y < 0 ||
+        !Number.isSafeInteger(width) || width <= 0 || !Number.isSafeInteger(height) || height <= 0 ||
+        width > frameWidth - x || height > frameHeight - y) {
+        throw new RangeError(`${description}: region ${x},${y} ${width}x${height} must fit within ${frameWidth}x${frameHeight}`);
+    }
+    if (!Number.isFinite(tolerance) || tolerance < 0) {
+        throw new RangeError(`${description}: color tolerance must be finite and non-negative`);
+    }
+
+    const sum: RGB = [0, 0, 0];
+    for (let dy = 0; dy < height; dy++) {
+        for (let dx = 0; dx < width; dx++) {
+            const offset = ((y + dy) * frameWidth + x + dx) * 3;
+            for (let channel = 0; channel < 3; channel++) sum[channel] += data[offset + channel];
+        }
+    }
+    const actual = sum.map(value => Math.round(value / (width * height)));
+    expected.forEach((value, channel) => {
+        expect(actual[channel], `${description}: expected ${expected}, got ${actual}`).to.be.closeTo(value, tolerance);
+    });
+}

tests/osn-tests/util/media_probe.ts

@@ -52,14 +52,41 @@ export function getVideoKeyframes(mediaFile: string): {
     };
 }
 
-// Resolves an ffmpeg executable. Honours FFMPEG_PATH (point it at OBS's bundled
-// ffmpeg when ffmpeg is not on PATH), otherwise relies on `ffmpeg` from PATH.
+// Prefer an explicit override, then the packaged executable, then PATH.
 function resolveFfmpeg(): string {
-    const override = process.env.FFMPEG_PATH;
-    if (override && fs.existsSync(override)) {
-        return override;
+    const executable = process.platform === 'win32' ? 'ffmpeg.exe' : 'ffmpeg';
+    return [
+        process.env.FFMPEG_PATH,
+        path.join(path.normalize(osn.wd), executable),
+        path.join(path.normalize(osn.wd), 'Frameworks', executable),
+    ].find(candidate => candidate && fs.existsSync(candidate)) || executable;
+}
+
+/** Decodes one unscaled RGB24 frame, checking its decoded width, height and payload length. */
+export function getVideoFrameRgb(mediaFile: string, timeSeconds: number, width: number, height: number): Buffer {
+    const expectedBytes = width * height * 3;
+    const frame = execFileSync(resolveFfmpeg(), [
+        '-hide_banner', '-loglevel', 'error', '-nostdin',
+        '-ss', String(timeSeconds), '-i', mediaFile, '-map', '0:v:0',
+        '-frames:v', '1', '-pix_fmt', 'rgb24', '-c:v', 'ppm', '-f', 'image2pipe', 'pipe:1',
+    ], { timeout: 10000, maxBuffer: expectedBytes + 1024, windowsHide: true });
+
+    // FFmpeg emits an 8-bit binary PPM header without comments. Consume exactly
+    // one separator after maxval: following whitespace or hash bytes are pixels.
+    const header = frame.subarray(0, 1024).toString('ascii').match(/^P6\s+(\d+)\s+(\d+)\s+255\s/);
+    if (!header) {
+        throw new Error(`Missing or invalid RGB24 PPM frame at ${timeSeconds}s in ${mediaFile}`);
+    }
+    const decodedWidth = Number(header[1]);
+    const decodedHeight = Number(header[2]);
+    if (decodedWidth !== width || decodedHeight !== height) {
+        throw new Error(`Expected a ${width}x${height} RGB frame at ${timeSeconds}s in ${mediaFile}, decoded ${decodedWidth}x${decodedHeight}`);
+    }
+    const pixels = frame.subarray(header[0].length);
+    if (pixels.length !== expectedBytes) {
+        throw new Error(`Expected a ${width}x${height} RGB frame at ${timeSeconds}s in ${mediaFile}, got ${pixels.length} bytes`);
     }
-    return 'ffmpeg';
+    return pixels;
 }
 
 // Returns the mean volume (dBFS) of the first audio stream of `mediaFile`, as