diff --git a/js/module.ts b/js/module.ts index 293c70210..9f488151b 100644 --- a/js/module.ts +++ b/js/module.ts @@ -2323,6 +2323,7 @@ interface IAutoOptimizerMeasurementEvidence { interface IAutoOptimizerMeasurement { mode: AutoOptimizerMeasurementMode; confidence: 'high' | 'medium' | 'low'; + /** Optional explanation code for an estimate or measured result, including why higher-quality settings were rejected. */ reason?: string; /** Provider measurements that contributed to the result. Detailed throughput and workload data remains internal to OSN. */ evidence?: IAutoOptimizerMeasurementEvidence[]; diff --git a/obs-studio-client/tests/auto-optimizer-client-contract-test.cpp b/obs-studio-client/tests/auto-optimizer-client-contract-test.cpp index 24cb46e1d..4c67ff82d 100644 --- a/obs-studio-client/tests/auto-optimizer-client-contract-test.cpp +++ b/obs-studio-client/tests/auto-optimizer-client-contract-test.cpp @@ -487,6 +487,25 @@ TEST_CASE("Auto Optimizer client requires the exact active Dual Output aggregate {"divergent preset", [](json &value) { value["legs"][1]["recommendation"]["preset"] = "veryfast"; }}}); } +TEST_CASE("Auto Optimizer client preserves measured Enhanced Broadcasting fallback explanations") +{ + for (const auto reason : + {"enhanced_broadcasting_transport_fallback", "enhanced_broadcasting_workload_fallback", "enhanced_broadcasting_transport_and_workload_fallback"}) { + CAPTURE(reason); + auto fixture = enhancedBroadcastingFixture(); + fixture.result["legs"][0]["measurement"]["reason"] = reason; + const auto result = contract::projectResult(fixture.result.dump(), "run", fixture.prepared.context); + REQUIRE(result.valid); + const json projected = json::parse(result.json); + const auto &output = projected["outputs"][0]; + CHECK(output["measurement"]["reason"] == reason); + CHECK(output["measurement"]["mode"] == "active"); + CHECK(output["measurement"]["confidence"] == "high"); + CHECK(output["measurement"]["evidence"][0]["success"] == true); + CHECK_FALSE(output.contains("encoding")); + } +} + TEST_CASE("Auto Optimizer client requires exact Enhanced Broadcasting combined workload proof") { const auto fixture = enhancedBroadcastingFixture(); diff --git a/obs-studio-client/tests/auto-optimizer-enhanced-broadcasting-policy-test.cpp b/obs-studio-client/tests/auto-optimizer-enhanced-broadcasting-policy-test.cpp index 12c4bd438..b16874c2e 100644 --- a/obs-studio-client/tests/auto-optimizer-enhanced-broadcasting-policy-test.cpp +++ b/obs-studio-client/tests/auto-optimizer-enhanced-broadcasting-policy-test.cpp @@ -138,3 +138,35 @@ TEST_CASE("Composite Enhanced Broadcasting maps only local joint-load failures t CHECK_FALSE(policy::isCompositeCandidateLoadFailure("enhanced_broadcasting_output_connect_failed")); CHECK_FALSE(policy::isCompositeCandidateLoadFailure("enhanced_broadcasting_transport_pressure")); } + +TEST_CASE("Enhanced Broadcasting fallback reports retain earlier upload instability") +{ + policy::CandidateFallbackEvidence evidence; + CHECK(evidence.reason().empty()); + evidence.record("enhanced_broadcasting_ladder_below_candidate"); + CHECK(evidence.reason().empty()); + evidence.record("enhanced_broadcasting_transport_pressure"); + CHECK(evidence.reason() == "enhanced_broadcasting_transport_fallback"); + evidence.record("enhanced_broadcasting_ladder_below_candidate"); + evidence.record(""); + CHECK(evidence.reason() == "enhanced_broadcasting_transport_fallback"); + CHECK(policy::CandidateFallbackEvidence{}.reason().empty()); +} + +TEST_CASE("Enhanced Broadcasting fallback reports distinguish workload and upload failures") +{ + for (const auto error : + {"enhanced_broadcasting_encoder_underload", "enhanced_broadcasting_render_overload", "enhanced_broadcasting_companion_overload"}) { + CAPTURE(error); + policy::CandidateFallbackEvidence evidence; + evidence.record(error); + CHECK(evidence.reason() == "enhanced_broadcasting_workload_fallback"); + evidence.record("enhanced_broadcasting_transport_pressure"); + CHECK(evidence.reason() == "enhanced_broadcasting_transport_and_workload_fallback"); + + policy::CandidateFallbackEvidence reversed; + reversed.record("enhanced_broadcasting_transport_pressure"); + reversed.record(error); + CHECK(reversed.reason() == evidence.reason()); + } +} diff --git a/obs-studio-server/source/nodeobs_auto_optimizer.cpp b/obs-studio-server/source/nodeobs_auto_optimizer.cpp index db7842db3..26f96c26d 100644 --- a/obs-studio-server/source/nodeobs_auto_optimizer.cpp +++ b/obs-studio-server/source/nodeobs_auto_optimizer.cpp @@ -80,8 +80,9 @@ constexpr int kYoutubeProbeTotalTimeoutMs = 100000; constexpr int kYoutubeProbeBudgetSlackMs = 250; constexpr int kYoutubeProbeMaximumConfirmationEpisodes = 2; constexpr int kYoutubeProbeBudgetEstimatePercent = 115; -constexpr float kProbeCongestionHigh = 0.20f; -constexpr float kProbeCongestionSevere = 0.50f; +// All probes and recovery checks share this cutoff. Keep the sample counters +// used by the duration-based policies, but count both from the same threshold. +constexpr float kProbeCongestionThreshold = 0.50f; constexpr int kTwitchProbeAudioBitrateKbps = 32; constexpr int kYoutubeProbeAudioBitrateKbps = 128; constexpr int kHardwareWarmupMs = 500; @@ -1378,6 +1379,7 @@ struct ProbeResult { uint64_t configuredAggregateBitrateKbps = 0; bool pairedCadenceEvidence = false; std::vector companionWorkloads; + enhancedBroadcastingPolicy::CandidateFallbackEvidence candidateFallback; }; static bool silentAudioCallback(void *, uint64_t startTimestamp, uint64_t, uint64_t *outputTimestamp, uint32_t, struct audio_data_mixes_outputs *) @@ -2737,10 +2739,10 @@ static bool runTwitchProbeSample(const std::shared_ptr &session, Scratc sample.congestionSamples = (uint32_t)congestionValues.size(); for (float congestion : congestionValues) { sample.maximumCongestion = std::max(sample.maximumCongestion, congestion); - if (congestion >= kProbeCongestionHigh) + if (congestion >= kProbeCongestionThreshold) { sample.congestionHighSamples++; - if (congestion >= kProbeCongestionSevere) sample.congestionSevereSamples++; + } } std::sort(congestionValues.begin(), congestionValues.end()); if (!congestionValues.empty()) { @@ -2850,7 +2852,7 @@ static bool waitForYoutubeRecoveryDrain(const std::shared_ptr &session, return false; } - if (recoveryGate.observe(lastCongestion < kProbeCongestionHigh, droppedFramesUnchanged)) { + if (recoveryGate.observe(lastCongestion < kProbeCongestionThreshold, droppedFramesUnchanged)) { recovered = true; break; } @@ -3026,10 +3028,10 @@ static bool runYoutubeProbeSample(const std::shared_ptr &session, Scrat sample.congestionSamples = (uint32_t)congestionValues.size(); for (float congestion : congestionValues) { sample.maximumCongestion = std::max(sample.maximumCongestion, congestion); - if (congestion >= kProbeCongestionHigh) + if (congestion >= kProbeCongestionThreshold) { sample.congestionHighSamples++; - if (congestion >= kProbeCongestionSevere) sample.congestionSevereSamples++; + } } std::sort(congestionValues.begin(), congestionValues.end()); if (!congestionValues.empty()) { @@ -3299,6 +3301,13 @@ static bool runEnhancedBroadcastingOutputAttempt(const std::shared_ptr return false; } + // Encoder groups synchronize startup to the renderer's clock, which the + // standalone synthetic input does not share. Release this probe-owned group + // before capture starts; the output retains the encoders and A/V pairing is + // unchanged. Texture probes keep the normal grouped startup. + if (!usePrivateTextureMix) + resources.multitrackVideoEncoderGroup.reset(); + std::vector canvasInputsBound(candidates.size(), false); for (size_t index = 0; index < config.encoder_configurations.size(); index++) { obs_encoder_t *encoder = obs_output_get_video_encoder2(resources.output, index); @@ -3593,7 +3602,7 @@ static bool runEnhancedBroadcastingOutputAttempt(const std::shared_ptr const char *outputError = obs_output_get_last_error(resources.output); const bool encoderFramesPassed = std::equal(attempt.encodedFrames.begin(), attempt.encodedFrames.end(), attempt.minimumEncodedFrames.begin(), [](uint32_t encoded, uint32_t minimum) { return encoded >= minimum; }); - const bool transportPassed = attempt.outputDroppedFrames == 0 && attempt.maximumCongestion < kProbeCongestionHigh; + const bool transportPassed = attempt.outputDroppedFrames == 0 && attempt.maximumCongestion < kProbeCongestionThreshold; bool companionsPassed = true; for (const CompanionSample &sample : companionSamples) { const uint32_t encoded = obs_encoder_get_encoded_frames(sample.resources->videoEncoder) - sample.encodedStart; @@ -3843,6 +3852,7 @@ static ProbeResult runEnhancedBroadcastingProbe(const std::shared_ptr & : textureAttempt.errorCode; if (!enhancedBroadcastingPolicy::allowsCandidateDescent(result.errorCode)) return result; + result.candidateFallback.record(result.errorCode); pushEvent(session, "progress", "bandwidth", candidateEnd, "enhanced_broadcasting_candidate_rejected", probe.legId, "active", probe.probeId, probe.provider, 0, &eventVideo, 0, 0, eventAdditionalVideoPtr); continue; @@ -3866,6 +3876,7 @@ static ProbeResult runEnhancedBroadcastingProbe(const std::shared_ptr & : exactAttempt.errorCode; if (!enhancedBroadcastingPolicy::allowsCandidateDescent(result.errorCode)) return result; + result.candidateFallback.record(result.errorCode); pushEvent(session, "progress", "bandwidth", candidateEnd, "enhanced_broadcasting_candidate_rejected", probe.legId, "active", probe.probeId, probe.provider, 0, &eventVideo, 0, 0, eventAdditionalVideoPtr); continue; @@ -4828,7 +4839,7 @@ static void runSession(const std::shared_ptr &session) const ProbeResult &tested = **enhancedResult; recommendation.measurementMode = "active"; recommendation.confidence = tested.pairedCadenceEvidence ? "medium" : "high"; - recommendation.reason.clear(); + recommendation.reason = tested.candidateFallback.reason(); recommendation.value.width = (int)tested.testedWidth; recommendation.value.height = (int)tested.testedHeight; recommendation.value.fpsNum = (int)tested.testedFpsNum; diff --git a/obs-studio-server/tests/test-osn-video-mix-lifecycle.cpp b/obs-studio-server/tests/test-osn-video-mix-lifecycle.cpp index 5a8d8b649..a4ea42055 100644 --- a/obs-studio-server/tests/test-osn-video-mix-lifecycle.cpp +++ b/obs-studio-server/tests/test-osn-video-mix-lifecycle.cpp @@ -4,9 +4,11 @@ #include #include +#include #include #include #include +#include #include #include "auto-optimizer-video-mix.hpp" @@ -25,6 +27,7 @@ constexpr char TEST_OUTPUT_ID[] = "osn_test_video_output"; constexpr char TEST_RAW_VIDEO_ENCODER_ID[] = "osn_test_raw_video_encoder"; constexpr char TEST_AUDIO_ENCODER_ID[] = "osn_test_audio_encoder"; constexpr char TEST_AV_OUTPUT_ID[] = "osn_test_av_output"; +constexpr char TEST_MULTITRACK_AV_OUTPUT_ID[] = "osn_test_multitrack_av_output"; constexpr uint32_t SOURCE_WIDTH = 1280; constexpr uint32_t SOURCE_HEIGHT = 720; constexpr uint32_t SCALED_WIDTH = 960; @@ -50,7 +53,7 @@ bool testEncoderEncodeTexture(void *, encoder_texture *, int64_t, uint64_t, uint struct TestOutputContext { obs_output_t *output = nullptr; - std::atomic videoPackets{0}; + std::array, MAX_OUTPUT_VIDEO_ENCODERS> videoPackets{}; std::atomic audioPackets{0}; }; @@ -89,8 +92,8 @@ void testOutputPacket(void *data, encoder_packet *packet) auto *context = static_cast(data); if (!packet) return; - if (packet->type == OBS_ENCODER_VIDEO) - context->videoPackets.fetch_add(1); + if (packet->type == OBS_ENCODER_VIDEO && packet->track_idx < context->videoPackets.size()) + context->videoPackets[packet->track_idx].fetch_add(1); else if (packet->type == OBS_ENCODER_AUDIO) context->audioPackets.fetch_add(1); } @@ -160,6 +163,10 @@ void registerTestTypes() outputInfo.id = TEST_AV_OUTPUT_ID; outputInfo.flags |= OBS_OUTPUT_AUDIO; obs_register_output(&outputInfo); + + outputInfo.id = TEST_MULTITRACK_AV_OUTPUT_ID; + outputInfo.flags |= OBS_OUTPUT_MULTI_TRACK_VIDEO; + obs_register_output(&outputInfo); } obs_video_info makeVideoInfo() @@ -279,13 +286,23 @@ class VideoMixResources { bool cleaned = false; }; +enum class AudioVideoWorkload { + SingleTrack, + Multitrack60Fps, +}; + class AudioVideoResources { public: ~AudioVideoResources() { cleanup(); } - bool initialize(bool standaloneVideo = true) + bool initialize(bool standaloneVideo = true, AudioVideoWorkload workload = AudioVideoWorkload::SingleTrack) { + const bool multitrack = workload == AudioVideoWorkload::Multitrack60Fps; + trackCount = multitrack ? videoEncoders.size() : 1; + inputFps = multitrack ? 60 : 30; obs_video_info info = makeVideoInfo(); + if (multitrack) + info.fps_num = 30; if (obs_reset_video(&info) != OBS_VIDEO_SUCCESS) return false; canvas = obs_get_video_info_by_index2(0); @@ -296,7 +313,7 @@ class AudioVideoResources { video_output_info videoInfo{}; videoInfo.name = "osn standalone A/V input"; videoInfo.format = VIDEO_FORMAT_NV12; - videoInfo.fps_num = 30; + videoInfo.fps_num = inputFps; videoInfo.fps_den = 1; videoInfo.width = 64; videoInfo.height = 64; @@ -316,44 +333,85 @@ class AudioVideoResources { if (audio_output_open(&audio, &audioInfo) != AUDIO_OUTPUT_SUCCESS) return false; - videoEncoder = obs_video_encoder_create(TEST_RAW_VIDEO_ENCODER_ID, "osn A/V video encoder", nullptr, nullptr); audioEncoder = obs_audio_encoder_create(TEST_AUDIO_ENCODER_ID, "osn A/V audio encoder", nullptr, 0, nullptr); - if (!videoEncoder || !audioEncoder) + if (!audioEncoder) return false; - if (ownedVideo) { - obs_encoder_set_video(videoEncoder, ownedVideo); - } else { - // obs_get_video() belongs to the core main canvas, which survives a - // partial video reset. Bind the registered canvas removed by this test. - obs_core_video_mix_t *mix = obs_video_mix_get(canvas, OBS_MAIN_VIDEO_RENDERING); - if (!mix) + for (size_t track = 0; track < trackCount; track++) { + const auto name = "osn A/V video encoder " + std::to_string(track); + auto *&encoder = videoEncoders[track]; + encoder = obs_video_encoder_create(TEST_RAW_VIDEO_ENCODER_ID, name.c_str(), nullptr, nullptr); + if (!encoder) return false; - obs_encoder_set_video_mix(videoEncoder, mix); + if (ownedVideo) { + obs_encoder_set_video(encoder, ownedVideo); + } else { + // obs_get_video() belongs to the core main canvas, which survives a + // partial video reset. Bind the registered canvas removed by this test. + obs_core_video_mix_t *mix = obs_video_mix_get(canvas, OBS_MAIN_VIDEO_RENDERING); + if (!mix) + return false; + obs_encoder_set_video_mix(encoder, mix); + } + // Match the 60/60/30/30 FPS ladder returned by Twitch for the failed + // promotion, without requiring NVENC, Twitch credentials, or a network. + obs_encoder_set_frame_rate_divisor(encoder, track < 2 ? 1 : 2); } obs_encoder_set_audio(audioEncoder, audio); - output = obs_output_create(TEST_AV_OUTPUT_ID, "osn A/V output", nullptr, nullptr); + output = obs_output_create(multitrack ? TEST_MULTITRACK_AV_OUTPUT_ID : TEST_AV_OUTPUT_ID, "osn A/V output", nullptr, nullptr); if (!output) return false; - obs_output_set_video_encoder(output, videoEncoder); + for (size_t track = 0; track < trackCount; track++) + obs_output_set_video_encoder2(output, videoEncoders[track], track); obs_output_set_audio_encoder(output, audioEncoder, 0); return true; } + bool groupVideoEncoders() + { + encoderGroup = obs_encoder_group_create(); + if (!encoderGroup) + return false; + for (size_t track = 0; track < trackCount; track++) { + if (!obs_encoder_set_group(videoEncoders[track], encoderGroup)) + return false; + } + return true; + } + + void releaseVideoEncoderGroup() + { + if (encoderGroup) { + obs_encoder_group_destroy(encoderGroup); + encoderGroup = nullptr; + } + } + bool initializeEncoders() { return obs_output_initialize_encoders(output, 0); } bool start() { return obs_output_start(output); } - bool hasVideoInput() const { return obs_encoder_video(videoEncoder) != nullptr; } + bool hasVideoInput() const { return obs_encoder_video(videoEncoders[0]) != nullptr; } + uint32_t inputFrames() const { return video_output_get_total_frames(ownedVideo); } + uint32_t videoPackets(size_t track) const { return static_cast(obs_obj_get_data(output))->videoPackets[track].load(); } + uint32_t audioPackets() const { return static_cast(obs_obj_get_data(output))->audioPackets.load(); } bool waitForAudioVideoPackets() { - auto *context = static_cast(obs_obj_get_data(output)); const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(3); + const auto frameDuration = std::chrono::nanoseconds(1000000000ULL / inputFps); + auto nextFrame = std::chrono::steady_clock::now(); + // This input deliberately has its own timestamp origin, just like the + // optimizer's synthetic feeder. Do not align it to the graphics thread: + // doing so would hide the grouped-encoder startup regression. + uint64_t timestamp = os_gettime_ns(); while (std::chrono::steady_clock::now() < deadline) { - if (context->videoPackets.load() >= 3 && context->audioPackets.load() >= 3) + bool allTracksDelivered = audioPackets() >= 3; + for (size_t track = 0; track < trackCount; track++) + allTracksDelivered = allTracksDelivered && videoPackets(track) >= 3; + if (allTracksDelivered) return true; video_frame frame{}; - if (video_output_lock_frame(ownedVideo, &frame, 1, os_gettime_ns())) { + if (video_output_lock_frame(ownedVideo, &frame, 1, timestamp)) { for (size_t row = 0; row < 64; row++) std::memset(frame.data[0] + row * frame.linesize[0], 16, 64); for (size_t row = 0; row < 32; row++) @@ -362,7 +420,9 @@ class AudioVideoResources { } // Feed on this bounded test loop, avoiding a worker that could outlive // an assertion failure and retain the standalone video input. - std::this_thread::sleep_for(std::chrono::milliseconds(33)); + timestamp += frameDuration.count(); + nextFrame += frameDuration; + std::this_thread::sleep_until(nextFrame); } return false; } @@ -387,9 +447,11 @@ class AudioVideoResources { obs_output_release(output); output = nullptr; } - if (videoEncoder) { - obs_encoder_release(videoEncoder); - videoEncoder = nullptr; + releaseVideoEncoderGroup(); + for (auto *&encoder : videoEncoders) { + if (encoder) + obs_encoder_release(encoder); + encoder = nullptr; } if (audioEncoder) { obs_encoder_release(audioEncoder); @@ -411,11 +473,46 @@ class AudioVideoResources { obs_video_info *canvas = nullptr; video_t *ownedVideo = nullptr; audio_t *audio = nullptr; - obs_encoder_t *videoEncoder = nullptr; + std::array videoEncoders{}; + size_t trackCount = 1; + uint32_t inputFps = 30; + obs_encoder_group_t *encoderGroup = nullptr; obs_encoder_t *audioEncoder = nullptr; obs_output_t *output = nullptr; }; +void checkStandaloneMultitrackPackets(bool releaseGroupBeforeStart) +{ + osn::tests::ObsSetup setup; + registerTestTypes(); + + // Exercise a fresh output twice to check that releasing the group and + // cleaning up capture leave no stale encoder, audio-pairing, or input state. + for (int iteration = 0; iteration < 2; iteration++) { + INFO("lifecycle iteration " << iteration << ", release group before start=" << releaseGroupBeforeStart); + AudioVideoResources resources; + REQUIRE(resources.initialize(true, AudioVideoWorkload::Multitrack60Fps)); + if (releaseGroupBeforeStart) { + REQUIRE(resources.groupVideoEncoders()); + // Match the optimizer's synthetic probe: release the renderer-clock + // group before capture while the output still owns all encoders. + resources.releaseVideoEncoderGroup(); + } + REQUIRE(resources.start()); + resources.waitForAudioVideoPackets(); + CHECK(resources.inputFrames() >= 3); + for (size_t track = 0; track < 4; track++) { + INFO("video track " << track); + CHECK(resources.videoPackets(track) >= 3); + } + CHECK(resources.audioPackets() >= 3); + const auto cleanupStart = std::chrono::steady_clock::now(); + CHECK(resources.cleanup() == OBS_VIDEO_SUCCESS); + CHECK(std::chrono::steady_clock::now() - cleanupStart < std::chrono::seconds(3)); + CHECK(obs_get_video_info_by_index2(0) == nullptr); + } +} + } // namespace TEST_CASE("Standalone video and audio inputs deliver encoded A/V packets", "[video-mix][standalone-av]") @@ -452,6 +549,16 @@ TEST_CASE("Encoded A/V output cannot start after its canvas input is removed", " CHECK(resources.cleanup() == OBS_VIDEO_SUCCESS); } +TEST_CASE("Ungrouped standalone 60 FPS ladder delivers A/V with a 30 FPS renderer", "[video-mix][standalone-av][multitrack-cadence]") +{ + checkStandaloneMultitrackPackets(false); +} + +TEST_CASE("Standalone 60 FPS ladder delivers A/V after releasing the renderer encoder group", "[video-mix][standalone-av][multitrack-cadence]") +{ + checkStandaloneMultitrackPackets(true); +} + TEST_CASE("Encoder GPU rescale supports a canvas-owned identity across reinitialization", "[video-mix][canvas-identity]") { osn::tests::ObsSetup setup; diff --git a/source/auto-optimizer-enhanced-broadcasting-policy.hpp b/source/auto-optimizer-enhanced-broadcasting-policy.hpp index 4909990d0..c3174397c 100644 --- a/source/auto-optimizer-enhanced-broadcasting-policy.hpp +++ b/source/auto-optimizer-enhanced-broadcasting-policy.hpp @@ -152,4 +152,29 @@ inline bool allowsCandidateDescent(std::string_view errorCode) errorCode == "enhanced_broadcasting_companion_overload"; } +// Keep earlier candidate failures after a lower-quality candidate passes so the +// measured recommendation can explain the fallback without claiming it failed. +struct CandidateFallbackEvidence { + bool transportPressure = false; + bool workloadPressure = false; + + void record(std::string_view errorCode) + { + transportPressure |= errorCode == "enhanced_broadcasting_transport_pressure"; + workloadPressure |= errorCode == "enhanced_broadcasting_encoder_underload" || errorCode == "enhanced_broadcasting_render_overload" || + errorCode == "enhanced_broadcasting_companion_overload"; + } + + std::string_view reason() const + { + if (transportPressure && workloadPressure) + return "enhanced_broadcasting_transport_and_workload_fallback"; + if (transportPressure) + return "enhanced_broadcasting_transport_fallback"; + if (workloadPressure) + return "enhanced_broadcasting_workload_fallback"; + return {}; + } +}; + } // namespace autoOptimizer::enhancedBroadcastingPolicy