diff --git a/Explorer/Assets/DCL/Infrastructure/Global/AppArgs/AppArgsFlags.cs b/Explorer/Assets/DCL/Infrastructure/Global/AppArgs/AppArgsFlags.cs index 376546396a..dbe7d7191f 100644 --- a/Explorer/Assets/DCL/Infrastructure/Global/AppArgs/AppArgsFlags.cs +++ b/Explorer/Assets/DCL/Infrastructure/Global/AppArgs/AppArgsFlags.cs @@ -5,6 +5,19 @@ public static class AppArgsFlags public const string DEBUG = "debug"; public const string DCL_EDITOR = "hub"; + // EXIT-DELAY INVESTIGATION (#8764): bisection flags for VoiceChatPlugin init. + // Used with the value 1..8 to stop InitializeAsync at successive stages and + // identify which component leaves livekit_ffi tokio threads attached to the + // IL2CPP runtime, preventing process shutdown. + // Remove once the offending component is identified and the dispose path is fixed. + public const string EXIT_TEST_VOICE_INIT_STOP = "exit-test-voice-init-stop"; + public const string EXIT_TEST_SKIP_NEARBY_VOICE_SYSTEMS = "exit-test-skip-nearby-voice-systems"; + public const string EXIT_TEST_NEARBY_INJECT_STOP = "exit-test-nearby-inject-stop"; + public const string EXIT_TEST_SKIP_AUDIO_SOURCE_CREATE = "exit-test-skip-audio-source-create"; + public const string EXIT_TEST_SKIP_GET_ACTIVE_STREAM = "exit-test-skip-get-active-stream"; + public const string EXIT_TEST_DISCONNECT_ROOMS_ON_QUIT = "exit-test-disconnect-rooms-on-quit"; + public const string EXIT_TEST_POST_DISCONNECT_DELAY_MS = "exit-test-post-disconnect-delay-ms"; + public const string SKIP_VERSION_CHECK = "skip-version-check"; public const string SIMULATE_VERSION = "simulateVersion"; public const string FORCE_MINIMUM_SPECS_SCREEN = "forceMinimumSpecsScreen"; diff --git a/Explorer/Assets/DCL/PluginSystem/Global/VoiceChatPlugin.cs b/Explorer/Assets/DCL/PluginSystem/Global/VoiceChatPlugin.cs index a2580784aa..28363638d7 100644 --- a/Explorer/Assets/DCL/PluginSystem/Global/VoiceChatPlugin.cs +++ b/Explorer/Assets/DCL/PluginSystem/Global/VoiceChatPlugin.cs @@ -21,6 +21,7 @@ using LiveKit.Rooms.Streaming; using LiveKit.Rooms.Streaming.Audio; using LiveKit.Rooms; +using Global.AppArgs; using System; using System.Collections.Generic; using System.Threading; @@ -88,6 +89,13 @@ public class VoiceChatPlugin : IDCLGlobalPlugin private CancellationTokenSource? nearbyTipCts; private VoiceChatConfiguration voiceChatConfiguration; + // EXIT-DELAY BISECTION (#8764): state for the wantsToQuit interception flow. + // disconnectsCompleted starts false; on the first wantsToQuit we cancel the quit, + // run the async room disconnects, set the flag, and re-issue Application.Quit(). + // The second wantsToQuit sees the flag and returns true so Unity proceeds with shutdown. + private bool disconnectsCompleted; + private bool disconnectsInFlight; + public VoiceChatPlugin( IRoomHub roomHub, VoiceChatPanelView voiceChatPanelView, @@ -135,10 +143,23 @@ public VoiceChatPlugin( this.volumeBus = volumeBus; voiceChatOrchestrator = voiceChatContainer.VoiceChatOrchestrator; + + // EXIT-DELAY BISECTION (#8764): hook Application.wantsToQuit so we can run the + // LiveKit room disconnects while the PlayerLoop is still pumping. Returning false + // cancels the quit, we drive the disconnects async, then re-issue Application.Quit() + // when complete. The second wantsToQuit invocation sees disconnectsCompleted=true + // and lets Unity proceed. This avoids the sync-over-async deadlock that .AsTask().Wait() + // hit in the previous attempt: blocking the main thread prevented UniTask continuations + // from resuming, so DisconnectInstruction.AwaitWithSuccess never completed. + if (HasExitTestDisconnectRoomsOnQuit()) + Application.wantsToQuit += OnWantsToQuitInterceptForRoomDisconnect; } public void Dispose() { + if (HasExitTestDisconnectRoomsOnQuit()) + Application.wantsToQuit -= OnWantsToQuitInterceptForRoomDisconnect; + nearbyTipCts.SafeCancelAndDispose(); pluginScope.Dispose(); @@ -148,19 +169,177 @@ public void Dispose() RustAudioClient.DeInit(); } + // EXIT-DELAY BISECTION (#8764): intercept the quit, run room disconnects asynchronously + // while the PlayerLoop is still active, then re-trigger Application.Quit() once complete. + private bool OnWantsToQuitInterceptForRoomDisconnect() + { + if (disconnectsCompleted) + { + ReportHub.LogWarning(ReportCategory.ALWAYS, "EXIT TEST: wantsToQuit second pass — disconnects complete, allowing quit"); + return true; + } + + if (disconnectsInFlight) + { + // Should not happen because Unity only invokes wantsToQuit once per Quit() call, + // but guard against re-entry just in case. + ReportHub.LogWarning(ReportCategory.ALWAYS, "EXIT TEST: wantsToQuit re-entered while disconnects in flight — blocking quit"); + return false; + } + + disconnectsInFlight = true; + ReportHub.LogWarning(ReportCategory.ALWAYS, "EXIT TEST: wantsToQuit — deferring quit to disconnect LiveKit rooms"); + DisconnectRoomsThenQuitAsync().Forget(); + return false; + } + + private async UniTaskVoid DisconnectRoomsThenQuitAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + try + { + await UniTask.WhenAll( + roomHub.IslandRoom().DisconnectAsync(cts.Token), + roomHub.VoiceChatRoom().Room().DisconnectAsync(cts.Token)); + ReportHub.LogWarning(ReportCategory.ALWAYS, "EXIT TEST: LiveKit room disconnects complete"); + } + catch (OperationCanceledException) + { + ReportHub.LogWarning(ReportCategory.ALWAYS, "EXIT TEST: LiveKit room disconnects timed out after 10s"); + } + catch (Exception ex) + { + ReportHub.LogException(ex, ReportCategory.ALWAYS); + } + + int postDelayMs = GetExitTestPostDisconnectDelayMs(); + if (postDelayMs > 0) + { + ReportHub.LogWarning(ReportCategory.ALWAYS, $"EXIT TEST: post-disconnect delay {postDelayMs}ms before Application.Quit()"); + try { await UniTask.Delay(postDelayMs, ignoreTimeScale: true); } + catch (Exception ex) { ReportHub.LogException(ex, ReportCategory.ALWAYS); } + ReportHub.LogWarning(ReportCategory.ALWAYS, "EXIT TEST: post-disconnect delay elapsed"); + } + + disconnectsCompleted = true; + Application.Quit(); + } + + // EXIT-DELAY BISECTION (#8764): helpers to read --exit-test-* flags directly + // from the process command line. Kept local to avoid plumbing IAppArgs through + // the plugin's DI graph for an investigation toggle. + private static int GetExitTestVoiceInitStopStage() + { + string[] args = Environment.GetCommandLineArgs(); + string prefix = "--" + AppArgsFlags.EXIT_TEST_VOICE_INIT_STOP + "="; + for (var i = 0; i < args.Length; i++) + { + if (args[i].StartsWith(prefix, StringComparison.Ordinal) + && int.TryParse(args[i].AsSpan(prefix.Length), out int n)) + return n; + } + return 0; // 0 = no stop, full init + } + + private static bool HasExitTestSkipNearbyVoiceSystems() + { + string[] args = Environment.GetCommandLineArgs(); + string dashed = "--" + AppArgsFlags.EXIT_TEST_SKIP_NEARBY_VOICE_SYSTEMS; + for (var i = 0; i < args.Length; i++) + if (args[i] == dashed) + return true; + return false; + } + + private static int GetExitTestNearbyInjectStopStage() + { + string[] args = Environment.GetCommandLineArgs(); + string prefix = "--" + AppArgsFlags.EXIT_TEST_NEARBY_INJECT_STOP + "="; + for (var i = 0; i < args.Length; i++) + { + if (args[i].StartsWith(prefix, StringComparison.Ordinal) + && int.TryParse(args[i].AsSpan(prefix.Length), out int n)) + return n; + } + return 0; // 0 = no stop, all systems injected + } + + private static bool HasExitTestDisconnectRoomsOnQuit() + { + string[] args = Environment.GetCommandLineArgs(); + string dashed = "--" + AppArgsFlags.EXIT_TEST_DISCONNECT_ROOMS_ON_QUIT; + for (var i = 0; i < args.Length; i++) + if (args[i] == dashed) + return true; + return false; + } + + // EXIT-DELAY BISECTION (#8764): post-disconnect delay in ms before Application.Quit(). + // Hypothesis: after DisconnectAsync returns, the FFI tokio runtime needs a brief window + // to wind down its worker threads. Returning 0 keeps the previous behavior. + private static int GetExitTestPostDisconnectDelayMs() + { + string[] args = Environment.GetCommandLineArgs(); + string prefix = "--" + AppArgsFlags.EXIT_TEST_POST_DISCONNECT_DELAY_MS + "="; + for (var i = 0; i < args.Length; i++) + { + if (args[i].StartsWith(prefix, StringComparison.Ordinal) + && int.TryParse(args[i].AsSpan(prefix.Length), out int n)) + return n; + } + return 0; + } + public void InjectToWorld(ref ArchSystemsWorldBuilder builder, in GlobalPluginArguments arguments) { + // EXIT-DELAY BISECTION (#8764): skip the nearby voice ECS systems independently + // of the InitializeAsync stop, and defensively guard against null when init was + // halted before the NEARBY block ran. + if (HasExitTestSkipNearbyVoiceSystems()) + { + ReportHub.LogWarning(ReportCategory.ALWAYS, "EXIT TEST: skipping NEARBY voice systems in InjectToWorld"); + return; + } + if (nearbyAudioStreamRegistry == null) + return; + if (FeaturesRegistry.Instance.IsEnabled(FeatureId.NEARBY_VOICE_CHAT)) { + // EXIT-DELAY BISECTION (#8764): + // Read --exit-test-nearby-inject-stop=N and stop after registering N ECS systems + // to identify which NEARBY voice system is leaving livekit_ffi tokio threads + // attached to the IL2CPP runtime. + int stopAfter = GetExitTestNearbyInjectStopStage(); + if (stopAfter > 0) + ReportHub.LogWarning(ReportCategory.ALWAYS, $"EXIT TEST: NEARBY inject will stop after stage {stopAfter}"); + var listenerState = new NearbyListenerState(); + // Stage 1: NearbyLivekitBridgeSystem (primary suspect, bridges to livekit_ffi) NearbyLivekitBridgeSystem.InjectToWorld(ref builder, nearbyAudioStreamRegistry!); + if (stopAfter == 1) return; + + // Stage 2: NearbyAudibleRangeSystem NearbyAudibleRangeSystem.InjectToWorld(ref builder, voiceChatConfiguration, listenerState); + if (stopAfter == 2) return; + + // Stage 3: NearbyAudioBindingSystem NearbyAudioBindingSystem.InjectToWorld(ref builder, nearbyAudioStreamRegistry!, nearbyAudioBindings!, userBlockingCache, nearbyStateModel!, nearbyAudioSourceFactory!, RoomMetadataCurrentScene.Instance); + if (stopAfter == 3) return; + + // Stage 4: NearbyAudioPositionSystem NearbyAudioPositionSystem.InjectToWorld(ref builder, nearbyMuteService!, listenerState); + if (stopAfter == 4) return; + + // Stage 5: NearbyAudioCleanupSystem NearbyAudioCleanupSystem.InjectToWorld(ref builder, nearbyAudioStreamRegistry!, nearbyAudioBindings!, userBlockingCache, nearbyStateModel!, nearbyAudioSourceFactory!, RoomMetadataCurrentScene.Instance); + if (stopAfter == 5) return; + + // Stage 6: NearbyVoiceChatNametagSystem NearbyVoiceChatNametagSystem.InjectToWorld(ref builder, playerEntity, nearbyAudioStreamRegistry!, nearbyStateModel!, nearbyMuteService!); + if (stopAfter == 6) return; + // Stage 7: NearbyVoiceChatDebugSystem NearbyVoiceChatDebugSystem.InjectToWorld(ref builder, voiceChatConfiguration, debugContainer, roomHub.IslandRoom(), nearbyStateModel!, nearbyAudioStreamRegistry!, entityParticipantTable); } } @@ -177,23 +356,40 @@ public async UniTask InitializeAsync(Settings settings, CancellationToken ct) VoiceChatPluginSettings pluginSettings = voiceChatPluginSettingsAsset.Value; voiceChatConfiguration = pluginSettings.VoiceChatConfiguration; + // EXIT-DELAY BISECTION (#8764): + // Read --exit-test-voice-init-stop=N and stop initialization after stage N to + // identify which voice chat component keeps livekit_ffi tokio threads attached + // to the IL2CPP runtime, preventing process shutdown. + int stopAfter = GetExitTestVoiceInitStopStage(); + if (stopAfter > 0) + ReportHub.LogWarning(ReportCategory.ALWAYS, $"EXIT TEST: VoiceChat init will stop after stage {stopAfter}"); + + // Stage 1: local-only handlers (no Room access) voiceChatHandler = new VoiceChatMicrophoneHandler(voiceChatConfiguration, voiceChatOrchestrator); pluginScope.Add(voiceChatHandler); microphoneStateManager = new VoiceChatMicrophoneStateManager(voiceChatHandler, voiceChatOrchestrator); pluginScope.Add(microphoneStateManager); + if (stopAfter == 1) return; + // Stage 2: MicrophoneTrackPublisher — first component that touches voice chat Room microphonePublisher = new MicrophoneTrackPublisher(roomHub.VoiceChatRoom().Room(), voiceChatConfiguration, VoiceChatType.COMMUNITY); + if (stopAfter == 2) return; + // Stage 3: RemoteTrackListener — second component that touches voice chat Room var callPlaybackSourcesHub = new PlaybackSourcesHub("Call", voiceChatConfiguration.ChatAudioMixerGroup.EnsureNotNull()); remoteListener = new RemoteTrackListener( roomHub.VoiceChatRoom().Room(), voiceChatConfiguration, callPlaybackSourcesHub); + if (stopAfter == 3) return; + // Stage 4: VoiceChatRoomManager (orchestrates publisher/listener) roomManager = new VoiceChatRoomManager(microphonePublisher, remoteListener, roomHub, roomHub.VoiceChatRoom().Room(), voiceChatOrchestrator, voiceChatConfiguration, microphoneStateManager, voiceChatHandler); pluginScope.Add(roomManager); + if (stopAfter == 4) return; + // Stage 5: VoiceChatNametagsHandler (subscribes to Room events) nametagsHandler = new VoiceChatNametagsHandler( roomHub.VoiceChatRoom().Room(), voiceChatOrchestrator, @@ -201,18 +397,25 @@ public async UniTask InitializeAsync(Settings settings, CancellationToken ct) world, playerEntity); pluginScope.Add(nametagsHandler); + if (stopAfter == 5) return; + // Stage 6: MicrophoneAudioToggleHandler (audio cues, no Room access) VoiceChatParticipantEntryView playerEntry = pluginSettings.PlayerEntryView; AudioClipConfig muteMicrophoneAudio = pluginSettings.MuteMicrophoneAudio; AudioClipConfig unmuteMicrophoneAudio = pluginSettings.UnmuteMicrophoneAudio; microphoneAudioToggleHandler = new MicrophoneAudioToggleHandler(voiceChatHandler.IsMicrophoneEnabled, muteMicrophoneAudio, unmuteMicrophoneAudio); pluginScope.Add(microphoneAudioToggleHandler); + if (stopAfter == 6) return; + // Stage 7: VoiceChatPanelPresenter (UI) voiceChatPanelPresenter = new VoiceChatPanelPresenter(voiceChatPanelView, profileDataProvider, communityDataProvider, imageControllerProvider, voiceChatOrchestrator, voiceChatHandler, roomManager, roomHub, playerEntry, chatSharedAreaEventBus); pluginScope.Add(voiceChatPanelPresenter); + if (stopAfter == 7) return; + // Stage 8: VoiceChatDebugContainer voiceChatDebugContainer = new VoiceChatDebugContainer(debugContainer, microphonePublisher, roomHub.VoiceChatRoom().Room(), callPlaybackSourcesHub); pluginScope.Add(voiceChatDebugContainer); + if (stopAfter == 8) return; if (FeaturesRegistry.Instance.IsEnabled(FeatureId.NEARBY_VOICE_CHAT)) { diff --git a/Explorer/Assets/DCL/VoiceChat/NearbyVoiceChat/Core/NearbyAudioSourceFactory.cs b/Explorer/Assets/DCL/VoiceChat/NearbyVoiceChat/Core/NearbyAudioSourceFactory.cs index f39eda647b..550866cc6b 100644 --- a/Explorer/Assets/DCL/VoiceChat/NearbyVoiceChat/Core/NearbyAudioSourceFactory.cs +++ b/Explorer/Assets/DCL/VoiceChat/NearbyVoiceChat/Core/NearbyAudioSourceFactory.cs @@ -78,6 +78,30 @@ public void Dispose(LivekitAudioSource? source) public void DisposeRoot() { + // Explicit teardown of LIVE and LEGACY instances before destroying the root. + // pool.Dispose() only runs the onRelease callback (Stop+Free) on instances + // currently in the pool; live and legacy instances would otherwise be + // destroyed via Unity.Destroy() on the parent container, which fires + // OnDestroy() (only DisposeWavWriter) but bypasses LivekitAudioSource.Stop() + // and LivekitAudioSource.Free(). The AudioSource then stays in Play state + // long enough for one or more OnAudioFilterRead callbacks to cross into + // livekit_ffi (AudioStream.ReadAudio), keeping the FFI subscriptions alive + // and attaching the consuming threads to the IL2CPP managed runtime. + // Those attached threads never detach, deadlocking il2cpp::vm::Runtime::Shutdown + // and producing the multi-second to minute-long EXIT freeze tracked in #8764. + foreach (LivekitAudioSource live in liveInstances) + { + if (live == null) continue; + live.Stop(); + live.Free(); + } + foreach (LivekitAudioSource legacy in legacyInstances) + { + if (legacy == null) continue; + legacy.Stop(); + legacy.Free(); + } + pool.Dispose(); UnityObjectUtils.SafeDestroyGameObject(pool.ParentContainer); liveInstances.Clear(); diff --git a/Explorer/Assets/DCL/VoiceChat/NearbyVoiceChat/Systems/NearbyAudioBindingSystem.cs b/Explorer/Assets/DCL/VoiceChat/NearbyVoiceChat/Systems/NearbyAudioBindingSystem.cs index c83c2288ee..c14624aa46 100644 --- a/Explorer/Assets/DCL/VoiceChat/NearbyVoiceChat/Systems/NearbyAudioBindingSystem.cs +++ b/Explorer/Assets/DCL/VoiceChat/NearbyVoiceChat/Systems/NearbyAudioBindingSystem.cs @@ -8,9 +8,11 @@ using DCL.VoiceChat.Nearby.Audio; using ECS.Abstract; using ECS.LifeCycle.Components; +using Global.AppArgs; using LiveKit.Rooms.Streaming; using LiveKit.Rooms.Streaming.Audio; using RichTypes; +using System; using System.Collections.Generic; namespace DCL.VoiceChat.Nearby.Systems @@ -51,6 +53,31 @@ protected override void OnDispose() sourceFactory.DisposeRoot(); } + private static bool HasExitTestSkipAudioSourceCreate() + { + string[] args = Environment.GetCommandLineArgs(); + string dashed = "--" + AppArgsFlags.EXIT_TEST_SKIP_AUDIO_SOURCE_CREATE; + for (var i = 0; i < args.Length; i++) + if (args[i] == dashed) + return true; + return false; + } + + // EXIT-DELAY BISECTION (#8764): skip the registry.GetActiveStream(key) call. The earlier + // SKIP_AUDIO_SOURCE_CREATE flag only skipped LivekitAudioSource.Create()/.Play(); it did + // NOT skip GetActiveStream, which lazily constructs an AudioStream and performs a + // synchronous FFI request. If this flag makes the exit consistently fast, GetActiveStream + // is what attaches the tokio workers to the IL2CPP runtime. + private static bool HasExitTestSkipGetActiveStream() + { + string[] args = Environment.GetCommandLineArgs(); + string dashed = "--" + AppArgsFlags.EXIT_TEST_SKIP_GET_ACTIVE_STREAM; + for (var i = 0; i < args.Length; i++) + if (args[i] == dashed) + return true; + return false; + } + protected override void Update(float t) { // Listening gate: skip the entire avatar query when nearby chat is SUPPRESSED or DISABLED. @@ -79,11 +106,27 @@ private void CreateAndBindAudioSourcesToStreamers(Entity avatarEntity, in Profil if (!bindings.Contains(key)) { + // EXIT-DELAY BISECTION (#8764): when --exit-test-skip-get-active-stream is set, + // bypass the registry.GetActiveStream(key) call entirely. That call lazily + // constructs a LiveKit AudioStream and performs a synchronous FFI request; + // we suspect it is what attaches tokio workers to the IL2CPP runtime. + if (HasExitTestSkipGetActiveStream()) + continue; + Weak stream = registry.GetActiveStream(key); // Track was unsubscribed between collection (snapshot read) and resolve (GetActiveStream); skip to avoid a one-frame ghost source. if (!stream.Resource.Has) continue; + // EXIT-DELAY BISECTION (#8764): when --exit-test-skip-audio-source-create is set, + // skip the LivekitAudioSource creation/Play() that triggers OnAudioFilterRead → FFI. + // The system stays registered and iterates avatars normally — only the Create call + // and the resulting NearbyAudioSourceComponent are bypassed. Used to isolate whether + // the active AudioSource binding is what keeps livekit_ffi tokio workers attached + // to the IL2CPP runtime, or whether the registration of the system alone is enough. + if (HasExitTestSkipAudioSourceCreate()) + continue; + LivekitAudioSource source = sourceFactory.Create(key, stream); World.Create(new NearbyAudioSourceComponent(key, avatarEntity, source));