Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,19 @@ public static class AppArgsFlags
public const string DEBUG = "debug";
public const string DCL_EDITOR = "hub";

// EXIT-DELAY INVESTIGATION (#8764): bisection flags for VoiceChatPlugin init.
// Used with the value 1..8 to stop InitializeAsync at successive stages and
// identify which component leaves livekit_ffi tokio threads attached to the
// IL2CPP runtime, preventing process shutdown.
// Remove once the offending component is identified and the dispose path is fixed.
public const string EXIT_TEST_VOICE_INIT_STOP = "exit-test-voice-init-stop";
public const string EXIT_TEST_SKIP_NEARBY_VOICE_SYSTEMS = "exit-test-skip-nearby-voice-systems";
public const string EXIT_TEST_NEARBY_INJECT_STOP = "exit-test-nearby-inject-stop";
public const string EXIT_TEST_SKIP_AUDIO_SOURCE_CREATE = "exit-test-skip-audio-source-create";
public const string EXIT_TEST_SKIP_GET_ACTIVE_STREAM = "exit-test-skip-get-active-stream";
public const string EXIT_TEST_DISCONNECT_ROOMS_ON_QUIT = "exit-test-disconnect-rooms-on-quit";
public const string EXIT_TEST_POST_DISCONNECT_DELAY_MS = "exit-test-post-disconnect-delay-ms";

public const string SKIP_VERSION_CHECK = "skip-version-check";
public const string SIMULATE_VERSION = "simulateVersion";
public const string FORCE_MINIMUM_SPECS_SCREEN = "forceMinimumSpecsScreen";
Expand Down
203 changes: 203 additions & 0 deletions Explorer/Assets/DCL/PluginSystem/Global/VoiceChatPlugin.cs
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
using LiveKit.Rooms.Streaming;
using LiveKit.Rooms.Streaming.Audio;
using LiveKit.Rooms;
using Global.AppArgs;
using System;
using System.Collections.Generic;
using System.Threading;
Expand Down Expand Up @@ -88,6 +89,13 @@ public class VoiceChatPlugin : IDCLGlobalPlugin<VoiceChatPlugin.Settings>
private CancellationTokenSource? nearbyTipCts;
private VoiceChatConfiguration voiceChatConfiguration;

// EXIT-DELAY BISECTION (#8764): state for the wantsToQuit interception flow.
// disconnectsCompleted starts false; on the first wantsToQuit we cancel the quit,
// run the async room disconnects, set the flag, and re-issue Application.Quit().
// The second wantsToQuit sees the flag and returns true so Unity proceeds with shutdown.
private bool disconnectsCompleted;
private bool disconnectsInFlight;

public VoiceChatPlugin(
IRoomHub roomHub,
VoiceChatPanelView voiceChatPanelView,
Expand Down Expand Up @@ -135,10 +143,23 @@ public VoiceChatPlugin(
this.volumeBus = volumeBus;

voiceChatOrchestrator = voiceChatContainer.VoiceChatOrchestrator;

// EXIT-DELAY BISECTION (#8764): hook Application.wantsToQuit so we can run the
// LiveKit room disconnects while the PlayerLoop is still pumping. Returning false
// cancels the quit, we drive the disconnects async, then re-issue Application.Quit()
// when complete. The second wantsToQuit invocation sees disconnectsCompleted=true
// and lets Unity proceed. This avoids the sync-over-async deadlock that .AsTask().Wait()
// hit in the previous attempt: blocking the main thread prevented UniTask continuations
// from resuming, so DisconnectInstruction.AwaitWithSuccess never completed.
if (HasExitTestDisconnectRoomsOnQuit())
Application.wantsToQuit += OnWantsToQuitInterceptForRoomDisconnect;
}

public void Dispose()
{
if (HasExitTestDisconnectRoomsOnQuit())
Application.wantsToQuit -= OnWantsToQuitInterceptForRoomDisconnect;

nearbyTipCts.SafeCancelAndDispose();
pluginScope.Dispose();

Expand All @@ -148,19 +169,177 @@ public void Dispose()
RustAudioClient.DeInit();
}

// EXIT-DELAY BISECTION (#8764): intercept the quit, run room disconnects asynchronously
// while the PlayerLoop is still active, then re-trigger Application.Quit() once complete.
private bool OnWantsToQuitInterceptForRoomDisconnect()
{
if (disconnectsCompleted)
{
ReportHub.LogWarning(ReportCategory.ALWAYS, "EXIT TEST: wantsToQuit second pass — disconnects complete, allowing quit");
return true;
}

if (disconnectsInFlight)
{
// Should not happen because Unity only invokes wantsToQuit once per Quit() call,
// but guard against re-entry just in case.
ReportHub.LogWarning(ReportCategory.ALWAYS, "EXIT TEST: wantsToQuit re-entered while disconnects in flight — blocking quit");
return false;
}

disconnectsInFlight = true;
ReportHub.LogWarning(ReportCategory.ALWAYS, "EXIT TEST: wantsToQuit — deferring quit to disconnect LiveKit rooms");
DisconnectRoomsThenQuitAsync().Forget();
return false;
}

private async UniTaskVoid DisconnectRoomsThenQuitAsync()
{
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10));
try
{
await UniTask.WhenAll(
roomHub.IslandRoom().DisconnectAsync(cts.Token),
roomHub.VoiceChatRoom().Room().DisconnectAsync(cts.Token));
ReportHub.LogWarning(ReportCategory.ALWAYS, "EXIT TEST: LiveKit room disconnects complete");
}
catch (OperationCanceledException)
{
ReportHub.LogWarning(ReportCategory.ALWAYS, "EXIT TEST: LiveKit room disconnects timed out after 10s");
}
catch (Exception ex)
{
ReportHub.LogException(ex, ReportCategory.ALWAYS);
}

int postDelayMs = GetExitTestPostDisconnectDelayMs();
if (postDelayMs > 0)
{
ReportHub.LogWarning(ReportCategory.ALWAYS, $"EXIT TEST: post-disconnect delay {postDelayMs}ms before Application.Quit()");
try { await UniTask.Delay(postDelayMs, ignoreTimeScale: true); }
catch (Exception ex) { ReportHub.LogException(ex, ReportCategory.ALWAYS); }
ReportHub.LogWarning(ReportCategory.ALWAYS, "EXIT TEST: post-disconnect delay elapsed");
}

disconnectsCompleted = true;
Application.Quit();
}

// EXIT-DELAY BISECTION (#8764): helpers to read --exit-test-* flags directly
// from the process command line. Kept local to avoid plumbing IAppArgs through
// the plugin's DI graph for an investigation toggle.
private static int GetExitTestVoiceInitStopStage()
{
string[] args = Environment.GetCommandLineArgs();
string prefix = "--" + AppArgsFlags.EXIT_TEST_VOICE_INIT_STOP + "=";
for (var i = 0; i < args.Length; i++)
{
if (args[i].StartsWith(prefix, StringComparison.Ordinal)
&& int.TryParse(args[i].AsSpan(prefix.Length), out int n))
return n;
}
return 0; // 0 = no stop, full init
}

private static bool HasExitTestSkipNearbyVoiceSystems()
{
string[] args = Environment.GetCommandLineArgs();
string dashed = "--" + AppArgsFlags.EXIT_TEST_SKIP_NEARBY_VOICE_SYSTEMS;
for (var i = 0; i < args.Length; i++)
if (args[i] == dashed)
return true;
return false;
}

private static int GetExitTestNearbyInjectStopStage()
{
string[] args = Environment.GetCommandLineArgs();
string prefix = "--" + AppArgsFlags.EXIT_TEST_NEARBY_INJECT_STOP + "=";
for (var i = 0; i < args.Length; i++)
{
if (args[i].StartsWith(prefix, StringComparison.Ordinal)
&& int.TryParse(args[i].AsSpan(prefix.Length), out int n))
return n;
}
return 0; // 0 = no stop, all systems injected
}

private static bool HasExitTestDisconnectRoomsOnQuit()
{
string[] args = Environment.GetCommandLineArgs();
string dashed = "--" + AppArgsFlags.EXIT_TEST_DISCONNECT_ROOMS_ON_QUIT;
for (var i = 0; i < args.Length; i++)
if (args[i] == dashed)
return true;
return false;
}

// EXIT-DELAY BISECTION (#8764): post-disconnect delay in ms before Application.Quit().
// Hypothesis: after DisconnectAsync returns, the FFI tokio runtime needs a brief window
// to wind down its worker threads. Returning 0 keeps the previous behavior.
private static int GetExitTestPostDisconnectDelayMs()
{
string[] args = Environment.GetCommandLineArgs();
string prefix = "--" + AppArgsFlags.EXIT_TEST_POST_DISCONNECT_DELAY_MS + "=";
for (var i = 0; i < args.Length; i++)
{
if (args[i].StartsWith(prefix, StringComparison.Ordinal)
&& int.TryParse(args[i].AsSpan(prefix.Length), out int n))
return n;
}
return 0;
}

public void InjectToWorld(ref ArchSystemsWorldBuilder<Arch.Core.World> builder, in GlobalPluginArguments arguments)
{
// EXIT-DELAY BISECTION (#8764): skip the nearby voice ECS systems independently
// of the InitializeAsync stop, and defensively guard against null when init was
// halted before the NEARBY block ran.
if (HasExitTestSkipNearbyVoiceSystems())
{
ReportHub.LogWarning(ReportCategory.ALWAYS, "EXIT TEST: skipping NEARBY voice systems in InjectToWorld");
return;
}
if (nearbyAudioStreamRegistry == null)
return;

if (FeaturesRegistry.Instance.IsEnabled(FeatureId.NEARBY_VOICE_CHAT))
{
// EXIT-DELAY BISECTION (#8764):
// Read --exit-test-nearby-inject-stop=N and stop after registering N ECS systems
// to identify which NEARBY voice system is leaving livekit_ffi tokio threads
// attached to the IL2CPP runtime.
int stopAfter = GetExitTestNearbyInjectStopStage();
if (stopAfter > 0)
ReportHub.LogWarning(ReportCategory.ALWAYS, $"EXIT TEST: NEARBY inject will stop after stage {stopAfter}");

var listenerState = new NearbyListenerState();

// Stage 1: NearbyLivekitBridgeSystem (primary suspect, bridges to livekit_ffi)
NearbyLivekitBridgeSystem.InjectToWorld(ref builder, nearbyAudioStreamRegistry!);
if (stopAfter == 1) return;

// Stage 2: NearbyAudibleRangeSystem
NearbyAudibleRangeSystem.InjectToWorld(ref builder, voiceChatConfiguration, listenerState);
if (stopAfter == 2) return;

// Stage 3: NearbyAudioBindingSystem
NearbyAudioBindingSystem.InjectToWorld(ref builder, nearbyAudioStreamRegistry!, nearbyAudioBindings!, userBlockingCache, nearbyStateModel!, nearbyAudioSourceFactory!, RoomMetadataCurrentScene.Instance);
if (stopAfter == 3) return;

// Stage 4: NearbyAudioPositionSystem
NearbyAudioPositionSystem.InjectToWorld(ref builder, nearbyMuteService!, listenerState);
if (stopAfter == 4) return;

// Stage 5: NearbyAudioCleanupSystem
NearbyAudioCleanupSystem.InjectToWorld(ref builder, nearbyAudioStreamRegistry!, nearbyAudioBindings!, userBlockingCache, nearbyStateModel!, nearbyAudioSourceFactory!, RoomMetadataCurrentScene.Instance);
if (stopAfter == 5) return;

// Stage 6: NearbyVoiceChatNametagSystem
NearbyVoiceChatNametagSystem.InjectToWorld(ref builder, playerEntity, nearbyAudioStreamRegistry!, nearbyStateModel!, nearbyMuteService!);
if (stopAfter == 6) return;

// Stage 7: NearbyVoiceChatDebugSystem
NearbyVoiceChatDebugSystem.InjectToWorld(ref builder, voiceChatConfiguration, debugContainer, roomHub.IslandRoom(), nearbyStateModel!, nearbyAudioStreamRegistry!, entityParticipantTable);
}
}
Expand All @@ -177,42 +356,66 @@ public async UniTask InitializeAsync(Settings settings, CancellationToken ct)
VoiceChatPluginSettings pluginSettings = voiceChatPluginSettingsAsset.Value;
voiceChatConfiguration = pluginSettings.VoiceChatConfiguration;

// EXIT-DELAY BISECTION (#8764):
// Read --exit-test-voice-init-stop=N and stop initialization after stage N to
// identify which voice chat component keeps livekit_ffi tokio threads attached
// to the IL2CPP runtime, preventing process shutdown.
int stopAfter = GetExitTestVoiceInitStopStage();
if (stopAfter > 0)
ReportHub.LogWarning(ReportCategory.ALWAYS, $"EXIT TEST: VoiceChat init will stop after stage {stopAfter}");

// Stage 1: local-only handlers (no Room access)
voiceChatHandler = new VoiceChatMicrophoneHandler(voiceChatConfiguration, voiceChatOrchestrator);
pluginScope.Add(voiceChatHandler);

microphoneStateManager = new VoiceChatMicrophoneStateManager(voiceChatHandler, voiceChatOrchestrator);
pluginScope.Add(microphoneStateManager);
if (stopAfter == 1) return;

// Stage 2: MicrophoneTrackPublisher — first component that touches voice chat Room
microphonePublisher = new MicrophoneTrackPublisher(roomHub.VoiceChatRoom().Room(), voiceChatConfiguration, VoiceChatType.COMMUNITY);
if (stopAfter == 2) return;

// Stage 3: RemoteTrackListener — second component that touches voice chat Room
var callPlaybackSourcesHub = new PlaybackSourcesHub("Call", voiceChatConfiguration.ChatAudioMixerGroup.EnsureNotNull());
remoteListener = new RemoteTrackListener(
roomHub.VoiceChatRoom().Room(),
voiceChatConfiguration,
callPlaybackSourcesHub);
if (stopAfter == 3) return;

// Stage 4: VoiceChatRoomManager (orchestrates publisher/listener)
roomManager = new VoiceChatRoomManager(microphonePublisher, remoteListener, roomHub, roomHub.VoiceChatRoom().Room(), voiceChatOrchestrator, voiceChatConfiguration, microphoneStateManager, voiceChatHandler);
pluginScope.Add(roomManager);
if (stopAfter == 4) return;

// Stage 5: VoiceChatNametagsHandler (subscribes to Room events)
nametagsHandler = new VoiceChatNametagsHandler(
roomHub.VoiceChatRoom().Room(),
voiceChatOrchestrator,
entityParticipantTable,
world,
playerEntity);
pluginScope.Add(nametagsHandler);
if (stopAfter == 5) return;

// Stage 6: MicrophoneAudioToggleHandler (audio cues, no Room access)
VoiceChatParticipantEntryView playerEntry = pluginSettings.PlayerEntryView;
AudioClipConfig muteMicrophoneAudio = pluginSettings.MuteMicrophoneAudio;
AudioClipConfig unmuteMicrophoneAudio = pluginSettings.UnmuteMicrophoneAudio;
microphoneAudioToggleHandler = new MicrophoneAudioToggleHandler(voiceChatHandler.IsMicrophoneEnabled, muteMicrophoneAudio, unmuteMicrophoneAudio);
pluginScope.Add(microphoneAudioToggleHandler);
if (stopAfter == 6) return;

// Stage 7: VoiceChatPanelPresenter (UI)
voiceChatPanelPresenter = new VoiceChatPanelPresenter(voiceChatPanelView, profileDataProvider, communityDataProvider, imageControllerProvider, voiceChatOrchestrator, voiceChatHandler, roomManager, roomHub, playerEntry, chatSharedAreaEventBus);
pluginScope.Add(voiceChatPanelPresenter);
if (stopAfter == 7) return;

// Stage 8: VoiceChatDebugContainer
voiceChatDebugContainer = new VoiceChatDebugContainer(debugContainer, microphonePublisher, roomHub.VoiceChatRoom().Room(), callPlaybackSourcesHub);
pluginScope.Add(voiceChatDebugContainer);
if (stopAfter == 8) return;

if (FeaturesRegistry.Instance.IsEnabled(FeatureId.NEARBY_VOICE_CHAT))
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,30 @@ public void Dispose(LivekitAudioSource? source)

public void DisposeRoot()
{
// Explicit teardown of LIVE and LEGACY instances before destroying the root.
// pool.Dispose() only runs the onRelease callback (Stop+Free) on instances
// currently in the pool; live and legacy instances would otherwise be
// destroyed via Unity.Destroy() on the parent container, which fires
// OnDestroy() (only DisposeWavWriter) but bypasses LivekitAudioSource.Stop()
// and LivekitAudioSource.Free(). The AudioSource then stays in Play state
// long enough for one or more OnAudioFilterRead callbacks to cross into
// livekit_ffi (AudioStream.ReadAudio), keeping the FFI subscriptions alive
// and attaching the consuming threads to the IL2CPP managed runtime.
// Those attached threads never detach, deadlocking il2cpp::vm::Runtime::Shutdown
// and producing the multi-second to minute-long EXIT freeze tracked in #8764.
foreach (LivekitAudioSource live in liveInstances)
{
if (live == null) continue;
live.Stop();
live.Free();
}
foreach (LivekitAudioSource legacy in legacyInstances)
{
if (legacy == null) continue;
legacy.Stop();
legacy.Free();
}

pool.Dispose();
UnityObjectUtils.SafeDestroyGameObject(pool.ParentContainer);
liveInstances.Clear();
Expand Down
Loading
Loading