Implement audio normalization using BASS

osu.Game/Audio/AudioNormalization.cs

@@ -0,0 +1,101 @@
+// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
+// See the LICENCE file in the repository root for full licence text.
+
+using System;
+using osu.Framework.Audio.Track;
+using osu.Framework.Logging;
+using osu.Game.Beatmaps;
+using osu.Game.Database;
+using Realms;
+
+namespace osu.Game.Audio
+{
+    /// <summary>
+    /// Audio Normalization Data
+    /// </summary>
+    public class AudioNormalization : EmbeddedObject, IEquatable<AudioNormalization>
+    {
+        /// <summary>
+        /// The target level for audio normalization
+        /// https://en.wikipedia.org/wiki/EBU_R_128
+        /// </summary>
+        public const int TARGET_LEVEL = -14;
+
+        /// <summary>
+        /// The integrated (average) loudness of the audio
+        /// </summary>
+        public float? IntegratedLoudness { get; init; }
+
+        /// <summary>
+        /// A volume offset that can be applied upon audio to reach <see cref="TARGET_LEVEL"/> (converted from integrated loudness).
+        /// </summary>
+        public double? IntegratedLoudnessInVolumeOffset => IntegratedLoudness != null ? TrackLoudness.ConvertToVolumeOffset(TARGET_LEVEL, (float)IntegratedLoudness) : null;
+
+        public AudioNormalization()
+        {
+        }
+
+        /// <summary>
+        /// Get the audio normalization for a beatmap. The loudness normalization value is stored in the object under <see cref="IntegratedLoudness"/>
+        /// </summary>
+        /// <param name="beatmapInfo">The <see cref="BeatmapInfo"/> of the beatmap</param>
+        /// <param name="beatmapSetInfo">The parent <see cref="BeatmapSetInfo"/> of the <see cref="BeatmapInfo"/> supplied</param>
+        /// <param name="realmFileStore">The <see cref="RealmFileStore"/> to use in getting the full path of the audio file</param>
+        public AudioNormalization(BeatmapInfo beatmapInfo, BeatmapSetInfo beatmapSetInfo, RealmFileStore realmFileStore)
+        {
+            string audiofile = beatmapInfo.Metadata.AudioFile;
+
+            if (string.IsNullOrEmpty(audiofile))
+            {
+                Logger.Log("Audio file not found for " + beatmapInfo.Metadata.Title, LoggingTarget.Runtime, LogLevel.Error);
+                return;
+            }
+
+            string? filepath = beatmapSetInfo.GetPathForFile(audiofile);
+
+            if (string.IsNullOrEmpty(filepath))
+            {
+                Logger.Log("File path not found for " + audiofile, LoggingTarget.Runtime, LogLevel.Error);
+                return;
+            }
+
+            using TrackLoudness loudness = new TrackLoudness(realmFileStore.Storage.GetStream(filepath));
+
+            float? integratedLoudness = loudness.GetIntegratedLoudness();
+
+            if (integratedLoudness == null)
+            {
+                Logger.Log("Failed to get loudness level for " + audiofile, LoggingTarget.Runtime, LogLevel.Error);
+                return;
+            }
+
+            IntegratedLoudness = integratedLoudness;
+        }
+
+        /// <summary>
+        /// Populate the audio normalization for every <see cref="BeatmapInfo"/> in a <see cref="BeatmapSetInfo"/>.
+        /// </summary>
+        /// <remarks>
+        /// This takes a <see cref="BeatmapInfo"/> and applies the same normalization to every <see cref="BeatmapInfo"/> in the <see cref="BeatmapSetInfo"/> that has the same audio file as the given <see cref="BeatmapInfo"/> to avoid having to calculate the loudness for an audio file multiple times
+        /// </remarks>
+        /// <param name="beatmapInfo">The <see cref="BeatmapInfo"/> to clone from</param>
+        /// <param name="beatmapSetInfo">The <see cref="BeatmapSetInfo"/> to populate</param>
+        public void PopulateSet(BeatmapInfo beatmapInfo, BeatmapSetInfo beatmapSetInfo)
+        {
+            foreach (BeatmapInfo beatmap in beatmapSetInfo.Beatmaps)
+            {
+                if (beatmap.AudioNormalization == null && beatmap.AudioEquals(beatmapInfo))
+                {
+                    beatmap.AudioNormalization = new AudioNormalization
+                    {
+                        IntegratedLoudness = IntegratedLoudness
+                    };
+                }
+            }
+        }
+
+
+        /// <inheritdoc />
+        public bool Equals(AudioNormalization? other) => other?.IntegratedLoudness != null && IntegratedLoudness != null && Math.Abs((float)(IntegratedLoudness - other.IntegratedLoudness)) < 0.0000001;
+    }
+}

osu.Game/Audio/AudioNormalizationManager.cs

@@ -0,0 +1,65 @@
+// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
+// See the LICENCE file in the repository root for full licence text.
+
+using osu.Framework.Audio.Track;
+using osu.Framework.Bindables;
+using osu.Framework.Logging;
+using osu.Game.Beatmaps;
+using osu.Game.Configuration;
+
+namespace osu.Game.Audio
+{
+    /// <summary>
+    /// Manages audio normalization.
+    /// </summary>
+    public class AudioNormalizationManager
+    {
+        /// <summary>
+        /// Fallback volume for tracks that <see cref="TrackLoudness"/> failed to measure loudness.
+        /// </summary>
+        public const double FALLBACK_VOLUME = 0.8;
+
+        private readonly Bindable<bool> audioNormalizationSetting;
+
+        /// <summary>
+        /// Samples assigned to hitobjects needs to bind to this bindable to normalize their volume in line with track.
+        /// </summary>
+        public readonly BindableDouble SampleNormalizeVolume = new BindableDouble(1.0);
+
+        /// <summary>
+        /// Creates a new <see cref="AudioNormalizationManager"/>.
+        /// </summary>
+        /// <param name="config">An osu config to get beatmap hitsounds bindable from.</param>
+        /// <param name="beatmap">A bindable for beatmap.</param>
+        public AudioNormalizationManager(OsuConfigManager config, IBindable<WorkingBeatmap> beatmap)
+        {
+            audioNormalizationSetting = config.GetBindable<bool>(OsuSetting.AudioNormalization);
+
+            updateNormalization(audioNormalizationSetting.Value, beatmap.Value);
+            audioNormalizationSetting.BindValueChanged(change => updateNormalization(change.NewValue, beatmap.Value));
+            beatmap.BindValueChanged(ev => onBeatmapChanged(ev.OldValue, ev.NewValue));
+        }
+
+        private void updateNormalization(bool value, WorkingBeatmap current)
+        {
+            if (value)
+            {
+                SampleNormalizeVolume.BindTo(current.TrackNormalizeVolume);
+                current.EnableTrackNormlization();
+                Logger.Log($"Normalization value: {(int)(current.TrackNormalizeVolume.Value * 100)}%");
+            }
+            else
+            {
+                SampleNormalizeVolume.UnbindFrom(current.TrackNormalizeVolume);
+                SampleNormalizeVolume.Value = 1.0;
+                current.DisableTrackNormalization();
+            }
+        }
+
+        private void onBeatmapChanged(WorkingBeatmap oldBeatmap, WorkingBeatmap newBeatmap)
+        {
+            SampleNormalizeVolume.UnbindFrom(oldBeatmap.TrackNormalizeVolume);
+            updateNormalization(audioNormalizationSetting.Value, newBeatmap);
+        }
+    }
+}

osu.Game/Beatmaps/BeatmapImporter.cs

@@ -12,6 +12,7 @@
 using osu.Framework.Extensions.IEnumerableExtensions;
 using osu.Framework.Logging;
 using osu.Framework.Platform;
+using osu.Game.Audio;
 using osu.Game.Beatmaps.Formats;
 using osu.Game.Collections;
 using osu.Game.Database;
@@ -173,6 +174,23 @@ protected override void Populate(BeatmapSetInfo beatmapSet, ArchiveReader? archi
                     b.Ruleset = realm.Find<RulesetInfo>(b.Ruleset.ShortName) ?? throw new ArgumentNullException(nameof(b.Ruleset));
             }
 
+            foreach (BeatmapInfo beatmapInfo in beatmapSet.Beatmaps)
+            {
+                if (beatmapInfo.AudioNormalization != null) continue;
+
+                AudioNormalization audioNormalization = new AudioNormalization(beatmapInfo, beatmapSet, Files);
+
+                if (audioNormalization.IntegratedLoudness == null)
+                {
+                    Logger.Log($"Failed to calculate audio normalization values for {beatmapInfo.Metadata.Title} [{beatmapInfo.DifficultyName}]", LoggingTarget.Runtime, LogLevel.Error);
+                    continue;
+                }
+
+                beatmapInfo.AudioNormalization = audioNormalization;
+                beatmapInfo.AudioNormalization.PopulateSet(beatmapInfo, beatmapSet);
+                Logger.Log($"Processed loudness values for {beatmapInfo.Metadata.Title} [{beatmapInfo.DifficultyName}]");
+            }
+
             validateOnlineIds(beatmapSet, realm);
 
             bool hadOnlineIDs = beatmapSet.Beatmaps.Any(b => b.OnlineID > 0);

osu.Game/Beatmaps/BeatmapInfo.cs

@@ -6,6 +6,7 @@
 using System.Linq;
 using JetBrains.Annotations;
 using Newtonsoft.Json;
+using osu.Game.Audio;
 using osu.Game.Beatmaps.ControlPoints;
 using osu.Game.Collections;
 using osu.Game.Database;
@@ -41,6 +42,8 @@ public class BeatmapInfo : RealmObject, IHasGuidPrimaryKey, IBeatmapInfo, IEquat
 
         public BeatmapMetadata Metadata { get; set; } = null!;
 
+        public AudioNormalization? AudioNormalization { get; set; }
+
         [JsonIgnore]
         [Backlink(nameof(ScoreInfo.BeatmapInfo))]
         public IQueryable<ScoreInfo> Scores { get; } = null!;

osu.Game/Beatmaps/WorkingBeatmap.cs

@@ -13,9 +13,11 @@
 using JetBrains.Annotations;
 using osu.Framework.Audio;
 using osu.Framework.Audio.Track;
+using osu.Framework.Bindables;
 using osu.Framework.Extensions;
 using osu.Framework.Graphics.Textures;
 using osu.Framework.Logging;
+using osu.Game.Audio;
 using osu.Game.Rulesets;
 using osu.Game.Rulesets.Mods;
 using osu.Game.Rulesets.UI;
@@ -111,9 +113,32 @@ public Track LoadTrack()
             waveform?.Dispose();
             waveform = null;
 
+            TrackNormalizeVolume = new BindableDouble(AudioNormalizationManager.FALLBACK_VOLUME);
+            track.AddAdjustment(AdjustableProperty.Volume, TrackNormalizeVolume);
+
             return track;
         }
 
+        // Normalization is here because it's not so good to apply normalization in global state through MusicController or TrackStore.
+        // Each beatmap has its own track and its own track loudness value.
+        public BindableDouble TrackNormalizeVolume { get; private set; }
+
+        public void EnableTrackNormlization()
+        {
+            if (TrackNormalizeVolume == null)
+                throw new InvalidOperationException($"{nameof(TrackNormalizeVolume)} is not available. You must call {nameof(LoadTrack)} before calling this.");
+
+            TrackNormalizeVolume.Value = BeatmapInfo.AudioNormalization?.IntegratedLoudnessInVolumeOffset ?? AudioNormalizationManager.FALLBACK_VOLUME;
+        }
+
+        public void DisableTrackNormalization()
+        {
+            if (TrackNormalizeVolume == null)
+                throw new InvalidOperationException($"{nameof(TrackNormalizeVolume)} is not available. You must call {nameof(LoadTrack)} before calling this.");
+
+            TrackNormalizeVolume.Value = AudioNormalizationManager.FALLBACK_VOLUME;
+        }
+
         public void PrepareTrackForPreview(bool looping, double offsetFromPreviewPoint = 0)
         {
             Track.Looping = looping;
@@ -144,6 +169,7 @@ public virtual bool TryTransferTrack([NotNull] WorkingBeatmap target)
             if (BeatmapInfo?.AudioEquals(target.BeatmapInfo) != true || Track.IsDummyDevice)
                 return false;
 
+            target.TrackNormalizeVolume = TrackNormalizeVolume;
             target.track = Track;
             return true;
         }

osu.Game/Configuration/OsuConfigManager.cs

@@ -103,6 +103,8 @@ protected override void InitialiseDefaults()
 
             SetDefault(OsuSetting.AudioOffset, 0, -500.0, 500.0, 1);
 
+            SetDefault(OsuSetting.AudioNormalization, true);
+
             // Input
             SetDefault(OsuSetting.MenuCursorSize, 1.0f, 0.5f, 2f, 0.01f);
             SetDefault(OsuSetting.GameplayCursorSize, 1.0f, 0.1f, 2f, 0.01f);
@@ -355,6 +357,7 @@ public enum OsuSetting
         /// This is added to the audio track's current time. Higher values will cause gameplay to occur earlier, relative to the audio track.
         /// </summary>
         AudioOffset,
+        AudioNormalization,
 
         VolumeInactive,
         MenuMusic,

osu.Game/Database/BackgroundDataStoreProcessor.cs

@@ -13,6 +13,7 @@
 using osu.Framework.Graphics;
 using osu.Framework.Logging;
 using osu.Framework.Platform;
+using osu.Game.Audio;
 using osu.Game.Beatmaps;
 using osu.Game.Extensions;
 using osu.Game.Online.API;
@@ -84,6 +85,7 @@ protected override void LoadComplete()
                 convertLegacyTotalScoreToStandardised();
                 upgradeScoreRanks();
                 backpopulateMissingSubmissionAndRankDates();
+                processAudioNormalization();
             }, TaskCreationOptions.LongRunning).ContinueWith(t =>
             {
                 if (t.Exception?.InnerException is ObjectDisposedException)
@@ -96,6 +98,71 @@ protected override void LoadComplete()
             });
         }
 
+        /// <summary>
+        /// Go through every beatmap and calculate the audio normalization values if they are missing
+        /// </summary>
+        private void processAudioNormalization()
+        {
+            var filestorage = new RealmFileStore(realmAccess, storage);
+            int beatmapsCount = 0;
+            int beatmapsToProcess = 0;
+
+            realmAccess.Run(r =>
+            {
+                foreach (BeatmapSetInfo beatmapSetInfo in r.All<BeatmapSetInfo>().Where(b => !b.DeletePending))
+                {
+                    beatmapsCount += beatmapSetInfo.Beatmaps.Count;
+                    beatmapsToProcess += beatmapSetInfo.Beatmaps.Count(b => b.AudioNormalization == null);
+                }
+            });
+
+            if (beatmapsToProcess == 0)
+            {
+                Logger.Log("No audio normalization processing required.");
+                return;
+            }
+
+            var notification = showProgressNotification(beatmapsCount, "Verifying loudness level for beatmaps", "all loudness levels have been verified");
+            int processedCount = 0;
+
+            realmAccess.Write(r =>
+            {
+                foreach (BeatmapSetInfo beatmapSetInfo in r.All<BeatmapSetInfo>().Where(b => !b.DeletePending))
+                {
+                    foreach (BeatmapInfo beatmapInfo in beatmapSetInfo.Beatmaps)
+                    {
+                        if (notification?.State == ProgressNotificationState.Cancelled)
+                            break;
+
+                        updateNotificationProgress(notification, processedCount, beatmapsCount);
+                        processedCount++;
+
+                        if (beatmapInfo.AudioNormalization != null) continue;
+
+                        sleepIfRequired();
+
+                        AudioNormalization audioNormalization = new AudioNormalization(beatmapInfo, beatmapSetInfo, filestorage);
+
+                        if (audioNormalization.IntegratedLoudness == null)
+                        {
+                            Logger.Log($"Failed to get loudness level for {beatmapSetInfo.Metadata.Title} [{beatmapInfo.DifficultyName}]", LoggingTarget.Runtime, LogLevel.Error);
+                            continue;
+                        }
+
+                        beatmapInfo.AudioNormalization = audioNormalization;
+                        audioNormalization.PopulateSet(beatmapInfo, beatmapSetInfo);
+                        ((IWorkingBeatmapCache)beatmapManager).Invalidate(beatmapSetInfo);
+                        Logger.Log($"Processed audio normalization for {beatmapSetInfo.Metadata.Title} [{beatmapInfo.DifficultyName}]");
+                    }
+                }
+
+                r.Refresh();
+            });
+
+            Logger.Log("Finished processing audio normalization readings");
+            completeNotification(notification, processedCount, beatmapsCount);
+        }
+
         /// <summary>
         /// Check whether the databased difficulty calculation version matches the latest ruleset provided version.
         /// If it doesn't, clear out any existing difficulties so they can be incrementally recalculated.

osu.Game/Database/RealmAccess.cs

@@ -94,8 +94,9 @@ public class RealmAccess : IDisposable
         /// 41   2024-04-17    Add ScoreInfo.TotalScoreWithoutMods for future mod multiplier rebalances.
         /// 42   2024-08-07    Update mania key bindings to reflect changes to ManiaAction
         /// 43   2024-10-14    Reset keybind for toggling FPS display to avoid conflict with "convert to stream" in the editor, if not already changed by user.
+        /// 44   2024-10-16    Add AudioNormalization to BeatmapInfo.
         /// </summary>
-        private const int schema_version = 43;
+        private const int schema_version = 44;
 
         /// <summary>
         /// Lock object which is held during <see cref="BlockAllOperations"/> sections, blocking realm retrieval during blocking periods.