Implement audio normalization using BASS

osu.Game/Audio/AudioNormalization.cs

@@ -0,0 +1,152 @@
+// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
+// See the LICENCE file in the repository root for full licence text.
+
+using System;
+using System.Linq;
+using ManagedBass;
+using ManagedBass.Fx;
+using osu.Framework.Audio.Mixing;
+using osu.Framework.Logging;
+using osu.Game.Beatmaps;
+using osu.Game.Database;
+using Realms;
+
+namespace osu.Game.Audio
+{
+    /// <summary>
+    /// Audio Normalization Manager
+    /// </summary>
+    public class AudioNormalization : EmbeddedObject, IAudioNormalization, IEquatable<AudioNormalization>
+    {
+        /// <summary>
+        /// The target level for audio normalization
+        /// https://en.wikipedia.org/wiki/EBU_R_128
+        /// </summary>
+        public const int TARGET_LEVEL = -14;
+
+        /// <summary>
+        /// The integrated (average) loudness of the audio
+        /// </summary>
+        public float? IntegratedLoudness { get; init; }
+
+        public AudioNormalization()
+        {
+        }
+
+        /// <summary>
+        /// Get the audio normalization for a beatmap. The loudness normalization value is stored in the object under <see cref="IntegratedLoudness"/>
+        /// </summary>
+        /// <param name="beatmapInfo">The <see cref="BeatmapInfo"/> of the beatmap</param>
+        /// <param name="beatmapSetInfo">The parent <see cref="BeatmapSetInfo"/> of the <see cref="BeatmapInfo"/> supplied</param>
+        /// <param name="realmFileStore">The <see cref="RealmFileStore"/> to use in getting the full path of the audio file</param>
+        public AudioNormalization(BeatmapInfo beatmapInfo, BeatmapSetInfo beatmapSetInfo, RealmFileStore realmFileStore)
+        {
+            string audiofile = beatmapInfo.Metadata.AudioFile;
+
+            if (string.IsNullOrEmpty(audiofile))
+            {
+                Logger.Log("Audio file not found for " + beatmapInfo.Metadata.Title, LoggingTarget.Runtime, LogLevel.Error);
+                return;
+            }
+
+            string? filepath = beatmapSetInfo.GetPathForFile(audiofile);
+
+            if (string.IsNullOrEmpty(filepath))
+            {
+                Logger.Log("File path not found for " + audiofile, LoggingTarget.Runtime, LogLevel.Error);
+                return;
+            }
+
+            filepath = realmFileStore.Storage.GetFullPath(filepath);
+
+            BassAudioNormalization loudnessDetection = new BassAudioNormalization(filepath);
+            float? integratedLoudness = loudnessDetection.IntegratedLoudness;
+
+            if (integratedLoudness == null)
+            {
+                Logger.Log("Failed to get loudness level for " + audiofile, LoggingTarget.Runtime, LogLevel.Error);
+                return;
+            }
+
+            IntegratedLoudness = integratedLoudness;
+        }
+
+        /// <summary>
+        /// Populate the audio normalization for every <see cref="BeatmapInfo"/> in a <see cref="BeatmapSetInfo"/>.
+        /// </summary>
+        /// <remarks>
+        /// This takes a <see cref="BeatmapInfo"/> and applies the same normalization to every <see cref="BeatmapInfo"/> in the <see cref="BeatmapSetInfo"/> that has the same audio file as the given <see cref="BeatmapInfo"/> to avoid having to calculate the loudness for an audio file multiple times
+        /// </remarks>
+        /// <param name="beatmapInfo">The <see cref="BeatmapInfo"/> to clone from</param>
+        /// <param name="beatmapSetInfo">The <see cref="BeatmapSetInfo"/> to populate</param>
+        public void PopulateSet(BeatmapInfo beatmapInfo, BeatmapSetInfo beatmapSetInfo)
+        {
+            foreach (BeatmapInfo beatmap in beatmapSetInfo.Beatmaps)
+            {
+                if (beatmap.AudioNormalization == null && beatmap.AudioEquals(beatmapInfo))
+                {
+                    beatmap.AudioNormalization = new AudioNormalization
+                    {
+                        IntegratedLoudness = IntegratedLoudness
+                    };
+                }
+            }
+        }
+
+        /// <summary>
+        /// Convert integrated loudness to a volume offset
+        /// </summary>
+        /// <param name="integratedLoudness">The integrated loudness value</param>
+        /// <returns>The volume offset needed to reach <see cref="TARGET_LEVEL"/></returns>
+        public static float IntegratedLoudnessToVolumeOffset(float integratedLoudness) => (float)Math.Pow(10, (TARGET_LEVEL - integratedLoudness) / 20);
+
+        /// <summary>
+        /// Get the loudness values of a <see cref="BeatmapInfo"/> and apply the audio normalization effect to an <see cref="IAudioMixer"/>
+        /// </summary>
+        /// <param name="beatmapInfo">The <see cref="BeatmapInfo"/> to get the loudness values of</param>
+        /// <param name="mixer">The <see cref="IAudioMixer"/> to apply the effect on</param>
+        public static void AddAudioNormalization(BeatmapInfo beatmapInfo, IAudioMixer mixer)
+        {
+            AudioNormalization? audioNormalizationModule = beatmapInfo.AudioNormalization;
+
+            VolumeParameters volumeParameters = new VolumeParameters
+            {
+                fTarget = audioNormalizationModule?.IntegratedLoudness != null ? IntegratedLoudnessToVolumeOffset((float)audioNormalizationModule.IntegratedLoudness) : 0.8f,
+                fCurrent = -1.0f,
+                fTime = 0.3f,
+                lCurve = 1,
+                lChannel = FXChannelFlags.All
+            };
+
+            addFx(volumeParameters, mixer);
+
+            Logger.Log("Normalization Status: " + (audioNormalizationModule?.IntegratedLoudness != null ? $"on ({Math.Round(IntegratedLoudnessToVolumeOffset((float)audioNormalizationModule.IntegratedLoudness) * 100)}%)" : "off"));
+        }
+
+        /// <summary>
+        /// Add an effect to a <see cref="IAudioMixer"/>, replacing it if it already exists
+        /// </summary>
+        /// <param name="effectParameter">The <see cref="IEffectParameter"/> to add to the <paramref name="mixer"/></param>
+        /// <param name="mixer">The <see cref="IAudioMixer"/> to add the effect to</param>
+        private static void addFx(IEffectParameter effectParameter, IAudioMixer mixer)
+        {
+            IEffectParameter? effect = mixer.Effects.SingleOrDefault(e => e.FXType == effectParameter.FXType);
+
+            if (effect != null)
+            {
+                int i = mixer.Effects.IndexOf(effect);
+                mixer.Effects[i] = effectParameter;
+            }
+            else
+            {
+                mixer.Effects.Add(effectParameter);
+            }
+        }
+
+        /// <inheritdoc />
+        public bool Equals(IAudioNormalization? other) => other is AudioNormalization audioNormalization && Equals(audioNormalization);
+
+        /// <inheritdoc />
+        public bool Equals(AudioNormalization? other) => other?.IntegratedLoudness != null && IntegratedLoudness != null && Math.Abs((float)(IntegratedLoudness - other.IntegratedLoudness)) < 0.0000001;
+    }
+}

osu.Game/Audio/BassAudioNormalization.cs

@@ -0,0 +1,74 @@
+// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
+// See the LICENCE file in the repository root for full licence text.
+
+using ManagedBass;
+using ManagedBass.Loud;
+using osu.Framework.Logging;
+
+namespace osu.Game.Audio
+{
+    /// <summary>
+    /// Audio Normalization Implementation using Bass
+    /// </summary>
+    public class BassAudioNormalization
+    {
+        /// <summary>
+        /// The integrated loudness of the audio
+        /// </summary>
+        /// <remarks>
+        /// Applicable range = -70 to -1<br/>Null if the loudness could not be calculated due to an error
+        /// </remarks>
+        public float? IntegratedLoudness { get; }
+
+        /// <summary>
+        /// Calculate the integrated loudness of an audio file using Bass
+        /// </summary>
+        /// <returns>The integrated loudness or 1 in <see cref="IntegratedLoudness"/></returns>
+        /// <param name="filePath">A path to an audio file</param>
+        public BassAudioNormalization(string filePath)
+        {
+            int decodeStream = Bass.CreateStream(filePath, 0, 0, BassFlags.Decode | BassFlags.Float);
+
+            if (decodeStream == 0)
+            {
+                Logger.Log("Failed to create stream for loudness measurement!\nError Code: " + Bass.LastError, LoggingTarget.Runtime, LogLevel.Error);
+                IntegratedLoudness = null;
+                Bass.StreamFree(decodeStream);
+                return;
+            }
+
+            int loudness = BassLoud.BASS_Loudness_Start(decodeStream, BassFlags.BassLoudnessIntegrated | BassFlags.BassLoudnessAutofree, 0);
+
+            if (loudness == 0)
+            {
+                Logger.Log("Failed to start loudness measurement!\nError Code: " + Bass.LastError, LoggingTarget.Runtime, LogLevel.Error);
+                IntegratedLoudness = null;
+                Bass.StreamFree(decodeStream);
+                return;
+            }
+
+            byte[] buffer = new byte[10000];
+
+            while (Bass.ChannelGetData(decodeStream, buffer, buffer.Length) >= 0)
+            {
+            }
+
+            float integratedLoudness = 0;
+            bool gotLevel = BassLoud.BASS_Loudness_GetLevel(loudness, BassFlags.BassLoudnessIntegrated, ref integratedLoudness);
+
+            if (!gotLevel || integratedLoudness == 0)
+            {
+                Logger.Log("Failed to get loudness level!\nError Code: " + Bass.LastError, LoggingTarget.Runtime, LogLevel.Error);
+                IntegratedLoudness = null;
+                Bass.StreamFree(decodeStream);
+                return;
+            }
+
+            IntegratedLoudness = integratedLoudness;
+
+            bool freedStream = Bass.StreamFree(decodeStream);
+            if (!freedStream)
+                Logger.Log("Failed to free stream!\nError Code: " + Bass.LastError, LoggingTarget.Runtime, LogLevel.Error);
+        }
+    }
+}

osu.Game/Audio/IAudioNormalization.cs

@@ -0,0 +1,12 @@
+// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
+// See the LICENCE file in the repository root for full licence text.
+
+using System;
+
+namespace osu.Game.Audio
+{
+    public interface IAudioNormalization : IEquatable<IAudioNormalization>
+    {
+        public float? IntegratedLoudness { get; }
+    }
+}

osu.Game/Beatmaps/BeatmapImporter.cs

@@ -12,6 +12,7 @@
 using osu.Framework.Extensions.IEnumerableExtensions;
 using osu.Framework.Logging;
 using osu.Framework.Platform;
+using osu.Game.Audio;
 using osu.Game.Beatmaps.Formats;
 using osu.Game.Collections;
 using osu.Game.Database;
@@ -165,6 +166,23 @@ protected override void Populate(BeatmapSetInfo beatmapSet, ArchiveReader? archi
                     b.Ruleset = realm.Find<RulesetInfo>(b.Ruleset.ShortName) ?? throw new ArgumentNullException(nameof(b.Ruleset));
             }
 
+            foreach (BeatmapInfo beatmapInfo in beatmapSet.Beatmaps)
+            {
+                if (beatmapInfo.AudioNormalization != null) continue;
+
+                AudioNormalization audioNormalization = new AudioNormalization(beatmapInfo, beatmapSet, Files);
+
+                if (audioNormalization.IntegratedLoudness == null)
+                {
+                    Logger.Log($"Failed to calculate audio normalization values for {beatmapInfo.Metadata.Title} [{beatmapInfo.DifficultyName}]", LoggingTarget.Runtime, LogLevel.Error);
+                    continue;
+                }
+
+                beatmapInfo.AudioNormalization = audioNormalization;
+                beatmapInfo.AudioNormalization.PopulateSet(beatmapInfo, beatmapSet);
+                Logger.Log($"Processed loudness values for {beatmapInfo.Metadata.Title} [{beatmapInfo.DifficultyName}]");
+            }
+
             validateOnlineIds(beatmapSet, realm);
 
             bool hadOnlineIDs = beatmapSet.Beatmaps.Any(b => b.OnlineID > 0);

osu.Game/Beatmaps/BeatmapInfo.cs

@@ -6,6 +6,7 @@
 using System.Linq;
 using JetBrains.Annotations;
 using Newtonsoft.Json;
+using osu.Game.Audio;
 using osu.Game.Beatmaps.ControlPoints;
 using osu.Game.Collections;
 using osu.Game.Database;
@@ -41,6 +42,8 @@ public class BeatmapInfo : RealmObject, IHasGuidPrimaryKey, IBeatmapInfo, IEquat
 
         public BeatmapMetadata Metadata { get; set; } = null!;
 
+        public AudioNormalization? AudioNormalization { get; set; }
+
         [JsonIgnore]
         [Backlink(nameof(ScoreInfo.BeatmapInfo))]
         public IQueryable<ScoreInfo> Scores { get; } = null!;

osu.Game/Database/BackgroundDataStoreProcessor.cs

@@ -11,6 +11,8 @@
 using osu.Framework.Bindables;
 using osu.Framework.Graphics;
 using osu.Framework.Logging;
+using osu.Framework.Platform;
+using osu.Game.Audio;
 using osu.Game.Beatmaps;
 using osu.Game.Extensions;
 using osu.Game.Online.API;
@@ -58,6 +60,9 @@ public partial class BackgroundDataStoreProcessor : Component
         [Resolved]
         private INotificationOverlay? notificationOverlay { get; set; }
 
+        [Resolved]
+        private Storage? storage { get; set; }
+
         [Resolved]
         private IAPIProvider api { get; set; } = null!;
 
@@ -78,6 +83,7 @@ protected override void LoadComplete()
                 processScoresWithMissingStatistics();
                 convertLegacyTotalScoreToStandardised();
                 upgradeScoreRanks();
+                processAudioNormalization();
             }, TaskCreationOptions.LongRunning).ContinueWith(t =>
             {
                 if (t.Exception?.InnerException is ObjectDisposedException)
@@ -90,6 +96,71 @@ protected override void LoadComplete()
             });
         }
 
+        /// <summary>
+        /// Go through every beatmap and calculate the audio normalization values if they are missing
+        /// </summary>
+        private void processAudioNormalization()
+        {
+            var filestorage = new RealmFileStore(realmAccess, storage!);
+            int beatmapsCount = 0;
+            int beatmapsToProcess = 0;
+
+            realmAccess.Run(r =>
+            {
+                foreach (BeatmapSetInfo beatmapSetInfo in r.All<BeatmapSetInfo>().Where(b => !b.DeletePending))
+                {
+                    beatmapsCount += beatmapSetInfo.Beatmaps.Count;
+                    beatmapsToProcess += beatmapSetInfo.Beatmaps.Count(b => b.AudioNormalization == null);
+                }
+            });
+
+            if (beatmapsToProcess == 0)
+            {
+                Logger.Log("No audio normalization processing required.");
+                return;
+            }
+
+            var notification = showProgressNotification(beatmapsCount, "Verifying loudness level for beatmaps", "all loudness levels have been verified");
+            int processedCount = 0;
+
+            realmAccess.Write(r =>
+            {
+                foreach (BeatmapSetInfo beatmapSetInfo in r.All<BeatmapSetInfo>().Where(b => !b.DeletePending))
+                {
+                    foreach (BeatmapInfo beatmapInfo in beatmapSetInfo.Beatmaps)
+                    {
+                        if (notification?.State == ProgressNotificationState.Cancelled)
+                            break;
+
+                        updateNotificationProgress(notification, processedCount, beatmapsCount);
+                        processedCount++;
+
+                        if (beatmapInfo.AudioNormalization != null) continue;
+
+                        sleepIfRequired();
+
+                        AudioNormalization audioNormalization = new AudioNormalization(beatmapInfo, beatmapSetInfo, filestorage);
+
+                        if (audioNormalization.IntegratedLoudness == null)
+                        {
+                            Logger.Log($"Failed to get loudness level for {beatmapSetInfo.Metadata.Title} [{beatmapInfo.DifficultyName}]", LoggingTarget.Runtime, LogLevel.Error);
+                            continue;
+                        }
+
+                        beatmapInfo.AudioNormalization = audioNormalization;
+                        audioNormalization.PopulateSet(beatmapInfo, beatmapSetInfo);
+                        ((IWorkingBeatmapCache)beatmapManager).Invalidate(beatmapInfo);
+                        Logger.Log($"Processed audio normalization for {beatmapSetInfo.Metadata.Title} [{beatmapInfo.DifficultyName}]");
+                    }
+                }
+
+                r.Refresh();
+            });
+
+            Logger.Log("Finished processing audio normalization readings");
+            completeNotification(notification, processedCount, beatmapsCount);
+        }
+
         /// <summary>
         /// Check whether the databased difficulty calculation version matches the latest ruleset provided version.
         /// If it doesn't, clear out any existing difficulties so they can be incrementally recalculated.

osu.Game/Database/RealmAccess.cs

@@ -91,8 +91,9 @@ public class RealmAccess : IDisposable
         /// 38   2023-12-10    Add EndTimeObjectCount and TotalObjectCount to BeatmapInfo.
         /// 39   2023-12-19    Migrate any EndTimeObjectCount and TotalObjectCount values of 0 to -1 to better identify non-calculated values.
         /// 40   2023-12-21    Add ScoreInfo.Version to keep track of which build scores were set on.
+        /// 41   2024-04-04    Add AudioNormalization to BeatmapInfo.
         /// </summary>
-        private const int schema_version = 40;
+        private const int schema_version = 41;
 
         /// <summary>
         /// Lock object which is held during <see cref="BlockAllOperations"/> sections, blocking realm retrieval during blocking periods.