Fix up midi stuff, support mild black midis (#1056)

Content.Client/GameObjects/Components/Instruments/InstrumentComponent.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Linq;
 using Content.Shared.GameObjects.Components.Instruments;
@@ -7,23 +8,26 @@ using JetBrains.Annotations;
 using NFluidsynth;
 using Robust.Shared.GameObjects;
 using Robust.Client.Audio.Midi;
+using Robust.Client.Player;
+using Robust.Shared.Interfaces.Log;
 using Robust.Shared.Interfaces.Network;
 using Robust.Shared.Interfaces.Timing;
 using Robust.Shared.IoC;
+using Robust.Shared.Log;
 using Robust.Shared.Players;
 using Robust.Shared.Serialization;
+using Robust.Shared.Timing;
 using Robust.Shared.ViewVariables;
 using Logger = Robust.Shared.Log.Logger;
 using MidiEvent = Robust.Shared.Audio.Midi.MidiEvent;
 using Timer = Robust.Shared.Timers.Timer;
 
-
 namespace Content.Client.GameObjects.Components.Instruments
 {
+
     [RegisterComponent]
     public class InstrumentComponent : SharedInstrumentComponent
     {
-        public const float TimeBetweenNetMessages = 1.0f;
 
         /// <summary>
         ///     Called when a midi song stops playing.
@@ -31,23 +35,29 @@ namespace Content.Client.GameObjects.Components.Instruments
         public event Action OnMidiPlaybackEnded;
 
 #pragma warning disable 649
-        [Dependency] private IMidiManager _midiManager;
+        [Dependency] private readonly IMidiManager _midiManager;
+
+        [Dependency] private readonly IGameTiming _gameTiming;
+
+        [Dependency] private readonly IClientNetManager _netManager;
 #pragma warning restore 649
 
         [CanBeNull]
         private IMidiRenderer _renderer;
+
         private byte _instrumentProgram = 1;
+
         private byte _instrumentBank = 0;
-        private uint _syncSequencerTick;
+
+        private uint _sequenceDelay = 0;
+
+        private uint _sequenceStartTick;
 
         /// <summary>
         ///     A queue of MidiEvents to be sent to the server.
         /// </summary>
         [ViewVariables]
-        private readonly Queue<MidiEvent> _midiQueue = new Queue<MidiEvent>();
+        private readonly List<MidiEvent> _midiEventBuffer = new List<MidiEvent>();
-
-        [ViewVariables]
-        private float _timer = 0f;
 
         /// <summary>
         ///     Whether a midi song will loop or not.
@@ -123,11 +133,12 @@ namespace Content.Client.GameObjects.Components.Instruments
             IoCManager.InjectDependencies(this);
         }
 
-        protected void SetupRenderer()
+        protected void SetupRenderer(bool fromStateChange = false)
         {
-            if (IsRendererAlive)
+            if (IsRendererAlive) return;
-                return;
 
+            _sequenceDelay = 0;
+            _sequenceStartTick = 0;
             _midiManager.OcclusionCollisionMask = (int) CollisionGroup.Impassable;
             _renderer = _midiManager.GetNewRenderer();
 
@@ -136,17 +147,32 @@ namespace Content.Client.GameObjects.Components.Instruments
                 _renderer.MidiBank = _instrumentBank;
                 _renderer.MidiProgram = _instrumentProgram;
                 _renderer.TrackingEntity = Owner;
-                _renderer.OnMidiPlayerFinished += () => { OnMidiPlaybackEnded?.Invoke(); EndRenderer(); SendNetworkMessage(new InstrumentStopMidiMessage()); };
+                _renderer.OnMidiPlayerFinished += () =>
+                {
+                    OnMidiPlaybackEnded?.Invoke();
+                    EndRenderer(fromStateChange);
+                };
+            }
+
+            if (!fromStateChange)
+            {
+                SendNetworkMessage(new InstrumentStartMidiMessage());
             }
         }
 
-        protected void EndRenderer()
+        protected void EndRenderer(bool fromStateChange = false)
         {
             if (IsInputOpen)
-                CloseInput();
+            {
+                CloseInput(fromStateChange);
+                return;
+            }
 
             if (IsMidiOpen)
-                CloseMidi();
+            {
+                CloseMidi(fromStateChange);
+                return;
+            }
 
             _renderer?.StopAllNotes();
 
@@ -155,7 +181,12 @@ namespace Content.Client.GameObjects.Components.Instruments
             // We dispose of the synth two seconds from now to allow the last notes to stop from playing.
             Timer.Spawn(2000, () => { renderer?.Dispose(); });
             _renderer = null;
-            _midiQueue.Clear();
+            _midiEventBuffer.Clear();
+
+            if (!fromStateChange)
+            {
+                SendNetworkMessage(new InstrumentStopMidiMessage());
+            }
         }
 
         protected override void Shutdown()
@@ -178,15 +209,65 @@ namespace Content.Client.GameObjects.Components.Instruments
             switch (message)
             {
                 case InstrumentMidiEventMessage midiEventMessage:
+                    if (IsRendererAlive)
+                    {
                         // If we're the ones sending the MidiEvents, we ignore this message.
-                    if (!IsRendererAlive || IsInputOpen || IsMidiOpen) break;
+                        if (IsInputOpen || IsMidiOpen) break;
+                    }
+                    else
+                    {
+                        // if we haven't started or finished some sequence
+                        if (_sequenceStartTick == 0)
+                        {
+                            // we may have arrived late
+                            SetupRenderer(true);
+                        }
+
+                        // might be our own notes after we already finished playing
+                        return;
+                    }
+
+                    if (_sequenceStartTick <= 0)
+                    {
+                        _sequenceStartTick = midiEventMessage.MidiEvent
+                            .Min(x => x.Tick) - 1;
+                    }
+
+                    var sqrtLag = MathF.Sqrt(_netManager.ServerChannel.Ping / 1000f);
+                    var delay = (uint) (_renderer!.SequencerTimeScale * (.2 + sqrtLag));
+                    var delta = delay - _sequenceStartTick;
+
+                    _sequenceDelay = Math.Max(_sequenceDelay, delta);
+
+                    var currentTick = _renderer.SequencerTick;
+
+                    // ReSharper disable once ForCanBeConvertedToForeach
                     for (var i = 0; i < midiEventMessage.MidiEvent.Length; i++)
                     {
                         var ev = midiEventMessage.MidiEvent[i];
-                        var delta = ((uint)TimeBetweenNetMessages*1250) + ev.Timestamp - _syncSequencerTick;
+                        var scheduled = ev.Tick + _sequenceDelay;
-                        _renderer?.ScheduleMidiEvent(ev, delta, true);
+
+                        if (scheduled <= currentTick)
+                        {
+                            _sequenceDelay += currentTick - ev.Tick;
+                            scheduled = ev.Tick + _sequenceDelay;
                         }
+
+
+                        _renderer?.ScheduleMidiEvent(ev, scheduled, true);
+                    }
+
                     break;
+                case InstrumentStartMidiMessage startMidiMessage:
+                {
+                    SetupRenderer(true);
+                    break;
+                }
+                case InstrumentStopMidiMessage stopMidiMessage:
+                {
+                    EndRenderer(true);
+                    break;
+                }
             }
         }
 
@@ -197,18 +278,18 @@ namespace Content.Client.GameObjects.Components.Instruments
 
             if (state.Playing)
             {
-                SetupRenderer();
+                SetupRenderer(true);
-                _syncSequencerTick = state.SequencerTick;
             }
             else
-                EndRenderer();
+            {
+                EndRenderer(true);
+            }
         }
 
         /// <inheritdoc cref="MidiRenderer.OpenInput"/>
         public bool OpenInput()
         {
             SetupRenderer();
-            SendNetworkMessage(new InstrumentStartMidiMessage());
 
             if (_renderer != null && _renderer.OpenInput())
             {
@@ -220,15 +301,14 @@ namespace Content.Client.GameObjects.Components.Instruments
         }
 
         /// <inheritdoc cref="MidiRenderer.CloseInput"/>
-        public bool CloseInput()
+        public bool CloseInput(bool fromStateChange = false)
         {
             if (_renderer == null || !_renderer.CloseInput())
             {
                 return false;
             }
 
-            EndRenderer();
+            EndRenderer(fromStateChange);
-            SendNetworkMessage(new InstrumentStopMidiMessage());
             return true;
         }
 
@@ -236,7 +316,6 @@ namespace Content.Client.GameObjects.Components.Instruments
         public bool OpenMidi(string filename)
         {
             SetupRenderer();
-            SendNetworkMessage(new InstrumentStartMidiMessage());
 
             if (_renderer == null || !_renderer.OpenMidi(filename))
             {
@@ -248,15 +327,14 @@ namespace Content.Client.GameObjects.Components.Instruments
         }
 
         /// <inheritdoc cref="MidiRenderer.CloseMidi"/>
-        public bool CloseMidi()
+        public bool CloseMidi(bool fromStateChange = false)
         {
             if (_renderer == null || !_renderer.CloseMidi())
             {
                 return false;
             }
 
-            EndRenderer();
+            EndRenderer(fromStateChange);
-            SendNetworkMessage(new InstrumentStopMidiMessage());
             return true;
         }
 
@@ -266,29 +344,83 @@ namespace Content.Client.GameObjects.Components.Instruments
         /// <param name="midiEvent">The received midi event</param>
         private void RendererOnMidiEvent(MidiEvent midiEvent)
         {
-            _midiQueue.Enqueue(midiEvent);
+            // avoid of out-of-band, unimportant or unsupported events
+            switch (midiEvent.Type)
+            {
+                case 0x80: // NOTE_OFF
+                case 0x90: // NOTE_ON
+                case 0xa0: // KEY_PRESSURE
+                case 0xb0: // CONTROL_CHANGE
+                case 0xd0: // CHANNEL_PRESSURE
+                case 0xe0: // PITCH_BEND
+                    break;
+                default:
+                    return;
             }
 
+            _midiEventBuffer.Add(midiEvent);
+        }
+
+        private TimeSpan _lastMeasured = TimeSpan.MinValue;
+
+        private int _sentWithinASec = 0;
+
+        private static readonly TimeSpan OneSecAgo = TimeSpan.FromSeconds(-1);
+
+        private static readonly Comparer<MidiEvent> SortMidiEventTick
+            = Comparer<MidiEvent>.Create((x, y)
+                => x.Tick.CompareTo(y.Tick));
+
         public override void Update(float delta)
         {
-            if (!IsMidiOpen && !IsInputOpen)
+            if (!IsMidiOpen && !IsInputOpen) return;
-                return;
 
-            _timer -= delta;
+            var now = _gameTiming.RealTime;
+            var oneSecAGo = now.Add(OneSecAgo);
 
-            if (_timer > 0f) return;
+            if (_lastMeasured <= oneSecAGo)
-
+            {
-            SendAllMidiMessages();
+                _lastMeasured = now;
-            _timer = TimeBetweenNetMessages;
+                _sentWithinASec = 0;
             }
 
-        private void SendAllMidiMessages()
+            if (_midiEventBuffer.Count == 0) return;
+
+            var max = Math.Min(MaxMidiEventsPerBatch, MaxMidiEventsPerSecond - _sentWithinASec);
+
+            if (max <= 0)
             {
-            if (_midiQueue.Count == 0) return;
+                // hit event/sec limit, have to lag the batch or drop events
-            var events = _midiQueue.ToArray();
+                return;
-            _midiQueue.Clear();
+            }
+
+            // fix cross-fade events generating retroactive events
+            // also handle any significant backlog of events after midi finished
+
+            _midiEventBuffer.Sort(SortMidiEventTick);
+            var bufferTicks = IsRendererAlive && _renderer!.Status != MidiRendererStatus.None
+                ? _renderer.SequencerTimeScale * .2f
+                : 0;
+            var bufferedTick = IsRendererAlive
+                ? _renderer!.SequencerTick - bufferTicks
+                : int.MaxValue;
+
+            var events = _midiEventBuffer
+                .TakeWhile(x => x.Tick < bufferedTick)
+                .Take(max)
+                .ToArray();
+
+            var eventCount = events.Length;
+
+            if (eventCount == 0) return;
 
             SendNetworkMessage(new InstrumentMidiEventMessage(events));
+
+            _sentWithinASec += eventCount;
+
+            _midiEventBuffer.RemoveRange(0, eventCount);
         }
+
     }
+
 }

Content.Server/GameObjects/Components/Instruments/InstrumentComponent.cs

@@ -1,3 +1,5 @@
+using System;
+using System.Linq;
 using Content.Server.GameObjects.Components.Mobs;
 using Content.Server.GameObjects.EntitySystems;
 using Content.Server.Interfaces;
@@ -14,7 +16,9 @@ using Robust.Server.Player;
 using Robust.Shared.Enums;
 using Robust.Shared.GameObjects;
 using Robust.Shared.Interfaces.GameObjects;
+using Robust.Shared.Interfaces.Log;
 using Robust.Shared.Interfaces.Network;
+using Robust.Shared.Interfaces.Timing;
 using Robust.Shared.IoC;
 using Robust.Shared.Players;
 using Robust.Shared.Serialization;
@@ -24,25 +28,33 @@ using MidiEvent = Robust.Shared.Audio.Midi.MidiEvent;
 
 namespace Content.Server.GameObjects.Components.Instruments
 {
+
     [RegisterComponent]
     [ComponentReference(typeof(IActivate))]
-    public class InstrumentComponent : SharedInstrumentComponent,
+    public class InstrumentComponent
-        IDropped, IHandSelected, IHandDeselected, IActivate, IUse, IThrown
+        : SharedInstrumentComponent,
+            IDropped,
+            IHandSelected,
+            IHandDeselected,
+            IActivate,
+            IUse,
+            IThrown
     {
+
 #pragma warning disable 649
-        [Dependency] private IServerNotifyManager _notifyManager;
+        [Dependency] private readonly IServerNotifyManager _notifyManager;
+
+        [Dependency] private readonly IGameTiming _gameTiming;
 #pragma warning restore 649
 
-        // These 2 values are quite high for now, and this could be easily abused. Change this if people are abusing it.
+        private static readonly TimeSpan OneSecAgo = TimeSpan.FromSeconds(-1);
-        public const int MaxMidiEventsPerSecond = 20;
-        public const int MaxMidiEventsPerBatch = 50;
-        public const int MaxMidiBatchDropped = 20;
 
         /// <summary>
         ///     The client channel currently playing the instrument, or null if there's none.
         /// </summary>
         [ViewVariables]
         private IPlayerSession _instrumentPlayer;
+
         private bool _handheld;
 
         [ViewVariables]
@@ -51,9 +63,15 @@ namespace Content.Server.GameObjects.Components.Instruments
         [ViewVariables]
         private float _timer = 0f;
 
+        [ViewVariables(VVAccess.ReadOnly)]
+        private TimeSpan _lastMeasured = TimeSpan.MinValue;
+
         [ViewVariables]
         private int _batchesDropped = 0;
 
+        [ViewVariables]
+        private int _laggedBatches = 0;
+
         [ViewVariables]
         private uint _lastSequencerTick = 0;
 
@@ -131,30 +149,87 @@ namespace Content.Server.GameObjects.Components.Instruments
             switch (message)
             {
                 case InstrumentMidiEventMessage midiEventMsg:
-                    if (!Playing || session != _instrumentPlayer)
+                    if (!Playing || session != _instrumentPlayer) return;
-                        return;
 
-                    if (++_midiEventCount <= MaxMidiEventsPerSecond &&
+                    var send = true;
-                        midiEventMsg.MidiEvent.Length < MaxMidiEventsPerBatch)
+
+                    var minTick = midiEventMsg.MidiEvent.Min(x => x.Tick);
+                    if (_lastSequencerTick > minTick)
+                    {
+                        var now = _gameTiming.RealTime;
+                        var oneSecAGo = now.Add(OneSecAgo);
+                        if (_lastMeasured < oneSecAGo)
+                        {
+                            _lastMeasured = now;
+                            _laggedBatches = 0;
+                            _batchesDropped = 0;
+                        }
+
+                        _laggedBatches++;
+                        switch (_laggedBatches)
+                        {
+                            case (int) (MaxMidiLaggedBatches * (1 / 3d)) + 1:
+                                _notifyManager.PopupMessage(Owner, InstrumentPlayer.AttachedEntity,
+                                    "Your fingers are beginning to a cramp a little!");
+                                break;
+                            case (int) (MaxMidiLaggedBatches * (2 / 3d)) + 1:
+                                _notifyManager.PopupMessage(Owner, InstrumentPlayer.AttachedEntity,
+                                    "Your fingers are seriously cramping up!");
+                                break;
+                        }
+
+                        if (_laggedBatches > MaxMidiLaggedBatches)
+                        {
+                            send = false;
+                        }
+                    }
+
+                    if (++_midiEventCount > MaxMidiEventsPerSecond
+                        || midiEventMsg.MidiEvent.Length > MaxMidiEventsPerBatch)
+                    {
+                        var now = _gameTiming.RealTime;
+                        var oneSecAGo = now.Add(OneSecAgo);
+                        if (_lastMeasured < oneSecAGo)
+                        {
+                            _lastMeasured = now;
+                            _laggedBatches = 0;
+                            _batchesDropped = 0;
+                        }
+
+                        _batchesDropped++;
+
+                        send = false;
+                    }
+
+                    if (send)
+                    {
                         SendNetworkMessage(midiEventMsg);
-                    else
+                    }
-                        _batchesDropped++; // Batch dropped!
 
-                    _lastSequencerTick = midiEventMsg.MidiEvent[^1].Timestamp;
+                    var maxTick = midiEventMsg.MidiEvent.Max(x => x.Tick);
+                    _lastSequencerTick = Math.Max(maxTick, minTick + 1);
                     break;
                 case InstrumentStartMidiMessage startMidi:
                     Playing = true;
                     break;
                 case InstrumentStopMidiMessage stopMidi:
                     Playing = false;
-                    _lastSequencerTick = 0;
+                    Clean();
                     break;
             }
         }
 
-        public void Dropped(DroppedEventArgs eventArgs)
+        private void Clean()
         {
             Playing = false;
+            _lastSequencerTick = 0;
+            _batchesDropped = 0;
+            _laggedBatches = 0;
+        }
+
+        public void Dropped(DroppedEventArgs eventArgs)
+        {
+            Clean();
             SendNetworkMessage(new InstrumentStopMidiMessage());
             InstrumentPlayer = null;
             _userInterface.CloseAll();
@@ -162,7 +237,7 @@ namespace Content.Server.GameObjects.Components.Instruments
 
         public void Thrown(ThrownEventArgs eventArgs)
         {
-            Playing = false;
+            Clean();
             SendNetworkMessage(new InstrumentStopMidiMessage());
             InstrumentPlayer = null;
             _userInterface.CloseAll();
@@ -179,18 +254,16 @@ namespace Content.Server.GameObjects.Components.Instruments
 
         public void HandDeselected(HandDeselectedEventArgs eventArgs)
         {
-            Playing = false;
+            Clean();
             SendNetworkMessage(new InstrumentStopMidiMessage());
             _userInterface.CloseAll();
         }
 
         public void Activate(ActivateEventArgs eventArgs)
         {
-            if (Handheld || !eventArgs.User.TryGetComponent(out IActorComponent actor))
+            if (Handheld || !eventArgs.User.TryGetComponent(out IActorComponent actor)) return;
-                return;
 
-            if (InstrumentPlayer != null)
+            if (InstrumentPlayer != null) return;
-                return;
 
             InstrumentPlayer = actor.playerSession;
             OpenUserInterface(actor.playerSession);
@@ -198,22 +271,23 @@ namespace Content.Server.GameObjects.Components.Instruments
 
         public bool UseEntity(UseEntityEventArgs eventArgs)
         {
-            if (!eventArgs.User.TryGetComponent(out IActorComponent actor))
+            if (!eventArgs.User.TryGetComponent(out IActorComponent actor)) return false;
-                return false;
 
             if (InstrumentPlayer == actor.playerSession)
+            {
                 OpenUserInterface(actor.playerSession);
+            }
+
             return false;
         }
 
         private void UserInterfaceOnClosed(IPlayerSession player)
         {
-            if (!Handheld && player == InstrumentPlayer)
+            if (Handheld || player != InstrumentPlayer) return;
-            {
+
+            Clean();
             InstrumentPlayer = null;
             SendNetworkMessage(new InstrumentStopMidiMessage());
-                Playing = false;
-            }
         }
 
         private void OpenUserInterface(IPlayerSession session)
@@ -226,19 +300,30 @@ namespace Content.Server.GameObjects.Components.Instruments
             base.Update(delta);
 
             if (_instrumentPlayer != null && !ActionBlockerSystem.CanInteract(_instrumentPlayer.AttachedEntity))
-                InstrumentPlayer = null;
-
-            if (_batchesDropped > MaxMidiBatchDropped && InstrumentPlayer != null)
             {
-                _batchesDropped = 0;
+                InstrumentPlayer = null;
+            }
+
+            if ((_batchesDropped >= MaxMidiBatchDropped
+                    || _laggedBatches >= MaxMidiLaggedBatches)
+                && InstrumentPlayer != null)
+            {
                 var mob = InstrumentPlayer.AttachedEntity;
 
+                SendNetworkMessage(new InstrumentStopMidiMessage());
+                Playing = false;
+
                 _userInterface.CloseAll();
 
                 if (mob.TryGetComponent(out StunnableComponent stun))
+                {
                     stun.Stun(1);
+                    Clean();
+                }
                 else
+                {
                     StandingStateHelper.DropAllItemsInHands(mob);
+                }
 
                 InstrumentPlayer = null;
 
@@ -247,8 +332,11 @@ namespace Content.Server.GameObjects.Components.Instruments
 
             _timer += delta;
             if (_timer < 1) return;
+
             _timer = 0f;
             _midiEventCount = 0;
         }
+
     }
+
 }

Content.Shared/GameObjects/Components/Instruments/SharedInstrumentComponent.cs

@@ -8,6 +8,13 @@ namespace Content.Shared.GameObjects.Components.Instruments
 {
     public class SharedInstrumentComponent : Component
     {
+
+        // These 2 values are quite high for now, and this could be easily abused. Change this if people are abusing it.
+        public const int MaxMidiEventsPerSecond = 1000;
+        public const int MaxMidiEventsPerBatch = 60;
+        public const int MaxMidiBatchDropped = 1;
+        public const int MaxMidiLaggedBatches = 8;
+
         public override string Name => "Instrument";
         public override uint? NetID => ContentNetIDs.INSTRUMENTS;
 
@@ -52,12 +59,10 @@ namespace Content.Shared.GameObjects.Components.Instruments
     public class InstrumentState : ComponentState
     {
         public bool Playing { get; }
-        public uint SequencerTick { get; }
 
         public InstrumentState(bool playing, uint sequencerTick = 0) : base(ContentNetIDs.INSTRUMENTS)
         {
             Playing = playing;
-            SequencerTick = sequencerTick;
         }
     }