Merge branches '2020-05-18-midi' and '20-05-22-midi-update'

Content.Client/GameObjects/Components/Instruments/InstrumentComponent.cs

@@ -1,17 +1,19 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using Content.Shared.GameObjects.Components.Instruments;
 using JetBrains.Annotations;
+using NFluidsynth;
 using Robust.Shared.GameObjects;
 using Robust.Client.Audio.Midi;
-using Robust.Shared.Audio.Midi;
-using Robust.Shared.Interfaces.GameObjects;
 using Robust.Shared.Interfaces.Network;
 using Robust.Shared.Interfaces.Timing;
 using Robust.Shared.IoC;
 using Robust.Shared.Players;
 using Robust.Shared.Serialization;
 using Robust.Shared.ViewVariables;
+using Logger = Robust.Shared.Log.Logger;
+using MidiEvent = Robust.Shared.Audio.Midi.MidiEvent;
 using Timer = Robust.Shared.Timers.Timer;
 
 
@@ -20,6 +22,8 @@ namespace Content.Client.GameObjects.Components.Instruments
     [RegisterComponent]
     public class InstrumentComponent : SharedInstrumentComponent
     {
+        public const float TimeBetweenNetMessages = 1.0f;
+
         /// <summary>
         ///     Called when a midi song stops playing.
         /// </summary>
@@ -27,17 +31,22 @@ namespace Content.Client.GameObjects.Components.Instruments
 
 #pragma warning disable 649
         [Dependency] private IMidiManager _midiManager;
-        [Dependency] private readonly IGameTiming _timing;
+        [Dependency] private readonly IGameTiming _gameTiming;
 #pragma warning restore 649
 
         [CanBeNull]
         private IMidiRenderer _renderer;
-        private int _instrumentProgram = 1;
+        private byte _instrumentProgram = 1;
+        private uint _syncSequencerTick;
 
         /// <summary>
         ///     A queue of MidiEvents to be sent to the server.
         /// </summary>
-        private Queue<MidiEvent> _eventQueue = new Queue<MidiEvent>();
+        [ViewVariables]
+        private readonly Queue<MidiEvent> _midiQueue = new Queue<MidiEvent>();
+
+        [ViewVariables]
+        private float _timer = 0f;
 
         /// <summary>
         ///     Whether a midi song will loop or not.
@@ -59,7 +68,7 @@ namespace Content.Client.GameObjects.Components.Instruments
         ///     Changes the instrument the midi renderer will play.
         /// </summary>
         [ViewVariables(VVAccess.ReadWrite)]
-        public int InstrumentProgram
+        public byte InstrumentProgram
         {
             get => _instrumentProgram;
             set
@@ -84,61 +93,102 @@ namespace Content.Client.GameObjects.Components.Instruments
         [ViewVariables]
         public bool IsInputOpen => _renderer?.Status == MidiRendererStatus.Input;
 
+        /// <summary>
+        ///     Whether the midi renderer is alive or not.
+        /// </summary>
+        [ViewVariables]
+        public bool IsRendererAlive => _renderer != null;
+
         public override void Initialize()
         {
             base.Initialize();
             IoCManager.InjectDependencies(this);
+        }
+
+        protected void SetupRenderer()
+        {
+            if (IsRendererAlive)
+                return;
+
             _renderer = _midiManager.GetNewRenderer();
 
             if (_renderer != null)
             {
                 _renderer.MidiProgram = _instrumentProgram;
                 _renderer.TrackingEntity = Owner;
-                _renderer.OnMidiPlayerFinished += () => { OnMidiPlaybackEnded?.Invoke(); };
+                _renderer.OnMidiPlayerFinished += () => { OnMidiPlaybackEnded?.Invoke(); EndRenderer(); SendNetworkMessage(new InstrumentStopMidiMessage()); };
             }
         }
 
+        protected void EndRenderer()
+        {
+            if (IsInputOpen)
+                CloseInput();
+
+            if (IsMidiOpen)
+                CloseMidi();
+
+            _renderer?.StopAllNotes();
+
+            var renderer = _renderer;
+
+            // We dispose of the synth two seconds from now to allow the last notes to stop from playing.
+            Timer.Spawn(2000, () => { renderer?.Dispose(); });
+            _renderer = null;
+            _midiQueue.Clear();
+        }
+
         protected override void Shutdown()
         {
             base.Shutdown();
-            _renderer?.Dispose();
+            EndRenderer();
         }
 
         public override void ExposeData(ObjectSerializer serializer)
         {
             base.ExposeData(serializer);
-            serializer.DataField(ref _instrumentProgram, "program", 1);
+            serializer.DataField(ref _instrumentProgram, "program", (byte)1);
         }
 
         public override void HandleNetworkMessage(ComponentMessage message, INetChannel channel, ICommonSession session = null)
         {
             base.HandleNetworkMessage(message, channel, session);
 
-            if (_renderer == null)
-            {
-                return;
-            }
-
             switch (message)
             {
                 case InstrumentMidiEventMessage midiEventMessage:
                     // If we're the ones sending the MidiEvents, we ignore this message.
-                    if (IsInputOpen || IsMidiOpen) break;
+                    if (!IsRendererAlive || IsInputOpen || IsMidiOpen) break;
-                    Timer.Spawn((int) (500 + _timing.CurTime.TotalMilliseconds - midiEventMessage.Timestamp),
+                    for (var i = 0; i < midiEventMessage.MidiEvent.Length; i++)
-                        () => _renderer.SendMidiEvent(midiEventMessage.MidiEvent));
+                    {
-                    break;
+                        var ev = midiEventMessage.MidiEvent[i];
-
+                        var delta = ((uint)TimeBetweenNetMessages*1250) + ev.Timestamp - _syncSequencerTick;
-                case InstrumentStopMidiMessage _:
+                        _renderer?.ScheduleMidiEvent(ev, delta, true);
-                    _renderer.StopAllNotes();
+                    }
-                    if (IsInputOpen) CloseInput();
-                    if (IsMidiOpen) CloseMidi();
                     break;
             }
         }
 
+        public override void HandleComponentState(ComponentState curState, ComponentState nextState)
+        {
+            base.HandleComponentState(curState, nextState);
+            if (!(curState is InstrumentState state)) return;
+
+            if (state.Playing)
+            {
+                SetupRenderer();
+                _syncSequencerTick = state.SequencerTick;
+            }
+            else
+                EndRenderer();
+        }
+
         /// <inheritdoc cref="MidiRenderer.OpenInput"/>
         public bool OpenInput()
         {
+            SetupRenderer();
+            SendNetworkMessage(new InstrumentStartMidiMessage());
+
             if (_renderer != null && _renderer.OpenInput())
             {
                 _renderer.OnMidiEvent += RendererOnMidiEvent;
@@ -156,13 +206,17 @@ namespace Content.Client.GameObjects.Components.Instruments
                 return false;
             }
 
-            _renderer.OnMidiEvent -= RendererOnMidiEvent;
+            EndRenderer();
+            SendNetworkMessage(new InstrumentStopMidiMessage());
             return true;
         }
 
         /// <inheritdoc cref="MidiRenderer.OpenMidi(string)"/>
         public bool OpenMidi(string filename)
         {
+            SetupRenderer();
+            SendNetworkMessage(new InstrumentStartMidiMessage());
+
             if (_renderer == null || !_renderer.OpenMidi(filename))
             {
                 return false;
@@ -180,7 +234,8 @@ namespace Content.Client.GameObjects.Components.Instruments
                 return false;
             }
 
-            _renderer.OnMidiEvent -= RendererOnMidiEvent;
+            EndRenderer();
+            SendNetworkMessage(new InstrumentStopMidiMessage());
             return true;
         }
 
@@ -190,7 +245,29 @@ namespace Content.Client.GameObjects.Components.Instruments
         /// <param name="midiEvent">The received midi event</param>
         private void RendererOnMidiEvent(MidiEvent midiEvent)
         {
-            SendNetworkMessage(new InstrumentMidiEventMessage(midiEvent, _timing.CurTime.TotalMilliseconds));
+            _midiQueue.Enqueue(midiEvent);
+        }
+
+        public override void Update(float delta)
+        {
+            if (!IsMidiOpen && !IsInputOpen)
+                return;
+
+            _timer -= delta;
+
+            if (_timer > 0f) return;
+
+            SendAllMidiMessages();
+            _timer = TimeBetweenNetMessages;
+        }
+
+        private void SendAllMidiMessages()
+        {
+            if (_midiQueue.Count == 0) return;
+            var events = _midiQueue.ToArray();
+            _midiQueue.Clear();
+
+            SendNetworkMessage(new InstrumentMidiEventMessage(events));
         }
     }
 }

Content.Client/GameObjects/EntitySystems/InstrumentSystem.cs

@@ -0,0 +1,25 @@
+using Content.Client.GameObjects.Components.Instruments;
+using Robust.Shared.GameObjects;
+using Robust.Shared.GameObjects.Systems;
+
+namespace Content.Client.GameObjects.EntitySystems
+{
+    public class InstrumentSystem : EntitySystem
+    {
+        public override void Initialize()
+        {
+            base.Initialize();
+            EntityQuery = new TypeEntityQuery(typeof(InstrumentComponent));
+        }
+
+        public override void Update(float frameTime)
+        {
+            base.Update(frameTime);
+
+            foreach (var entity in RelevantEntities)
+            {
+                entity.GetComponent<InstrumentComponent>().Update(frameTime);
+            }
+        }
+    }
+}

Content.Server/GameObjects/Components/Instruments/InstrumentComponent.cs

@@ -1,15 +1,25 @@
+using Content.Server.GameObjects.Components.Mobs;
 using Content.Server.GameObjects.EntitySystems;
+using Content.Server.Interfaces;
+using Content.Server.Interfaces.GameObjects;
+using Content.Server.Mobs;
 using Content.Shared.GameObjects.Components.Instruments;
+using NFluidsynth;
 using Robust.Server.GameObjects;
 using Robust.Server.GameObjects.Components.UserInterface;
 using Robust.Server.Interfaces.GameObjects;
 using Robust.Server.Interfaces.Player;
+using Robust.Server.Player;
+using Robust.Shared.Enums;
 using Robust.Shared.GameObjects;
 using Robust.Shared.Interfaces.GameObjects;
 using Robust.Shared.Interfaces.Network;
+using Robust.Shared.IoC;
 using Robust.Shared.Players;
 using Robust.Shared.Serialization;
 using Robust.Shared.ViewVariables;
+using Logger = Robust.Shared.Log.Logger;
+using MidiEvent = Robust.Shared.Audio.Midi.MidiEvent;
 
 namespace Content.Server.GameObjects.Components.Instruments
 {
@@ -18,12 +28,35 @@ namespace Content.Server.GameObjects.Components.Instruments
     public class InstrumentComponent : SharedInstrumentComponent,
         IDropped, IHandSelected, IHandDeselected, IActivate, IUse, IThrown
     {
+        [Dependency] private IServerNotifyManager _notifyManager;
+
+        // These 2 values are quite high for now, and this could be easily abused. Change this if people are abusing it.
+        public const int MaxMidiEventsPerSecond = 20;
+        public const int MaxMidiEventsPerBatch = 50;
+        public const int MaxMidiBatchDropped = 20;
+
         /// <summary>
         ///     The client channel currently playing the instrument, or null if there's none.
         /// </summary>
-        private ICommonSession _instrumentPlayer;
+        [ViewVariables]
+        private IPlayerSession _instrumentPlayer;
         private bool _handheld;
 
+        [ViewVariables]
+        private bool _playing = false;
+
+        [ViewVariables]
+        private float _timer = 0f;
+
+        [ViewVariables]
+        private int _batchesDropped = 0;
+
+        [ViewVariables]
+        private uint _lastSequencerTick = 0;
+
+        [ViewVariables]
+        private int _midiEventCount = 0;
+
         [ViewVariables]
         private BoundUserInterface _userInterface;
 
@@ -33,6 +66,43 @@ namespace Content.Server.GameObjects.Components.Instruments
         [ViewVariables]
         public bool Handheld => _handheld;
 
+        /// <summary>
+        ///     Whether the instrument is currently playing or not.
+        /// </summary>
+        [ViewVariables]
+        public bool Playing
+        {
+            get => _playing;
+            set
+            {
+                _playing = value;
+                Dirty();
+            }
+        }
+
+        public IPlayerSession InstrumentPlayer
+        {
+            get => _instrumentPlayer;
+            private set
+            {
+                Playing = false;
+
+                if(_instrumentPlayer != null)
+                    _instrumentPlayer.PlayerStatusChanged -= OnPlayerStatusChanged;
+
+                _instrumentPlayer = value;
+
+                if(value != null)
+                    _instrumentPlayer.PlayerStatusChanged += OnPlayerStatusChanged;
+            }
+        }
+
+        private void OnPlayerStatusChanged(object? sender, SessionStatusEventArgs e)
+        {
+            if (e.NewStatus == SessionStatus.Disconnected)
+                InstrumentPlayer = null;
+        }
+
         public override void Initialize()
         {
             base.Initialize();
@@ -46,31 +116,52 @@ namespace Content.Server.GameObjects.Components.Instruments
             serializer.DataField(ref _handheld, "handheld", false);
         }
 
+        public override ComponentState GetComponentState()
+        {
+            return new InstrumentState(Playing, _lastSequencerTick);
+        }
+
         public override void HandleNetworkMessage(ComponentMessage message, INetChannel channel, ICommonSession session = null)
         {
             base.HandleNetworkMessage(message, channel, session);
 
-            // If the client that sent the message isn't the client playing this instrument, we ignore it.
-            if (session != _instrumentPlayer) return;
             switch (message)
             {
                 case InstrumentMidiEventMessage midiEventMsg:
+                    if (!Playing || session != _instrumentPlayer)
+                        return;
+
+                    if (++_midiEventCount <= MaxMidiEventsPerSecond &&
+                        midiEventMsg.MidiEvent.Length < MaxMidiEventsPerBatch)
                         SendNetworkMessage(midiEventMsg);
+                    else
+                        _batchesDropped++; // Batch dropped!
+
+                    _lastSequencerTick = midiEventMsg.MidiEvent[^1].Timestamp;
+                    break;
+                case InstrumentStartMidiMessage startMidi:
+                    Playing = true;
+                    break;
+                case InstrumentStopMidiMessage stopMidi:
+                    Playing = false;
+                    _lastSequencerTick = 0;
                     break;
             }
         }
 
         public void Dropped(DroppedEventArgs eventArgs)
         {
+            Playing = false;
             SendNetworkMessage(new InstrumentStopMidiMessage());
-            _instrumentPlayer = null;
+            InstrumentPlayer = null;
             _userInterface.CloseAll();
         }
 
         public void Thrown(ThrownEventArgs eventArgs)
         {
+            Playing = false;
             SendNetworkMessage(new InstrumentStopMidiMessage());
-            _instrumentPlayer = null;
+            InstrumentPlayer = null;
             _userInterface.CloseAll();
         }
 
@@ -80,11 +171,12 @@ namespace Content.Server.GameObjects.Components.Instruments
 
             if (session == null) return;
 
-            _instrumentPlayer = session;
+            InstrumentPlayer = session;
         }
 
         public void HandDeselected(HandDeselectedEventArgs eventArgs)
         {
+            Playing = false;
             SendNetworkMessage(new InstrumentStopMidiMessage());
             _userInterface.CloseAll();
         }
@@ -94,10 +186,10 @@ namespace Content.Server.GameObjects.Components.Instruments
             if (Handheld || !eventArgs.User.TryGetComponent(out IActorComponent actor))
                 return;
 
-            if (_instrumentPlayer != null)
+            if (InstrumentPlayer != null)
                 return;
 
-            _instrumentPlayer = actor.playerSession;
+            InstrumentPlayer = actor.playerSession;
             OpenUserInterface(actor.playerSession);
         }
 
@@ -106,17 +198,18 @@ namespace Content.Server.GameObjects.Components.Instruments
             if (!eventArgs.User.TryGetComponent(out IActorComponent actor))
                 return false;
 
-            if(_instrumentPlayer == actor.playerSession)
+            if(InstrumentPlayer == actor.playerSession)
                 OpenUserInterface(actor.playerSession);
             return false;
         }
 
         private void UserInterfaceOnClosed(IPlayerSession player)
         {
-            if (!Handheld && player == _instrumentPlayer)
+            if (!Handheld && player == InstrumentPlayer)
             {
-                _instrumentPlayer = null;
+                InstrumentPlayer = null;
                 SendNetworkMessage(new InstrumentStopMidiMessage());
+                Playing = false;
             }
         }
 
@@ -124,5 +217,35 @@ namespace Content.Server.GameObjects.Components.Instruments
         {
             _userInterface.Open(session);
         }
+
+        public override void Update(float delta)
+        {
+            base.Update(delta);
+
+            if (_instrumentPlayer != null && !ActionBlockerSystem.CanInteract(_instrumentPlayer.AttachedEntity))
+                InstrumentPlayer = null;
+
+            if (_batchesDropped > MaxMidiBatchDropped && InstrumentPlayer != null)
+            {
+                _batchesDropped = 0;
+                var mob = InstrumentPlayer.AttachedEntity;
+
+                _userInterface.CloseAll();
+
+                if (mob.TryGetComponent(out StunnableComponent stun))
+                    stun.Stun(1);
+                else
+                    StandingStateHelper.DropAllItemsInHands(mob);
+
+                InstrumentPlayer = null;
+
+                _notifyManager.PopupMessage(Owner, mob, "Your fingers cramp up from playing!");
+            }
+
+            _timer += delta;
+            if (_timer < 1) return;
+            _timer = 0f;
+            _midiEventCount = 0;
+        }
     }
 }

Content.Server/GameObjects/EntitySystems/InstrumentSystem.cs

@@ -0,0 +1,25 @@
+using Content.Server.GameObjects.Components.Instruments;
+using Robust.Shared.GameObjects;
+using Robust.Shared.GameObjects.Systems;
+
+namespace Content.Server.GameObjects.EntitySystems
+{
+    public class InstrumentSystem : EntitySystem
+    {
+        public override void Initialize()
+        {
+            base.Initialize();
+            EntityQuery = new TypeEntityQuery(typeof(InstrumentComponent));
+        }
+
+        public override void Update(float frameTime)
+        {
+            base.Update(frameTime);
+
+            foreach (var entity in RelevantEntities)
+            {
+                entity.GetComponent<InstrumentComponent>().Update(frameTime);
+            }
+        }
+    }
+}

Content.Shared/GameObjects/Components/Instruments/SharedInstrumentComponent.cs

@@ -1,4 +1,5 @@
 using System;
+using Content.Shared.BodySystem;
 using Robust.Shared.Audio.Midi;
 using Robust.Shared.GameObjects;
 using Robust.Shared.Serialization;
@@ -9,6 +10,10 @@ namespace Content.Shared.GameObjects.Components.Instruments
     {
         public override string Name => "Instrument";
         public override uint? NetID => ContentNetIDs.INSTRUMENTS;
+
+        public virtual void Update(float delta)
+        {
+        }
     }
 
 
@@ -20,19 +25,39 @@ namespace Content.Shared.GameObjects.Components.Instruments
     {
     }
 
+    /// <summary>
+    ///     This message is sent to the client to start the synth.
+    /// </summary>
+    [Serializable, NetSerializable]
+    public class InstrumentStartMidiMessage : ComponentMessage
+    {
+
+    }
+
     /// <summary>
     ///     This message carries a MidiEvent to be played on clients.
     /// </summary>
     [Serializable, NetSerializable]
     public class InstrumentMidiEventMessage : ComponentMessage
     {
-        public MidiEvent MidiEvent;
+        public MidiEvent[] MidiEvent;
-        public double Timestamp;
 
-        public InstrumentMidiEventMessage(MidiEvent midiEvent, double timestamp)
+        public InstrumentMidiEventMessage(MidiEvent[] midiEvent)
         {
             MidiEvent = midiEvent;
-            Timestamp = timestamp;
+        }
+    }
+
+    [Serializable, NetSerializable]
+    public class InstrumentState : ComponentState
+    {
+        public bool Playing { get; }
+        public uint SequencerTick { get; }
+
+        public InstrumentState(bool playing, uint sequencerTick = 0) : base(ContentNetIDs.INSTRUMENTS)
+        {
+            Playing = playing;
+            SequencerTick = sequencerTick;
         }
     }