diff --git a/Content.Client/GameObjects/Components/Instruments/InstrumentComponent.cs b/Content.Client/GameObjects/Components/Instruments/InstrumentComponent.cs index dbb5d7e536..ed530f9237 100644 --- a/Content.Client/GameObjects/Components/Instruments/InstrumentComponent.cs +++ b/Content.Client/GameObjects/Components/Instruments/InstrumentComponent.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using Content.Shared.GameObjects.Components.Instruments; @@ -7,23 +8,26 @@ using JetBrains.Annotations; using NFluidsynth; using Robust.Shared.GameObjects; using Robust.Client.Audio.Midi; +using Robust.Client.Player; +using Robust.Shared.Interfaces.Log; using Robust.Shared.Interfaces.Network; using Robust.Shared.Interfaces.Timing; using Robust.Shared.IoC; +using Robust.Shared.Log; using Robust.Shared.Players; using Robust.Shared.Serialization; +using Robust.Shared.Timing; using Robust.Shared.ViewVariables; using Logger = Robust.Shared.Log.Logger; using MidiEvent = Robust.Shared.Audio.Midi.MidiEvent; using Timer = Robust.Shared.Timers.Timer; - namespace Content.Client.GameObjects.Components.Instruments { + [RegisterComponent] public class InstrumentComponent : SharedInstrumentComponent { - public const float TimeBetweenNetMessages = 1.0f; /// /// Called when a midi song stops playing. @@ -31,23 +35,29 @@ namespace Content.Client.GameObjects.Components.Instruments public event Action OnMidiPlaybackEnded; #pragma warning disable 649 - [Dependency] private IMidiManager _midiManager; + [Dependency] private readonly IMidiManager _midiManager; + + [Dependency] private readonly IGameTiming _gameTiming; + + [Dependency] private readonly IClientNetManager _netManager; #pragma warning restore 649 [CanBeNull] private IMidiRenderer _renderer; + private byte _instrumentProgram = 1; + private byte _instrumentBank = 0; - private uint _syncSequencerTick; + + private uint _sequenceDelay = 0; + + private uint _sequenceStartTick; /// /// A queue of MidiEvents to be sent to the server. /// [ViewVariables] - private readonly Queue _midiQueue = new Queue(); - - [ViewVariables] - private float _timer = 0f; + private readonly List _midiEventBuffer = new List(); /// /// Whether a midi song will loop or not. @@ -123,11 +133,12 @@ namespace Content.Client.GameObjects.Components.Instruments IoCManager.InjectDependencies(this); } - protected void SetupRenderer() + protected void SetupRenderer(bool fromStateChange = false) { - if (IsRendererAlive) - return; + if (IsRendererAlive) return; + _sequenceDelay = 0; + _sequenceStartTick = 0; _midiManager.OcclusionCollisionMask = (int) CollisionGroup.Impassable; _renderer = _midiManager.GetNewRenderer(); @@ -136,17 +147,32 @@ namespace Content.Client.GameObjects.Components.Instruments _renderer.MidiBank = _instrumentBank; _renderer.MidiProgram = _instrumentProgram; _renderer.TrackingEntity = Owner; - _renderer.OnMidiPlayerFinished += () => { OnMidiPlaybackEnded?.Invoke(); EndRenderer(); SendNetworkMessage(new InstrumentStopMidiMessage()); }; + _renderer.OnMidiPlayerFinished += () => + { + OnMidiPlaybackEnded?.Invoke(); + EndRenderer(fromStateChange); + }; + } + + if (!fromStateChange) + { + SendNetworkMessage(new InstrumentStartMidiMessage()); } } - protected void EndRenderer() + protected void EndRenderer(bool fromStateChange = false) { if (IsInputOpen) - CloseInput(); + { + CloseInput(fromStateChange); + return; + } if (IsMidiOpen) - CloseMidi(); + { + CloseMidi(fromStateChange); + return; + } _renderer?.StopAllNotes(); @@ -155,7 +181,12 @@ namespace Content.Client.GameObjects.Components.Instruments // We dispose of the synth two seconds from now to allow the last notes to stop from playing. Timer.Spawn(2000, () => { renderer?.Dispose(); }); _renderer = null; - _midiQueue.Clear(); + _midiEventBuffer.Clear(); + + if (!fromStateChange) + { + SendNetworkMessage(new InstrumentStopMidiMessage()); + } } protected override void Shutdown() @@ -167,8 +198,8 @@ namespace Content.Client.GameObjects.Components.Instruments public override void ExposeData(ObjectSerializer serializer) { base.ExposeData(serializer); - serializer.DataField(ref _instrumentProgram, "program", (byte)1); - serializer.DataField(ref _instrumentBank, "bank", (byte)0); + serializer.DataField(ref _instrumentProgram, "program", (byte) 1); + serializer.DataField(ref _instrumentBank, "bank", (byte) 0); } public override void HandleNetworkMessage(ComponentMessage message, INetChannel channel, ICommonSession session = null) @@ -178,15 +209,65 @@ namespace Content.Client.GameObjects.Components.Instruments switch (message) { case InstrumentMidiEventMessage midiEventMessage: - // If we're the ones sending the MidiEvents, we ignore this message. - if (!IsRendererAlive || IsInputOpen || IsMidiOpen) break; + if (IsRendererAlive) + { + // If we're the ones sending the MidiEvents, we ignore this message. + if (IsInputOpen || IsMidiOpen) break; + } + else + { + // if we haven't started or finished some sequence + if (_sequenceStartTick == 0) + { + // we may have arrived late + SetupRenderer(true); + } + + // might be our own notes after we already finished playing + return; + } + + if (_sequenceStartTick <= 0) + { + _sequenceStartTick = midiEventMessage.MidiEvent + .Min(x => x.Tick) - 1; + } + + var sqrtLag = MathF.Sqrt(_netManager.ServerChannel.Ping / 1000f); + var delay = (uint) (_renderer!.SequencerTimeScale * (.2 + sqrtLag)); + var delta = delay - _sequenceStartTick; + + _sequenceDelay = Math.Max(_sequenceDelay, delta); + + var currentTick = _renderer.SequencerTick; + + // ReSharper disable once ForCanBeConvertedToForeach for (var i = 0; i < midiEventMessage.MidiEvent.Length; i++) { var ev = midiEventMessage.MidiEvent[i]; - var delta = ((uint)TimeBetweenNetMessages*1250) + ev.Timestamp - _syncSequencerTick; - _renderer?.ScheduleMidiEvent(ev, delta, true); + var scheduled = ev.Tick + _sequenceDelay; + + if (scheduled <= currentTick) + { + _sequenceDelay += currentTick - ev.Tick; + scheduled = ev.Tick + _sequenceDelay; + } + + + _renderer?.ScheduleMidiEvent(ev, scheduled, true); } + break; + case InstrumentStartMidiMessage startMidiMessage: + { + SetupRenderer(true); + break; + } + case InstrumentStopMidiMessage stopMidiMessage: + { + EndRenderer(true); + break; + } } } @@ -197,18 +278,18 @@ namespace Content.Client.GameObjects.Components.Instruments if (state.Playing) { - SetupRenderer(); - _syncSequencerTick = state.SequencerTick; + SetupRenderer(true); } else - EndRenderer(); + { + EndRenderer(true); + } } /// public bool OpenInput() { SetupRenderer(); - SendNetworkMessage(new InstrumentStartMidiMessage()); if (_renderer != null && _renderer.OpenInput()) { @@ -220,15 +301,14 @@ namespace Content.Client.GameObjects.Components.Instruments } /// - public bool CloseInput() + public bool CloseInput(bool fromStateChange = false) { if (_renderer == null || !_renderer.CloseInput()) { return false; } - EndRenderer(); - SendNetworkMessage(new InstrumentStopMidiMessage()); + EndRenderer(fromStateChange); return true; } @@ -236,7 +316,6 @@ namespace Content.Client.GameObjects.Components.Instruments public bool OpenMidi(string filename) { SetupRenderer(); - SendNetworkMessage(new InstrumentStartMidiMessage()); if (_renderer == null || !_renderer.OpenMidi(filename)) { @@ -248,15 +327,14 @@ namespace Content.Client.GameObjects.Components.Instruments } /// - public bool CloseMidi() + public bool CloseMidi(bool fromStateChange = false) { if (_renderer == null || !_renderer.CloseMidi()) { return false; } - EndRenderer(); - SendNetworkMessage(new InstrumentStopMidiMessage()); + EndRenderer(fromStateChange); return true; } @@ -266,29 +344,83 @@ namespace Content.Client.GameObjects.Components.Instruments /// The received midi event private void RendererOnMidiEvent(MidiEvent midiEvent) { - _midiQueue.Enqueue(midiEvent); + // avoid of out-of-band, unimportant or unsupported events + switch (midiEvent.Type) + { + case 0x80: // NOTE_OFF + case 0x90: // NOTE_ON + case 0xa0: // KEY_PRESSURE + case 0xb0: // CONTROL_CHANGE + case 0xd0: // CHANNEL_PRESSURE + case 0xe0: // PITCH_BEND + break; + default: + return; + } + + _midiEventBuffer.Add(midiEvent); } + private TimeSpan _lastMeasured = TimeSpan.MinValue; + + private int _sentWithinASec = 0; + + private static readonly TimeSpan OneSecAgo = TimeSpan.FromSeconds(-1); + + private static readonly Comparer SortMidiEventTick + = Comparer.Create((x, y) + => x.Tick.CompareTo(y.Tick)); + public override void Update(float delta) { - if (!IsMidiOpen && !IsInputOpen) + if (!IsMidiOpen && !IsInputOpen) return; + + var now = _gameTiming.RealTime; + var oneSecAGo = now.Add(OneSecAgo); + + if (_lastMeasured <= oneSecAGo) + { + _lastMeasured = now; + _sentWithinASec = 0; + } + + if (_midiEventBuffer.Count == 0) return; + + var max = Math.Min(MaxMidiEventsPerBatch, MaxMidiEventsPerSecond - _sentWithinASec); + + if (max <= 0) + { + // hit event/sec limit, have to lag the batch or drop events return; + } - _timer -= delta; + // fix cross-fade events generating retroactive events + // also handle any significant backlog of events after midi finished - if (_timer > 0f) return; + _midiEventBuffer.Sort(SortMidiEventTick); + var bufferTicks = IsRendererAlive && _renderer!.Status != MidiRendererStatus.None + ? _renderer.SequencerTimeScale * .2f + : 0; + var bufferedTick = IsRendererAlive + ? _renderer!.SequencerTick - bufferTicks + : int.MaxValue; - SendAllMidiMessages(); - _timer = TimeBetweenNetMessages; - } + var events = _midiEventBuffer + .TakeWhile(x => x.Tick < bufferedTick) + .Take(max) + .ToArray(); - private void SendAllMidiMessages() - { - if (_midiQueue.Count == 0) return; - var events = _midiQueue.ToArray(); - _midiQueue.Clear(); + var eventCount = events.Length; + + if (eventCount == 0) return; SendNetworkMessage(new InstrumentMidiEventMessage(events)); + + _sentWithinASec += eventCount; + + _midiEventBuffer.RemoveRange(0, eventCount); } + } + } diff --git a/Content.Server/GameObjects/Components/Instruments/InstrumentComponent.cs b/Content.Server/GameObjects/Components/Instruments/InstrumentComponent.cs index 5c7f05872c..b0a28c2c4d 100644 --- a/Content.Server/GameObjects/Components/Instruments/InstrumentComponent.cs +++ b/Content.Server/GameObjects/Components/Instruments/InstrumentComponent.cs @@ -1,3 +1,5 @@ +using System; +using System.Linq; using Content.Server.GameObjects.Components.Mobs; using Content.Server.GameObjects.EntitySystems; using Content.Server.Interfaces; @@ -14,7 +16,9 @@ using Robust.Server.Player; using Robust.Shared.Enums; using Robust.Shared.GameObjects; using Robust.Shared.Interfaces.GameObjects; +using Robust.Shared.Interfaces.Log; using Robust.Shared.Interfaces.Network; +using Robust.Shared.Interfaces.Timing; using Robust.Shared.IoC; using Robust.Shared.Players; using Robust.Shared.Serialization; @@ -24,25 +28,33 @@ using MidiEvent = Robust.Shared.Audio.Midi.MidiEvent; namespace Content.Server.GameObjects.Components.Instruments { + [RegisterComponent] [ComponentReference(typeof(IActivate))] - public class InstrumentComponent : SharedInstrumentComponent, - IDropped, IHandSelected, IHandDeselected, IActivate, IUse, IThrown + public class InstrumentComponent + : SharedInstrumentComponent, + IDropped, + IHandSelected, + IHandDeselected, + IActivate, + IUse, + IThrown { + #pragma warning disable 649 - [Dependency] private IServerNotifyManager _notifyManager; + [Dependency] private readonly IServerNotifyManager _notifyManager; + + [Dependency] private readonly IGameTiming _gameTiming; #pragma warning restore 649 - // These 2 values are quite high for now, and this could be easily abused. Change this if people are abusing it. - public const int MaxMidiEventsPerSecond = 20; - public const int MaxMidiEventsPerBatch = 50; - public const int MaxMidiBatchDropped = 20; + private static readonly TimeSpan OneSecAgo = TimeSpan.FromSeconds(-1); /// /// The client channel currently playing the instrument, or null if there's none. /// [ViewVariables] private IPlayerSession _instrumentPlayer; + private bool _handheld; [ViewVariables] @@ -51,9 +63,15 @@ namespace Content.Server.GameObjects.Components.Instruments [ViewVariables] private float _timer = 0f; + [ViewVariables(VVAccess.ReadOnly)] + private TimeSpan _lastMeasured = TimeSpan.MinValue; + [ViewVariables] private int _batchesDropped = 0; + [ViewVariables] + private int _laggedBatches = 0; + [ViewVariables] private uint _lastSequencerTick = 0; @@ -90,12 +108,12 @@ namespace Content.Server.GameObjects.Components.Instruments { Playing = false; - if(_instrumentPlayer != null) + if (_instrumentPlayer != null) _instrumentPlayer.PlayerStatusChanged -= OnPlayerStatusChanged; _instrumentPlayer = value; - if(value != null) + if (value != null) _instrumentPlayer.PlayerStatusChanged += OnPlayerStatusChanged; } } @@ -131,30 +149,87 @@ namespace Content.Server.GameObjects.Components.Instruments switch (message) { case InstrumentMidiEventMessage midiEventMsg: - if (!Playing || session != _instrumentPlayer) - return; + if (!Playing || session != _instrumentPlayer) return; - if (++_midiEventCount <= MaxMidiEventsPerSecond && - midiEventMsg.MidiEvent.Length < MaxMidiEventsPerBatch) + var send = true; + + var minTick = midiEventMsg.MidiEvent.Min(x => x.Tick); + if (_lastSequencerTick > minTick) + { + var now = _gameTiming.RealTime; + var oneSecAGo = now.Add(OneSecAgo); + if (_lastMeasured < oneSecAGo) + { + _lastMeasured = now; + _laggedBatches = 0; + _batchesDropped = 0; + } + + _laggedBatches++; + switch (_laggedBatches) + { + case (int) (MaxMidiLaggedBatches * (1 / 3d)) + 1: + _notifyManager.PopupMessage(Owner, InstrumentPlayer.AttachedEntity, + "Your fingers are beginning to a cramp a little!"); + break; + case (int) (MaxMidiLaggedBatches * (2 / 3d)) + 1: + _notifyManager.PopupMessage(Owner, InstrumentPlayer.AttachedEntity, + "Your fingers are seriously cramping up!"); + break; + } + + if (_laggedBatches > MaxMidiLaggedBatches) + { + send = false; + } + } + + if (++_midiEventCount > MaxMidiEventsPerSecond + || midiEventMsg.MidiEvent.Length > MaxMidiEventsPerBatch) + { + var now = _gameTiming.RealTime; + var oneSecAGo = now.Add(OneSecAgo); + if (_lastMeasured < oneSecAGo) + { + _lastMeasured = now; + _laggedBatches = 0; + _batchesDropped = 0; + } + + _batchesDropped++; + + send = false; + } + + if (send) + { SendNetworkMessage(midiEventMsg); - else - _batchesDropped++; // Batch dropped! + } - _lastSequencerTick = midiEventMsg.MidiEvent[^1].Timestamp; + var maxTick = midiEventMsg.MidiEvent.Max(x => x.Tick); + _lastSequencerTick = Math.Max(maxTick, minTick + 1); break; case InstrumentStartMidiMessage startMidi: Playing = true; break; case InstrumentStopMidiMessage stopMidi: Playing = false; - _lastSequencerTick = 0; + Clean(); break; } } - public void Dropped(DroppedEventArgs eventArgs) + private void Clean() { Playing = false; + _lastSequencerTick = 0; + _batchesDropped = 0; + _laggedBatches = 0; + } + + public void Dropped(DroppedEventArgs eventArgs) + { + Clean(); SendNetworkMessage(new InstrumentStopMidiMessage()); InstrumentPlayer = null; _userInterface.CloseAll(); @@ -162,7 +237,7 @@ namespace Content.Server.GameObjects.Components.Instruments public void Thrown(ThrownEventArgs eventArgs) { - Playing = false; + Clean(); SendNetworkMessage(new InstrumentStopMidiMessage()); InstrumentPlayer = null; _userInterface.CloseAll(); @@ -179,18 +254,16 @@ namespace Content.Server.GameObjects.Components.Instruments public void HandDeselected(HandDeselectedEventArgs eventArgs) { - Playing = false; + Clean(); SendNetworkMessage(new InstrumentStopMidiMessage()); _userInterface.CloseAll(); } public void Activate(ActivateEventArgs eventArgs) { - if (Handheld || !eventArgs.User.TryGetComponent(out IActorComponent actor)) - return; + if (Handheld || !eventArgs.User.TryGetComponent(out IActorComponent actor)) return; - if (InstrumentPlayer != null) - return; + if (InstrumentPlayer != null) return; InstrumentPlayer = actor.playerSession; OpenUserInterface(actor.playerSession); @@ -198,22 +271,23 @@ namespace Content.Server.GameObjects.Components.Instruments public bool UseEntity(UseEntityEventArgs eventArgs) { - if (!eventArgs.User.TryGetComponent(out IActorComponent actor)) - return false; + if (!eventArgs.User.TryGetComponent(out IActorComponent actor)) return false; - if(InstrumentPlayer == actor.playerSession) + if (InstrumentPlayer == actor.playerSession) + { OpenUserInterface(actor.playerSession); + } + return false; } private void UserInterfaceOnClosed(IPlayerSession player) { - if (!Handheld && player == InstrumentPlayer) - { - InstrumentPlayer = null; - SendNetworkMessage(new InstrumentStopMidiMessage()); - Playing = false; - } + if (Handheld || player != InstrumentPlayer) return; + + Clean(); + InstrumentPlayer = null; + SendNetworkMessage(new InstrumentStopMidiMessage()); } private void OpenUserInterface(IPlayerSession session) @@ -226,19 +300,30 @@ namespace Content.Server.GameObjects.Components.Instruments base.Update(delta); if (_instrumentPlayer != null && !ActionBlockerSystem.CanInteract(_instrumentPlayer.AttachedEntity)) - InstrumentPlayer = null; - - if (_batchesDropped > MaxMidiBatchDropped && InstrumentPlayer != null) { - _batchesDropped = 0; + InstrumentPlayer = null; + } + + if ((_batchesDropped >= MaxMidiBatchDropped + || _laggedBatches >= MaxMidiLaggedBatches) + && InstrumentPlayer != null) + { var mob = InstrumentPlayer.AttachedEntity; + SendNetworkMessage(new InstrumentStopMidiMessage()); + Playing = false; + _userInterface.CloseAll(); if (mob.TryGetComponent(out StunnableComponent stun)) + { stun.Stun(1); + Clean(); + } else + { StandingStateHelper.DropAllItemsInHands(mob); + } InstrumentPlayer = null; @@ -247,8 +332,11 @@ namespace Content.Server.GameObjects.Components.Instruments _timer += delta; if (_timer < 1) return; + _timer = 0f; _midiEventCount = 0; } + } + } diff --git a/Content.Shared/GameObjects/Components/Instruments/SharedInstrumentComponent.cs b/Content.Shared/GameObjects/Components/Instruments/SharedInstrumentComponent.cs index ef94e31e86..c431a25fc8 100644 --- a/Content.Shared/GameObjects/Components/Instruments/SharedInstrumentComponent.cs +++ b/Content.Shared/GameObjects/Components/Instruments/SharedInstrumentComponent.cs @@ -8,6 +8,13 @@ namespace Content.Shared.GameObjects.Components.Instruments { public class SharedInstrumentComponent : Component { + + // These 2 values are quite high for now, and this could be easily abused. Change this if people are abusing it. + public const int MaxMidiEventsPerSecond = 1000; + public const int MaxMidiEventsPerBatch = 60; + public const int MaxMidiBatchDropped = 1; + public const int MaxMidiLaggedBatches = 8; + public override string Name => "Instrument"; public override uint? NetID => ContentNetIDs.INSTRUMENTS; @@ -52,12 +59,10 @@ namespace Content.Shared.GameObjects.Components.Instruments public class InstrumentState : ComponentState { public bool Playing { get; } - public uint SequencerTick { get; } public InstrumentState(bool playing, uint sequencerTick = 0) : base(ContentNetIDs.INSTRUMENTS) { Playing = playing; - SequencerTick = sequencerTick; } } diff --git a/RobustToolbox b/RobustToolbox index 31c5c9373f..35c23bd905 160000 --- a/RobustToolbox +++ b/RobustToolbox @@ -1 +1 @@ -Subproject commit 31c5c9373f661aac6f59168bfb7720bb923b78eb +Subproject commit 35c23bd905643f10e025ba030c9e1cc6ad0a423a