diff --git a/Content.Server/Administration/Logs/AdminLogManager.cs b/Content.Server/Administration/Logs/AdminLogManager.cs index f7d2346568..06f58f10b2 100644 --- a/Content.Server/Administration/Logs/AdminLogManager.cs +++ b/Content.Server/Administration/Logs/AdminLogManager.cs @@ -12,7 +12,6 @@ using Robust.Shared; using Robust.Shared.Configuration; using Robust.Shared.Reflection; using Robust.Shared.Timing; -using Robust.Shared.Utility; namespace Content.Server.Administration.Logs; @@ -66,6 +65,7 @@ public sealed partial class AdminLogManager : SharedAdminLogManager, IAdminLogMa private TimeSpan _queueSendDelay; private int _queueMax; private int _preRoundQueueMax; + private int _dropThreshold; // Per update private TimeSpan _nextUpdateTime; @@ -78,6 +78,10 @@ public sealed partial class AdminLogManager : SharedAdminLogManager, IAdminLogMa private int NextLogId => Interlocked.Increment(ref _currentLogId); private GameRunLevel _runLevel = GameRunLevel.PreRoundLobby; + // 1 when saving, 0 otherwise + private int _savingLogs; + private int _logsDropped; + public void Initialize() { _sawmill = _logManager.GetSawmill(SawmillId); @@ -94,6 +98,8 @@ public sealed partial class AdminLogManager : SharedAdminLogManager, IAdminLogMa value => _queueMax = value, true); _configuration.OnValueChanged(CCVars.AdminLogsPreRoundQueueMax, value => _preRoundQueueMax = value, true); + _configuration.OnValueChanged(CCVars.AdminLogsDropThreshold, + value => _dropThreshold = value, true); if (_metricsEnabled) { @@ -132,7 +138,7 @@ public sealed partial class AdminLogManager : SharedAdminLogManager, IAdminLogMa if (_timing.RealTime >= _nextUpdateTime) { - await SaveLogs(); + await TrySaveLogs(); return; } @@ -143,8 +149,7 @@ public sealed partial class AdminLogManager : SharedAdminLogManager, IAdminLogMa QueueCapReached.Inc(); } - _sawmill.Warning($"Maximum cap of {_queueMax} reached for admin logs."); - await SaveLogs(); + await TrySaveLogs(); } } @@ -163,8 +168,22 @@ public sealed partial class AdminLogManager : SharedAdminLogManager, IAdminLogMa PreRoundQueueCapReached.Inc(); } - _sawmill.Warning($"Maximum cap of {_preRoundQueueMax} reached for pre-round admin logs."); - await SaveLogs(); + await TrySaveLogs(); + } + + private async Task TrySaveLogs() + { + if (Interlocked.Exchange(ref _savingLogs, 1) == 1) + return; + + try + { + await SaveLogs(); + } + finally + { + Interlocked.Exchange(ref _savingLogs, 0); + } } private async Task SaveLogs() @@ -173,10 +192,18 @@ public sealed partial class AdminLogManager : SharedAdminLogManager, IAdminLogMa // TODO ADMIN LOGS array pool var copy = new List(_logQueue.Count + _preRoundLogQueue.Count); - copy.AddRange(_logQueue); - _logQueue.Clear(); - Queue.Set(0); + + if (_logQueue.Count >= _queueMax) + { + _sawmill.Warning($"In-round cap of {_queueMax} reached for admin logs."); + } + + var dropped = Interlocked.Exchange(ref _logsDropped, 0); + if (dropped > 0) + { + _sawmill.Error($"Dropped {dropped} logs. Current max threshold: {_dropThreshold}"); + } if (_runLevel == GameRunLevel.PreRoundLobby && !_preRoundLogQueue.IsEmpty) { @@ -193,10 +220,12 @@ public sealed partial class AdminLogManager : SharedAdminLogManager, IAdminLogMa copy.AddRange(_preRoundLogQueue); } + _logQueue.Clear(); + Queue.Set(0); + _preRoundLogQueue.Clear(); PreRoundQueue.Set(0); - // ship the logs to Azkaban var task = _db.AddAdminLogs(copy); _sawmill.Debug($"Saving {copy.Count} admin logs."); @@ -251,6 +280,14 @@ public sealed partial class AdminLogManager : SharedAdminLogManager, IAdminLogMa private void Add(LogType type, LogImpact impact, string message, JsonDocument json, HashSet players, List entities) { + var preRound = _runLevel == GameRunLevel.PreRoundLobby; + var count = preRound ? _preRoundLogQueue.Count : _logQueue.Count; + if (count >= _dropThreshold) + { + Interlocked.Increment(ref _logsDropped); + return; + } + var log = new AdminLog { Id = NextLogId, @@ -275,7 +312,7 @@ public sealed partial class AdminLogManager : SharedAdminLogManager, IAdminLogMa log.Players.Add(player); } - if (_runLevel == GameRunLevel.PreRoundLobby) + if (preRound) { _preRoundLogQueue.Enqueue(log); } diff --git a/Content.Server/Administration/Logs/AdminLogSystem.cs b/Content.Server/Administration/Logs/AdminLogSystem.cs index cf5da2322f..c9ecec320e 100644 --- a/Content.Server/Administration/Logs/AdminLogSystem.cs +++ b/Content.Server/Administration/Logs/AdminLogSystem.cs @@ -26,6 +26,7 @@ public sealed class AdminLogSystem : EntitySystem public override void Shutdown() { + base.Shutdown(); _adminLogs.Shutdown(); } } diff --git a/Content.Shared/CCVar/CCVars.cs b/Content.Shared/CCVar/CCVars.cs index 9e8ad01006..ce768b418f 100644 --- a/Content.Shared/CCVar/CCVars.cs +++ b/Content.Shared/CCVar/CCVars.cs @@ -1,6 +1,5 @@ using Robust.Shared; using Robust.Shared.Configuration; -using Robust.Shared.Utility; namespace Content.Shared.CCVar { @@ -866,12 +865,18 @@ namespace Content.Shared.CCVar public static readonly CVarDef AdminLogsQueueSendDelay = CVarDef.Create("adminlogs.queue_send_delay_seconds", 5f, CVar.SERVERONLY); + // When to skip the waiting time to save in-round admin logs, if no admin logs are currently being saved public static readonly CVarDef AdminLogsQueueMax = CVarDef.Create("adminlogs.queue_max", 5000, CVar.SERVERONLY); + // When to skip the waiting time to save pre-round admin logs, if no admin logs are currently being saved public static readonly CVarDef AdminLogsPreRoundQueueMax = CVarDef.Create("adminlogs.pre_round_queue_max", 5000, CVar.SERVERONLY); + // When to start dropping logs + public static readonly CVarDef AdminLogsDropThreshold = + CVarDef.Create("adminlogs.drop_threshold", 20000, CVar.SERVERONLY); + // How many logs to send to the client at once public static readonly CVarDef AdminLogsClientBatchSize = CVarDef.Create("adminlogs.client_batch_size", 1000, CVar.SERVERONLY);