Limit saving admin logs to one task (#19146)

Content.Server/Administration/Logs/AdminLogManager.cs

@@ -12,7 +12,6 @@ using Robust.Shared;
 using Robust.Shared.Configuration;
 using Robust.Shared.Reflection;
 using Robust.Shared.Timing;
-using Robust.Shared.Utility;
 
 namespace Content.Server.Administration.Logs;
 
@@ -66,6 +65,7 @@ public sealed partial class AdminLogManager : SharedAdminLogManager, IAdminLogMa
     private TimeSpan _queueSendDelay;
     private int _queueMax;
     private int _preRoundQueueMax;
+    private int _dropThreshold;
 
     // Per update
     private TimeSpan _nextUpdateTime;
@@ -78,6 +78,10 @@ public sealed partial class AdminLogManager : SharedAdminLogManager, IAdminLogMa
     private int NextLogId => Interlocked.Increment(ref _currentLogId);
     private GameRunLevel _runLevel = GameRunLevel.PreRoundLobby;
 
+    // 1 when saving, 0 otherwise
+    private int _savingLogs;
+    private int _logsDropped;
+
     public void Initialize()
     {
         _sawmill = _logManager.GetSawmill(SawmillId);
@@ -94,6 +98,8 @@ public sealed partial class AdminLogManager : SharedAdminLogManager, IAdminLogMa
             value => _queueMax = value, true);
         _configuration.OnValueChanged(CCVars.AdminLogsPreRoundQueueMax,
             value => _preRoundQueueMax = value, true);
+        _configuration.OnValueChanged(CCVars.AdminLogsDropThreshold,
+            value => _dropThreshold = value, true);
 
         if (_metricsEnabled)
         {
@@ -132,7 +138,7 @@ public sealed partial class AdminLogManager : SharedAdminLogManager, IAdminLogMa
 
         if (_timing.RealTime >= _nextUpdateTime)
         {
-            await SaveLogs();
+            await TrySaveLogs();
             return;
         }
 
@@ -143,8 +149,7 @@ public sealed partial class AdminLogManager : SharedAdminLogManager, IAdminLogMa
                 QueueCapReached.Inc();
             }
 
-            _sawmill.Warning($"Maximum cap of {_queueMax} reached for admin logs.");
+            await TrySaveLogs();
-            await SaveLogs();
         }
     }
 
@@ -163,8 +168,22 @@ public sealed partial class AdminLogManager : SharedAdminLogManager, IAdminLogMa
             PreRoundQueueCapReached.Inc();
         }
 
-        _sawmill.Warning($"Maximum cap of {_preRoundQueueMax} reached for pre-round admin logs.");
+        await TrySaveLogs();
-        await SaveLogs();
+    }
+
+    private async Task TrySaveLogs()
+    {
+        if (Interlocked.Exchange(ref _savingLogs, 1) == 1)
+            return;
+
+        try
+        {
+            await SaveLogs();
+        }
+        finally
+        {
+            Interlocked.Exchange(ref _savingLogs, 0);
+        }
     }
 
     private async Task SaveLogs()
@@ -173,10 +192,18 @@ public sealed partial class AdminLogManager : SharedAdminLogManager, IAdminLogMa
 
         // TODO ADMIN LOGS array pool
         var copy = new List<AdminLog>(_logQueue.Count + _preRoundLogQueue.Count);
-
         copy.AddRange(_logQueue);
-        _logQueue.Clear();
+
-        Queue.Set(0);
+        if (_logQueue.Count >= _queueMax)
+        {
+            _sawmill.Warning($"In-round cap of {_queueMax} reached for admin logs.");
+        }
+
+        var dropped = Interlocked.Exchange(ref _logsDropped, 0);
+        if (dropped > 0)
+        {
+            _sawmill.Error($"Dropped {dropped} logs. Current max threshold: {_dropThreshold}");
+        }
 
         if (_runLevel == GameRunLevel.PreRoundLobby && !_preRoundLogQueue.IsEmpty)
         {
@@ -193,10 +220,12 @@ public sealed partial class AdminLogManager : SharedAdminLogManager, IAdminLogMa
             copy.AddRange(_preRoundLogQueue);
         }
 
+        _logQueue.Clear();
+        Queue.Set(0);
+
         _preRoundLogQueue.Clear();
         PreRoundQueue.Set(0);
 
-        // ship the logs to Azkaban
         var task = _db.AddAdminLogs(copy);
 
         _sawmill.Debug($"Saving {copy.Count} admin logs.");
@@ -251,6 +280,14 @@ public sealed partial class AdminLogManager : SharedAdminLogManager, IAdminLogMa
 
     private void Add(LogType type, LogImpact impact, string message, JsonDocument json, HashSet<Guid> players, List<AdminLogEntity> entities)
     {
+        var preRound = _runLevel == GameRunLevel.PreRoundLobby;
+        var count = preRound ? _preRoundLogQueue.Count : _logQueue.Count;
+        if (count >= _dropThreshold)
+        {
+            Interlocked.Increment(ref _logsDropped);
+            return;
+        }
+
         var log = new AdminLog
         {
             Id = NextLogId,
@@ -275,7 +312,7 @@ public sealed partial class AdminLogManager : SharedAdminLogManager, IAdminLogMa
             log.Players.Add(player);
         }
 
-        if (_runLevel == GameRunLevel.PreRoundLobby)
+        if (preRound)
         {
             _preRoundLogQueue.Enqueue(log);
         }

Content.Server/Administration/Logs/AdminLogSystem.cs

@@ -26,6 +26,7 @@ public sealed class AdminLogSystem : EntitySystem
 
     public override void Shutdown()
     {
+        base.Shutdown();
         _adminLogs.Shutdown();
     }
 }

Content.Shared/CCVar/CCVars.cs

@@ -1,6 +1,5 @@
 using Robust.Shared;
 using Robust.Shared.Configuration;
-using Robust.Shared.Utility;
 
 namespace Content.Shared.CCVar
 {
@@ -866,12 +865,18 @@ namespace Content.Shared.CCVar
         public static readonly CVarDef<float> AdminLogsQueueSendDelay =
             CVarDef.Create("adminlogs.queue_send_delay_seconds", 5f, CVar.SERVERONLY);
 
+        // When to skip the waiting time to save in-round admin logs, if no admin logs are currently being saved
         public static readonly CVarDef<int> AdminLogsQueueMax =
             CVarDef.Create("adminlogs.queue_max", 5000, CVar.SERVERONLY);
 
+        // When to skip the waiting time to save pre-round admin logs, if no admin logs are currently being saved
         public static readonly CVarDef<int> AdminLogsPreRoundQueueMax =
             CVarDef.Create("adminlogs.pre_round_queue_max", 5000, CVar.SERVERONLY);
 
+        // When to start dropping logs
+        public static readonly CVarDef<int> AdminLogsDropThreshold =
+            CVarDef.Create("adminlogs.drop_threshold", 20000, CVar.SERVERONLY);
+
         // How many logs to send to the client at once
         public static readonly CVarDef<int> AdminLogsClientBatchSize =
             CVarDef.Create("adminlogs.client_batch_size", 1000, CVar.SERVERONLY);