diff --git a/Content.Client/Administration/UI/Logs/AdminLogsControl.xaml.cs b/Content.Client/Administration/UI/Logs/AdminLogsControl.xaml.cs index eb0167fbab..8b8fea5efd 100644 --- a/Content.Client/Administration/UI/Logs/AdminLogsControl.xaml.cs +++ b/Content.Client/Administration/UI/Logs/AdminLogsControl.xaml.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Runtime.InteropServices; using Content.Client.Administration.UI.CustomControls; using Content.Shared.Administration.Logs; using Content.Shared.Database; @@ -390,11 +391,12 @@ public partial class AdminLogsControl : Control UpdateLogs(); } - public void AddLogs(SharedAdminLog[] logs) + public void AddLogs(List logs) { - for (var i = 0; i < logs.Length; i++) + var span = CollectionsMarshal.AsSpan(logs); + for (var i = 0; i < span.Length; i++) { - ref var log = ref logs[i]; + ref var log = ref span[i]; var separator = new HSeparator(); var label = new AdminLogLabel(ref log, separator); label.Visible = ShouldShowLog(label); @@ -404,7 +406,7 @@ public partial class AdminLogsControl : Control } } - public void SetLogs(SharedAdminLog[] logs) + public void SetLogs(List logs) { LogsContainer.RemoveAllChildren(); AddLogs(logs); diff --git a/Content.IntegrationTests/Tests/Administration/Logs/AddTests.cs b/Content.IntegrationTests/Tests/Administration/Logs/AddTests.cs index c32c5932dd..57bc5f113c 100644 --- a/Content.IntegrationTests/Tests/Administration/Logs/AddTests.cs +++ b/Content.IntegrationTests/Tests/Administration/Logs/AddTests.cs @@ -5,6 +5,7 @@ using System.Threading.Tasks; using Content.Server.Administration.Logs; using Content.Server.Database; using Content.Server.GameTicking; +using Content.Shared.Administration.Logs; using Content.Shared.CCVar; using Content.Shared.Database; using NUnit.Framework; @@ -89,6 +90,7 @@ public class AddTests : ContentIntegrationTest var sSystems = server.ResolveDependency(); var sAdminLogSystem = sSystems.GetEntitySystem(); + var sGamerTicker = sSystems.GetEntitySystem(); var guid = Guid.NewGuid(); @@ -100,27 +102,27 @@ public class AddTests : ContentIntegrationTest sAdminLogSystem.Add(LogType.Unknown, $"{entity} test log: {guid}"); }); - LogRecord log = null; + SharedAdminLog log = default; await WaitUntil(server, async () => { - var logs = sAdminLogSystem.CurrentRoundLogs(new LogFilter + var logs = await sAdminLogSystem.CurrentRoundLogs(new LogFilter { Search = guid.ToString() }); - await foreach (var found in logs) + if (logs.Count == 0) { - log = found; - return true; + return false; } - return false; + log = logs.First(); + return true; }); var filter = new LogFilter { - Round = log.RoundId, + Round = sGamerTicker.RoundId, Search = log.Message, Types = new HashSet {log.Type}, }; @@ -137,9 +139,8 @@ public class AddTests : ContentIntegrationTest } [Test] - [TestCase(500, false)] - [TestCase(500, true)] - public async Task BulkAddLogs(int amount, bool parallel) + [TestCase(500)] + public async Task BulkAddLogs(int amount) { var server = StartServer(new ServerContentIntegrationOption { @@ -162,33 +163,16 @@ public class AddTests : ContentIntegrationTest var coordinates = GetMainEntityCoordinates(sMaps); var entity = sEntities.SpawnEntity(null, coordinates); - if (parallel) + for (var i = 0; i < amount; i++) { - Parallel.For(0, amount, _ => - { - sAdminLogSystem.Add(LogType.Unknown, $"{entity:Entity} test log."); - }); - } - else - { - for (var i = 0; i < amount; i++) - { - sAdminLogSystem.Add(LogType.Unknown, $"{entity:Entity} test log."); - } + sAdminLogSystem.Add(LogType.Unknown, $"{entity:Entity} test log."); } }); await WaitUntil(server, async () => { - var messages = sAdminLogSystem.CurrentRoundLogs(); - var count = 0; - - await foreach (var _ in messages) - { - count++; - } - - return count >= amount; + var messages = await sAdminLogSystem.CurrentRoundLogs(); + return messages.Count >= amount; }); } @@ -225,15 +209,14 @@ public class AddTests : ContentIntegrationTest await WaitUntil(server, async () => { - var logs = sAdminLogSystem.CurrentRoundLogs(); - - await foreach (var log in logs) + var logs = await sAdminLogSystem.CurrentRoundLogs(); + if (logs.Count == 0) { - Assert.That(log.Players, Does.Contain(playerGuid)); - return true; + return false; } - return false; + Assert.That(logs.First().Players, Does.Contain(playerGuid)); + return true; }); } @@ -273,27 +256,27 @@ public class AddTests : ContentIntegrationTest sGamerTicker.StartRound(true); }); - LogRecord log = null; + SharedAdminLog log = default; await WaitUntil(server, async () => { - var logs = sAdminLogSystem.CurrentRoundLogs(new LogFilter + var logs = await sAdminLogSystem.CurrentRoundLogs(new LogFilter { Search = guid.ToString() }); - await foreach (var found in logs) + if (logs.Count == 0) { - log = found; - return true; + return false; } - return false; + log = logs.First(); + return true; }); var filter = new LogFilter { - Round = log.RoundId, + Round = sGamerTicker.RoundId, Search = log.Message, Types = new HashSet {log.Type}, }; diff --git a/Content.IntegrationTests/Tests/Administration/Logs/FilterTests.cs b/Content.IntegrationTests/Tests/Administration/Logs/FilterTests.cs index b56009565a..7487a2ff15 100644 --- a/Content.IntegrationTests/Tests/Administration/Logs/FilterTests.cs +++ b/Content.IntegrationTests/Tests/Administration/Logs/FilterTests.cs @@ -1,5 +1,4 @@ using System; -using System.Threading; using System.Threading.Tasks; using Content.Server.Administration.Logs; using Content.Shared.Administration.Logs; @@ -84,13 +83,13 @@ public class FilterTests : ContentIntegrationTest var firstFound = false; var secondFound = false; - var both = sAdminLogSystem.CurrentRoundLogs(new LogFilter + var both = await sAdminLogSystem.CurrentRoundLogs(new LogFilter { Search = commonGuidStr, DateOrder = order }); - await foreach (var log in both) + foreach (var log in both) { if (!log.Message.Contains(commonGuidStr)) { diff --git a/Content.IntegrationTests/Tests/Administration/Logs/QueryTests.cs b/Content.IntegrationTests/Tests/Administration/Logs/QueryTests.cs index 390384abc1..522eedd3f8 100644 --- a/Content.IntegrationTests/Tests/Administration/Logs/QueryTests.cs +++ b/Content.IntegrationTests/Tests/Administration/Logs/QueryTests.cs @@ -59,7 +59,7 @@ public class QueryTests : ContentIntegrationTest await WaitUntil(server, async () => { - await foreach (var _ in sAdminLogSystem.All(filter)) + foreach (var _ in await sAdminLogSystem.All(filter)) { return true; } diff --git a/Content.Server/Administration/Logs/AdminLogSystem.Cache.cs b/Content.Server/Administration/Logs/AdminLogSystem.Cache.cs new file mode 100644 index 0000000000..67853790af --- /dev/null +++ b/Content.Server/Administration/Logs/AdminLogSystem.Cache.cs @@ -0,0 +1,151 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using Content.Server.Database; +using Content.Shared.Administration.Logs; +using Content.Shared.Database; +using Prometheus; + +namespace Content.Server.Administration.Logs; + +public partial class AdminLogSystem +{ + private const int MaxRoundsCached = 3; + private const int LogListInitialSize = 30_000; + + private readonly int _logTypes = Enum.GetValues().Length; + + // TODO ADMIN LOGS make this thread safe or remove thread safety from the main partial class + private readonly Dictionary> _roundsLogCache = new(MaxRoundsCached); + + private static readonly Gauge CacheRoundCount = Metrics.CreateGauge( + "admin_logs_cache_round_count", + "How many rounds are in cache."); + + private static readonly Gauge CacheLogCount = Metrics.CreateGauge( + "admin_logs_cache_log_count", + "How many logs are in cache."); + + // TODO ADMIN LOGS cache previous {MaxRoundsCached} rounds on startup + private void CacheNewRound() + { + List list; + var oldestRound = CurrentRoundId - MaxRoundsCached; + + if (_roundsLogCache.Remove(oldestRound, out var oldestList)) + { + list = oldestList; + list.Clear(); + } + else + { + list = new List(LogListInitialSize); + } + + _roundsLogCache.Add(CurrentRoundId, list); + CacheRoundCount.Set(_roundsLogCache.Count); + } + + private void CacheLog(AdminLog log) + { + var players = log.Players.Select(player => player.PlayerUserId).ToArray(); + var record = new SharedAdminLog(log.Id, log.Type, log.Impact, log.Date, log.Message, players); + + CacheLog(record); + } + + private void CacheLog(QueuedLog log) + { + CacheLog(log.Log); + } + + private void CacheLog(SharedAdminLog log) + { + // TODO ADMIN LOGS remove redundant data and don't do a dictionary lookup per log + var cache = _roundsLogCache[CurrentRoundId]; + cache.Add(log); + CacheLogCount.Set(cache.Count); + } + + private void CacheLogs(IEnumerable logs) + { + var cache = _roundsLogCache[CurrentRoundId]; + cache.AddRange(logs); + CacheLogCount.Set(cache.Count); + } + + private bool TryGetCache(int roundId, [NotNullWhen(true)] out List? cache) + { + return _roundsLogCache.TryGetValue(roundId, out cache); + } + + private bool TrySearchCache(LogFilter? filter, [NotNullWhen(true)] out List? results) + { + if (filter?.Round == null || !TryGetCache(filter.Round.Value, out var cache)) + { + results = null; + return false; + } + + // TODO ADMIN LOGS a better heuristic than linq spaghetti + var query = cache.AsEnumerable(); + + query = filter.DateOrder switch + { + DateOrder.Ascending => query, + DateOrder.Descending => query.Reverse(), + _ => throw new ArgumentOutOfRangeException(nameof(filter), + $"Unknown {nameof(DateOrder)} value {filter.DateOrder}") + }; + + if (filter.LogsSent != 0) + { + query = query.Skip(filter.LogsSent); + } + + if (filter.Search != null) + { + query = query.Where(log => log.Message.Contains(filter.Search, StringComparison.OrdinalIgnoreCase)); + } + + if (filter.Types != null && filter.Types.Count != _logTypes) + { + query = query.Where(log => filter.Types.Contains(log.Type)); + } + + if (filter.Impacts != null) + { + query = query.Where(log => filter.Impacts.Contains(log.Impact)); + } + + if (filter.Before != null) + { + query = query.Where(log => log.Date < filter.Before); + } + + if (filter.After != null) + { + query = query.Where(log => log.Date > filter.After); + } + + if (filter.AnyPlayers != null) + { + query = query.Where(log => filter.AnyPlayers.Any(filterPlayer => log.Players.Contains(filterPlayer))); + } + + if (filter.AllPlayers != null) + { + query = query.Where(log => filter.AllPlayers.All(filterPlayer => log.Players.Contains(filterPlayer))); + } + + if (filter.Limit != null) + { + query = query.Take(filter.Limit.Value); + } + + // TODO ADMIN LOGS array pool + results = query.ToList(); + return true; + } +} diff --git a/Content.Server/Administration/Logs/AdminLogSystem.cs b/Content.Server/Administration/Logs/AdminLogSystem.cs index 2fed0d4b18..d437e4ac54 100644 --- a/Content.Server/Administration/Logs/AdminLogSystem.cs +++ b/Content.Server/Administration/Logs/AdminLogSystem.cs @@ -2,6 +2,7 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Text.Json; +using System.Threading; using System.Threading.Tasks; using Content.Server.Database; using Content.Server.GameTicking; @@ -75,7 +76,10 @@ public partial class AdminLogSystem : SharedAdminLogSystem private readonly ConcurrentQueue _logQueue = new(); private readonly ConcurrentQueue _preRoundLogQueue = new(); + // Per round private int CurrentRoundId => _gameTicker.RoundId; + private int _currentLogId; + private int NextLogId => Interlocked.Increment(ref _currentLogId); public override void Initialize() { @@ -195,6 +199,7 @@ public partial class AdminLogSystem : SharedAdminLogSystem foreach (var queued in _preRoundLogQueue) { queued.Log.RoundId = _gameTicker.RoundId; + CacheLog(queued); } copy.AddRange(_preRoundLogQueue); @@ -227,6 +232,9 @@ public partial class AdminLogSystem : SharedAdminLogSystem private void RoundStarting(RoundStartingEvent ev) { + Interlocked.Exchange(ref _currentLogId, 0); + CacheNewRound(); + if (_metricsEnabled) { PreRoundQueueCapReached.Set(0); @@ -237,12 +245,16 @@ public partial class AdminLogSystem : SharedAdminLogSystem private async void Add(LogType type, LogImpact impact, string message, JsonDocument json, List players, List<(int id, string? name)> entities) { + var logId = NextLogId; + var date = DateTime.UtcNow; + var log = new AdminLog { + Id = logId, RoundId = CurrentRoundId, Type = type, Impact = impact, - Date = DateTime.UtcNow, + Date = date, Message = message, Json = json, Players = new List(players.Count) @@ -254,6 +266,7 @@ public partial class AdminLogSystem : SharedAdminLogSystem { var player = new AdminLogPlayer { + LogId = logId, PlayerUserId = id }; @@ -267,6 +280,7 @@ public partial class AdminLogSystem : SharedAdminLogSystem else { _logQueue.Enqueue(queued); + CacheLog(log); } } @@ -289,9 +303,22 @@ public partial class AdminLogSystem : SharedAdminLogSystem Add(type, LogImpact.Medium, ref handler); } - public IAsyncEnumerable All(LogFilter? filter = null) + public async Task> All(LogFilter? filter = null) { - return _db.GetAdminLogs(filter); + if (TrySearchCache(filter, out var results)) + { + return results; + } + + var initialSize = Math.Min(filter?.Limit ?? 0, 1000); + var list = new List(initialSize); + + await foreach (var log in _db.GetAdminLogs(filter).WithCancellation(filter?.CancellationToken ?? default)) + { + list.Add(log); + } + + return list; } public IAsyncEnumerable AllMessages(LogFilter? filter = null) @@ -309,7 +336,7 @@ public partial class AdminLogSystem : SharedAdminLogSystem return _db.GetRound(roundId); } - public IAsyncEnumerable CurrentRoundLogs(LogFilter? filter = null) + public Task> CurrentRoundLogs(LogFilter? filter = null) { filter ??= new LogFilter(); filter.Round = CurrentRoundId; diff --git a/Content.Server/Administration/Logs/AdminLogsEui.cs b/Content.Server/Administration/Logs/AdminLogsEui.cs index 72b169b158..97260873fe 100644 --- a/Content.Server/Administration/Logs/AdminLogsEui.cs +++ b/Content.Server/Administration/Logs/AdminLogsEui.cs @@ -147,21 +147,17 @@ public sealed class AdminLogsEui : BaseEui stopwatch.Start(); // TODO ADMIN LOGS array pool - var logs = new List(_clientBatchSize); + List logs = default!; await Task.Run(async () => { - var results = await Task.Run(() => _logSystem.All(_filter)); - - await foreach (var record in results.WithCancellation(_logSendCancellation.Token)) - { - var log = new SharedAdminLog(record.Id, record.Type, record.Impact, record.Date, record.Message, record.Players); - logs.Add(log); - } + logs = await _logSystem.All(_filter); }, _filter.CancellationToken); if (logs.Count > 0) { + _filter.LogsSent += logs.Count; + var largestId = _filter.DateOrder switch { DateOrder.Ascending => ^1, @@ -172,7 +168,7 @@ public sealed class AdminLogsEui : BaseEui _filter.LastLogId = logs[largestId].Id; } - var message = new NewLogs(logs.ToArray(), replace); + var message = new NewLogs(logs, replace); SendMessage(message); diff --git a/Content.Server/Administration/Logs/LogFilter.cs b/Content.Server/Administration/Logs/LogFilter.cs index b917ff36b1..9dc609e917 100644 --- a/Content.Server/Administration/Logs/LogFilter.cs +++ b/Content.Server/Administration/Logs/LogFilter.cs @@ -28,6 +28,8 @@ public sealed class LogFilter public int? LastLogId { get; set; } + public int LogsSent { get; set; } + public int? Limit { get; set; } public DateOrder DateOrder { get; set; } = DateOrder.Descending; diff --git a/Content.Server/Administration/Logs/LogRecord.cs b/Content.Server/Administration/Logs/LogRecord.cs deleted file mode 100644 index 6f32939350..0000000000 --- a/Content.Server/Administration/Logs/LogRecord.cs +++ /dev/null @@ -1,36 +0,0 @@ -using System; -using System.Collections.Generic; -using Content.Server.Database; -using Content.Shared.Administration.Logs; -using Content.Shared.Database; - -namespace Content.Server.Administration.Logs; - -public class LogRecord -{ - public LogRecord( - int id, - int roundId, - LogType type, - LogImpact impact, - DateTime date, - string message, - Guid[] players) - { - Id = id; - RoundId = roundId; - Type = type; - Impact = impact; - Date = date; - Message = message; - Players = players; - } - - public int Id { get; } - public int RoundId { get; } - public LogType Type { get; } - public LogImpact Impact { get; } - public DateTime Date { get; } - public string Message { get; } - public Guid[] Players { get; } -} diff --git a/Content.Server/Database/ServerDbBase.cs b/Content.Server/Database/ServerDbBase.cs index 9b219ba4d8..a86d3c3a4e 100644 --- a/Content.Server/Database/ServerDbBase.cs +++ b/Content.Server/Database/ServerDbBase.cs @@ -640,7 +640,7 @@ namespace Content.Server.Database } } - public async IAsyncEnumerable GetAdminLogs(LogFilter? filter = null) + public async IAsyncEnumerable GetAdminLogs(LogFilter? filter = null) { await using var db = await GetDb(); var query = await GetAdminLogsQuery(db.DbContext, filter); @@ -654,7 +654,7 @@ namespace Content.Server.Database players[i] = log.Players[i].PlayerUserId; } - yield return new LogRecord(log.Id, log.RoundId, log.Type, log.Impact, log.Date, log.Message, players); + yield return new SharedAdminLog(log.Id, log.Type, log.Impact, log.Date, log.Message, players); } } diff --git a/Content.Server/Database/ServerDbManager.cs b/Content.Server/Database/ServerDbManager.cs index 701eaa3f58..a9a978edd9 100644 --- a/Content.Server/Database/ServerDbManager.cs +++ b/Content.Server/Database/ServerDbManager.cs @@ -7,6 +7,7 @@ using System.Text.Json; using System.Threading; using System.Threading.Tasks; using Content.Server.Administration.Logs; +using Content.Shared.Administration.Logs; using Content.Shared.CCVar; using Content.Shared.Preferences; using Microsoft.Data.Sqlite; @@ -19,7 +20,6 @@ using Robust.Shared.IoC; using Robust.Shared.Log; using Robust.Shared.Maths; using Robust.Shared.Network; -using Logger = Robust.Shared.Log.Logger; using LogLevel = Robust.Shared.Log.LogLevel; using MSLogLevel = Microsoft.Extensions.Logging.LogLevel; @@ -135,7 +135,7 @@ namespace Content.Server.Database Task AddAdminLogs(List logs); IAsyncEnumerable GetAdminLogMessages(LogFilter? filter = null); - IAsyncEnumerable GetAdminLogs(LogFilter? filter = null); + IAsyncEnumerable GetAdminLogs(LogFilter? filter = null); IAsyncEnumerable GetAdminLogsJson(LogFilter? filter = null); #endregion @@ -346,7 +346,7 @@ namespace Content.Server.Database return _db.GetAdminLogMessages(filter); } - public IAsyncEnumerable GetAdminLogs(LogFilter? filter = null) + public IAsyncEnumerable GetAdminLogs(LogFilter? filter = null) { return _db.GetAdminLogs(filter); } diff --git a/Content.Server/GameTicking/Events/RoundStartingEvent.cs b/Content.Server/GameTicking/Events/RoundStartingEvent.cs index da7395e7b6..4edf1371a3 100644 --- a/Content.Server/GameTicking/Events/RoundStartingEvent.cs +++ b/Content.Server/GameTicking/Events/RoundStartingEvent.cs @@ -3,7 +3,7 @@ namespace Content.Server.GameTicking.Events; /// -/// Raised at the start of +/// Raised at the start of , after round id has been incremented /// public class RoundStartingEvent : EntityEventArgs { diff --git a/Content.Shared/Administration/AdminLogsEuiState.cs b/Content.Shared/Administration/AdminLogsEuiState.cs index 551a5ecbb0..3921a3bbb5 100644 --- a/Content.Shared/Administration/AdminLogsEuiState.cs +++ b/Content.Shared/Administration/AdminLogsEuiState.cs @@ -33,13 +33,13 @@ public static class AdminLogsEuiMsg [Serializable, NetSerializable] public sealed class NewLogs : EuiMessageBase { - public NewLogs(SharedAdminLog[] logs, bool replace) + public NewLogs(List logs, bool replace) { Logs = logs; Replace = replace; } - public SharedAdminLog[] Logs { get; set; } + public List Logs { get; set; } public bool Replace { get; set; } }