Kick on ban for entire server group (#28649)

* Start work on PostgresNotificationManager
Implement initial version of init and listening code

* Finish implementing PostgresNotificationManager
Implement ban insert trigger

* Implement ignoring notifications if the ban was from the same server

* Address reviews

* Fixes and refactorings

Fix typo in migration SQL

Pull new code in BanManager out into its own partial file.

Unify logic to kick somebody with that when a new ban is placed directly on the server.

New bans are now checked against all parameters (IP, HWID) instead of just user ID.

Extracted SQLite ban matching code into a new class so that it can mostly be re-used by the ban notification code. No copy-paste here.

Database notifications are now not implicitly sent to the main thread, this means basic checks will happen in the thread pool beforehand.

Bans without user ID are now sent to servers. Bans are rate limited to avoid undue work from mass ban imports, beyond the rate limit they are dropped.

Improved error handling and logging for the whole system.

Matching bans against connected players requires knowing their ban exemption flags. These are now cached when the player connects.

ServerBanDef now has exemption flags, again to allow matching full ban details for ban notifications.

Made database notifications a proper struct type to reduce copy pasting a tuple.

Remove copy pasted connection string building code by just... passing the string into the constructor.

Add lock around _notificationHandlers just in case.

Fixed postgres connection wait not being called in a loop and therefore spamming LISTEN commands for every received notification.

Added more error handling and logging to notification listener.

Removed some copy pasting from SQLite database layer too while I was at it because god forbid we expect anybody else to do all the work in this project.

Sorry Julian

---------

Co-authored-by: Pieter-Jan Briers <pieterjan.briers+git@gmail.com>

Content.Server/Database/ServerDbPostgres.Notifications.cs

@@ -0,0 +1,121 @@
+using System.Data;
+using System.Threading;
+using System.Threading.Tasks;
+using Content.Server.Administration.Managers;
+using Npgsql;
+
+namespace Content.Server.Database;
+
+/// Listens for ban_notification containing the player id and the banning server id using postgres listen/notify.
+/// Players a ban_notification got received for get banned, except when the current server id and the one in the notification payload match.
+
+public sealed partial class ServerDbPostgres
+{
+    /// <summary>
+    /// The list of notify channels to subscribe to.
+    /// </summary>
+    private static readonly string[] NotificationChannels =
+    [
+        BanManager.BanNotificationChannel,
+    ];
+
+    private static readonly TimeSpan ReconnectWaitIncrease = TimeSpan.FromSeconds(10);
+
+    private readonly CancellationTokenSource _notificationTokenSource = new();
+
+    private NpgsqlConnection? _notificationConnection;
+    private TimeSpan _reconnectWaitTime = TimeSpan.Zero;
+
+    /// <summary>
+    /// Sets up the database connection and the notification handler
+    /// </summary>
+    private void InitNotificationListener(string connectionString)
+    {
+        _notificationConnection = new NpgsqlConnection(connectionString);
+        _notificationConnection.Notification += OnNotification;
+
+        var cancellationToken = _notificationTokenSource.Token;
+        Task.Run(() => NotificationListener(cancellationToken), cancellationToken);
+    }
+
+    /// <summary>
+    /// Listens to the notification channel with basic error handling and reopens the connection if it got closed
+    /// </summary>
+    private async Task NotificationListener(CancellationToken cancellationToken)
+    {
+        if (_notificationConnection == null)
+            return;
+
+        _notifyLog.Verbose("Starting notification listener");
+
+        while (!cancellationToken.IsCancellationRequested)
+        {
+            try
+            {
+                if (_notificationConnection.State == ConnectionState.Broken)
+                {
+                    _notifyLog.Debug("Notification listener entered broken state, closing...");
+                    await _notificationConnection.CloseAsync();
+                }
+
+                if (_notificationConnection.State == ConnectionState.Closed)
+                {
+                    _notifyLog.Debug("Opening notification listener connection...");
+                    if (_reconnectWaitTime != TimeSpan.Zero)
+                    {
+                        _notifyLog.Verbose($"_reconnectWaitTime is {_reconnectWaitTime}");
+                        await Task.Delay(_reconnectWaitTime, cancellationToken);
+                    }
+
+                    await _notificationConnection.OpenAsync(cancellationToken);
+                    _reconnectWaitTime = TimeSpan.Zero;
+                    _notifyLog.Verbose($"Notification connection opened...");
+                }
+
+                foreach (var channel in NotificationChannels)
+                {
+                    _notifyLog.Verbose($"Listening on channel {channel}");
+                    await using var cmd = new NpgsqlCommand($"LISTEN {channel}", _notificationConnection);
+                    await cmd.ExecuteNonQueryAsync(cancellationToken);
+                }
+
+                while (!cancellationToken.IsCancellationRequested)
+                {
+                    _notifyLog.Verbose("Waiting on notifications...");
+                    await _notificationConnection.WaitAsync(cancellationToken);
+                }
+            }
+            catch (OperationCanceledException)
+            {
+                // Abort loop on cancel.
+                _notifyLog.Verbose($"Shutting down notification listener due to cancellation");
+                return;
+            }
+            catch (Exception e)
+            {
+                _reconnectWaitTime += ReconnectWaitIncrease;
+                _notifyLog.Error($"Error in notification listener: {e}");
+            }
+        }
+    }
+
+    private void OnNotification(object _, NpgsqlNotificationEventArgs notification)
+    {
+        _notifyLog.Verbose($"Received notification on channel {notification.Channel}");
+        NotificationReceived(new DatabaseNotification
+        {
+            Channel = notification.Channel,
+            Payload = notification.Payload,
+        });
+    }
+
+    public override void Shutdown()
+    {
+        _notificationTokenSource.Cancel();
+        if (_notificationConnection == null)
+            return;
+
+        _notificationConnection.Notification -= OnNotification;
+        _notificationConnection.Dispose();
+    }
+}