Fix server update restarts when server paused. (#8509)

Content.Server/Entry/EntryPoint.cs

@@ -18,6 +18,7 @@ using Content.Server.LandMines;
 using Content.Server.Maps;
 using Content.Server.NodeContainer.NodeGroups;
 using Content.Server.Preferences.Managers;
+using Content.Server.ServerUpdates;
 using Content.Server.Voting.Managers;
 using Content.Shared.Administration;
 using Content.Shared.CCVar;
@@ -38,6 +39,7 @@ namespace Content.Server.Entry
     {
         private EuiManager _euiManager = default!;
         private IVoteManager _voteManager = default!;
+        private ServerUpdateManager _updateManager = default!;
 
         /// <inheritdoc />
         public override void Init()
@@ -75,6 +77,7 @@ namespace Content.Server.Entry
             {
                 _euiManager = IoCManager.Resolve<EuiManager>();
                 _voteManager = IoCManager.Resolve<IVoteManager>();
+                _updateManager = IoCManager.Resolve<ServerUpdateManager>();
 
                 var playerManager = IoCManager.Resolve<IPlayerManager>();
 
@@ -92,6 +95,7 @@ namespace Content.Server.Entry
                 IoCManager.Resolve<GhostKickManager>().Initialize();
 
                 _voteManager.Initialize();
+                _updateManager.Initialize();
             }
         }
 
@@ -145,6 +149,10 @@ namespace Content.Server.Entry
                     _voteManager.Update();
                     break;
                 }
+
+                case ModUpdateLevel.FramePostEngine:
+                    _updateManager.Update();
+                    break;
             }
         }
     }

Content.Server/GameTicking/GameTicker.Player.cs

@@ -26,11 +26,6 @@ namespace Content.Server.GameTicking
 
             switch (args.NewStatus)
             {
-                case SessionStatus.Connecting:
-                    // Cancel shutdown update timer in progress.
-                    _updateShutdownCts?.Cancel();
-                    break;
-
                 case SessionStatus.Connected:
                 {
                     AddPlayerToDb(args.Session.UserId.UserId);
@@ -95,7 +90,6 @@ namespace Content.Server.GameTicking
 
                     _chatManager.SendAdminAnnouncement(Loc.GetString("player-leave-message", ("name", args.Session.Name)));
 
-                    ServerEmptyUpdateRestartCheck();
                     _prefsManager.OnClientDisconnected(session);
                     break;
                 }

Content.Server/GameTicking/GameTicker.RoundFlow.cs

@@ -335,11 +335,9 @@ namespace Content.Server.GameTicking
             if (DummyTicker)
                 return;
 
-            if (_updateOnRoundEnd)
+            // Handle restart for server update
-            {
+            if (_serverUpdates.RoundEnded())
-                _baseServer.Shutdown(Loc.GetString("game-ticker-shutdown-server-update"));
                 return;
-            }
 
             _sawmill.Info("Restarting round!");
 

Content.Server/GameTicking/GameTicker.Updates.cs

@@ -1,58 +0,0 @@
-using System.Linq;
-using System.Threading;
-using Robust.Shared.Enums;
-using Timer = Robust.Shared.Timing.Timer;
-
-namespace Content.Server.GameTicking
-{
-    public sealed partial class GameTicker
-    {
-        private static readonly TimeSpan UpdateRestartDelay = TimeSpan.FromSeconds(20);
-
-        [ViewVariables]
-        private bool _updateOnRoundEnd;
-        private CancellationTokenSource? _updateShutdownCts;
-
-        private void InitializeUpdates()
-        {
-            _watchdogApi.UpdateReceived += WatchdogApiOnUpdateReceived;
-        }
-
-        private void WatchdogApiOnUpdateReceived()
-        {
-            _chatManager.DispatchServerAnnouncement(Loc.GetString("game-ticker-restart-round-server-update"));
-            _updateOnRoundEnd = true;
-            ServerEmptyUpdateRestartCheck();
-        }
-
-        /// <summary>
-        ///     Checks whether there are still players on the server,
-        /// and if not starts a timer to automatically reboot the server if an update is available.
-        /// </summary>
-        private void ServerEmptyUpdateRestartCheck()
-        {
-            // Can't simple check the current connected player count since that doesn't update
-            // before PlayerStatusChanged gets fired.
-            // So in the disconnect handler we'd still see a single player otherwise.
-            var playersOnline = _playerManager.Sessions.Any(p => p.Status != SessionStatus.Disconnected);
-            if (playersOnline || !_updateOnRoundEnd)
-            {
-                // Still somebody online.
-                return;
-            }
-
-            if (_updateShutdownCts is {IsCancellationRequested: false})
-            {
-                // Do nothing because I guess we already have a timer running..?
-                return;
-            }
-
-            _updateShutdownCts = new CancellationTokenSource();
-
-            Timer.Spawn(UpdateRestartDelay, () =>
-            {
-                _baseServer.Shutdown(Loc.GetString("game-ticker-shutdown-server-update"));
-            }, _updateShutdownCts.Token);
-        }
-    }
-}

Content.Server/GameTicking/GameTicker.cs

@@ -5,6 +5,7 @@ using Content.Server.Database;
 using Content.Server.Ghost;
 using Content.Server.Maps;
 using Content.Server.Preferences.Managers;
+using Content.Server.ServerUpdates;
 using Content.Server.Station.Systems;
 using Content.Shared.Chat;
 using Content.Shared.Damage;
@@ -12,7 +13,6 @@ using Content.Shared.GameTicking;
 using Content.Shared.Roles;
 using Robust.Server;
 using Robust.Server.Maps;
-using Robust.Server.ServerStatus;
 using Robust.Shared.Configuration;
 using Robust.Shared.Console;
 #if EXCEPTION_TOLERANCE
@@ -55,7 +55,6 @@ namespace Content.Server.GameTicking
             DebugTools.Assert(_prototypeManager.Index<JobPrototype>(FallbackOverflowJob).Name == Loc.GetString(FallbackOverflowJobName),
                 "Overflow role does not have the correct name!");
             InitializeGameRules();
-            InitializeUpdates();
 
             _initialized = true;
         }
@@ -101,7 +100,6 @@ namespace Content.Server.GameTicking
         [Dependency] private readonly IRobustRandom _robustRandom = default!;
         [Dependency] private readonly IServerPreferencesManager _prefsManager = default!;
         [Dependency] private readonly IBaseServer _baseServer = default!;
-        [Dependency] private readonly IWatchdogApi _watchdogApi = default!;
         [Dependency] private readonly IGameMapManager _gameMapManager = default!;
         [Dependency] private readonly IServerDbManager _db = default!;
         [Dependency] private readonly IAdminLogManager _adminLogger = default!;
@@ -116,5 +114,6 @@ namespace Content.Server.GameTicking
         [Dependency] private readonly DamageableSystem _damageable = default!;
         [Dependency] private readonly GhostSystem _ghosts = default!;
         [Dependency] private readonly RoleBanManager _roleBanManager = default!;
+        [Dependency] private readonly ServerUpdateManager _serverUpdates = default!;
     }
 }

Content.Server/IoC/ServerContentIoC.cs

@@ -19,6 +19,7 @@ using Content.Server.NodeContainer.NodeGroups;
 using Content.Server.Objectives;
 using Content.Server.Objectives.Interfaces;
 using Content.Server.Preferences.Managers;
+using Content.Server.ServerUpdates;
 using Content.Server.Voting.Managers;
 using Content.Shared.Administration;
 using Content.Shared.Administration.Logs;
@@ -42,6 +43,7 @@ namespace Content.Server.IoC
             IoCManager.Register<BlackboardManager, BlackboardManager>();
             IoCManager.Register<ConsiderationsManager, ConsiderationsManager>();
             IoCManager.Register<IConnectionManager, ConnectionManager>();
+            IoCManager.Register<ServerUpdateManager>();
             IoCManager.Register<IObjectivesManager, ObjectivesManager>();
             IoCManager.Register<IAdminManager, AdminManager>();
             IoCManager.Register<EuiManager, EuiManager>();

Content.Server/ServerUpdates/ServerUpdateManager.cs

@@ -0,0 +1,109 @@
+using System.Linq;
+using Content.Server.Chat.Managers;
+using Content.Shared.CCVar;
+using Robust.Server;
+using Robust.Server.Player;
+using Robust.Server.ServerStatus;
+using Robust.Shared.Configuration;
+using Robust.Shared.Enums;
+using Robust.Shared.Timing;
+
+namespace Content.Server.ServerUpdates;
+
+/// <summary>
+/// Responsible for restarting the server for update, when not disruptive.
+/// </summary>
+public sealed class ServerUpdateManager
+{
+    [Dependency] private readonly IGameTiming _gameTiming = default!;
+    [Dependency] private readonly IWatchdogApi _watchdog = default!;
+    [Dependency] private readonly IPlayerManager _playerManager = default!;
+    [Dependency] private readonly IChatManager _chatManager = default!;
+    [Dependency] private readonly IBaseServer _server = default!;
+    [Dependency] private readonly IConfigurationManager _cfg = default!;
+
+    [ViewVariables]
+    private bool _updateOnRoundEnd;
+
+    private TimeSpan? _restartTime;
+
+    public void Initialize()
+    {
+        _watchdog.UpdateReceived += WatchdogOnUpdateReceived;
+        _playerManager.PlayerStatusChanged += PlayerManagerOnPlayerStatusChanged;
+    }
+
+    public void Update()
+    {
+        if (_restartTime != null && _restartTime < _gameTiming.RealTime)
+        {
+            DoShutdown();
+        }
+    }
+
+    /// <summary>
+    /// Notify that the round just ended, which is a great time to restart if necessary!
+    /// </summary>
+    /// <returns>True if the server is going to restart.</returns>
+    public bool RoundEnded()
+    {
+        if (_updateOnRoundEnd)
+        {
+            DoShutdown();
+            return true;
+        }
+
+        return false;
+    }
+
+    private void PlayerManagerOnPlayerStatusChanged(object? sender, SessionStatusEventArgs e)
+    {
+        switch (e.NewStatus)
+        {
+            case SessionStatus.Connecting:
+                _restartTime = null;
+                break;
+            case SessionStatus.Disconnected:
+                ServerEmptyUpdateRestartCheck();
+                break;
+        }
+    }
+
+    private void WatchdogOnUpdateReceived()
+    {
+        _chatManager.DispatchServerAnnouncement(Loc.GetString("server-updates-received"));
+        _updateOnRoundEnd = true;
+        ServerEmptyUpdateRestartCheck();
+    }
+
+    /// <summary>
+    ///     Checks whether there are still players on the server,
+    /// and if not starts a timer to automatically reboot the server if an update is available.
+    /// </summary>
+    private void ServerEmptyUpdateRestartCheck()
+    {
+        // Can't simple check the current connected player count since that doesn't update
+        // before PlayerStatusChanged gets fired.
+        // So in the disconnect handler we'd still see a single player otherwise.
+        var playersOnline = _playerManager.Sessions.Any(p => p.Status != SessionStatus.Disconnected);
+        if (playersOnline || !_updateOnRoundEnd)
+        {
+            // Still somebody online.
+            return;
+        }
+
+        if (_restartTime != null)
+        {
+            // Do nothing because I guess we already have a timer running..?
+            return;
+        }
+
+        var restartDelay = TimeSpan.FromSeconds(_cfg.GetCVar(CCVars.UpdateRestartDelay));
+        _restartTime = restartDelay + _gameTiming.RealTime;
+    }
+
+    private void DoShutdown()
+    {
+        _server.Shutdown(Loc.GetString("server-updates-shutdown"));
+    }
+}

Content.Shared/CCVar/CCVars.cs

@@ -961,5 +961,15 @@ namespace Content.Shared.CCVar
         /// </summary>
         public static readonly CVarDef<float> DragDropDeadZone =
             CVarDef.Create("control.drag_dead_zone", 12f, CVar.CLIENTONLY | CVar.ARCHIVE);
+
+        /*
+         * UPDATE
+         */
+
+        /// <summary>
+        /// If a server update restart is pending, the delay after the last player leaves before we actually restart. In seconds.
+        /// </summary>
+        public static readonly CVarDef<float> UpdateRestartDelay =
+            CVarDef.Create("update.restart_delay", 20f, CVar.SERVERONLY);
     }
 }

Resources/Locale/en-US/game-ticking/game-ticker.ftl

@@ -1,5 +1,3 @@
-game-ticker-restart-round-server-update = Update has been received, server will automatically restart for update at the end of this round.
-game-ticker-shutdown-server-update = Server is shutting down for update and will automatically restart.
 game-ticker-restart-round = Restarting round...
 game-ticker-start-round = The round is starting now...
 game-ticker-start-round-cannot-start-game-mode-fallback = Failed to start {$failedGameMode} mode! Defaulting to {$fallbackMode}...

Resources/Locale/en-US/server-updates/server-updates.ftl

@@ -0,0 +1,2 @@
+server-updates-received = Update has been received, server will automatically restart for update at the end of this round.
+server-updates-shutdown = Server is shutting down for update and will automatically restart.