Load Maps on Round Start, not Round Restart v3 (#6989)

* Load Maps on Round Start, not Round Restart

* Fix admin log test.
It assumed maps/grids existed during pre-round, wihch is not a valid assumption anymore after this PR.

* Shutdown server if round fails to start 5 times.

* Fix bugs with round starting flag.

* Make StartRound not async, synchronously get new round ID from DB.

* Handle StationId.Invalid in PickBestAvailableJob
Instead of crashing, return null.
SpawnPlayer will handle this by making the player an observer or returning them to the lobby.

Content.IntegrationTests/Tests/Administration/Logs/AddTests.cs

@@ -234,8 +234,6 @@ public sealed class AddTests : ContentIntegrationTest
         await server.WaitIdleAsync();
 
         var sDatabase = server.ResolveDependency<IServerDbManager>();
-        var sEntities = server.ResolveDependency<IEntityManager>();
-        var sMaps = server.ResolveDependency<IMapManager>();
         var sSystems = server.ResolveDependency<IEntitySystemManager>();
 
         var sAdminLogSystem = sSystems.GetEntitySystem<AdminLogSystem>();
@@ -245,10 +243,7 @@ public sealed class AddTests : ContentIntegrationTest
 
         await server.WaitPost(() =>
         {
-            var coordinates = GetMainEntityCoordinates(sMaps);
+            sAdminLogSystem.Add(LogType.Unknown, $"test log: {guid}");
-            var entity = sEntities.SpawnEntity(null, coordinates);
-
-            sAdminLogSystem.Add(LogType.Unknown, $"{entity} test log: {guid}");
         });
 
         await server.WaitPost(() =>
@@ -285,7 +280,6 @@ public sealed class AddTests : ContentIntegrationTest
         {
             var root = json.RootElement;
             
-            Assert.That(root.TryGetProperty("entity", out _), Is.True);
             Assert.That(root.TryGetProperty("guid", out _), Is.True);
 
             json.Dispose();

Content.Server/GameTicking/GameTicker.CVars.cs

@@ -27,6 +27,11 @@ namespace Content.Server.GameTicking
         [ViewVariables]
         public float MaxStationOffset { get; private set; } = 0f;
 
+#if EXCEPTION_TOLERANCE
+        [ViewVariables]
+        public int RoundStartFailShutdownCount { get; private set; } = 0;
+#endif
+
         private void InitializeCVars()
         {
             _configurationManager.OnValueChanged(CCVars.GameLobbyEnabled, value => LobbyEnabled = value, true);
@@ -37,6 +42,9 @@ namespace Content.Server.GameTicking
             _configurationManager.OnValueChanged(CCVars.StationOffset, value => StationOffset = value, true);
             _configurationManager.OnValueChanged(CCVars.StationRotation, value => StationRotation = value, true);
             _configurationManager.OnValueChanged(CCVars.MaxStationOffset, value => MaxStationOffset = value, true);
+#if EXCEPTION_TOLERANCE
+            _configurationManager.OnValueChanged(CCVars.RoundStartFailShutdownCount, value => RoundStartFailShutdownCount = value, true);
+#endif
         }
     }
 }

Content.Server/GameTicking/GameTicker.JobController.cs

@@ -108,6 +108,9 @@ namespace Content.Server.GameTicking
         private string? PickBestAvailableJob(IPlayerSession playerSession, HumanoidCharacterProfile profile,
             StationId station)
         {
+            if (station == StationId.Invalid)
+                return null;
+
             var available = _stationSystem.StationInfo[station].JobList;
 
             bool TryPick(JobPriority priority, [NotNullWhen(true)] out string? jobId)

Content.Server/GameTicking/GameTicker.RoundFlow.cs

@@ -1,31 +1,21 @@
-using System;
-using System.Collections.Generic;
 using System.Linq;
-using Content.Server.Database;
 using Content.Server.GameTicking.Events;
 using Content.Server.Ghost;
 using Content.Server.Maps;
 using Content.Server.Mind;
 using Content.Server.Players;
 using Content.Server.Station;
-using Content.Shared.CCVar;
 using Content.Shared.Coordinates;
 using Content.Shared.GameTicking;
 using Content.Shared.Preferences;
 using Content.Shared.Station;
 using Prometheus;
 using Robust.Server.Player;
-using Robust.Shared.GameObjects;
-using Robust.Shared.IoC;
-using Robust.Shared.Localization;
-using Robust.Shared.Log;
 using Robust.Shared.Map;
-using Robust.Shared.Maths;
 using Robust.Shared.Network;
 using Robust.Shared.Player;
 using Robust.Shared.Random;
 using Robust.Shared.Utility;
-using Robust.Shared.ViewVariables;
 
 namespace Content.Server.GameTicking
 {
@@ -39,7 +29,10 @@ namespace Content.Server.GameTicking
             "ss14_round_length",
             "Round length in seconds.");
 
-        [Dependency] private readonly IServerDbManager _db = default!;
+#if EXCEPTION_TOLERANCE
+        [ViewVariables]
+        private int _roundStartFailCount = 0;
+#endif
 
         [ViewVariables]
         private TimeSpan _roundStartTimeSpan;
@@ -75,7 +68,6 @@ namespace Content.Server.GameTicking
 
             DefaultMap = _mapManager.CreateMap();
             _mapManager.AddUninitializedMap(DefaultMap);
-            _startingRound = false;
             var startTime = _gameTiming.RealTime;
             var maps = new List<GameMapPrototype>() { _gameMapManager.GetSelectedMapChecked(true) };
 
@@ -173,7 +165,7 @@ namespace Content.Server.GameTicking
             }
         }
 
-        public async void StartRound(bool force = false)
+        public void StartRound(bool force = false)
         {
 #if EXCEPTION_TOLERANCE
             try
@@ -190,12 +182,14 @@ namespace Content.Server.GameTicking
 
             SendServerMessage(Loc.GetString("game-ticker-start-round"));
 
+            LoadMaps();
+
             StartGamePresetRules();
 
             RoundLengthMetric.Set(0);
 
             var playerIds = _playersInLobby.Keys.Select(player => player.UserId.UserId).ToArray();
-                RoundId = await _db.AddNewRound(playerIds);
+            RoundId = _db.AddNewRound(playerIds).Result;
 
             var startingEvent = new RoundStartingEvent();
             RaiseLocalEvent(startingEvent);
@@ -246,8 +240,6 @@ namespace Content.Server.GameTicking
             _roundStartDateTime = DateTime.UtcNow;
             RunLevel = GameRunLevel.InRound;
 
-                _startingRound = false;
-
             _roundStartTimeSpan = _gameTiming.RealTime;
             SendStatusToAll();
             ReqWindowAttentionAll();
@@ -256,14 +248,30 @@ namespace Content.Server.GameTicking
 
 #if EXCEPTION_TOLERANCE
             }
-            catch(Exception e)
+            catch (Exception e)
             {
+                _roundStartFailCount++;
 
-                Logger.WarningS("ticker", $"Exception caught while trying to start the round! Restarting...");
+                if (RoundStartFailShutdownCount > 0 && _roundStartFailCount >= RoundStartFailShutdownCount)
+                {
+                    Logger.FatalS("ticker",
+                        $"Failed to start a round {_roundStartFailCount} time(s) in a row... Shutting down!");
                     _runtimeLog.LogException(e, nameof(GameTicker));
-                RestartRound();
+                    _baseServer.Shutdown("Restarting server");
+                    return;
                 }
+
+                Logger.WarningS("ticker", $"Exception caught while trying to start the round! Restarting round...");
+                _runtimeLog.LogException(e, nameof(GameTicker));
+                _startingRound = false;
+                RestartRound();
+                return;
+            }
+
+            // Round started successfully! Reset counter...
+            _roundStartFailCount = 0;
 #endif
+            _startingRound = false;
         }
 
         private void RefreshLateJoinAllowed()
@@ -373,7 +381,6 @@ namespace Content.Server.GameTicking
             RunLevel = GameRunLevel.PreRoundLobby;
             LobbySong = _robustRandom.Pick(_lobbyMusicCollection.PickFiles).ToString();
             ResettingCleanup();
-            LoadMaps();
 
             if (!LobbyEnabled)
             {
@@ -411,18 +418,16 @@ namespace Content.Server.GameTicking
                 unCastData.ContentData()?.WipeMind();
             }
 
-            // Delete all entities.
+            _mapManager.Restart();
-            foreach (var entity in EntityManager.GetEntities().ToList())
+
+            // Delete all remaining entities.
+            foreach (var entity in EntityManager.GetEntities().ToArray())
             {
                 // TODO: Maybe something less naive here?
                 // FIXME: Actually, definitely.
                 EntityManager.DeleteEntity(entity);
             }
 
-            _startingRound = false;
-
-            _mapManager.Restart();
-
             _roleBanManager.Restart();
 
             // Clear up any game rules.
@@ -465,8 +470,7 @@ namespace Content.Server.GameTicking
                 RoundLengthMetric.Inc(frameTime);
             }
 
-            if (RunLevel != GameRunLevel.PreRoundLobby ||
+            if (RunLevel != GameRunLevel.PreRoundLobby || Paused ||
-                Paused ||
                 _roundStartTime > _gameTiming.CurTime ||
                 _roundStartCountdownHasNotStartedYetDueToNoPlayers)
             {

Content.Server/GameTicking/GameTicker.cs

@@ -2,6 +2,7 @@ using Content.Server.Administration.Logs;
 using Content.Server.Administration.Managers;
 using Content.Server.CharacterAppearance.Systems;
 using Content.Server.Chat.Managers;
+using Content.Server.Database;
 using Content.Server.Ghost;
 using Content.Server.Maps;
 using Content.Server.PDA;
@@ -89,6 +90,7 @@ namespace Content.Server.GameTicking
         [Dependency] private readonly IBaseServer _baseServer = default!;
         [Dependency] private readonly IWatchdogApi _watchdogApi = default!;
         [Dependency] private readonly IGameMapManager _gameMapManager = default!;
+        [Dependency] private readonly IServerDbManager _db = default!;
 #if EXCEPTION_TOLERANCE
         [Dependency] private readonly IRuntimeLog _runtimeLog = default!;
 #endif

Content.Shared/CCVar/CCVars.cs

@@ -60,6 +60,9 @@ namespace Content.Shared.CCVar
         public static readonly CVarDef<bool>
             EventsEnabled = CVarDef.Create("events.enabled", true, CVar.ARCHIVE | CVar.SERVERONLY);
 
+        /// <summary>
+        ///     Disables most functionality in the GameTicker.
+        /// </summary>
         public static readonly CVarDef<bool>
             GameDummyTicker = CVarDef.Create("game.dummyticker", false, CVar.ARCHIVE | CVar.SERVERONLY);
 
@@ -154,6 +157,15 @@ namespace Content.Shared.CCVar
         public static readonly CVarDef<int> SoftMaxPlayers =
             CVarDef.Create("game.soft_max_players", 30, CVar.SERVERONLY | CVar.ARCHIVE);
 
+#if EXCEPTION_TOLERANCE
+        /// <summary>
+        ///     Amount of times round start must fail before the server is shut down.
+        ///     Set to 0 or a negative number to disable.
+        /// </summary>
+        public static readonly CVarDef<int> RoundStartFailShutdownCount =
+            CVarDef.Create("game.round_start_fail_shutdown_count", 5, CVar.SERVERONLY | CVar.SERVER);
+#endif
+
         /*
          * Discord
          */