Load Maps on Round Start, not Round Restart v3 (#6989)

* Load Maps on Round Start, not Round Restart

* Fix admin log test.
It assumed maps/grids existed during pre-round, wihch is not a valid assumption anymore after this PR.

* Shutdown server if round fails to start 5 times.

* Fix bugs with round starting flag.

* Make StartRound not async, synchronously get new round ID from DB.

* Handle StationId.Invalid in PickBestAvailableJob
Instead of crashing, return null.
SpawnPlayer will handle this by making the player an observer or returning them to the lobby.

Content.IntegrationTests/Tests/Administration/Logs/AddTests.cs

@@ -234,8 +234,6 @@ public sealed class AddTests : ContentIntegrationTest
         await server.WaitIdleAsync();
 
         var sDatabase = server.ResolveDependency<IServerDbManager>();
-        var sEntities = server.ResolveDependency<IEntityManager>();
-        var sMaps = server.ResolveDependency<IMapManager>();
         var sSystems = server.ResolveDependency<IEntitySystemManager>();
 
         var sAdminLogSystem = sSystems.GetEntitySystem<AdminLogSystem>();
@@ -245,10 +243,7 @@ public sealed class AddTests : ContentIntegrationTest
 
         await server.WaitPost(() =>
         {
-            var coordinates = GetMainEntityCoordinates(sMaps);
+            sAdminLogSystem.Add(LogType.Unknown, $"test log: {guid}");
-            var entity = sEntities.SpawnEntity(null, coordinates);
-
-            sAdminLogSystem.Add(LogType.Unknown, $"{entity} test log: {guid}");
         });
 
         await server.WaitPost(() =>
@@ -284,8 +279,7 @@ public sealed class AddTests : ContentIntegrationTest
         await foreach (var json in sDatabase.GetAdminLogsJson(filter))
         {
             var root = json.RootElement;
-
+            
-            Assert.That(root.TryGetProperty("entity", out _), Is.True);
             Assert.That(root.TryGetProperty("guid", out _), Is.True);
 
             json.Dispose();

Content.Server/GameTicking/GameTicker.CVars.cs

@@ -27,6 +27,11 @@ namespace Content.Server.GameTicking
         [ViewVariables]
         public float MaxStationOffset { get; private set; } = 0f;
 
+#if EXCEPTION_TOLERANCE
+        [ViewVariables]
+        public int RoundStartFailShutdownCount { get; private set; } = 0;
+#endif
+
         private void InitializeCVars()
         {
             _configurationManager.OnValueChanged(CCVars.GameLobbyEnabled, value => LobbyEnabled = value, true);
@@ -37,6 +42,9 @@ namespace Content.Server.GameTicking
             _configurationManager.OnValueChanged(CCVars.StationOffset, value => StationOffset = value, true);
             _configurationManager.OnValueChanged(CCVars.StationRotation, value => StationRotation = value, true);
             _configurationManager.OnValueChanged(CCVars.MaxStationOffset, value => MaxStationOffset = value, true);
+#if EXCEPTION_TOLERANCE
+            _configurationManager.OnValueChanged(CCVars.RoundStartFailShutdownCount, value => RoundStartFailShutdownCount = value, true);
+#endif
         }
     }
 }

Content.Server/GameTicking/GameTicker.JobController.cs

@@ -108,6 +108,9 @@ namespace Content.Server.GameTicking
         private string? PickBestAvailableJob(IPlayerSession playerSession, HumanoidCharacterProfile profile,
             StationId station)
         {
+            if (station == StationId.Invalid)
+                return null;
+
             var available = _stationSystem.StationInfo[station].JobList;
 
             bool TryPick(JobPriority priority, [NotNullWhen(true)] out string? jobId)

Content.Server/GameTicking/GameTicker.RoundFlow.cs

@@ -1,31 +1,21 @@
-using System;
-using System.Collections.Generic;
 using System.Linq;
-using Content.Server.Database;
 using Content.Server.GameTicking.Events;
 using Content.Server.Ghost;
 using Content.Server.Maps;
 using Content.Server.Mind;
 using Content.Server.Players;
 using Content.Server.Station;
-using Content.Shared.CCVar;
 using Content.Shared.Coordinates;
 using Content.Shared.GameTicking;
 using Content.Shared.Preferences;
 using Content.Shared.Station;
 using Prometheus;
 using Robust.Server.Player;
-using Robust.Shared.GameObjects;
-using Robust.Shared.IoC;
-using Robust.Shared.Localization;
-using Robust.Shared.Log;
 using Robust.Shared.Map;
-using Robust.Shared.Maths;
 using Robust.Shared.Network;
 using Robust.Shared.Player;
 using Robust.Shared.Random;
 using Robust.Shared.Utility;
-using Robust.Shared.ViewVariables;
 
 namespace Content.Server.GameTicking
 {
@@ -39,7 +29,10 @@ namespace Content.Server.GameTicking
             "ss14_round_length",
             "Round length in seconds.");
 
-        [Dependency] private readonly IServerDbManager _db = default!;
+#if EXCEPTION_TOLERANCE
+        [ViewVariables]
+        private int _roundStartFailCount = 0;
+#endif
 
         [ViewVariables]
         private TimeSpan _roundStartTimeSpan;
@@ -75,7 +68,6 @@ namespace Content.Server.GameTicking
 
             DefaultMap = _mapManager.CreateMap();
             _mapManager.AddUninitializedMap(DefaultMap);
-            _startingRound = false;
             var startTime = _gameTiming.RealTime;
             var maps = new List<GameMapPrototype>() { _gameMapManager.GetSelectedMapChecked(true) };
 
@@ -173,97 +165,113 @@ namespace Content.Server.GameTicking
             }
         }
 
-        public async void StartRound(bool force = false)
+        public void StartRound(bool force = false)
         {
 #if EXCEPTION_TOLERANCE
             try
             {
 #endif
-                // If this game ticker is a dummy or the round is already being started, do nothing!
+            // If this game ticker is a dummy or the round is already being started, do nothing!
-                if (DummyTicker || _startingRound)
+            if (DummyTicker || _startingRound)
-                    return;
+                return;
 
-                _startingRound = true;
+            _startingRound = true;
 
-                DebugTools.Assert(RunLevel == GameRunLevel.PreRoundLobby);
+            DebugTools.Assert(RunLevel == GameRunLevel.PreRoundLobby);
-                Logger.InfoS("ticker", "Starting round!");
+            Logger.InfoS("ticker", "Starting round!");
 
-                SendServerMessage(Loc.GetString("game-ticker-start-round"));
+            SendServerMessage(Loc.GetString("game-ticker-start-round"));
 
-                StartGamePresetRules();
+            LoadMaps();
 
-                RoundLengthMetric.Set(0);
+            StartGamePresetRules();
 
-                var playerIds = _playersInLobby.Keys.Select(player => player.UserId.UserId).ToArray();
+            RoundLengthMetric.Set(0);
-                RoundId = await _db.AddNewRound(playerIds);
 
-                var startingEvent = new RoundStartingEvent();
+            var playerIds = _playersInLobby.Keys.Select(player => player.UserId.UserId).ToArray();
-                RaiseLocalEvent(startingEvent);
+            RoundId = _db.AddNewRound(playerIds).Result;
 
-                List<IPlayerSession> readyPlayers;
+            var startingEvent = new RoundStartingEvent();
-                if (LobbyEnabled)
+            RaiseLocalEvent(startingEvent);
+
+            List<IPlayerSession> readyPlayers;
+            if (LobbyEnabled)
+            {
+                readyPlayers = _playersInLobby.Where(p => p.Value == LobbyPlayerStatus.Ready).Select(p => p.Key)
+                    .ToList();
+            }
+            else
+            {
+                readyPlayers = _playersInLobby.Keys.ToList();
+            }
+
+            readyPlayers.RemoveAll(p =>
+            {
+                if (_roleBanManager.GetRoleBans(p.UserId) != null)
+                    return false;
+                Logger.ErrorS("RoleBans", $"Role bans for player {p} {p.UserId} have not been loaded yet.");
+                return true;
+            });
+
+            // Get the profiles for each player for easier lookup.
+            var profiles = _prefsManager.GetSelectedProfilesForPlayers(
+                    readyPlayers
+                        .Select(p => p.UserId).ToList())
+                .ToDictionary(p => p.Key, p => (HumanoidCharacterProfile) p.Value);
+
+            foreach (var readyPlayer in readyPlayers)
+            {
+                if (!profiles.ContainsKey(readyPlayer.UserId))
                 {
-                    readyPlayers = _playersInLobby.Where(p => p.Value == LobbyPlayerStatus.Ready).Select(p => p.Key)
+                    profiles.Add(readyPlayer.UserId, HumanoidCharacterProfile.Random());
-                        .ToList();
-                }
-                else
-                {
-                    readyPlayers = _playersInLobby.Keys.ToList();
                 }
+            }
 
-                readyPlayers.RemoveAll(p =>
+            var origReadyPlayers = readyPlayers.ToArray();
-                {
-                    if (_roleBanManager.GetRoleBans(p.UserId) != null)
-                        return false;
-                    Logger.ErrorS("RoleBans", $"Role bans for player {p} {p.UserId} have not been loaded yet.");
-                    return true;
-                });
 
-                // Get the profiles for each player for easier lookup.
+            if (!StartPreset(origReadyPlayers, force))
-                var profiles = _prefsManager.GetSelectedProfilesForPlayers(
+                return;
-                        readyPlayers
-                            .Select(p => p.UserId).ToList())
-                    .ToDictionary(p => p.Key, p => (HumanoidCharacterProfile) p.Value);
 
-                foreach (var readyPlayer in readyPlayers)
+            // MapInitialize *before* spawning players, our codebase is too shit to do it afterwards...
-                {
+            _mapManager.DoMapInitialize(DefaultMap);
-                    if (!profiles.ContainsKey(readyPlayer.UserId))
-                    {
-                        profiles.Add(readyPlayer.UserId, HumanoidCharacterProfile.Random());
-                    }
-                }
 
-                var origReadyPlayers = readyPlayers.ToArray();
+            SpawnPlayers(readyPlayers, origReadyPlayers, profiles, force);
 
-                if (!StartPreset(origReadyPlayers, force))
+            _roundStartDateTime = DateTime.UtcNow;
-                    return;
+            RunLevel = GameRunLevel.InRound;
 
-                // MapInitialize *before* spawning players, our codebase is too shit to do it afterwards...
+            _roundStartTimeSpan = _gameTiming.RealTime;
-                _mapManager.DoMapInitialize(DefaultMap);
+            SendStatusToAll();
-
+            ReqWindowAttentionAll();
-                SpawnPlayers(readyPlayers, origReadyPlayers, profiles, force);
+            UpdateLateJoinStatus();
-
+            UpdateJobsAvailable();
-                _roundStartDateTime = DateTime.UtcNow;
-                RunLevel = GameRunLevel.InRound;
-
-                _startingRound = false;
-
-                _roundStartTimeSpan = _gameTiming.RealTime;
-                SendStatusToAll();
-                ReqWindowAttentionAll();
-                UpdateLateJoinStatus();
-                UpdateJobsAvailable();
 
 #if EXCEPTION_TOLERANCE
             }
-            catch(Exception e)
+            catch (Exception e)
             {
+                _roundStartFailCount++;
 
-                Logger.WarningS("ticker", $"Exception caught while trying to start the round! Restarting...");
+                if (RoundStartFailShutdownCount > 0 && _roundStartFailCount >= RoundStartFailShutdownCount)
+                {
+                    Logger.FatalS("ticker",
+                        $"Failed to start a round {_roundStartFailCount} time(s) in a row... Shutting down!");
+                    _runtimeLog.LogException(e, nameof(GameTicker));
+                    _baseServer.Shutdown("Restarting server");
+                    return;
+                }
+
+                Logger.WarningS("ticker", $"Exception caught while trying to start the round! Restarting round...");
                 _runtimeLog.LogException(e, nameof(GameTicker));
+                _startingRound = false;
                 RestartRound();
+                return;
             }
+
+            // Round started successfully! Reset counter...
+            _roundStartFailCount = 0;
 #endif
+            _startingRound = false;
         }
 
         private void RefreshLateJoinAllowed()
@@ -373,7 +381,6 @@ namespace Content.Server.GameTicking
             RunLevel = GameRunLevel.PreRoundLobby;
             LobbySong = _robustRandom.Pick(_lobbyMusicCollection.PickFiles).ToString();
             ResettingCleanup();
-            LoadMaps();
 
             if (!LobbyEnabled)
             {
@@ -411,18 +418,16 @@ namespace Content.Server.GameTicking
                 unCastData.ContentData()?.WipeMind();
             }
 
-            // Delete all entities.
+            _mapManager.Restart();
-            foreach (var entity in EntityManager.GetEntities().ToList())
+
+            // Delete all remaining entities.
+            foreach (var entity in EntityManager.GetEntities().ToArray())
             {
                 // TODO: Maybe something less naive here?
                 // FIXME: Actually, definitely.
                 EntityManager.DeleteEntity(entity);
             }
 
-            _startingRound = false;
-
-            _mapManager.Restart();
-
             _roleBanManager.Restart();
 
             // Clear up any game rules.
@@ -465,8 +470,7 @@ namespace Content.Server.GameTicking
                 RoundLengthMetric.Inc(frameTime);
             }
 
-            if (RunLevel != GameRunLevel.PreRoundLobby ||
+            if (RunLevel != GameRunLevel.PreRoundLobby || Paused ||
-                Paused ||
                 _roundStartTime > _gameTiming.CurTime ||
                 _roundStartCountdownHasNotStartedYetDueToNoPlayers)
             {

Content.Server/GameTicking/GameTicker.cs

@@ -2,6 +2,7 @@ using Content.Server.Administration.Logs;
 using Content.Server.Administration.Managers;
 using Content.Server.CharacterAppearance.Systems;
 using Content.Server.Chat.Managers;
+using Content.Server.Database;
 using Content.Server.Ghost;
 using Content.Server.Maps;
 using Content.Server.PDA;
@@ -89,6 +90,7 @@ namespace Content.Server.GameTicking
         [Dependency] private readonly IBaseServer _baseServer = default!;
         [Dependency] private readonly IWatchdogApi _watchdogApi = default!;
         [Dependency] private readonly IGameMapManager _gameMapManager = default!;
+        [Dependency] private readonly IServerDbManager _db = default!;
 #if EXCEPTION_TOLERANCE
         [Dependency] private readonly IRuntimeLog _runtimeLog = default!;
 #endif

Content.Shared/CCVar/CCVars.cs

@@ -60,6 +60,9 @@ namespace Content.Shared.CCVar
         public static readonly CVarDef<bool>
             EventsEnabled = CVarDef.Create("events.enabled", true, CVar.ARCHIVE | CVar.SERVERONLY);
 
+        /// <summary>
+        ///     Disables most functionality in the GameTicker.
+        /// </summary>
         public static readonly CVarDef<bool>
             GameDummyTicker = CVarDef.Create("game.dummyticker", false, CVar.ARCHIVE | CVar.SERVERONLY);
 
@@ -154,6 +157,15 @@ namespace Content.Shared.CCVar
         public static readonly CVarDef<int> SoftMaxPlayers =
             CVarDef.Create("game.soft_max_players", 30, CVar.SERVERONLY | CVar.ARCHIVE);
 
+#if EXCEPTION_TOLERANCE
+        /// <summary>
+        ///     Amount of times round start must fail before the server is shut down.
+        ///     Set to 0 or a negative number to disable.
+        /// </summary>
+        public static readonly CVarDef<int> RoundStartFailShutdownCount =
+            CVarDef.Create("game.round_start_fail_shutdown_count", 5, CVar.SERVERONLY | CVar.SERVER);
+#endif
+
         /*
          * Discord
          */