diff --git a/Content.Client/UserInterface/Systems/Inventory/InventoryUIController.cs b/Content.Client/UserInterface/Systems/Inventory/InventoryUIController.cs
index 5d7a775104..49a2fba898 100644
--- a/Content.Client/UserInterface/Systems/Inventory/InventoryUIController.cs
+++ b/Content.Client/UserInterface/Systems/Inventory/InventoryUIController.cs
@@ -21,6 +21,7 @@ using Robust.Client.UserInterface.Controls;
 using Robust.Shared.Input;
 using Robust.Shared.Input.Binding;
 using Robust.Shared.Map;
+using Robust.Shared.Player;
 using Robust.Shared.Utility;
 using static Content.Client.Inventory.ClientInventorySystem;
 
@@ -399,6 +400,9 @@ public sealed class InventoryUIController : UIController, IOnStateEntered<Gamepl
         foreach (var slotData in clientInv.SlotData.Values)
         {
             AddSlot(slotData);
+
+            if (_inventoryButton != null)
+                _inventoryButton.Visible = true;
         }
 
         UpdateInventoryHotbar(_playerInventory);
@@ -406,6 +410,9 @@ public sealed class InventoryUIController : UIController, IOnStateEntered<Gamepl
 
     private void UnloadSlots()
     {
+        if (_inventoryButton != null)
+            _inventoryButton.Visible = false;
+
         _playerUid = null;
         _playerInventory = null;
         foreach (var slotGroup in _slotGroups.Values)
diff --git a/Content.Client/UserInterface/Systems/Inventory/Widgets/InventoryGui.xaml b/Content.Client/UserInterface/Systems/Inventory/Widgets/InventoryGui.xaml
index 576d06d816..d4fb68c38d 100644
--- a/Content.Client/UserInterface/Systems/Inventory/Widgets/InventoryGui.xaml
+++ b/Content.Client/UserInterface/Systems/Inventory/Widgets/InventoryGui.xaml
@@ -9,9 +9,11 @@
     Orientation="Horizontal"
     HorizontalAlignment="Center">
     <Control HorizontalAlignment="Center">
+        <!-- Needs to default to invisible because if we attach to a non-slots entity this will never get unset -->
         <controls:SlotButton
             Name="InventoryButton"
             Access="Public"
+            Visible="False"
             VerticalAlignment="Bottom"
             HorizontalExpand="False"
             VerticalExpand="False"