Adds Network Resource Uploading for admins. (#6904)

Co-authored-by: Pieter-Jan Briers <pieterjan.briers@gmail.com>
Co-authored-by: Pieter-Jan Briers <pieterjan.briers+git@gmail.com>

Content.Client/Administration/Commands/UploadFile.cs

@@ -0,0 +1,62 @@
+using Content.Shared.Administration;
+using Content.Shared.CCVar;
+using Robust.Client.UserInterface;
+using Robust.Shared.Configuration;
+using Robust.Shared.Console;
+using Robust.Shared.Network;
+using Robust.Shared.Utility;
+
+namespace Content.Client.Administration.Commands;
+
+public sealed class UploadFile : IConsoleCommand
+{
+    public string Command => "uploadfile";
+    public string Description => "Uploads a resource to the server.";
+    public string Help => $"{Command} [relative path for the resource]";
+
+    public async void Execute(IConsoleShell shell, string argStr, string[] args)
+    {
+        var cfgMan = IoCManager.Resolve<IConfigurationManager>();
+
+        if (!cfgMan.GetCVar(CCVars.ResourceUploadingEnabled))
+        {
+            shell.WriteError("Network Resource Uploading is currently disabled by the server.");
+            return;
+        }
+
+        if (args.Length != 1)
+        {
+            shell.WriteError("Wrong number of arguments!");
+            return;
+        }
+
+        var dialog = IoCManager.Resolve<IFileDialogManager>();
+
+        var filters = new FileDialogFilters(new FileDialogFilters.Group("*"));
+        await using var file = await dialog.OpenFile(filters);
+
+        if (file == null)
+        {
+            shell.WriteError("Error picking file!");
+            return;
+        }
+
+        var sizeLimit = cfgMan.GetCVar(CCVars.ResourceUploadingLimitMb);
+
+        if (sizeLimit > 0f && file.Length * SharedNetworkResourceManager.BytesToMegabytes > sizeLimit)
+        {
+            shell.WriteError($"File above the current size limit! It must be smaller than {sizeLimit} MB.");
+            return;
+        }
+
+        var data = file.CopyToArray();
+
+        var netManager = IoCManager.Resolve<INetManager>();
+        var msg = netManager.CreateNetMessage<NetworkResourceUploadMessage>();
+
+        msg.RelativePath = new ResourcePath(args[0]).ToRelativePath();
+        msg.Data = data;
+
+        netManager.ClientSendMessage(msg);
+    }
+}

Content.Client/Administration/Managers/NetworkResourceManager.cs

@@ -0,0 +1,15 @@
+using Content.Shared.Administration;
+
+namespace Content.Client.Administration.Managers;
+
+public sealed class NetworkResourceManager : SharedNetworkResourceManager
+{
+    /// <summary>
+    ///     Callback for when the server sends a new resource.
+    /// </summary>
+    /// <param name="msg">The network message containing the data.</param>
+    protected override void ResourceUploadMsg(NetworkResourceUploadMessage msg)
+    {
+        ContentRoot.AddOrUpdateFile(msg.RelativePath, msg.Data);
+    }
+}

Content.Client/Entry/EntryPoint.cs

@@ -194,6 +194,7 @@ namespace Content.Client.Entry
             IoCManager.Resolve<EuiManager>().Initialize();
             IoCManager.Resolve<IVoteManager>().Initialize();
             IoCManager.Resolve<IGamePrototypeLoadManager>().Initialize();
+            IoCManager.Resolve<NetworkResourceManager>().Initialize();
 
             _baseClient.RunLevelChanged += (_, args) =>
             {

Content.Client/IoC/ClientContentIoC.cs

@@ -15,7 +15,6 @@ using Content.Client.StationEvents.Managers;
 using Content.Client.Stylesheets;
 using Content.Client.Viewport;
 using Content.Client.Voting;
-using Content.Shared.Actions;
 using Content.Shared.Administration;
 using Content.Shared.Module;
 
@@ -43,6 +42,7 @@ namespace Content.Client.IoC
             IoCManager.Register<RulesManager, RulesManager>();
             IoCManager.Register<ViewportManager, ViewportManager>();
             IoCManager.Register<IGamePrototypeLoadManager, GamePrototypeLoadManager>();
+            IoCManager.Register<NetworkResourceManager>();
         }
     }
 }

Content.Server.Database/Migrations/Postgres/20220326104916_UploadedResourcesLog.cs

@@ -0,0 +1,36 @@
+using System;
+using Microsoft.EntityFrameworkCore.Migrations;
+using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
+
+#nullable disable
+
+namespace Content.Server.Database.Migrations.Postgres
+{
+    public partial class UploadedResourcesLog : Migration
+    {
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.CreateTable(
+                name: "uploaded_resource_log",
+                columns: table => new
+                {
+                    uploaded_resource_log_id = table.Column<int>(type: "integer", nullable: false)
+                        .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn),
+                    date = table.Column<DateTime>(type: "timestamp with time zone", nullable: false),
+                    user_id = table.Column<Guid>(type: "uuid", nullable: false),
+                    path = table.Column<string>(type: "text", nullable: false),
+                    data = table.Column<byte[]>(type: "bytea", nullable: false)
+                },
+                constraints: table =>
+                {
+                    table.PrimaryKey("PK_uploaded_resource_log", x => x.uploaded_resource_log_id);
+                });
+        }
+
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropTable(
+                name: "uploaded_resource_log");
+        }
+    }
+}

Content.Server.Database/Migrations/Postgres/PostgresServerDbContextModelSnapshot.cs

@@ -793,6 +793,39 @@ namespace Content.Server.Database.Migrations.Postgres
                     b.ToTable("server_unban", (string)null);
                 });
 
+            modelBuilder.Entity("Content.Server.Database.UploadedResourceLog", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("integer")
+                        .HasColumnName("uploaded_resource_log_id");
+
+                    NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property<int>("Id"));
+
+                    b.Property<byte[]>("Data")
+                        .IsRequired()
+                        .HasColumnType("bytea")
+                        .HasColumnName("data");
+
+                    b.Property<DateTime>("Date")
+                        .HasColumnType("timestamp with time zone")
+                        .HasColumnName("date");
+
+                    b.Property<string>("Path")
+                        .IsRequired()
+                        .HasColumnType("text")
+                        .HasColumnName("path");
+
+                    b.Property<Guid>("UserId")
+                        .HasColumnType("uuid")
+                        .HasColumnName("user_id");
+
+                    b.HasKey("Id")
+                        .HasName("PK_uploaded_resource_log");
+
+                    b.ToTable("uploaded_resource_log", (string)null);
+                });
+
             modelBuilder.Entity("Content.Server.Database.Whitelist", b =>
                 {
                     b.Property<Guid>("UserId")

Content.Server.Database/Migrations/Sqlite/20220326104908_UploadedResourcesLog.cs

@@ -0,0 +1,35 @@
+using System;
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace Content.Server.Database.Migrations.Sqlite
+{
+    public partial class UploadedResourcesLog : Migration
+    {
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.CreateTable(
+                name: "uploaded_resource_log",
+                columns: table => new
+                {
+                    uploaded_resource_log_id = table.Column<int>(type: "INTEGER", nullable: false)
+                        .Annotation("Sqlite:Autoincrement", true),
+                    date = table.Column<DateTime>(type: "TEXT", nullable: false),
+                    user_id = table.Column<Guid>(type: "TEXT", nullable: false),
+                    path = table.Column<string>(type: "TEXT", nullable: false),
+                    data = table.Column<byte[]>(type: "BLOB", nullable: false)
+                },
+                constraints: table =>
+                {
+                    table.PrimaryKey("PK_uploaded_resource_log", x => x.uploaded_resource_log_id);
+                });
+        }
+
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropTable(
+                name: "uploaded_resource_log");
+        }
+    }
+}

Content.Server.Database/Migrations/Sqlite/SqliteServerDbContextModelSnapshot.cs

@@ -737,6 +737,37 @@ namespace Content.Server.Database.Migrations.Sqlite
                     b.ToTable("server_unban", (string)null);
                 });
 
+            modelBuilder.Entity("Content.Server.Database.UploadedResourceLog", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("INTEGER")
+                        .HasColumnName("uploaded_resource_log_id");
+
+                    b.Property<byte[]>("Data")
+                        .IsRequired()
+                        .HasColumnType("BLOB")
+                        .HasColumnName("data");
+
+                    b.Property<DateTime>("Date")
+                        .HasColumnType("TEXT")
+                        .HasColumnName("date");
+
+                    b.Property<string>("Path")
+                        .IsRequired()
+                        .HasColumnType("TEXT")
+                        .HasColumnName("path");
+
+                    b.Property<Guid>("UserId")
+                        .HasColumnType("TEXT")
+                        .HasColumnName("user_id");
+
+                    b.HasKey("Id")
+                        .HasName("PK_uploaded_resource_log");
+
+                    b.ToTable("uploaded_resource_log", (string)null);
+                });
+
             modelBuilder.Entity("Content.Server.Database.Whitelist", b =>
                 {
                     b.Property<Guid>("UserId")

Content.Server.Database/Model.cs

@@ -33,6 +33,7 @@ namespace Content.Server.Database
         public DbSet<ServerBanHit> ServerBanHit { get; set; } = default!;
         public DbSet<ServerRoleBan> RoleBan { get; set; } = default!;
         public DbSet<ServerRoleUnban> RoleUnban { get; set; } = default!;
+        public DbSet<UploadedResourceLog> UploadedResourceLog { get; set; } = default!;
 
         protected override void OnModelCreating(ModelBuilder modelBuilder)
         {
@@ -459,4 +460,19 @@ namespace Content.Server.Database
 
         public DateTime UnbanTime { get; set; }
     }
+
+    [Table("uploaded_resource_log")]
+    public sealed class UploadedResourceLog
+    {
+        [Key, DatabaseGenerated(DatabaseGeneratedOption.Identity)]
+        public int Id { get; set; }
+
+        public DateTime Date { get; set; }
+
+        public Guid UserId { get; set; }
+
+        public string Path { get; set; } = string.Empty;
+
+        public byte[] Data { get; set; } = default!;
+    }
 }

Content.Server/Administration/GamePrototypeLoadManager.cs

@@ -1,11 +1,6 @@
-using System;
-using System.Collections.Generic;
 using Content.Server.Administration.Managers;
 using Content.Shared.Administration;
 using Robust.Server.Player;
-using Robust.Shared.IoC;
-using Robust.Shared.Localization;
-using Robust.Shared.Log;
 using Robust.Shared.Network;
 using Robust.Shared.Prototypes;
 

Content.Server/Administration/NetworkResourceManager.cs

@@ -0,0 +1,90 @@
+using Content.Server.Administration.Managers;
+using Content.Server.Database;
+using Content.Shared.Administration;
+using Content.Shared.CCVar;
+using Robust.Server.Player;
+using Robust.Shared.Configuration;
+using Robust.Shared.Network;
+
+namespace Content.Server.Administration;
+
+public sealed class NetworkResourceManager : SharedNetworkResourceManager
+{
+    [Dependency] private readonly IPlayerManager _playerManager = default!;
+    [Dependency] private readonly IAdminManager _adminManager = default!;
+    [Dependency] private readonly IServerNetManager _serverNetManager = default!;
+    [Dependency] private readonly IConfigurationManager _cfgManager = default!;
+    [Dependency] private readonly IServerDbManager _serverDb = default!;
+
+    [ViewVariables] public bool Enabled { get; private set; } = true;
+    [ViewVariables] public float SizeLimit { get; private set; } = 0f;
+    [ViewVariables] public bool StoreUploaded { get; set; } = true;
+
+    public override void Initialize()
+    {
+        base.Initialize();
+
+        _serverNetManager.Connected += ServerNetManagerOnConnected;
+        _cfgManager.OnValueChanged(CCVars.ResourceUploadingEnabled, value => Enabled = value, true);
+        _cfgManager.OnValueChanged(CCVars.ResourceUploadingLimitMb, value => SizeLimit = value, true);
+        _cfgManager.OnValueChanged(CCVars.ResourceUploadingStoreEnabled, value => StoreUploaded = value, true);
+
+        AutoDelete(_cfgManager.GetCVar(CCVars.ResourceUploadingStoreDeletionDays));
+    }
+
+    /// <summary>
+    ///     Callback for when a client attempts to upload a resource.
+    /// </summary>
+    /// <param name="msg"></param>
+    /// <exception cref="NotImplementedException"></exception>
+    protected override async void ResourceUploadMsg(NetworkResourceUploadMessage msg)
+    {
+        // Do not allow uploading any new resources if it has been disabled.
+        // Note: Any resources uploaded before being disabled will still be kept and sent.
+        if (!Enabled)
+            return;
+
+        if (!_playerManager.TryGetSessionByChannel(msg.MsgChannel, out var session))
+            return;
+
+        // +QUERY only for now.
+        if (!_adminManager.HasAdminFlag(session, AdminFlags.Query))
+            return;
+
+        // Ensure the data is under the current size limit, if it's currently enabled.
+        if (SizeLimit > 0f && msg.Data.Length * BytesToMegabytes > SizeLimit)
+            return;
+
+        ContentRoot.AddOrUpdateFile(msg.RelativePath, msg.Data);
+
+        // Now we broadcast the message!
+        foreach (var channel in _serverNetManager.Channels)
+        {
+            channel.SendMessage(msg);
+        }
+
+        if (!StoreUploaded)
+            return;
+
+        await _serverDb.AddUploadedResourceLogAsync(session.UserId, DateTime.Now, msg.RelativePath.ToString(), msg.Data);
+    }
+
+    private void ServerNetManagerOnConnected(object? sender, NetChannelArgs e)
+    {
+        foreach (var (path, data) in ContentRoot.GetAllFiles())
+        {
+            var msg = _serverNetManager.CreateNetMessage<NetworkResourceUploadMessage>();
+            msg.RelativePath = path;
+            msg.Data = data;
+            e.Channel.SendMessage(msg);
+        }
+    }
+
+    private async void AutoDelete(int days)
+    {
+        if (days <= 0)
+            return; // auto-deletion disabled...
+
+        await _serverDb.PurgeUploadedResourceLogAsync(days);
+    }
+}

Content.Server/Database/ServerDbBase.cs

@@ -776,6 +776,34 @@ namespace Content.Server.Database
 
         #endregion
 
+        #region Uploaded Resources Logs
+
+        public async Task AddUploadedResourceLogAsync(NetUserId user, DateTime date, string path, byte[] data)
+        {
+            await using var db = await GetDb();
+
+            db.DbContext.UploadedResourceLog.Add(new UploadedResourceLog() { UserId = user, Date = date, Path = path, Data = data });
+            await db.DbContext.SaveChangesAsync();
+        }
+
+        public async Task PurgeUploadedResourceLogAsync(int days)
+        {
+            await using var db = await GetDb();
+
+            var date = DateTime.Now.Subtract(TimeSpan.FromDays(days));
+
+            await foreach (var log in db.DbContext.UploadedResourceLog
+                               .Where(l => date > l.Date)
+                               .AsAsyncEnumerable())
+            {
+                db.DbContext.UploadedResourceLog.Remove(log);
+            }
+
+            await db.DbContext.SaveChangesAsync();
+        }
+
+        #endregion
+
         protected abstract Task<DbGuard> GetDb();
 
         protected abstract class DbGuard : IAsyncDisposable

Content.Server/Database/ServerDbManager.cs

@@ -185,6 +185,14 @@ namespace Content.Server.Database
         Task RemoveFromWhitelistAsync(NetUserId player);
 
         #endregion
+
+        #region Uploaded Resources Logs
+
+        Task AddUploadedResourceLogAsync(NetUserId user, DateTime date, string path, byte[] data);
+
+        Task PurgeUploadedResourceLogAsync(int days);
+
+        #endregion
     }
 
     public sealed class ServerDbManager : IServerDbManager
@@ -455,6 +463,16 @@ namespace Content.Server.Database
             return _db.RemoveFromWhitelistAsync(player);
         }
 
+        public Task AddUploadedResourceLogAsync(NetUserId user, DateTime date, string path, byte[] data)
+        {
+            return _db.AddUploadedResourceLogAsync(user, date, path, data);
+        }
+
+        public Task PurgeUploadedResourceLogAsync(int days)
+        {
+            return _db.PurgeUploadedResourceLogAsync(days);
+        }
+
         private DbContextOptions<PostgresServerDbContext> CreatePostgresOptions()
         {
             var host = _cfg.GetCVar(CCVars.DatabasePgHost);

Content.Server/Entry/EntryPoint.cs

@@ -1,4 +1,5 @@
 using System.IO;
+using Content.Server.Administration;
 using Content.Server.Administration.Managers;
 using Content.Server.Afk;
 using Content.Server.AI.Utility;
@@ -86,6 +87,7 @@ namespace Content.Server.Entry
                 IoCManager.Resolve<IServerPreferencesManager>().Init();
                 IoCManager.Resolve<INodeGroupFactory>().Initialize();
                 IoCManager.Resolve<IGamePrototypeLoadManager>().Initialize();
+                IoCManager.Resolve<NetworkResourceManager>().Initialize();
                 _voteManager.Initialize();
             }
         }

Content.Server/IoC/ServerContentIoC.cs

@@ -50,6 +50,7 @@ namespace Content.Server.IoC
             IoCManager.Register<IGamePrototypeLoadManager, GamePrototypeLoadManager>();
             IoCManager.Register<RulesManager, RulesManager>();
             IoCManager.Register<RoleBanManager, RoleBanManager>();
+            IoCManager.Register<NetworkResourceManager>();
         }
     }
 }

Content.Shared/Administration/NetworkResourceUploadMessage.cs

@@ -0,0 +1,39 @@
+using Lidgren.Network;
+using Robust.Shared.Network;
+using Robust.Shared.Utility;
+
+namespace Content.Shared.Administration;
+
+public sealed class NetworkResourceUploadMessage : NetMessage
+{
+    public override NetDeliveryMethod DeliveryMethod => NetDeliveryMethod.ReliableUnordered;
+    public override MsgGroups MsgGroup => MsgGroups.Command;
+
+    public byte[] Data { get; set; } = Array.Empty<byte>();
+    public ResourcePath RelativePath { get; set; } = ResourcePath.Self;
+
+    public NetworkResourceUploadMessage()
+    {
+    }
+
+    public NetworkResourceUploadMessage(byte[] data, ResourcePath relativePath)
+    {
+        Data = data;
+        RelativePath = relativePath;
+    }
+
+    public override void ReadFromBuffer(NetIncomingMessage buffer)
+    {
+        var dataLength = buffer.ReadVariableInt32();
+        Data = buffer.ReadBytes(dataLength);
+        RelativePath = new ResourcePath(buffer.ReadString(), buffer.ReadString());
+    }
+
+    public override void WriteToBuffer(NetOutgoingMessage buffer)
+    {
+        buffer.WriteVariableInt32(Data.Length);
+        buffer.Write(Data);
+        buffer.Write(RelativePath.ToString());
+        buffer.Write(RelativePath.Separator);
+    }
+}

Content.Shared/Administration/SharedNetworkResourceManager.cs

@@ -0,0 +1,41 @@
+using Robust.Shared.ContentPack;
+using Robust.Shared.Network;
+using Robust.Shared.Utility;
+
+namespace Content.Shared.Administration;
+
+/// <summary>
+///     Manager that allows resources to be added at runtime by admins.
+///     They will be sent to all clients automatically.
+/// </summary>
+public abstract class SharedNetworkResourceManager : IDisposable
+{
+    [Dependency] private readonly INetManager _netManager = default!;
+    [Dependency] protected readonly IResourceManager ResourceManager = default!;
+
+    public const double BytesToMegabytes = 0.000001d;
+
+    /// <summary>
+    ///     The prefix for any and all downloaded network resources.
+    /// </summary>
+    private static readonly ResourcePath Prefix = ResourcePath.Root / "Uploaded";
+
+    protected readonly MemoryContentRoot ContentRoot = new();
+
+    public virtual void Initialize()
+    {
+        _netManager.RegisterNetMessage<NetworkResourceUploadMessage>(ResourceUploadMsg);
+
+        // Add our content root to the resource manager.
+        ResourceManager.AddRoot(Prefix, ContentRoot);
+    }
+
+    protected abstract void ResourceUploadMsg(NetworkResourceUploadMessage msg);
+
+    public void Dispose()
+    {
+        // This is called automatically when the IoCManager's dependency collection is cleared.
+        // MemoryContentRoot uses a ReaderWriterLockSlim, which we need to dispose of.
+        ContentRoot.Dispose();
+    }
+}

Content.Shared/CCVar/CCVars.cs

@@ -737,5 +737,37 @@ namespace Content.Shared.CCVar
 
         public static readonly CVarDef<string> DestinationFile =
             CVarDef.Create("autogen.destination_file", "", CVar.SERVER | CVar.SERVERONLY);
+
+        /*
+         * Network Resource Manager
+         */
+
+        /// <summary>
+        /// Controls whether new resources can be uploaded by admins.
+        /// Does not prevent already uploaded resources from being sent.
+        /// </summary>
+        public static readonly CVarDef<bool> ResourceUploadingEnabled =
+            CVarDef.Create("netres.enabled", true, CVar.REPLICATED | CVar.SERVER);
+
+        /// <summary>
+        /// Controls the data size limit in megabytes for uploaded resources. If they're too big, they will be dropped.
+        /// Set to zero or a negative value to disable limit.
+        /// </summary>
+        public static readonly CVarDef<float> ResourceUploadingLimitMb =
+            CVarDef.Create("netres.limit", 3f, CVar.REPLICATED | CVar.SERVER);
+
+        /// <summary>
+        /// Whether uploaded files will be stored in the server's database.
+        /// This is useful to keep "logs" on what files admins have uploaded in the past.
+        /// </summary>
+        public static readonly CVarDef<bool> ResourceUploadingStoreEnabled =
+            CVarDef.Create("netres.store_enabled", true, CVar.SERVER | CVar.SERVERONLY);
+
+        /// <summary>
+        /// Numbers of days before stored uploaded files are deleted. Set to zero or negative to disable auto-delete.
+        /// This is useful to free some space automatically. Auto-deletion runs only on server boot.
+        /// </summary>
+        public static readonly CVarDef<int> ResourceUploadingStoreDeletionDays =
+            CVarDef.Create("netres.store_deletion_days", 30, CVar.SERVER | CVar.SERVERONLY);
     }
 }

Resources/clientCommandPerms.yml

@@ -28,3 +28,7 @@
 - Flags: ADMIN
   Commands:
     - togglehealthoverlay
+
+- Flags: QUERY
+  Commands:
+    - uploadfile