Port python packaging to Content.Packaging (#21458)

.github/workflows/publish.yml

@@ -29,10 +29,17 @@ jobs:
         cd RobustToolbox
         git fetch --depth=1
 
+    - name: Install dependencies
+      run: dotnet restore
+
+    - name: Build Packaging
+      run: dotnet build Content.Packaging --configuration Release --no-restore /m
+
+    - name: Package server
+      run: dotnet run --project Content.Packaging server --platform win-x64 --platform linux-x64 --platform osx-x64 --platform linux-arm64
+
     - name: Package client
-      run: |
+      run: dotnet run --project Content.Packaging client --no-wipe-release
-        Tools/package_server_build.py -p win-x64 linux-x64 osx-x64 linux-arm64
-        Tools/package_client_build.py
 
     - name: Update Build Info
       run: Tools/gen_build_info.py

.github/workflows/test-packaging.yml

@@ -56,10 +56,14 @@ jobs:
       - name: Install dependencies
         run: dotnet restore
 
+      - name: Build Packaging
+        run: dotnet build Content.Packaging --configuration Release --no-restore /m
+
+      - name: Package server
+        run: dotnet run --project Content.Packaging server --platform win-x64 --platform linux-x64 --platform osx-x64 --platform linux-arm64
+
       - name: Package client
-        run: |
+        run: dotnet run --project Content.Packaging client --no-wipe-release
-          Tools/package_server_build.py -p win-x64 linux-x64 osx-x64 linux-arm64
-          Tools/package_client_build.py
       
       - name: Update Build Info
         run: Tools/gen_build_info.py

Content.Packaging/ClientPackaging.cs

@@ -0,0 +1,79 @@
+using System.Diagnostics;
+using System.IO.Compression;
+using Robust.Packaging;
+using Robust.Packaging.AssetProcessing;
+using Robust.Packaging.AssetProcessing.Passes;
+using Robust.Packaging.Utility;
+using Robust.Shared.Timing;
+
+namespace Content.Packaging;
+
+public static class ClientPackaging
+{
+    /// <summary>
+    /// Be advised this can be called from server packaging during a HybridACZ build.
+    /// </summary>
+    public static async Task PackageClient(bool skipBuild, IPackageLogger logger)
+    {
+        logger.Info("Building client...");
+
+        if (!skipBuild)
+        {
+            await ProcessHelpers.RunCheck(new ProcessStartInfo
+            {
+                FileName = "dotnet",
+                ArgumentList =
+                {
+                    "build",
+                    Path.Combine("Content.Client", "Content.Client.csproj"),
+                    "-c", "Release",
+                    "--nologo",
+                    "/v:m",
+                    "/t:Rebuild",
+                    "/p:FullRelease=true",
+                    "/m"
+                }
+            });
+        }
+
+        logger.Info("Packaging client...");
+
+        var sw = RStopwatch.StartNew();
+        {
+            await using var zipFile =
+                File.Open(Path.Combine("release", "SS14.Client.zip"), FileMode.Create, FileAccess.ReadWrite);
+            using var zip = new ZipArchive(zipFile, ZipArchiveMode.Update);
+            var writer = new AssetPassZipWriter(zip);
+
+            await WriteResources("", writer, logger, default);
+            await writer.FinishedTask;
+        }
+
+        logger.Info($"Finished packaging client in {sw.Elapsed}");
+    }
+
+    public static async Task WriteResources(
+        string contentDir,
+        AssetPass pass,
+        IPackageLogger logger,
+        CancellationToken cancel)
+    {
+        var graph = new RobustClientAssetGraph();
+        pass.Dependencies.Add(new AssetPassDependency(graph.Output.Name));
+
+        AssetGraph.CalculateGraph(graph.AllPasses.Append(pass).ToArray(), logger);
+
+        var inputPass = graph.Input;
+
+        await RobustSharedPackaging.WriteContentAssemblies(
+            inputPass,
+            contentDir,
+            "Content.Client",
+            new[] { "Content.Client", "Content.Shared", "Content.Shared.Database" },
+            cancel: cancel);
+
+        await RobustClientPackaging.WriteClientResources(contentDir, pass, cancel);
+
+        inputPass.InjectFinished();
+    }
+}

Content.Packaging/CommandLineArgs.cs

@@ -0,0 +1,139 @@
+using System.Diagnostics.CodeAnalysis;
+
+namespace Content.Packaging;
+
+public sealed class CommandLineArgs
+{
+    // PJB forgib me
+
+    /// <summary>
+    /// Generate client or server.
+    /// </summary>
+    public bool Client { get; set; }
+
+    /// <summary>
+    /// Should we also build the relevant project.
+    /// </summary>
+    public bool SkipBuild { get; set; }
+
+    /// <summary>
+    /// Should we wipe the release folder or ignore it.
+    /// </summary>
+    public bool WipeRelease { get; set; }
+
+    /// <summary>
+    /// Platforms for server packaging.
+    /// </summary>
+    public List<string>? Platforms { get; set; }
+
+    /// <summary>
+    /// Use HybridACZ for server packaging.
+    /// </summary>
+    public bool HybridAcz { get; set; }
+
+    // CommandLineArgs, 3rd of her name.
+    public static bool TryParse(IReadOnlyList<string> args, [NotNullWhen(true)] out CommandLineArgs? parsed)
+    {
+        parsed = null;
+        bool? client = null;
+        var skipBuild = false;
+        var wipeRelease = true;
+        var hybridAcz = false;
+        List<string>? platforms = null;
+
+        using var enumerator = args.GetEnumerator();
+        var i = -1;
+
+        while (enumerator.MoveNext())
+        {
+            i++;
+            var arg = enumerator.Current;
+            if (i == 0)
+            {
+                if (arg == "client")
+                {
+                    client = true;
+                }
+                else if (arg == "server")
+                {
+                    client = false;
+                }
+                else
+                {
+                    return false;
+                }
+
+                continue;
+            }
+
+            if (arg == "--skip-build")
+            {
+                skipBuild = true;
+            }
+            else if (arg == "--no-wipe-release")
+            {
+                wipeRelease = false;
+            }
+            else if (arg == "--hybrid-acz")
+            {
+                hybridAcz = true;
+            }
+            else if (arg == "--platform")
+            {
+                if (!enumerator.MoveNext())
+                {
+                    Console.WriteLine("No platform provided");
+                    return false;
+                }
+
+                platforms ??= new List<string>();
+                platforms.Add(enumerator.Current);
+            }
+            else if (arg == "--help")
+            {
+                PrintHelp();
+                return false;
+            }
+            else
+            {
+                Console.WriteLine("Unknown argument: {0}", arg);
+            }
+        }
+
+        if (client == null)
+        {
+            Console.WriteLine("Client / server packaging unspecified.");
+            return false;
+        }
+
+        parsed = new CommandLineArgs(client.Value, skipBuild, wipeRelease, hybridAcz, platforms);
+        return true;
+    }
+
+    private static void PrintHelp()
+    {
+        Console.WriteLine(@"
+Usage: Content.Packaging [client/server] [options]
+
+Options:
+  --skip-build          Should we skip building the project and use what's already there.
+  --no-wipe-release     Don't wipe the release folder before creating files.
+  --hybrid-acz          Use HybridACZ for server builds.
+  --platform            Platform for server builds. Default will output several x64 targets.
+");
+    }
+
+    private CommandLineArgs(
+        bool client,
+        bool skipBuild,
+        bool wipeRelease,
+        bool hybridAcz,
+        List<string>? platforms)
+    {
+        Client = client;
+        SkipBuild = skipBuild;
+        WipeRelease = wipeRelease;
+        HybridAcz = hybridAcz;
+        Platforms = platforms;
+    }
+}

Content.Packaging/Content.Packaging.csproj

@@ -5,6 +5,9 @@
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
+  <ItemGroup>
+    <PackageReference Include="NVorbis" Version="0.10.1" PrivateAssets="compile" />
+  </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\RobustToolbox\Robust.Packaging\Robust.Packaging.csproj" />
   </ItemGroup>

Content.Packaging/ContentPackaging.cs

@@ -1,32 +0,0 @@
-using Robust.Packaging;
-using Robust.Packaging.AssetProcessing;
-
-namespace Content.Packaging;
-
-public static class ContentPackaging
-{
-    public static async Task WriteResources(
-        string contentDir,
-        AssetPass pass,
-        IPackageLogger logger,
-        CancellationToken cancel)
-    {
-        var graph = new RobustClientAssetGraph();
-        pass.Dependencies.Add(new AssetPassDependency(graph.Output.Name));
-
-        AssetGraph.CalculateGraph(graph.AllPasses.Append(pass).ToArray(), logger);
-
-        var inputPass = graph.Input;
-
-        await RobustClientPackaging.WriteContentAssemblies(
-            inputPass,
-            contentDir,
-            "Content.Client",
-            new[] { "Content.Client", "Content.Shared", "Content.Shared.Database" },
-            cancel);
-
-        await RobustClientPackaging.WriteClientResources(contentDir, inputPass, cancel);
-
-        inputPass.InjectFinished();
-    }
-}

Content.Packaging/Program.cs

@@ -1,68 +1,44 @@
-using System.Diagnostics;
+using Content.Packaging;
-using System.IO.Compression;
-using Content.Packaging;
 using Robust.Packaging;
-using Robust.Packaging.AssetProcessing.Passes;
-using Robust.Packaging.Utility;
-using Robust.Shared.Timing;
 
 IPackageLogger logger = new PackageLoggerConsole();
 
-logger.Info("Clearing release/ directory");
+if (!CommandLineArgs.TryParse(args, out var parsed))
-Directory.CreateDirectory("release");
-
-var skipBuild = args.Contains("--skip-build");
-
-if (!skipBuild)
-    WipeBin();
-
-await Build(skipBuild);
-
-async Task Build(bool skipBuild)
 {
-    logger.Info("Building project...");
+    logger.Error("Unable to parse args, aborting.");
-
+    return;
-    if (!skipBuild)
-    {
-        await ProcessHelpers.RunCheck(new ProcessStartInfo
-        {
-            FileName = "dotnet",
-            ArgumentList =
-            {
-                "build",
-                Path.Combine("Content.Client", "Content.Client.csproj"),
-                "-c", "Release",
-                "--nologo",
-                "/v:m",
-                "/t:Rebuild",
-                "/p:FullRelease=true",
-                "/m"
-            }
-        });
-    }
-
-    logger.Info("Packaging client...");
-
-    var sw = RStopwatch.StartNew();
-
-    {
-        using var zipFile =
-            File.Open(Path.Combine("release", "SS14.Client.zip"), FileMode.Create, FileAccess.ReadWrite);
-        using var zip = new ZipArchive(zipFile, ZipArchiveMode.Update);
-        var writer = new AssetPassZipWriter(zip);
-
-        await ContentPackaging.WriteResources("", writer, logger, default);
-
-        await writer.FinishedTask;
-    }
-
-    logger.Info($"Finished packaging in {sw.Elapsed}");
 }
 
+if (parsed.WipeRelease)
+    WipeRelease();
+
+if (!parsed.SkipBuild)
+    WipeBin();
+
+if (parsed.Client)
+{
+    await ClientPackaging.PackageClient(parsed.SkipBuild, logger);
+}
+else
+{
+    await ServerPackaging.PackageServer(parsed.SkipBuild, parsed.HybridAcz, logger, parsed.Platforms);
+}
 
 void WipeBin()
 {
     logger.Info("Clearing old build artifacts (if any)...");
 
+    if (Directory.Exists("bin"))
         Directory.Delete("bin", recursive: true);
 }
+
+void WipeRelease()
+{
+    if (Directory.Exists("release"))
+    {
+        logger.Info("Cleaning old release packages (release/)...");
+        Directory.Delete("release", recursive: true);
+    }
+
+    Directory.CreateDirectory("release");
+}

Content.Packaging/ServerPackaging.cs

@@ -0,0 +1,226 @@
+using System.Diagnostics;
+using System.Globalization;
+using System.IO.Compression;
+using Robust.Packaging;
+using Robust.Packaging.AssetProcessing;
+using Robust.Packaging.AssetProcessing.Passes;
+using Robust.Packaging.Utility;
+using Robust.Shared.Audio;
+using Robust.Shared.Serialization;
+using Robust.Shared.Timing;
+using YamlDotNet.Core;
+using YamlDotNet.RepresentationModel;
+
+namespace Content.Packaging;
+
+public static class ServerPackaging
+{
+    private static readonly List<PlatformReg> Platforms = new()
+    {
+        new PlatformReg("win-x64", "Windows", true),
+        new PlatformReg("linux-x64", "Linux", true),
+        new PlatformReg("linux-arm64", "Linux", true),
+        new PlatformReg("osx-x64", "MacOS", true),
+        // Non-default platforms (i.e. for Watchdog Git)
+        new PlatformReg("win-x86", "Windows", false),
+        new PlatformReg("linux-x86", "Linux", false),
+        new PlatformReg("linux-arm", "Linux", false),
+    };
+
+    private static List<string> PlatformRids => Platforms
+        .Select(o => o.Rid)
+        .ToList();
+
+    private static List<string> PlatformRidsDefault => Platforms
+        .Where(o => o.BuildByDefault)
+        .Select(o => o.Rid)
+        .ToList();
+
+    private static readonly List<string> ServerContentAssemblies = new()
+    {
+        "Content.Server.Database",
+        "Content.Server",
+        "Content.Shared",
+        "Content.Shared.Database",
+    };
+
+    private static readonly List<string> ServerExtraAssemblies = new()
+    {
+        // Python script had Npgsql. though we want Npgsql.dll as well soooo
+        "Npgsql",
+        "Microsoft",
+    };
+
+    private static readonly List<string> ServerNotExtraAssemblies = new()
+    {
+        "Microsoft.CodeAnalysis",
+    };
+
+    private static readonly HashSet<string> BinSkipFolders = new()
+    {
+        // Roslyn localization files, screw em.
+        "cs",
+        "de",
+        "es",
+        "fr",
+        "it",
+        "ja",
+        "ko",
+        "pl",
+        "pt-BR",
+        "ru",
+        "tr",
+        "zh-Hans",
+        "zh-Hant"
+    };
+
+    public static async Task PackageServer(bool skipBuild, bool hybridAcz, IPackageLogger logger, List<string>? platforms = null)
+    {
+        if (platforms == null)
+        {
+            platforms ??= PlatformRidsDefault;
+        }
+
+        if (hybridAcz)
+        {
+            // Hybrid ACZ involves a file "Content.Client.zip" in the server executable directory.
+            // Rather than hosting the client ZIP on the watchdog or on a separate server,
+            //  Hybrid ACZ uses the ACZ hosting functionality to host it as part of the status host,
+            //  which means that features such as automatic UPnP forwarding still work properly.
+            await ClientPackaging.PackageClient(skipBuild, logger);
+        }
+
+        // Good variable naming right here.
+        foreach (var platform in Platforms)
+        {
+            if (!platforms.Contains(platform.Rid))
+                continue;
+
+            await BuildPlatform(platform, skipBuild, hybridAcz, logger);
+        }
+    }
+
+    private static async Task BuildPlatform(PlatformReg platform, bool skipBuild, bool hybridAcz, IPackageLogger logger)
+    {
+        logger.Info($"Building project for {platform}...");
+
+        if (!skipBuild)
+        {
+            await ProcessHelpers.RunCheck(new ProcessStartInfo
+            {
+                FileName = "dotnet",
+                ArgumentList =
+                {
+                    "build",
+                    Path.Combine("Content.Server", "Content.Server.csproj"),
+                    "-c", "Release",
+                    "--nologo",
+                    "/v:m",
+                    $"/p:TargetOs={platform.TargetOs}",
+                    "/t:Rebuild",
+                    "/p:FullRelease=true",
+                    "/m"
+                }
+            });
+
+            await PublishClientServer(platform.Rid, platform.TargetOs);
+        }
+
+        logger.Info($"Packaging {platform.Rid} server...");
+
+        var sw = RStopwatch.StartNew();
+        {
+            await using var zipFile =
+                File.Open(Path.Combine("release", $"SS14.Server_{platform.Rid}.zip"), FileMode.Create, FileAccess.ReadWrite);
+            using var zip = new ZipArchive(zipFile, ZipArchiveMode.Update);
+            var writer = new AssetPassZipWriter(zip);
+
+            await WriteServerResources(platform, "", writer, logger, hybridAcz, default);
+            await writer.FinishedTask;
+        }
+
+        logger.Info($"Finished packaging server in {sw.Elapsed}");
+    }
+
+    private static async Task PublishClientServer(string runtime, string targetOs)
+    {
+        await ProcessHelpers.RunCheck(new ProcessStartInfo
+        {
+            FileName = "dotnet",
+            ArgumentList =
+            {
+                "publish",
+                "--runtime", runtime,
+                "--no-self-contained",
+                "-c", "Release",
+                $"/p:TargetOs={targetOs}",
+                "/p:FullRelease=True",
+                "/m",
+                "RobustToolbox/Robust.Server/Robust.Server.csproj"
+            }
+        });
+    }
+
+    private static async Task WriteServerResources(
+        PlatformReg platform,
+        string contentDir,
+        AssetPass pass,
+        IPackageLogger logger,
+        bool hybridAcz,
+        CancellationToken cancel)
+    {
+        var graph = new RobustClientAssetGraph();
+        var passes = graph.AllPasses.ToList();
+
+        pass.Dependencies.Add(new AssetPassDependency(graph.Output.Name));
+        passes.Add(pass);
+
+        AssetGraph.CalculateGraph(passes, logger);
+
+        var inputPass = graph.Input;
+        var contentAssemblies = new List<string>(ServerContentAssemblies);
+
+        // Additional assemblies that need to be copied such as EFCore.
+        var sourcePath = Path.Combine(contentDir, "bin", "Content.Server");
+
+        // Should this be an asset pass?
+        // For future archaeologists I just want audio rework to work and need the audio pass so
+        // just porting this as is from python.
+        foreach (var fullPath in Directory.EnumerateFiles(sourcePath, "*.*", SearchOption.AllDirectories))
+        {
+            var fileName = Path.GetFileNameWithoutExtension(fullPath);
+
+            if (!ServerNotExtraAssemblies.Any(o => fileName.StartsWith(o)) && ServerExtraAssemblies.Any(o => fileName.StartsWith(o)))
+            {
+                contentAssemblies.Add(fileName);
+            }
+        }
+
+        await RobustSharedPackaging.DoResourceCopy(
+            Path.Combine("RobustToolbox", "bin", "Server",
+            platform.Rid,
+            "publish"),
+            inputPass,
+            BinSkipFolders,
+            cancel: cancel);
+
+        await RobustSharedPackaging.WriteContentAssemblies(
+            inputPass,
+            contentDir,
+            "Content.Server",
+            contentAssemblies,
+            Path.Combine("Resources", "Assemblies"),
+            cancel);
+
+        await RobustServerPackaging.WriteServerResources(contentDir, inputPass, cancel);
+
+        if (hybridAcz)
+        {
+            inputPass.InjectFileFromDisk("Content.Client.zip", Path.Combine("release", "SS14.Client.zip"));
+        }
+
+        inputPass.InjectFinished();
+    }
+
+    private readonly record struct PlatformReg(string Rid, string TargetOs, bool BuildByDefault);
+}

Content.Server/Acz/ContentMagicAczProvider.cs

@@ -20,6 +20,6 @@ public sealed class ContentMagicAczProvider : IMagicAczProvider
     {
         var contentDir = DefaultMagicAczProvider.FindContentRootPath(_deps);
 
-        await ContentPackaging.WriteResources(contentDir, pass, logger, cancel);
+        await ClientPackaging.WriteResources(contentDir, pass, logger, cancel);
     }
 }

SpaceStation14.sln

@@ -63,8 +63,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Build", "Build", "{806ED41A
 	ProjectSection(SolutionItems) = preProject
 		Tools\gen_build_info.py = Tools\gen_build_info.py
 		Tools\generate_hashes.ps1 = Tools\generate_hashes.ps1
-		Tools\package_client_build.py = Tools\package_client_build.py
-		Tools\package_server_build.py = Tools\package_server_build.py
 	EndProjectSection
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Robust.Shared.Scripting", "RobustToolbox\Robust.Shared.Scripting\Robust.Shared.Scripting.csproj", "{41B450C0-A361-4CD7-8121-7072B8995CFC}"