diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index ae4c4e270f..8283763839 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -29,10 +29,17 @@ jobs: cd RobustToolbox git fetch --depth=1 + - name: Install dependencies + run: dotnet restore + + - name: Build Packaging + run: dotnet build Content.Packaging --configuration Release --no-restore /m + + - name: Package server + run: dotnet run --project Content.Packaging server --platform win-x64 --platform linux-x64 --platform osx-x64 --platform linux-arm64 + - name: Package client - run: | - Tools/package_server_build.py -p win-x64 linux-x64 osx-x64 linux-arm64 - Tools/package_client_build.py + run: dotnet run --project Content.Packaging client --no-wipe-release - name: Update Build Info run: Tools/gen_build_info.py diff --git a/.github/workflows/test-packaging.yml b/.github/workflows/test-packaging.yml index 815b6a4adc..b22f307de5 100644 --- a/.github/workflows/test-packaging.yml +++ b/.github/workflows/test-packaging.yml @@ -56,11 +56,15 @@ jobs: - name: Install dependencies run: dotnet restore - - name: Package client - run: | - Tools/package_server_build.py -p win-x64 linux-x64 osx-x64 linux-arm64 - Tools/package_client_build.py + - name: Build Packaging + run: dotnet build Content.Packaging --configuration Release --no-restore /m + - name: Package server + run: dotnet run --project Content.Packaging server --platform win-x64 --platform linux-x64 --platform osx-x64 --platform linux-arm64 + + - name: Package client + run: dotnet run --project Content.Packaging client --no-wipe-release + - name: Update Build Info run: Tools/gen_build_info.py diff --git a/Content.Packaging/ClientPackaging.cs b/Content.Packaging/ClientPackaging.cs new file mode 100644 index 0000000000..a989ebd968 --- /dev/null +++ b/Content.Packaging/ClientPackaging.cs @@ -0,0 +1,79 @@ +using System.Diagnostics; +using System.IO.Compression; +using Robust.Packaging; +using Robust.Packaging.AssetProcessing; +using Robust.Packaging.AssetProcessing.Passes; +using Robust.Packaging.Utility; +using Robust.Shared.Timing; + +namespace Content.Packaging; + +public static class ClientPackaging +{ + /// + /// Be advised this can be called from server packaging during a HybridACZ build. + /// + public static async Task PackageClient(bool skipBuild, IPackageLogger logger) + { + logger.Info("Building client..."); + + if (!skipBuild) + { + await ProcessHelpers.RunCheck(new ProcessStartInfo + { + FileName = "dotnet", + ArgumentList = + { + "build", + Path.Combine("Content.Client", "Content.Client.csproj"), + "-c", "Release", + "--nologo", + "/v:m", + "/t:Rebuild", + "/p:FullRelease=true", + "/m" + } + }); + } + + logger.Info("Packaging client..."); + + var sw = RStopwatch.StartNew(); + { + await using var zipFile = + File.Open(Path.Combine("release", "SS14.Client.zip"), FileMode.Create, FileAccess.ReadWrite); + using var zip = new ZipArchive(zipFile, ZipArchiveMode.Update); + var writer = new AssetPassZipWriter(zip); + + await WriteResources("", writer, logger, default); + await writer.FinishedTask; + } + + logger.Info($"Finished packaging client in {sw.Elapsed}"); + } + + public static async Task WriteResources( + string contentDir, + AssetPass pass, + IPackageLogger logger, + CancellationToken cancel) + { + var graph = new RobustClientAssetGraph(); + pass.Dependencies.Add(new AssetPassDependency(graph.Output.Name)); + + AssetGraph.CalculateGraph(graph.AllPasses.Append(pass).ToArray(), logger); + + var inputPass = graph.Input; + + await RobustSharedPackaging.WriteContentAssemblies( + inputPass, + contentDir, + "Content.Client", + new[] { "Content.Client", "Content.Shared", "Content.Shared.Database" }, + cancel: cancel); + + await RobustClientPackaging.WriteClientResources(contentDir, pass, cancel); + + inputPass.InjectFinished(); + } +} diff --git a/Content.Packaging/CommandLineArgs.cs b/Content.Packaging/CommandLineArgs.cs new file mode 100644 index 0000000000..9f2b075535 --- /dev/null +++ b/Content.Packaging/CommandLineArgs.cs @@ -0,0 +1,139 @@ +using System.Diagnostics.CodeAnalysis; + +namespace Content.Packaging; + +public sealed class CommandLineArgs +{ + // PJB forgib me + + /// + /// Generate client or server. + /// + public bool Client { get; set; } + + /// + /// Should we also build the relevant project. + /// + public bool SkipBuild { get; set; } + + /// + /// Should we wipe the release folder or ignore it. + /// + public bool WipeRelease { get; set; } + + /// + /// Platforms for server packaging. + /// + public List? Platforms { get; set; } + + /// + /// Use HybridACZ for server packaging. + /// + public bool HybridAcz { get; set; } + + // CommandLineArgs, 3rd of her name. + public static bool TryParse(IReadOnlyList args, [NotNullWhen(true)] out CommandLineArgs? parsed) + { + parsed = null; + bool? client = null; + var skipBuild = false; + var wipeRelease = true; + var hybridAcz = false; + List? platforms = null; + + using var enumerator = args.GetEnumerator(); + var i = -1; + + while (enumerator.MoveNext()) + { + i++; + var arg = enumerator.Current; + if (i == 0) + { + if (arg == "client") + { + client = true; + } + else if (arg == "server") + { + client = false; + } + else + { + return false; + } + + continue; + } + + if (arg == "--skip-build") + { + skipBuild = true; + } + else if (arg == "--no-wipe-release") + { + wipeRelease = false; + } + else if (arg == "--hybrid-acz") + { + hybridAcz = true; + } + else if (arg == "--platform") + { + if (!enumerator.MoveNext()) + { + Console.WriteLine("No platform provided"); + return false; + } + + platforms ??= new List(); + platforms.Add(enumerator.Current); + } + else if (arg == "--help") + { + PrintHelp(); + return false; + } + else + { + Console.WriteLine("Unknown argument: {0}", arg); + } + } + + if (client == null) + { + Console.WriteLine("Client / server packaging unspecified."); + return false; + } + + parsed = new CommandLineArgs(client.Value, skipBuild, wipeRelease, hybridAcz, platforms); + return true; + } + + private static void PrintHelp() + { + Console.WriteLine(@" +Usage: Content.Packaging [client/server] [options] + +Options: + --skip-build Should we skip building the project and use what's already there. + --no-wipe-release Don't wipe the release folder before creating files. + --hybrid-acz Use HybridACZ for server builds. + --platform Platform for server builds. Default will output several x64 targets. +"); + } + + private CommandLineArgs( + bool client, + bool skipBuild, + bool wipeRelease, + bool hybridAcz, + List? platforms) + { + Client = client; + SkipBuild = skipBuild; + WipeRelease = wipeRelease; + HybridAcz = hybridAcz; + Platforms = platforms; + } +} diff --git a/Content.Packaging/Content.Packaging.csproj b/Content.Packaging/Content.Packaging.csproj index 1b5acec3fd..82edfb4add 100644 --- a/Content.Packaging/Content.Packaging.csproj +++ b/Content.Packaging/Content.Packaging.csproj @@ -5,6 +5,9 @@ enable + + + diff --git a/Content.Packaging/ContentPackaging.cs b/Content.Packaging/ContentPackaging.cs deleted file mode 100644 index 0f8c7c747b..0000000000 --- a/Content.Packaging/ContentPackaging.cs +++ /dev/null @@ -1,32 +0,0 @@ -using Robust.Packaging; -using Robust.Packaging.AssetProcessing; - -namespace Content.Packaging; - -public static class ContentPackaging -{ - public static async Task WriteResources( - string contentDir, - AssetPass pass, - IPackageLogger logger, - CancellationToken cancel) - { - var graph = new RobustClientAssetGraph(); - pass.Dependencies.Add(new AssetPassDependency(graph.Output.Name)); - - AssetGraph.CalculateGraph(graph.AllPasses.Append(pass).ToArray(), logger); - - var inputPass = graph.Input; - - await RobustClientPackaging.WriteContentAssemblies( - inputPass, - contentDir, - "Content.Client", - new[] { "Content.Client", "Content.Shared", "Content.Shared.Database" }, - cancel); - - await RobustClientPackaging.WriteClientResources(contentDir, inputPass, cancel); - - inputPass.InjectFinished(); - } -} diff --git a/Content.Packaging/Program.cs b/Content.Packaging/Program.cs index f965ec9995..ba5924ec3e 100644 --- a/Content.Packaging/Program.cs +++ b/Content.Packaging/Program.cs @@ -1,68 +1,44 @@ -using System.Diagnostics; -using System.IO.Compression; -using Content.Packaging; +using Content.Packaging; using Robust.Packaging; -using Robust.Packaging.AssetProcessing.Passes; -using Robust.Packaging.Utility; -using Robust.Shared.Timing; IPackageLogger logger = new PackageLoggerConsole(); -logger.Info("Clearing release/ directory"); -Directory.CreateDirectory("release"); - -var skipBuild = args.Contains("--skip-build"); - -if (!skipBuild) - WipeBin(); - -await Build(skipBuild); - -async Task Build(bool skipBuild) +if (!CommandLineArgs.TryParse(args, out var parsed)) { - logger.Info("Building project..."); - - if (!skipBuild) - { - await ProcessHelpers.RunCheck(new ProcessStartInfo - { - FileName = "dotnet", - ArgumentList = - { - "build", - Path.Combine("Content.Client", "Content.Client.csproj"), - "-c", "Release", - "--nologo", - "/v:m", - "/t:Rebuild", - "/p:FullRelease=true", - "/m" - } - }); - } - - logger.Info("Packaging client..."); - - var sw = RStopwatch.StartNew(); - - { - using var zipFile = - File.Open(Path.Combine("release", "SS14.Client.zip"), FileMode.Create, FileAccess.ReadWrite); - using var zip = new ZipArchive(zipFile, ZipArchiveMode.Update); - var writer = new AssetPassZipWriter(zip); - - await ContentPackaging.WriteResources("", writer, logger, default); - - await writer.FinishedTask; - } - - logger.Info($"Finished packaging in {sw.Elapsed}"); + logger.Error("Unable to parse args, aborting."); + return; } +if (parsed.WipeRelease) + WipeRelease(); + +if (!parsed.SkipBuild) + WipeBin(); + +if (parsed.Client) +{ + await ClientPackaging.PackageClient(parsed.SkipBuild, logger); +} +else +{ + await ServerPackaging.PackageServer(parsed.SkipBuild, parsed.HybridAcz, logger, parsed.Platforms); +} void WipeBin() { logger.Info("Clearing old build artifacts (if any)..."); - Directory.Delete("bin", recursive: true); + if (Directory.Exists("bin")) + Directory.Delete("bin", recursive: true); +} + +void WipeRelease() +{ + if (Directory.Exists("release")) + { + logger.Info("Cleaning old release packages (release/)..."); + Directory.Delete("release", recursive: true); + } + + Directory.CreateDirectory("release"); } diff --git a/Content.Packaging/ServerPackaging.cs b/Content.Packaging/ServerPackaging.cs new file mode 100644 index 0000000000..d75b425561 --- /dev/null +++ b/Content.Packaging/ServerPackaging.cs @@ -0,0 +1,226 @@ +using System.Diagnostics; +using System.Globalization; +using System.IO.Compression; +using Robust.Packaging; +using Robust.Packaging.AssetProcessing; +using Robust.Packaging.AssetProcessing.Passes; +using Robust.Packaging.Utility; +using Robust.Shared.Audio; +using Robust.Shared.Serialization; +using Robust.Shared.Timing; +using YamlDotNet.Core; +using YamlDotNet.RepresentationModel; + +namespace Content.Packaging; + +public static class ServerPackaging +{ + private static readonly List Platforms = new() + { + new PlatformReg("win-x64", "Windows", true), + new PlatformReg("linux-x64", "Linux", true), + new PlatformReg("linux-arm64", "Linux", true), + new PlatformReg("osx-x64", "MacOS", true), + // Non-default platforms (i.e. for Watchdog Git) + new PlatformReg("win-x86", "Windows", false), + new PlatformReg("linux-x86", "Linux", false), + new PlatformReg("linux-arm", "Linux", false), + }; + + private static List PlatformRids => Platforms + .Select(o => o.Rid) + .ToList(); + + private static List PlatformRidsDefault => Platforms + .Where(o => o.BuildByDefault) + .Select(o => o.Rid) + .ToList(); + + private static readonly List ServerContentAssemblies = new() + { + "Content.Server.Database", + "Content.Server", + "Content.Shared", + "Content.Shared.Database", + }; + + private static readonly List ServerExtraAssemblies = new() + { + // Python script had Npgsql. though we want Npgsql.dll as well soooo + "Npgsql", + "Microsoft", + }; + + private static readonly List ServerNotExtraAssemblies = new() + { + "Microsoft.CodeAnalysis", + }; + + private static readonly HashSet BinSkipFolders = new() + { + // Roslyn localization files, screw em. + "cs", + "de", + "es", + "fr", + "it", + "ja", + "ko", + "pl", + "pt-BR", + "ru", + "tr", + "zh-Hans", + "zh-Hant" + }; + + public static async Task PackageServer(bool skipBuild, bool hybridAcz, IPackageLogger logger, List? platforms = null) + { + if (platforms == null) + { + platforms ??= PlatformRidsDefault; + } + + if (hybridAcz) + { + // Hybrid ACZ involves a file "Content.Client.zip" in the server executable directory. + // Rather than hosting the client ZIP on the watchdog or on a separate server, + // Hybrid ACZ uses the ACZ hosting functionality to host it as part of the status host, + // which means that features such as automatic UPnP forwarding still work properly. + await ClientPackaging.PackageClient(skipBuild, logger); + } + + // Good variable naming right here. + foreach (var platform in Platforms) + { + if (!platforms.Contains(platform.Rid)) + continue; + + await BuildPlatform(platform, skipBuild, hybridAcz, logger); + } + } + + private static async Task BuildPlatform(PlatformReg platform, bool skipBuild, bool hybridAcz, IPackageLogger logger) + { + logger.Info($"Building project for {platform}..."); + + if (!skipBuild) + { + await ProcessHelpers.RunCheck(new ProcessStartInfo + { + FileName = "dotnet", + ArgumentList = + { + "build", + Path.Combine("Content.Server", "Content.Server.csproj"), + "-c", "Release", + "--nologo", + "/v:m", + $"/p:TargetOs={platform.TargetOs}", + "/t:Rebuild", + "/p:FullRelease=true", + "/m" + } + }); + + await PublishClientServer(platform.Rid, platform.TargetOs); + } + + logger.Info($"Packaging {platform.Rid} server..."); + + var sw = RStopwatch.StartNew(); + { + await using var zipFile = + File.Open(Path.Combine("release", $"SS14.Server_{platform.Rid}.zip"), FileMode.Create, FileAccess.ReadWrite); + using var zip = new ZipArchive(zipFile, ZipArchiveMode.Update); + var writer = new AssetPassZipWriter(zip); + + await WriteServerResources(platform, "", writer, logger, hybridAcz, default); + await writer.FinishedTask; + } + + logger.Info($"Finished packaging server in {sw.Elapsed}"); + } + + private static async Task PublishClientServer(string runtime, string targetOs) + { + await ProcessHelpers.RunCheck(new ProcessStartInfo + { + FileName = "dotnet", + ArgumentList = + { + "publish", + "--runtime", runtime, + "--no-self-contained", + "-c", "Release", + $"/p:TargetOs={targetOs}", + "/p:FullRelease=True", + "/m", + "RobustToolbox/Robust.Server/Robust.Server.csproj" + } + }); + } + + private static async Task WriteServerResources( + PlatformReg platform, + string contentDir, + AssetPass pass, + IPackageLogger logger, + bool hybridAcz, + CancellationToken cancel) + { + var graph = new RobustClientAssetGraph(); + var passes = graph.AllPasses.ToList(); + + pass.Dependencies.Add(new AssetPassDependency(graph.Output.Name)); + passes.Add(pass); + + AssetGraph.CalculateGraph(passes, logger); + + var inputPass = graph.Input; + var contentAssemblies = new List(ServerContentAssemblies); + + // Additional assemblies that need to be copied such as EFCore. + var sourcePath = Path.Combine(contentDir, "bin", "Content.Server"); + + // Should this be an asset pass? + // For future archaeologists I just want audio rework to work and need the audio pass so + // just porting this as is from python. + foreach (var fullPath in Directory.EnumerateFiles(sourcePath, "*.*", SearchOption.AllDirectories)) + { + var fileName = Path.GetFileNameWithoutExtension(fullPath); + + if (!ServerNotExtraAssemblies.Any(o => fileName.StartsWith(o)) && ServerExtraAssemblies.Any(o => fileName.StartsWith(o))) + { + contentAssemblies.Add(fileName); + } + } + + await RobustSharedPackaging.DoResourceCopy( + Path.Combine("RobustToolbox", "bin", "Server", + platform.Rid, + "publish"), + inputPass, + BinSkipFolders, + cancel: cancel); + + await RobustSharedPackaging.WriteContentAssemblies( + inputPass, + contentDir, + "Content.Server", + contentAssemblies, + Path.Combine("Resources", "Assemblies"), + cancel); + + await RobustServerPackaging.WriteServerResources(contentDir, inputPass, cancel); + + if (hybridAcz) + { + inputPass.InjectFileFromDisk("Content.Client.zip", Path.Combine("release", "SS14.Client.zip")); + } + + inputPass.InjectFinished(); + } + + private readonly record struct PlatformReg(string Rid, string TargetOs, bool BuildByDefault); +} diff --git a/Content.Server/Acz/ContentMagicAczProvider.cs b/Content.Server/Acz/ContentMagicAczProvider.cs index ffd3123c57..57b5c1ec56 100644 --- a/Content.Server/Acz/ContentMagicAczProvider.cs +++ b/Content.Server/Acz/ContentMagicAczProvider.cs @@ -20,6 +20,6 @@ public sealed class ContentMagicAczProvider : IMagicAczProvider { var contentDir = DefaultMagicAczProvider.FindContentRootPath(_deps); - await ContentPackaging.WriteResources(contentDir, pass, logger, cancel); + await ClientPackaging.WriteResources(contentDir, pass, logger, cancel); } } diff --git a/SpaceStation14.sln b/SpaceStation14.sln index a94daa316d..10c4ea1c2c 100644 --- a/SpaceStation14.sln +++ b/SpaceStation14.sln @@ -63,8 +63,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Build", "Build", "{806ED41A ProjectSection(SolutionItems) = preProject Tools\gen_build_info.py = Tools\gen_build_info.py Tools\generate_hashes.ps1 = Tools\generate_hashes.ps1 - Tools\package_client_build.py = Tools\package_client_build.py - Tools\package_server_build.py = Tools\package_server_build.py EndProjectSection EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Robust.Shared.Scripting", "RobustToolbox\Robust.Shared.Scripting\Robust.Shared.Scripting.csproj", "{41B450C0-A361-4CD7-8121-7072B8995CFC}" diff --git a/Tools/package_client_build.py b/Tools/package_client_build.py old mode 100755 new mode 100644 diff --git a/Tools/package_server_build.py b/Tools/package_server_build.py old mode 100755 new mode 100644 index 32a61722f6..78ef15d8d0 --- a/Tools/package_server_build.py +++ b/Tools/package_server_build.py @@ -286,4 +286,4 @@ def copy_content_assemblies(target, zipf): if __name__ == '__main__': - main() + main() \ No newline at end of file