tommy/tbd-station-14
2
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Port python packaging to Content.Packaging (#21458)

Browse Source
This commit is contained in:
metalgearsloth
2023-11-07 09:53:59 +11:00
committed by GitHub
parent df46c52dd7
commit b9c38879e5
12 changed files with 498 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
.github/workflows/publish.yml vendored
View File

@@ -29,10 +29,17 @@ jobs:
cd RobustToolbox
git fetch --depth=1
- name: Install dependencies
run: dotnet restore
- name: Build Packaging
run: dotnet build Content.Packaging --configuration Release --no-restore /m
- name: Package server
run: dotnet run --project Content.Packaging server --platform win-x64 --platform linux-x64 --platform osx-x64 --platform linux-arm64
- name: Package client
run: |
Tools/package_server_build.py -p win-x64 linux-x64 osx-x64 linux-arm64
Tools/package_client_build.py
run: dotnet run --project Content.Packaging client --no-wipe-release
- name: Update Build Info
run: Tools/gen_build_info.py

12
.github/workflows/test-packaging.yml vendored
View File

@@ -56,11 +56,15 @@ jobs:
- name: Install dependencies
run: dotnet restore
- name: Package client
run: |
Tools/package_server_build.py -p win-x64 linux-x64 osx-x64 linux-arm64
Tools/package_client_build.py
- name: Build Packaging
run: dotnet build Content.Packaging --configuration Release --no-restore /m
- name: Package server
run: dotnet run --project Content.Packaging server --platform win-x64 --platform linux-x64 --platform osx-x64 --platform linux-arm64
- name: Package client
run: dotnet run --project Content.Packaging client --no-wipe-release
- name: Update Build Info
run: Tools/gen_build_info.py

79
Content.Packaging/ClientPackaging.cs Normal file
View File

@@ -0,0 +1,79 @@
using System.Diagnostics;
using System.IO.Compression;
using Robust.Packaging;
using Robust.Packaging.AssetProcessing;
using Robust.Packaging.AssetProcessing.Passes;
using Robust.Packaging.Utility;
using Robust.Shared.Timing;
namespace Content.Packaging;
public static class ClientPackaging
{
/// <summary>
/// Be advised this can be called from server packaging during a HybridACZ build.
/// </summary>
public static async Task PackageClient(bool skipBuild, IPackageLogger logger)
{
logger.Info("Building client...");
if (!skipBuild)
{
await ProcessHelpers.RunCheck(new ProcessStartInfo
{
FileName = "dotnet",
ArgumentList =
{
"build",
Path.Combine("Content.Client", "Content.Client.csproj"),
"-c", "Release",
"--nologo",
"/v:m",
"/t:Rebuild",
"/p:FullRelease=true",
"/m"
}
});
}
logger.Info("Packaging client...");
var sw = RStopwatch.StartNew();
{
await using var zipFile =
File.Open(Path.Combine("release", "SS14.Client.zip"), FileMode.Create, FileAccess.ReadWrite);
using var zip = new ZipArchive(zipFile, ZipArchiveMode.Update);
var writer = new AssetPassZipWriter(zip);
await WriteResources("", writer, logger, default);
await writer.FinishedTask;
}
logger.Info($"Finished packaging client in {sw.Elapsed}");
}
public static async Task WriteResources(
string contentDir,
AssetPass pass,
IPackageLogger logger,
CancellationToken cancel)
{
var graph = new RobustClientAssetGraph();
pass.Dependencies.Add(new AssetPassDependency(graph.Output.Name));
AssetGraph.CalculateGraph(graph.AllPasses.Append(pass).ToArray(), logger);
var inputPass = graph.Input;
await RobustSharedPackaging.WriteContentAssemblies(
inputPass,
contentDir,
"Content.Client",
new[] { "Content.Client", "Content.Shared", "Content.Shared.Database" },
cancel: cancel);
await RobustClientPackaging.WriteClientResources(contentDir, pass, cancel);
inputPass.InjectFinished();
}
}

139
Content.Packaging/CommandLineArgs.cs Normal file
View File

@@ -0,0 +1,139 @@
using System.Diagnostics.CodeAnalysis;
namespace Content.Packaging;
public sealed class CommandLineArgs
{
// PJB forgib me
/// <summary>
/// Generate client or server.
/// </summary>
public bool Client { get; set; }
/// <summary>
/// Should we also build the relevant project.
/// </summary>
public bool SkipBuild { get; set; }
/// <summary>
/// Should we wipe the release folder or ignore it.
/// </summary>
public bool WipeRelease { get; set; }
/// <summary>
/// Platforms for server packaging.
/// </summary>
public List<string>? Platforms { get; set; }
/// <summary>
/// Use HybridACZ for server packaging.
/// </summary>
public bool HybridAcz { get; set; }
// CommandLineArgs, 3rd of her name.
public static bool TryParse(IReadOnlyList<string> args, [NotNullWhen(true)] out CommandLineArgs? parsed)
{
parsed = null;
bool? client = null;
var skipBuild = false;
var wipeRelease = true;
var hybridAcz = false;
List<string>? platforms = null;
using var enumerator = args.GetEnumerator();
var i = -1;
while (enumerator.MoveNext())
{
i++;
var arg = enumerator.Current;
if (i == 0)
{
if (arg == "client")
{
client = true;
}
else if (arg == "server")
{
client = false;
}
else
{
return false;
}
continue;
}
if (arg == "--skip-build")
{
skipBuild = true;
}
else if (arg == "--no-wipe-release")
{
wipeRelease = false;
}
else if (arg == "--hybrid-acz")
{
hybridAcz = true;
}
else if (arg == "--platform")
{
if (!enumerator.MoveNext())
{
Console.WriteLine("No platform provided");
return false;
}
platforms ??= new List<string>();
platforms.Add(enumerator.Current);
}
else if (arg == "--help")
{
PrintHelp();
return false;
}
else
{
Console.WriteLine("Unknown argument: {0}", arg);
}
}
if (client == null)
{
Console.WriteLine("Client / server packaging unspecified.");
return false;
}
parsed = new CommandLineArgs(client.Value, skipBuild, wipeRelease, hybridAcz, platforms);
return true;
}
private static void PrintHelp()
{
Console.WriteLine(@"
Usage: Content.Packaging [client/server] [options]
Options:
--skip-build Should we skip building the project and use what's already there.
--no-wipe-release Don't wipe the release folder before creating files.
--hybrid-acz Use HybridACZ for server builds.
--platform Platform for server builds. Default will output several x64 targets.
");
}
private CommandLineArgs(
bool client,
bool skipBuild,
bool wipeRelease,
bool hybridAcz,
List<string>? platforms)
{
Client = client;
SkipBuild = skipBuild;
WipeRelease = wipeRelease;
HybridAcz = hybridAcz;
Platforms = platforms;
}
}

3
Content.Packaging/Content.Packaging.csproj
View File

@@ -5,6 +5,9 @@
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="NVorbis" Version="0.10.1" PrivateAssets="compile" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\RobustToolbox\Robust.Packaging\Robust.Packaging.csproj" />
</ItemGroup>

32
Content.Packaging/ContentPackaging.cs
View File

@@ -1,32 +0,0 @@
using Robust.Packaging;
using Robust.Packaging.AssetProcessing;
namespace Content.Packaging;
public static class ContentPackaging
{
public static async Task WriteResources(
string contentDir,
AssetPass pass,
IPackageLogger logger,
CancellationToken cancel)
{
var graph = new RobustClientAssetGraph();
pass.Dependencies.Add(new AssetPassDependency(graph.Output.Name));
AssetGraph.CalculateGraph(graph.AllPasses.Append(pass).ToArray(), logger);
var inputPass = graph.Input;
await RobustClientPackaging.WriteContentAssemblies(
inputPass,
contentDir,
"Content.Client",
new[] { "Content.Client", "Content.Shared", "Content.Shared.Database" },
cancel);
await RobustClientPackaging.WriteClientResources(contentDir, inputPass, cancel);
inputPass.InjectFinished();
}
}

86
Content.Packaging/Program.cs
View File

@@ -1,68 +1,44 @@
using System.Diagnostics;
using System.IO.Compression;
using Content.Packaging;
using Content.Packaging;
using Robust.Packaging;
using Robust.Packaging.AssetProcessing.Passes;
using Robust.Packaging.Utility;
using Robust.Shared.Timing;
IPackageLogger logger = new PackageLoggerConsole();
logger.Info("Clearing release/ directory");
Directory.CreateDirectory("release");
var skipBuild = args.Contains("--skip-build");
if (!skipBuild)
WipeBin();
await Build(skipBuild);
async Task Build(bool skipBuild)
if (!CommandLineArgs.TryParse(args, out var parsed))
{
logger.Info("Building project...");
if (!skipBuild)
{
await ProcessHelpers.RunCheck(new ProcessStartInfo
{
FileName = "dotnet",
ArgumentList =
{
"build",
Path.Combine("Content.Client", "Content.Client.csproj"),
"-c", "Release",
"--nologo",
"/v:m",
"/t:Rebuild",
"/p:FullRelease=true",
"/m"
}
});
}
logger.Info("Packaging client...");
var sw = RStopwatch.StartNew();
{
using var zipFile =
File.Open(Path.Combine("release", "SS14.Client.zip"), FileMode.Create, FileAccess.ReadWrite);
using var zip = new ZipArchive(zipFile, ZipArchiveMode.Update);
var writer = new AssetPassZipWriter(zip);
await ContentPackaging.WriteResources("", writer, logger, default);
await writer.FinishedTask;
}
logger.Info($"Finished packaging in {sw.Elapsed}");
logger.Error("Unable to parse args, aborting.");
return;
}
if (parsed.WipeRelease)
WipeRelease();
if (!parsed.SkipBuild)
WipeBin();
if (parsed.Client)
{
await ClientPackaging.PackageClient(parsed.SkipBuild, logger);
}
else
{
await ServerPackaging.PackageServer(parsed.SkipBuild, parsed.HybridAcz, logger, parsed.Platforms);
}
void WipeBin()
{
logger.Info("Clearing old build artifacts (if any)...");
Directory.Delete("bin", recursive: true);
if (Directory.Exists("bin"))
Directory.Delete("bin", recursive: true);
}
void WipeRelease()
{
if (Directory.Exists("release"))
{
logger.Info("Cleaning old release packages (release/)...");
Directory.Delete("release", recursive: true);
}
Directory.CreateDirectory("release");
}

226
Content.Packaging/ServerPackaging.cs Normal file
View File

@@ -0,0 +1,226 @@
using System.Diagnostics;
using System.Globalization;
using System.IO.Compression;
using Robust.Packaging;
using Robust.Packaging.AssetProcessing;
using Robust.Packaging.AssetProcessing.Passes;
using Robust.Packaging.Utility;
using Robust.Shared.Audio;
using Robust.Shared.Serialization;
using Robust.Shared.Timing;
using YamlDotNet.Core;
using YamlDotNet.RepresentationModel;
namespace Content.Packaging;
public static class ServerPackaging
{
private static readonly List<PlatformReg> Platforms = new()
{
new PlatformReg("win-x64", "Windows", true),
new PlatformReg("linux-x64", "Linux", true),
new PlatformReg("linux-arm64", "Linux", true),
new PlatformReg("osx-x64", "MacOS", true),
// Non-default platforms (i.e. for Watchdog Git)
new PlatformReg("win-x86", "Windows", false),
new PlatformReg("linux-x86", "Linux", false),
new PlatformReg("linux-arm", "Linux", false),
};
private static List<string> PlatformRids => Platforms
.Select(o => o.Rid)
.ToList();
private static List<string> PlatformRidsDefault => Platforms
.Where(o => o.BuildByDefault)
.Select(o => o.Rid)
.ToList();
private static readonly List<string> ServerContentAssemblies = new()
{
"Content.Server.Database",
"Content.Server",
"Content.Shared",
"Content.Shared.Database",
};
private static readonly List<string> ServerExtraAssemblies = new()
{
// Python script had Npgsql. though we want Npgsql.dll as well soooo
"Npgsql",
"Microsoft",
};
private static readonly List<string> ServerNotExtraAssemblies = new()
{
"Microsoft.CodeAnalysis",
};
private static readonly HashSet<string> BinSkipFolders = new()
{
// Roslyn localization files, screw em.
"cs",
"de",
"es",
"fr",
"it",
"ja",
"ko",
"pl",
"pt-BR",
"ru",
"tr",
"zh-Hans",
"zh-Hant"
};
public static async Task PackageServer(bool skipBuild, bool hybridAcz, IPackageLogger logger, List<string>? platforms = null)
{
if (platforms == null)
{
platforms ??= PlatformRidsDefault;
}
if (hybridAcz)
{
// Hybrid ACZ involves a file "Content.Client.zip" in the server executable directory.
// Rather than hosting the client ZIP on the watchdog or on a separate server,
// Hybrid ACZ uses the ACZ hosting functionality to host it as part of the status host,
// which means that features such as automatic UPnP forwarding still work properly.
await ClientPackaging.PackageClient(skipBuild, logger);
}
// Good variable naming right here.
foreach (var platform in Platforms)
{
if (!platforms.Contains(platform.Rid))
continue;
await BuildPlatform(platform, skipBuild, hybridAcz, logger);
}
}
private static async Task BuildPlatform(PlatformReg platform, bool skipBuild, bool hybridAcz, IPackageLogger logger)
{
logger.Info($"Building project for {platform}...");
if (!skipBuild)
{
await ProcessHelpers.RunCheck(new ProcessStartInfo
{
FileName = "dotnet",
ArgumentList =
{
"build",
Path.Combine("Content.Server", "Content.Server.csproj"),
"-c", "Release",
"--nologo",
"/v:m",
$"/p:TargetOs={platform.TargetOs}",
"/t:Rebuild",
"/p:FullRelease=true",
"/m"
}
});
await PublishClientServer(platform.Rid, platform.TargetOs);
}
logger.Info($"Packaging {platform.Rid} server...");
var sw = RStopwatch.StartNew();
{
await using var zipFile =
File.Open(Path.Combine("release", $"SS14.Server_{platform.Rid}.zip"), FileMode.Create, FileAccess.ReadWrite);
using var zip = new ZipArchive(zipFile, ZipArchiveMode.Update);
var writer = new AssetPassZipWriter(zip);
await WriteServerResources(platform, "", writer, logger, hybridAcz, default);
await writer.FinishedTask;
}
logger.Info($"Finished packaging server in {sw.Elapsed}");
}
private static async Task PublishClientServer(string runtime, string targetOs)
{
await ProcessHelpers.RunCheck(new ProcessStartInfo
{
FileName = "dotnet",
ArgumentList =
{
"publish",
"--runtime", runtime,
"--no-self-contained",
"-c", "Release",
$"/p:TargetOs={targetOs}",
"/p:FullRelease=True",
"/m",
"RobustToolbox/Robust.Server/Robust.Server.csproj"
}
});
}
private static async Task WriteServerResources(
PlatformReg platform,
string contentDir,
AssetPass pass,
IPackageLogger logger,
bool hybridAcz,
CancellationToken cancel)
{
var graph = new RobustClientAssetGraph();
var passes = graph.AllPasses.ToList();
pass.Dependencies.Add(new AssetPassDependency(graph.Output.Name));
passes.Add(pass);
AssetGraph.CalculateGraph(passes, logger);
var inputPass = graph.Input;
var contentAssemblies = new List<string>(ServerContentAssemblies);
// Additional assemblies that need to be copied such as EFCore.
var sourcePath = Path.Combine(contentDir, "bin", "Content.Server");
// Should this be an asset pass?
// For future archaeologists I just want audio rework to work and need the audio pass so
// just porting this as is from python.
foreach (var fullPath in Directory.EnumerateFiles(sourcePath, "*.*", SearchOption.AllDirectories))
{
var fileName = Path.GetFileNameWithoutExtension(fullPath);
if (!ServerNotExtraAssemblies.Any(o => fileName.StartsWith(o)) && ServerExtraAssemblies.Any(o => fileName.StartsWith(o)))
{
contentAssemblies.Add(fileName);
}
}
await RobustSharedPackaging.DoResourceCopy(
Path.Combine("RobustToolbox", "bin", "Server",
platform.Rid,
"publish"),
inputPass,
BinSkipFolders,
cancel: cancel);
await RobustSharedPackaging.WriteContentAssemblies(
inputPass,
contentDir,
"Content.Server",
contentAssemblies,
Path.Combine("Resources", "Assemblies"),
cancel);
await RobustServerPackaging.WriteServerResources(contentDir, inputPass, cancel);
if (hybridAcz)
{
inputPass.InjectFileFromDisk("Content.Client.zip", Path.Combine("release", "SS14.Client.zip"));
}
inputPass.InjectFinished();
}
private readonly record struct PlatformReg(string Rid, string TargetOs, bool BuildByDefault);
}

2
Content.Server/Acz/ContentMagicAczProvider.cs
View File

@@ -20,6 +20,6 @@ public sealed class ContentMagicAczProvider : IMagicAczProvider
{
var contentDir = DefaultMagicAczProvider.FindContentRootPath(_deps);
await ContentPackaging.WriteResources(contentDir, pass, logger, cancel);
await ClientPackaging.WriteResources(contentDir, pass, logger, cancel);
}
}

2
SpaceStation14.sln
View File

@@ -63,8 +63,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Build", "Build", "{806ED41A
ProjectSection(SolutionItems) = preProject
Tools\gen_build_info.py = Tools\gen_build_info.py
Tools\generate_hashes.ps1 = Tools\generate_hashes.ps1
Tools\package_client_build.py = Tools\package_client_build.py
Tools\package_server_build.py = Tools\package_server_build.py
EndProjectSection
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Robust.Shared.Scripting", "RobustToolbox\Robust.Shared.Scripting\Robust.Shared.Scripting.csproj", "{41B450C0-A361-4CD7-8121-7072B8995CFC}"

0
Tools/package_client_build.py Executable file → Normal file
View File

2
Tools/package_server_build.py Executable file → Normal file
View File

@@ -286,4 +286,4 @@ def copy_content_assemblies(target, zipf):
if __name__ == '__main__':
main()
main()