Displacement map prototype (#26709)

Requires https://github.com/space-wizards/RobustToolbox/pull/5023

This uses the new engine features (above) to add a displacement map shader. This allows deforming a sprite based on another sprite.

Primary use case is automatically adapting human clothing sprites to different species, something we want to make species like Vox a reality.

A basic example of wiring this up with Vox has been added. The system is however incredibly simple and **will** need more work by a content developer to select and toggle displacement maps when appropriate. I am leaving that to somebody else. For example right now the displacement map is applied even if a species already has custom-fit sprites for a piece of clothing, such as the grey jumpsuit for Vox.

Basic Aseprite plugins to help with authoring displacement maps have also been made.

Content.Client/Clothing/ClientClothingSystem.cs

@@ -11,6 +11,7 @@ using Content.Shared.Item;
 using Robust.Client.GameObjects;
 using Robust.Client.Graphics;
 using Robust.Client.ResourceManagement;
+using Robust.Shared.Serialization.Manager;
 using Robust.Shared.Serialization.TypeSerializers.Implementations;
 using Robust.Shared.Utility;
 using static Robust.Client.GameObjects.SpriteComponent;
@@ -46,6 +47,7 @@ public sealed class ClientClothingSystem : ClothingSystem
     };
 
     [Dependency] private readonly IResourceCache _cache = default!;
+    [Dependency] private readonly ISerializationManager _serialization = default!;
     [Dependency] private readonly InventorySystem _inventorySystem = default!;
 
     public override void Initialize()
@@ -265,6 +267,7 @@ public sealed class ClientClothingSystem : ClothingSystem
         // temporary, until layer draw depths get added. Basically: a layer with the key "slot" is being used as a
         // bookmark to determine where in the list of layers we should insert the clothing layers.
         bool slotLayerExists = sprite.LayerMapTryGet(slot, out var index);
+        var displacementData = inventory.Displacements.GetValueOrDefault(slot);
 
         // add the new layers
         foreach (var (key, layerData) in ev.Layers)
@@ -304,10 +307,29 @@ public sealed class ClientClothingSystem : ClothingSystem
             // Sprite layer redactor when
             // Sprite "redactor" just a week away.
             if (slot == Jumpsuit)
-                layerData.Shader ??= "StencilDraw";
+                layerData.Shader ??= inventory.JumpsuitShader;
 
             sprite.LayerSetData(index, layerData);
             layer.Offset += slotDef.Offset;
+
+            if (displacementData != null)
+            {
+                var displacementKey = $"{key}-displacement";
+                if (!revealedLayers.Add(displacementKey))
+                {
+                    Log.Warning($"Duplicate key for clothing visuals DISPLACEMENT: {displacementKey}.");
+                    continue;
+                }
+
+                var displacementLayer = _serialization.CreateCopy(displacementData.Layer, notNullableOverride: true);
+                displacementLayer.CopyToShaderParameters!.LayerKey = key;
+
+                // Add before main layer for this item.
+                sprite.AddLayer(displacementLayer, index);
+                sprite.LayerMapSet(displacementKey, index);
+
+                revealedLayers.Add(displacementKey);
+            }
         }
 
         RaiseLocalEvent(equipment, new EquipmentVisualsUpdatedEvent(equipee, slot, revealedLayers), true);

Content.Shared/Inventory/InventoryComponent.cs

@@ -13,6 +13,16 @@ public sealed partial class InventoryComponent : Component
 
     [DataField("speciesId")] public string? SpeciesId { get; set; }
 
+    [DataField] public string JumpsuitShader = "StencilDraw";
+    [DataField] public Dictionary<string, SlotDisplacementData> Displacements = [];
+
     public SlotDefinition[] Slots = Array.Empty<SlotDefinition>();
     public ContainerSlot[] Containers = Array.Empty<ContainerSlot>();
+
+    [DataDefinition]
+    public sealed partial class SlotDisplacementData
+    {
+        [DataField(required: true)]
+        public PrototypeLayerData Layer = default!;
+    }
 }

Resources/Prototypes/Entities/Mobs/Species/vox.yml

@@ -16,6 +16,17 @@
     #- type: VoxAccent # Not yet coded
   - type: Inventory
     speciesId: vox
+    jumpsuitShader: DisplacedStencilDraw
+    displacements:
+      jumpsuit:
+        layer:
+          sprite: Mobs/Species/Vox/displacement.rsi
+          state: jumpsuit
+          copyToShaderParameters:
+            # Value required, provide a dummy. Gets overridden when applied.
+            layerKey: dummy
+            parameterTexture: displacementMap
+            parameterUV: displacementUV
   - type: Speech
     speechVerb: Vox
     speechSounds: Vox

Resources/Prototypes/Shaders/displacement.yml

@@ -0,0 +1,10 @@
+- type: shader
+  id: DisplacedStencilDraw
+  kind: source
+  path: "/Textures/Shaders/displacement.swsl"
+  stencil:
+    ref: 1
+    op: Keep
+    func: NotEqual
+  params:
+    displacementSize: 127

Resources/Textures/Mobs/Species/Vox/displacement.rsi/meta.json

@@ -0,0 +1,18 @@
+{
+    "version": 1,
+    "license": "CC-BY-SA-3.0",
+    "copyright": "Made by PJB3005",
+    "size": {
+        "x": 32,
+        "y": 32
+    },
+    "load": {
+        "srgb": false
+    },
+    "states": [
+        {
+            "name": "jumpsuit",
+            "directions": 4
+        }
+    ]
+}

Resources/Textures/Shaders/displacement.swsl

@@ -0,0 +1,18 @@
+uniform sampler2D displacementMap;
+uniform highp float displacementSize;
+uniform highp vec4 displacementUV;
+
+varying highp vec2 displacementUVOut;
+
+void vertex() {
+    displacementUVOut = mix(displacementUV.xy, displacementUV.zw, tCoord2);
+}
+
+void fragment() {
+    highp vec4 displacementSample = texture2D(displacementMap, displacementUVOut);
+    highp vec2 displacementValue = (displacementSample.xy - vec2(128.0 / 255.0)) / (1.0 - 128.0 / 255.0);
+    COLOR = zTexture(UV + displacementValue * TEXTURE_PIXEL_SIZE * displacementSize * vec2(1.0, -1.0));
+    COLOR.a *= displacementSample.a;
+}
+
+

Tools/SS14 Aseprite Plugins/Displacement Map Flip.lua

@@ -0,0 +1,78 @@
+local sprite = app.editor.sprite
+local cel = app.cel
+
+if sprite.selection.isEmpty then
+    print("You need to select something sorry")
+    return
+end
+
+local diag = Dialog{
+    title = "Flip Displacement Map"
+}
+
+diag:check{
+    id = "horizontal",
+    label = "flip horizontal?"
+}
+
+diag:check{
+    id = "vertical",
+    label = "flip vertical?"
+}
+
+diag:button{
+    text = "ok",
+    focus = true,
+    onclick = function(ev)
+        local horizontal = diag.data["horizontal"]
+        local vertical = diag.data["vertical"]
+
+        local selection = sprite.selection
+        local image = cel.image:clone()
+
+        for x = 0, selection.bounds.width do
+            for y = 0, selection.bounds.height do
+                local xSel = x + selection.origin.x
+                local ySel = y + selection.origin.y
+
+                local xImg = xSel - cel.position.x
+                local yImg = ySel - cel.position.y
+
+                if xImg < 0 or xImg >= image.width or yImg < 0 or yImg >= image.height then
+                    goto continue
+                end
+
+                local imgValue = image:getPixel(xImg, yImg)
+                local color = Color(imgValue)
+
+                if horizontal then
+                    color.red = 128 + -(color.red - 128)
+                end
+
+                if vertical then
+                    color.green = 128 + -(color.green - 128)
+                end
+
+                image:drawPixel(
+                    xImg,
+                    yImg,
+                    app.pixelColor.rgba(color.red, color.green, color.blue, color.alpha))
+
+                ::continue::
+            end
+        end
+
+        cel.image = image
+
+        diag:close()
+    end
+}
+
+diag:button{
+    text = "cancel",
+    onclick = function(ev)
+        diag:close()
+    end
+}
+
+diag:show()

Tools/SS14 Aseprite Plugins/Displacement Map Visualizer.lua

@@ -0,0 +1,135 @@
+-- Displacement Map Visualizer
+--
+-- This script will create a little preview window that will test a displacement map.
+--
+-- TODO: Handling of sizes != 127 doesn't work properly and rounds differently from the real shader. Ah well.
+
+local scale = 4
+
+-- This script requires UI
+if not app.isUIAvailable then
+    return
+end
+
+local getOffsetPixel = function(x, y, image, rect)
+    local posX = x - rect.x
+    local posY = y - rect.y
+
+    if posX < 0 or posX >= image.width or posY < 0 or posY >= image.height then
+        return image.spec.transparentColor
+    end
+
+    return image:getPixel(posX, posY)
+end
+
+local pixelValueToColor = function(sprite, value)
+    return Color(value)
+end
+
+local applyDisplacementMap = function(width, height, size, displacement, displacementRect, target, targetRect)
+    -- print(Color(displacement:getPixel(17, 15)).red)
+    local image = target:clone()
+    image:resize(width, height)
+    image:clear()
+
+    for x = 0, width - 1 do
+        for y = 0, height - 1 do
+            local value = getOffsetPixel(x, y, displacement, displacementRect)
+            local color = pixelValueToColor(sprite, value)
+
+            if color.alpha ~= 0 then
+                local offset_x = (color.red - 128) / 127 * size
+                local offset_y = (color.green - 128) / 127 * size
+
+                local colorValue = getOffsetPixel(x + offset_x, y + offset_y, target, targetRect)
+                image:drawPixel(x, y, colorValue)
+            end
+        end
+    end
+
+    return image
+end
+
+local dialog = nil
+
+local sprite = app.editor.sprite
+local spriteChanged = sprite.events:on("change",
+    function(ev)
+        dialog:repaint()
+    end)
+
+local layers = {}
+for i,layer in ipairs(sprite.layers) do
+    table.insert(layers, 1, layer.name)
+end
+
+local findLayer = function(sprite, name)
+    for i, layer in ipairs(sprite.layers) do
+        if layer.name == name then
+            return layer
+        end
+    end
+
+    return nil
+end
+
+dialog = Dialog{
+    title = "Displacement map preview",
+    onclose = function(ev)
+        sprite.events:off(spriteChanged)
+    end}
+
+dialog:canvas{
+    id = "canvas",
+    width = sprite.width * scale,
+    height = sprite.height * scale,
+    onpaint = function(ev)
+        local context = ev.context
+
+        local layerDisplacement = findLayer(sprite, dialog.data["displacement-select"])
+        local layerTarget = findLayer(sprite, dialog.data["reference-select"])
+        -- print(layerDisplacement.name)
+        -- print(layerTarget.name)
+        local celDisplacement = layerDisplacement:cel(1)
+        local celTarget = layerTarget:cel(1)
+        local image = applyDisplacementMap(
+            sprite.width, sprite.height,
+            dialog.data["size"],
+            celDisplacement.image, celDisplacement.bounds,
+            celTarget.image, celTarget.bounds)
+
+        context:drawImage(image, 0, 0, image.width, image.height, 0, 0, image.width * scale, context.width, context.height)
+    end
+}
+
+dialog:combobox{
+    id = "displacement-select",
+    label = "displacement layer",
+    options = layers,
+    onchange = function(ev)
+        dialog:repaint()
+    end
+}
+
+dialog:combobox{
+    id = "reference-select",
+    label = "reference layer",
+    options = layers,
+    onchange = function(ev)
+        dialog:repaint()
+    end
+}
+
+
+dialog:slider{
+    id = "size",
+    label = "displacement size",
+    min = 1,
+    max = 127,
+    value = 127,
+    onchange = function(ev)
+        dialog:repaint()
+    end
+}
+
+dialog:show{wait = false}