diff --git a/src/ShaderRipper.Tests/BundleTests.cs b/src/ShaderRipper.Tests/BundleTests.cs index 267f68b..9ab0865 100644 --- a/src/ShaderRipper.Tests/BundleTests.cs +++ b/src/ShaderRipper.Tests/BundleTests.cs @@ -22,7 +22,6 @@ private sealed class Builder public uint Version = 8; public bool PadTo16 = true; public bool BlocksInfoAtEnd; - public bool CompressInfo; public List<(string path, byte[] data)> Nodes = new(); public byte[] Build() diff --git a/src/ShaderRipper.Tests/SharedPassProgramTests.cs b/src/ShaderRipper.Tests/SharedPassProgramTests.cs new file mode 100644 index 0000000..176b340 --- /dev/null +++ b/src/ShaderRipper.Tests/SharedPassProgramTests.cs @@ -0,0 +1,80 @@ +using System.Collections.Generic; +using System.Linq; +using NUnit.Framework; +using ShaderRipper.Data; +using ShaderRipper.Game; + +namespace ShaderRipper.Tests; + +/// +/// Two passes whose compiled programs are byte-identical share one blob slot in +/// the serialized asset. The pass map must therefore carry every owner per +/// slot: keying a slot by a single pass silently dropped the program from every +/// other pass that declared it. +/// +[TestFixture] +public class SharedPassProgramTests +{ + private static Pass PassWithBlob(int blobIndex, int cbCount) + { + var sp = new SubProgram { BlobIndex = blobIndex }; + for (int i = 0; i < cbCount; i++) + sp.ConstantBuffers.Add(new AssetConstantBuffer + { + Name = $"cb{i}", + Members = { new AssetCbMember { Name = $"m{i}" } }, + }); + return new Pass { SubPrograms = { sp } }; + } + + private static Shader ShaderWith(params Pass[] passes) + { + return new Shader + { + ParsedForm = new ParsedForm + { + Name = "Test/SharedPasses", + SubShaders = { new SubShader { Passes = new List(passes) } }, + }, + }; + } + + [Test] + public void SharedBlobSlotListsEveryOwnerPass() + { + var passA = PassWithBlob(7, 2); + var passB = PassWithBlob(7, 2); + var sh = ShaderWith(passA, passB); + + var (passOf, spOf) = GameExport.PassMaps(sh); + + Assert.That(passOf[7], Is.EquivalentTo(new[] { passA, passB })); + // Equal reflection: the first pass seen keeps the slot. + Assert.That(spOf[7], Is.SameAs(passA.SubPrograms[0])); + } + + [Test] + public void DistinctBlobSlotsKeepTheirOwnPasses() + { + var passA = PassWithBlob(1, 2); + var passB = PassWithBlob(2, 2); + var sh = ShaderWith(passA, passB); + + var (passOf, _) = GameExport.PassMaps(sh); + + Assert.That(passOf[1], Is.EquivalentTo(new[] { passA })); + Assert.That(passOf[2], Is.EquivalentTo(new[] { passB })); + } + + [Test] + public void RicherReflectionWinsTheSharedSlot() + { + var thin = PassWithBlob(3, 1); + var rich = PassWithBlob(3, 4); + var sh = ShaderWith(thin, rich); + + var (_, spOf) = GameExport.PassMaps(sh); + + Assert.That(spOf[3], Is.SameAs(rich.SubPrograms[0])); + } +} diff --git a/src/ShaderRipper/Game/GameExport.cs b/src/ShaderRipper/Game/GameExport.cs index 53367c7..a0bf5c2 100644 --- a/src/ShaderRipper/Game/GameExport.cs +++ b/src/ShaderRipper/Game/GameExport.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Text.RegularExpressions; using ShaderRipper.Data; using ShaderRipper.Emit; using ShaderRipper.Resolve; @@ -1058,6 +1059,40 @@ private static void ExportShader(Shader sh, string name, string rel, string dirN // only thing the decoded slot tells us. var (passOf, spOf) = PassMaps(sh); + // Cross-platform sampler names: the GL-family variants of the same + // shader keep the sampler names in their GLSL source + // (UNITY_LOCATION(n) uniform sampler2D ) where the DXBC payload + // strips them. GL locations and DXBC texture registers are assigned + // in the same property order, so location N names tN. A + // Windows-only build has no GL-family slot, and the DXBC route then + // falls back to reflection and (honestly noted) inference. + var glslFs = new Dictionary(); + var glslVs = new Dictionary(); + for (int gi = 0; gi < (sh.Platforms?.Length ?? 0); gi++) + { + var ginfo = Plugins.PlatformCatalog.Get((int)sh.Platforms[gi]); + if (ginfo == null || ginfo.Format != "glsl") continue; + byte[] gdecoded; + try { gdecoded = BlobTool.DecodeBlob(sh, gi); } + catch { continue; } + if (gdecoded == null || gdecoded.Length == 0) continue; + foreach (var pr in ProgramSource.Extract(gdecoded)) + { + if (pr.Format != BlobFormat.Glsl || string.IsNullOrEmpty(pr.Text)) continue; + // Stage split: a vertex program's texture table is not the + // fragment program's, and both compile from the same source. + var target = pr.Text.Contains("SV_Target") ? glslFs + : pr.Text.Contains("gl_Position") ? glslVs : null; + if (target == null) continue; + foreach (Match m in Regex.Matches(pr.Text, + @"(?:UNITY_LOCATION\((\d+)\)|layout\(location\s*=\s*(\d+)\))\s*uniform\s+sampler2D\s+([A-Za-z_]\w*)")) + { + int loc = int.Parse(m.Groups[1].Success ? m.Groups[1].Value : m.Groups[2].Value); + if (!target.TryGetValue(loc, out var nm)) target[loc] = m.Groups[3].Value; + } + } + } + // Every platform the game shipped, one at a time: d3d11 drives the // DXBC route, the GLES/GL platforms drive the GLSL route, everything // else is noted and skipped (no translator for it in this project). @@ -1079,7 +1114,7 @@ private static void ExportShader(Shader sh, string name, string rel, string dirN if (decoded == null || decoded.Length == 0) { s.Skipped++; continue; } if (info.Format == "dxbc") ExportDxbc(sh, name, info, decoded, dir, dirName, - passOf, spOf, kwIndex, props, perPass, programs, origAsm, s); + passOf, spOf, kwIndex, props, perPass, programs, origAsm, s, null, null, glslVs, glslFs); else if (info.Format == "glsl") ExportGlsl(sh, name, info, decoded, dir, passOf, spOf, kwIndex, perPass, s, origAsm); else if (info.Format == "d3d9") ExportD3d9(sh, name, info, decoded, dir, passOf, spOf, perPass, origAsm, s); else ExportPlugin(sh, name, info, decoded, dir, passOf, spOf, kwIndex, props, perPass, origAsm, s); @@ -1265,7 +1300,7 @@ internal static string TranslateOne(Shader sh, string name, Stats s) if (pre != null && pre.Count == 0) { s.Skipped++; continue; } if (info.Format == "dxbc") ExportDxbc(sh, name, info, decoded, null, null, - passOf, spOf, kwIndex, props, perPass, false, false, s, null, pre); + passOf, spOf, kwIndex, props, perPass, false, false, s, null, pre, null, null); else if (info.Format == "glsl") ExportGlsl(sh, name, info, decoded, null, passOf, spOf, kwIndex, perPass, s, false); else if (info.Format == "d3d9") ExportD3d9(sh, name, info, decoded, null, passOf, spOf, perPass, false, s); else ExportPlugin(sh, name, info, decoded, null, passOf, spOf, kwIndex, props, perPass, false, s, null); @@ -1363,7 +1398,7 @@ internal static void TranslateProgramsOf(Shader sh, Stats s, /// mapping is therefore a list per slot: keeping only one owner silently /// dropped the program from every other pass that declared it. /// - private static (Dictionary> passOf, Dictionary spOf) PassMaps(Shader sh, int platformIndex = -1) + internal static (Dictionary> passOf, Dictionary spOf) PassMaps(Shader sh, int platformIndex = -1) { var passOf = new Dictionary>(); var spOf = new Dictionary(); @@ -1456,7 +1491,8 @@ private static void ExportDxbc(Shader sh, string name, PlatformInfo info, byte[] Dictionary kwIndex, List props, Dictionary perPass, bool programs, bool origAsm, Stats s, List<(int blobIndex, string stage, string hlsl)> collected = null, - List preSlots = null) + List preSlots = null, + Dictionary glslVs = null, Dictionary glslFs = null) { int platformIndex = Array.IndexOf(sh.Platforms, (uint)info.Id); if (sh.ParsedForm?.SubShaders.SelectMany(ss => ss.Passes).SelectMany(p => p.SubPrograms) @@ -1491,8 +1527,10 @@ private static void ExportDxbc(Shader sh, string name, PlatformInfo info, byte[] List? owners = null; passOf.TryGetValue(slot.BlobIndex, out owners); var pass = owners is { Count: > 0 } ? owners[0] : null; + var glslTexNames = module.Stage is "vs" ? glslVs + : module.Stage is "ps" ? glslFs : null; module.Cb = NameConstantBuffers(module, name, pass, sub, props, paramCbs, allParamCbs, - matrixRows); + matrixRows, glslTexNames); // Two consumers, two shapes. The ShaderLab wrapper merges this text // into a file that includes `UnityCG.cginc` -- matrices are @@ -2352,7 +2390,8 @@ private static Resolve.CbResolver NameConstantBuffers( Ast.Module m, string shaderName, Pass pass, SubProgram sub, List props, Dictionary> paramCbs, List allParamCbs, - Dictionary matrixRows = null) + Dictionary matrixRows = null, + Dictionary glslTexNames = null) { var nameTable = pass?.NameIndices != null ? new HashSet(pass.NameIndices.Keys, StringComparer.Ordinal) @@ -2366,7 +2405,7 @@ private static Resolve.CbResolver NameConstantBuffers( var r = Resolve.CbResolver.Resolve( m, nameTable, props, own, -1, BestGlobals(sub), null, - allParamCbs, true, null, sub?.TextureParams, matrixRows); + allParamCbs, true, null, sub?.TextureParams, matrixRows, glslTexNames); if (Environment.GetEnvironmentVariable("SHADERRIP_CBDBG") == "1") Console.Error.WriteLine($"[cb] {shaderName}/{sub?.BlobIndex}: " diff --git a/src/ShaderRipper/Resolve/CbResolver.cs b/src/ShaderRipper/Resolve/CbResolver.cs index 03f0158..9666fc1 100644 --- a/src/ShaderRipper/Resolve/CbResolver.cs +++ b/src/ShaderRipper/Resolve/CbResolver.cs @@ -230,7 +230,8 @@ public static CbResolver Resolve(Ast.Module prog, ICollection nameTable, bool tmpLayout = false, TmpLayouts.Layout layoutHint = null, System.Collections.Generic.List<(string Name, int Index, int Sampler)> texParams = null, - Dictionary matrixRows = null) + Dictionary matrixRows = null, + Dictionary glslTexNames = null) { var r = new CbResolver(new HashSet(nameTable ?? new List(), StringComparer.Ordinal)); r.MatrixRows = matrixRows; @@ -334,7 +335,7 @@ public static CbResolver Resolve(Ast.Module prog, ICollection nameTable, r.FullyMapped.Add(d.Register); } - r.MapTextures(prog, props, texParams); + r.MapTextures(prog, props, texParams, glslTexNames); if (Environment.GetEnvironmentVariable("SHADERRIP_CBDBG") == "1") { @@ -1247,15 +1248,22 @@ private List NamedGlobals(IReadOnlyList props) /// Give sampler registers the property name they belong to. /// - /// Priority: the asset's own m_TextureParams reflection (name + t# - /// register straight out of the serialized program parameters -- the same - /// table UnityPlayer's runtime consumes), falling back to property - /// declaration order only when the asset kept no texture list. Property + /// Priority: the shader's own GLSL source (GL-family variants keep the + /// sampler names in UNITY_LOCATION(n) uniform sampler2D lines where + /// the DXBC payload strips them; GL locations and DXBC texture registers + /// follow the same property order, so location N names tN); then the + /// asset's own m_TextureParams reflection (name + t# register + /// straight out of the serialized program parameters); then, only when no + /// GLSL source exists, the main-texture convention ([MainTexture] + /// attribute, else the literal _MainTex -- the rule + /// Material.mainTexture documents, applied as an explicit guess, + /// never as an engine answer); then property declaration order. Property /// order is NOT register order: one commercial title's FloorMesh declares _TileTex first but /// binds it to t2, which silently swapped three samplers and broke the look. /// private void MapTextures(Ast.Module prog, IReadOnlyList props, - System.Collections.Generic.List<(string Name, int Index, int Sampler)> texParams) + System.Collections.Generic.List<(string Name, int Index, int Sampler)> texParams, + Dictionary glslTexNames = null) { // A name is trustworthy when the shader itself declares it. The // property table is authoritative for material textures; the name table @@ -1263,10 +1271,38 @@ private void MapTextures(Ast.Module prog, IReadOnlyList props, var declared = new HashSet(StringComparer.Ordinal); if (props != null) foreach (var p in props) declared.Add(p.Name); + var regs = prog.Decls.Where(d => d.Kind != null && d.Kind.StartsWith("texture")) + .Select(d => d.Register).Distinct().OrderBy(x => x).ToList(); + if (regs.Count == 0) return; + + // 1. GL-family source names -- the strongest source in the asset. The + // GLSL text is Unity's own translation and keeps the property names + // the DXBC payload strips; locations and texture registers are + // assigned in the same property order, so they pair up positionally. + if (glslTexNames != null && glslTexNames.Count > 0) + { + var locs = glslTexNames.Keys.OrderBy(x => x).ToList(); + int g = Math.Min(regs.Count, locs.Count); + if (g > 0) + { + var named = new List(); + for (int i = 0; i < g; i++) + { + TexName[regs[i]] = glslTexNames[locs[i]]; + named.Add($"t{regs[i]}={glslTexNames[locs[i]]}"); + } + FileNotes.Add("texture names from the shader's own GLSL source: " + + string.Join(", ", named)); + } + } + + // 2. The asset's own reflection fills whatever the GLSL source did not + // cover (a Windows-only asset ships no GL-family variant to read). if (texParams != null && texParams.Count > 0) { foreach (var (nm, idx, _) in texParams) - if (idx >= 0 && idx < 128 && (declared.Contains(nm) || _nameTable.Contains(nm))) + if (idx >= 0 && idx < 128 && !TexName.ContainsKey(idx) && + (declared.Contains(nm) || _nameTable.Contains(nm))) TexName[idx] = nm; // Do NOT stop here. The asset records only the textures its material // inspector exposes, so a partial list is the normal case: `_AlphaTex` @@ -1275,7 +1311,14 @@ private void MapTextures(Ast.Module prog, IReadOnlyList props, // The property-order pass below fills the remainder. } - // The declaration order is the only signal left. Weak, but measured: + var open = regs.Where(r => !TexName.ContainsKey(r)).ToList(); + if (open.Count == 0) return; + + bool guessed = (texParams == null || texParams.Count == 0) && + (glslTexNames == null || glslTexNames.Count == 0); + + // The declaration order is the only signal left for everything past the + // main texture. Weak, but measured: // over every pass of the reference corpus whose reflection names t0, the // property table lists that texture FIRST -- 849 of 849, whatever the // property flags say. `[PerRendererData]` is no exception: `_MainTex` @@ -1286,22 +1329,42 @@ private void MapTextures(Ast.Module prog, IReadOnlyList props, if (props != null) foreach (var p in props.Where(p => TextureTypes.Contains(p.Type))) texProps.Add(p.Name); - bool guessed = texParams == null || texParams.Count == 0; - var regs = prog.Decls.Where(d => d.Kind != null && d.Kind.StartsWith("texture")) - .Select(d => d.Register).Distinct().OrderBy(x => x).ToList(); - if (regs.Count == 0) return; - // Registers the reflection already named are settled; the property-order - // pass only fills the rest, in order, so a name from the asset is never - // overwritten by a guess. - var open = regs.Where(r => !TexName.ContainsKey(r)).ToList(); - if (open.Count == 0) return; // A property the reflection already bound must not be handed out again: // two registers with one name do not compile (`redefinition of // '_MainTex'`), and the second binding is a leftover sampler anyway. var used = new HashSet(TexName.Values, StringComparer.Ordinal); var spare = texProps.Where(p => !used.Contains(p)).ToList(); - // The two lists rarely line up. A texture property the shader never + + // 3. Main-texture convention, only when no GLSL source existed: + // Material.mainTexture resolves the main texture as the + // `[MainTexture]`-flagged property, else the literal `_MainTex` -- + // applied here as an explicit guess, NOT as an engine answer: the + // asset keeps no binding table, and a game that only ever + // SetTexture()s a later property writes a slot nothing samples. + // (Gated on the sprite-pipeline marker, a `UnityPerDrawSprite` entry + // in the pass name table, where the convention is observed.) + string mainName = null; + int mainReg = open.Count > 0 ? open[0] : -1; + if (open.Count > 0 && (glslTexNames == null || glslTexNames.Count == 0) && + _nameTable.Contains("UnityPerDrawSprite")) + { + string flagged = spare.FirstOrDefault(p => + { + var q = props?.FirstOrDefault(x => x.Name == p); + return q != null && (q.Flags & 0x80) != 0; + }); + string literal = flagged ?? (spare.Contains("_MainTex") ? "_MainTex" : null); + if (literal != null) + { + TexName[mainReg] = literal; + mainName = literal; + spare.Remove(literal); + open = open.Skip(1).ToList(); + } + } + + // 4. The two lists rarely line up. A texture property the shader never // samples in this variant (a `multi_compile` branch that is compiled out) // leaves more properties than registers; a built-in sampler Unity injects // (camera depth, a grab texture) leaves more registers than properties. @@ -1310,15 +1373,19 @@ private void MapTextures(Ast.Module prog, IReadOnlyList props, // own textures right and the leftovers stay `tN`. int n = Math.Min(spare.Count, open.Count); for (int i = 0; i < n; i++) TexName[open[i]] = spare[i]; - if (n == 0) + if (n == 0 && mainName == null) FileNotes.Add($"{open.Count} texture register(s) the asset did not name -- left as tN"); else if (guessed) - // The names above are an inference, not a reading: this pass kept no - // reflection, so nothing in the asset proves the slot holds that - // property rather than one the pass never uses. Say so in the file - // instead of letting the result look authoritative. - FileNotes.Add("texture names inferred from property order: " + - string.Join(", ", Enumerable.Range(0, n).Select(i => $"t{open[i]}={spare[i]}"))); + { + var parts = new List(); + if (mainName != null) + parts.Add($"t{mainReg}={mainName} (main-texture convention, unverified: the asset keeps no binding)"); + if (n > 0) + parts.Add("remaining texture names inferred from property order: " + + string.Join(", ", Enumerable.Range(0, n).Select(i => $"t{open[i]}={spare[i]}"))); + if (parts.Count > 0) + FileNotes.Add(string.Join("; ", parts)); + } } } }