Skip to content

Commit

Permalink
porting async rmt2 generation
Browse files Browse the repository at this point in the history
  • Loading branch information
@Pedro413
Pedro413 committed May 23, 2024
1 parent 094e22c commit a710826
Show file tree
Hide file tree
Showing 4 changed files with 256 additions and 119 deletions.
81 changes: 79 additions & 2 deletions TagTool/Commands/Porting/PortTagCommand.Shader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,67 @@
using TagTool.Shaders.ShaderMatching;
using System;
using TagTool.Shaders.ShaderConverter;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using System.Linq;
using TagTool.Commands.Common;

namespace TagTool.Commands.Porting
{
partial class PortTagCommand
{
public List<string> PendingTemplates = new List<string>();
public Dictionary<CachedTag, (CachedTag, object, object)> DeferredRenderMethods = new Dictionary<CachedTag, (CachedTag, object, object)>(); // format: base cache tag, (blam tag, converted definition, blam cache definition)
public Dictionary<string, Task> TemplateConversionTasks = new Dictionary<string, Task>();

public class TemplateConversionResult
{
public RenderMethodTemplate Definition;
public PixelShader PixelShaderDefinition;
public VertexShader VertexShaderDefinition;
public CachedTag Tag;
}

public void WaitForPendingTemplateConversion()
{
try
{
Task.WaitAll(TemplateConversionTasks.Values.ToArray());
}
catch (AggregateException ex)
{
foreach (var inner in ex.InnerExceptions)
new TagToolError(CommandError.CustomError, inner.Message);
throw (ex);
}
}

public void FinishConvertTemplate(TemplateConversionResult result,
string tagName,
out RenderMethodTemplate rmt2,
out PixelShader pixl,
out VertexShader vtsh)
{
var task = TemplateConversionTasks[tagName];
TemplateConversionTasks.Remove(tagName);

if (!task.IsFaulted)
{
rmt2 = result.Definition;
pixl = result.PixelShaderDefinition;
vtsh = result.VertexShaderDefinition;
}
else
{
// rethrow the exception
task.GetAwaiter().GetResult();
rmt2 = null;
pixl = null;
vtsh = null;
}
}

public ShaderMatcherNew Matcher = new ShaderMatcherNew();

private RasterizerGlobals ConvertRasterizerGlobals(RasterizerGlobals rasg)
Expand All @@ -21,8 +77,22 @@ private RasterizerGlobals ConvertRasterizerGlobals(RasterizerGlobals rasg)
return rasg;
}

private object ConvertShader(Stream cacheStream, Stream blamCacheStream, object definition, CachedTag blamTag, object blamDefinition)
private void FinalizeRenderMethods(Stream cacheStream, Stream blamCacheStream)
{
foreach (var deferredRm in DeferredRenderMethods)
{
var definition = ConvertShaderInternal(cacheStream, blamCacheStream, (RenderMethod)deferredRm.Value.Item2, deferredRm.Value.Item1, (RenderMethod)deferredRm.Value.Item3);

CacheContext.Serialize(cacheStream, deferredRm.Key, definition);

if (FlagIsSet(PortingFlags.Print))
Console.WriteLine($"['{deferredRm.Key.Group.Tag}', 0x{deferredRm.Key.Index:X4}] {deferredRm.Key.Name}.{(deferredRm.Key.Group as Cache.Gen3.TagGroupGen3).Name}");
}
}

private object ConvertShader(Stream cacheStream, Stream blamCacheStream, object definition, CachedTag blamTag, object blamDefinition, CachedTag edTag, out bool isDeferred)
{
isDeferred = false;
switch (definition)
{
case ShaderFoliage rmfl:
Expand All @@ -37,6 +107,13 @@ private object ConvertShader(Stream cacheStream, Stream blamCacheStream, object
case ShaderScreen rmss:
case ShaderZonly rmzo:
case ShaderCortana rmct:
var rmDef = (RenderMethod)definition;
if (rmDef.ShaderProperties.Count > 0 && PendingTemplates.Contains(rmDef.ShaderProperties[0].Template.Name))
{
DeferredRenderMethods.Add(edTag, (blamTag, definition, blamDefinition)); // easier to defer and convert later
isDeferred = true;
return definition;
}
return ConvertShaderInternal(cacheStream, blamCacheStream, (RenderMethod)definition, blamTag, (RenderMethod)blamDefinition);

case ContrailSystem cntl:
Expand Down Expand Up @@ -151,7 +228,7 @@ private CachedTag FindClosestRmt2(Stream cacheStream, Stream blamCacheStream, Ca
{
// Verify that the ShaderMatcher is ready to use
if (!Matcher.IsInitialized)
Matcher.Init(CacheContext, BlamCache, cacheStream, blamCacheStream, FlagIsSet(PortingFlags.Ms30), FlagIsSet(PortingFlags.PefectShaderMatchOnly));
Matcher.Init(CacheContext, BlamCache, cacheStream, blamCacheStream, this, FlagIsSet(PortingFlags.Ms30), FlagIsSet(PortingFlags.PefectShaderMatchOnly));

return Matcher.FindClosestTemplate(blamRmt2, BlamCache.Deserialize<RenderMethodTemplate>(blamCacheStream, blamRmt2), FlagIsSet(PortingFlags.GenerateShaders));
}
Expand Down
8 changes: 5 additions & 3 deletions TagTool/Commands/Porting/PortTagCommand.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ public partial class PortTagCommand : Command
private readonly List<Tag> ResourceTagGroups = new List<Tag> { new Tag("snd!"), new Tag("bitm"), new Tag("Lbsp") }; // for null tag detection

private DirectoryInfo TempDirectory { get; } = new DirectoryInfo(Path.GetTempPath());
private BlockingCollection<Action> _deferredActions = new BlockingCollection<Action>();
internal BlockingCollection<Action> _deferredActions = new BlockingCollection<Action>();

internal SemaphoreSlim ConcurrencyLimiter;

Expand Down Expand Up @@ -131,7 +131,9 @@ public override object Execute(List<string> args)

WaitForPendingSoundConversion();
WaitForPendingBitmapConversion();
WaitForPendingTemplateConversion();
ProcessDeferredActions();
FinalizeRenderMethods(cacheStream, blamCacheStream);
if (BlamCache is GameCacheGen3 gen3Cache)
gen3Cache.ResourceCacheGen3.ResourcePageCache.Clear();

Expand Down Expand Up @@ -1392,9 +1394,9 @@ public CachedTag ConvertTagInternal(Stream cacheStream, Stream blamCacheStream,
{
// Verify that the ShaderMatcher is ready to use
if (!Matcher.IsInitialized)
Matcher.Init(CacheContext, BlamCache, cacheStream, blamCacheStream, FlagIsSet(PortingFlags.Ms30), FlagIsSet(PortingFlags.PefectShaderMatchOnly));
Matcher.Init(CacheContext, BlamCache, cacheStream, blamCacheStream, this, FlagIsSet(PortingFlags.Ms30), FlagIsSet(PortingFlags.PefectShaderMatchOnly));

blamDefinition = ConvertShader(cacheStream, blamCacheStream, blamDefinition, blamTag, BlamCache.Deserialize(blamCacheStream, blamTag));
blamDefinition = ConvertShader(cacheStream, blamCacheStream, blamDefinition, blamTag, BlamCache.Deserialize(blamCacheStream, blamTag), edTag, out isDeferred);
if (blamDefinition == null) // convert shader failed
return GetDefaultShader(blamTag.Group.Tag);
}
Expand Down
101 changes: 37 additions & 64 deletions TagTool/Shaders/ShaderGenerator/ShaderGenerator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1047,6 +1047,43 @@ private static void AccumRuntimeParameterType(GameCache cache, Dictionary<string
return allRmopParameters;
}

public static List<RenderMethodOption.ParameterBlock> GatherParametersAsync(Dictionary<string, RenderMethodOption> renderMethodOptions, RenderMethodDefinition rmdf, List<byte> options, bool includeGlobal = true)
{
List<RenderMethodOption.ParameterBlock> allRmopParameters = new List<RenderMethodOption.ParameterBlock>();

if (includeGlobal)
{
if (rmdf.GlobalOptions != null)
{
var globalRmop = renderMethodOptions[rmdf.GlobalOptions.Name];
allRmopParameters.AddRange(globalRmop.Parameters);
}
}

for (int i = 0; i < rmdf.Categories.Count; i++)
{
if (rmdf.Categories[i].ShaderOptions.Count == 0)
continue;

var option = rmdf.Categories[i].ShaderOptions[i < options.Count ? options[i] : 0];

if (option.Option != null)
{
var rmop = renderMethodOptions[option.Option.Name];

foreach (var parameter in rmop.Parameters)
{
if (allRmopParameters.Any(x => x.Name == parameter.Name)) // prevent duplicates
continue;

allRmopParameters.Add(parameter);
}
}
}

return allRmopParameters;
}

/// <summary>
/// Non async
/// </summary>
Expand Down Expand Up @@ -1550,70 +1587,6 @@ public static GlobalVertexShader GenerateSharedVertexShaders(GameCache cache, Re
return glvs;
}

public static bool VerifyRmt2Routing(GameCache cache, Stream stream, RenderMethodTemplate rmt2, RenderMethodDefinition rmdf, List<byte> options)
{
bool anyMissing = false;

var allParameters = GatherParameters(cache, stream, rmdf, options);

var pixl = cache.Deserialize<PixelShader>(stream, rmt2.PixelShader);

foreach (var entry in rmdf.EntryPoints)
{
if (rmt2.EntryPoints[(int)entry.EntryPoint].Count > 0)
{
int iEnd = rmt2.EntryPoints[(int)entry.EntryPoint].Count + rmt2.EntryPoints[(int)entry.EntryPoint].Offset;
for (int i = rmt2.EntryPoints[(int)entry.EntryPoint].Offset; i < iEnd; i++)
{
var pass = rmt2.Passes[i];

if (pass.Values[(int)ParameterUsage.PS_Real].Count > 0)
{
foreach (var constant in pixl.Shaders[pixl.EntryPointShaders[(int)entry.EntryPoint].Offset].PCConstantTable.Constants)
{
if (constant.RegisterType != ShaderParameter.RType.Vector)
continue;

string constantName = cache.StringTable.GetString(constant.ParameterName);
bool found = false;

int jEnd = pass.Values[(int)ParameterUsage.PS_Real].Offset + pass.Values[(int)ParameterUsage.PS_Real].Count;
for (int j = pass.Values[(int)ParameterUsage.PS_Real].Offset; j < jEnd; j++)
{
if (rmt2.RoutingInfo[j].DestinationIndex == constant.RegisterIndex)
{
found = true;
break;
}
}

if (!found)
{
jEnd = pass.Values[(int)ParameterUsage.PS_RealExtern].Offset + pass.Values[(int)ParameterUsage.PS_RealExtern].Count;
for (int j = pass.Values[(int)ParameterUsage.PS_RealExtern].Offset; j < jEnd; j++)
{
if (rmt2.RoutingInfo[j].DestinationIndex == constant.RegisterIndex)
{
found = true;
break;
}
}
}

if (!found)
{
Console.WriteLine($"WARNING: {constantName} not bound in rmt2");
anyMissing = true;
}
}
}
}
}
}

return !anyMissing;
}

public static void GenerateExplicitShader(GameCache cache, Stream stream, string explicitShader, out PixelShader pixl, out VertexShader vtsh)
{
ExplicitGenerator generator = new ExplicitGenerator();
Expand Down
Loading

0 comments on commit a710826

Please sign in to comment.