From a6a9c2dbcae67d6d8d1c0f2d52d08157b5afe400 Mon Sep 17 00:00:00 2001 From: marcussacana Date: Fri, 16 Aug 2019 07:24:17 -0300 Subject: [PATCH] New Good Feature Now the SRL duplicate lines with different translation in the same database! See the Readme for more info. --- Help/Readme.txt | 23 ++- SRL/Compiler.cs | 4 +- SRL/DuplicableDictionary.cs | 286 ++++++++++++++++++++++++++++++++++++ SRL/FileWorker.cs | 8 +- SRL/Main.cs | 5 +- SRL/PipeService.cs | 45 ++++-- SRL/Reloader.cs | 15 +- SRL/SRL.csproj | 1 + SRL/SRL.ini | 1 + SRL/Types.cs | 27 ++++ SRL/Variables.cs | 40 ++--- SRLTracer/Form1.cs | 2 + 12 files changed, 407 insertions(+), 50 deletions(-) create mode 100644 SRL/DuplicableDictionary.cs diff --git a/Help/Readme.txt b/Help/Readme.txt index 69ef6ff..194bab9 100644 --- a/Help/Readme.txt +++ b/Help/Readme.txt @@ -86,9 +86,30 @@ At StringsReloader -"AsianInput" Hint the Dialogue Detection algorithm saying if the game is a japanese game or not. -"AutoUnks" Automatically create the unknow char reload list based on your reloads. -"CaseSensitive" The SRL database match can match with case senstive or not, just change this --"NotCachedOnly" Use the pointer cache to don't allow the SRL process again the same string +-"NotCachedOnly" Use the pointer cache to don't allow the SRL process again the same string, usefull when the SRL injection point is inside a for each char loop of the string -"SetOutEncoding" If the Debug Low Window (aka console) display invalid chars, set true -"AllowEmptyReloads" Keep in the database Reloads that don't change nothing +-"AllowDuplicates" Allow a single database contains more than one reload for the same match line + +About the AllowDuplicates +Well, After basically 2 years devlopling the SRL, I added support to the SRL can match +different text for a same line, This feature is disabled by default because the old method +to match the string is 15x faster than the new method, in others words enabling this feature +make the SRL match the game text more slow, and of couse, will increase the cpu usage too. +Wow shit! 15x?! Yes, but you don't need worry much since the SRL is very, but very fast +to match a string, in a small database in the old method he can match like 500000 times +the reloads using only 0.01 ms, and when using the new method he will use 0.18 ms. +Then it's really more slow but isn't something that you will notice when use... +Oh Good! But if we have more gains than lost with this new method, why is disable by default? +Well, Well... Basically the SRL already can match a duplicate line in the old method, +You just need split the database in parts and the SRL will give priority to the last database +that he found a translation, and you can need use the ::SETDB-??:: sometimes too, like when +the duplicate is just after the last, But Enabling this new feature you don't need +split the database in many .lst's to match a duplicate, and don't need use the ::SETDB-??::, +The SRL will give priority to the closest line of the lastest matched line in the same database, +Keep in mind, when the SRL change to other database he don't reset the 'last match' position +of the related database, then if the game return to a .lst that has already matched something +can give unexpected results, then keep sure to test everything before you relase your patch. At WordWrap -"Enable" Disable or Enable the SRL Auto Wordwrap engine diff --git a/SRL/Compiler.cs b/SRL/Compiler.cs index 8260a56..3088df4 100644 --- a/SRL/Compiler.cs +++ b/SRL/Compiler.cs @@ -296,13 +296,13 @@ static void LoadData() { Log("Chars Reloads Initialized, Total entries: {0} + {1}", true, UnkRld.Count, CharRld.Count); Log("Processing String Reloads...", true); List Temp = new List(); - StrRld = new Dictionary(); + StrRld = CreateDictionary(); long ReloadEntries = 0, MaskEntries = 0; foreach (SRLDatabase2 Database in Data.Databases) { for (uint i = 0; i < Database.Original.LongLength; i++) { Application.DoEvents(); string str = SimplfyMatch(Database.Original[i]); - if (!ContainsKey(str, true)) { + if (AllowDuplicates || !ContainsKey(str, true)) { if (IsMask(Database.Original[i])) { if (LiteralMaskMatch) { AddEntry(str, ReplaceChars(Database.Replace[i])); diff --git a/SRL/DuplicableDictionary.cs b/SRL/DuplicableDictionary.cs new file mode 100644 index 0000000..53e2bea --- /dev/null +++ b/SRL/DuplicableDictionary.cs @@ -0,0 +1,286 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; + +namespace SRL { + public class DuplicableDictionary : IDictionary { + public int LastKeyIndex { get; private set; } = 0; + + Collection KeyCollection = new Collection(); + Collection ValueCollection = new Collection(); + Dictionary> Base = new Dictionary>(); + + public bool NearToNext = true; + + public TValue this[TKey key] { + get { + if (TryGetValue(key, out TValue value)) + return value; + throw new KeyNotFoundException(); + } + set => throw new NotImplementedException("You Must use the Item Index"); + } + + public TValue this[int index] { + get { + return ValueCollection[index]; + } + set { + ValueCollection[index] = value; + var Entry = Base[KeyCollection[index]]; + Entry.Update(index, value); + Base[KeyCollection[index]] = Entry; + } + } + + public ICollection Keys => KeyCollection; + + public ICollection Values => ValueCollection; + + public int Count => ValueCollection.Count; + + public bool IsReadOnly => false; + + public void Add(TKey key, TValue value) { + Add(new KeyValuePair(key, value)); + } + + public void Add(KeyValuePair item) { + int ID = Count; + KeyCollection.Add(item.Key); + ValueCollection.Add(item.Value); + + if (Base.ContainsKey(item.Key)) { + var Entry = Base[item.Key]; + Entry.Insert(item.Value, ID); + Base[item.Key] = Entry; + } else + Base[item.Key] = new KeyValuePairs(item.Key, item.Value, ID); + } + + public void Clear() { + Base.Clear(); + KeyCollection.Clear(); + ValueCollection.Clear(); + LastKeyIndex = 0; + } + + public bool Contains(KeyValuePair item) { + if (!Base.ContainsKey(item.Key)) + return false; + foreach (var Pair in Base[item.Key].GetKeyPairs()) { + if (Pair.Key.Equals(item.Key) && Pair.Value.Equals(item.Value)) + return true; + } + + return false; + } + + public bool ContainsKey(TKey key) { + return Base.ContainsKey(key); + } + + public void CopyTo(KeyValuePair[] array, int arrayIndex) { + for (int i = 0; i < Count; i++) + array[i + arrayIndex] = new KeyValuePair(KeyCollection[i], ValueCollection[i]); + } + + public IEnumerator> GetEnumerator() { + for (int i = 0; i < Count; i++) + yield return new KeyValuePair(KeyCollection[i], ValueCollection[i]); + } + + /// + /// This don't remove the entry, just disable the match + /// + public bool Remove(TKey key) { + if (!ContainsKey(key)) + return false; + + return Base.Remove(key); + } + + /// + /// This don't remove the entry, just disable the match + /// + public void RemoveAt(int Index) { + var Key = KeyCollection[Index]; + Base[Key].Remove(Index); + if (Base[Key].Count == 0) + Base.Remove(Key); + } + + /// + /// This don't remove the entry, just disable the match + /// + public bool Remove(KeyValuePair item) { + if (!Base.ContainsKey(item.Key)) + return false; + + int Removed = 0; + + var Entry = Base[item.Key]; + + foreach (var Pair in Entry.GetIdPairs()) { + if (Pair.Value.Equals(item.Value)) { + Entry.Remove(Pair.Key); + Removed++; + } + } + + if (Entry.Count == 0) + Base.Remove(item.Key); + + return Removed > 0; + } + + public bool TryGetValue(TKey key, out TValue value) { + if (!Base.ContainsKey(key)) { + value = default; + return false; + } + + if (KeyCollection[LastKeyIndex].Equals(key)) { + value = ValueCollection[LastKeyIndex]; + if (NearToNext && LastKeyIndex + 1 < Keys.Count) + LastKeyIndex++; + return true; + } + + var Entry = Base[key]; + + var Closest = ClosestTo(Entry.Identifiers, LastKeyIndex); + if (Closest == int.MaxValue) + Closest = Entry.Identifiers.First(); + + LastKeyIndex = Closest; + if (NearToNext && LastKeyIndex + 1 < Keys.Count) + LastKeyIndex++; + + foreach (var Pair in Entry.GetIdPairs()) { + if (Pair.Key == Closest) { + value = Pair.Value; + return true; + } + } + value = default; + return false; + } + + public static int ClosestTo(IEnumerable collection, int target) { + // NB Method will return int.MaxValue for a sequence containing no elements. + // Apply any defensive coding here as necessary. + var closest = int.MaxValue; + var minDifference = int.MaxValue; + foreach (var element in collection) { + var difference = Math.Abs((long)element - target); + if (minDifference > difference) { + minDifference = (int)difference; + closest = element; + } + } + + return closest; + } + + IEnumerator IEnumerable.GetEnumerator() { + return GetEnumerator(); + } + } + + internal class Collection : ICollection { + List List = new List(); + public int Count => List.Count; + + public bool IsReadOnly => false; + public Type this[int index] { get => List[index]; set => List[index] = value; } + + public void Add(Type item) { + List.Add(item); + } + + public void Clear() { + List.Clear(); + } + + public bool Contains(Type item) { + return List.Contains(item); + } + + public void CopyTo(Type[] array, int arrayIndex) { + List.CopyTo(array, arrayIndex); + } + + public IEnumerator GetEnumerator() { + return List.GetEnumerator(); + } + + public bool Remove(Type item) { + return List.Remove(item); + } + + public void RemoveAt(int index) { + List.RemoveAt(index); + } + + IEnumerator IEnumerable.GetEnumerator() { + return List.GetEnumerator(); + } + } + + internal struct KeyValuePairs { + public TKey Key { get; private set; } + public TValue[] Values { get; private set; } + + public int[] Identifiers { get; private set; } + + public int Count => Values.Length; + + public KeyValuePairs(TKey Key, TValue Value, int Identifier) { + this.Key = Key; + Values = new TValue[] { Value }; + Identifiers = new int[] { Identifier }; + } + + public void Insert(TValue Value, int Identifier) => Insert(new TValue[] { Value }, new int[] { Identifier }); + public void Insert(TValue[] Values, int[] Identifiers) { + this.Values = this.Values.Concat(Values).ToArray(); + this.Identifiers = this.Identifiers.Concat(Identifiers).ToArray(); + } + + public void Update(int Identifier, TValue Value) { + for (int i = 0; i < Values.Length; i++) { + if (Identifiers[i] == Identifier) { + Values[i] = Value; + break; + } + } + } + + public void Remove(int Identifier) { + for (int i = 0; i < Values.Length; i++) { + if (Identifiers[i] == Identifier) { + var tmpVal = Values.ToList(); + var tmpIds = Identifiers.ToList(); + + tmpVal.RemoveAt(i); + tmpIds.RemoveAt(i); + + Values = tmpVal.ToArray(); + Identifiers = tmpIds.ToArray(); + break; + } + } + } + + public IEnumerable> GetKeyPairs() { + foreach (var Value in Values) + yield return new KeyValuePair(Key, Value); + } + public IEnumerable> GetIdPairs() { + for (int i = 0; i < Values.Length; i++) + yield return new KeyValuePair(Identifiers[i], Values[i]); + } + } +} \ No newline at end of file diff --git a/SRL/FileWorker.cs b/SRL/FileWorker.cs index bbfbc8b..2de2c7f 100644 --- a/SRL/FileWorker.cs +++ b/SRL/FileWorker.cs @@ -29,7 +29,7 @@ static void ReadDump(string Path, ref List In, ref List Out, boo if (L2 != L1 || (IgnoreMask && IsMask(L1))) { if (!string.IsNullOrWhiteSpace(L2) || AllowEmpty) { - if (!In.Contains(L1)) { + if (!In.Contains(L1) || AllowDuplicates) { In.Add(L1); Out.Add(L2); } @@ -304,6 +304,12 @@ private static void LoadConfig() { Log("Empty Reloader Filter Disabled", true); } + if (!Initialized && Settings.AllowDuplicates) { + AllowDuplicates = true; + Log("Duplicate Reload Support Enabled", true); + } else if (Settings.AllowDuplicates != AllowDuplicates) + Warning("Duplicate Reload Support Changed - Restart Required"); + if (Settings.SetOutputEncoding) { Log("Console Output Encoding Changed", false); if (!Debugging) { diff --git a/SRL/Main.cs b/SRL/Main.cs index 4c95a3a..2c68fe9 100644 --- a/SRL/Main.cs +++ b/SRL/Main.cs @@ -97,7 +97,10 @@ internal static IntPtr ProcessReal(IntPtr Target) { return Target; if (LogAll || LogOutput) { - Log("Output: {0}", true, Reloaded); + if (AllowDuplicates) + Log("Output: {0}\r\nDB Current Index: {1}", true, Reloaded); + else + Log("Output: {0}", true, Reloaded); } } diff --git a/SRL/PipeService.cs b/SRL/PipeService.cs index bee3a76..d621946 100644 --- a/SRL/PipeService.cs +++ b/SRL/PipeService.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.IO; using System.IO.Pipes; using System.Linq; @@ -8,6 +9,11 @@ namespace SRL { static partial class StringReloader { public static int ServiceCall(string ID) { + if (ID.Contains(ServiceDuplicateFlag)) { + ID = ID.Split('|').First(); + AllowDuplicates = true; + } + string SName = string.Format(ServiceMask, ID); if (Debugging) @@ -18,9 +24,9 @@ public static int ServiceCall(string ID) { } private static void Service(string ServiceName) { - StrRld = new System.Collections.Generic.Dictionary(); - MskRld = new System.Collections.Generic.Dictionary(); - Missed = new System.Collections.Generic.List(); + StrRld = CreateDictionary(); + MskRld = CreateDictionary(); + Missed = new List(); NamedPipeServerStream Server = new NamedPipeServerStream(ServiceName, PipeDirection.InOut, 2, PipeTransmissionMode.Byte); Server.WaitForConnection(); @@ -37,7 +43,7 @@ private static void Service(string ServiceName) { OK = true; string Original = Reader.ReadString(); string Reloader = Reader.ReadString(); - if (!StrRld.ContainsKey(Original)) + if (!StrRld.ContainsKey(Original) || AllowDuplicates) StrRld.Add(Original, Reloader); Log("Command Finished, In: {0}, Out: {1}", true, 2, 0); break; @@ -123,7 +129,7 @@ private static void Service(string ServiceName) { OK = true; string Input = Reader.ReadString(); string Reload = Reader.ReadString(); - if (!MskRld.ContainsKey(Input)) + if (!MskRld.ContainsKey(Input) || AllowDuplicates) MskRld.Add(Input, Reload); Log("Command Finished, In {0}, Out: {1}", true, 2, 0); break; @@ -162,6 +168,11 @@ private static void Service(string ServiceName) { DBID = Reader.ReadInt32(); Log("Command Finished, In: {0}, Out: {1}", true, 2, 0); break; + case PipeCommands.GetDBIndex: + Writer.Write(((DuplicableDictionary)StrRld).LastKeyIndex); + Writer.Flush(); + Log("Command Finished, In: {0}, Out: {1}", true, 1, 1); + break; default: if (!OK) MessageBox.Show("Recived Invalid Command to the pipe service...", "SRL Engine", MessageBoxButtons.OK, MessageBoxIcon.Error); @@ -284,7 +295,7 @@ private static bool ValidateMask(string String) { } - private static string ProcesMask(string Original) { + private static string ProcessMask(string Original) { if (Multithread) { string Mask = (from x in MskRld.Keys where MaskMatch(x, Original) select x).FirstOrDefault(); return MaskReplace(Mask, Original, MskRld[Mask]); @@ -387,6 +398,19 @@ private static void SetDBID(int ID) PipeWriter.Write(ID); PipeWriter.Flush(); } + + private static int GetCurrentDBIndex() { + if (!AllowDuplicates) + return -1; + + if (Multithread) { + return ((DuplicableDictionary)StrRld).LastKeyIndex; + } + + PipeWriter.Write((byte)PipeCommands.GetDBIndex); + PipeWriter.Flush(); + return PipeReader.ReadInt32(); + } private static int GetPipeID() { return new Random().Next(0, int.MaxValue); } @@ -394,9 +418,9 @@ private static int GetPipeID() { private static void StartPipe() { if (Multithread) { Log("Pipe Service Disabled.", true); - StrRld = new System.Collections.Generic.Dictionary(); - MskRld = new System.Collections.Generic.Dictionary(); - Missed = new System.Collections.Generic.List(); + StrRld = CreateDictionary(); + MskRld = CreateDictionary(); + Missed = new List(); return; } @@ -406,6 +430,9 @@ private static void StartPipe() { int ServiceID = GetPipeID(); string Service = string.Format(ServiceMask, ServiceID); + if (AllowDuplicates) + Service += ServiceDuplicateFlag; + #if DEBUG new System.Threading.Thread(() => ServiceCall(ServiceID.ToString())).Start(); #else diff --git a/SRL/Reloader.cs b/SRL/Reloader.cs index 4d01982..1167e3e 100644 --- a/SRL/Reloader.cs +++ b/SRL/Reloader.cs @@ -99,7 +99,7 @@ internal static string StrMap(string Input, IntPtr InputPtr, bool Native) { if ((LogAll || LogInput) && (!DumpStrOnly || IsDialog)) { if (!DumpStrOnly || !InCache("LOG: " + Str)) { - Log("[{1}] Input: {0}", true, Input, IsDialog ? "D" : "S"); + Log("[{1}|{2}] Input: {0}", true, Input, IsDialog ? "D" : "S", GetCurrentDBIndex()); if (DumpStrOnly) CacheReply("LOG: " + Str); } @@ -141,7 +141,7 @@ internal static string StrMap(string Input, IntPtr InputPtr, bool Native) { if (!DisableMasks && ValidateMask(Input)) { try { - string Result = ProcesMask(Input); + string Result = ProcessMask(Input); if (Result.StartsWith(MaskWordWrap)) { Result = Result.Substring(MaskWordWrap.Length, Result.Length - MaskWordWrap.Length); @@ -343,7 +343,7 @@ internal static void Init() { else { NoDatabase = true; - StrRld = new Dictionary(); + StrRld = CreateDictionary(); Warning("Can't Compile Strings because the SRL don't found any LST."); } } @@ -375,7 +375,7 @@ internal static void Init() { for (int i = 0; i < Ori.Count; i++) { string Match = SimplfyMatch(Ori[i]); - if (!ContainsKey(Match)) + if (AllowDuplicates || !ContainsKey(Match)) AddEntry(Match, ReplaceChars(TL[i])); } } @@ -514,6 +514,11 @@ private static void ProcessOver(object sender, EventArgs e) { System.Diagnostics.Process.GetCurrentProcess().Kill(); } } - + + private static IDictionary CreateDictionary() { + return AllowDuplicates ? + (IDictionary) new DuplicableDictionary() : + (IDictionary) new Dictionary(); + } } } \ No newline at end of file diff --git a/SRL/SRL.csproj b/SRL/SRL.csproj index 5d4ed33..5513218 100644 --- a/SRL/SRL.csproj +++ b/SRL/SRL.csproj @@ -63,6 +63,7 @@ + diff --git a/SRL/SRL.ini b/SRL/SRL.ini index ca4ea1e..f028402 100644 --- a/SRL/SRL.ini +++ b/SRL/SRL.ini @@ -42,6 +42,7 @@ CaseSensitive=true NotCachedOnly=false SetOutEncoding=false AllowEmptyReloads=false +AllowDuplicates=false [WordWrap] Enable=false diff --git a/SRL/Types.cs b/SRL/Types.cs index c6986e6..07e4cfa 100644 --- a/SRL/Types.cs +++ b/SRL/Types.cs @@ -26,6 +26,31 @@ internal static void AppendArray(ref T[] Arr, T Val, bool CheckDouble = false NArr[Arr.Length] = Val; Arr = NArr; } + + enum PipeCommands : byte { + FindMissed = 0, + AddMissed = 1, + FindReload = 2, + AddReload = 3, + GetReload = 4, + True = 5, + False = 6, + AddPtr = 7, + GetPtrs = 8, + EndPipe = 9, + AddMask = 10, + ChkMask = 11, + RldMask = 12, + AdvDB = 13, + GetDBID = 14, + SetDBID = 15, + GetDBIndex = 16 + } + + struct Range { + internal uint Min; + internal uint Max; + } internal struct Quote { public char Start; public char End; @@ -279,6 +304,8 @@ struct SRLSettings { [FieldParmaters(DefaultValue = false, Name = "AllowEmptyReloads;AllowEmptyReload;AllowEmpty;DisableEmptyFilter")] public bool AllowEmptyReloads; + [FieldParmaters(DefaultValue = false, Name = "AllowDuplicates;AllowDuplicate;AllowDuplicateReloads;AcceptDuplicates;AcceptDuplicate")] + public bool AllowDuplicates; } [FieldParmaters(Name = "WordWrap")] diff --git a/SRL/Variables.cs b/SRL/Variables.cs index 7f7484c..5c3cf4e 100644 --- a/SRL/Variables.cs +++ b/SRL/Variables.cs @@ -19,38 +19,15 @@ partial class StringReloader { const string CfgName = "StringReloader"; const string ServiceMask = "StringReloaderPipeID-{0}"; + const string ServiceDuplicateFlag = "|Duplicate"; const int CacheLength = 200; - enum PipeCommands : byte { - FindMissed = 0, - AddMissed = 1, - FindReload = 2, - AddReload = 3, - GetReload = 4, - True = 5, - False = 6, - AddPtr = 7, - GetPtrs = 8, - EndPipe = 9, - AddMask = 10, - ChkMask = 11, - RldMask = 12, - AdvDB = 13, - GetDBID = 14, - SetDBID = 15 - } - - struct Range { - internal uint Min; - internal uint Max; - } - static int GamePID = System.Diagnostics.Process.GetCurrentProcess().Id; static Dictionary CharRld; static Dictionary UnkRld; - static Dictionary MskRld = null; + static IDictionary MskRld = null; static Dictionary DBNames = null; static Dictionary FontReplaces = new Dictionary(); @@ -103,6 +80,7 @@ struct Range { static bool CaseSensitive = false; static bool NotCachedOnly = false; static bool AllowEmpty = false; + static bool AllowDuplicates = false; static bool OverlayEnabled = false; static bool OverlayInitialized = false; @@ -207,14 +185,14 @@ struct Range { static int LastDBID = 0; static int DBID = 0; - static List> Databases = null; - static Dictionary StrRld { + static List> Databases = null; + static IDictionary StrRld { get { if (Databases == null) - Databases = new List>() { null }; + Databases = new List>() { null }; if (DBID >= Databases.Count) - Databases.Add(new Dictionary()); + Databases.Add(CreateDictionary()); if (DBID >= Databases.Count) throw new Exception("GET - Invalid Database ID"); @@ -223,10 +201,10 @@ static Dictionary StrRld { } set { if (Databases == null) - Databases = new List>() { null }; + Databases = new List>() { null }; if (DBID >= Databases.Count) - Databases.Add(new Dictionary()); + Databases.Add(CreateDictionary()); if (DBID >= Databases.Count) throw new Exception("SET - Invalid Database ID"); diff --git a/SRLTracer/Form1.cs b/SRLTracer/Form1.cs index 8387634..1427380 100644 --- a/SRLTracer/Form1.cs +++ b/SRLTracer/Form1.cs @@ -1,6 +1,8 @@ //#define DEBUG using SacanaWrapper; +using SRL; using System; +using System.Collections.Generic; using System.Windows.Forms; namespace SRLTracer {