Propagate changes to the decompiler package structure into un.rpyc, s…

…mall grammar fixes, and fix an error in minimize.py related to obfsucation.

decompiler/renpycompat.py

@@ -1,5 +1,5 @@
 # This module holds some special classes and shorthand functions for support of renpy compatiblity.
-# They're separate so there will be less code duplication, simpler dependencys between files and
+# They're separate so there will be less code duplication, simpler dependencies between files and
 # to avoid middle-of-file imports.
 
 from . import magic
@@ -12,7 +12,7 @@
 SPECIAL_CLASSES = [set, frozenset]
 
 
-# ren'py _annoyingly_ enables fix_imports even in ren'py v8,and still defaults to pickle protocol 2.
+# ren'py _annoyingly_ enables fix_imports even in ren'py v8 and still defaults to pickle protocol 2.
 # so set/frozenset get mapped to the wrong location (__builtins__ instead of builtins)
 # we don't want to enable that option as we want control over what the pickler is allowed to
 # unpickle

un.rpyc/compile.py

@@ -114,12 +114,6 @@ def Exec(code):
                 sys.files.append((abspath, fn, dir, bindata))
 ''', globals())
 
-_0 # decompiler shim
-_1 # decompiler.translate shim
-_2 # decompiler.astdump shim
-_3 # decompiler.util
-_4 # decompiler.magic
-
 renpy_modules = modules.copy()
 
 exec('''
@@ -133,21 +127,22 @@ def Exec(code):
 package = __package__
 __package__ = ""
 
-import traceback as traceback
-import codecs as codecs
-
-from decompiler.magic import fake_package, remove_fake_package
-# fake the prescense of renpy
-fake_package("renpy")
-
-_5 # decompiler.testcasedecompiler
-_6 # decompiler.atldecompiler
-_7 # decompiler.sl2decompiler
-_8 # decompiler
-_9 # unrpyc
+_0 # decompiler shim
+_1 # decompiler.translate shim
+_2 # decompiler.astdump shim
+_3 # decompiler.util
+_4 # decompiler.magic
+_5 # decompiler.renpycompat
+_6 # decompiler.testcasedecompiler
+_7 # decompiler.atldecompiler
+_8 # decompiler.sl2decompiler
+_9 # decompiler
+_10 # unrpyc
 
 from unrpyc import decompile_game
 decompile_game()
+
+from decompiler.magic import remove_fake_package
 remove_fake_package("renpy")
 
 modules.update(renpy_modules)
@@ -161,6 +156,7 @@ def Exec(code):
     p.DeclareModule("decompiler.astdump"),
     Module("decompiler.util", BASE_FOLDER / "decompiler/util.py"),
     Module("decompiler.magic", BASE_FOLDER / "decompiler/magic.py", False),
+    Module("decompiler.renpycompat", BASE_FOLDER / "decompiler/renpycompat.py"),
     Module("decompiler.testcasedecompiler", BASE_FOLDER / "decompiler/testcasedecompiler.py"),
     Module("decompiler.atldecompiler", BASE_FOLDER / "decompiler/atldecompiler.py"),
     Module("decompiler.sl2decompiler", BASE_FOLDER / "decompiler/sl2decompiler.py"),
@@ -176,15 +172,13 @@ def Exec(code):
 
 decompiler_rpyb = p.ExecTranspile(base + "(None, [])", decompiler_items)
 
-rpy_one = p.GetItem(p.Sequence(
+rpy_items = p.GetItem(p.Sequence(
     p.DeclareModule("decompiler"),
     p.DeclareModule("decompiler.translate"),
     p.DeclareModule("decompiler.astdump"),
     Module("decompiler.util", BASE_FOLDER / "decompiler/util.py"),
     Module("decompiler.magic", BASE_FOLDER / "decompiler/magic.py", False),
-), "decompiler.magic")
-
-rpy_two = p.GetItem(p.Sequence(
+    Module("decompiler.renpycompat", BASE_FOLDER / "decompiler/renpycompat.py"),
     Module("decompiler.testcasedecompiler", BASE_FOLDER / "decompiler/testcasedecompiler.py"),
     Module("decompiler.atldecompiler", BASE_FOLDER / "decompiler/atldecompiler.py"),
     Module("decompiler.sl2decompiler", BASE_FOLDER / "decompiler/sl2decompiler.py"),
@@ -221,24 +215,19 @@ def Exec(code):
                     bindata = renpy.game.script.read_rpyc_data(file, 1)
                     sys.files.append((abspath, fn, dir, bindata))
 
-    # ???
-
-    magic = pickle.loads(base64.b64decode({}))
-
     renpy_modules = sys.modules.copy()
     for i in renpy_modules:
         if "renpy" in i and not "renpy.execution" in i:
             sys.modules.pop(i)
 
     renpy_loader = sys.meta_path.pop()
 
-    magic.fake_package("renpy")
-
     # ?????????
     unrpyc = pickle.loads(base64.b64decode({}))
     unrpyc.decompile_game()
 
-    magic.remove_fake_package("renpy")
+    from decompiler.magic import remove_fake_package
+    remove_fake_package("renpy")
 
     sys.modules.update(renpy_modules)
     sys.meta_path.append(renpy_loader)
@@ -257,8 +246,7 @@ def Exec(code):
 9)
 
 unrpy = rpy_base.format(
-    repr(base64.b64encode(p.optimize(p.dumps(rpy_one, protocol), protocol))),
-    repr(base64.b64encode(p.optimize(p.dumps(rpy_two, protocol), protocol)))
+    repr(base64.b64encode(p.optimize(p.dumps(rpy_items, protocol), protocol)))
 )
 
 
@@ -299,9 +287,6 @@ def Exec(code):
     with (PACK_FOLDER / "un.dis3").open("w", encoding="utf-8") as f:
         p.pprint(decompiler_rpyc, f)
 
-    with (PACK_FOLDER / "un.rpy.dis1").open("w", encoding="utf-8") as f:
-        pickletools.dis(p.dumps(rpy_one, protocol), f)
-
-    with (PACK_FOLDER / "un.rpy.dis2").open("w", encoding="utf-8") as f:
-        pickletools.dis(p.dumps(rpy_one, protocol), f)
+    with (PACK_FOLDER / "un.rpy.dis").open("w", encoding="utf-8") as f:
+        pickletools.dis(p.dumps(rpy_items, protocol), f)
 

un.rpyc/minimize.py

@@ -26,13 +26,14 @@
 
 # Main API
 
-def minimize(code, remove_docstrings=True, obfuscate_globals=False,
+def minimize(code, remove_docs=True, obfuscate_globals=False,
              obfuscate_builtins=False, obfuscate_imports=False):
     # convert the code to an AST
     tree = ast.parse(code)
-    if remove_docstrings:
+    if remove_docs:
         # optimize the ast by removing docstrings
         tree = DocstringRemover().visit(tree)
+        # also remove annotations
     # perform variable name optimization
     tree = ScopeAnalyzer().analyze(
         tree, not obfuscate_globals, not obfuscate_builtins,
@@ -50,6 +51,11 @@ def visit_Expr(self, node):
         else:
             return self.generic_visit(node)
 
+    def visit_arg(self, node):
+        # also remove annotations
+        node.annotation = None
+        return node
+
 # Scope analysis implementation
 
 BUILTIN  = 0
@@ -283,7 +289,7 @@ def analyze(self, node, protect_globals, protect_builtins, protect_imports, mung
         if not protect_builtins:
             # append the nodes to rename builtins
             extra_nodes = [ast.Assign([ast.Name(value, ast.Store())], ast.Name(key, ast.Load()))
-                           for key, value in self.builtin_scope.bound_vars.iteritems() if key != value]
+                           for key, value in self.builtin_scope.bound_vars.items() if key != value]
             # ensure any "from __future__ import thing" statements are at the start of the
             futures = [future for future in node.body if
                        isinstance(future, ast.ImportFrom) and future.module == "__future__"]
@@ -443,7 +449,7 @@ def visit_arg(self, node, protected=False):
             node.arg = self.new_name(node.arg)
 
         if node.annotation:
-            self.visit(annotation)
+            self.visit(node.annotation)
 
         return node
 

un.rpyc/unrpyc-compile.py

@@ -24,79 +24,10 @@
 import struct
 
 import decompiler
-from decompiler import magic
-
-# these named classes need some special handling for us to be able to reconstruct ren'py ASTs from pickles
-SPECIAL_CLASSES = [set, frozenset]
-
-class oldset(set):
-    __module__ = "__builtin__"
-oldset.__name__ = "set"
-SPECIAL_CLASSES.append(oldset)
-
-class oldfrozenset(frozenset):
-    __module__ = "__builtin__"
-oldfrozenset.__name__ = "frozenset"
-SPECIAL_CLASSES.append(oldfrozenset)
-
-@SPECIAL_CLASSES.append
-class PyExpr(magic.FakeStrict, str):
-    __module__ = "renpy.ast"
-    def __new__(cls, s, filename, linenumber, py=None):
-        self = str.__new__(cls, s)
-        self.filename = filename
-        self.linenumber = linenumber
-        self.py = py
-        return self
-
-@SPECIAL_CLASSES.append
-class PyCode(magic.FakeStrict):
-    __module__ = "renpy.ast"
-    def __setstate__(self, state):
-        if len(state) == 4:
-            (_, self.source, self.location, self.mode) = state
-            self.py = None
-        else:
-            (_, self.source, self.location, self.mode, self.py) = state
-        self.bytecode = None
-
-@SPECIAL_CLASSES.append
-class Sentinel(magic.FakeStrict, object):
-    __module__ = "renpy.object"
-    def __new__(cls, name):
-        obj = object.__new__(cls)
-        obj.name = name
-        return obj
-
-# but they live in renpy.revertable now
-@SPECIAL_CLASSES.append
-class RevertableList(magic.FakeStrict, list):
-    __module__ = "renpy.revertable"
-    def __new__(cls):
-        return list.__new__(cls)
-
-@SPECIAL_CLASSES.append
-class RevertableDict(magic.FakeStrict, dict):
-    __module__ = "renpy.revertable"
-    def __new__(cls):
-        return dict.__new__(cls)
-
-@SPECIAL_CLASSES.append
-class RevertableSet(magic.FakeStrict, set):
-    __module__ = "renpy.revertable"
-    def __new__(cls):
-        return set.__new__(cls)
-
-    def __setstate__(self, state):
-        if isinstance(state, tuple):
-            self.update(state[0].keys())
-        else:
-            self.update(state)
-
-factory = magic.FakeClassFactory(SPECIAL_CLASSES, magic.FakeStrict)
+from decompiler.renpycompat import pickle_safe_loads
 
 def read_ast_from_file(raw_contents):
-    data, stmts = magic.safe_loads(raw_contents, factory, {"_ast", "collections"})
+    _, stmts = pickle_safe_loads(raw_contents)
     return stmts
 
 def ensure_dir(filename):