Add un.rpyc/rpy/rpyb testing harness. It just emulates running them i…

…nside ren'py, but that's enough to catch most issues.

.github/workflows/python-app.yaml

@@ -24,8 +24,17 @@ jobs:
         python-version: "3.9"
     - name: Test by decompiling a script and building un.rpyc
       run: |
+        # test the command line tool
         ./unrpyc.py --clobber "testcases/compiled/**/*.rpyc"
         diff -ur testcases/expected testcases/compiled -x "*.rpyc"
+        # compile un.rpyc/rpy/rpyb
         cd un.rpyc;
         ./compile.py -p 1
         cd ..
+        # test all the different iterations of it
+        ./testcases/test_un_rpyc.py --unrpyc un.rpyc/un.rpyc "testcases/compiled/**/*.rpyc"
+        diff -ur testcases/expected testcases/compiled -x "*.rpyc"
+        ./testcases/test_un_rpyc.py --unrpyb un.rpyc/bytecode-39.rpyb "testcases/compiled/**/*.rpyc"
+        diff -ur testcases/expected testcases/compiled -x "*.rpyc"
+        ./testcases/test_un_rpyc.py --unrpy un.rpyc/un.rpy "testcases/compiled/**/*.rpyc"
+        diff -ur testcases/expected testcases/compiled -x "*.rpyc"

testcases/test_un_rpyc.py

@@ -0,0 +1,298 @@
+#!/usr/bin/env python
+
+# It is hard to test un.rpyc with a full renpy instance. But some automated testing would be nice.
+# Therefore, this file attempts to construct the absolute minimum renpy environment necessary to
+# load un.rpyc
+
+import types
+import sys
+import os
+import zlib
+import argparse
+import glob
+import io
+import struct
+import pickle
+import traceback
+
+from pathlib import Path
+
+
+SCRIPT_VERSION = 5003000
+SCRIPT_KEY = b"somerandomnonsense"
+
+
+def main():
+    parser = argparse.ArgumentParser(description="un.rypc testing framework")
+    parser.add_argument("file", type=str, nargs='+', help="The files to provide for the test")
+    group = parser.add_mutually_exclusive_group(required=True)
+    group.add_argument("--unrpyc", type=Path, help="Selects the un.rpyc file to test")
+    group.add_argument("--unrpy", type=Path, help="Selects the un.rpy file to test")
+    group.add_argument("--unrpyb", type=Path, help="Selects the un.rpyb file to tes")
+    args = parser.parse_args()
+
+    # we need to mess with the working directory later, so resolve these paths
+    if args.unrpyc:
+        args.unrpyc = args.unrpyc.resolve()
+    if args.unrpy:
+        args.unrpy = args.unrpy.resolve()
+    if args.unrpyb:
+        args.unrpyb = args.unrpyb.resolve()
+
+    # resolve any globs and create our file list
+    filelist = []
+    for file in args.file:
+        globbed = [Path(i).resolve() for i in glob.iglob(file, recursive=True)]
+        if not globbed:
+            raise Exception(f"File not found: {file}")
+
+        filelist.extend(globbed)
+
+    # move to the `testcases` directory
+    os.chdir(Path(__file__).parent)
+
+    # if the output file already exists, clear it for now
+    output_path = Path("game/unrpyc.log.txt")
+    if output_path.exists():
+        output_path.unlink()
+
+    # build our ren'py environment
+    build_renpy_environment(filelist)
+
+    # prevent a false positive in modules loaded by
+    # importing the latin-1 codec (pickle protocol <3 can cause that to get loaded)
+    import encodings.latin_1
+
+    # make some backups of the environment
+    meta_path = sys.meta_path.copy()
+    modules = sys.modules.copy()
+    cwd = Path.cwd()
+
+    # run the respective test case
+    if args.unrpyc:
+        test_unrpyc(args.unrpyc)
+
+    elif args.unrpyb:
+        test_unrpyb(args.unrpyb)
+
+    elif args.unrpy:
+        test_unrpy(args.unrpy)
+
+    # check if the environment was left clean
+    assert meta_path == sys.meta_path, "sys.meta_path was changed"
+    assert modules == sys.modules, "sys.modules was changed"
+    assert cwd == Path.cwd()
+
+    # validate the expected output
+    if not output_path.exists():
+        raise Exception("No output log file was created")
+
+    with Path("game/unrpyc.log.txt").open("r", encoding="utf-8") as f:
+        lines = list(f)
+
+    # validate the output format
+    assert lines[0] == "Beginning decompiling\n"
+    assert lines[-1] == "end decompiling\n"
+
+    # rudinemtary parse of the log file contents
+    success = []
+    failure = []
+    for line in lines[1:-1]:
+        if line.startswith("Failed at decompiling "):
+            failure.append(line[len("Failed at decompiling "):].rstrip())
+
+        elif line.startswith("Decompiled "):
+            success.append(line[len("Decompiled  "):].rstrip())
+
+    assert len(failure) == 0, f"{len(failure)} files failed decompilation"
+    assert len(success) == len(filelist), "Not all files were decompiled"
+
+    print(f"{len(success)} files successfully decompiled")
+
+# testing protocols for the different files
+
+def test_unrpyc(unrpyc):
+    from renpy.game import script
+    from renpy import loader
+
+    with loader.load(unrpyc) as f:
+        contents = script.read_rpyc_data(f, 1)
+
+    # magic happens here
+    data, stmts = loads(contents)
+
+    # verify expected output
+    assert data == dict(version=SCRIPT_VERSION, key=SCRIPT_KEY)
+    assert stmts == []
+
+def test_unrpyb(unrpyb):
+    from renpy import loader
+
+    with loader.load(unrpyb) as f:
+        contents = zlib.decompress(f.read())
+
+    # magic happens here
+    version, cache = loads(contents)
+
+    # verify expected output
+    assert version is None
+    assert cache == []
+
+def test_unrpy(unrpy):
+    # this one is the hardest to mockup properly, as we don't want to ship an entire ren'py runtime
+    # luckily, this file basically is just a python file with a header.
+    # so we strip the header and extra indentation, and then just compile -> execute it.
+    from renpy import loader
+
+    with loader.load(unrpy) as f:
+        decoded = f.read().decode("utf-8")
+
+    # need to strip the "init python early hide:" header and remove one layer of indentation
+    contents = []
+    for line in decoded.splitlines(keepends=True):
+        if line.startswith("    "):
+            # if indented, strip one layer of indentation
+            contents.append(line[4:])
+        elif not line.strip():
+            # keep empty lines unchanged
+            contents.append(line)
+        else:
+            # comment out "init python early hide""
+            contents.append("#" + line)
+
+    contents = "".join(contents)
+
+    # compile unrpy
+    code = compile(contents, unrpy, "exec")
+
+    # run it
+    exec(code, {})
+
+# utilies
+
+def loads(buffer):
+    # pickle wrapper that does the same thing as ren'py
+    f = io.BytesIO(buffer)
+    unpickler = pickle._Unpickler(f, fix_imports=True, encoding="utf-8", errors="surrogateescape")
+
+    try:
+        return unpickler.load()
+    except Exception as e:
+        raise
+
+def build_module(name, **items):
+    # Construct a module with name `name`, and module contents `items`
+    module = types.ModuleType(name, f"totally legit module {name}")
+    module.__dict__.update(items)
+    sys.modules[name] = module
+    return module
+
+class RenpyLoader:
+    __module__ = "renpy.loader"
+
+    # A meta path finder that does literally nothing.
+    def find_spec(self, fullname, path, target=None):
+        return None
+
+    def invalidate_caches(self):
+        pass
+
+# shenanigans
+
+def build_renpy_environment(filelist):
+    # construct a module environment as it would be found in renpy
+    # filelist: a list of Path objects that will be returned
+    # by renpy.loader.listdirfiles()
+
+    def load(name, directory=None, tl=True):
+        # renpy.loader.load
+        # essentially ren'py's `open`
+
+        # unrpyc doesn't use these
+        assert directory is None
+        assert tl is True
+
+        return open(name, "rb")
+
+    def listdirfiles(common=True):
+        # renpy.loader.listdirfiles
+        # returns a list of (directory, filename)
+        # filename can be a longer relative path
+        # directory is only the root directory from a search path
+        # 
+        # un.rpyc expects to be able to load a file just from its filename,
+        # and uses either dirname/filename or ./filename to put it back
+        #
+        # if common is True: the list also contains common files
+        # (engine files loaded by the game)
+        root = Path.cwd()
+        return [(str(root), str(p.relative_to(root))) for p in filelist]
+        # alternative
+        # return [(None, str(p.relative_to(root, walk_up=True))) for p in filelist]
+
+    class Script:
+        # renpy.game.script = renpy.script.Script()
+        def __init__(self):
+            self.key = SCRIPT_KEY
+
+        def read_rpyc_data(self, f, slot):
+            # reads the data stored in slot `slot` of a rpyc file
+
+            # adapted from unrpyc.py - read_ast_from_file
+
+            raw_contents = f.read()
+
+            # ren'py 8 only uses RPYC 2 files, but un.rpyc is a RPYC 1 file, so we need
+            # to support both for now still
+            if raw_contents.startswith(b"RENPY RPC2"):
+
+                # parse the archive structure
+                pos = 10
+                chunks = {}
+                while True:
+                    slot_index, start, length = struct.unpack("III", raw_contents[pos: pos + 12])
+                    if slot_index == 0:
+                        break
+
+                    pos += 12
+
+                    chunks[slot_index] = raw_contents[start: start + length]
+
+                slot_contents = chunks[slot]
+
+            else:
+                if slot != 1:
+                    return None
+
+                slot_contents = raw_contents
+
+            return zlib.decompress(slot_contents)
+
+
+    game = build_module(
+        "renpy.game",
+        script=Script(),
+    )
+    loader = build_module(
+        "renpy.loader",
+        load=load,
+        listdirfiles=listdirfiles
+    )
+    renpy = build_module(
+        "renpy",
+        game=game,
+        loader=loader,
+        script_version=SCRIPT_VERSION
+    )
+
+    # modern ren'py versions tend to have two loaders inserted at the start of sys.meta_path
+    sys.meta_path.insert(0, RenpyLoader())
+    sys.meta_path.insert(0, RenpyLoader())
+
+    # older versions have a single one inserted at the end of sys.meta_path
+    sys.meta_path.append(RenpyLoader())
+
+    return renpy
+
+if __name__ == '__main__':
+    main()