diff --git a/.github/workflows/python-app.yaml b/.github/workflows/python-app.yaml index c6305c74..39d8e26e 100644 --- a/.github/workflows/python-app.yaml +++ b/.github/workflows/python-app.yaml @@ -24,8 +24,17 @@ jobs: python-version: "3.9" - name: Test by decompiling a script and building un.rpyc run: | + # test the command line tool ./unrpyc.py --clobber "testcases/compiled/**/*.rpyc" diff -ur testcases/expected testcases/compiled -x "*.rpyc" + # compile un.rpyc/rpy/rpyb cd un.rpyc; ./compile.py -p 1 cd .. + # test all the different iterations of it + ./testcases/test_un_rpyc.py --unrpyc un.rpyc/un.rpyc "testcases/compiled/**/*.rpyc" + diff -ur testcases/expected testcases/compiled -x "*.rpyc" + ./testcases/test_un_rpyc.py --unrpyb un.rpyc/bytecode-39.rpyb "testcases/compiled/**/*.rpyc" + diff -ur testcases/expected testcases/compiled -x "*.rpyc" + ./testcases/test_un_rpyc.py --unrpy un.rpyc/un.rpy "testcases/compiled/**/*.rpyc" + diff -ur testcases/expected testcases/compiled -x "*.rpyc" diff --git a/testcases/test_un_rpyc.py b/testcases/test_un_rpyc.py new file mode 100755 index 00000000..dfa20afe --- /dev/null +++ b/testcases/test_un_rpyc.py @@ -0,0 +1,298 @@ +#!/usr/bin/env python + +# It is hard to test un.rpyc with a full renpy instance. But some automated testing would be nice. +# Therefore, this file attempts to construct the absolute minimum renpy environment necessary to +# load un.rpyc + +import types +import sys +import os +import zlib +import argparse +import glob +import io +import struct +import pickle +import traceback + +from pathlib import Path + + +SCRIPT_VERSION = 5003000 +SCRIPT_KEY = b"somerandomnonsense" + + +def main(): + parser = argparse.ArgumentParser(description="un.rypc testing framework") + parser.add_argument("file", type=str, nargs='+', help="The files to provide for the test") + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--unrpyc", type=Path, help="Selects the un.rpyc file to test") + group.add_argument("--unrpy", type=Path, help="Selects the un.rpy file to test") + group.add_argument("--unrpyb", type=Path, help="Selects the un.rpyb file to tes") + args = parser.parse_args() + + # we need to mess with the working directory later, so resolve these paths + if args.unrpyc: + args.unrpyc = args.unrpyc.resolve() + if args.unrpy: + args.unrpy = args.unrpy.resolve() + if args.unrpyb: + args.unrpyb = args.unrpyb.resolve() + + # resolve any globs and create our file list + filelist = [] + for file in args.file: + globbed = [Path(i).resolve() for i in glob.iglob(file, recursive=True)] + if not globbed: + raise Exception(f"File not found: {file}") + + filelist.extend(globbed) + + # move to the `testcases` directory + os.chdir(Path(__file__).parent) + + # if the output file already exists, clear it for now + output_path = Path("game/unrpyc.log.txt") + if output_path.exists(): + output_path.unlink() + + # build our ren'py environment + build_renpy_environment(filelist) + + # prevent a false positive in modules loaded by + # importing the latin-1 codec (pickle protocol <3 can cause that to get loaded) + import encodings.latin_1 + + # make some backups of the environment + meta_path = sys.meta_path.copy() + modules = sys.modules.copy() + cwd = Path.cwd() + + # run the respective test case + if args.unrpyc: + test_unrpyc(args.unrpyc) + + elif args.unrpyb: + test_unrpyb(args.unrpyb) + + elif args.unrpy: + test_unrpy(args.unrpy) + + # check if the environment was left clean + assert meta_path == sys.meta_path, "sys.meta_path was changed" + assert modules == sys.modules, "sys.modules was changed" + assert cwd == Path.cwd() + + # validate the expected output + if not output_path.exists(): + raise Exception("No output log file was created") + + with Path("game/unrpyc.log.txt").open("r", encoding="utf-8") as f: + lines = list(f) + + # validate the output format + assert lines[0] == "Beginning decompiling\n" + assert lines[-1] == "end decompiling\n" + + # rudinemtary parse of the log file contents + success = [] + failure = [] + for line in lines[1:-1]: + if line.startswith("Failed at decompiling "): + failure.append(line[len("Failed at decompiling "):].rstrip()) + + elif line.startswith("Decompiled "): + success.append(line[len("Decompiled "):].rstrip()) + + assert len(failure) == 0, f"{len(failure)} files failed decompilation" + assert len(success) == len(filelist), "Not all files were decompiled" + + print(f"{len(success)} files successfully decompiled") + +# testing protocols for the different files + +def test_unrpyc(unrpyc): + from renpy.game import script + from renpy import loader + + with loader.load(unrpyc) as f: + contents = script.read_rpyc_data(f, 1) + + # magic happens here + data, stmts = loads(contents) + + # verify expected output + assert data == dict(version=SCRIPT_VERSION, key=SCRIPT_KEY) + assert stmts == [] + +def test_unrpyb(unrpyb): + from renpy import loader + + with loader.load(unrpyb) as f: + contents = zlib.decompress(f.read()) + + # magic happens here + version, cache = loads(contents) + + # verify expected output + assert version is None + assert cache == [] + +def test_unrpy(unrpy): + # this one is the hardest to mockup properly, as we don't want to ship an entire ren'py runtime + # luckily, this file basically is just a python file with a header. + # so we strip the header and extra indentation, and then just compile -> execute it. + from renpy import loader + + with loader.load(unrpy) as f: + decoded = f.read().decode("utf-8") + + # need to strip the "init python early hide:" header and remove one layer of indentation + contents = [] + for line in decoded.splitlines(keepends=True): + if line.startswith(" "): + # if indented, strip one layer of indentation + contents.append(line[4:]) + elif not line.strip(): + # keep empty lines unchanged + contents.append(line) + else: + # comment out "init python early hide"" + contents.append("#" + line) + + contents = "".join(contents) + + # compile unrpy + code = compile(contents, unrpy, "exec") + + # run it + exec(code, {}) + +# utilies + +def loads(buffer): + # pickle wrapper that does the same thing as ren'py + f = io.BytesIO(buffer) + unpickler = pickle._Unpickler(f, fix_imports=True, encoding="utf-8", errors="surrogateescape") + + try: + return unpickler.load() + except Exception as e: + raise + +def build_module(name, **items): + # Construct a module with name `name`, and module contents `items` + module = types.ModuleType(name, f"totally legit module {name}") + module.__dict__.update(items) + sys.modules[name] = module + return module + +class RenpyLoader: + __module__ = "renpy.loader" + + # A meta path finder that does literally nothing. + def find_spec(self, fullname, path, target=None): + return None + + def invalidate_caches(self): + pass + +# shenanigans + +def build_renpy_environment(filelist): + # construct a module environment as it would be found in renpy + # filelist: a list of Path objects that will be returned + # by renpy.loader.listdirfiles() + + def load(name, directory=None, tl=True): + # renpy.loader.load + # essentially ren'py's `open` + + # unrpyc doesn't use these + assert directory is None + assert tl is True + + return open(name, "rb") + + def listdirfiles(common=True): + # renpy.loader.listdirfiles + # returns a list of (directory, filename) + # filename can be a longer relative path + # directory is only the root directory from a search path + # + # un.rpyc expects to be able to load a file just from its filename, + # and uses either dirname/filename or ./filename to put it back + # + # if common is True: the list also contains common files + # (engine files loaded by the game) + root = Path.cwd() + return [(str(root), str(p.relative_to(root))) for p in filelist] + # alternative + # return [(None, str(p.relative_to(root, walk_up=True))) for p in filelist] + + class Script: + # renpy.game.script = renpy.script.Script() + def __init__(self): + self.key = SCRIPT_KEY + + def read_rpyc_data(self, f, slot): + # reads the data stored in slot `slot` of a rpyc file + + # adapted from unrpyc.py - read_ast_from_file + + raw_contents = f.read() + + # ren'py 8 only uses RPYC 2 files, but un.rpyc is a RPYC 1 file, so we need + # to support both for now still + if raw_contents.startswith(b"RENPY RPC2"): + + # parse the archive structure + pos = 10 + chunks = {} + while True: + slot_index, start, length = struct.unpack("III", raw_contents[pos: pos + 12]) + if slot_index == 0: + break + + pos += 12 + + chunks[slot_index] = raw_contents[start: start + length] + + slot_contents = chunks[slot] + + else: + if slot != 1: + return None + + slot_contents = raw_contents + + return zlib.decompress(slot_contents) + + + game = build_module( + "renpy.game", + script=Script(), + ) + loader = build_module( + "renpy.loader", + load=load, + listdirfiles=listdirfiles + ) + renpy = build_module( + "renpy", + game=game, + loader=loader, + script_version=SCRIPT_VERSION + ) + + # modern ren'py versions tend to have two loaders inserted at the start of sys.meta_path + sys.meta_path.insert(0, RenpyLoader()) + sys.meta_path.insert(0, RenpyLoader()) + + # older versions have a single one inserted at the end of sys.meta_path + sys.meta_path.append(RenpyLoader()) + + return renpy + +if __name__ == '__main__': + main()