From 307120296a471b01cd7f9207a78e7b2e8f41c55c Mon Sep 17 00:00:00 2001 From: Henrique Gemignani Passos Lima Date: Thu, 3 Aug 2023 02:01:49 +0300 Subject: [PATCH 1/6] Progress in BMSAD for SR --- .../formats/bmsad.py | 53 +++++++++++++++---- 1 file changed, 42 insertions(+), 11 deletions(-) diff --git a/src/mercury_engine_data_structures/formats/bmsad.py b/src/mercury_engine_data_structures/formats/bmsad.py index d37cfef9..9c648156 100644 --- a/src/mercury_engine_data_structures/formats/bmsad.py +++ b/src/mercury_engine_data_structures/formats/bmsad.py @@ -8,6 +8,7 @@ Float32l, Hex, IfThenElse, + Int8ul, Int16ul, Int32ul, PrefixedArray, @@ -23,6 +24,15 @@ from mercury_engine_data_structures.formats.property_enum import PropertyEnum from mercury_engine_data_structures.game_check import Game + +def SR_or_Dread(sr, dread): + return IfThenElse( + game_check.current_game_at_most(Game.SAMUS_RETURNS), + sr, + dread, + ) + + FunctionArgument = Struct( type=Char, value=Switch( @@ -40,7 +50,11 @@ name=StrId, unk=Int16ul, params=common_types.DictAdapter(common_types.make_vector( - common_types.DictElement(FunctionArgument, key=PropertyEnum) + common_types.DictElement( + FunctionArgument, + # FIXME: PropertyEnum should properly support SR + key=SR_or_Dread(Hex(Int32ul), PropertyEnum), + ) )), )) @@ -150,7 +164,7 @@ def component_type(this): )) ), functions=Functions, - dependencies=Dependencies() + dependencies=Dependencies(), ) CCharClass = Struct( @@ -189,13 +203,32 @@ def component_type(this): "CActorDef": CActorDef } # -BMSAD = Struct( + +SR_Component = Struct( + type=StrId, + unk_1=Hex(Int32ul)[2], + functions=Functions, + unk_2=Hex(Int32ul), + unk_3=construct.If(construct.this.type == "CAnimationComponent", Hex(Int32ul)), +) + +BMSAD_SR = Struct( _magic=Const(b"MSAD"), - version=IfThenElse( - game_check.current_game_at_most(Game.SAMUS_RETURNS), - Const(0x002C0001, Hex(Int32ul)), - Const(0x0200000F, Hex(Int32ul)), - ), + version=Const(0x002C0001, Hex(Int32ul)), + + name=StrId, + model_name=StrId, + unk_1=Hex(Int32ul)[14], + unk_2=Hex(Int8ul)[3], + components=make_dict(SR_Component), + + rest=construct.GreedyBytes, + _end=construct.Terminated, +) + +BMSAD_Dread = Struct( + _magic=Const(b"MSAD"), + version=Const(0x0200000F, Hex(Int32ul)), # # gameeditor::CGameModelRoot # root_type=construct.Const('Root', PropertyEnum), @@ -209,11 +242,9 @@ def component_type(this): property_types, ErrorWithMessage(lambda ctx: f"Unknown property type: {ctx.type}"), ), - # rest=Peek(construct.GreedyBytes), - - # z=Probe(), _end=construct.Terminated, ) +BMSAD = SR_or_Dread(BMSAD_SR, BMSAD_Dread) # BMSAD = game_model_root.create('CActorDef', 0x02000031) From 83ea69f45b237bc0b0bdc6a96d7d9fdb9b93c3a4 Mon Sep 17 00:00:00 2001 From: Henrique Gemignani Passos Lima Date: Fri, 4 Aug 2023 15:27:55 +0300 Subject: [PATCH 2/6] Fixes for bmsad --- .../formats/bmsad.py | 73 +++++++++++++------ 1 file changed, 52 insertions(+), 21 deletions(-) diff --git a/src/mercury_engine_data_structures/formats/bmsad.py b/src/mercury_engine_data_structures/formats/bmsad.py index 9c648156..fe6bdb62 100644 --- a/src/mercury_engine_data_structures/formats/bmsad.py +++ b/src/mercury_engine_data_structures/formats/bmsad.py @@ -74,7 +74,7 @@ def find_charclass_for_type(type_name: str): ) -def Dependencies(): +def DreadDependencies(): component_dependencies = { "CFXComponent": make_vector(Struct( "file" / StrId, @@ -132,7 +132,23 @@ def component_type(this): return Switch(component_type, component_dependencies) -Component = Struct( +ExtraFields = common_types.DictAdapter(common_types.make_vector( + common_types.DictElement(Struct( + "type" / StrId, + "value" / Switch( + construct.this.type, + { + "bool": Flag, + "string": StrId, + "float": Float, + }, + ErrorWithMessage(lambda ctx: f"Unknown argument type: {ctx.type}", construct.SwitchError) + ) + )) +)) + + +DreadComponent = Struct( type=StrId, unk_1=Array(2, Hex(Int32ul)), fields=PrefixedAllowZeroLen( @@ -149,22 +165,10 @@ def component_type(this): ), extra_fields=construct.If( lambda this: type_lib.is_child_of(this.type, "CComponent"), - common_types.DictAdapter(common_types.make_vector( - common_types.DictElement(Struct( - "type" / StrId, - "value" / Switch( - construct.this.type, - { - "bool": Flag, - "string": StrId - }, - ErrorWithMessage(lambda ctx: f"Unknown argument type: {ctx.type}", construct.SwitchError) - ) - )) - )) + ExtraFields, ), functions=Functions, - dependencies=Dependencies(), + dependencies=DreadDependencies(), ) CCharClass = Struct( @@ -179,7 +183,7 @@ def component_type(this): unk_6=StrId, unk_7=Byte, - components=make_dict(Component), + components=make_dict(DreadComponent), binaries=make_vector(StrId), sources=make_vector(StrId >> Byte), @@ -192,7 +196,7 @@ def component_type(this): sub_actors=PrefixedArray(Int32ul, StrId), unk_4=StrId, - components=make_dict(Component), + components=make_dict(DreadComponent), binaries=make_vector(StrId), sources=make_vector(StrId >> Byte), @@ -204,12 +208,34 @@ def component_type(this): } # + +def SRDependencies(): + component_dependencies = { + "CAnimationComponent": Hex(Int32ul), + "CFXComponent": make_vector(Struct( + "file" / StrId, + "unk1" / Int32ul, + "unk2" / Int32ul, + )), + "CCollisionComponent": Struct( + "file" / StrId, + "unk" / Int16ul + ), + "CGrabComponent": make_vector(Struct( + "unk" / Const(0, Int32ul), + )), + } + + return Switch(construct.this.type, component_dependencies) + + SR_Component = Struct( type=StrId, unk_1=Hex(Int32ul)[2], functions=Functions, - unk_2=Hex(Int32ul), - unk_3=construct.If(construct.this.type == "CAnimationComponent", Hex(Int32ul)), + + extra_fields=ExtraFields, + dependencies=SRDependencies(), ) BMSAD_SR = Struct( @@ -218,8 +244,13 @@ def component_type(this): name=StrId, model_name=StrId, - unk_1=Hex(Int32ul)[14], + unk_1=Hex(Int32ul)[12], unk_2=Hex(Int8ul)[3], + sub_actors=make_vector(StrId), + unk_3=Hex(Int32ul), + + # count=Int32ul, + # comp=(StrId >> SR_Component)[6], components=make_dict(SR_Component), rest=construct.GreedyBytes, From 59680ee98928326a3ee1af56f23ed99e6690e45b Mon Sep 17 00:00:00 2001 From: Henrique Gemignani Passos Lima Date: Fri, 4 Aug 2023 15:59:08 +0300 Subject: [PATCH 3/6] Additional parsing for BMSAD --- .../formats/bmsad.py | 17 +++++++++----- tests/conftest.py | 7 +++++- tests/formats/test_bmsad.py | 22 ++++++++++++++++--- 3 files changed, 37 insertions(+), 9 deletions(-) diff --git a/src/mercury_engine_data_structures/formats/bmsad.py b/src/mercury_engine_data_structures/formats/bmsad.py index fe6bdb62..ec0cf163 100644 --- a/src/mercury_engine_data_structures/formats/bmsad.py +++ b/src/mercury_engine_data_structures/formats/bmsad.py @@ -141,6 +141,9 @@ def component_type(this): "bool": Flag, "string": StrId, "float": Float, + + "int": construct.Int32sl, + "vec3": Float[3], }, ErrorWithMessage(lambda ctx: f"Unknown argument type: {ctx.type}", construct.SwitchError) ) @@ -244,13 +247,17 @@ def SRDependencies(): name=StrId, model_name=StrId, - unk_1=Hex(Int32ul)[12], - unk_2=Hex(Int8ul)[3], + + unk_1=Int8ul, + unk_2=Float[9], + unk_3=Int8ul, + unk_4=Int32ul, + other_magic=Const(0xFFFFFFFF, Hex(Int32ul)), + unk_5=Int16ul, + unk_6=StrId[2], + unk_7=Int8ul, sub_actors=make_vector(StrId), - unk_3=Hex(Int32ul), - # count=Int32ul, - # comp=(StrId >> SR_Component)[6], components=make_dict(SR_Component), rest=construct.GreedyBytes, diff --git a/tests/conftest.py b/tests/conftest.py index 7df8a67e..2c218240 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,7 +18,7 @@ def get_env_or_skip(env_name): return os.environ[env_name] -@pytest.fixture() +@pytest.fixture(scope="session") def samus_returns_path(): return Path(get_env_or_skip("SAMUS_RETURNS_PATH")) @@ -28,6 +28,11 @@ def dread_path(): return Path(get_env_or_skip("DREAD_PATH")) +@pytest.fixture(scope="session") +def samus_returns_tree(samus_returns_path): + return FileTreeEditor(samus_returns_path, Game.SAMUS_RETURNS) + + @pytest.fixture(scope="session") def dread_file_tree(dread_path): return FileTreeEditor(dread_path, Game.DREAD) diff --git a/tests/formats/test_bmsad.py b/tests/formats/test_bmsad.py index 6fd00137..834f46cb 100644 --- a/tests/formats/test_bmsad.py +++ b/tests/formats/test_bmsad.py @@ -4,20 +4,23 @@ import pytest from tests.test_lib import parse_build_compare_editor -from mercury_engine_data_structures import dread_data +from mercury_engine_data_structures import dread_data, samus_returns_data from mercury_engine_data_structures.formats.bmsad import Bmsad +all_sr_bmsad = [name for name in samus_returns_data.all_name_to_asset_id().keys() + if name.endswith(".bmsad")] + all_dread_bmsad = [name for name in dread_data.all_name_to_asset_id().keys() if name.endswith(".bmsad")] -expected_failures = { +expected_dread_failures = { "actors/props/pf_mushr_fr/charclasses/pf_mushr_fr.bmsad", } @pytest.mark.parametrize("bmsad_path", all_dread_bmsad) def test_compare_dread_all(dread_file_tree, bmsad_path): - if bmsad_path in expected_failures: + if bmsad_path in expected_dread_failures: expectation = pytest.raises(construct.ConstructError) else: expectation = contextlib.nullcontext() @@ -26,3 +29,16 @@ def test_compare_dread_all(dread_file_tree, bmsad_path): parse_build_compare_editor( Bmsad, dread_file_tree, bmsad_path ) + + +@pytest.mark.parametrize("bmsad_path", all_sr_bmsad) +def test_compare_sr_all(samus_returns_tree, bmsad_path): + if not samus_returns_tree.does_asset_exists(bmsad_path): + return pytest.skip() + + expectation = contextlib.nullcontext() + + with expectation: + parse_build_compare_editor( + Bmsad, samus_returns_tree, bmsad_path + ) From fe40a295c8190aa44973627fa52ef2e66772c4ab Mon Sep 17 00:00:00 2001 From: Henrique Gemignani Passos Lima Date: Fri, 4 Aug 2023 16:00:26 +0300 Subject: [PATCH 4/6] Oops don't skip components --- src/mercury_engine_data_structures/formats/bmsad.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mercury_engine_data_structures/formats/bmsad.py b/src/mercury_engine_data_structures/formats/bmsad.py index ec0cf163..2d19cc3a 100644 --- a/src/mercury_engine_data_structures/formats/bmsad.py +++ b/src/mercury_engine_data_structures/formats/bmsad.py @@ -257,6 +257,7 @@ def SRDependencies(): unk_6=StrId[2], unk_7=Int8ul, sub_actors=make_vector(StrId), + unk_8=Int32ul, components=make_dict(SR_Component), From 0585b17754f775c0e5696af839be2a7329587a2c Mon Sep 17 00:00:00 2001 From: Henrique Gemignani Passos Lima Date: Fri, 4 Aug 2023 17:22:32 +0300 Subject: [PATCH 5/6] Add allow_duplicates to DictAdapter --- src/mercury_engine_data_structures/common_types.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/mercury_engine_data_structures/common_types.py b/src/mercury_engine_data_structures/common_types.py index 72f5f05c..7257e9ca 100644 --- a/src/mercury_engine_data_structures/common_types.py +++ b/src/mercury_engine_data_structures/common_types.py @@ -48,16 +48,24 @@ def items(self): class DictAdapter(Adapter): + def __init__(self, subcon, *, allow_duplicates: bool = False): + super().__init__(subcon) + self.allow_duplicates = allow_duplicates + def _decode(self, obj: construct.ListContainer, context, path): result = construct.Container() for item in obj: key = item.key if key in result: + if self.allow_duplicates: + return obj raise construct.ConstructError(f"Key {key} found twice in object", path) result[key] = item.value return result def _encode(self, obj: construct.Container, context, path): + if self.allow_duplicates and isinstance(obj, list): + return obj return construct.ListContainer( construct.Container(key=type_, value=item) for type_, item in obj.items() From ac5d12920a980abb8e8bdf7f3793670876ab3ce0 Mon Sep 17 00:00:00 2001 From: Henrique Gemignani Passos Lima Date: Fri, 4 Aug 2023 17:22:40 +0300 Subject: [PATCH 6/6] Parse the rest of BMSAD --- .../formats/bmsad.py | 55 ++++++++++++++++++- tests/formats/test_bmsad.py | 9 ++- 2 files changed, 60 insertions(+), 4 deletions(-) diff --git a/src/mercury_engine_data_structures/formats/bmsad.py b/src/mercury_engine_data_structures/formats/bmsad.py index 2d19cc3a..391830d7 100644 --- a/src/mercury_engine_data_structures/formats/bmsad.py +++ b/src/mercury_engine_data_structures/formats/bmsad.py @@ -148,7 +148,7 @@ def component_type(this): ErrorWithMessage(lambda ctx: f"Unknown argument type: {ctx.type}", construct.SwitchError) ) )) -)) +), allow_duplicates=True) DreadComponent = Struct( @@ -214,7 +214,7 @@ def component_type(this): def SRDependencies(): component_dependencies = { - "CAnimationComponent": Hex(Int32ul), + "CAnimationComponent": make_vector(StrId), "CFXComponent": make_vector(Struct( "file" / StrId, "unk1" / Int32ul, @@ -225,9 +225,54 @@ def SRDependencies(): "unk" / Int16ul ), "CGrabComponent": make_vector(Struct( - "unk" / Const(0, Int32ul), + a=StrId, + b=StrId, + c=StrId, + d=Hex(Int32ul)[2], + e=Float[8], + f=Hex(Int32ul)[9], )), + "CGlowflyAnimationComponent": make_vector(StrId), + "CSceneModelAnimationComponent": make_dict(make_dict(StrId)), + + "CBillboardComponent": Struct( + "id1" / StrId, + "unk1" / make_dict(Struct( + "unk1" / Int32ul[3], + "unk2" / Byte, + "unk3" / Int32ul[2], + "unk4" / Float32l + )), + "id2" / StrId, + "unk3" / make_vector(Struct( + "id" / StrId, + "unk1" / Byte, + "unk2" / make_vector(Struct( + "a" / Float, + "b" / Float, + "c" / Float, + )), + )), + ), + + "CSwarmControllerComponent": Struct( + "unk1" / make_vector(StrId), + "unk2" / make_vector(StrId), + "unk3" / make_vector(StrId) + ), } + for dep in [ + "CTsumuriAcidDroolCollision", + "CBillboardCollisionComponent", + "CQueenPlasmaArcCollision", + ]: + component_dependencies[dep] = component_dependencies["CCollisionComponent"] + + for dep in [ + "CFlockingSwarmControllerComponent", + "CBeeSwarmControllerComponent", + ]: + component_dependencies[dep] = component_dependencies["CSwarmControllerComponent"] return Switch(construct.this.type, component_dependencies) @@ -259,8 +304,12 @@ def SRDependencies(): sub_actors=make_vector(StrId), unk_8=Int32ul, + # count=Int32ul, + # component=(StrId >> SR_Component)[4], components=make_dict(SR_Component), + binaries=make_vector(StrId), + rest=construct.GreedyBytes, _end=construct.Terminated, ) diff --git a/tests/formats/test_bmsad.py b/tests/formats/test_bmsad.py index 834f46cb..b031885e 100644 --- a/tests/formats/test_bmsad.py +++ b/tests/formats/test_bmsad.py @@ -16,6 +16,10 @@ expected_dread_failures = { "actors/props/pf_mushr_fr/charclasses/pf_mushr_fr.bmsad", } +expected_sr_failures = { + "actors/items/adn/charclasses/adn.bmsad", + "actors/props/ridleystorm/charclasses/ridleystorm.bmsad", +} @pytest.mark.parametrize("bmsad_path", all_dread_bmsad) @@ -36,7 +40,10 @@ def test_compare_sr_all(samus_returns_tree, bmsad_path): if not samus_returns_tree.does_asset_exists(bmsad_path): return pytest.skip() - expectation = contextlib.nullcontext() + if bmsad_path in expected_sr_failures: + expectation = pytest.raises(construct.core.ConstError) + else: + expectation = contextlib.nullcontext() with expectation: parse_build_compare_editor(