From 3d970f0d016f85dcd6cf90df9c05c0c32e949a10 Mon Sep 17 00:00:00 2001 From: Ananya2003Gupta Date: Tue, 26 Sep 2023 22:53:12 +0530 Subject: [PATCH] Added --upstream-edm code generation support --- python/podio_class_generator.py | 91 ++++++++++++++---------- python/templates/MutableStruct.jl.jinja2 | 18 ++++- python/templates/ParentModule.jl.jinja2 | 66 +++++++++++++++-- python/templates/macros/params.jinja2 | 9 ++- tests/CMakeLists.txt | 9 ++- 5 files changed, 142 insertions(+), 51 deletions(-) diff --git a/python/podio_class_generator.py b/python/podio_class_generator.py index ff448b9b7..7731aa229 100755 --- a/python/podio_class_generator.py +++ b/python/podio_class_generator.py @@ -28,6 +28,12 @@ Used {yamlfile} to create {nclasses} classes in {installdir}/ Read instructions in the README.md to run your first example! """ +REPORT_TEXT_JULIA = """ + PODIO Data Model + ================ + Used {yamlfile} to create {nclasses} julia files in {installdir}/ + Read instructions in the README.md to run your first example! +""" def get_clang_format(): @@ -134,6 +140,10 @@ def process(self): datamodel = {} datamodel['class'] = DataType(self.package_name.capitalize()) + datamodel['upstream_edm'] = self.upstream_edm + datamodel['upstream_edm_name'] = '' + if self.upstream_edm: + datamodel['upstream_edm_name'] = self.upstream_edm.options["includeSubfolder"].split("/")[-2].capitalize() datamodel['components'] = [] datamodel['datatypes'] = [] for name, component in self.datamodel.components.items(): @@ -149,11 +159,11 @@ def process(self): self._write_edm_def_file() if 'ROOT' in self.io_handlers: + self.prepare_iorules() self._create_selection_xml() self._write_cmake_lists_file() - self.process_schema_evolution() self.print_report() def process_schema_evolution(self): @@ -192,11 +202,16 @@ def print_report(self): """Print a summary report about the generated code""" if not self.verbose: return - - nclasses = 5 * len(self.datamodel.datatypes) + len(self.datamodel.components) - text = REPORT_TEXT.format(yamlfile=self.yamlfile, - nclasses=nclasses, - installdir=self.install_dir) + if self.proglang == "julia": + nclasses = len(self.datamodel.datatypes) + len(self.datamodel.components) + 1 + text = REPORT_TEXT_JULIA.format(yamlfile=self.yamlfile, + nclasses=nclasses, + installdir=self.install_dir) + if self.proglang == "cpp": + nclasses = 5 * len(self.datamodel.datatypes) + len(self.datamodel.components) + text = REPORT_TEXT.format(yamlfile=self.yamlfile, + nclasses=nclasses, + installdir=self.install_dir) for summaryline in text.splitlines(): print(summaryline) @@ -250,7 +265,6 @@ def get_fn_format(tmpl): endings = { 'Data': ('h',), - 'Component': ('h',), 'PrintInfo': ('h',), 'MutableStruct': ('jl',), 'ParentModule': ('jl',), @@ -292,29 +306,34 @@ def _process_component(self, name, component): component['includes'] = self._sort_includes(includes) component['includes_jl'] = {'struct': sorted(includes_jl)} component['class'] = DataType(name) + component['upstream_edm'] = self.upstream_edm + component['upstream_edm_name'] = '' + if self.upstream_edm: + component['upstream_edm_name'] = self.upstream_edm.options["includeSubfolder"].split("/")[-2].capitalize() - if self.proglang == "cpp": - self._fill_templates('Component', component) if self.proglang == "julia": self._fill_templates('MutableStruct', component) - return component - - # Add potentially older schema for schema evolution - # based on ROOT capabilities for now - if name in self.root_schema_dict: - schema_evolutions = self.root_schema_dict[name] - component = copy.deepcopy(component) - for schema_evolution in schema_evolutions: - if isinstance(schema_evolution, RenamedMember): - for member in component['Members']: - if member.name == schema_evolution.member_name_new: - member.name = schema_evolution.member_name_old - component['class'] = DataType(name + self.old_schema_version) - else: - raise NotImplementedError + if self.proglang == "cpp": self._fill_templates('Component', component) - self.root_schema_component_names.add(name + self.old_schema_version) + + # Add potentially older schema for schema evolution + # based on ROOT capabilities for now + if name in self.root_schema_dict: + schema_evolutions = self.root_schema_dict[name] + component = copy.deepcopy(component) + for schema_evolution in schema_evolutions: + if isinstance(schema_evolution, RenamedMember): + for member in component['Members']: + if member.name == schema_evolution.member_name_new: + member.name = schema_evolution.member_name_old + component['class'] = DataType(name + self.old_schema_version) + else: + raise NotImplementedError + self._fill_templates('Component', component) + self.root_schema_component_names.add(name + self.old_schema_version) + + return component @staticmethod def _replace_component_in_paths(oldname, newname, paths): @@ -345,7 +364,7 @@ def _process_datatype(self, name, definition): if member.array_type in self.root_schema_dict: needs_schema_evolution = True self._replace_component_in_paths(member.array_type, member.array_type + self.old_schema_version, - schema_evolution_datatype['includes_data']) + schema_evolution_datatype['includes_data']) member.full_type = member.full_type.replace(member.array_type, member.array_type + self.old_schema_version) member.array_type = member.array_type + self.old_schema_version @@ -354,7 +373,7 @@ def _process_datatype(self, name, definition): needs_schema_evolution = True # prepare the ROOT I/O rule self._replace_component_in_paths(member.full_type, member.full_type + self.old_schema_version, - schema_evolution_datatype['includes_data']) + schema_evolution_datatype['includes_data']) member.full_type = member.full_type + self.old_schema_version member.bare_type = member.bare_type + self.old_schema_version @@ -416,7 +435,7 @@ def _get_julia_params(datatype): params = set() for relation in datatype['OneToManyRelations'] + datatype['OneToOneRelations']: if not relation.is_builtin: - params.add(relation.bare_type) + params.add((relation.bare_type, relation.full_type)) return list(params) def _preprocess_for_obj(self, datatype): @@ -551,7 +570,11 @@ def _preprocess_datatype(self, name, definition): data['class'] = DataType(name) data['includes_data'] = self._get_member_includes(definition["Members"]) data['includes_jl'] = {'struct': self._get_member_includes(definition["Members"], julia=True)} - data['params_jl'] = sorted(self._get_julia_params(data)) + data['params_jl'] = sorted(self._get_julia_params(data), key=lambda x: x[0]) + data['upstream_edm'] = self.upstream_edm + data['upstream_edm_name'] = '' + if self.upstream_edm: + data['upstream_edm_name'] = self.upstream_edm.options["includeSubfolder"].split("/")[-2].capitalize() self._preprocess_for_class(data) self._preprocess_for_obj(data) self._preprocess_for_collection(data) @@ -678,14 +701,8 @@ def _build_julia_include_for_class(self, classname, include_from: IncludeFrom) - """Return the include statement for julia for this specific class""" if include_from == IncludeFrom.INTERNAL: # If we have an internal include all includes should be relative - inc_folder = '' - if include_from == IncludeFrom.EXTERNAL: - inc_folder = f'{self.upstream_edm.options["includeSubfolder"]}' - if include_from == IncludeFrom.NOWHERE: - # We don't need an include in this case - return '' - - return f'include("{inc_folder}{classname}Struct.jl")' + return f'include("{classname}Struct.jl")' + return '' def _sort_includes(self, includes): """Sort the includes in order to try to have the std includes at the bottom""" diff --git a/python/templates/MutableStruct.jl.jinja2 b/python/templates/MutableStruct.jl.jinja2 index 8ddc13cb2..e003b801b 100644 --- a/python/templates/MutableStruct.jl.jinja2 +++ b/python/templates/MutableStruct.jl.jinja2 @@ -2,13 +2,23 @@ {% for include in includes_jl['struct'] %} {{ include }} {% endfor %} -mutable struct {{ class.bare_type }}Struct{{ params.julia_parameters(params_jl,"" ,"T" ) }} +mutable struct {{ class.bare_type }}Struct{{ params.julia_parameters(params_jl,"T" ) }} {% for member in Members %} - {% if member.is_array or member.is_builtin %} + {% if member.is_array %} + {% if not member.is_builtin_array and upstream_edm and (member.array_type in upstream_edm.components or member.array_type in upstream_edm.datatypes) %} + {{ member.name }}::MVector{{ '{' }}{{ member.array_size }}, {{ upstream_edm_name }}.{{ member.array_bare_type }}Struct{{ '}' }} + {% else %} + {{ member.name }}::{{ member.julia_type }} + {% endif %} + {% elif member.is_builtin %} {{ member.name }}::{{ member.julia_type }} {% else %} + {% if upstream_edm and (member.full_type in upstream_edm.components or member.full_type in upstream_edm.datatypes) %} + {{ member.name }}::{{ upstream_edm_name }}.{{ member.julia_type }}Struct + {% else %} {{ member.name }}::{{ member.julia_type }}Struct {% endif %} + {% endif %} {% endfor %} {% for relation in OneToManyRelations %} {{ relation.name }}::Vector{ {{ relation.julia_type }}T } @@ -20,7 +30,11 @@ mutable struct {{ class.bare_type }}Struct{{ params.julia_parameters(params_jl," {% if member.is_builtin %} {{ member.name }}::Vector{ {{ member.julia_type }} } {% else %} + {% if upstream_edm and (member.full_type in upstream_edm.components or member.full_type in upstream_edm.datatypes) %} + {{ member.name }}::Vector{ {{ upstream_edm_name }}.{{ member.julia_type }}Struct } + {% else %} {{ member.name }}::Vector{ {{ member.julia_type }}Struct } {% endif %} + {% endif %} {% endfor %} end diff --git a/python/templates/ParentModule.jl.jinja2 b/python/templates/ParentModule.jl.jinja2 index 6a43e9633..b30a81a6e 100644 --- a/python/templates/ParentModule.jl.jinja2 +++ b/python/templates/ParentModule.jl.jinja2 @@ -1,5 +1,9 @@ {% import "macros/params.jinja2" as params %} {% import "macros/abstracttypes.jinja2" as abstdatatype %} +{% if upstream_edm %} +include("{{ upstream_edm.options["includeSubfolder"] }}{{ upstream_edm_name }}.jl") +{% endif %} + module {{ class.bare_type }} {% for component in components %} export {{ component['class'].bare_type }} @@ -9,6 +13,10 @@ export {{ datatype['class'].bare_type }} export {{ datatype['class'].bare_type }}Collection {% endfor %} +{% if upstream_edm %} +import ..{{ upstream_edm_name }} +{% endif %} + {% for component in components %} include("{{ component['class'].bare_type }}Struct.jl") {% endfor %} @@ -18,30 +26,54 @@ include("{{ datatype['class'].bare_type }}Struct.jl") {% for component in components %} function {{ component['class'].bare_type }}( - {% for member in component['Members'] %} +{% for member in component['Members'] %} {% if member.is_array %} + {% if member.is_builtin_array %} {{ member.name }}::{{ member.julia_type }} = {{ member.julia_type }}(undef), + {% else %} + {% if upstream_edm and (member.array_type in upstream_edm.components or member.array_type in upstream_edm.datatypes) %} + {{ member.name }}::MVector{{ '{' }}{{ member.array_size }}, {{ upstream_edm_name }}.{{ member.array_bare_type }}Struct{{ '}' }} = MVector{{ '{' }}{{ member.array_size }}, {{ upstream_edm_name }}.{{ member.array_bare_type }}Struct{{ '}' }}(undef), + {% else %} + {{ member.name }}::{{ member.julia_type }} = {{ member.julia_type }}(undef), + {% endif %} + {% endif %} {% elif member.is_builtin %} {{ member.name }}::{{ abstdatatype.classify_data_type(member.julia_type) }} = {{ member.julia_type }}(0), {% else %} + {% if upstream_edm and (member.full_type in upstream_edm.components or member.full_type in upstream_edm.datatypes) %} + {{ member.name }}::{{ upstream_edm_name }}.{{ member.julia_type }}Struct = {{ upstream_edm_name }}.{{ member.julia_type }}(), + {% else %} {{ member.name }}::{{ member.julia_type }}Struct = {{ member.julia_type }}(), {% endif %} + {% endif %} {% endfor %} {% for relation in component['OneToManyRelations'] %} + {% if upstream_edm and (relation.full_type in upstream_edm.components or relation.full_type in upstream_edm.datatypes) %} + {{ relation.name }}::Vector{ {{ upstream_edm_name }}.{{ relation.julia_type }}Struct } = Vector{ {{ upstream_edm_name }}.{{ relation.julia_type }}Struct }(), + {% else %} {{ relation.name }}::Vector{ {{ relation.julia_type }}Struct } = Vector{ {{ relation.julia_type }}Struct }(), + {% endif %} {% endfor %} {% for relation in component['OneToOneRelations'] %} + {% if upstream_edm and (relation.full_type in upstream_edm.components or relation.full_type in upstream_edm.datatypes) %} + {{ relation.name }}::Union{Nothing, {{ upstream_edm_name }}.{{ relation.julia_type }}Struct } = nothing, + {% else %} {{ relation.name }}::Union{Nothing, {{ relation.julia_type }}Struct } = nothing, + {% endif %} {% endfor %} {% for member in component['VectorMembers'] %} {% if member.is_builtin %} {{ member.name }}::Vector{ {{ member.julia_type }} } = Vector{ {{ member.julia_type }} }([]), {% else %} + {% if upstream_edm and (member.full_type in upstream_edm.components or member.full_type in upstream_edm.datatypes) %} + {{ member.name }}::Vector{ {{ upstream_edm_name }}.{{ member.julia_type }}Struct } = Vector{ {{ upstream_edm_name }}.{{ member.julia_type }}Struct }([]), + {% else %} {{ member.name }}::Vector{ {{ member.julia_type }}Struct } = Vector{ {{ member.julia_type }}Struct }([]), {% endif %} + {% endif %} {% endfor %} ) - return {{ component['class'].bare_type }}Struct{{ params.julia_parameters(component['params_jl'], "", "Struct") }}( + return {{ component['class'].bare_type }}Struct{{ params.julia_parameters(component['params_jl'], "Struct", upstream_edm, upstream_edm_name) }}( {% for member in component['Members'] %} {% if member.is_builtin %} {{ member.julia_type }}({{ member.name }}), @@ -65,30 +97,54 @@ end {% for datatype in datatypes %} function {{ datatype['class'].bare_type }}( - {% for member in datatype['Members'] %} +{% for member in datatype['Members'] %} {% if member.is_array %} + {% if member.is_builtin_array %} + {{ member.name }}::{{ member.julia_type }} = {{ member.julia_type }}(undef), + {% else %} + {% if upstream_edm and (member.array_type in upstream_edm.components or member.array_type in upstream_edm.datatypes) %} + {{ member.name }}::MVector{{ '{' }}{{ member.array_size }}, {{ upstream_edm_name }}.{{ member.array_bare_type }}Struct{{ '}' }} = MVector{{ '{' }}{{ member.array_size }}, {{ upstream_edm_name }}.{{ member.array_bare_type }}Struct{{ '}' }}(undef), + {% else %} {{ member.name }}::{{ member.julia_type }} = {{ member.julia_type }}(undef), + {% endif %} + {% endif %} {% elif member.is_builtin %} {{ member.name }}::{{ abstdatatype.classify_data_type(member.julia_type) }} = {{ member.julia_type }}(0), {% else %} + {% if upstream_edm and (member.full_type in upstream_edm.components or member.full_type in upstream_edm.datatypes) %} + {{ member.name }}::{{ upstream_edm_name }}.{{ member.julia_type }}Struct = {{ upstream_edm_name }}.{{ member.julia_type }}(), + {% else %} {{ member.name }}::{{ member.julia_type }}Struct = {{ member.julia_type }}(), {% endif %} + {% endif %} {% endfor %} {% for relation in datatype['OneToManyRelations'] %} + {% if upstream_edm and (relation.full_type in upstream_edm.components or relation.full_type in upstream_edm.datatypes) %} + {{ relation.name }}::Vector{ {{ upstream_edm_name }}.{{ relation.julia_type }}Struct } = Vector{ {{ upstream_edm_name }}.{{ relation.julia_type }}Struct }(), + {% else %} {{ relation.name }}::Vector{ {{ relation.julia_type }}Struct } = Vector{ {{ relation.julia_type }}Struct }(), + {% endif %} {% endfor %} {% for relation in datatype['OneToOneRelations'] %} + {% if upstream_edm and (relation.full_type in upstream_edm.components or relation.full_type in upstream_edm.datatypes) %} + {{ relation.name }}::Union{Nothing, {{ upstream_edm_name }}.{{ relation.julia_type }}Struct } = nothing, + {% else %} {{ relation.name }}::Union{Nothing, {{ relation.julia_type }}Struct } = nothing, + {% endif %} {% endfor %} {% for member in datatype['VectorMembers'] %} {% if member.is_builtin %} {{ member.name }}::Vector{ {{ member.julia_type }} } = Vector{ {{ member.julia_type }} }([]), {% else %} + {% if upstream_edm and (member.full_type in upstream_edm.components or member.full_type in upstream_edm.datatypes) %} + {{ member.name }}::Vector{ {{ upstream_edm_name }}.{{ member.julia_type }}Struct } = Vector{ {{ upstream_edm_name }}.{{ member.julia_type }}Struct }([]), + {% else %} {{ member.name }}::Vector{ {{ member.julia_type }}Struct } = Vector{ {{ member.julia_type }}Struct }([]), {% endif %} + {% endif %} {% endfor %} ) - return {{ datatype['class'].bare_type }}Struct{{ params.julia_parameters(datatype['params_jl'], "", "Struct") }}( + return {{ datatype['class'].bare_type }}Struct{{ params.julia_parameters(datatype['params_jl'], "Struct", upstream_edm, upstream_edm_name) }}( {% for member in datatype['Members'] %} {% if member.is_builtin %} {{ member.julia_type }}({{ member.name }}), @@ -108,7 +164,7 @@ function {{ datatype['class'].bare_type }}( ) end -{{ datatype['class'].bare_type }}Collection = Vector{ {{ datatype['class'].bare_type }}Struct{{ params.julia_parameters(datatype['params_jl'], prefix="", suffix="Struct") }} } +{{ datatype['class'].bare_type }}Collection = Vector{ {{ datatype['class'].bare_type }}Struct{{ params.julia_parameters(datatype['params_jl'], "Struct", upstream_edm, upstream_edm_name) }} } {% endfor %} end \ No newline at end of file diff --git a/python/templates/macros/params.jinja2 b/python/templates/macros/params.jinja2 index 7aba769c9..28b3d36d0 100644 --- a/python/templates/macros/params.jinja2 +++ b/python/templates/macros/params.jinja2 @@ -1,8 +1,13 @@ -{% macro julia_parameters(params, prefix, suffix) %} +{% macro julia_parameters(params, suffix, upstream_edm=None, upstream_edm_name=None) %} {%- if params -%}{ {%- set comma = joiner(',') -%} {%- for par in params -%} -{{ comma() }}{{ prefix }}{{ par }}{{ suffix }} +{{comma() }} + {%- if upstream_edm and (par[1] in upstream_edm.components or par[1] in upstream_edm.datatypes) %} + {{- upstream_edm_name }}.{{ par[0] }}{{ suffix -}} + {%- else %} + {{- par[0] }}{{ suffix -}} + {%- endif %} {%- endfor -%} }{%- endif -%} {%- endmacro -%} \ No newline at end of file diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 22e758c76..16ec5c456 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -66,20 +66,16 @@ CREATE_PODIO_TEST(write_ascii.cpp "") PODIO_GENERATE_DATAMODEL(datamodeljulia datalayout.yaml headers sources IO_BACKEND_HANDLERS ${PODIO_IO_HANDLERS} - OLD_DESCRIPTION datalayout_old.yaml - SCHEMA_EVOLUTION schema_evolution.yaml LANG julia ) PODIO_GENERATE_DATAMODEL(extensionmodeljulia datalayout_extension.yaml ext_headers ext_sources - UPSTREAM_EDM datamodel:datalayout.yaml + UPSTREAM_EDM ../../datamodeljulia:datalayout.yaml IO_BACKEND_HANDLERS ${PODIO_IO_HANDLERS} OUTPUT_FOLDER ${CMAKE_CURRENT_SOURCE_DIR}/extension_model LANG julia ) -# Customize CTest to potentially disable some of the tests with known problems -configure_file(CTestCustom.cmake ${PROJECT_BINARY_DIR}/CTestCustom.cmake @ONLY) find_program(Julia_EXECUTABLE julia) if (Julia_EXECUTABLE) message(STATUS "Found Julia, will add the Julia tests") @@ -88,3 +84,6 @@ if (Julia_EXECUTABLE) WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} ) endif() + +# Customize CTest to potentially disable some of the tests with known problems +configure_file(CTestCustom.cmake ${PROJECT_BINARY_DIR}/CTestCustom.cmake @ONLY)