Added --upstream-edm code generation support

python/podio_class_generator.py

@@ -28,6 +28,12 @@
   Used {yamlfile} to create {nclasses} classes in {installdir}/
   Read instructions in the README.md to run your first example!
 """
+REPORT_TEXT_JULIA = """
+  PODIO Data Model
+  ================
+  Used {yamlfile} to create {nclasses} julia files in {installdir}/
+  Read instructions in the README.md to run your first example!
+"""
 
 
 def get_clang_format():
@@ -134,6 +140,10 @@ def process(self):
 
     datamodel = {}
     datamodel['class'] = DataType(self.package_name.capitalize())
+    datamodel['upstream_edm'] = self.upstream_edm
+    datamodel['upstream_edm_name'] = ''
+    if self.upstream_edm:
+      datamodel['upstream_edm_name'] = self.upstream_edm.options["includeSubfolder"].split("/")[-2].capitalize()
     datamodel['components'] = []
     datamodel['datatypes'] = []
     for name, component in self.datamodel.components.items():
@@ -149,11 +159,11 @@ def process(self):
       self._write_edm_def_file()
 
       if 'ROOT' in self.io_handlers:
+        self.prepare_iorules()
         self._create_selection_xml()
 
       self._write_cmake_lists_file()
 
-    self.process_schema_evolution()
     self.print_report()
 
   def process_schema_evolution(self):
@@ -192,11 +202,16 @@ def print_report(self):
     """Print a summary report about the generated code"""
     if not self.verbose:
       return
-
-    nclasses = 5 * len(self.datamodel.datatypes) + len(self.datamodel.components)
-    text = REPORT_TEXT.format(yamlfile=self.yamlfile,
-                              nclasses=nclasses,
-                              installdir=self.install_dir)
+    if self.proglang == "julia":
+      nclasses = len(self.datamodel.datatypes) + len(self.datamodel.components) + 1
+      text = REPORT_TEXT_JULIA.format(yamlfile=self.yamlfile,
+                                      nclasses=nclasses,
+                                      installdir=self.install_dir)
+    if self.proglang == "cpp":
+      nclasses = 5 * len(self.datamodel.datatypes) + len(self.datamodel.components)
+      text = REPORT_TEXT.format(yamlfile=self.yamlfile,
+                                nclasses=nclasses,
+                                installdir=self.install_dir)
 
     for summaryline in text.splitlines():
       print(summaryline)
@@ -250,7 +265,6 @@ def get_fn_format(tmpl):
 
     endings = {
         'Data': ('h',),
-        'Component': ('h',),
         'PrintInfo': ('h',),
         'MutableStruct': ('jl',),
         'ParentModule': ('jl',),
@@ -292,29 +306,34 @@ def _process_component(self, name, component):
     component['includes'] = self._sort_includes(includes)
     component['includes_jl'] = {'struct': sorted(includes_jl)}
     component['class'] = DataType(name)
+    component['upstream_edm'] = self.upstream_edm
+    component['upstream_edm_name'] = ''
+    if self.upstream_edm:
+      component['upstream_edm_name'] = self.upstream_edm.options["includeSubfolder"].split("/")[-2].capitalize()
 
-    if self.proglang == "cpp":
-      self._fill_templates('Component', component)
     if self.proglang == "julia":
       self._fill_templates('MutableStruct', component)
 
-    return component
-
-    # Add potentially older schema for schema evolution
-    # based on ROOT capabilities for now
-    if name in self.root_schema_dict:
-      schema_evolutions = self.root_schema_dict[name]
-      component = copy.deepcopy(component)
-      for schema_evolution in schema_evolutions:
-        if isinstance(schema_evolution, RenamedMember):
-          for member in component['Members']:
-            if member.name == schema_evolution.member_name_new:
-              member.name = schema_evolution.member_name_old
-          component['class'] = DataType(name + self.old_schema_version)
-        else:
-          raise NotImplementedError
+    if self.proglang == "cpp":
       self._fill_templates('Component', component)
-      self.root_schema_component_names.add(name + self.old_schema_version)
+
+      # Add potentially older schema for schema evolution
+      # based on ROOT capabilities for now
+      if name in self.root_schema_dict:
+        schema_evolutions = self.root_schema_dict[name]
+        component = copy.deepcopy(component)
+        for schema_evolution in schema_evolutions:
+          if isinstance(schema_evolution, RenamedMember):
+            for member in component['Members']:
+              if member.name == schema_evolution.member_name_new:
+                member.name = schema_evolution.member_name_old
+            component['class'] = DataType(name + self.old_schema_version)
+          else:
+            raise NotImplementedError
+        self._fill_templates('Component', component)
+        self.root_schema_component_names.add(name + self.old_schema_version)
+
+    return component
 
   @staticmethod
   def _replace_component_in_paths(oldname, newname, paths):
@@ -345,7 +364,7 @@ def _process_datatype(self, name, definition):
           if member.array_type in self.root_schema_dict:
             needs_schema_evolution = True
             self._replace_component_in_paths(member.array_type, member.array_type + self.old_schema_version,
-                                            schema_evolution_datatype['includes_data'])
+                                             schema_evolution_datatype['includes_data'])
             member.full_type = member.full_type.replace(member.array_type, member.array_type + self.old_schema_version)
             member.array_type = member.array_type + self.old_schema_version
 
@@ -354,7 +373,7 @@ def _process_datatype(self, name, definition):
             needs_schema_evolution = True
             # prepare the ROOT I/O rule
             self._replace_component_in_paths(member.full_type, member.full_type + self.old_schema_version,
-                                            schema_evolution_datatype['includes_data'])
+                                             schema_evolution_datatype['includes_data'])
             member.full_type = member.full_type + self.old_schema_version
             member.bare_type = member.bare_type + self.old_schema_version
 
@@ -416,7 +435,7 @@ def _get_julia_params(datatype):
     params = set()
     for relation in datatype['OneToManyRelations'] + datatype['OneToOneRelations']:
       if not relation.is_builtin:
-        params.add(relation.bare_type)
+        params.add((relation.bare_type, relation.full_type))
     return list(params)
 
   def _preprocess_for_obj(self, datatype):
@@ -551,7 +570,11 @@ def _preprocess_datatype(self, name, definition):
     data['class'] = DataType(name)
     data['includes_data'] = self._get_member_includes(definition["Members"])
     data['includes_jl'] = {'struct': self._get_member_includes(definition["Members"], julia=True)}
-    data['params_jl'] = sorted(self._get_julia_params(data))
+    data['params_jl'] = sorted(self._get_julia_params(data), key=lambda x: x[0])
+    data['upstream_edm'] = self.upstream_edm
+    data['upstream_edm_name'] = ''
+    if self.upstream_edm:
+      data['upstream_edm_name'] = self.upstream_edm.options["includeSubfolder"].split("/")[-2].capitalize()
     self._preprocess_for_class(data)
     self._preprocess_for_obj(data)
     self._preprocess_for_collection(data)
@@ -678,14 +701,8 @@ def _build_julia_include_for_class(self, classname, include_from: IncludeFrom) -
     """Return the include statement for julia for this specific class"""
     if include_from == IncludeFrom.INTERNAL:
       # If we have an internal include all includes should be relative
-      inc_folder = ''
-    if include_from == IncludeFrom.EXTERNAL:
-      inc_folder = f'{self.upstream_edm.options["includeSubfolder"]}'
-    if include_from == IncludeFrom.NOWHERE:
-      # We don't need an include in this case
-      return ''
-
-    return f'include("{inc_folder}{classname}Struct.jl")'
+      return f'include("{classname}Struct.jl")'
+    return ''
 
   def _sort_includes(self, includes):
     """Sort the includes in order to try to have the std includes at the bottom"""

python/templates/MutableStruct.jl.jinja2

@@ -2,13 +2,23 @@
 {% for include in includes_jl['struct'] %}
 {{ include }}
 {% endfor %}
-mutable struct {{ class.bare_type }}Struct{{ params.julia_parameters(params_jl,"" ,"T" ) }}
+mutable struct {{ class.bare_type }}Struct{{ params.julia_parameters(params_jl,"T" ) }}
 {% for member in Members %}
-	{% if member.is_array or member.is_builtin %}
+	{% if member.is_array %}
+	{% if not member.is_builtin_array and upstream_edm and (member.array_type in upstream_edm.components or member.array_type in upstream_edm.datatypes) %}
+	{{ member.name }}::MVector{{ '{' }}{{ member.array_size }}, {{ upstream_edm_name }}.{{ member.array_bare_type }}Struct{{ '}' }}
+	{% else %}
+	{{ member.name }}::{{ member.julia_type }}
+	{% endif %}
+	{% elif member.is_builtin %}
 	{{ member.name }}::{{ member.julia_type }}
 	{% else %}
+	{% if upstream_edm and (member.full_type in upstream_edm.components or member.full_type in upstream_edm.datatypes) %}
+	{{ member.name }}::{{ upstream_edm_name }}.{{ member.julia_type }}Struct
+	{% else %}
 	{{ member.name }}::{{ member.julia_type }}Struct
 	{% endif %}
+	{% endif %}
 {% endfor %}
 {% for relation in OneToManyRelations %}
 	{{ relation.name }}::Vector{ {{ relation.julia_type }}T }
@@ -20,7 +30,11 @@ mutable struct {{ class.bare_type }}Struct{{ params.julia_parameters(params_jl,"
 	{% if member.is_builtin %}
 	{{ member.name }}::Vector{ {{ member.julia_type }} }
 	{% else %}
+	{% if upstream_edm and (member.full_type in upstream_edm.components or member.full_type in upstream_edm.datatypes) %}
+	{{ member.name }}::Vector{ {{ upstream_edm_name }}.{{ member.julia_type }}Struct }
+	{% else %}
 	{{ member.name }}::Vector{ {{ member.julia_type }}Struct }
 	{% endif %}
+	{% endif %}
 {% endfor %}
 end

python/templates/ParentModule.jl.jinja2

@@ -1,5 +1,9 @@
 {% import "macros/params.jinja2" as params %}
 {% import "macros/abstracttypes.jinja2" as abstdatatype %}
+{% if upstream_edm %}
+include("{{ upstream_edm.options["includeSubfolder"] }}{{ upstream_edm_name }}.jl")
+{% endif %}
+
 module {{ class.bare_type }}
 {% for component in components %}
 export {{ component['class'].bare_type }}
@@ -9,6 +13,10 @@ export {{ datatype['class'].bare_type }}
 export {{ datatype['class'].bare_type }}Collection
 {% endfor %}
 
+{% if upstream_edm %}
+import ..{{ upstream_edm_name }}
+{% endif %}
+
 {% for component in components %}
 include("{{ component['class'].bare_type }}Struct.jl")
 {% endfor %}
@@ -18,30 +26,54 @@ include("{{ datatype['class'].bare_type }}Struct.jl")
 
 {% for component in components %}
 function {{ component['class'].bare_type }}(
-    {% for member in component['Members'] %}
+{% for member in component['Members'] %}
 	{% if member.is_array %}
+	{% if member.is_builtin_array %}
 	{{ member.name }}::{{ member.julia_type }} = {{ member.julia_type }}(undef),
+	{% else %}
+	{% if upstream_edm and (member.array_type in upstream_edm.components or member.array_type in upstream_edm.datatypes) %}
+	{{ member.name }}::MVector{{ '{' }}{{ member.array_size }}, {{ upstream_edm_name }}.{{ member.array_bare_type }}Struct{{ '}' }} = MVector{{ '{' }}{{ member.array_size }}, {{ upstream_edm_name }}.{{ member.array_bare_type }}Struct{{ '}' }}(undef),
+	{% else %}
+	{{ member.name }}::{{ member.julia_type }} = {{ member.julia_type }}(undef),
+	{% endif %}
+	{% endif %}
 	{% elif member.is_builtin %}
 	{{ member.name }}::{{ abstdatatype.classify_data_type(member.julia_type) }} = {{ member.julia_type }}(0),
 	{% else %}
+	{% if upstream_edm and (member.full_type in upstream_edm.components or member.full_type in upstream_edm.datatypes) %}
+	{{ member.name }}::{{ upstream_edm_name }}.{{ member.julia_type }}Struct = {{ upstream_edm_name }}.{{ member.julia_type }}(),
+	{% else %}
 	{{ member.name }}::{{ member.julia_type }}Struct = {{ member.julia_type }}(),
 	{% endif %}
+	{% endif %}
 {% endfor %}
 {% for relation in component['OneToManyRelations'] %}
+	{% if upstream_edm and (relation.full_type in upstream_edm.components or relation.full_type in upstream_edm.datatypes) %}
+	{{ relation.name }}::Vector{ {{ upstream_edm_name }}.{{ relation.julia_type }}Struct } = Vector{ {{ upstream_edm_name }}.{{ relation.julia_type }}Struct }(),
+	{% else %}
 	{{ relation.name }}::Vector{ {{ relation.julia_type }}Struct } = Vector{ {{ relation.julia_type }}Struct }(),
+	{% endif %}
 {% endfor %}
 {% for relation in component['OneToOneRelations'] %}
+	{% if upstream_edm and (relation.full_type in upstream_edm.components or relation.full_type in upstream_edm.datatypes) %}
+	{{ relation.name }}::Union{Nothing, {{ upstream_edm_name }}.{{ relation.julia_type }}Struct } = nothing,
+	{% else %}
 	{{ relation.name }}::Union{Nothing, {{ relation.julia_type }}Struct } = nothing,
+	{% endif %}
 {% endfor %}
 {% for member in component['VectorMembers'] %}
 	{% if member.is_builtin %}
 	{{ member.name }}::Vector{ {{ member.julia_type }} } = Vector{ {{ member.julia_type }} }([]),
 	{% else %}
+	{% if upstream_edm and (member.full_type in upstream_edm.components or member.full_type in upstream_edm.datatypes) %}
+	{{ member.name }}::Vector{ {{ upstream_edm_name }}.{{ member.julia_type }}Struct } = Vector{ {{ upstream_edm_name }}.{{ member.julia_type }}Struct }([]),
+	{% else %}
 	{{ member.name }}::Vector{ {{ member.julia_type }}Struct } = Vector{ {{ member.julia_type }}Struct }([]),
 	{% endif %}
+	{% endif %}
 {% endfor %}
 )
-	return {{ component['class'].bare_type }}Struct{{ params.julia_parameters(component['params_jl'], "", "Struct") }}(
+	return {{ component['class'].bare_type }}Struct{{ params.julia_parameters(component['params_jl'], "Struct", upstream_edm, upstream_edm_name) }}(
 	{% for member in component['Members'] %}
 	{% if member.is_builtin %}
 	{{ member.julia_type }}({{ member.name }}),
@@ -65,30 +97,54 @@ end
 
 {% for datatype in datatypes %}
 function {{ datatype['class'].bare_type }}(
-    {% for member in datatype['Members'] %}
+{% for member in datatype['Members'] %}
 	{% if member.is_array %}
+	{% if member.is_builtin_array %}
+	{{ member.name }}::{{ member.julia_type }} = {{ member.julia_type }}(undef),
+	{% else %}
+	{% if upstream_edm and (member.array_type in upstream_edm.components or member.array_type in upstream_edm.datatypes) %}
+	{{ member.name }}::MVector{{ '{' }}{{ member.array_size }}, {{ upstream_edm_name }}.{{ member.array_bare_type }}Struct{{ '}' }} = MVector{{ '{' }}{{ member.array_size }}, {{ upstream_edm_name }}.{{ member.array_bare_type }}Struct{{ '}' }}(undef),
+	{% else %}
 	{{ member.name }}::{{ member.julia_type }} = {{ member.julia_type }}(undef),
+	{% endif %}
+	{% endif %}
 	{% elif member.is_builtin %}
 	{{ member.name }}::{{ abstdatatype.classify_data_type(member.julia_type) }} = {{ member.julia_type }}(0),
 	{% else %}
+	{% if upstream_edm and (member.full_type in upstream_edm.components or member.full_type in upstream_edm.datatypes) %}
+	{{ member.name }}::{{ upstream_edm_name }}.{{ member.julia_type }}Struct = {{ upstream_edm_name }}.{{ member.julia_type }}(),
+	{% else %}
 	{{ member.name }}::{{ member.julia_type }}Struct = {{ member.julia_type }}(),
 	{% endif %}
+	{% endif %}
 {% endfor %}
 {% for relation in datatype['OneToManyRelations'] %}
+	{% if upstream_edm and (relation.full_type in upstream_edm.components or relation.full_type in upstream_edm.datatypes) %}
+	{{ relation.name }}::Vector{ {{ upstream_edm_name }}.{{ relation.julia_type }}Struct } = Vector{ {{ upstream_edm_name }}.{{ relation.julia_type }}Struct }(),
+	{% else %}
 	{{ relation.name }}::Vector{ {{ relation.julia_type }}Struct } = Vector{ {{ relation.julia_type }}Struct }(),
+	{% endif %}
 {% endfor %}
 {% for relation in datatype['OneToOneRelations'] %}
+	{% if upstream_edm and (relation.full_type in upstream_edm.components or relation.full_type in upstream_edm.datatypes) %}
+	{{ relation.name }}::Union{Nothing, {{ upstream_edm_name }}.{{ relation.julia_type }}Struct } = nothing,
+	{% else %}
 	{{ relation.name }}::Union{Nothing, {{ relation.julia_type }}Struct } = nothing,
+	{% endif %}
 {% endfor %}
 {% for member in datatype['VectorMembers'] %}
 	{% if member.is_builtin %}
 	{{ member.name }}::Vector{ {{ member.julia_type }} } = Vector{ {{ member.julia_type }} }([]),
 	{% else %}
+	{% if upstream_edm and (member.full_type in upstream_edm.components or member.full_type in upstream_edm.datatypes) %}
+	{{ member.name }}::Vector{ {{ upstream_edm_name }}.{{ member.julia_type }}Struct } = Vector{ {{ upstream_edm_name }}.{{ member.julia_type }}Struct }([]),
+	{% else %}
 	{{ member.name }}::Vector{ {{ member.julia_type }}Struct } = Vector{ {{ member.julia_type }}Struct }([]),
 	{% endif %}
+	{% endif %}
 {% endfor %}
 )
-	return {{ datatype['class'].bare_type }}Struct{{ params.julia_parameters(datatype['params_jl'], "", "Struct") }}(
+	return {{ datatype['class'].bare_type }}Struct{{ params.julia_parameters(datatype['params_jl'], "Struct", upstream_edm, upstream_edm_name) }}(
 	{% for member in datatype['Members'] %}
 	{% if member.is_builtin %}
 	{{ member.julia_type }}({{ member.name }}),
@@ -108,7 +164,7 @@ function {{ datatype['class'].bare_type }}(
 	)
 end
 
-{{ datatype['class'].bare_type }}Collection = Vector{ {{ datatype['class'].bare_type }}Struct{{ params.julia_parameters(datatype['params_jl'], prefix="", suffix="Struct") }} }
+{{ datatype['class'].bare_type }}Collection = Vector{ {{ datatype['class'].bare_type }}Struct{{ params.julia_parameters(datatype['params_jl'], "Struct", upstream_edm, upstream_edm_name) }} }
 
 {% endfor %}
 end

python/templates/macros/params.jinja2

@@ -1,8 +1,13 @@
-{% macro julia_parameters(params, prefix, suffix) %}
+{% macro julia_parameters(params, suffix, upstream_edm=None, upstream_edm_name=None) %}
 {%- if params -%}{
 {%- set comma = joiner(',') -%}
 {%- for par in params -%}
-{{ comma() }}{{ prefix }}{{ par }}{{ suffix }}
+{{comma() }}
+    {%- if upstream_edm and (par[1] in upstream_edm.components or par[1] in upstream_edm.datatypes) %}
+        {{- upstream_edm_name }}.{{ par[0] }}{{ suffix -}}
+    {%- else %}
+        {{- par[0] }}{{ suffix -}}
+    {%- endif %}
 {%- endfor -%}
 }{%- endif -%}
 {%- endmacro -%}