From c7328d62f17061c8360eb2ac0a3fe0ce931fd2ae Mon Sep 17 00:00:00 2001 From: hegner Date: Wed, 15 Mar 2023 15:51:56 +0100 Subject: [PATCH] add stripped down schema evolution (#341) * add stripped down schema evolution; relying on ROOT for the moment --------- Co-authored-by: Thomas Madlener --- .github/scripts/pylint.rc | 8 +- .gitignore | 1 + CMakeLists.txt | 4 +- cmake/podioMacros.cmake | 16 +- include/podio/ASCIIWriter.h | 1 + include/podio/CollectionBase.h | 9 + include/podio/CollectionBuffers.h | 3 + include/podio/ROOTFrameWriter.h | 2 +- include/podio/ROOTLegacyReader.h | 2 +- include/podio/ROOTReader.h | 2 +- include/podio/SchemaEvolution.h | 14 + include/podio/UserDataCollection.h | 11 + python/CMakeLists.txt | 1 + python/podio/generator_utils.py | 7 +- python/podio/podio_config_reader.py | 10 +- python/podio_class_generator.py | 58 ++- python/podio_schema_evolution.py | 370 ++++++++++++++++++ python/templates/Collection.cc.jinja2 | 10 + python/templates/Collection.h.jinja2 | 7 + .../schemaevolution/EvolvePOD.h.jinja2 | 0 python/templates/selection.xml.jinja2 | 5 + src/ROOTFrameReader.cc | 2 +- src/ROOTFrameWriter.cc | 3 +- src/ROOTLegacyReader.cc | 2 +- src/ROOTReader.cc | 40 +- src/ROOTWriter.cc | 2 +- src/rootUtils.h | 12 +- src/selection.xml | 2 + tests/CMakeLists.txt | 5 +- tests/datalayout.yaml | 1 + tests/datalayout_old.yaml | 203 ++++++++++ tests/schema_evolution.yaml | 13 + 32 files changed, 784 insertions(+), 42 deletions(-) create mode 100644 include/podio/SchemaEvolution.h create mode 100755 python/podio_schema_evolution.py create mode 100644 python/templates/schemaevolution/EvolvePOD.h.jinja2 create mode 100755 tests/datalayout_old.yaml create mode 100644 tests/schema_evolution.yaml diff --git a/.github/scripts/pylint.rc b/.github/scripts/pylint.rc index c5c1e86b3..2db65ccd7 100644 --- a/.github/scripts/pylint.rc +++ b/.github/scripts/pylint.rc @@ -263,14 +263,14 @@ exclude-protected=_asdict,_fields,_replace,_source,_make [DESIGN] # Maximum number of arguments for function / method -max-args=8 +max-args=10 # Argument names that match this expression will be ignored. Default to name # with leading underscore ignored-argument-names=_.* # Maximum number of locals for function / method body -max-locals=20 +max-locals=25 # Maximum number of return / yield for function / method body max-returns=8 @@ -285,10 +285,10 @@ max-statements=50 max-parents=7 # Maximum number of attributes for a class (see R0902). -max-attributes=20 +max-attributes=25 # Minimum number of public methods for a class (see R0903). -min-public-methods=1 +min-public-methods=0 # Maximum number of public methods for a class (see R0904). max-public-methods=20 diff --git a/.gitignore b/.gitignore index b665c600f..22f536c88 100644 --- a/.gitignore +++ b/.gitignore @@ -56,6 +56,7 @@ spack* # Tooling /.clangd/ /compile_commands.json +.vscode /.cache/ # Generated files diff --git a/CMakeLists.txt b/CMakeLists.txt index 3730ab000..e8a943b61 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,8 +7,8 @@ project(podio) #--- Version ------------------------------------------------------------------- SET( ${PROJECT_NAME}_VERSION_MAJOR 0 ) -SET( ${PROJECT_NAME}_VERSION_MINOR 16 ) -SET( ${PROJECT_NAME}_VERSION_PATCH 3 ) +SET( ${PROJECT_NAME}_VERSION_MINOR 17 ) +SET( ${PROJECT_NAME}_VERSION_PATCH 0 ) SET( ${PROJECT_NAME}_VERSION "${${PROJECT_NAME}_VERSION_MAJOR}.${${PROJECT_NAME}_VERSION_MINOR}.${${PROJECT_NAME}_VERSION_PATCH}" ) diff --git a/cmake/podioMacros.cmake b/cmake/podioMacros.cmake index 0541317e6..bb217801c 100644 --- a/cmake/podioMacros.cmake +++ b/cmake/podioMacros.cmake @@ -119,6 +119,7 @@ set_property(CACHE PODIO_USE_CLANG_FORMAT PROPERTY STRINGS AUTO ON OFF) # RETURN_HEADERS variable that will be filled with the list of created headers files: ${datamodel}/*.h # RETURN_SOURCES variable that will be filled with the list of created source files : src/*.cc # Parameters: +# OLD_DESCRIPTION OPTIONAL: The path to the yaml file describing a previous datamodel version # OUTPUT_FOLDER OPTIONAL: The folder in which the output files should be placed # Default is ${CMAKE_CURRENT_SOURCE_DIR} # UPSTREAM_EDM OPTIONAL: The upstream edm and its package name that are passed to the @@ -126,13 +127,14 @@ set_property(CACHE PODIO_USE_CLANG_FORMAT PROPERTY STRINGS AUTO ON OFF) # IO_BACKEND_HANDLERS OPTIONAL: The I/O backend handlers that should be generated. The list is # passed directly to podio_class_generator.py and validated there # Default is ROOT +# SCHEMA_EVOLUTION OPTIONAL: The path to the yaml file declaring the necessary schema evolution # ) # # Note that the create_${datamodel} target will always be called, but if the YAML_FILE has not changed # this is essentially a no-op, and should not cause re-compilation. #--------------------------------------------------------------------------------------------------- function(PODIO_GENERATE_DATAMODEL datamodel YAML_FILE RETURN_HEADERS RETURN_SOURCES) - CMAKE_PARSE_ARGUMENTS(ARG "" "OUTPUT_FOLDER;UPSTREAM_EDM" "IO_BACKEND_HANDLERS" ${ARGN}) + CMAKE_PARSE_ARGUMENTS(ARG "" "OLD_DESCRIPTION;OUTPUT_FOLDER;UPSTREAM_EDM;SCHEMA_EVOLUTION" "IO_BACKEND_HANDLERS" ${ARGN}) IF(NOT ARG_OUTPUT_FOLDER) SET(ARG_OUTPUT_FOLDER ${CMAKE_CURRENT_SOURCE_DIR}) ENDIF() @@ -141,11 +143,21 @@ function(PODIO_GENERATE_DATAMODEL datamodel YAML_FILE RETURN_HEADERS RETURN_SOUR SET(UPSTREAM_EDM_ARG "--upstream-edm=${ARG_UPSTREAM_EDM}") ENDIF() + SET(OLD_DESCRIPTION_ARG "") + IF (ARG_OLD_DESCRIPTION) + SET(OLD_DESCRIPTION_ARG "--old-description=${ARG_OLD_DESCRIPTION}") + ENDIF() + IF(NOT ARG_IO_BACKEND_HANDLERS) # At least build the ROOT selection.xml by default for now SET(ARG_IO_BACKEND_HANDLERS "ROOT") ENDIF() + SET(SCHEMA_EVOLUTION_ARG "") + IF (ARG_SCHEMA_EVOLUTION) + SET(SCHEMA_EVOLUTION_ARG "--evolution_file=${ARG_SCHEMA_EVOLUTION}") + ENDIF() + set(CLANG_FORMAT_ARG "") if (PODIO_USE_CLANG_FORMAT STREQUAL AUTO OR PODIO_USE_CLANG_FORMAT) find_program(CLANG_FORMAT_EXE NAMES "clang-format") @@ -189,7 +201,7 @@ function(PODIO_GENERATE_DATAMODEL datamodel YAML_FILE RETURN_HEADERS RETURN_SOUR message(STATUS "Creating '${datamodel}' datamodel") # we need to boostrap the data model, so this has to be executed in the cmake run execute_process( - COMMAND ${Python_EXECUTABLE} ${podio_PYTHON_DIR}/podio_class_generator.py ${CLANG_FORMAT_ARG} ${UPSTREAM_EDM_ARG} ${YAML_FILE} ${ARG_OUTPUT_FOLDER} ${datamodel} ${ARG_IO_BACKEND_HANDLERS} + COMMAND ${Python_EXECUTABLE} ${podio_PYTHON_DIR}/podio_class_generator.py ${CLANG_FORMAT_ARG} ${OLD_DESCRIPTION_ARG} ${SCHEMA_EVOLUTION_ARG} ${UPSTREAM_EDM_ARG} ${YAML_FILE} ${ARG_OUTPUT_FOLDER} ${datamodel} ${ARG_IO_BACKEND_HANDLERS} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} RESULT_VARIABLE podio_generate_command_retval ) diff --git a/include/podio/ASCIIWriter.h b/include/podio/ASCIIWriter.h index 54fd2040b..e941bb6aa 100644 --- a/include/podio/ASCIIWriter.h +++ b/include/podio/ASCIIWriter.h @@ -2,6 +2,7 @@ #define PODIO_ASCIIWRITER_H #include "podio/EventStore.h" +#include "podio/SchemaEvolution.h" #include "podio/utilities/Deprecated.h" #include diff --git a/include/podio/CollectionBase.h b/include/podio/CollectionBase.h index fcb81401a..d2dc1a626 100644 --- a/include/podio/CollectionBase.h +++ b/include/podio/CollectionBase.h @@ -3,6 +3,7 @@ #include "podio/CollectionBuffers.h" #include "podio/ObjectID.h" +#include "podio/SchemaEvolution.h" #include #include @@ -49,6 +50,12 @@ class CollectionBase { /// Create (empty) collection buffers from which a collection can be constructed virtual podio::CollectionReadBuffers createBuffers() /*const*/ = 0; + /// Create (empty) collection buffers from which a collection can be constructed + /// Versioned to support schema evolution + virtual podio::CollectionReadBuffers createSchemaEvolvableBuffers(int readSchemaVersion, + podio::Backend backend) /*const*/ + = 0; + /// check for validity of the container after read virtual bool isValid() const = 0; @@ -61,6 +68,8 @@ class CollectionBase { virtual std::string getValueTypeName() const = 0; /// fully qualified type name of stored POD elements - with namespace virtual std::string getDataTypeName() const = 0; + /// schema version of the collection + virtual SchemaVersionT getSchemaVersion() const = 0; /// destructor virtual ~CollectionBase() = default; diff --git a/include/podio/CollectionBuffers.h b/include/podio/CollectionBuffers.h index d69ff0288..80b94c6dd 100644 --- a/include/podio/CollectionBuffers.h +++ b/include/podio/CollectionBuffers.h @@ -2,6 +2,7 @@ #define PODIO_COLLECTIONBUFFERS_H #include "podio/ObjectID.h" +#include "podio/SchemaEvolution.h" #include #include @@ -41,7 +42,9 @@ struct CollectionWriteBuffers { }; struct CollectionReadBuffers { + bool needsSchemaEvolution{false}; void* data{nullptr}; + void* data_oldschema{nullptr}; CollRefCollection* references{nullptr}; VectorMembersInfo* vectorMembers{nullptr}; diff --git a/include/podio/ROOTFrameWriter.h b/include/podio/ROOTFrameWriter.h index 2546613d8..3b0fde4ba 100644 --- a/include/podio/ROOTFrameWriter.h +++ b/include/podio/ROOTFrameWriter.h @@ -54,7 +54,7 @@ class ROOTFrameWriter { // collectionID, collectionType, subsetCollection // NOTE: same as in rootUtils.h private header! - using CollectionInfoT = std::tuple; + using CollectionInfoT = std::tuple; /** * Helper struct to group together all necessary state to write / process a diff --git a/include/podio/ROOTLegacyReader.h b/include/podio/ROOTLegacyReader.h index 06f9c015b..b6fed99f1 100644 --- a/include/podio/ROOTLegacyReader.h +++ b/include/podio/ROOTLegacyReader.h @@ -91,7 +91,7 @@ class ROOTLegacyReader { private: std::pair getLocalTreeAndEntry(const std::string& treename); - void createCollectionBranches(const std::vector>& collInfo); + void createCollectionBranches(const std::vector>& collInfo); podio::GenericParameters readEventMetaData(); diff --git a/include/podio/ROOTReader.h b/include/podio/ROOTReader.h index ba33cae16..03a5d5557 100644 --- a/include/podio/ROOTReader.h +++ b/include/podio/ROOTReader.h @@ -85,7 +85,7 @@ class ROOTReader : public IReader { std::map* readRunMetaData() override; private: - void createCollectionBranches(const std::vector>& collInfo); + void createCollectionBranches(const std::vector>& collInfo); std::pair getLocalTreeAndEntry(const std::string& treename); // Information about the data vector as wall as the collection class type diff --git a/include/podio/SchemaEvolution.h b/include/podio/SchemaEvolution.h new file mode 100644 index 000000000..fd77fddb6 --- /dev/null +++ b/include/podio/SchemaEvolution.h @@ -0,0 +1,14 @@ +#ifndef PODIO_SCHEMAEVOLUTION_H +#define PODIO_SCHEMAEVOLUTION_H + +#include + +namespace podio { + +enum class Backend { ROOT, SIO }; + +using SchemaVersionT = uint32_t; + +} // namespace podio + +#endif \ No newline at end of file diff --git a/include/podio/UserDataCollection.h b/include/podio/UserDataCollection.h index 7d28e2c99..b3d910ed8 100644 --- a/include/podio/UserDataCollection.h +++ b/include/podio/UserDataCollection.h @@ -118,6 +118,12 @@ class UserDataCollection : public CollectionBase { }}; } + podio::CollectionReadBuffers createSchemaEvolvableBuffers(__attribute__((unused)) int readSchemaVersion, + __attribute__((unused)) + podio::Backend backend) /*const*/ final { + return createBuffers(); + } + /// check for validity of the container after read bool isValid() const override { return true; @@ -157,6 +163,11 @@ class UserDataCollection : public CollectionBase { void setSubsetCollection(bool) override { } + /// The schema version is fixed manually + SchemaVersionT getSchemaVersion() const final { + return 1; + } + /// Print this collection to the passed stream void print(std::ostream& os = std::cout, bool flush = true) const override { os << "["; diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 54547e833..43b706390 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -4,6 +4,7 @@ SET(podio_PYTHON_DIR ${CMAKE_CURRENT_LIST_DIR} PARENT_SCOPE) set(to_install podio_class_generator.py + podio_schema_evolution.py figure.txt EventStore.py) diff --git a/python/podio/generator_utils.py b/python/podio/generator_utils.py index e50b139e7..600d11ca8 100644 --- a/python/podio/generator_utils.py +++ b/python/podio/generator_utils.py @@ -71,9 +71,10 @@ def _is_fixed_width_type(type_name): class DataType: """Simple class to hold information about a datatype or component that is defined in the datamodel.""" - def __init__(self, klass): + def __init__(self, klass, schema_version): self.full_type = klass self.namespace, self.bare_type = _get_namespace_class(self.full_type) + self.schema_version = schema_version def __str__(self): if self.namespace: @@ -195,8 +196,7 @@ def _to_json(self): class DataModel: # pylint: disable=too-few-public-methods """A class for holding a complete datamodel read from a configuration file""" - - def __init__(self, datatypes=None, components=None, options=None): + def __init__(self, datatypes=None, components=None, options=None, schema_version=None): self.options = options or { # should getters / setters be prefixed with get / set? "getSyntax": False, @@ -205,6 +205,7 @@ def __init__(self, datatypes=None, components=None, options=None): # use subfolder when including package header files "includeSubfolder": False, } + self.schema_version = schema_version self.components = components or {} self.datatypes = datatypes or {} diff --git a/python/podio/podio_config_reader.py b/python/podio/podio_config_reader.py index 3992a3aa9..79f43d1b2 100644 --- a/python/podio/podio_config_reader.py +++ b/python/podio/podio_config_reader.py @@ -409,6 +409,14 @@ def _read_datatype(cls, value): @classmethod def parse_model(cls, model_dict, package_name, upstream_edm=None): """Parse a model from the dictionary, e.g. read from a yaml file.""" + + if "schema_version" in model_dict: + schema_version = model_dict["schema_version"] + else: + warnings.warn("Please provide a schema_version entry. It will become mandatory. Setting it to 0 as default", + FutureWarning, stacklevel=3) + schema_version = 0 + components = {} if "components" in model_dict: for klassname, value in model_dict["components"].items(): @@ -432,7 +440,7 @@ def parse_model(cls, model_dict, package_name, upstream_edm=None): # If this doesn't raise an exception everything should in principle work out validator = ClassDefinitionValidator() - datamodel = DataModel(datatypes, components, options) + datamodel = DataModel(datatypes, components, options, schema_version) validator.validate(datamodel, upstream_edm) return datamodel diff --git a/python/podio_class_generator.py b/python/podio_class_generator.py index 44408c915..ad8877042 100755 --- a/python/podio_class_generator.py +++ b/python/podio_class_generator.py @@ -18,6 +18,7 @@ from podio.podio_config_reader import PodioConfigReader from podio.generator_utils import DataType, DefinitionError, DataModelJSONEncoder +from podio_schema_evolution import DataModelComparator # dealing with cyclic imports THIS_DIR = os.path.dirname(os.path.abspath(__file__)) TEMPLATE_DIR = os.path.join(THIS_DIR, 'templates') @@ -87,13 +88,20 @@ class IncludeFrom(IntEnum): class ClassGenerator: """The entry point for reading a datamodel definition and generating the necessary source code from it.""" - def __init__(self, yamlfile, install_dir, package_name, io_handlers, verbose, dryrun, upstream_edm): + def __init__(self, yamlfile, install_dir, package_name, io_handlers, verbose, dryrun, + upstream_edm, old_description, evolution_file): self.install_dir = install_dir self.package_name = package_name self.io_handlers = io_handlers self.verbose = verbose self.dryrun = dryrun self.yamlfile = yamlfile + # schema evolution specific code + self.old_yamlfile = old_description + self.evolution_file = evolution_file + self.old_datamodel = None + self.old_datamodels_components = set() + self.old_datamodels_datatypes = set() try: self.datamodel = PodioConfigReader.read(yamlfile, package_name, upstream_edm) @@ -110,6 +118,7 @@ def __init__(self, yamlfile, install_dir, package_name, io_handlers, verbose, dr self.incfolder = self.datamodel.options['includeSubfolder'] self.expose_pod_members = self.datamodel.options["exposePODMembers"] self.upstream_edm = upstream_edm + self.schema_version = self.datamodel.schema_version self.clang_format = [] self.generated_files = [] @@ -127,9 +136,36 @@ def process(self): if 'ROOT' in self.io_handlers: self._create_selection_xml() - self.print_report() self._write_cmake_lists_file() + self.process_schema_evolution() + + self.print_report() + + def process_schema_evolution(self): + """Process the schema evolution""" + # have to make all necessary comparisons + # which are the ones that changed? + # have to extend the selection xml file + if self.old_yamlfile: + comparator = DataModelComparator(self.yamlfile, self.old_yamlfile, + evolution_file=self.evolution_file) + comparator.read() + comparator.compare() + + # some sanity checks + if len(comparator.errors) > 0: + print(f"The given datamodels '{self.yamlfile}' and '{self.old_yamlfile}' \ +have unresolvable schema evolution incompatibilities:") + for error in comparator.errors: + print(error) + sys.exit(-1) + if len(comparator.warnings) > 0: + print(f"The given datamodels '{self.yamlfile}' and '{self.old_yamlfile}' \ +have resolvable schema evolution incompatibilities:") + for warning in comparator.warnings: + print(warning) + sys.exit(-1) def print_report(self): """Print a summary report about the generated code""" @@ -228,7 +264,7 @@ def _process_component(self, name, component): includes.update(component.get("ExtraCode", {}).get("includes", "").split('\n')) component['includes'] = self._sort_includes(includes) - component['class'] = DataType(name) + component['class'] = DataType(name, self.schema_version) self._fill_templates('Component', component) @@ -375,7 +411,7 @@ def _preprocess_datatype(self, name, definition): # Make a copy here and add the preprocessing steps to that such that the # original definition can be left untouched data = deepcopy(definition) - data['class'] = DataType(name) + data['class'] = DataType(name, self.schema_version) data['includes_data'] = self._get_member_includes(definition["Members"]) self._preprocess_for_class(data) self._preprocess_for_obj(data) @@ -459,8 +495,10 @@ def _needs_include(self, classname) -> IncludeFrom: def _create_selection_xml(self): """Create the selection xml that is necessary for ROOT I/O""" - data = {'components': [DataType(c) for c in self.datamodel.components], - 'datatypes': [DataType(d) for d in self.datamodel.datatypes]} + data = {'components': [DataType(c, self.schema_version) for c in self.datamodel.components], + 'datatypes': [DataType(d, self.schema_version) for d in self.datamodel.datatypes], + 'old_schema_components': [DataType(d, self.schema_version) for d in + self.old_datamodels_datatypes | self.old_datamodels_components]} self._write_file('selection.xml', self._eval_template('selection.xml.jinja2', data)) def _build_include(self, member): @@ -539,6 +577,11 @@ def read_upstream_edm(name_path): ' EDM. Format is \':\'. ' 'Note that only the code for the current EDM will be generated', default=None, type=read_upstream_edm) + parser.add_argument('--old-description', + help='Provide schema evolution relative to the old yaml file.', + default=None, action='store') + parser.add_argument('-e', '--evolution_file', help='yaml file clarifying schema evolutions', + default=None, action='store') args = parser.parse_args() @@ -551,7 +594,8 @@ def read_upstream_edm(name_path): os.makedirs(directory) gen = ClassGenerator(args.description, args.targetdir, args.packagename, args.iohandlers, - verbose=args.verbose, dryrun=args.dryrun, upstream_edm=args.upstream_edm) + verbose=args.verbose, dryrun=args.dryrun, upstream_edm=args.upstream_edm, + old_description=args.old_description, evolution_file=args.evolution_file) if args.clangformat: gen.clang_format = get_clang_format() gen.process() diff --git a/python/podio_schema_evolution.py b/python/podio_schema_evolution.py new file mode 100755 index 000000000..7bd5b676b --- /dev/null +++ b/python/podio_schema_evolution.py @@ -0,0 +1,370 @@ +#!/usr/bin/env python +""" +Provides infrastructure for analyzing schema definitions for schema evolution +""" + +import yaml + +from podio.podio_config_reader import PodioConfigReader + + +# @TODO: not really a good class model here +# this is a remnant from previous more-sophisticated setups + + +class SchemaChange: + """The base class for all schema changes providing a brief description as representation""" + def __init__(self, description): + self.description = description + + def __str__(self) -> str: + return self.description + + def __repr__(self) -> str: + return self.description + + +class AddedComponent(SchemaChange): + """Class representing an added component""" + def __init__(self, component, name): + self.component = component + self.name = name + super().__init__(f"'{self.component.name}' has been added") + + +class DroppedComponent(SchemaChange): + """Class representing a dropped component""" + def __init__(self, component, name): + self.component = component + self.name = name + self.klassname = name + super().__init__(f"'{self.name}' has been dropped") + + +class AddedDatatype(SchemaChange): + """Class representing an added datatype""" + def __init__(self, datatype, name): + self.datatype = datatype + self.name = name + self.klassname = name + super().__init__(f"'{self.name}' has been added") + + +class DroppedDatatype(SchemaChange): + """Class representing a dropped datatype""" + def __init__(self, datatype, name): + self.datatype = datatype + self.name = name + self.klassname = name + super().__init__(f"'{self.name}' has been dropped") + + +class RenamedDataType(SchemaChange): + """Class representing a renamed datatype""" + def __init__(self, name_old, name_new): + self.name_old = name_old + self.name_new = name_new + super().__init__(f"'{self.name_new}': datatype '{self.name_old}' renamed to '{self.name_new}'.") + + +class AddedMember(SchemaChange): + """Class representing an added member""" + def __init__(self, member, definition_name): + self.member = member + self.definition_name = definition_name + self.klassname = definition_name + super().__init__(f"'{self.definition_name}' has an added member '{self.member.name}'") + + +class DroppedMember(SchemaChange): + """Class representing a dropped member""" + def __init__(self, member, definition_name): + self.member = member + self.definition_name = definition_name + self.klassname = definition_name + super().__init__(f"'{self.definition_name}' has a dropped member '{self.member.name}") + + +class ChangedMember(SchemaChange): + """Class representing a type change in a member""" + def __init__(self, name, member_name, old_member, new_member): + self.name = name + self.member_name = member_name + self.old_member = old_member + self.new_member = new_member + self.klassname = name + super().__init__(f"'{self.name}.{self.member_name}' changed type from '+\ + '{self.old_member.full_type} to {self.new_member.full_type}") + + +class RenamedMember(SchemaChange): + """Class representing a renamed member""" + def __init__(self, name, member_name_old, member_name_new): + self.name = name + self.member_name_old = member_name_old + self.member_name_new = member_name_new + self.klassname = name + super().__init__(f"'{self.name}': member '{self.member_name_old}' renamed to '{self.member_name_new}'.") + + +def sio_filter(schema_changes): + """ + Checks what is required/supported for the SIO backend + + At this point in time all schema changes have to be handled on PODIO side + + """ + return schema_changes + + +def root_filter(schema_changes): + """ + Checks what is required/supported for the ROOT backend + + At this point in time we are only interested in renames. + Everything else will be done by ROOT automatically + """ + relevant_schema_changes = [] + for schema_change in schema_changes: + if isinstance(schema_change, RenamedMember): + relevant_schema_changes.append(schema_change) + return relevant_schema_changes + + +class DataModelComparator: + """ + Compares two datamodels and extracts required schema evolution + """ + def __init__(self, yamlfile_new, yamlfile_old, evolution_file=None) -> None: + self.yamlfile_new = yamlfile_new + self.yamlfile_old = yamlfile_old + self.evolution_file = evolution_file + self.reader = PodioConfigReader() + + self.datamodel_new = None + self.datamodel_old = None + self.detected_schema_changes = [] + self.read_schema_changes = [] + self.schema_changes = [] + + self.warnings = [] + self.errors = [] + + def compare(self) -> None: + """execute the comparison on-preloaded datamodel definitions""" + self._compare_components() + self._compare_datatypes() + self.heuristics() + + def _compare_components(self) -> None: + """compare component definitions of old and new datamodel""" + # first check for dropped, added and kept components + added_components, dropped_components, kept_components = self._compare_keys(self.datamodel_new.components.keys(), + self.datamodel_old.components.keys()) + # Make findings known globally + self.detected_schema_changes.extend([AddedComponent(self.datamodel_new.components[name], name) + for name in added_components]) + self.detected_schema_changes.extend([DroppedComponent(self.datamodel_old.components[name], name) + for name in dropped_components]) + + self._compare_definitions(kept_components, self.datamodel_new.components, self.datamodel_old.components, "Members") + + def _compare_datatypes(self) -> None: + """compare datatype definitions of old and new datamodel""" + # first check for dropped, added and kept components + added_types, dropped_types, kept_types = self._compare_keys(self.datamodel_new.datatypes.keys(), + self.datamodel_old.datatypes.keys()) + # Make findings known globally + self.detected_schema_changes.extend([AddedDatatype(self.datamodel_new.datatypes[name], name) + for name in added_types]) + self.detected_schema_changes.extend([DroppedDatatype(self.datamodel_old.datatypes[name], name) + for name in dropped_types]) + + self._compare_definitions(kept_types, self.datamodel_new.datatypes, self.datamodel_old.datatypes, "Members") + + def _compare_definitions(self, definitions, first, second, category) -> None: + """compare member definitions in old and new datamodel""" + for name in definitions: + # we are only interested in members not the extracode + members1 = {member.name: member for member in first[name][category]} + members2 = {member.name: member for member in second[name][category]} + added_members, dropped_members, kept_members = self._compare_keys(members1.keys(), + members2.keys()) + # Make findings known globally + self.detected_schema_changes.extend([AddedMember(members1[member], name) for member in added_members]) + self.detected_schema_changes.extend([DroppedMember(members2[member], name) for member in dropped_members]) + + # now let's compare old and new for the kept members + for member_name in kept_members: + new = members1[member_name] + old = members2[member_name] + if old.full_type != new.full_type: + self.detected_schema_changes.append(ChangedMember(name, member_name, old, new)) + + @staticmethod + def _compare_keys(keys1, keys2): + """compare keys of two given dicts. return added, dropped and overlapping keys""" + added = set(keys1).difference(keys2) + dropped = set(keys2).difference(keys1) + kept = set(keys1).intersection(keys2) + return added, dropped, kept + + def get_changed_schemata(self, schema_filter=None): + """return the schemata which actually changed""" + if schema_filter: + schema_changes = schema_filter(self.schema_changes) + else: + schema_changes = self.schema_changes + changed_klasses = {} + for schema_change in schema_changes: + changed_klass = changed_klasses.setdefault(schema_change.klassname, []) + changed_klass.append(schema_change) + return changed_klasses + + def heuristics_members(self, added_members, dropped_members, schema_changes): + """make analysis of member changes in a given data type """ + for dropped_member in dropped_members: + added_members_in_definition = [member for member in added_members if + dropped_member.definition_name == member.definition_name] + for added_member in added_members_in_definition: + if added_member.member.full_type == dropped_member.member.full_type: + # this is a rename candidate. So let's see whether it has been explicitly declared by the user + is_rename = False + for schema_change in self.read_schema_changes: + if isinstance(schema_change, RenamedMember) and \ + (schema_change.name == dropped_member.definition_name) and \ + (schema_change.member_name_old == dropped_member.member.name) and \ + (schema_change.member_name_new == added_member.member.name): + # remove the dropping/adding from the schema changes and replace it by the rename + schema_changes.remove(dropped_member) + schema_changes.remove(added_member) + schema_changes.append(schema_change) + is_rename = True + if not is_rename: + self.warnings.append(f"Definition '{dropped_member.definition_name}' has a potential rename " + f"'{dropped_member.member.name}' -> '{added_member.member.name}' of type " + f"'{dropped_member.member.full_type}'.") + + def heuristics(self): + """make an analysis of the data model changes: + - check which can be auto-resolved + - check which need extra information from the user + - check which one are plain forbidden/impossible + """ + # let's analyse the changes in more detail + # make a copy that can be altered along the way + schema_changes = self.detected_schema_changes.copy() + # are there dropped/added member pairs that could be interpreted as rename? + dropped_members = [change for change in schema_changes if isinstance(change, DroppedMember)] + added_members = [change for change in schema_changes if isinstance(change, AddedMember)] + self.heuristics_members(added_members, dropped_members, schema_changes) + + # are the member changes actually supported/supportable? + changed_members = [change for change in schema_changes if isinstance(change, ChangedMember)] + for change in changed_members: + # changes between arrays and basic types are forbidden + if change.old_member.is_array != change.new_member.is_array: + self.errors.append(f"Forbidden schema change in '{change.name}' for '{change.member_name}' from " + f"'{change.old_member.full_type}' to '{change.new_member.full_type}'") + + # are there dropped/added datatype pairs that could be interpreted as rename? + # for now assuming no change to the individual datatype definition + # I do not think more complicated heuristics are needed at this point in time + dropped_datatypes = [change for change in schema_changes if isinstance(change, DroppedDatatype)] + added_datatypes = [change for change in schema_changes if isinstance(change, AddedDatatype)] + + for dropped in dropped_datatypes: + dropped_members = {member.name: member for member in dropped.datatype["Members"]} + is_known_evolution = False + for added in added_datatypes: + added_members = {member.name: member for member in added.datatype["Members"]} + if set(dropped_members.keys()) == set(added_members.keys()): + for schema_change in self.read_schema_changes: + if isinstance(schema_change, RenamedDataType) and \ + (schema_change.name_old == dropped.name and schema_change.name_new == added.name): + schema_changes.remove(dropped) + schema_changes.remove(added) + schema_changes.append(schema_change) + is_known_evolution = True + if not is_known_evolution: + self.warnings.append(f"Potential rename of '{dropped.name}' into '{added.name}'.") + + # are there dropped/added component pairs that could be interpreted as rename? + dropped_components = [change for change in schema_changes if isinstance(change, DroppedComponent)] + added_components = [change for change in schema_changes if isinstance(change, AddedComponent)] + + for dropped in dropped_components: + dropped_members = {member.name: member for member in dropped.component["Members"]} + for added in added_components: + added_members = {member.name: member for member in added.component["Members"]} + if set(dropped_members.keys()) == set(added_members.keys()): + self.warnings.append(f"Potential rename of '{dropped.name}' into '{added.name}'.") + + # make the results of the heuristics known to the instance + self.schema_changes = schema_changes + + def print_comparison(self): + """print the result of the datamodel comparison""" + print(f"Comparing datamodel versions {self.datamodel_new.schema_version} and {self.datamodel_old.schema_version}") + + print(f"Detected {len(self.schema_changes)} schema changes:") + for change in self.schema_changes: + print(f" - {change}") + + if len(self.warnings) > 0: + print("Warnings:") + for warning in self.warnings: + print(f" - {warning}") + + if len(self.errors) > 0: + print("ERRORS:") + for error in self.errors: + print(f" - {error}") + + def read(self) -> None: + """read datamodels from yaml files""" + self.datamodel_new = self.reader.read(self.yamlfile_new, package_name="new") + self.datamodel_old = self.reader.read(self.yamlfile_old, package_name="old") + if self.evolution_file: + self.read_evolution_file() + + def read_evolution_file(self) -> None: + """read and parse evolution file""" + supported_operations = ('member_rename', 'class_renamed_to') + with open(self.evolution_file, "r", encoding='utf-8') as stream: + content = yaml.load(stream, yaml.SafeLoader) + from_schema_version = content["from_schema_version"] + to_schema_version = content["to_schema_version"] + if (from_schema_version != self.datamodel_old.schema_version) or (to_schema_version != self.datamodel_new.schema_version): # nopep8 # noqa + raise BaseException("Versions in schema evolution file do not match versions in data model descriptions.") # nopep8 # noqa + + if "evolutions" in content: + for klassname, value in content["evolutions"].items(): + # now let's go through the various supported evolutions + for operation, details in value.items(): + if operation not in supported_operations: + raise BaseException(f'Schema evolution operation {operation} in {klassname} unknown or not supported') # nopep8 # noqa + if operation == 'member_rename': + schema_change = RenamedMember(klassname, details[0], details[1]) + self.read_schema_changes.append(schema_change) + elif operation == 'class_renamed_to': + schema_change = RenamedDataType(klassname, details) + self.read_schema_changes.append(schema_change) + + +########################## +if __name__ == "__main__": + import argparse + parser = argparse.ArgumentParser(description='Given two yaml files this script analyzes ' + 'the difference of the two datamodels') + + parser.add_argument('new', help='yaml file describing the new datamodel') + parser.add_argument('old', help='yaml file describing the old datamodel') + parser.add_argument('-e', '--evo', help='yaml file clarifying schema evolutions', action='store') + args = parser.parse_args() + + comparator = DataModelComparator(args.new, args.old, evolution_file=args.evo) + comparator.read() + comparator.compare() + comparator.print_comparison() + print(comparator.get_changed_schemata(schema_filter=root_filter)) diff --git a/python/templates/Collection.cc.jinja2 b/python/templates/Collection.cc.jinja2 index 8c121de20..185b98971 100644 --- a/python/templates/Collection.cc.jinja2 +++ b/python/templates/Collection.cc.jinja2 @@ -175,6 +175,16 @@ podio::CollectionReadBuffers {{ collection_type }}::createBuffers() /*const*/ { return readBuffers; } +podio::CollectionReadBuffers {{ collection_type }}::createSchemaEvolvableBuffers(int readSchemaVersion, podio::Backend /*backend*/) /*const*/ { + // no version difference -> no-op + if (readSchemaVersion == {{ class.schema_version }}) { + return createBuffers(); + } + // default is no-op as well + return createBuffers(); +} + + {% for member in Members %} {{ macros.vectorized_access(class, member) }} {% endfor %} diff --git a/python/templates/Collection.h.jinja2 b/python/templates/Collection.h.jinja2 index 2c1a80e3b..f66a4fb89 100644 --- a/python/templates/Collection.h.jinja2 +++ b/python/templates/Collection.h.jinja2 @@ -82,6 +82,8 @@ public: std::string getValueTypeName() const final { return std::string("{{ (class | string ).strip(':') }}"); } /// fully qualified type name of stored POD elements - with namespace std::string getDataTypeName() const final { return std::string("{{ (class | string ).strip(':')+"Data" }}"); } + /// schema version + unsigned int getSchemaVersion() const final { return {{ class.schema_version }}; }; bool isSubsetCollection() const final { return m_isSubsetColl; @@ -112,6 +114,11 @@ public: /// Create (empty) collection buffers from which a collection can be constructed podio::CollectionReadBuffers createBuffers() /*const*/ final; + /// Create (empty) collection buffers from which a collection can be constructed + /// Versioned to support schema evolution + podio::CollectionReadBuffers createSchemaEvolvableBuffers(int readSchemaVersion, podio::Backend backend) /*const*/ final; + + void setID(unsigned ID) final { m_collectionID = ID; if (!m_isSubsetColl) { diff --git a/python/templates/schemaevolution/EvolvePOD.h.jinja2 b/python/templates/schemaevolution/EvolvePOD.h.jinja2 new file mode 100644 index 000000000..e69de29bb diff --git a/python/templates/selection.xml.jinja2 b/python/templates/selection.xml.jinja2 index 1d6f4e4f6..b0da9ff74 100644 --- a/python/templates/selection.xml.jinja2 +++ b/python/templates/selection.xml.jinja2 @@ -23,7 +23,12 @@ {# we have to declare them here, otherwise they cannot be easily imported #} {{ class_selection(class) }} {{ class_selection(class, postfix='Collection') }} +{% endfor %} + +{% for class in old_schema_components %} +{{ class_selection(class) }} {% endfor %} + diff --git a/src/ROOTFrameReader.cc b/src/ROOTFrameReader.cc index f8880133c..f4ba00bc4 100644 --- a/src/ROOTFrameReader.cc +++ b/src/ROOTFrameReader.cc @@ -264,7 +264,7 @@ createCollectionBranches(TChain* chain, const podio::CollectionIDTable& idTable, std::vector> storedClasses; storedClasses.reserve(collInfo.size()); - for (const auto& [collID, collType, isSubsetColl] : collInfo) { + for (const auto& [collID, collType, isSubsetColl, collSchemaVersion] : collInfo) { // We only write collections that are in the collectionIDTable, so no need // to check here const auto name = idTable.name(collID); diff --git a/src/ROOTFrameWriter.cc b/src/ROOTFrameWriter.cc index 3f552d69f..e6fa85de6 100644 --- a/src/ROOTFrameWriter.cc +++ b/src/ROOTFrameWriter.cc @@ -95,7 +95,8 @@ void ROOTFrameWriter::initBranches(CategoryInfo& catInfo, const std::vectorgetTypeName(), coll->isSubsetCollection()); + catInfo.collInfo.emplace_back(catInfo.idTable.collectionID(name), coll->getTypeName(), coll->isSubsetCollection(), + coll->getSchemaVersion()); } // Also make branches for the parameters diff --git a/src/ROOTLegacyReader.cc b/src/ROOTLegacyReader.cc index f25c1b6bc..0a9380bd1 100644 --- a/src/ROOTLegacyReader.cc +++ b/src/ROOTLegacyReader.cc @@ -185,7 +185,7 @@ unsigned ROOTLegacyReader::getEntries(const std::string& name) const { void ROOTLegacyReader::createCollectionBranches(const std::vector& collInfo) { size_t collectionIndex{0}; - for (const auto& [collID, collType, isSubsetColl] : collInfo) { + for (const auto& [collID, collType, isSubsetColl, collSchemaVersion] : collInfo) { // We only write collections that are in the collectionIDTable, so no need // to check here const auto name = m_table->name(collID); diff --git a/src/ROOTReader.cc b/src/ROOTReader.cc index b2cb33100..781c1aea1 100644 --- a/src/ROOTReader.cc +++ b/src/ROOTReader.cc @@ -163,25 +163,45 @@ void ROOTReader::openFiles(const std::vector& filenames) { podio::version::Version* versionPtr{nullptr}; if (auto* versionBranch = root_utils::getBranch(metadatatree, "PodioVersion")) { versionBranch->SetAddress(&versionPtr); + metadatatree->GetEntry(0); } + m_fileVersion = versionPtr ? *versionPtr : podio::version::Version{0, 0, 0}; + + // Read the collection type info + // For versions <0.13.1 it does not exist and has to be rebuilt from scratch + if (m_fileVersion < podio::version::Version{0, 13, 1}) { + + std::cout << "PODIO: Reconstructing CollectionTypeInfo branch from other sources in file: \'" + << m_chain->GetFile()->GetName() << "\'" << std::endl; + metadatatree->GetEntry(0); + const auto collectionInfo = root_utils::reconstructCollectionInfo(m_chain, *m_table); + createCollectionBranches(collectionInfo); - // Check if the CollectionTypeInfo branch is there and assume that the file - // has been written with with podio pre #197 (<0.13.1) if that is not the case - if (auto* collInfoBranch = root_utils::getBranch(metadatatree, "CollectionTypeInfo")) { + } else if (m_fileVersion < podio::version::Version{0, 17, 0}) { + + auto* collInfoBranch = root_utils::getBranch(metadatatree, "CollectionTypeInfo"); + auto collectionInfoWithoutSchema = new std::vector; auto collectionInfo = new std::vector; - collInfoBranch->SetAddress(&collectionInfo); + collInfoBranch->SetAddress(&collectionInfoWithoutSchema); metadatatree->GetEntry(0); + for (const auto& [collID, collType, isSubsetColl] : *collectionInfoWithoutSchema) { + collectionInfo->emplace_back(collID, collType, isSubsetColl, 0); + } createCollectionBranches(*collectionInfo); + delete collectionInfoWithoutSchema; delete collectionInfo; + } else { - std::cout << "PODIO: Reconstructing CollectionTypeInfo branch from other sources in file: \'" - << m_chain->GetFile()->GetName() << "\'" << std::endl; + + auto* collInfoBranch = root_utils::getBranch(metadatatree, "CollectionTypeInfo"); + + auto collectionInfo = new std::vector; + collInfoBranch->SetAddress(&collectionInfo); metadatatree->GetEntry(0); - const auto collectionInfo = root_utils::reconstructCollectionInfo(m_chain, *m_table); - createCollectionBranches(collectionInfo); + createCollectionBranches(*collectionInfo); + delete collectionInfo; } - m_fileVersion = versionPtr ? *versionPtr : podio::version::Version{0, 0, 0}; delete versionPtr; } @@ -226,7 +246,7 @@ void ROOTReader::goToEvent(unsigned eventNumber) { void ROOTReader::createCollectionBranches(const std::vector& collInfo) { size_t collectionIndex{0}; - for (const auto& [collID, collType, isSubsetColl] : collInfo) { + for (const auto& [collID, collType, isSubsetColl, collSchemaVersion] : collInfo) { // We only write collections that are in the collectionIDTable, so no need // to check here const auto name = m_table->name(collID); diff --git a/src/ROOTWriter.cc b/src/ROOTWriter.cc index 46842b370..cf0a768e6 100644 --- a/src/ROOTWriter.cc +++ b/src/ROOTWriter.cc @@ -112,7 +112,7 @@ void ROOTWriter::finish() { m_store->get(name, coll); const auto collType = coll->getTypeName(); // const auto collType = "std::vector<" + coll->getDataTypeName() + ">"; - collectionInfo.emplace_back(collID, std::move(collType), coll->isSubsetCollection()); + collectionInfo.emplace_back(collID, std::move(collType), coll->isSubsetCollection(), coll->getSchemaVersion()); } m_metadatatree->Branch("CollectionTypeInfo", &collectionInfo); diff --git a/src/rootUtils.h b/src/rootUtils.h index 215c7fea6..2ad69389b 100644 --- a/src/rootUtils.h +++ b/src/rootUtils.h @@ -101,9 +101,11 @@ inline void setCollectionAddresses(const BufferT& collBuffers, const CollectionB } // A collection of additional information that describes the collection: the -// collectionID, the collection (data) type, and whether it is a subset -// collection -using CollectionInfoT = std::tuple; +// collectionID, the collection (data) type, whether it is a subset +// collection, and its schema version +using CollectionInfoT = std::tuple; +// for backwards compatibility +using CollectionInfoTWithoutSchema = std::tuple; inline void readBranchesData(const CollectionBranches& branches, Long64_t entry) { // Read all data @@ -140,8 +142,8 @@ inline auto reconstructCollectionInfo(TTree* eventTree, podio::CollectionIDTable std::string_view dataClass = bufferClassName; dataClass.remove_suffix(5); const auto collClass = std::string(dataClass.substr(7)) + "Collection"; - // Assume that there are no subset collections in "old files" - collInfo.emplace_back(collID, std::move(collClass), false); + // Assume that there are no subset collections in "old files" and schema is 0 + collInfo.emplace_back(collID, std::move(collClass), false, 0); } else { std::cerr << "Problems reconstructing collection info for collection: \'" << name << "\'\n"; } diff --git a/src/selection.xml b/src/selection.xml index d198bfab6..92ca68764 100644 --- a/src/selection.xml +++ b/src/selection.xml @@ -14,6 +14,8 @@ + + diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 2b056bc74..c3944d4ea 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -6,7 +6,10 @@ foreach( _conf ${CMAKE_CONFIGURATION_TYPES} ) endforeach() PODIO_GENERATE_DATAMODEL(datamodel datalayout.yaml headers sources - IO_BACKEND_HANDLERS ${PODIO_IO_HANDLERS}) + IO_BACKEND_HANDLERS ${PODIO_IO_HANDLERS} + OLD_DESCRIPTION datalayout_old.yaml + SCHEMA_EVOLUTION schema_evolution.yaml + ) # Use the cmake building blocks to add the different parts (conditionally) PODIO_ADD_DATAMODEL_CORE_LIB(TestDataModel "${headers}" "${sources}") diff --git a/tests/datalayout.yaml b/tests/datalayout.yaml index 4028ec7d1..2bcf7cb11 100755 --- a/tests/datalayout.yaml +++ b/tests/datalayout.yaml @@ -1,4 +1,5 @@ --- +schema_version : 1 options : # should getters / setters be prefixed with get / set? diff --git a/tests/datalayout_old.yaml b/tests/datalayout_old.yaml new file mode 100755 index 000000000..81a9d5707 --- /dev/null +++ b/tests/datalayout_old.yaml @@ -0,0 +1,203 @@ +--- +schema_version : 0 + +options : + # should getters / setters be prefixed with get / set? + getSyntax: False + # should POD members be exposed with getters/setters in classes that have them as members? + exposePODMembers: True + includeSubfolder: True + +components : + + ToBeDroppedStruct: + Members: + - int x + + SimpleStruct: + Members: + - int x + - int z + - std::array p + # can also add c'tors: + ExtraCode : + declaration: " + SimpleStruct() : x(0),y(0),z(0) {}\n + SimpleStruct( const int* v) : x(v[0]),y(v[1]),z(v[2]) {} + " + + NotSoSimpleStruct: + Members: + - SimpleStruct data // component members can have descriptions + + ex2::NamespaceStruct: + Members: + - int x + - int y_old + + ex2::NamespaceInNamespaceStruct: + Members: + - ex2::NamespaceStruct data + + StructWithFixedWithTypes: + Members: + - uint16_t fixedUnsigned16 // unsigned int with exactly 16 bits + - std::int64_t fixedInteger64 // int with exactly 64 bits + - int32_t fixedInteger32 // int with exactly 32 bits + + CompWithInit: + Members: + - int i{42} // is there even another value to initialize ints to? + - std::array arr {1.2, 3.4} // half initialized double array + +datatypes : + + EventInfoOldName: + Description : "Event info" + Author : "B. Hegner" + Members : + - int Number // event number + MutableExtraCode : + declaration: "void setNumber(int n) { Number( n ) ; } " + ExtraCode: + declaration: "int getNumber() const;" + implementation: "int {name}::getNumber() const { return Number(); }" + + ExampleHit : + Description : "Example Hit" + Author : "B. Hegner" + Members: + - unsigned long long cellID // cellID + - double x // x-coordinate + - double y // y-coordinate + - double z // z-coordinate + - double energy // measured energy deposit + + ExampleMC : + Description : "Example MC-particle" + Author: "F.Gaede" + Members: + - double energy // energy + - int PDG // PDG code + OneToManyRelations: + - ExampleMC parents // parents + - ExampleMC daughters // daughters + + ExampleCluster : + Description : "Cluster" + Author : "B. Hegner" + Members: + - double energy // cluster energy + OneToManyRelations: + - ExampleHit Hits // hits contained in the cluster + - ExampleCluster Clusters // sub clusters used to create this cluster + + ExampleReferencingType : + Description : "Referencing Type" + Author : "B. Hegner" + OneToManyRelations: + - ExampleCluster Clusters // some refs to Clusters + - ExampleReferencingType Refs // refs into same type + + ExampleWithVectorMember : + Description : "Type with a vector member" + Author : "B. Hegner" + VectorMembers: + - int count // various ADC counts + + ExampleWithOneRelation : + Description : "Type with one relation member" + Author : "Benedikt Hegner" + OneToOneRelations: + - ExampleCluster cluster // a particular cluster + + ExampleWithArrayComponent: + Description: "A type that has a component with an array" + Author: "Thomas Madlener" + Members: + - SimpleStruct s // a component that has an array + + ExampleWithComponent : + Description : "Type with one component" + Author : "Benedikt Hegner" + Members : + - NotSoSimpleStruct component // a component + + ExampleForCyclicDependency1 : + Description : "Type for cyclic dependency" + Author : "Benedikt Hegner" + OneToOneRelations : + - ExampleForCyclicDependency2 ref // a ref + + ExampleForCyclicDependency2 : + Description : "Type for cyclic dependency" + Author : "Benedikt Hegner" + OneToOneRelations : + - ExampleForCyclicDependency1 ref // a ref + +# ExampleWithArray : +# Description : "Type with an array" +# Author : "Benedikt Hegner" +# Members: +# - std::array array // the array + + ex42::ExampleWithNamespace : + Description : "Type with namespace and namespaced member" + Author : "Joschka Lingemann" + Members: + - ex2::NamespaceStruct component // a component + + ex42::ExampleWithARelation : + Description : "Type with namespace and namespaced relation" + Author : "Joschka Lingemann" + Members: + - float number // just a number + OneToOneRelations : + - ex42::ExampleWithNamespace ref // a ref in a namespace + OneToManyRelations : + - ex42::ExampleWithNamespace refs // multiple refs in a namespace + + ExampleWithDifferentNamespaceRelations: + Description: "Datatype using a namespaced relation without being in the same namespace" + Author: "Thomas Madlener" + OneToOneRelations: + - ex42::ExampleWithNamespace rel // a relation in a different namespace + OneToManyRelations: + - ex42::ExampleWithNamespace rels // relations in a different namespace + + ExampleWithArray: + Description: "Datatype with an array member" + Author: "Joschka Lingemann" + Members: + - NotSoSimpleStruct arrayStruct // component that contains an array + - std::array myArray // array-member without space to test regex + - std::array anotherArray2 // array-member with space to test regex + - std::array snail_case_array // snail case to test regex + - std::array snail_case_Array3 // mixing things up for regex + - std::array structArray // an array containing structs + + ExampleWithFixedWidthIntegers: + Description: "Datatype using fixed width integer types as members" + Author: "Thomas Madlener" + Members: + - std::int16_t fixedI16 // some integer with exactly 16 bits + - uint64_t fixedU64 // unsigned int with exactly 64 bits + - uint32_t fixedU32 // unsigned int with exactly 32 bits + - StructWithFixedWithTypes fixedWidthStruct // struct with more fixed width types + - std::array fixedWidthArray // 32 bits split into two times 16 bits + + ExampleWithUserInit: + Description: "Datatype with user defined initialization values" + Author: "Thomas Madlener" + Members: + - std::int16_t i16Val{42} // some int16 value + - std::array floats {3.14f, 1.23f} // some float values + - ex2::NamespaceStruct s{10, 11} // one that we happen to know works + - double d{9.876e5} // double val + - CompWithInit comp // To make sure that the default initializer of the component does what it should + + ExampleOfDroppedType: + Description: "Datatype with user defined initialization values" + Author: "Thomas Madlener" + Members: + - int x // some member \ No newline at end of file diff --git a/tests/schema_evolution.yaml b/tests/schema_evolution.yaml new file mode 100644 index 000000000..8a9e925f8 --- /dev/null +++ b/tests/schema_evolution.yaml @@ -0,0 +1,13 @@ +--- +from_schema_version : 0 +to_schema_version : 1 + +evolutions: + + ex2::NamespaceStruct: + member_rename: + - y_old + - y + + EventInfoOldName: + class_renamed_to: EventInfo