From 2f1d05ee126276b7be8be3cf0431b41004d22ac3 Mon Sep 17 00:00:00 2001 From: MBorne Date: Fri, 9 Jul 2021 18:49:38 +0200 Subject: [PATCH] add tables with a FeatureType on MultiTableModel + add a pre-process to check and generate FeatureType according to data in order to allow geometric validation and post-process without providing table models (refs #231) --- .../main/java/fr/ign/validator/Context.java | 4 + .../validator/data/file/MultiTableFile.java | 5 +- .../fr/ign/validator/data/file/TableFile.java | 4 - .../fr/ign/validator/database/Database.java | 140 ++++++++--- .../ign/validator/io/AbstractModelReader.java | 12 +- .../fr/ign/validator/io/JsonModelReader.java | 28 ++- .../fr/ign/validator/io/XmlModelReader.java | 15 +- .../fr/ign/validator/model/FeatureType.java | 7 + .../fr/ign/validator/model/FileModel.java | 16 -- .../model/file/EmbeddedTableModel.java | 56 +++++ .../validator/model/file/MultiTableModel.java | 35 +++ .../ign/validator/model/file/TableModel.java | 20 ++ .../normalize/DocumentNormalizer.java | 46 ++-- .../process/CheckFeatureTypesPreProcess.java | 231 ++++++++++++++++++ .../ign/validator/tools/AutoFeatureType.java | 8 + .../ign/validator/tools/MultiTableReader.java | 2 +- .../fr/ign/validator/tools/TableReader.java | 2 +- .../database/AttributeReferenceValidator.java | 13 +- .../database/AttributeUniqueValidator.java | 13 +- .../java/fr/ign/validator/ContextTest.java | 17 +- .../ign/validator/io/JsonModelReaderTest.java | 15 +- .../ign/validator/io/XmlModelReaderTest.java | 21 +- .../normalize/TableNormalizerTest.java | 6 +- .../regress/ValidatePCRSRegressTest.java | 3 +- .../AttributeUniqueValidatorTest.java | 4 +- .../resources/config-json/pcrs-2.0/files.json | 71 +++++- .../config-json/pcrs-2.0/tables.json | 59 +++++ .../process/CreateShapefilesPostProcess.java | 3 +- .../process/CustomizeIdurbaPreProcess.java | 2 +- .../cnig/process/SupRelationsPostProcess.java | 7 +- .../validation/attribute/InseeValidator.java | 6 +- .../fr/ign/validator/cnig/tools/VRTTest.java | 6 +- .../dgpr/process/CreateTheGeomColumn.java | 2 +- .../attribute/NumericCustomizer.java | 21 +- 34 files changed, 747 insertions(+), 153 deletions(-) create mode 100644 validator-core/src/main/java/fr/ign/validator/model/file/EmbeddedTableModel.java create mode 100644 validator-core/src/main/java/fr/ign/validator/process/CheckFeatureTypesPreProcess.java create mode 100644 validator-core/src/test/resources/config-json/pcrs-2.0/tables.json diff --git a/validator-core/src/main/java/fr/ign/validator/Context.java b/validator-core/src/main/java/fr/ign/validator/Context.java index 239e006c..7179a8a3 100644 --- a/validator-core/src/main/java/fr/ign/validator/Context.java +++ b/validator-core/src/main/java/fr/ign/validator/Context.java @@ -30,6 +30,7 @@ import fr.ign.validator.model.FileModel; import fr.ign.validator.model.Model; import fr.ign.validator.model.Projection; +import fr.ign.validator.process.CheckFeatureTypesPreProcess; import fr.ign.validator.process.DocumentInfoExtractorPostProcess; import fr.ign.validator.process.FilterMetadataPreProcess; import fr.ign.validator.process.MetadataPreProcess; @@ -208,6 +209,9 @@ private void registerDefaultListeners() { // Info and warning about CRS addListener(new ProjectionPreProcess()); + // Check and complete FeatureType definitions + addListener(new CheckFeatureTypesPreProcess()); + // generate CSV addListener(new NormalizePostProcess()); // produce document-info.json diff --git a/validator-core/src/main/java/fr/ign/validator/data/file/MultiTableFile.java b/validator-core/src/main/java/fr/ign/validator/data/file/MultiTableFile.java index cdfb8703..8fca41e1 100644 --- a/validator-core/src/main/java/fr/ign/validator/data/file/MultiTableFile.java +++ b/validator-core/src/main/java/fr/ign/validator/data/file/MultiTableFile.java @@ -20,9 +20,12 @@ public MultiTableModel getFileModel() { return fileModel; } + /** + * TODO complete to validate according to {@link MultiTableModel} + */ @Override protected void validateContent(Context context) { - // TODO complete + } } diff --git a/validator-core/src/main/java/fr/ign/validator/data/file/TableFile.java b/validator-core/src/main/java/fr/ign/validator/data/file/TableFile.java index 664086e6..6f097562 100644 --- a/validator-core/src/main/java/fr/ign/validator/data/file/TableFile.java +++ b/validator-core/src/main/java/fr/ign/validator/data/file/TableFile.java @@ -38,9 +38,6 @@ protected void validateContent(Context context) { * @param matchingFile */ protected void validateTable(Context context, TableModel tableModel, File matchingFile) { - /* - * csv file validation - */ try { TableReader reader = TableReader.createTableReader(matchingFile, context.getEncoding()); if (!reader.isCharsetValid()) { @@ -90,7 +87,6 @@ protected void validateTable(Context context, TableModel tableModel, File matchi context.createError(CoreErrorCodes.FILE_NOT_OPENED) .setMessageParam("FILEPATH", context.relativize(matchingFile)) ); - return; } } diff --git a/validator-core/src/main/java/fr/ign/validator/database/Database.java b/validator-core/src/main/java/fr/ign/validator/database/Database.java index 74011459..52131e77 100644 --- a/validator-core/src/main/java/fr/ign/validator/database/Database.java +++ b/validator-core/src/main/java/fr/ign/validator/database/Database.java @@ -22,11 +22,16 @@ import fr.ign.validator.Context; import fr.ign.validator.data.Document; import fr.ign.validator.data.DocumentFile; +import fr.ign.validator.data.file.MultiTableFile; import fr.ign.validator.data.file.TableFile; import fr.ign.validator.model.AttributeType; import fr.ign.validator.model.DocumentModel; +import fr.ign.validator.model.FeatureType; import fr.ign.validator.model.FileModel; +import fr.ign.validator.model.file.EmbeddedTableModel; +import fr.ign.validator.model.file.MultiTableModel; import fr.ign.validator.model.file.TableModel; +import fr.ign.validator.normalize.DocumentNormalizer; import fr.ign.validator.tools.TableReader; /** @@ -262,8 +267,9 @@ public void createTables(DocumentModel documentModel) throws SQLException { List fileModels = documentModel.getFileModels(); for (FileModel fileModel : fileModels) { if (fileModel instanceof TableModel) { - TableModel tableModel = (TableModel) fileModel; - createTable(tableModel); + createTable((TableModel) fileModel); + } else if (fileModel instanceof MultiTableModel) { + createTables((MultiTableModel) fileModel); } } } @@ -274,17 +280,53 @@ public void createTables(DocumentModel documentModel) throws SQLException { * @param tableModel * @throws SQLException */ - public void createTable(TableModel tableModel) throws SQLException { - log.info(MARKER, "Create table for the TableModel '{}' ...", tableModel.getName()); - List> attributes = tableModel.getFeatureType().getAttributes(); - List columns = new ArrayList(attributes.size()); + private void createTable(TableModel tableModel) throws SQLException { + log.info(MARKER, "Create table for {} ...", tableModel); - for (AttributeType attribute : attributes) { - columns.add(attribute.getName() + " TEXT"); + String tableName = tableModel.getName(); + FeatureType featureType = tableModel.getFeatureType(); + createTable(tableName, featureType); + } + + /** + * Create tables according to MultiTableModel. + * + * TODO ensure table naming consistency with {@link DocumentNormalizer} + * + * @param fileModel + */ + private void createTables(MultiTableModel fileModel) throws SQLException { + log.info(MARKER, "Create tables for {} ...", fileModel); + for (EmbeddedTableModel tableModel : fileModel.getTableModels()) { + String tableName = fileModel.getName() + "_" + tableModel.getName(); + log.info( + MARKER, "Create table {} for {} in {} ...", + tableName, + tableModel, + fileModel + ); + createTable(tableName, tableModel.getFeatureType()); } + } - String sql = "CREATE TABLE " + tableModel.getName() + " (" + String.join(",", columns) + ");"; + /** + * Create table for a given FeatureType. + * + * @param tableName + * @param featureType + * @throws SQLException + */ + private void createTable(String tableName, FeatureType featureType) throws SQLException { + if (featureType.isEmpty()) { + log.warn(MARKER, "Create table {} skipped (empty FeatureType)", tableName); + return; + } + List columns = new ArrayList<>(featureType.getAttributeCount()); + for (AttributeType attribute : featureType.getAttributes()) { + columns.add(attribute.getName() + " TEXT"); + } + String sql = "CREATE TABLE " + tableName + " (" + String.join(",", columns) + ");"; update(sql); connection.commit(); } @@ -313,29 +355,44 @@ public void createTable(String tableName, List columnNames) throws SQLEx /** * Create indexes for all tables on unique fields * + * TODO add support for {@link MultiTableModel} + * * @param document * @throws SQLException */ public void createIndexes(DocumentModel documentModel) throws SQLException { + log.info(MARKER, "Create indexes for {} ...", documentModel); for (FileModel file : documentModel.getFileModels()) { - if (!(file instanceof TableModel)) { - continue; + if (file instanceof TableModel) { + createIndexes((TableModel) file); + } else if (file instanceof MultiTableModel) { + log.warn(MARKER, "Create indexes for {} is not yet supported!", file); } - List> attributes = file.getFeatureType().getAttributes(); - /* - * create indexes according to constraints - */ - for (AttributeType attributeType : attributes) { - // create index for unique values - if (attributeType.getConstraints().isUnique()) { - createIndex(file.getName(), attributeType.getName()); - } - // create index for referenced values - if (!StringUtils.isEmpty(attributeType.getConstraints().getReference())) { - String targetTableName = attributeType.getTableReference(); - String targetColumnName = attributeType.getAttributeReference(); - createIndex(targetTableName, targetColumnName); - } + } + } + + /** + * Create indexes for a given table. + * + * @param tableModel + * @throws SQLException + */ + public void createIndexes(TableModel tableModel) throws SQLException { + log.info(MARKER, "Create indexes for {} ...", tableModel); + List> attributes = tableModel.getFeatureType().getAttributes(); + /* + * create indexes according to constraints + */ + for (AttributeType attributeType : attributes) { + // create index for unique values + if (attributeType.getConstraints().isUnique()) { + createIndex(tableModel.getName(), attributeType.getName()); + } + // create index for referenced values + if (!StringUtils.isEmpty(attributeType.getConstraints().getReference())) { + String targetTableName = attributeType.getTableReference(); + String targetColumnName = attributeType.getAttributeReference(); + createIndex(targetTableName, targetColumnName); } } } @@ -347,6 +404,7 @@ public void createIndexes(DocumentModel documentModel) throws SQLException { * @param columnName */ public void createIndex(String tableName, String columnName) throws SQLException { + log.info(MARKER, "Create index on {}.{} ...", tableName, columnName); String indexName = "idx_" + tableName + "_" + columnName; String sql = "CREATE INDEX IF NOT EXISTS " + indexName + " ON " + tableName + " (" + columnName + ")"; @@ -357,15 +415,18 @@ public void createIndex(String tableName, String columnName) throws SQLException /** * Load an entire document to the database (insert mode) * + * TODO add support for {@link MultiTableFile} + * * @throws IOException * @throws SQLException */ public void load(Context context, Document document) throws IOException, SQLException { - log.info(MARKER, "Load document '{}'...", document.getDocumentPath()); - List files = document.getDocumentFiles(); - for (DocumentFile file : files) { - if (file instanceof TableFile) { - load(context, file); + log.info(MARKER, "Loading data from document '{}'...", document.getDocumentPath()); + for (DocumentFile documentFile : document.getDocumentFiles()) { + if (documentFile instanceof TableFile) { + load(context, (TableFile) documentFile); + } else if (documentFile instanceof MultiTableFile) { + log.warn(MARKER, "Loading data from {} is not yet supported!", documentFile); } } } @@ -378,9 +439,9 @@ public void load(Context context, Document document) throws IOException, SQLExce * @throws IOException * @throws SQLException */ - public void load(Context context, DocumentFile documentFile) throws IOException, SQLException { - FileModel fileModel = documentFile.getFileModel(); - loadFile(fileModel.getName(), documentFile.getPath(), context.getEncoding()); + public void load(Context context, TableFile tableFile) throws IOException, SQLException { + FileModel fileModel = tableFile.getFileModel(); + loadFile(fileModel.getName(), tableFile.getPath(), context.getEncoding()); } /** @@ -393,7 +454,7 @@ public void load(Context context, DocumentFile documentFile) throws IOException, * @throws SQLException */ public void loadFile(String tableName, File path, Charset charset) throws IOException, SQLException { - log.info(MARKER, "loadFile({},{},{})...", tableName, path.getAbsolutePath(), charset.toString()); + log.info(MARKER, "Load table '{}' from '{}' (charset={})...", tableName, path.getAbsolutePath(), charset); /* * Create table reader */ @@ -455,6 +516,15 @@ public void loadFile(String tableName, File path, Charset charset) throws IOExce } sth.executeBatch(); connection.commit(); + + log.info( + MARKER, + "Load table '{}' from '{}' (charset={}) : completed, {} row(s) loaded", + tableName, + path.getAbsolutePath(), + charset, + count + ); } /** diff --git a/validator-core/src/main/java/fr/ign/validator/io/AbstractModelReader.java b/validator-core/src/main/java/fr/ign/validator/io/AbstractModelReader.java index 33bbcb1a..652605af 100644 --- a/validator-core/src/main/java/fr/ign/validator/io/AbstractModelReader.java +++ b/validator-core/src/main/java/fr/ign/validator/io/AbstractModelReader.java @@ -10,7 +10,7 @@ import fr.ign.validator.exception.ModelNotFoundException; import fr.ign.validator.model.DocumentModel; import fr.ign.validator.model.FeatureType; -import fr.ign.validator.model.FileModel; +import fr.ign.validator.model.file.TableModel; /** * Common implementation for JSON and XML ModelReader. @@ -40,28 +40,28 @@ public FeatureType loadFeatureType(File path) throws ModelNotFoundException, Inv /** * Resolve FeatureType URL for a given FileModel * - * TODO support explicit featureType reference in FileModel + * TODO support explicit featureType reference in TableModel * * @param documentModelUrl * @param documentModel - * @param documentFile + * @param tableModel * @return * @throws MalformedURLException */ - protected URL resolveFeatureTypeUrl(URL documentModelUrl, DocumentModel documentModel, FileModel documentFile) + protected URL resolveFeatureTypeUrl(URL documentModelUrl, DocumentModel documentModel, TableModel tableModel) throws MalformedURLException { String parentUrl = getParentURL(documentModelUrl); if (documentModelUrl.getProtocol().equals("file")) { /* config export convention */ // validator-config-cnig/config/cnig_PLU_2017/files.xml // validator-config-cnig/config/cnig_PLU_2017/types/ZONE_URBA.xml - return new URL(parentUrl + "/types/" + documentFile.getName() + "." + getFormat()); + return new URL(parentUrl + "/types/" + tableModel.getName() + "." + getFormat()); } else { /* URL convention */ // https://www.geoportail-urbanisme.gouv.fr/standard/cnig_PLU_2017.xml // https://www.geoportail-urbanisme.gouv.fr/standard/cnig_PLU_2017/types/ZONE_URBA.xml return new URL( - parentUrl + "/" + documentModel.getName() + "/types/" + documentFile.getName() + "." + getFormat() + parentUrl + "/" + documentModel.getName() + "/types/" + tableModel.getName() + "." + getFormat() ); } } diff --git a/validator-core/src/main/java/fr/ign/validator/io/JsonModelReader.java b/validator-core/src/main/java/fr/ign/validator/io/JsonModelReader.java index 119c9a2e..102eeaff 100644 --- a/validator-core/src/main/java/fr/ign/validator/io/JsonModelReader.java +++ b/validator-core/src/main/java/fr/ign/validator/io/JsonModelReader.java @@ -17,8 +17,15 @@ import fr.ign.validator.model.DocumentModel; import fr.ign.validator.model.FeatureType; import fr.ign.validator.model.FileModel; +import fr.ign.validator.model.file.MultiTableModel; import fr.ign.validator.model.file.TableModel; +/** + * Helper class to load {@link DocumentModel} and {@link FeatureType} from JSON. + * + * @author MBorne + * + */ public class JsonModelReader extends AbstractModelReader { private static final Logger log = LogManager.getRootLogger(); private static final Marker MARKER = MarkerManager.getMarker("JsonModelReader"); @@ -41,17 +48,20 @@ public DocumentModel loadDocumentModel(URL documentModelUrl) throws ModelNotFoun InputStream is = getInputStream(documentModelUrl); try { DocumentModel documentModel = objectMapper.readValue(is, DocumentModel.class); - /* - * load feature types for TableModel - */ - for (FileModel documentFile : documentModel.getFileModels()) { - if (!(documentFile instanceof TableModel)) { - continue; + log.info(MARKER, "Loading FeatureTypes for {} ...", documentModel); + for (FileModel fileModel : documentModel.getFileModels()) { + if (fileModel instanceof TableModel) { + log.info(MARKER, "Loading FeatureType for {} ...", fileModel); + TableModel tableModel = (TableModel) fileModel; + URL featureTypeUrl = resolveFeatureTypeUrl(documentModelUrl, documentModel, tableModel); + FeatureType featureType = loadFeatureType(featureTypeUrl); + tableModel.setFeatureType(featureType); + } else if (fileModel instanceof MultiTableModel) { + log.warn(MARKER, "Loading MultiTableModel for {} is not yet supported!", fileModel); + // TODO allows to specify FeatureType for each table in MultiTableModel } - URL featureTypeUrl = resolveFeatureTypeUrl(documentModelUrl, documentModel, documentFile); - FeatureType featureType = loadFeatureType(featureTypeUrl); - documentFile.setFeatureType(featureType); } + log.info(MARKER, "Loading FeatureTypes for {} : completed.", documentModel); return documentModel; } catch (IOException e) { String message = String.format("Fail to parse DocumentModel : %1s : %2s", documentModelUrl, e.getMessage()); diff --git a/validator-core/src/main/java/fr/ign/validator/io/XmlModelReader.java b/validator-core/src/main/java/fr/ign/validator/io/XmlModelReader.java index 2e9cb879..5dec11a9 100644 --- a/validator-core/src/main/java/fr/ign/validator/io/XmlModelReader.java +++ b/validator-core/src/main/java/fr/ign/validator/io/XmlModelReader.java @@ -33,6 +33,7 @@ public class XmlModelReader extends AbstractModelReader { private Unmarshaller unmarshaller; public XmlModelReader() { + log.trace(MARKER, "Initializing XmlModelReader..."); try { this.context = JAXBContext.newInstance(FeatureType.class, DocumentModel.class); this.unmarshaller = context.createUnmarshaller(); @@ -48,7 +49,7 @@ public String getFormat() { @Override public DocumentModel loadDocumentModel(URL documentModelUrl) { - log.info(MARKER, "loadDocumentModel({}) ...", documentModelUrl); + log.info(MARKER, "Loading DocumentModel from {} ...", documentModelUrl); /* * loading documentModel @@ -59,13 +60,13 @@ public DocumentModel loadDocumentModel(URL documentModelUrl) { /* * load feature types for TableModel */ - for (FileModel documentFile : documentModel.getFileModels()) { - if (!(documentFile instanceof TableModel)) { - continue; + for (FileModel fileModel : documentModel.getFileModels()) { + if (fileModel instanceof TableModel) { + TableModel tableModel = (TableModel) fileModel; + URL featureTypeUrl = resolveFeatureTypeUrl(documentModelUrl, documentModel, tableModel); + FeatureType featureType = loadFeatureType(featureTypeUrl); + tableModel.setFeatureType(featureType); } - URL featureTypeUrl = resolveFeatureTypeUrl(documentModelUrl, documentModel, documentFile); - FeatureType featureType = loadFeatureType(featureTypeUrl); - documentFile.setFeatureType(featureType); } return documentModel; } catch (JAXBException e) { diff --git a/validator-core/src/main/java/fr/ign/validator/model/FeatureType.java b/validator-core/src/main/java/fr/ign/validator/model/FeatureType.java index 84b3b159..9e932d94 100644 --- a/validator-core/src/main/java/fr/ign/validator/model/FeatureType.java +++ b/validator-core/src/main/java/fr/ign/validator/model/FeatureType.java @@ -145,6 +145,13 @@ public void addAttribute(AttributeType attribute) { this.attributes.add(attribute); } + /** + * True if no {@link FeatureType} is empty. + */ + public boolean isEmpty() { + return getAttributeCount() == 0; + } + /** * Gets number of attributes (manages inheritance) * diff --git a/validator-core/src/main/java/fr/ign/validator/model/FileModel.java b/validator-core/src/main/java/fr/ign/validator/model/FileModel.java index 4125363a..d8ec2c09 100644 --- a/validator-core/src/main/java/fr/ign/validator/model/FileModel.java +++ b/validator-core/src/main/java/fr/ign/validator/model/FileModel.java @@ -9,7 +9,6 @@ import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlEnum; -import javax.xml.bind.annotation.XmlTransient; import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; import org.apache.logging.log4j.LogManager; @@ -86,11 +85,6 @@ public enum MandatoryMode { */ private MandatoryMode mandatory = MandatoryMode.WARN; - /** - * Table model (optional, for tables only) - */ - private FeatureType featureType = null; - /** * XSD schema (optional, for XML/GML files only) * @@ -148,16 +142,6 @@ public void setMandatory(MandatoryMode mandatory) { this.mandatory = mandatory; } - @JsonIgnore - public FeatureType getFeatureType() { - return featureType; - } - - @XmlTransient - public void setFeatureType(FeatureType featureType) { - this.featureType = featureType; - } - public URL getXsdSchema() { return xsdSchema; } diff --git a/validator-core/src/main/java/fr/ign/validator/model/file/EmbeddedTableModel.java b/validator-core/src/main/java/fr/ign/validator/model/file/EmbeddedTableModel.java new file mode 100644 index 00000000..cb8243e5 --- /dev/null +++ b/validator-core/src/main/java/fr/ign/validator/model/file/EmbeddedTableModel.java @@ -0,0 +1,56 @@ +package fr.ign.validator.model.file; + +import javax.xml.bind.annotation.XmlTransient; + +import com.fasterxml.jackson.annotation.JsonIgnore; + +import fr.ign.validator.model.FeatureType; +import fr.ign.validator.model.Model; + +/** + * A table in a {@link MultiTableModel} + * + * @author MBorne + * + */ +public class EmbeddedTableModel implements Model { + + /** + * Nom of the table. + */ + private String name; + + /** + * Table model + */ + private FeatureType featureType; + + public EmbeddedTableModel() { + // nothing to initialiaze + } + + @Override + public String getName() { + return name; + } + + @XmlTransient + public void setName(String name) { + this.name = name; + } + + @JsonIgnore + public FeatureType getFeatureType() { + return featureType; + } + + public void setFeatureType(FeatureType featureType) { + this.featureType = featureType; + } + + @Override + public String toString() { + return name + " (" + getClass().getSimpleName() + ")"; + } + +} diff --git a/validator-core/src/main/java/fr/ign/validator/model/file/MultiTableModel.java b/validator-core/src/main/java/fr/ign/validator/model/file/MultiTableModel.java index 9d0b672c..94c91d75 100644 --- a/validator-core/src/main/java/fr/ign/validator/model/file/MultiTableModel.java +++ b/validator-core/src/main/java/fr/ign/validator/model/file/MultiTableModel.java @@ -1,6 +1,13 @@ package fr.ign.validator.model.file; import java.io.File; +import java.util.ArrayList; +import java.util.List; + +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlElementWrapper; + +import com.fasterxml.jackson.annotation.JsonProperty; import fr.ign.validator.data.DocumentFile; import fr.ign.validator.data.file.MultiTableFile; @@ -16,6 +23,8 @@ public class MultiTableModel extends FileModel { public static final String TYPE = "multi_table"; + private List tableModels = new ArrayList<>(); + public MultiTableModel() { super(); } @@ -35,4 +44,30 @@ public DocumentFile createDocumentFile(File path) { return new MultiTableFile(this, path); } + @XmlElementWrapper(name = "tables") + @XmlElement(name = "table") + @JsonProperty("tables") + public List getTableModels() { + return tableModels; + } + + /** + * Find table model by name. + * + * @param name + * @return + */ + public EmbeddedTableModel getTableModelByName(String name) { + for (EmbeddedTableModel tableModel : tableModels) { + if (tableModel.getName().equalsIgnoreCase(name)) { + return tableModel; + } + } + return null; + } + + public void setTableModels(List tableModels) { + this.tableModels = tableModels; + } + } diff --git a/validator-core/src/main/java/fr/ign/validator/model/file/TableModel.java b/validator-core/src/main/java/fr/ign/validator/model/file/TableModel.java index 22898992..8a65bdde 100644 --- a/validator-core/src/main/java/fr/ign/validator/model/file/TableModel.java +++ b/validator-core/src/main/java/fr/ign/validator/model/file/TableModel.java @@ -2,8 +2,13 @@ import java.io.File; +import javax.xml.bind.annotation.XmlTransient; + +import com.fasterxml.jackson.annotation.JsonIgnore; + import fr.ign.validator.data.DocumentFile; import fr.ign.validator.data.file.TableFile; +import fr.ign.validator.model.FeatureType; import fr.ign.validator.model.FileModel; /** @@ -15,6 +20,11 @@ public class TableModel extends FileModel { public static final String TYPE = "table"; + /** + * Table model (optional, for tables only) + */ + private FeatureType featureType = null; + public TableModel() { super(); } @@ -24,6 +34,16 @@ public String getType() { return TYPE; } + @JsonIgnore + public FeatureType getFeatureType() { + return featureType; + } + + @XmlTransient + public void setFeatureType(FeatureType featureType) { + this.featureType = featureType; + } + @Override public String getRegexpSuffix() { return "\\.(dbf|DBF|tab|TAB|gml|GML|csv|CSV)"; diff --git a/validator-core/src/main/java/fr/ign/validator/normalize/DocumentNormalizer.java b/validator-core/src/main/java/fr/ign/validator/normalize/DocumentNormalizer.java index 9f39c688..3d57d524 100644 --- a/validator-core/src/main/java/fr/ign/validator/normalize/DocumentNormalizer.java +++ b/validator-core/src/main/java/fr/ign/validator/normalize/DocumentNormalizer.java @@ -15,11 +15,11 @@ import fr.ign.validator.data.DocumentFile; import fr.ign.validator.model.FeatureType; import fr.ign.validator.model.FileModel; +import fr.ign.validator.model.file.EmbeddedTableModel; import fr.ign.validator.model.file.MetadataModel; import fr.ign.validator.model.file.MultiTableModel; import fr.ign.validator.model.file.PdfModel; import fr.ign.validator.model.file.TableModel; -import fr.ign.validator.tools.AutoFeatureType; import fr.ign.validator.tools.MultiTableReader; /** @@ -102,7 +102,6 @@ private void normalizeTable(Context context, TableModel fileModel, List documentFiles) throws IOException { - if (documentFiles.isEmpty() && documentFiles.size() > 1) { + if (documentFiles.size() != 1) { log.warn( - MARKER, "{} - skipped (found {} files, normalization not supported for MultiTable)", - fileModel.getName(), + MARKER, "Skip normalization for {} (found {} files, normalization supported only for one DocumentFile)", + fileModel, documentFiles.size() ); return; } + File outputDir = new File(context.getDataDirectory(), fileModel.getName()); + if (!outputDir.exists()) { + outputDir.mkdirs(); + } + DocumentFile documentFile = documentFiles.get(0); MultiTableReader reader = MultiTableReader.createMultiTableReader(documentFile.getPath()); for (String tableName : reader.getTableNames()) { @@ -142,25 +146,27 @@ private void normalizeMultiTable(Context context, MultiTableModel fileModel, Lis * Retrieve source path for CSV converted table. */ File sourceFile = reader.getTablePath(tableName); - /* - * Detected FeatureType from CSV. - * - * TODO allow user to provide featureTypes. - */ - FeatureType featureType = AutoFeatureType.createFeatureTypeFromTable(sourceFile); - - /* - * Prepare output directory for the FileModel DATA/{fileModel.name} - */ - File outputDir = new File(context.getDataDirectory(), fileModel.getName()); - if (!outputDir.exists()) { - outputDir.mkdirs(); + EmbeddedTableModel tableModel = fileModel.getTableModelByName(tableName); + if (tableModel == null) { + log.warn( + MARKER, "Skip table {} in '{}' (table model not found in {})", + tableName, + documentFile.getPath(), + fileModel + ); + continue; } + /* - * Create normalized CSV file. + * + * /* Create normalized CSV file. */ File outputFile = new File(outputDir, tableName + ".csv"); - TableNormalizer normalizer = new TableNormalizer(context, featureType, outputFile); + TableNormalizer normalizer = new TableNormalizer( + context, + tableModel.getFeatureType(), + outputFile + ); normalizer.append(sourceFile); normalizer.close(); } diff --git a/validator-core/src/main/java/fr/ign/validator/process/CheckFeatureTypesPreProcess.java b/validator-core/src/main/java/fr/ign/validator/process/CheckFeatureTypesPreProcess.java new file mode 100644 index 00000000..640e96c5 --- /dev/null +++ b/validator-core/src/main/java/fr/ign/validator/process/CheckFeatureTypesPreProcess.java @@ -0,0 +1,231 @@ +package fr.ign.validator.process; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.MarkerManager; + +import fr.ign.validator.Context; +import fr.ign.validator.ValidatorListener; +import fr.ign.validator.data.Document; +import fr.ign.validator.data.DocumentFile; +import fr.ign.validator.model.DocumentModel; +import fr.ign.validator.model.FeatureType; +import fr.ign.validator.model.FileModel; +import fr.ign.validator.model.file.EmbeddedTableModel; +import fr.ign.validator.model.file.MultiTableModel; +import fr.ign.validator.model.file.TableModel; +import fr.ign.validator.tools.AutoFeatureType; +import fr.ign.validator.tools.MultiTableReader; + +/** + * Ensure that FeatureType are defined for each {@link TableModel} and + * {@link EmbeddedTableModel} in {@link MultiTableModel}. + * + * If not, a FeatureType is determined from data in order to allow table + * validation of some aspects (geometry,...) without providing a complete model. + * + * @author MBorne + * + */ +public class CheckFeatureTypesPreProcess implements ValidatorListener { + + public static final Logger log = LogManager.getRootLogger(); + private static final Marker MARKER = MarkerManager.getMarker("CheckFeatureTypesPreProcess"); + + @Override + public void beforeMatching(Context context, Document document) throws Exception { + // nothing to (data required) + } + + @Override + public void beforeValidate(Context context, Document document) throws Exception { + process(context, document, document.getDocumentModel()); + } + + @Override + public void afterValidate(Context context, Document document) throws Exception { + // nothing to (data required) + } + + /** + * Process DocumentModel components. + * + * @param document + * @throws IOException + */ + private void process(Context context, Document document, DocumentModel documentModel) throws IOException { + log.info(MARKER, "Process {}...", documentModel); + context.beginModel(documentModel); + for (FileModel fileModel : documentModel.getFileModels()) { + if (fileModel instanceof TableModel) { + process(context, document, (TableModel) fileModel); + } else if (fileModel instanceof MultiTableModel) { + process(context, document, (MultiTableModel) fileModel); + } + } + context.endModel(documentModel); + log.info(MARKER, "Process {} : completed", documentModel); + } + + /** + * Process FeatureType associated to TableModel. + * + * @param context + * @param document + * @param tableModel + * @throws IOException + */ + private void process(Context context, Document document, TableModel tableModel) throws IOException { + log.info(MARKER, "Process {}...", tableModel); + + if (tableModel.getFeatureType() != null) { + log.info( + MARKER, "Process {} : FeatureType is already defined.", + tableModel + ); + return; + } + + DocumentFile documentFile = getFirstDocumentFile(document, tableModel); + if (documentFile == null) { + log.warn( + MARKER, + "Process {} : No DocumentFile found to detect FeatureType, create empty FeatureType.", + tableModel + ); + tableModel.setFeatureType(createEmptyFeatureType(tableModel.getName())); + return; + } + + FeatureType featureType = AutoFeatureType.createFeatureTypeFromTable(documentFile.getPath()); + featureType.setName(tableModel.getName()); + log.info( + MARKER, "Process {} : FeatureType detected from {}", + tableModel, + documentFile.getPath() + ); + } + + private void process(Context context, Document document, MultiTableModel multiTableModel) throws IOException { + log.info(MARKER, "Process {}...", multiTableModel); + if (haveAllFeatureFeatureTypeDefined(multiTableModel)) { + log.info( + MARKER, "Process {} : completed (all FeatureType are already defined).", + multiTableModel + ); + return; + } + + DocumentFile documentFile = getFirstDocumentFile(document, multiTableModel); + + /* + * create empty FeatureType when no DocumentFile is available. + */ + if (documentFile == null) { + for (EmbeddedTableModel tableModel : multiTableModel.getTableModels()) { + if (tableModel.getFeatureType() != null) { + continue; + } + log.warn( + MARKER, + "Process {} : No DocumentFile found to detect FeatureType, create empty FeatureType for {}", + multiTableModel, + tableModel + ); + tableModel.setFeatureType(createEmptyFeatureType(tableModel.getName())); + } + return; + } + + log.info( + MARKER, + "Process {} : Create missing FeatureTypes from {}...", + multiTableModel, + documentFile.getPath() + ); + MultiTableReader reader = MultiTableReader.createMultiTableReader(documentFile.getPath()); + for (EmbeddedTableModel tableModel : multiTableModel.getTableModels()) { + if (tableModel.getFeatureType() != null) { + continue; + } + File path = reader.getTablePath(tableModel.getName()); + if (!path.exists()) { + log.warn( + MARKER, "Process {} : Create empty FeatureType for {} ('{}' not found)", + tableModel, + tableModel.getName(), + path + ); + tableModel.setFeatureType(createEmptyFeatureType(tableModel.getName())); + continue; + } + + FeatureType featureType = AutoFeatureType.createFeatureTypeFromTable(path); + featureType.setName(tableModel.getName()); + log.info( + MARKER, "Process {} : FeatureType create for {} from {}", + tableModel, + tableModel.getName(), + documentFile.getPath() + ); + tableModel.setFeatureType(featureType); + } + } + + /** + * Get first {@link DocumentFile} matched to {@link FileModel} + * + * @param document + * @param fileModel + * @return + */ + private DocumentFile getFirstDocumentFile(Document document, FileModel fileModel) { + List documentFiles = document.getDocumentFilesByModel(fileModel); + if (documentFiles.isEmpty()) { + return null; + } + if (documentFiles.size() > 1) { + log.warn( + MARKER, + "Process {} : found {} document files, using first one to detect FeatureType", + fileModel, + documentFiles.size() + ); + } + return documentFiles.get(0); + } + + /** + * False if {@link MultiTableModel} as at least one missing {@link FeatureType} + * definition. + * + * @param multiTableModel + * @return + */ + private boolean haveAllFeatureFeatureTypeDefined(MultiTableModel multiTableModel) { + for (EmbeddedTableModel tableModel : multiTableModel.getTableModels()) { + if (tableModel.getFeatureType() == null) { + return false; + } + } + return true; + } + + /** + * Create empty FeatureType to avoid risk of crash during runtime. + * + * @param name + * @return + */ + private FeatureType createEmptyFeatureType(String name) { + FeatureType featureType = new FeatureType(); + featureType.setName(name); + return featureType; + } + +} diff --git a/validator-core/src/main/java/fr/ign/validator/tools/AutoFeatureType.java b/validator-core/src/main/java/fr/ign/validator/tools/AutoFeatureType.java index 05b3ced6..89fbfacc 100644 --- a/validator-core/src/main/java/fr/ign/validator/tools/AutoFeatureType.java +++ b/validator-core/src/main/java/fr/ign/validator/tools/AutoFeatureType.java @@ -5,6 +5,10 @@ import java.nio.charset.StandardCharsets; import org.apache.commons.io.FilenameUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.MarkerManager; import fr.ign.validator.model.AttributeType; import fr.ign.validator.model.FeatureType; @@ -20,6 +24,9 @@ */ public class AutoFeatureType { + public static final Logger log = LogManager.getRootLogger(); + private static final Marker MARKER = MarkerManager.getMarker("AutoFeatureType"); + private AutoFeatureType() { // helper class grouping static helpers } @@ -37,6 +44,7 @@ private AutoFeatureType() { * @throws IOException */ public static FeatureType createFeatureTypeFromTable(File path) throws IOException { + log.info(MARKER, "Create FeatureType reading data from {}", path); FeatureType result = new FeatureType(); result.setName(FilenameUtils.getBaseName(path.getName())); diff --git a/validator-core/src/main/java/fr/ign/validator/tools/MultiTableReader.java b/validator-core/src/main/java/fr/ign/validator/tools/MultiTableReader.java index 9a28cccb..8ed66f58 100644 --- a/validator-core/src/main/java/fr/ign/validator/tools/MultiTableReader.java +++ b/validator-core/src/main/java/fr/ign/validator/tools/MultiTableReader.java @@ -98,7 +98,7 @@ public File getTablePath(String tableName) { * @throws IOException */ public static MultiTableReader createMultiTableReader(File file) throws IOException { - log.info( + log.debug( MARKER, "Create MultiTableReader for '{}' (charset={})...", file.getAbsoluteFile(), StandardCharsets.UTF_8 diff --git a/validator-core/src/main/java/fr/ign/validator/tools/TableReader.java b/validator-core/src/main/java/fr/ign/validator/tools/TableReader.java index 4a8c4320..7dfafc87 100644 --- a/validator-core/src/main/java/fr/ign/validator/tools/TableReader.java +++ b/validator-core/src/main/java/fr/ign/validator/tools/TableReader.java @@ -207,7 +207,7 @@ public int findColumnRequired(String name) throws ColumnNotFoundException { * @throws IOException */ public static TableReader createTableReader(File file, Charset preferedCharset) throws IOException { - log.info( + log.debug( MARKER, "Create TableReader for '{}' (preferedCharset={})...", file.getAbsoluteFile(), preferedCharset diff --git a/validator-core/src/main/java/fr/ign/validator/validation/database/AttributeReferenceValidator.java b/validator-core/src/main/java/fr/ign/validator/validation/database/AttributeReferenceValidator.java index ce06a5e4..58e92671 100644 --- a/validator-core/src/main/java/fr/ign/validator/validation/database/AttributeReferenceValidator.java +++ b/validator-core/src/main/java/fr/ign/validator/validation/database/AttributeReferenceValidator.java @@ -18,6 +18,7 @@ import fr.ign.validator.model.AttributeType; import fr.ign.validator.model.FeatureType; import fr.ign.validator.model.FileModel; +import fr.ign.validator.model.file.MultiTableModel; import fr.ign.validator.model.file.TableModel; import fr.ign.validator.tools.EnvelopeUtils; import fr.ign.validator.validation.Validator; @@ -49,6 +50,15 @@ public void validate(Context context, Database database) { } } + /** + * + * TODO add support for {@link MultiTableModel} + * + * @param context + * @param database + * @throws SQLException + * @throws IOException + */ private void doValidate(Context context, Database database) throws SQLException, IOException { log.info(MARKER, "Looking for attributes with reference constraints..."); @@ -60,8 +70,9 @@ private void doValidate(Context context, Database database) throws SQLException, continue; } + TableModel tableModel = (TableModel) fileModel; context.beginModel(fileModel); - FeatureType featureType = fileModel.getFeatureType(); + FeatureType featureType = tableModel.getFeatureType(); for (AttributeType attribute : featureType.getAttributes()) { String reference = attribute.getConstraints().getReference(); if (StringUtils.isEmpty(reference)) { diff --git a/validator-core/src/main/java/fr/ign/validator/validation/database/AttributeUniqueValidator.java b/validator-core/src/main/java/fr/ign/validator/validation/database/AttributeUniqueValidator.java index 0e6f5a62..1fbfdbcc 100644 --- a/validator-core/src/main/java/fr/ign/validator/validation/database/AttributeUniqueValidator.java +++ b/validator-core/src/main/java/fr/ign/validator/validation/database/AttributeUniqueValidator.java @@ -17,6 +17,7 @@ import fr.ign.validator.error.ErrorScope; import fr.ign.validator.model.AttributeType; import fr.ign.validator.model.FileModel; +import fr.ign.validator.model.file.MultiTableModel; import fr.ign.validator.model.file.TableModel; import fr.ign.validator.validation.Validator; @@ -49,6 +50,15 @@ public void validate(Context context, Database database) { } } + /** + * + * TODO add support for {@link MultiTableModel} + * + * @param context + * @param database + * @throws SQLException + * @throws IOException + */ private void doValidate(Context context, Database database) throws SQLException, IOException { log.info(MARKER, "Looking for attributes with unique constraints..."); @@ -60,8 +70,9 @@ private void doValidate(Context context, Database database) throws SQLException, continue; } + TableModel tableModel = (TableModel) fileModel; context.beginModel(fileModel); - for (AttributeType attribute : fileModel.getFeatureType().getAttributes()) { + for (AttributeType attribute : tableModel.getFeatureType().getAttributes()) { if (!attribute.getConstraints().isUnique()) { continue; } diff --git a/validator-core/src/test/java/fr/ign/validator/ContextTest.java b/validator-core/src/test/java/fr/ign/validator/ContextTest.java index c97b5501..8f95a2d6 100644 --- a/validator-core/src/test/java/fr/ign/validator/ContextTest.java +++ b/validator-core/src/test/java/fr/ign/validator/ContextTest.java @@ -7,6 +7,7 @@ import org.mockito.Mockito; import fr.ign.validator.process.MetadataPreProcess; +import fr.ign.validator.process.CheckFeatureTypesPreProcess; import fr.ign.validator.process.DocumentInfoExtractorPostProcess; import fr.ign.validator.process.FilterMetadataPreProcess; import fr.ign.validator.process.NormalizePostProcess; @@ -31,14 +32,15 @@ public void testDefaultConstructor() { public void testDefaultListeners() { Context context = new Context(); List listeners = context.getValidatorListeners(); - Assert.assertEquals(6, listeners.size()); + Assert.assertEquals(7, listeners.size()); // order is important between FilterMetadataPreProcess and CharsetPreProcess Assert.assertEquals(RemovePreviousFilesPreProcess.class, listeners.get(0).getClass()); Assert.assertEquals(FilterMetadataPreProcess.class, listeners.get(1).getClass()); Assert.assertEquals(MetadataPreProcess.class, listeners.get(2).getClass()); Assert.assertEquals(ProjectionPreProcess.class, listeners.get(3).getClass()); - Assert.assertEquals(NormalizePostProcess.class, listeners.get(4).getClass()); - Assert.assertEquals(DocumentInfoExtractorPostProcess.class, listeners.get(5).getClass()); + Assert.assertEquals(CheckFeatureTypesPreProcess.class, listeners.get(4).getClass()); + Assert.assertEquals(NormalizePostProcess.class, listeners.get(5).getClass()); + Assert.assertEquals(DocumentInfoExtractorPostProcess.class, listeners.get(6).getClass()); } /** @@ -51,14 +53,15 @@ public void testAddListenerBefore() { context.addListenerBefore(fakeListener, NormalizePostProcess.class); List listeners = context.getValidatorListeners(); - Assert.assertEquals(7, listeners.size()); + Assert.assertEquals(8, listeners.size()); Assert.assertEquals(RemovePreviousFilesPreProcess.class, listeners.get(0).getClass()); // order is important between FilterMetadataPreProcess and CharsetPreProcess Assert.assertEquals(FilterMetadataPreProcess.class, listeners.get(1).getClass()); Assert.assertEquals(MetadataPreProcess.class, listeners.get(2).getClass()); Assert.assertEquals(ProjectionPreProcess.class, listeners.get(3).getClass()); - Assert.assertSame(fakeListener, listeners.get(4)); - Assert.assertEquals(NormalizePostProcess.class, listeners.get(5).getClass()); - Assert.assertEquals(DocumentInfoExtractorPostProcess.class, listeners.get(6).getClass()); + Assert.assertEquals(CheckFeatureTypesPreProcess.class, listeners.get(4).getClass()); + Assert.assertSame(fakeListener, listeners.get(5)); + Assert.assertEquals(NormalizePostProcess.class, listeners.get(6).getClass()); + Assert.assertEquals(DocumentInfoExtractorPostProcess.class, listeners.get(7).getClass()); } } diff --git a/validator-core/src/test/java/fr/ign/validator/io/JsonModelReaderTest.java b/validator-core/src/test/java/fr/ign/validator/io/JsonModelReaderTest.java index 41d3e23c..9418f755 100644 --- a/validator-core/src/test/java/fr/ign/validator/io/JsonModelReaderTest.java +++ b/validator-core/src/test/java/fr/ign/validator/io/JsonModelReaderTest.java @@ -18,6 +18,7 @@ import fr.ign.validator.model.DocumentModel; import fr.ign.validator.model.FeatureType; import fr.ign.validator.model.FileModel; +import fr.ign.validator.model.file.EmbeddedTableModel; import fr.ign.validator.model.file.MetadataModel; import fr.ign.validator.model.file.MultiTableModel; import fr.ign.validator.model.file.TableModel; @@ -91,11 +92,11 @@ public void testLoadDocumentModelAdresse() { /* perform checks on FileModel "adresse" */ FileModel fileModel = documentModel.getFileModelByName("ADRESSE"); Assert.assertNotNull(fileModel); - Assert.assertTrue(fileModel instanceof TableModel); Assert.assertEquals("ADRESSE(_+[0-9])?", fileModel.getPath()); /* perform checks on FeatureType "adresse" */ - FeatureType featureType = fileModel.getFeatureType(); + Assert.assertTrue(fileModel instanceof TableModel); + FeatureType featureType = ((TableModel) fileModel).getFeatureType(); Assert.assertNotNull(featureType); assertExceptedFeatureTypeAdresse(featureType); } @@ -185,7 +186,7 @@ public void testLoadDocumentModelPCRS() { assertTrue(fileModel instanceof MetadataModel); } - // check METADONNEES + // check DONNEES { FileModel fileModel = documentModel.getFileModelByName("DONNEES"); assertNotNull(fileModel); @@ -194,8 +195,12 @@ public void testLoadDocumentModelPCRS() { "https://cnigfr.github.io/PCRS/schemas/CNIG_PCRS_v2.0.xsd", fileModel.getXsdSchema().toString() ); - } + // check embedded tables + MultiTableModel multiTableModel = (MultiTableModel) fileModel; + List tableModels = multiTableModel.getTableModels(); + Assert.assertEquals(17, tableModels.size()); + } } /** @@ -211,7 +216,7 @@ private void assertIsValid(DocumentModel documentModel) { Assert.assertNotNull(fileModel.getName()); Assert.assertNotNull(fileModel.getMandatory()); if (fileModel instanceof TableModel) { - FeatureType featureType = fileModel.getFeatureType(); + FeatureType featureType = ((TableModel) fileModel).getFeatureType(); Assert.assertNotNull(featureType); assertIsValid(featureType); } diff --git a/validator-core/src/test/java/fr/ign/validator/io/XmlModelReaderTest.java b/validator-core/src/test/java/fr/ign/validator/io/XmlModelReaderTest.java index c246d882..291db7c8 100644 --- a/validator-core/src/test/java/fr/ign/validator/io/XmlModelReaderTest.java +++ b/validator-core/src/test/java/fr/ign/validator/io/XmlModelReaderTest.java @@ -121,7 +121,8 @@ public void testLoadDocumentModelAdresse() { FileModel fileModel = documentModel.getFileModels().get(0); Assert.assertEquals("ADRESSE", fileModel.getName()); - FeatureType featureType = fileModel.getFeatureType(); + Assert.assertTrue(fileModel instanceof TableModel); + FeatureType featureType = ((TableModel) fileModel).getFeatureType(); Assert.assertNotNull(featureType); Assert.assertEquals(2, featureType.getAttributeCount()); @@ -166,23 +167,27 @@ public void testLoadDocumentModelSampleDocument() { { FileModel fileModel = documentModel.getFileModels().get(index++); Assert.assertEquals("SIMPLE", fileModel.getName()); - Assert.assertNotNull(fileModel.getFeatureType()); + Assert.assertTrue(fileModel instanceof TableModel); + FeatureType featureType = ((TableModel) fileModel).getFeatureType(); + Assert.assertNotNull(featureType); Assert.assertEquals(MandatoryMode.WARN, fileModel.getMandatory()); - Assert.assertEquals("SIMPLE", fileModel.getFeatureType().getName()); + Assert.assertEquals("SIMPLE", featureType.getName()); } { FileModel fileModel = documentModel.getFileModels().get(index++); Assert.assertEquals("Donnees_geographiques", fileModel.getName()); Assert.assertEquals(MandatoryMode.ERROR, fileModel.getMandatory()); - Assert.assertNull(fileModel.getFeatureType()); } { FileModel fileModel = documentModel.getFileModels().get(index++); Assert.assertEquals("COMMUNE", fileModel.getName()); Assert.assertEquals(MandatoryMode.WARN, fileModel.getMandatory()); - Assert.assertNotNull(fileModel.getFeatureType()); - Assert.assertEquals("COMMUNE", fileModel.getFeatureType().getName()); - assertExceptedFeatureTypeCommune(fileModel.getFeatureType()); + + Assert.assertTrue(fileModel instanceof TableModel); + FeatureType featureType = ((TableModel) fileModel).getFeatureType(); + Assert.assertNotNull(featureType); + Assert.assertEquals("COMMUNE", featureType.getName()); + assertExceptedFeatureTypeCommune(featureType); } } @@ -290,7 +295,7 @@ private void assertIsValid(DocumentModel documentModel) { Assert.assertNotNull(fileModel.getName()); Assert.assertNotNull(fileModel.getMandatory()); if (fileModel instanceof TableModel) { - FeatureType featureType = fileModel.getFeatureType(); + FeatureType featureType = ((TableModel) fileModel).getFeatureType(); Assert.assertNotNull(featureType); assertIsValid(featureType); } diff --git a/validator-core/src/test/java/fr/ign/validator/normalize/TableNormalizerTest.java b/validator-core/src/test/java/fr/ign/validator/normalize/TableNormalizerTest.java index 94cbbc8f..0cb8b6fb 100644 --- a/validator-core/src/test/java/fr/ign/validator/normalize/TableNormalizerTest.java +++ b/validator-core/src/test/java/fr/ign/validator/normalize/TableNormalizerTest.java @@ -15,8 +15,10 @@ import fr.ign.validator.io.ModelReader; import fr.ign.validator.io.XmlModelReader; import fr.ign.validator.model.DocumentModel; +import fr.ign.validator.model.FeatureType; import fr.ign.validator.model.FileModel; import fr.ign.validator.model.Projection; +import fr.ign.validator.model.file.TableModel; import fr.ign.validator.report.InMemoryReportBuilder; import fr.ign.validator.tools.ResourceHelper; @@ -57,9 +59,11 @@ public void setUp() throws Exception { public void testNormalise() throws Exception { FileModel fileModel = document.getDocumentModel().getFileModelByName("ADRESSE"); Assert.assertNotNull(fileModel); + Assert.assertTrue(fileModel instanceof TableModel); + FeatureType featureType = ((TableModel) fileModel).getFeatureType(); File targetFile = new File(document.getDocumentPath(), "adresse_normalized.csv"); - TableNormalizer csvNormalizer = new TableNormalizer(context, fileModel.getFeatureType(), targetFile); + TableNormalizer csvNormalizer = new TableNormalizer(context, featureType, targetFile); File csvFile1 = new File(document.getDocumentPath(), "adresse_1.csv"); Assert.assertTrue(csvFile1.exists()); diff --git a/validator-core/src/test/java/fr/ign/validator/regress/ValidatePCRSRegressTest.java b/validator-core/src/test/java/fr/ign/validator/regress/ValidatePCRSRegressTest.java index 1315a811..078f6bd4 100644 --- a/validator-core/src/test/java/fr/ign/validator/regress/ValidatePCRSRegressTest.java +++ b/validator-core/src/test/java/fr/ign/validator/regress/ValidatePCRSRegressTest.java @@ -42,6 +42,8 @@ public void setUp() { /* create report */ report = new InMemoryReportBuilder(); context.setReportBuilder(report); + context.setNormalizeEnabled(true); + /* load document model */ { JsonModelReader modelReader = new JsonModelReader(); @@ -62,7 +64,6 @@ public void testLyon01() throws Exception { File validationDirectory = new File(documentPath.getParentFile(), "validation"); validationDirectory.mkdirs(); context.setValidationDirectory(validationDirectory); - context.setReportBuilder(report); document.validate(context); diff --git a/validator-core/src/test/java/fr/ign/validator/validation/database/AttributeUniqueValidatorTest.java b/validator-core/src/test/java/fr/ign/validator/validation/database/AttributeUniqueValidatorTest.java index 04bb8026..b90451de 100644 --- a/validator-core/src/test/java/fr/ign/validator/validation/database/AttributeUniqueValidatorTest.java +++ b/validator-core/src/test/java/fr/ign/validator/validation/database/AttributeUniqueValidatorTest.java @@ -65,12 +65,12 @@ public void setUp() throws NoSuchAuthorityCodeException, FactoryException { featureType2.addAttribute(attribute); // creates a FileModel with the first Feature Type - FileModel fileModel = new TableModel(); + TableModel fileModel = new TableModel(); fileModel.setName("TEST"); fileModel.setFeatureType(featureType); // reates a FileModel with the second Feature Type - FileModel fileModel2 = new TableModel(); + TableModel fileModel2 = new TableModel(); fileModel2.setName("RELATION"); fileModel2.setFeatureType(featureType2); diff --git a/validator-core/src/test/resources/config-json/pcrs-2.0/files.json b/validator-core/src/test/resources/config-json/pcrs-2.0/files.json index c7d2e5b3..9f6684db 100644 --- a/validator-core/src/test/resources/config-json/pcrs-2.0/files.json +++ b/validator-core/src/test/resources/config-json/pcrs-2.0/files.json @@ -1,21 +1,74 @@ { - "name": "pcrs-2.0", - "title": "PCRS 2.0", + "name": "prcs-2.0", + "title": "PCRS version 2.0 -21 septembre 2017", "description": "Modèle de test pour PCRS v2.0", "files": [ - { - "name": "DONNEES", - "type": "multi_table", - "path": "[^\\/]*", - "mandatory": "ERROR", - "xsdSchema": "https://cnigfr.github.io/PCRS/schemas/CNIG_PCRS_v2.0.xsd" - }, { "title": "METADONNEES", "name": "METADONNEES", "path": "[^\\/]*", "type": "metadata", "mandatory": "OPTIONAL" + }, + { + "name": "DONNEES", + "type": "multi_table", + "path": "[^\\/]*", + "mandatory": "ERROR", + "xsdSchema": "https://cnigfr.github.io/PCRS/schemas/CNIG_PCRS_v2.0.xsd", + "tables": [ + { + "name": "AffleurantPCRS" + }, + { + "name": "AffleurantGeometriquePCRS" + }, + { + "name": "AffleurantEnveloppePCRS" + }, + { + "name": "AffleurantSymbolePCRS" + }, + { + "name": "AffleurantPointPCRS" + }, + { + "name": "ArbrePCRS" + }, + { + "name": "EmpriseEchangePCRS" + }, + { + "name": "FacadePCRS" + }, + { + "name": "LimiteVoiriePCRS" + }, + { + "name": "MurPCRS" + }, + { + "name": "NomVoiriePCRS" + }, + { + "name": "NumeroVoiriePCRS" + }, + { + "name": "PilierParticulierPCRS" + }, + { + "name": "ProeminenceBatiPCRS" + }, + { + "name": "RailPCRS" + }, + { + "name": "RasterPCRS" + }, + { + "name": "SeuilPCRS" + } + ] } ] } \ No newline at end of file diff --git a/validator-core/src/test/resources/config-json/pcrs-2.0/tables.json b/validator-core/src/test/resources/config-json/pcrs-2.0/tables.json new file mode 100644 index 00000000..066670ad --- /dev/null +++ b/validator-core/src/test/resources/config-json/pcrs-2.0/tables.json @@ -0,0 +1,59 @@ +[ + { + "name": "AffleurantPCRS" + }, + { + "name": "AffleurantGeometriquePCRS" + }, + { + "name": "AffleurantEnveloppePCRS" + }, + { + "name": "AffleurantSymbolePCRS" + }, + { + "name": "AffleurantPointPCRS" + }, + { + "name": "ArbrePCRS" + }, + { + "name": "EmpriseEchangePCRS" + }, + { + "name": "EmpriseEchangePCRS" + }, + { + "name": "FacadePCRS" + }, + { + "name": "LimiteVoiriePCRS" + }, + { + "name": "MurPCRS" + }, + { + "name": "NomVoiriePCRS" + }, + { + "name": "NumeroVoiriePCRS" + }, + { + "name": "PilierParticulierPCRS" + }, + { + "name": "ProeminenceBatiPCRS" + }, + { + "name": "RailPCRS" + }, + { + "name": "RasterPCRS" + }, + { + "name": "SeuilPCRS" + } +}, +{ + "name": "FacadePCRS" +] \ No newline at end of file diff --git a/validator-plugin-cnig/src/main/java/fr/ign/validator/cnig/process/CreateShapefilesPostProcess.java b/validator-plugin-cnig/src/main/java/fr/ign/validator/cnig/process/CreateShapefilesPostProcess.java index a468a865..f9ccb4d7 100644 --- a/validator-plugin-cnig/src/main/java/fr/ign/validator/cnig/process/CreateShapefilesPostProcess.java +++ b/validator-plugin-cnig/src/main/java/fr/ign/validator/cnig/process/CreateShapefilesPostProcess.java @@ -16,6 +16,7 @@ import fr.ign.validator.data.Document; import fr.ign.validator.model.FeatureType; import fr.ign.validator.model.FileModel; +import fr.ign.validator.model.file.TableModel; import fr.ign.validator.tools.FileConverter; /** @@ -56,7 +57,7 @@ public void afterValidate(Context context, Document document) throws Exception { // get FeatureType String typeName = FilenameUtils.getBaseName(csvFile.getName()); FileModel fileModel = context.getDocumentModel().getFileModelByName(typeName); - FeatureType featureType = fileModel.getFeatureType(); + FeatureType featureType = ((TableModel) fileModel).getFeatureType(); // create vrt file File vrtFile = new File(csvFile.getParent(), FilenameUtils.getBaseName(csvFile.getName()) + ".vrt"); diff --git a/validator-plugin-cnig/src/main/java/fr/ign/validator/cnig/process/CustomizeIdurbaPreProcess.java b/validator-plugin-cnig/src/main/java/fr/ign/validator/cnig/process/CustomizeIdurbaPreProcess.java index 28dbeab0..2e69c807 100644 --- a/validator-plugin-cnig/src/main/java/fr/ign/validator/cnig/process/CustomizeIdurbaPreProcess.java +++ b/validator-plugin-cnig/src/main/java/fr/ign/validator/cnig/process/CustomizeIdurbaPreProcess.java @@ -57,7 +57,7 @@ public void beforeMatching(Context context, Document document) throws Exception List fileModels = document.getDocumentModel().getFileModels(); for (FileModel fileModel : fileModels) { if (fileModel instanceof TableModel) { - FeatureType featureType = fileModel.getFeatureType(); + FeatureType featureType = ((TableModel) fileModel).getFeatureType(); AttributeType attribute = featureType.getAttribute("IDURBA"); if (attribute == null) { continue; diff --git a/validator-plugin-cnig/src/main/java/fr/ign/validator/cnig/process/SupRelationsPostProcess.java b/validator-plugin-cnig/src/main/java/fr/ign/validator/cnig/process/SupRelationsPostProcess.java index 6a578c12..72697888 100644 --- a/validator-plugin-cnig/src/main/java/fr/ign/validator/cnig/process/SupRelationsPostProcess.java +++ b/validator-plugin-cnig/src/main/java/fr/ign/validator/cnig/process/SupRelationsPostProcess.java @@ -23,6 +23,7 @@ import fr.ign.validator.model.DocumentModel; import fr.ign.validator.model.FeatureType; import fr.ign.validator.model.FileModel; +import fr.ign.validator.model.file.TableModel; import fr.ign.validator.validation.Validator; /** @@ -57,7 +58,11 @@ public void beforeMatching(Context context, Document document) throws Exception log.info(MARKER, "Disable unique constraint validation for IDGEN and IDASS"); for (FileModel fileModel : context.getDocumentModel().getFileModels()) { - FeatureType featureType = fileModel.getFeatureType(); + if (!(fileModel instanceof TableModel)) { + continue; + } + + FeatureType featureType = ((TableModel) fileModel).getFeatureType(); if (featureType == null) { continue; } diff --git a/validator-plugin-cnig/src/main/java/fr/ign/validator/cnig/validation/attribute/InseeValidator.java b/validator-plugin-cnig/src/main/java/fr/ign/validator/cnig/validation/attribute/InseeValidator.java index b97f0bf4..522afff2 100644 --- a/validator-plugin-cnig/src/main/java/fr/ign/validator/cnig/validation/attribute/InseeValidator.java +++ b/validator-plugin-cnig/src/main/java/fr/ign/validator/cnig/validation/attribute/InseeValidator.java @@ -34,7 +34,7 @@ public class InseeValidator implements Validator>, ValidatorLi @Override public void beforeMatching(Context context, Document document) throws Exception { - + // nothing to do } @Override @@ -42,7 +42,7 @@ public void beforeValidate(Context context, Document document) throws Exception List fileModels = document.getDocumentModel().getFileModels(); for (FileModel fileModel : fileModels) { if (fileModel instanceof TableModel) { - FeatureType featureType = fileModel.getFeatureType(); + FeatureType featureType = ((TableModel) fileModel).getFeatureType(); AttributeType attributeType = featureType.getAttribute(ATTRIBUTE_INSEE); if (null != attributeType && attributeType instanceof StringType) { log.info(MARKER, "Ajout de InseeValidator à {}", attributeType.getName()); @@ -54,7 +54,7 @@ public void beforeValidate(Context context, Document document) throws Exception @Override public void afterValidate(Context context, Document document) throws Exception { - + // nothing to do } @Override diff --git a/validator-plugin-cnig/src/test/java/fr/ign/validator/cnig/tools/VRTTest.java b/validator-plugin-cnig/src/test/java/fr/ign/validator/cnig/tools/VRTTest.java index 29adb3b1..46e88883 100644 --- a/validator-plugin-cnig/src/test/java/fr/ign/validator/cnig/tools/VRTTest.java +++ b/validator-plugin-cnig/src/test/java/fr/ign/validator/cnig/tools/VRTTest.java @@ -14,6 +14,8 @@ import fr.ign.validator.cnig.CnigRegressHelper; import fr.ign.validator.model.DocumentModel; import fr.ign.validator.model.FeatureType; +import fr.ign.validator.model.FileModel; +import fr.ign.validator.model.file.TableModel; import fr.ign.validator.tools.ResourceHelper; /** @@ -40,7 +42,9 @@ public void testDocUrba() throws Exception { // Retrieve FeatureType DocumentModel documentModel = CnigRegressHelper.getDocumentModel("cnig_PLU_2017"); - FeatureType featureType = documentModel.getFileModelByName("DOC_URBA").getFeatureType(); + FileModel fileModel = documentModel.getFileModelByName("DOC_URBA"); + assertTrue(fileModel instanceof TableModel); + FeatureType featureType = ((TableModel) fileModel).getFeatureType(); File vrtFile = VRT.createVRTfile(csvFile, featureType); assertTrue(vrtFile.exists()); diff --git a/validator-plugin-dgpr/src/main/java/fr/ign/validator/dgpr/process/CreateTheGeomColumn.java b/validator-plugin-dgpr/src/main/java/fr/ign/validator/dgpr/process/CreateTheGeomColumn.java index 3d02a3b6..f3f20088 100644 --- a/validator-plugin-dgpr/src/main/java/fr/ign/validator/dgpr/process/CreateTheGeomColumn.java +++ b/validator-plugin-dgpr/src/main/java/fr/ign/validator/dgpr/process/CreateTheGeomColumn.java @@ -53,7 +53,7 @@ public void afterValidate(Context context, Document document) throws Exception { continue; } String tableName = fileModel.getName(); - GeometryType geometryAttribute = fileModel.getFeatureType().getDefaultGeometry(); + GeometryType geometryAttribute = ((TableModel) fileModel).getFeatureType().getDefaultGeometry(); if (geometryAttribute == null) { log.info(MARKER, "skip {} (not spatial)", tableName); continue; diff --git a/validator-plugin-dgpr/src/main/java/fr/ign/validator/dgpr/validation/attribute/NumericCustomizer.java b/validator-plugin-dgpr/src/main/java/fr/ign/validator/dgpr/validation/attribute/NumericCustomizer.java index 0d8a08f6..16021182 100644 --- a/validator-plugin-dgpr/src/main/java/fr/ign/validator/dgpr/validation/attribute/NumericCustomizer.java +++ b/validator-plugin-dgpr/src/main/java/fr/ign/validator/dgpr/validation/attribute/NumericCustomizer.java @@ -28,18 +28,19 @@ public void beforeMatching(Context context, Document document) throws Exception if (!(fileModel instanceof TableModel)) { continue; } + TableModel tableModel = (TableModel) fileModel; switch (fileModel.getName()) { case "N_prefixTri_COTE_VIT_DEB_P_ddd": - addDebLinValidator(fileModel); - addAzimuthValidator(fileModel); + addDebLinValidator(tableModel); + addAzimuthValidator(tableModel); break; case "N_prefixTri_CHAMP_VIT_P_ddd": - addVitesseMinValidator(fileModel); + addVitesseMinValidator(tableModel); break; case "N_prefixTri_ISO_DEB_S_ddd": - addDebLinMinValidator(fileModel); - addDebLinMaxValidator(fileModel); + addDebLinMinValidator(tableModel); + addDebLinMaxValidator(tableModel); break; default: @@ -57,7 +58,7 @@ public void beforeValidate(Context context, Document document) throws Exception public void afterValidate(Context context, Document document) throws Exception { } - private void addDebLinMaxValidator(FileModel fileModel) { + private void addDebLinMaxValidator(TableModel fileModel) { // looking for N_prefixTri_ISO_DEB_S_ddd.DEBLIN_MAX AttributeType attribute = fileModel.getFeatureType().getAttribute("DEBLIN_MAX"); if (attribute == null) { @@ -74,7 +75,7 @@ private void addDebLinMaxValidator(FileModel fileModel) { } } - private void addDebLinMinValidator(FileModel fileModel) { + private void addDebLinMinValidator(TableModel fileModel) { // looking for N_prefixTri_ISO_DEB_S_ddd.DEBLIN_MIN or // N_prefixTri_COTE_VIT_DEB_P_ddd.DEBLIN @@ -92,7 +93,7 @@ private void addDebLinMinValidator(FileModel fileModel) { } } - private void addDebLinValidator(FileModel fileModel) { + private void addDebLinValidator(TableModel fileModel) { // looking for N_prefixTri_ISO_DEB_S_ddd.DEBLIN_MIN or // N_prefixTri_COTE_VIT_DEB_P_ddd.DEBLIN @@ -110,7 +111,7 @@ private void addDebLinValidator(FileModel fileModel) { } } - private void addAzimuthValidator(FileModel fileModel) { + private void addAzimuthValidator(TableModel fileModel) { // looking for N_prefixTri_COTE_VIT_DEB_P_ddd AttributeType attribute = fileModel.getFeatureType().getAttribute("AZIMUTH"); if (attribute == null) { @@ -126,7 +127,7 @@ private void addAzimuthValidator(FileModel fileModel) { } } - private void addVitesseMinValidator(FileModel fileModel) { + private void addVitesseMinValidator(TableModel fileModel) { // looking for N_prefixTri_COTE_VIT_DEB_P_ddd.VITESS_MIN AttributeType attribute = fileModel.getFeatureType().getAttribute("VITESS_MIN"); if (attribute == null) {