Skip to content

Commit

Permalink
add tables with a FeatureType on MultiTableModel + add a pre-process …
Browse files Browse the repository at this point in the history 
…to check and generate FeatureType according to data in order to allow geometric validation and post-process without providing table models (refs #231)
  • Loading branch information
@mborne
mborne committed Jul 9, 2021
1 parent 0999358 commit 2f1d05e
Show file tree
Hide file tree
Showing 34 changed files with 747 additions and 153 deletions.
4 changes: 4 additions & 0 deletions validator-core/src/main/java/fr/ign/validator/Context.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import fr.ign.validator.model.FileModel;
import fr.ign.validator.model.Model;
import fr.ign.validator.model.Projection;
import fr.ign.validator.process.CheckFeatureTypesPreProcess;
import fr.ign.validator.process.DocumentInfoExtractorPostProcess;
import fr.ign.validator.process.FilterMetadataPreProcess;
import fr.ign.validator.process.MetadataPreProcess;
Expand Down Expand Up @@ -208,6 +209,9 @@ private void registerDefaultListeners() {
// Info and warning about CRS
addListener(new ProjectionPreProcess());

// Check and complete FeatureType definitions
addListener(new CheckFeatureTypesPreProcess());

// generate CSV
addListener(new NormalizePostProcess());
// produce document-info.json
Expand Down
5 changes: 4 additions & 1 deletion validator-core/src/main/java/fr/ign/validator/data/file/MultiTableFile.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,12 @@ public MultiTableModel getFileModel() {
return fileModel;
}

/**
* TODO complete to validate according to {@link MultiTableModel}
*/
@Override
protected void validateContent(Context context) {
// TODO complete

}

}
4 changes: 0 additions & 4 deletions validator-core/src/main/java/fr/ign/validator/data/file/TableFile.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,6 @@ protected void validateContent(Context context) {
* @param matchingFile
*/
protected void validateTable(Context context, TableModel tableModel, File matchingFile) {
/*
* csv file validation
*/
try {
TableReader reader = TableReader.createTableReader(matchingFile, context.getEncoding());
if (!reader.isCharsetValid()) {
Expand Down Expand Up @@ -90,7 +87,6 @@ protected void validateTable(Context context, TableModel tableModel, File matchi
context.createError(CoreErrorCodes.FILE_NOT_OPENED)
.setMessageParam("FILEPATH", context.relativize(matchingFile))
);
return;
}
}

Expand Down
140 changes: 105 additions & 35 deletions validator-core/src/main/java/fr/ign/validator/database/Database.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,16 @@
import fr.ign.validator.Context;
import fr.ign.validator.data.Document;
import fr.ign.validator.data.DocumentFile;
import fr.ign.validator.data.file.MultiTableFile;
import fr.ign.validator.data.file.TableFile;
import fr.ign.validator.model.AttributeType;
import fr.ign.validator.model.DocumentModel;
import fr.ign.validator.model.FeatureType;
import fr.ign.validator.model.FileModel;
import fr.ign.validator.model.file.EmbeddedTableModel;
import fr.ign.validator.model.file.MultiTableModel;
import fr.ign.validator.model.file.TableModel;
import fr.ign.validator.normalize.DocumentNormalizer;
import fr.ign.validator.tools.TableReader;

/**
Expand Down Expand Up @@ -262,8 +267,9 @@ public void createTables(DocumentModel documentModel) throws SQLException {
List<FileModel> fileModels = documentModel.getFileModels();
for (FileModel fileModel : fileModels) {
if (fileModel instanceof TableModel) {
TableModel tableModel = (TableModel) fileModel;
createTable(tableModel);
createTable((TableModel) fileModel);
} else if (fileModel instanceof MultiTableModel) {
createTables((MultiTableModel) fileModel);
}
}
}
Expand All @@ -274,17 +280,53 @@ public void createTables(DocumentModel documentModel) throws SQLException {
* @param tableModel
* @throws SQLException
*/
public void createTable(TableModel tableModel) throws SQLException {
log.info(MARKER, "Create table for the TableModel '{}' ...", tableModel.getName());
List<AttributeType<?>> attributes = tableModel.getFeatureType().getAttributes();
List<String> columns = new ArrayList<String>(attributes.size());
private void createTable(TableModel tableModel) throws SQLException {
log.info(MARKER, "Create table for {} ...", tableModel);

for (AttributeType<?> attribute : attributes) {
columns.add(attribute.getName() + " TEXT");
String tableName = tableModel.getName();
FeatureType featureType = tableModel.getFeatureType();
createTable(tableName, featureType);
}

/**
* Create tables according to MultiTableModel.
*
* TODO ensure table naming consistency with {@link DocumentNormalizer}
*
* @param fileModel
*/
private void createTables(MultiTableModel fileModel) throws SQLException {
log.info(MARKER, "Create tables for {} ...", fileModel);
for (EmbeddedTableModel tableModel : fileModel.getTableModels()) {
String tableName = fileModel.getName() + "_" + tableModel.getName();
log.info(
MARKER, "Create table {} for {} in {} ...",
tableName,
tableModel,
fileModel
);
createTable(tableName, tableModel.getFeatureType());
}
}

String sql = "CREATE TABLE " + tableModel.getName() + " (" + String.join(",", columns) + ");";
/**
* Create table for a given FeatureType.
*
* @param tableName
* @param featureType
* @throws SQLException
*/
private void createTable(String tableName, FeatureType featureType) throws SQLException {
if (featureType.isEmpty()) {
log.warn(MARKER, "Create table {} skipped (empty FeatureType)", tableName);
return;
}

List<String> columns = new ArrayList<>(featureType.getAttributeCount());
for (AttributeType<?> attribute : featureType.getAttributes()) {
columns.add(attribute.getName() + " TEXT");
}
String sql = "CREATE TABLE " + tableName + " (" + String.join(",", columns) + ");";
update(sql);
connection.commit();
}
Expand Down Expand Up @@ -313,29 +355,44 @@ public void createTable(String tableName, List<String> columnNames) throws SQLEx
/**
* Create indexes for all tables on unique fields
*
* TODO add support for {@link MultiTableModel}
*
* @param document
* @throws SQLException
*/
public void createIndexes(DocumentModel documentModel) throws SQLException {
log.info(MARKER, "Create indexes for {} ...", documentModel);
for (FileModel file : documentModel.getFileModels()) {
if (!(file instanceof TableModel)) {
continue;
if (file instanceof TableModel) {
createIndexes((TableModel) file);
} else if (file instanceof MultiTableModel) {
log.warn(MARKER, "Create indexes for {} is not yet supported!", file);
}
List<AttributeType<?>> attributes = file.getFeatureType().getAttributes();
/*
* create indexes according to constraints
*/
for (AttributeType<?> attributeType : attributes) {
// create index for unique values
if (attributeType.getConstraints().isUnique()) {
createIndex(file.getName(), attributeType.getName());
}
// create index for referenced values
if (!StringUtils.isEmpty(attributeType.getConstraints().getReference())) {
String targetTableName = attributeType.getTableReference();
String targetColumnName = attributeType.getAttributeReference();
createIndex(targetTableName, targetColumnName);
}
}
}

/**
* Create indexes for a given table.
*
* @param tableModel
* @throws SQLException
*/
public void createIndexes(TableModel tableModel) throws SQLException {
log.info(MARKER, "Create indexes for {} ...", tableModel);
List<AttributeType<?>> attributes = tableModel.getFeatureType().getAttributes();
/*
* create indexes according to constraints
*/
for (AttributeType<?> attributeType : attributes) {
// create index for unique values
if (attributeType.getConstraints().isUnique()) {
createIndex(tableModel.getName(), attributeType.getName());
}
// create index for referenced values
if (!StringUtils.isEmpty(attributeType.getConstraints().getReference())) {
String targetTableName = attributeType.getTableReference();
String targetColumnName = attributeType.getAttributeReference();
createIndex(targetTableName, targetColumnName);
}
}
}
Expand All @@ -347,6 +404,7 @@ public void createIndexes(DocumentModel documentModel) throws SQLException {
* @param columnName
*/
public void createIndex(String tableName, String columnName) throws SQLException {
log.info(MARKER, "Create index on {}.{} ...", tableName, columnName);
String indexName = "idx_" + tableName + "_" + columnName;
String sql = "CREATE INDEX IF NOT EXISTS " + indexName + " ON " + tableName + " (" + columnName + ")";

Expand All @@ -357,15 +415,18 @@ public void createIndex(String tableName, String columnName) throws SQLException
/**
* Load an entire document to the database (insert mode)
*
* TODO add support for {@link MultiTableFile}
*
* @throws IOException
* @throws SQLException
*/
public void load(Context context, Document document) throws IOException, SQLException {
log.info(MARKER, "Load document '{}'...", document.getDocumentPath());
List<DocumentFile> files = document.getDocumentFiles();
for (DocumentFile file : files) {
if (file instanceof TableFile) {
load(context, file);
log.info(MARKER, "Loading data from document '{}'...", document.getDocumentPath());
for (DocumentFile documentFile : document.getDocumentFiles()) {
if (documentFile instanceof TableFile) {
load(context, (TableFile) documentFile);
} else if (documentFile instanceof MultiTableFile) {
log.warn(MARKER, "Loading data from {} is not yet supported!", documentFile);
}
}
}
Expand All @@ -378,9 +439,9 @@ public void load(Context context, Document document) throws IOException, SQLExce
* @throws IOException
* @throws SQLException
*/
public void load(Context context, DocumentFile documentFile) throws IOException, SQLException {
FileModel fileModel = documentFile.getFileModel();
loadFile(fileModel.getName(), documentFile.getPath(), context.getEncoding());
public void load(Context context, TableFile tableFile) throws IOException, SQLException {
FileModel fileModel = tableFile.getFileModel();
loadFile(fileModel.getName(), tableFile.getPath(), context.getEncoding());
}

/**
Expand All @@ -393,7 +454,7 @@ public void load(Context context, DocumentFile documentFile) throws IOException,
* @throws SQLException
*/
public void loadFile(String tableName, File path, Charset charset) throws IOException, SQLException {
log.info(MARKER, "loadFile({},{},{})...", tableName, path.getAbsolutePath(), charset.toString());
log.info(MARKER, "Load table '{}' from '{}' (charset={})...", tableName, path.getAbsolutePath(), charset);
/*
* Create table reader
*/
Expand Down Expand Up @@ -455,6 +516,15 @@ public void loadFile(String tableName, File path, Charset charset) throws IOExce
}
sth.executeBatch();
connection.commit();

log.info(
MARKER,
"Load table '{}' from '{}' (charset={}) : completed, {} row(s) loaded",
tableName,
path.getAbsolutePath(),
charset,
count
);
}

/**
Expand Down
12 changes: 6 additions & 6 deletions validator-core/src/main/java/fr/ign/validator/io/AbstractModelReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
import fr.ign.validator.exception.ModelNotFoundException;
import fr.ign.validator.model.DocumentModel;
import fr.ign.validator.model.FeatureType;
import fr.ign.validator.model.FileModel;
import fr.ign.validator.model.file.TableModel;

/**
* Common implementation for JSON and XML ModelReader.
Expand Down Expand Up @@ -40,28 +40,28 @@ public FeatureType loadFeatureType(File path) throws ModelNotFoundException, Inv
/**
* Resolve FeatureType URL for a given FileModel
*
* TODO support explicit featureType reference in FileModel
* TODO support explicit featureType reference in TableModel
*
* @param documentModelUrl
* @param documentModel
* @param documentFile
* @param tableModel
* @return
* @throws MalformedURLException
*/
protected URL resolveFeatureTypeUrl(URL documentModelUrl, DocumentModel documentModel, FileModel documentFile)
protected URL resolveFeatureTypeUrl(URL documentModelUrl, DocumentModel documentModel, TableModel tableModel)
throws MalformedURLException {
String parentUrl = getParentURL(documentModelUrl);
if (documentModelUrl.getProtocol().equals("file")) {
/* config export convention */
// validator-config-cnig/config/cnig_PLU_2017/files.xml
// validator-config-cnig/config/cnig_PLU_2017/types/ZONE_URBA.xml
return new URL(parentUrl + "/types/" + documentFile.getName() + "." + getFormat());
return new URL(parentUrl + "/types/" + tableModel.getName() + "." + getFormat());
} else {
/* URL convention */
// https://www.geoportail-urbanisme.gouv.fr/standard/cnig_PLU_2017.xml
// https://www.geoportail-urbanisme.gouv.fr/standard/cnig_PLU_2017/types/ZONE_URBA.xml
return new URL(
parentUrl + "/" + documentModel.getName() + "/types/" + documentFile.getName() + "." + getFormat()
parentUrl + "/" + documentModel.getName() + "/types/" + tableModel.getName() + "." + getFormat()
);
}
}
Expand Down
28 changes: 19 additions & 9 deletions validator-core/src/main/java/fr/ign/validator/io/JsonModelReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,15 @@
import fr.ign.validator.model.DocumentModel;
import fr.ign.validator.model.FeatureType;
import fr.ign.validator.model.FileModel;
import fr.ign.validator.model.file.MultiTableModel;
import fr.ign.validator.model.file.TableModel;

/**
* Helper class to load {@link DocumentModel} and {@link FeatureType} from JSON.
*
* @author MBorne
*
*/
public class JsonModelReader extends AbstractModelReader {
private static final Logger log = LogManager.getRootLogger();
private static final Marker MARKER = MarkerManager.getMarker("JsonModelReader");
Expand All @@ -41,17 +48,20 @@ public DocumentModel loadDocumentModel(URL documentModelUrl) throws ModelNotFoun
InputStream is = getInputStream(documentModelUrl);
try {
DocumentModel documentModel = objectMapper.readValue(is, DocumentModel.class);
/*
* load feature types for TableModel
*/
for (FileModel documentFile : documentModel.getFileModels()) {
if (!(documentFile instanceof TableModel)) {
continue;
log.info(MARKER, "Loading FeatureTypes for {} ...", documentModel);
for (FileModel fileModel : documentModel.getFileModels()) {
if (fileModel instanceof TableModel) {
log.info(MARKER, "Loading FeatureType for {} ...", fileModel);
TableModel tableModel = (TableModel) fileModel;
URL featureTypeUrl = resolveFeatureTypeUrl(documentModelUrl, documentModel, tableModel);
FeatureType featureType = loadFeatureType(featureTypeUrl);
tableModel.setFeatureType(featureType);
} else if (fileModel instanceof MultiTableModel) {
log.warn(MARKER, "Loading MultiTableModel for {} is not yet supported!", fileModel);
// TODO allows to specify FeatureType for each table in MultiTableModel
}
URL featureTypeUrl = resolveFeatureTypeUrl(documentModelUrl, documentModel, documentFile);
FeatureType featureType = loadFeatureType(featureTypeUrl);
documentFile.setFeatureType(featureType);
}
log.info(MARKER, "Loading FeatureTypes for {} : completed.", documentModel);
return documentModel;
} catch (IOException e) {
String message = String.format("Fail to parse DocumentModel : %1s : %2s", documentModelUrl, e.getMessage());
Expand Down
15 changes: 8 additions & 7 deletions validator-core/src/main/java/fr/ign/validator/io/XmlModelReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ public class XmlModelReader extends AbstractModelReader {
private Unmarshaller unmarshaller;

public XmlModelReader() {
log.trace(MARKER, "Initializing XmlModelReader...");
try {
this.context = JAXBContext.newInstance(FeatureType.class, DocumentModel.class);
this.unmarshaller = context.createUnmarshaller();
Expand All @@ -48,7 +49,7 @@ public String getFormat() {

@Override
public DocumentModel loadDocumentModel(URL documentModelUrl) {
log.info(MARKER, "loadDocumentModel({}) ...", documentModelUrl);
log.info(MARKER, "Loading DocumentModel from {} ...", documentModelUrl);

/*
* loading documentModel
Expand All @@ -59,13 +60,13 @@ public DocumentModel loadDocumentModel(URL documentModelUrl) {
/*
* load feature types for TableModel
*/
for (FileModel documentFile : documentModel.getFileModels()) {
if (!(documentFile instanceof TableModel)) {
continue;
for (FileModel fileModel : documentModel.getFileModels()) {
if (fileModel instanceof TableModel) {
TableModel tableModel = (TableModel) fileModel;
URL featureTypeUrl = resolveFeatureTypeUrl(documentModelUrl, documentModel, tableModel);
FeatureType featureType = loadFeatureType(featureTypeUrl);
tableModel.setFeatureType(featureType);
}
URL featureTypeUrl = resolveFeatureTypeUrl(documentModelUrl, documentModel, documentFile);
FeatureType featureType = loadFeatureType(featureTypeUrl);
documentFile.setFeatureType(featureType);
}
return documentModel;
} catch (JAXBException e) {
Expand Down
Loading

0 comments on commit 2f1d05e

Please sign in to comment.