diff --git a/validator-core/src/main/java/fr/ign/validator/data/Document.java b/validator-core/src/main/java/fr/ign/validator/data/Document.java index 76865080..16bebf47 100644 --- a/validator-core/src/main/java/fr/ign/validator/data/Document.java +++ b/validator-core/src/main/java/fr/ign/validator/data/Document.java @@ -53,7 +53,7 @@ public class Document implements Validatable { /** * Files related to Document (defined after matching step) */ - private List documentFiles = new ArrayList(); + private List documentFiles = new ArrayList<>(); /** * Additional informations @@ -109,7 +109,7 @@ public List getDocumentFiles() { * @return */ public List getDocumentFiles(Class type) { - List result = new ArrayList(); + List result = new ArrayList<>(); for (DocumentFile documentFile : documentFiles) { if (type.isAssignableFrom(documentFile.getFileModel().getClass())) { result.add(documentFile); @@ -132,7 +132,7 @@ public void removeDocumentFile(DocumentFile documentFile) { * @return */ public List getDocumentFilesByModel(FileModel fileModel) { - List result = new ArrayList(); + List result = new ArrayList<>(); for (DocumentFile documentFile : documentFiles) { if (documentFile.getFileModel() == fileModel) { result.add(documentFile); @@ -233,6 +233,7 @@ protected void triggerBeforeMatching(Context context) throws Exception { for (ValidatorListener validatorListener : context.getValidatorListeners()) { validatorListener.beforeMatching(context, this); } + log.info(MARKER, "Run 'BeforeMatching' pre-processes : completed"); } /** @@ -246,6 +247,7 @@ protected void triggerBeforeValidate(Context context) throws Exception { for (ValidatorListener validatorListener : context.getValidatorListeners()) { validatorListener.beforeValidate(context, this); } + log.info(MARKER, "Run 'beforeValidate' pre-processes : completed"); } /** @@ -259,6 +261,7 @@ protected void triggerAfterValidate(Context context) throws Exception { for (ValidatorListener validatorListener : context.getValidatorListeners()) { validatorListener.afterValidate(context, this); } + log.info(MARKER, "Run 'afterValidate' pre-processes : completed"); } /** @@ -269,9 +272,9 @@ protected void triggerAfterValidate(Context context) throws Exception { public void findDocumentFiles(Context context) { clearFiles(); - File documentPath = getDocumentPath(); - log.info(MARKER, "findDocumentFiles('{}')...", documentPath); + log.info(MARKER, "findDocumentFiles('{}') : list files and directories...", documentPath); Collection files = FileUtils.listFilesAndDirs(documentPath, allowedExtensions); + log.info(MARKER, "findDocumentFiles('{}') : {} file(s) found", documentPath, files.size()); /* * find match with FileModel @@ -315,7 +318,11 @@ public void findDocumentFiles(Context context) { ); } - log.info(MARKER, "findDocumentFiles('{}') complete, found {} files.", documentPath, documentFiles.size()); + log.info( + MARKER, "findDocumentFiles('{}') : completed, {} file(s) found.", + documentPath, + documentFiles.size() + ); } private void addDocumentFile(FileModel fileModel, File path) { diff --git a/validator-core/src/main/java/fr/ign/validator/data/DocumentFile.java b/validator-core/src/main/java/fr/ign/validator/data/DocumentFile.java index 620401d6..dc60af2d 100644 --- a/validator-core/src/main/java/fr/ign/validator/data/DocumentFile.java +++ b/validator-core/src/main/java/fr/ign/validator/data/DocumentFile.java @@ -20,7 +20,7 @@ public abstract class DocumentFile implements Validatable { public static final Marker MARKER = MarkerManager.getMarker("DocumentFile"); /** - * filepath + * Path to the given file. */ private File path; @@ -37,7 +37,7 @@ protected DocumentFile(File path) { /** * @return the fileModel */ - abstract public FileModel getFileModel(); + public abstract FileModel getFileModel(); /** * @return the path diff --git a/validator-core/src/main/java/fr/ign/validator/tools/FileConverter.java b/validator-core/src/main/java/fr/ign/validator/tools/FileConverter.java index 821d8da3..87453611 100644 --- a/validator-core/src/main/java/fr/ign/validator/tools/FileConverter.java +++ b/validator-core/src/main/java/fr/ign/validator/tools/FileConverter.java @@ -32,14 +32,23 @@ * */ public class FileConverter { + public static final Marker MARKER = MarkerManager.getMarker("FileConverter"); public static final Logger log = LogManager.getRootLogger(); + private static final String DRIVER_CSV = "CSV"; + private static final String DRIVER_SHAPEFILE = "ESRI Shapefile"; + public static final String ENCODING_UTF8 = "UTF-8"; public static final String ENCODING_LATIN1 = "ISO-8859-1"; private static FileConverter instance = new FileConverter(); + /** + * Path to ogr2ogr + */ + private String ogr2ogrPath; + /** * ogr2ogr version */ @@ -49,6 +58,8 @@ public class FileConverter { * Default constructor */ private FileConverter() { + log.info(MARKER, "Instanciate FileConverter ensuring that ogr2ogr version is supported..."); + this.ogr2ogrPath = retrieveOgr2ogrPath(); this.version = retrieveAndValidateOgrVersion(); } @@ -61,23 +72,6 @@ public static FileConverter getInstance() { return instance; } - /** - * Get path to ogr2ogr. Default is ogr2ogr, it can be specified with : - *
    - *
  • Environment variable OGR2OGR_PATH
    • - *
  • System property ogr2ogr_path
    • - *
- * - * @return - */ - private String getOgr2ogrPath() { - String result = System.getenv("OGR2OGR_PATH"); - if (result != null) { - return result; - } - return System.getProperty("ogr2ogr_path", "ogr2ogr"); - } - /** * returns ogr2ogr version * @@ -87,46 +81,6 @@ public OgrVersion getVersion() { return this.version; } - /** - * Récupération de la version de ogr2ogr - * - * @return - */ - private OgrVersion retrieveAndValidateOgrVersion() { - String fullVersion = retrieveFullVersion(); - OgrVersion version = new OgrVersion(fullVersion); - version.ensureVersionIsSupported(); - return version; - } - - /** - * Call `ogr2ogr --version` to get GDAL version - * - * @return - */ - private String retrieveFullVersion() { - log.info(MARKER, "Run 'ogr2ogr --version' to retrieve GDAL version..."); - String[] args = new String[] { - getOgr2ogrPath(), "--version" - }; - ProcessBuilder builder = new ProcessBuilder(args); - try { - Process process = builder.start(); - - process.waitFor(); - - InputStream stdout = process.getInputStream(); - BufferedReader stdoutReader = new BufferedReader(new InputStreamReader(stdout)); - String version = stdoutReader.readLine(); - stdoutReader.close(); - return version; - } catch (IOException e) { - return null; - } catch (InterruptedException e) { - return null; - } - } - /** * Convert a source file with a given sourceCharset to an UTF-8 encoded CSV * target @@ -154,9 +108,9 @@ public void convertToCSV(File source, File target, Charset sourceCharset) throws CompanionFileUtils.removeCompanionFile(source, "cpg"); CompanionFileUtils.removeCompanionFile(source, "CPG"); - String[] args = getArguments(source, target, "CSV"); - Map envs = new HashMap(); - // encoding is specified in UTF-8 so that ogr2ogr doesn't convert + String[] args = getArguments(source, target, DRIVER_CSV); + Map envs = new HashMap<>(); + if (sourceExtension.equals("dbf") || sourceExtension.equals("shp")) { envs.put("SHAPE_ENCODING", toEncoding(sourceCharset)); } @@ -170,20 +124,6 @@ public void convertToCSV(File source, File target, Charset sourceCharset) throws } } - /** - * Convert java charset to GDAL encoding - * - * @param sourceCharset - * @return - */ - private String toEncoding(Charset sourceCharset) { - if (sourceCharset.equals(StandardCharsets.ISO_8859_1)) { - return ENCODING_LATIN1; - } else { - return ENCODING_UTF8; - } - } - /** * Converts a source file in LATIN1 encoded shapefile * @@ -196,7 +136,7 @@ public void convertToShapefile(File source, File target) throws IOException { fixGML(source); } - String[] args = getArguments(source, target, "ESRI Shapefile"); + String[] args = getArguments(source, target, DRIVER_SHAPEFILE); Map envs = new HashMap(); envs.put("SHAPE_ENCODING", ENCODING_LATIN1); runCommand(args, envs); @@ -215,6 +155,78 @@ public void convertToShapefile(File source, File target) throws IOException { FileUtils.writeStringToFile(cpgFile, ENCODING_LATIN1, StandardCharsets.UTF_8); } + /** + * Get path to ogr2ogr. Default is ogr2ogr, it can be specified with : + *
    + *
  • Environment variable OGR2OGR_PATH
    • + *
  • System property ogr2ogr_path
    • + *
+ * + * @return + */ + private String retrieveOgr2ogrPath() { + String result = System.getenv("OGR2OGR_PATH"); + if (result != null) { + return result; + } + return System.getProperty("ogr2ogr_path", "ogr2ogr"); + } + + /** + * Récupération de la version de ogr2ogr + * + * @return + */ + private OgrVersion retrieveAndValidateOgrVersion() { + String fullVersion = retrieveFullVersion(); + log.info(MARKER, "ogr2ogr --version : {}", fullVersion); + OgrVersion result = new OgrVersion(fullVersion); + result.ensureVersionIsSupported(); + return result; + } + + /** + * Call `ogr2ogr --version` to get GDAL version + * + * @return + */ + private String retrieveFullVersion() { + log.info(MARKER, "Run 'ogr2ogr --version' to retrieve GDAL version..."); + String[] args = new String[] { + ogr2ogrPath, "--version" + }; + ProcessBuilder builder = new ProcessBuilder(args); + try { + Process process = builder.start(); + + process.waitFor(); + + InputStream stdout = process.getInputStream(); + BufferedReader stdoutReader = new BufferedReader(new InputStreamReader(stdout)); + String result = stdoutReader.readLine(); + stdoutReader.close(); + return result; + } catch (IOException e) { + return null; + } catch (InterruptedException e) { + return null; + } + } + + /** + * Convert java charset to GDAL encoding + * + * @param sourceCharset + * @return + */ + private String toEncoding(Charset sourceCharset) { + if (sourceCharset.equals(StandardCharsets.ISO_8859_1)) { + return ENCODING_LATIN1; + } else { + return ENCODING_UTF8; + } + } + /** * * Any invalid csv file blocks ogr2ogr use A valid file with header without data @@ -243,7 +255,7 @@ private void createFalseCSV(File target) throws IOException { */ private String[] getArguments(File source, File target, String driver) { List arguments = new ArrayList(); - arguments.add(getOgr2ogrPath()); + arguments.add(ogr2ogrPath); // Otherwise, some ogr2ogr versions transforms 01 to 1... if (FilenameUtils.getExtension(source.getName()).toLowerCase().equals("gml")) { @@ -257,7 +269,7 @@ private String[] getArguments(File source, File target, String driver) { /* * Getting format-specific parameters */ - if (driver.equals("CSV")) { + if (driver.equals(DRIVER_CSV)) { if (hasSpatialColumn(source)) { // unsure conversion to WKT arguments.add("-lco"); @@ -346,9 +358,7 @@ private void runCommand(String[] args, Map envs) throws IOExcept if (process.exitValue() != 0) { log.error(MARKER, "command fail!"); } - } catch (IOException e1) { - throw new RuntimeException("ogr2ogr command fails", e1); - } catch (InterruptedException e) { + } catch (IOException | InterruptedException e) { throw new RuntimeException("ogr2ogr command fails", e); } } diff --git a/validator-core/src/main/java/fr/ign/validator/tools/FileUtils.java b/validator-core/src/main/java/fr/ign/validator/tools/FileUtils.java index d3c5a126..16a99226 100644 --- a/validator-core/src/main/java/fr/ign/validator/tools/FileUtils.java +++ b/validator-core/src/main/java/fr/ign/validator/tools/FileUtils.java @@ -10,11 +10,23 @@ import fr.ign.validator.tools.filter.FileByExtensionAndDirectoryFilter; /** + * Helper class to find Document files. * * @see http://techblog.sharpmind.de/?p=228 */ public class FileUtils { + /** + * List files and directories filtering files according to an extension. + */ + public static Collection listFilesAndDirs(File directory, String[] allowedExtensions) { + return listFilesAndDirs( + directory, + new FileByExtensionAndDirectoryFilter(allowedExtensions), + TrueFileFilter.INSTANCE + ); + } + /** * Finds files and directories within a given directory (and optionally its * subdirectories). All files/dirs found are filtered by an IOFileFilter. @@ -24,7 +36,7 @@ public class FileUtils { * @param dirFilter in which dirs the algorithm should traverse * @return the list of found file objects */ - public static Collection listFilesAndDirs(File directory, + private static Collection listFilesAndDirs(File directory, IOFileFilter fileFilter, IOFileFilter dirFilter) { Collection files = new ArrayList(); @@ -45,12 +57,4 @@ public static Collection listFilesAndDirs(File directory, return files; } - public static Collection listFilesAndDirs(File directory, String[] allowedExtensions) { - return listFilesAndDirs( - directory, - new FileByExtensionAndDirectoryFilter(allowedExtensions), - TrueFileFilter.INSTANCE - ); - } - }