Skip to content

Commit

Permalink
review logs and fix some sonar issues (refs #231)
Browse files Browse the repository at this point in the history
  • Loading branch information
@mborne
mborne committed Jul 6, 2021
1 parent 5005c2d commit 88de70f
Show file tree
Hide file tree
Showing 4 changed files with 118 additions and 97 deletions.
19 changes: 13 additions & 6 deletions validator-core/src/main/java/fr/ign/validator/data/Document.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ public class Document implements Validatable {
/**
* Files related to Document (defined after matching step)
*/
private List<DocumentFile> documentFiles = new ArrayList<DocumentFile>();
private List<DocumentFile> documentFiles = new ArrayList<>();

/**
* Additional informations
Expand Down Expand Up @@ -109,7 +109,7 @@ public List<DocumentFile> getDocumentFiles() {
* @return
*/
public <T extends FileModel> List<DocumentFile> getDocumentFiles(Class<T> type) {
List<DocumentFile> result = new ArrayList<DocumentFile>();
List<DocumentFile> result = new ArrayList<>();
for (DocumentFile documentFile : documentFiles) {
if (type.isAssignableFrom(documentFile.getFileModel().getClass())) {
result.add(documentFile);
Expand All @@ -132,7 +132,7 @@ public void removeDocumentFile(DocumentFile documentFile) {
* @return
*/
public List<DocumentFile> getDocumentFilesByModel(FileModel fileModel) {
List<DocumentFile> result = new ArrayList<DocumentFile>();
List<DocumentFile> result = new ArrayList<>();
for (DocumentFile documentFile : documentFiles) {
if (documentFile.getFileModel() == fileModel) {
result.add(documentFile);
Expand Down Expand Up @@ -233,6 +233,7 @@ protected void triggerBeforeMatching(Context context) throws Exception {
for (ValidatorListener validatorListener : context.getValidatorListeners()) {
validatorListener.beforeMatching(context, this);
}
log.info(MARKER, "Run 'BeforeMatching' pre-processes : completed");
}

/**
Expand All @@ -246,6 +247,7 @@ protected void triggerBeforeValidate(Context context) throws Exception {
for (ValidatorListener validatorListener : context.getValidatorListeners()) {
validatorListener.beforeValidate(context, this);
}
log.info(MARKER, "Run 'beforeValidate' pre-processes : completed");
}

/**
Expand All @@ -259,6 +261,7 @@ protected void triggerAfterValidate(Context context) throws Exception {
for (ValidatorListener validatorListener : context.getValidatorListeners()) {
validatorListener.afterValidate(context, this);
}
log.info(MARKER, "Run 'afterValidate' pre-processes : completed");
}

/**
Expand All @@ -269,9 +272,9 @@ protected void triggerAfterValidate(Context context) throws Exception {
public void findDocumentFiles(Context context) {
clearFiles();

File documentPath = getDocumentPath();
log.info(MARKER, "findDocumentFiles('{}')...", documentPath);
log.info(MARKER, "findDocumentFiles('{}') : list files and directories...", documentPath);
Collection<File> files = FileUtils.listFilesAndDirs(documentPath, allowedExtensions);
log.info(MARKER, "findDocumentFiles('{}') : {} file(s) found", documentPath, files.size());

/*
* find match with FileModel
Expand Down Expand Up @@ -315,7 +318,11 @@ public void findDocumentFiles(Context context) {
);
}

log.info(MARKER, "findDocumentFiles('{}') complete, found {} files.", documentPath, documentFiles.size());
log.info(
MARKER, "findDocumentFiles('{}') : completed, {} file(s) found.",
documentPath,
documentFiles.size()
);
}

private void addDocumentFile(FileModel fileModel, File path) {
Expand Down
4 changes: 2 additions & 2 deletions validator-core/src/main/java/fr/ign/validator/data/DocumentFile.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ public abstract class DocumentFile implements Validatable {
public static final Marker MARKER = MarkerManager.getMarker("DocumentFile");

/**
* filepath
* Path to the given file.
*/
private File path;

Expand All @@ -37,7 +37,7 @@ protected DocumentFile(File path) {
/**
* @return the fileModel
*/
abstract public FileModel getFileModel();
public abstract FileModel getFileModel();

/**
* @return the path
Expand Down
170 changes: 90 additions & 80 deletions validator-core/src/main/java/fr/ign/validator/tools/FileConverter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,14 +32,23 @@
*
*/
public class FileConverter {

public static final Marker MARKER = MarkerManager.getMarker("FileConverter");
public static final Logger log = LogManager.getRootLogger();

private static final String DRIVER_CSV = "CSV";
private static final String DRIVER_SHAPEFILE = "ESRI Shapefile";

public static final String ENCODING_UTF8 = "UTF-8";
public static final String ENCODING_LATIN1 = "ISO-8859-1";

private static FileConverter instance = new FileConverter();

/**
* Path to ogr2ogr
*/
private String ogr2ogrPath;

/**
* ogr2ogr version
*/
Expand All @@ -49,6 +58,8 @@ public class FileConverter {
* Default constructor
*/
private FileConverter() {
log.info(MARKER, "Instanciate FileConverter ensuring that ogr2ogr version is supported...");
this.ogr2ogrPath = retrieveOgr2ogrPath();
this.version = retrieveAndValidateOgrVersion();
}

Expand All @@ -61,23 +72,6 @@ public static FileConverter getInstance() {
return instance;
}

/**
* Get path to ogr2ogr. Default is ogr2ogr, it can be specified with :
* <ul>
* <li>Environment variable OGR2OGR_PATH</li>
* <li>System property ogr2ogr_path</li>
* </ul>
*
* @return
*/
private String getOgr2ogrPath() {
String result = System.getenv("OGR2OGR_PATH");
if (result != null) {
return result;
}
return System.getProperty("ogr2ogr_path", "ogr2ogr");
}

/**
* returns ogr2ogr version
*
Expand All @@ -87,46 +81,6 @@ public OgrVersion getVersion() {
return this.version;
}

/**
* Récupération de la version de ogr2ogr
*
* @return
*/
private OgrVersion retrieveAndValidateOgrVersion() {
String fullVersion = retrieveFullVersion();
OgrVersion version = new OgrVersion(fullVersion);
version.ensureVersionIsSupported();
return version;
}

/**
* Call `ogr2ogr --version` to get GDAL version
*
* @return
*/
private String retrieveFullVersion() {
log.info(MARKER, "Run 'ogr2ogr --version' to retrieve GDAL version...");
String[] args = new String[] {
getOgr2ogrPath(), "--version"
};
ProcessBuilder builder = new ProcessBuilder(args);
try {
Process process = builder.start();

process.waitFor();

InputStream stdout = process.getInputStream();
BufferedReader stdoutReader = new BufferedReader(new InputStreamReader(stdout));
String version = stdoutReader.readLine();
stdoutReader.close();
return version;
} catch (IOException e) {
return null;
} catch (InterruptedException e) {
return null;
}
}

/**
* Convert a source file with a given sourceCharset to an UTF-8 encoded CSV
* target
Expand Down Expand Up @@ -154,9 +108,9 @@ public void convertToCSV(File source, File target, Charset sourceCharset) throws
CompanionFileUtils.removeCompanionFile(source, "cpg");
CompanionFileUtils.removeCompanionFile(source, "CPG");

String[] args = getArguments(source, target, "CSV");
Map<String, String> envs = new HashMap<String, String>();
// encoding is specified in UTF-8 so that ogr2ogr doesn't convert
String[] args = getArguments(source, target, DRIVER_CSV);
Map<String, String> envs = new HashMap<>();

if (sourceExtension.equals("dbf") || sourceExtension.equals("shp")) {
envs.put("SHAPE_ENCODING", toEncoding(sourceCharset));
}
Expand All @@ -170,20 +124,6 @@ public void convertToCSV(File source, File target, Charset sourceCharset) throws
}
}

/**
* Convert java charset to GDAL encoding
*
* @param sourceCharset
* @return
*/
private String toEncoding(Charset sourceCharset) {
if (sourceCharset.equals(StandardCharsets.ISO_8859_1)) {
return ENCODING_LATIN1;
} else {
return ENCODING_UTF8;
}
}

/**
* Converts a source file in LATIN1 encoded shapefile
*
Expand All @@ -196,7 +136,7 @@ public void convertToShapefile(File source, File target) throws IOException {
fixGML(source);
}

String[] args = getArguments(source, target, "ESRI Shapefile");
String[] args = getArguments(source, target, DRIVER_SHAPEFILE);
Map<String, String> envs = new HashMap<String, String>();
envs.put("SHAPE_ENCODING", ENCODING_LATIN1);
runCommand(args, envs);
Expand All @@ -215,6 +155,78 @@ public void convertToShapefile(File source, File target) throws IOException {
FileUtils.writeStringToFile(cpgFile, ENCODING_LATIN1, StandardCharsets.UTF_8);
}

/**
* Get path to ogr2ogr. Default is ogr2ogr, it can be specified with :
* <ul>
* <li>Environment variable OGR2OGR_PATH</li>
* <li>System property ogr2ogr_path</li>
* </ul>
*
* @return
*/
private String retrieveOgr2ogrPath() {
String result = System.getenv("OGR2OGR_PATH");
if (result != null) {
return result;
}
return System.getProperty("ogr2ogr_path", "ogr2ogr");
}

/**
* Récupération de la version de ogr2ogr
*
* @return
*/
private OgrVersion retrieveAndValidateOgrVersion() {
String fullVersion = retrieveFullVersion();
log.info(MARKER, "ogr2ogr --version : {}", fullVersion);
OgrVersion result = new OgrVersion(fullVersion);
result.ensureVersionIsSupported();
return result;
}

/**
* Call `ogr2ogr --version` to get GDAL version
*
* @return
*/
private String retrieveFullVersion() {
log.info(MARKER, "Run 'ogr2ogr --version' to retrieve GDAL version...");
String[] args = new String[] {
ogr2ogrPath, "--version"
};
ProcessBuilder builder = new ProcessBuilder(args);
try {
Process process = builder.start();

process.waitFor();

InputStream stdout = process.getInputStream();
BufferedReader stdoutReader = new BufferedReader(new InputStreamReader(stdout));
String result = stdoutReader.readLine();
stdoutReader.close();
return result;
} catch (IOException e) {
return null;
} catch (InterruptedException e) {
return null;
}
}

/**
* Convert java charset to GDAL encoding
*
* @param sourceCharset
* @return
*/
private String toEncoding(Charset sourceCharset) {
if (sourceCharset.equals(StandardCharsets.ISO_8859_1)) {
return ENCODING_LATIN1;
} else {
return ENCODING_UTF8;
}
}

/**
*
* Any invalid csv file blocks ogr2ogr use A valid file with header without data
Expand Down Expand Up @@ -243,7 +255,7 @@ private void createFalseCSV(File target) throws IOException {
*/
private String[] getArguments(File source, File target, String driver) {
List<String> arguments = new ArrayList<String>();
arguments.add(getOgr2ogrPath());
arguments.add(ogr2ogrPath);

// Otherwise, some ogr2ogr versions transforms 01 to 1...
if (FilenameUtils.getExtension(source.getName()).toLowerCase().equals("gml")) {
Expand All @@ -257,7 +269,7 @@ private String[] getArguments(File source, File target, String driver) {
/*
* Getting format-specific parameters
*/
if (driver.equals("CSV")) {
if (driver.equals(DRIVER_CSV)) {
if (hasSpatialColumn(source)) {
// unsure conversion to WKT
arguments.add("-lco");
Expand Down Expand Up @@ -346,9 +358,7 @@ private void runCommand(String[] args, Map<String, String> envs) throws IOExcept
if (process.exitValue() != 0) {
log.error(MARKER, "command fail!");
}
} catch (IOException e1) {
throw new RuntimeException("ogr2ogr command fails", e1);
} catch (InterruptedException e) {
} catch (IOException | InterruptedException e) {
throw new RuntimeException("ogr2ogr command fails", e);
}
}
Expand Down
22 changes: 13 additions & 9 deletions validator-core/src/main/java/fr/ign/validator/tools/FileUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,23 @@
import fr.ign.validator.tools.filter.FileByExtensionAndDirectoryFilter;

/**
* Helper class to find Document files.
*
* @see http://techblog.sharpmind.de/?p=228
*/
public class FileUtils {

/**
* List files and directories filtering files according to an extension.
*/
public static Collection<File> listFilesAndDirs(File directory, String[] allowedExtensions) {
return listFilesAndDirs(
directory,
new FileByExtensionAndDirectoryFilter(allowedExtensions),
TrueFileFilter.INSTANCE
);
}

/**
* Finds files and directories within a given directory (and optionally its
* subdirectories). All files/dirs found are filtered by an IOFileFilter.
Expand All @@ -24,7 +36,7 @@ public class FileUtils {
* @param dirFilter in which dirs the algorithm should traverse
* @return the list of found file objects
*/
public static Collection<File> listFilesAndDirs(File directory,
private static Collection<File> listFilesAndDirs(File directory,
IOFileFilter fileFilter,
IOFileFilter dirFilter) {
Collection<File> files = new ArrayList<File>();
Expand All @@ -45,12 +57,4 @@ public static Collection<File> listFilesAndDirs(File directory,
return files;
}

public static Collection<File> listFilesAndDirs(File directory, String[] allowedExtensions) {
return listFilesAndDirs(
directory,
new FileByExtensionAndDirectoryFilter(allowedExtensions),
TrueFileFilter.INSTANCE
);
}

}

0 comments on commit 88de70f

Please sign in to comment.