Merge branch 'master' into stable

.travis.yml

@@ -12,11 +12,11 @@ script:
 deploy:
   - provider: script
     skip_cleanup: true
-    script: gradle -Drepos_pwd=${repos_pwd} -Drepos_usr=jql_jars-INTE upload
+    script: gradle -Drepos_pwd=${repos_pwd} -Drepos_usr=jql_jars-INTE uploadArchives
     on:
       branch: master
   - provider: script
     skip_cleanup: true
-    script: gradle -Drepos_pwd=${repos_pwd} -Drepos_usr=jql_jars-INTE '-Drelease=' upload
+    script: gradle -Drepos_pwd=${repos_pwd} -Drepos_usr=jql_jars-INTE '-Drelease=' -Ddownloads_pwd=${downloads_pwd} -Ddownloads_usr=jql_downloads uploadArchives uploadBin
     on:
       branch: stable

build.gradle

@@ -7,10 +7,11 @@ allprojects {
 	targetCompatibility = "1.6"
 }
 
-version '1.11.5'+System.getProperty('release','-SNAPSHOT')
+version '1.11.6'+System.getProperty('release','-SNAPSHOT')
 
 configurations {
     deployerJars
+    ftpAntTask
 }
 
 // to get the latest SNAPSHOT uncomment the following lines
@@ -21,13 +22,15 @@ configurations {
 
 dependencies {
     compile group: 'ch.ehi', name: 'ehibasics', version: '1.3.0'
-    compile group: 'ch.interlis', name: 'iox-ili', version: '1.20.18'
-    compile group: 'ch.interlis', name: 'ili2c-tool', version: "5.0.8"
-    compile group: 'ch.interlis', name: 'ili2c-core', version: "5.0.8"
+    compile group: 'ch.interlis', name: 'iox-ili', version: '1.21.0'
+    compile group: 'ch.interlis', name: 'ili2c-tool', version: "5.1.1"
+    compile group: 'ch.interlis', name: 'ili2c-core', version: "5.1.1"
     testCompile group: 'junit', name: 'junit', version: '4.12'
     deployerJars "org.apache.maven.wagon:wagon-ftp:3.3.3"
     deployerJars "org.apache.maven.wagon:wagon-ssh:3.3.3"
 
+    ftpAntTask "org.apache.ant:ant-commons-net:1.10.7"
+
     testRuntime project('demoplugin')
 }
 
@@ -47,6 +50,8 @@ if(propFile.exists()){
 def git = System.getProperty('git',properties.get('git','git'))
 def repos_pwd = System.getProperty('repos_pwd',properties.get('repos_pwd','repos_pwd'))
 def repos_usr = System.getProperty('repos_usr',properties.get('repos_usr','repos_usr'))
+def downloads_pwd = System.getProperty('downloads_pwd',properties.get('downloads_pwd','downloads_pwd'))
+def downloads_usr = System.getProperty('downloads_usr',properties.get('downloads_usr','downloads_usr'))
 
 def generatedResources = "$buildDir/generated-resources/main"
 
@@ -142,4 +147,26 @@ uploadArchives {
             }
         }
     }
+}
+
+
+
+task uploadBin(dependsOn: bindist) {
+	doLast {
+		ant.taskdef(name: 'ftp',
+					classname: 'org.apache.tools.ant.taskdefs.optional.net.FTP',
+					classpath: configurations.ftpAntTask.asPath)
+		def dist= bindist.archiveFile.get().asFile.parent
+		def name=bindist.archiveFile.get().asFile.name
+		def json = groovy.json.JsonOutput.toJson([filename: 'https://downloads.interlis.ch/ilivalidator/'+name, version: project.version ,date: new Date().format( 'yyyy-MM-dd' )])
+		def releaseFile = new File(dist,project.name+"-release.json")
+		releaseFile.write(json)
+		ant.ftp(server: "jql.ftp.infomaniak.com", userid: downloads_usr, password: downloads_pwd, 
+			remoteDir: "/ilivalidator", passive:"yes") {
+				fileset(dir: dist ) {
+					include(name: name)
+					include(name: releaseFile.name)
+				}
+		}
+	}
 }

docs/CHANGELOG.txt

@@ -1,6 +1,14 @@
 LATEST VERSION
 -----------------------------
 
+ilivalidator 1.11.6 (2020-05-01)
+-----------------------------
+- fails to lookup ili24 model if xtf24 transfer file (#239)
+- Check Base64 input character (#246)
+- No object found with OID null (#249)
+- iox-ili-1.21.0
+- ili2c-5.1.1
+
 ilivalidator 1.11.5 (2020-04-01)
 -----------------------------
 - Missing check when write to "unwriteable" log-file (#228)

src/org/interlis2/validator/CreateIliDataTool.java

@@ -29,7 +29,7 @@
 import ch.interlis.iox.IoxReader;
 import ch.interlis.iox_j.EndBasketEvent;
 import ch.interlis.iox_j.EndTransferEvent;
-import ch.interlis.iox_j.IoxUtility;
+import ch.interlis.iox_j.utility.IoxUtility;
 import ch.interlis.iox_j.ObjectEvent;
 import ch.interlis.iox_j.StartBasketEvent;
 import ch.interlis.iox_j.StartTransferEvent;

src/org/interlis2/validator/UpdateIliDataTool.java

@@ -22,7 +22,7 @@
 import ch.interlis.iox.IoxEvent;
 import ch.interlis.iox.IoxReader;
 import ch.interlis.iox_j.EndBasketEvent;
-import ch.interlis.iox_j.IoxUtility;
+import ch.interlis.iox_j.utility.IoxUtility;
 import ch.interlis.iox_j.ObjectEvent;
 import ch.interlis.iox_j.StartBasketEvent;
 import ch.interlis.models.DatasetIdx16.Code_;

src/org/interlis2/validator/Validator.java

@@ -17,22 +17,28 @@
 import ch.interlis.ili2c.Ili2cException;
 import ch.interlis.ili2c.Ili2cFailure;
 import ch.interlis.ili2c.gui.UserSettings;
+import ch.interlis.ili2c.metamodel.Model;
 import ch.interlis.ili2c.metamodel.TransferDescription;
 import ch.interlis.iom_j.itf.ItfReader;
 import ch.interlis.iom_j.itf.ItfReader2;
+import ch.interlis.iom_j.xtf.Xtf23Reader;
+import ch.interlis.iom_j.xtf.Xtf24Reader;
+import ch.interlis.iom_j.xtf.XtfReader;
 import ch.interlis.iox.EndTransferEvent;
 import ch.interlis.iox.IoxEvent;
 import ch.interlis.iox.IoxException;
 import ch.interlis.iox.IoxLogging;
 import ch.interlis.iox.IoxReader;
 import ch.interlis.iox_j.IoxIliReader;
 import ch.interlis.iox_j.PipelinePool;
+import ch.interlis.iox_j.StartTransferEvent;
 import ch.interlis.iox_j.logging.FileLogger;
 import ch.interlis.iox_j.logging.LogEventFactory;
 import ch.interlis.iox_j.logging.StdLogger;
 import ch.interlis.iox_j.logging.XtfErrorsLogger;
 import ch.interlis.iox_j.plugins.PluginLoader;
 import ch.interlis.iox_j.statistics.IoxStatistics;
+import ch.interlis.iox_j.utility.IoxUtility;
 import ch.interlis.iox_j.utility.ReaderFactory;
 import ch.interlis.iox_j.validator.ValidationConfig;
 
@@ -138,7 +144,7 @@ public boolean validate(
 		    // give user important info (such as input files or program version)
 			EhiLogger.logState(Main.APP_NAME+"-"+Main.getVersion());
 			EhiLogger.logState("ili2c-"+ch.interlis.ili2c.Ili2c.getVersion());
-			EhiLogger.logState("iox-ili-"+ch.interlis.iox_j.IoxUtility.getVersion());
+			EhiLogger.logState("iox-ili-"+ch.interlis.iox_j.utility.IoxUtility.getVersion());
             EhiLogger.logState("User <"+java.lang.System.getProperty("user.name")+">");
             String DATE_FORMAT = "yyyy-MM-dd HH:mm";
             SimpleDateFormat dateFormat = new java.text.SimpleDateFormat(DATE_FORMAT);
@@ -204,9 +210,14 @@ public boolean validate(
 				userFunctions=PluginLoader.getInterlisFunctions(loader.getAllPlugins());
 				settings.setTransientObject(ch.interlis.iox_j.validator.Validator.CONFIG_CUSTOM_FUNCTIONS, userFunctions);
 			}
+            IoxLogging errHandler=new ch.interlis.iox_j.logging.Log2EhiLogger();
+            LogEventFactory errFactory=new LogEventFactory();
+            errFactory.setLogger(errHandler);
+
+			String modelVersion = IoxUtility.getModelVersion(dataFiles, errFactory);
 
 			// read ili models
-			td=compileIli(modelnames, null,new File(dataFiles[0]).getAbsoluteFile().getParentFile().getAbsolutePath(),appHome, settings);
+			td=compileIli(modelVersion,modelnames, null,new File(dataFiles[0]).getAbsoluteFile().getParentFile().getAbsolutePath(),appHome, settings);
 			if(td==null){
 				return false;
 			}
@@ -244,9 +255,6 @@ public boolean validate(
 				    }
 				}
 
-				IoxLogging errHandler=new ch.interlis.iox_j.logging.Log2EhiLogger();
-				LogEventFactory errFactory=new LogEventFactory();
-				errFactory.setLogger(errHandler);
 				PipelinePool pool=new PipelinePool();
 				validator=new ch.interlis.iox_j.validator.Validator(td,modelConfig, errHandler, errFactory, pool,settings);
 				validator.setAutoSecondPass(false);
@@ -321,6 +329,7 @@ public boolean validate(
 		return ret;
 	}
 
+
 	private static boolean isWriteable(File f) throws IOException {
         f.createNewFile();
         return f.canWrite();
@@ -382,6 +391,7 @@ private static List<String> getSpecifiedModelNames(String modelNames) {
 	}
 
 	/** Compiles the required Interlis models.
+	 * @param iliVersion version of required ili model (null, "1.0","2.2", "2.3", "2.4")
 	 * @param aclass Interlis qualified class name of a required class.
 	 * @param ilifile Interlis model file to read. null if not known.
 	 * @param itfDir Folder with Interlis model files or null.
@@ -390,7 +400,7 @@ private static List<String> getSpecifiedModelNames(String modelNames) {
 	 * @return root object of java representation of Interlis model.
 	 * @see #SETTING_ILIDIRS
 	 */
-	public static TransferDescription compileIli(List<String> modelNames,File ilifile,String itfDir,String appHome,Settings settings) {
+	public static TransferDescription compileIli(String iliVersion,List<String> modelNames,File ilifile,String itfDir,String appHome,Settings settings) {
 		ArrayList modeldirv=new ArrayList();
 		String ilidirs=settings.getValue(Validator.SETTING_ILIDIRS);
 		if(ilidirs==null){
@@ -453,10 +463,14 @@ public static TransferDescription compileIli(List<String> modelNames,File ilifil
 				modelv.addAll(modelNames);
 			}
 			try {
+			    double version=0.0;
+			    if(iliVersion!=null) {
+			        version=Double.parseDouble(iliVersion);
+			    }
 				//ili2cConfig=ch.interlis.ili2c.ModelScan.getConfig(modeldirv, modelv);
 				ch.interlis.ilirepository.IliManager modelManager=new ch.interlis.ilirepository.IliManager();
 				modelManager.setRepositories((String[])modeldirv.toArray(new String[]{}));
-				ili2cConfig=modelManager.getConfig(modelv, 0.0);
+				ili2cConfig=modelManager.getConfig(modelv, version);
 				ili2cConfig.setGenerateWarnings(false);
 			} catch (Ili2cException ex) {
 				EhiLogger.logError(ex);

test/data/ilirepository/Simple10a.itf

@@ -0,0 +1,11 @@
+SCNT
+////
+MTID INTERLIS1
+MODL Simple
+TOPI TestA
+TABL ClassA1
+OBJE 1 0
+ETAB
+ETOP
+EMOD
+ENDE

test/data/ilirepository/Simple22a.xtf

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<TRANSFER xmlns="http://www.interlis.ch/INTERLIS2.2">
+	<HEADERSECTION SENDER="ili2c-4.5.22-20160520" VERSION="2.3">
+		<MODELS>
+			<MODEL NAME="Simple" VERSION="2011-12-22" URI="http://interlis.ch"></MODEL>
+		</MODELS>
+	</HEADERSECTION>
+	<DATASECTION>
+		<Simple.TestA BID="xb1">
+			<Simple.TestA.ClassA1 TID="xo1">
+				<attr22>a22</attr22>
+			</Simple.TestA.ClassA1>
+		</Simple.TestA>
+	</DATASECTION>
+</TRANSFER>

test/data/ilirepository/Simple23a.xtf

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<TRANSFER xmlns="http://www.interlis.ch/INTERLIS2.3">
+	<HEADERSECTION SENDER="ili2c-4.5.22-20160520" VERSION="2.3">
+		<MODELS>
+			<MODEL NAME="Simple" VERSION="2011-12-22" URI="http://interlis.ch"></MODEL>
+		</MODELS>
+	</HEADERSECTION>
+	<DATASECTION>
+		<Simple.TestA BID="b1">
+			<Simple.TestA.ClassA1 TID="o1">
+				<attr23>a23</attr23>
+			</Simple.TestA.ClassA1>
+		</Simple.TestA>
+	</DATASECTION>
+</TRANSFER>

test/data/ilirepository/Simple24a.xtf

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ili:transfer xmlns="http://www.interlis.ch/xtf/2.4/Simple" xmlns:ili="http://www.interlis.ch/xtf/2.4/INTERLIS">
+	<ili:headersection SENDER="ili2c-4.5.22-20160520" VERSION="2.3">
+		<ili:models>
+			<ili:model>Simple</ili:model>
+		</ili:models>
+	</ili:headersection>
+	<ili:datasection>
+		<TestA ili:bid="b1">
+			<ClassA1 ili:tid="o1">
+				<attr24>a24</attr24>
+			</ClassA1>
+		</TestA>
+	</ili:datasection>
+</ili:transfer>

test/data/ilirepository/repos/Basis22-20200322.ili

@@ -0,0 +1,7 @@
+INTERLIS 2.2;
+
+MODEL Basis =
+  DOMAIN
+  	Domain22 = (a22);
+
+END Basis.

test/data/ilirepository/repos/Basis23-20200323.ili

@@ -0,0 +1,8 @@
+INTERLIS 2.3;
+
+MODEL Basis
+  AT "mailto:[email protected]" VERSION "2020-03-23" =
+  DOMAIN
+  	Domain23 = (a23);
+
+END Basis.

test/data/ilirepository/repos/Basis24-20200324.ili

@@ -0,0 +1,8 @@
+INTERLIS 2.4;
+
+MODEL Basis
+  AT "mailto:[email protected]" VERSION "2020-03-24" =
+  DOMAIN
+  	Domain24 = (a24);
+
+END Basis.

test/data/ilirepository/repos/Simple10-20200310.ili

@@ -0,0 +1,25 @@
+TRANSFER INTERLIS1;
+
+MODEL Simple
+
+  DOMAIN
+  	Domain10 = (a10);
+
+  TOPIC TestA =
+
+    TABLE ClassA1 =
+    	attr10 : Domain10;
+    	NO IDENT
+    END ClassA1;
+
+  END TestA.
+
+
+END Simple.
+
+FORMAT FREE;
+
+CODE
+  BLANK = DEFAULT, UNDEFINED = DEFAULT, CONTINUE = DEFAULT;
+  TID = ANY;
+END.

test/data/ilirepository/repos/Simple22-20200322.ili

@@ -0,0 +1,15 @@
+INTERLIS 2.2;
+
+MODEL Simple =
+  IMPORTS Basis;
+
+  TOPIC TestA =
+
+    CLASS ClassA1 =
+    	attr22 : Basis.Domain22;
+    END ClassA1;
+
+  END TestA;
+
+
+END Simple.

test/data/ilirepository/repos/Simple23-20200323.ili

@@ -0,0 +1,16 @@
+INTERLIS 2.3;
+
+MODEL Simple
+  AT "mailto:[email protected]" VERSION "2020-03-23" =
+  IMPORTS Basis;
+
+  TOPIC TestA =
+
+    CLASS ClassA1 =
+    	attr23 : Basis.Domain23;
+    END ClassA1;
+
+  END TestA;
+
+
+END Simple.

test/data/ilirepository/repos/Simple24-20200324.ili

@@ -0,0 +1,16 @@
+INTERLIS 2.4;
+
+MODEL Simple
+  AT "mailto:[email protected]" VERSION "2020-03-24" =
+  IMPORTS Basis;
+
+  TOPIC TestA =
+
+    CLASS ClassA1 =
+    	attr24 : Basis.Domain24;
+    END ClassA1;
+
+  END TestA;
+
+
+END Simple.