[SYSTEMDS-3135] Reader Generation for Custom Text Formats

This commit contains the the updated implementation of GIO,
a system for generating efficient matrix and frame readers
for custom data formats by example. Our GIO paper was published
in SIGMOD 2023. The main improvements and extensions over the
previous implementation are

1. Support for both frame and matrix reader generation
2. Support for projection over the selected fields
3. Improved code generation that adapts to single/multi-threaded configuration
4. Support for single/multi-line record representations

Closes #1865

pom.xml

@@ -634,6 +634,9 @@
 								<exclude>src/test/scripts/functions/jmlc/tfmtd_frame_example/tfmtd_frame</exclude>
 								<!-- csv test input not captured by *.csv -->
 								<exclude>src/test/scripts/functions/io/csv/in/*/*</exclude>
+								<!-- IOGEN input examples -->
+								<exclude>src/test/scripts/functions/iogen/in/*/*/*</exclude>
+								<exclude>src/test/scripts/functions/iogen/in/*/*</exclude>
 								<!-- Python bindings lineage test result comparison -->
 								<exclude>src/main/python/tests/lt*.txt</exclude>
 								<!-- Perftest requirement file -->

src/main/java/org/apache/sysds/runtime/iogen/ColIndexStructure.java

@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.runtime.iogen;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+
+public class ColIndexStructure {
+
+	public enum IndexProperties {
+		Identity, // col number of sample raw data equal to the row index of matrix/frame
+		CellWiseExist; // col index of every cell values are in the sample raw data
+		@Override
+		public String toString() {
+			return this.name().toUpperCase();
+		}
+	}
+
+	public ColIndexStructure() {
+		this.properties = null;
+		this.keyPattern = null;
+		this.colIndexBegin = 0;
+	}
+
+	private IndexProperties properties;
+
+	private HashSet<String> endWithValueString = null;
+	private ArrayList<String> keyPattern;
+	private int colIndexBegin;
+
+	// when the index properties is CellWiseExist:
+	private String indexDelim;
+	private String valueDelim;
+
+	public HashSet<String> endWithValueStrings() {
+		return  endWithValueString;
+	}
+
+	public IndexProperties getProperties() {
+		return properties;
+	}
+
+	public void setProperties(IndexProperties properties) {
+		this.properties = properties;
+	}
+
+	public ArrayList<String> getKeyPattern() {
+		return keyPattern;
+	}
+
+	public void setKeyPattern(ArrayList<String> keyPattern) {
+		this.keyPattern = keyPattern;
+	}
+
+	public int getColIndexBegin() {
+		return colIndexBegin;
+	}
+
+	public void setColIndexBegin(int colIndexBegin) {
+		this.colIndexBegin = colIndexBegin;
+	}
+
+	public String getIndexDelim() {
+		return indexDelim;
+	}
+
+	public void setIndexDelim(String indexDelim) {
+		this.indexDelim = indexDelim;
+	}
+
+	public String getValueDelim() {
+		return valueDelim;
+	}
+
+	public void setValueDelim(String valueDelim) {
+		this.valueDelim = valueDelim;
+	}
+
+	public HashSet<String> endWithValueString() {
+		return endWithValueString;
+	}
+
+	public void setEndWithValueString(HashSet<String> endWithValueString) {
+		this.endWithValueString = endWithValueString;
+	}
+}

src/main/java/org/apache/sysds/runtime/iogen/CustomProperties.java

@@ -19,117 +19,113 @@
 
 package org.apache.sysds.runtime.iogen;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import org.apache.sysds.common.Types;
 import org.apache.sysds.runtime.io.FileFormatProperties;
-
 import java.io.Serializable;
+import java.util.ArrayList;
 import java.util.HashSet;
 
 public class CustomProperties extends FileFormatProperties implements Serializable {
-	protected static final Log LOG = LogFactory.getLog(CustomProperties.class.getName());
-	private static final long serialVersionUID = -4447926749068752721L;
 
-	private String delim;
-	private String indexDelim;
-	private HashSet<String> naStrings;
-	private int firstColIndex;
-	private int firstRowIndex;
+	private MappingProperties mappingProperties;
+	private RowIndexStructure rowIndexStructure;
+	private ColIndexStructure colIndexStructure;
+	private HashSet<String>[] endWithValueStrings;
+	private ArrayList<String>[] colKeyPatterns;
+	private ArrayList<String> valueKeyPattern;
+	private Types.ValueType[] schema;
 
-	protected enum GRPattern {
-		Regular, Irregular;
+	private int ncols;
+	private boolean sparse;
+	private boolean parallel;
 
-		@Override
-		public String toString() {
-			return this.name().toLowerCase();
-		}
+	public CustomProperties(MappingProperties mappingProperties, RowIndexStructure rowIndexStructure,
+		ColIndexStructure colIndexStructure) {
+		this.mappingProperties = mappingProperties;
+		this.rowIndexStructure = rowIndexStructure;
+		this.colIndexStructure = colIndexStructure;
 	}
 
-	protected enum GRSymmetry {
-		GENERAL, SYMMETRIC, SKEW_SYMMETRIC;
+	public MappingProperties getMappingProperties() {
+		return mappingProperties;
+	}
 
-		@Override
-		public String toString() {
-			return this.name().toLowerCase().replaceAll("_", "-");
-		}
+	public void setMappingProperties(MappingProperties mappingProperties) {
+		this.mappingProperties = mappingProperties;
 	}
 
-	private GRPattern rowPattern;
-	private GRPattern colPattern;
-	private GRSymmetry grSymmetry;
+	public RowIndexStructure getRowIndexStructure() {
+		return rowIndexStructure;
+	}
+
+	public void setRowIndexStructure(RowIndexStructure rowIndexStructure) {
+		this.rowIndexStructure = rowIndexStructure;
+	}
+
+	public ColIndexStructure getColIndexStructure() {
+		return colIndexStructure;
+	}
+
+	public void setColIndexStructure(ColIndexStructure colIndexStructure) {
+		this.colIndexStructure = colIndexStructure;
+	}
 
-	public CustomProperties() {
+	public ArrayList<String>[] getColKeyPatterns() {
+		return colKeyPatterns;
 	}
 
-	// Row & Col Regular Format
-	public CustomProperties(GRPattern rowPattern, String delim, HashSet<String> naStrings) {
-		this.delim = delim;
-		this.naStrings = naStrings;
-		this.rowPattern = rowPattern;
-		this.colPattern = GRPattern.Regular;
-		this.grSymmetry = GRSymmetry.GENERAL;
-		this.firstRowIndex = 0;
-		this.firstColIndex = 0;
+	public void setColKeyPatterns(ArrayList<String>[] colKeyPatterns) {
+		this.colKeyPatterns = colKeyPatterns;
 	}
 
-	// Row Regular & Col Irregular Format
-	public CustomProperties(GRPattern rowPattern, String delim, String indexDelim, int firstColIndex) {
-		this.delim = delim;
-		this.indexDelim = indexDelim;
-		this.rowPattern = rowPattern;
-		this.colPattern = GRPattern.Irregular;
-		this.grSymmetry = GRSymmetry.GENERAL;
-		this.firstColIndex = firstColIndex;
-		this.firstRowIndex = 0;
+	public Types.ValueType[] getSchema() {
+		return schema;
 	}
 
-	// Row Irregular format
-	public CustomProperties(GRSymmetry grSymmetry, String delim, int firstRowIndex, int firstColIndex) {
-		this.delim = delim;
-		this.grSymmetry = grSymmetry;
-		this.colPattern = GRPattern.Regular;
-		this.rowPattern = GRPattern.Irregular;
-		this.firstColIndex = firstColIndex;
-		this.firstRowIndex = firstRowIndex;
+	public void setSchema(Types.ValueType[] schema) {
+		this.schema = schema;
 	}
 
-	public String getDelim() {
-		return delim;
+	public HashSet<String>[] endWithValueStrings() {
+		if(endWithValueStrings !=null)
+			return this.endWithValueStrings;
+		else
+			return null;
 	}
 
-	public String getIndexDelim() {
-		return indexDelim;
+	public ArrayList<String> getValueKeyPattern() {
+		return valueKeyPattern;
 	}
 
-	public HashSet<String> getNaStrings() {
-		return naStrings;
+	public void setValueKeyPattern(ArrayList<String> valueKeyPattern) {
+		this.valueKeyPattern = valueKeyPattern;
 	}
 
-	public GRPattern getRowPattern() {
-		return rowPattern;
+	public int getNcols() {
+		return ncols;
 	}
 
-	public GRPattern getColPattern() {
-		return colPattern;
+	public void setNcols(int ncols) {
+		this.ncols = ncols;
 	}
 
-	public GRSymmetry getGrSymmetry() {
-		return grSymmetry;
+	public boolean isSparse() {
+		return sparse;
 	}
 
-	public int getFirstColIndex() {
-		return firstColIndex;
+	public void setSparse(boolean sparse) {
+		this.sparse = sparse;
 	}
 
-	public void setFirstColIndex(int firstColIndex) {
-		this.firstColIndex = firstColIndex;
+	public boolean isParallel() {
+		return parallel;
 	}
 
-	public int getFirstRowIndex() {
-		return firstRowIndex;
+	public void setParallel(boolean parallel) {
+		this.parallel = parallel;
 	}
 
-	public void setFirstRowIndex(int firstRowIndex) {
-		this.firstRowIndex = firstRowIndex;
+	public void setEndWithValueStrings(HashSet<String>[] endWithValueStrings) {
+		this.endWithValueStrings = endWithValueStrings;
 	}
 }