Skip to content

Commit

Permalink
HPCC4J-657 DFSClient: Reading indices with blobs fails
Browse files Browse the repository at this point in the history 
- Updated RecordDefinitionTranslator to maintain blob information
- Added isBlob property to FieldDef
- Added translation test for indices with blobs
- Added index read test for indices with blobs
- Updated data generation to create an index with blobs

Signed-off-by: James McMullan [email protected]
  • Loading branch information
@jpmcmu
jpmcmu committed Oct 24, 2024
1 parent 6fdaf72 commit 31344aa
Show file tree
Hide file tree
Showing 7 changed files with 171 additions and 26 deletions.
20 changes: 20 additions & 0 deletions commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/FieldDef.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ public class FieldDef implements Serializable
private long len = 0;
private boolean fixedLength = false;
private boolean isUnsigned = false;
private boolean isBlob = false;
private int additionalFlags = 0;

/**
Expand All @@ -59,6 +60,7 @@ public FieldDef(FieldDef rhs)
this.len = rhs.len;
this.fixedLength = rhs.fixedLength;
this.isUnsigned = rhs.isUnsigned;
this.isBlob = rhs.isBlob;
this.additionalFlags = rhs.additionalFlags;
}

Expand Down Expand Up @@ -330,6 +332,24 @@ public boolean isBiased()
return isNonStandardInt();
}

/**
* Is the field stored in a blob?
*
* @return true when blob
*/
public boolean isBlob()
{
return this.isBlob;
}

/**
* Sets the blob flag.
*/
public void setIsBlob(boolean blob)
{
this.isBlob = blob;
}

/**
*
*
Expand Down
104 changes: 79 additions & 25 deletions commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/RecordDefinitionTranslator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ public class RecordDefinitionTranslator
private static final String NAME_KEY = "name";
private static final String TYPE_KEY = "type";
private static final String CHILD_KEY = "child";
private static final String XPATH_KEY = "xpath";
private static final String FLAGS_KEY = "flags";

private static final String ESP_TYPE_NAME_PREFIX = "ty";
Expand All @@ -48,6 +49,7 @@ public class RecordDefinitionTranslator
final private static int type_keyedint = 10; // Convert to integer
final private static int type_record = 13;
final private static int type_varstring = 14;
final private static int type_blob = 15;
final private static int type_data = 16;
final private static int type_table = 20;
final private static int type_set = 21;
Expand Down Expand Up @@ -264,28 +266,32 @@ public static String toECLRecord(FieldDef field) throws Exception
*/
private static String getEClTypeDefinition(FieldDef field, HashMap<String, String> recordDefinitionMap) throws Exception
{
String type = "";
switch (field.getFieldType())
{
case SET:
{
return "SET OF " + getEClTypeDefinition(field.getDef(0), recordDefinitionMap);
type = "SET OF " + getEClTypeDefinition(field.getDef(0), recordDefinitionMap);
break;
}
case DATASET:
{
return "DATASET(" + getEClTypeDefinition(field.getDef(0), recordDefinitionMap) + ")";
type = "DATASET(" + getEClTypeDefinition(field.getDef(0), recordDefinitionMap) + ")";
break;
}
case BINARY:
{
type = "DATA";
if (field.isFixed())
{
return "DATA" + field.getDataLen();
type += field.getDataLen();
}

return "DATA";
break;
}
case BOOLEAN:
{
return "BOOLEAN";
type = "BOOLEAN";
break;
}
case INTEGER:
{
Expand All @@ -300,7 +306,8 @@ private static String getEClTypeDefinition(FieldDef field, HashMap<String, Strin
throw new Exception("Error: Unsupported integer size: " + field.getDataLen() + " must 1-8.");
}

return root + field.getDataLen();
type = root + field.getDataLen();
break;
}
case FILEPOS:
{
Expand All @@ -314,7 +321,8 @@ private static String getEClTypeDefinition(FieldDef field, HashMap<String, Strin
throw new Exception("Error: Unsupported filepos size: " + field.getDataLen() + " must be 8.");
}

return "UNSIGNED8";
type = "UNSIGNED8";
break;
}
case DECIMAL:
{
Expand All @@ -323,28 +331,33 @@ private static String getEClTypeDefinition(FieldDef field, HashMap<String, Strin
{
root = "U" + root;
}
return root + field.getPrecision() + "_" + field.getScale();
type = root + field.getPrecision() + "_" + field.getScale();
break;
}
case REAL:
{
if (field.getDataLen() == 4)
{
return "REAL4";
type = "REAL4";
}
else if (field.getDataLen() == 8)
{
return "REAL8";
type = "REAL8";
}
else
{
throw new Exception("Error: Unsupported real size: " + field.getDataLen() + " must 4 or 8.");
}
break;

throw new Exception("Error: Unsupported real size: " + field.getDataLen() + " must 4 or 8.");
}
case CHAR:
{
return "STRING1";
type = "STRING1";
break;
}
case STRING:
{
String type = "";
HpccSrcType srcType = field.getSourceType();
if (srcType == HpccSrcType.SINGLE_BYTE_CHAR)
{
Expand All @@ -371,11 +384,10 @@ else if (srcType == HpccSrcType.QSTRING)
{
type += field.getDataLen();
}
return type;
break;
}
case VAR_STRING:
{
String type = "";
HpccSrcType srcType = field.getSourceType();
if (srcType == HpccSrcType.SINGLE_BYTE_CHAR)
{
Expand All @@ -394,7 +406,7 @@ else if (srcType == HpccSrcType.UTF16LE || srcType == HpccSrcType.UTF16BE)
{
type += field.getDataLen();
}
return type;
break;
}
case RECORD:
{
Expand All @@ -416,13 +428,21 @@ else if (srcType == HpccSrcType.UTF16LE || srcType == HpccSrcType.UTF16BE)
String recordDefnName = "##" + hash + "##";

recordDefinitionMap.put(recordDefnName, definition);
return recordDefnName;
type = recordDefnName;
break;
}
default:
{
throw new Exception("Unable to generate ECL unknown field type: " + field.getFieldType().description());
}
}

if (field.isBlob())
{
type += " {blob}";
}

return type;
}

/**
Expand Down Expand Up @@ -648,7 +668,7 @@ else if (srcType == HpccSrcType.UTF16LE || srcType == HpccSrcType.UTF16BE)
*/
private static int getTypeHash(FieldDef field) throws Exception
{
int numHashComponents = 4 + field.getNumDefs();
int numHashComponents = 5 + field.getNumDefs();
if (field.getFieldType() == FieldType.DECIMAL)
{
numHashComponents += 2;
Expand All @@ -659,8 +679,9 @@ private static int getTypeHash(FieldDef field) throws Exception
hashComponents[1] = field.getDataLen();
hashComponents[2] = field.getSourceType().ordinal();
hashComponents[3] = field.getAdditionalFlags();
hashComponents[4] = field.isBlob() ? 1 : 0;

int hashCompIndex = 4;
int hashCompIndex = 5;
for (int i = 0; i < field.getNumDefs(); i++, hashCompIndex++)
{
hashComponents[hashCompIndex] = getTypeHash(field.getDef(i));
Expand Down Expand Up @@ -698,6 +719,26 @@ private static int getJsonTypeDefinition(FieldDef field, HashMap<Integer, Intege
return typeHash;
}

if (field.isBlob())
{
FieldDef nonBlobField = new FieldDef(field);
nonBlobField.setIsBlob(false);

int nonBlobTypeHash = getJsonTypeDefinition(nonBlobField, typeDefinitionMap, typeDefinitions);
int nonBlobTypeIndex = typeDefinitionMap.get(nonBlobTypeHash);
String nonBlobTypeName = ESP_TYPE_NAME_PREFIX + (nonBlobTypeIndex + 1);

JSONObject typeDef = new JSONObject();
typeDef.put("fieldType", type_blob);
typeDef.put("length", 8);
typeDef.put("child", nonBlobTypeName);

int newTypeIndex = typeDefinitions.size();
typeDefinitions.add(typeDef);
typeDefinitionMap.put(typeHash, newTypeIndex);
return typeHash;
}

JSONObject typeDef = new JSONObject();
int typeID = getTypeID(field);
switch (field.getFieldType())
Expand Down Expand Up @@ -753,25 +794,30 @@ private static int getJsonTypeDefinition(FieldDef field, HashMap<Integer, Intege
int childTypeHash = getJsonTypeDefinition(childField, typeDefinitionMap, typeDefinitions);
int childTypeIndex = typeDefinitionMap.get(childTypeHash);
String childTypeName = ESP_TYPE_NAME_PREFIX + (childTypeIndex + 1);

int childTypeID = getTypeID(childField);
if (childField.isBlob())
{
childTypeID = type_blob;
}

JSONObject childJson = new JSONObject();
childJson.put("name", childField.getFieldName());
childJson.put("type", childTypeName);
childJson.put(NAME_KEY, childField.getFieldName());
childJson.put(TYPE_KEY, childTypeName);

int flags = childTypeID | childField.getAdditionalFlags();
if (flags > 0)
{
childJson.put("flags", flags);
childJson.put(FLAGS_KEY, flags);
}

if (childField.getFieldType() == FieldType.DATASET)
{
childJson.put("xpath", childField.getFieldName() + XPATH_DELIMITER + "Row");
childJson.put(XPATH_KEY, childField.getFieldName() + XPATH_DELIMITER + "Row");
}
else if (childField.getFieldType() == FieldType.SET)
{
childJson.put("xpath", childField.getFieldName() + XPATH_DELIMITER + "Item");
childJson.put(XPATH_KEY, childField.getFieldName() + XPATH_DELIMITER + "Item");
}

fields.put(childJson);
Expand Down Expand Up @@ -954,6 +1000,14 @@ private static FieldDef parseJsonTypeDefinition(JSONObject jsonTypeDefinitions,
int typeID = typeDef.getInt(FIELD_TYPE_KEY);
long length = typeDef.getLong(LENGTH_KEY);

if (typeID == type_blob)
{
String blobType = typeDef.getString(CHILD_KEY);
FieldDef def = getOrParseJsonTypeDefintion(blobType, jsonTypeDefinitions, protoTypeDefs);
def.setIsBlob(true);
return def;
}

FieldType fieldType = getFieldType(typeID);
switch (fieldType)
{
Expand Down
21 changes: 21 additions & 0 deletions commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/TestFieldDefinitions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,22 @@ public class TestFieldDefinitions
+ " \"name\": \"isactive\",\r\n \"type\": \"ty15\",\r\n \"flags\": 65536\r\n },\r\n {\r\n \"name\": \"__internal_fpos__\",\r\n \"type\": \"ty16\",\r\n"
+ " \"flags\": 65821\r\n }\r\n ]\r\n}";

private static final String blobIndexDefinitionStr = "{\n \"ty6\": { \"vinit\": 2, \"length\": 8, \"fieldType\": 285 },\n"
+ " \"ty5\": { \"length\": 8, \"fieldType\": 15, \"child\": \"ty4\" },\n"
+ " \"length\": 36,\n"
+ " \"ty2\": { \"length\": 0, \"fieldType\": 1028 },\n"
+ " \"fields\": [\n"
+ " { \"name\": \"str12\", \"flags\": 4, \"type\": \"ty1\" },\n"
+ " { \"name\": \"content_string\", \"flags\": 65551, \"type\": \"ty3\" },\n"
+ " { \"name\": \"content_data\", \"flags\": 65551, \"type\": \"ty5\" },\n"
+ " { \"name\": \"__internal_fpos__\", \"flags\": 65821, \"type\": \"ty6\" }\n"
+ " ],\n"
+ " \"ty1\": { \"length\": 12, \"fieldType\": 4 },\n"
+ " \"fieldType\": 13,\n"
+ " \"ty4\": { \"length\": 0, \"fieldType\": 1040 },\n"
+ " \"ty3\": { \"length\": 8, \"fieldType\": 15, \"child\": \"ty2\" }\n"
+ "}\n";

/**
* Gets the complex record definition json.
*
Expand All @@ -79,6 +95,11 @@ public static String getAllTypesIndexRecordDefinitionJson()
return allTypesIndexRecordDefinitionStr;
}

public static String getBlobIndexDefinitionJson()
{
return blobIndexDefinitionStr;
}

/**
* Gets the complex record definition.
*
Expand Down
3 changes: 2 additions & 1 deletion commons-hpcc/src/test/java/org/hpccsystems/commons/ecl/RecordDefinitionTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,8 @@ public void setup()
public void testJsonRecordParsing() throws Exception
{
String[] recordDefStrs = new String[] { TestFieldDefinitions.getComplexRecordDefinitionJson(),
TestFieldDefinitions.getAllTypesIndexRecordDefinitionJson() };
TestFieldDefinitions.getAllTypesIndexRecordDefinitionJson(),
TestFieldDefinitions.getBlobIndexDefinitionJson() };
for (String recordDefStr : recordDefStrs)
{
try
Expand Down
4 changes: 4 additions & 0 deletions dfsclient/src/main/java/org/hpccsystems/dfs/client/HPCCFile.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,10 @@ private void updateProjectedRecordDef() throws Exception
field.setSourceType(HpccSrcType.LITTLE_ENDIAN);
}

// if (field.isBlob())
// {
// field.setIsBlob(false);
// }
}
}

Expand Down
21 changes: 21 additions & 0 deletions dfsclient/src/test/java/org/hpccsystems/dfs/client/DFSIndexTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,27 @@ public void biasedIntTest() throws Exception
}
}

@Test
public void indexBlobTest() throws Exception
{
HPCCFile file = new HPCCFile("~test::index::blobs::key", connString , hpccUser, hpccPass);

DataPartition[] fileParts = file.getFileParts();

List<HPCCRecord> records = new ArrayList<HPCCRecord>();
FieldDef originalRD = file.getRecordDefinition();
for (int j = 0; j < fileParts.length; j++)
{
HPCCRecordBuilder recordBuilder = new HPCCRecordBuilder(file.getProjectedRecordDefinition());
HpccRemoteFileReader<HPCCRecord> fileReader = new HpccRemoteFileReader<HPCCRecord>(fileParts[j], originalRD, recordBuilder);
while (fileReader.hasNext())
{
records.add(fileReader.next());
}
fileReader.close();
}
}

private String partitionListToString(List<DataPartition> partitions)
{
String matchedPartitionStr = "[ ";
Expand Down
Loading

0 comments on commit 31344aa

Please sign in to comment.