Skip to content

Commit

Permalink
HPCC4J-650 DFSClient should retain integer subtypes
Browse files Browse the repository at this point in the history 
- Modified record translation code to maintain integer subtype
- Fixed record translation test case
- Added index record translation to test case

Signed-off-by: James McMullan [email protected]
  • Loading branch information
@jpmcmu
jpmcmu committed Sep 24, 2024
1 parent 434557e commit 168d835
Show file tree
Hide file tree
Showing 4 changed files with 76 additions and 22 deletions.
6 changes: 6 additions & 0 deletions commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/HpccSrcType.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,12 @@ public enum HpccSrcType
"little endian", false
), BINARY_CODED_DECIMAL (
"Binary coded decimal", false
), KEYED_INTEGER (
"Non-payload integer field within a key", false
), SWAPPED_INTEGER (
"Byte swapped integer, used within keys", false
), BIAS_SWAPPED_INTEGER (
"Byte swapped integer, deprecated", false
), UNKNOWN (
"Unkown", false
);
Expand Down
27 changes: 24 additions & 3 deletions commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/RecordDefinitionTranslator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,9 +134,12 @@ private static HpccSrcType getSourceType(int typeID)
case type_int:
case type_real:
return HpccSrcType.LITTLE_ENDIAN;
case type_swapint:
case type_biasedswapint:
return HpccSrcType.BIAS_SWAPPED_INTEGER;
case type_swapint:
return HpccSrcType.SWAPPED_INTEGER;
case type_keyedint:
return HpccSrcType.KEYED_INTEGER;
case type_filepos:
return HpccSrcType.BIG_ENDIAN;
case type_utf8:
Expand Down Expand Up @@ -272,7 +275,7 @@ private static String getEClTypeDefinition(FieldDef field, HashMap<String, Strin
{
return "DATA" + field.getDataLen();
}

return "DATA";
}
case BOOLEAN:
Expand All @@ -298,7 +301,7 @@ private static String getEClTypeDefinition(FieldDef field, HashMap<String, Strin
{
if (field.isUnsigned() == false)
{
throw new Exception("Error: Filepos must be unsigned");
throw new Exception("Error: Filepos must be unsigned");
}

if (field.getDataLen() != 8)
Expand Down Expand Up @@ -518,6 +521,20 @@ private static int getTypeID(FieldDef field) throws Exception
{
typeID |= FLAG_UNSIGNED;
}

HpccSrcType srcType = field.getSourceType();
if (srcType == HpccSrcType.KEYED_INTEGER)
{
typeID = type_keyedint;
}
else if (srcType == HpccSrcType.SWAPPED_INTEGER)
{
typeID = type_swapint;
}
else if (srcType == HpccSrcType.BIAS_SWAPPED_INTEGER)
{
typeID = type_biasedswapint;
}
break;
}
case DECIMAL:
Expand Down Expand Up @@ -743,6 +760,10 @@ private static int getJsonTypeDefinition(FieldDef field, HashMap<Integer, Intege
{
char delim = 0x0001;
childJson.put("xpath", childField.getFieldName() + delim + "Row");
} else if (childField.getFieldType() == FieldType.SET)
{
char delim = 0x0001;
childJson.put("xpath", childField.getFieldName() + delim + "Item");
}

fields.put(childJson);
Expand Down
23 changes: 23 additions & 0 deletions commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/TestFieldDefinitions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,24 @@ public class TestFieldDefinitions
+ "\"type\": \"ty17\",\r\n \"xpath\": \"childdataset\\u0001Row\",\r\n \"flags\": 1044\r\n },\r\n {\r\n \"name\": \"int2set\",\r\n "
+ "\"type\": \"ty19\",\r\n \"xpath\": \"int2set\\u0001Item\",\r\n \"flags\": 1045\r\n }\r\n ]\r\n}";

private static final String allTypesIndexRecordDefinitionStr = "{\r\n \"ty1\": {\r\n \"fieldType\": 1,\r\n \"length\": 8\r\n },\r\n \"ty2\":"
+ " {\r\n \"fieldType\": 10,\r\n \"length\": 8,\r\n \"child\": \"ty1\"\r\n },\r\n \"ty3\": {\r\n \"fieldType\": 257,\r\n \"length\":"
+ " 8\r\n },\r\n \"ty4\": {\r\n \"fieldType\": 266,\r\n \"length\": 8,\r\n \"child\": \"ty3\"\r\n },\r\n \"ty5\": {\r\n \"fieldType\": "
+ "1,\r\n \"length\": 4\r\n },\r\n \"ty6\": {\r\n \"fieldType\": 10,\r\n \"length\": 4,\r\n \"child\": \"ty5\"\r\n },\r\n \"ty7\": {\r\n "
+ " \"fieldType\": 257,\r\n \"length\": 4\r\n },\r\n \"ty8\": {\r\n \"fieldType\": 266,\r\n \"length\": 4,\r\n \"child\": \"ty7\"\r\n },\r\n"
+ " \"ty9\": {\r\n \"fieldType\": 1,\r\n \"length\": 2\r\n },\r\n \"ty10\": {\r\n \"fieldType\": 10,\r\n \"length\": 2,\r\n \"child\": "
+ "\"ty9\"\r\n },\r\n \"ty11\": {\r\n \"fieldType\": 257,\r\n \"length\": 2\r\n },\r\n \"ty12\": {\r\n \"fieldType\": 266,\r\n \"length\": "
+ "2,\r\n \"child\": \"ty11\"\r\n },\r\n \"ty13\": {\r\n \"fieldType\": 259,\r\n \"length\": 524304\r\n },\r\n \"ty14\": {\r\n \"fieldType\":"
+ " 4,\r\n \"length\": 8\r\n },\r\n \"ty15\": {\r\n \"fieldType\": 0,\r\n \"length\": 1\r\n },\r\n \"ty16\": {\r\n \"fieldType\": 285,\r\n "
+ "\"length\": 8\r\n },\r\n \"fieldType\": 13,\r\n \"length\": 61,\r\n \"fields\": [\r\n {\r\n \"name\": \"int8\",\r\n \"type\": \"ty2\",\r\n"
+ " \"flags\": 10\r\n },\r\n {\r\n \"name\": \"uint8\",\r\n \"type\": \"ty4\",\r\n \"flags\": 266\r\n },\r\n {\r\n \"name\": \"int4\","
+ "\r\n \"type\": \"ty6\",\r\n \"flags\": 10\r\n },\r\n {\r\n \"name\": \"uint4\",\r\n \"type\": \"ty8\",\r\n \"flags\": 266\r\n },\r\n"
+ " {\r\n \"name\": \"int2\",\r\n \"type\": \"ty10\",\r\n \"flags\": 10\r\n },\r\n {\r\n \"name\": \"uint2\",\r\n \"type\": \"ty12\",\r\n"
+ " \"flags\": 266\r\n },\r\n {\r\n \"name\": \"udec16\",\r\n \"type\": \"ty13\",\r\n \"flags\": 259\r\n },\r\n {\r\n \"name\": \"fixstr8\",\r\n"
+ " \"type\": \"ty14\",\r\n \"flags\": 4\r\n },\r\n {\r\n \"name\": \"recptr\",\r\n \"type\": \"ty4\",\r\n \"flags\": 266\r\n },\r\n {\r\n"
+ " \"name\": \"isactive\",\r\n \"type\": \"ty15\",\r\n \"flags\": 65536\r\n },\r\n {\r\n \"name\": \"__internal_fpos__\",\r\n \"type\": \"ty16\",\r\n"
+ " \"flags\": 65821\r\n }\r\n ]\r\n}";

/**
* Gets the complex record definition json.
*
Expand All @@ -56,6 +74,11 @@ public static String getComplexRecordDefinitionJson()
return complexRecordDefinitionStr;
}

public static String getAllTypesIndexRecordDefinitionJson()
{
return allTypesIndexRecordDefinitionStr;
}

/**
* Gets the complex record definition.
*
Expand Down
42 changes: 23 additions & 19 deletions commons-hpcc/src/test/java/org/hpccsystems/commons/ecl/RecordDefinitionTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,30 +38,34 @@ public void setup()
@Test
public void testJsonRecordParsing() throws Exception
{
String jsonRecDefStr = TestFieldDefinitions.getComplexRecordDefinitionJson();
try
String[] recordDefStrs = new String[] { TestFieldDefinitions.getComplexRecordDefinitionJson(),
TestFieldDefinitions.getAllTypesIndexRecordDefinitionJson() };
for (String recordDefStr : recordDefStrs)
{
JSONObject jsonRecDef = new JSONObject(jsonRecDefStr);
FieldDef recDef = RecordDefinitionTranslator.parseJsonRecordDefinition(jsonRecDef);
try
{
JSONObject expectedRefDef = new JSONObject(recordDefStr);

JSONObject tJsonRecDef = RecordDefinitionTranslator.toJsonRecord(recDef);
FieldDef recDef = RecordDefinitionTranslator.parseJsonRecordDefinition(expectedRefDef);
JSONObject actualRecDef = RecordDefinitionTranslator.toJsonRecord(recDef);

if (tJsonRecDef.equals(jsonRecDef))
// Use similar instead of equals because the order of fields in the JSON object may differ
if (!actualRecDef.similar(expectedRefDef))
{
System.out.println("Expected Record Def: " + expectedRefDef + "\n");
System.out.println("Actual Record Def: " + actualRecDef + "\n");
Assert.fail("Translated JSON record definition differs from original");
}
}
catch (UnparsableContentException e)
{
System.out.println(jsonRecDef + "\n");
System.out.println(tJsonRecDef + "\n");
Assert.fail("Translated JSON record definition differs from original");
Assert.fail("Encountered invalid record definition: '" + recordDefStr + "'");
}
catch (Exception e)
{
System.out.println(e.getMessage());
Assert.fail("Exception occurred: '" + e.getMessage() + "'");
}
}
catch (UnparsableContentException e)
{
Assert.fail("Encountered invalid record definition: '" + jsonRecDefStr + "'");
}
catch (Exception e)
{
System.out.println(e.getMessage());
Assert.fail("Exception occurred: '" + e.getMessage() + "'");
}

}
Expand Down

0 comments on commit 168d835

Please sign in to comment.