From 168d8355c9922f439bfe1bb5307a0be5b10fd500 Mon Sep 17 00:00:00 2001
From: James McMullan <jpmcmu@gmail.com>
Date: Mon, 23 Sep 2024 21:58:42 -0400
Subject: [PATCH] HPCC4J-650 DFSClient should retain integer subtypes

- Modified record translation code to maintain integer subtype
- Fixed record translation test case
- Added index record translation to test case

Signed-off-by: James McMullan James.McMullan@lexisnexis.com
---
 .../hpccsystems/commons/ecl/HpccSrcType.java  |  6 +++
 .../ecl/RecordDefinitionTranslator.java       | 27 ++++++++++--
 .../commons/ecl/TestFieldDefinitions.java     | 23 ++++++++++
 .../commons/ecl/RecordDefinitionTests.java    | 42 ++++++++++---------
 4 files changed, 76 insertions(+), 22 deletions(-)

diff --git a/commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/HpccSrcType.java b/commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/HpccSrcType.java
index 678c90de1..e0615fc36 100644
--- a/commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/HpccSrcType.java
+++ b/commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/HpccSrcType.java
@@ -36,6 +36,12 @@ public enum HpccSrcType
             "little endian", false
     ), BINARY_CODED_DECIMAL (
             "Binary coded decimal", false
+    ), KEYED_INTEGER (
+            "Non-payload integer field within a key", false
+    ), SWAPPED_INTEGER (
+            "Byte swapped integer, used within keys", false
+    ), BIAS_SWAPPED_INTEGER (
+            "Byte swapped integer, deprecated", false
     ), UNKNOWN (
             "Unkown", false
     );
diff --git a/commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/RecordDefinitionTranslator.java b/commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/RecordDefinitionTranslator.java
index 7db62895c..543e9fe0a 100644
--- a/commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/RecordDefinitionTranslator.java
+++ b/commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/RecordDefinitionTranslator.java
@@ -134,9 +134,12 @@ private static HpccSrcType getSourceType(int typeID)
             case type_int:
             case type_real:
                 return HpccSrcType.LITTLE_ENDIAN;
-            case type_swapint:
             case type_biasedswapint:
+                return HpccSrcType.BIAS_SWAPPED_INTEGER;
+            case type_swapint:
+                return HpccSrcType.SWAPPED_INTEGER;
             case type_keyedint:
+                return HpccSrcType.KEYED_INTEGER;
             case type_filepos:
                 return HpccSrcType.BIG_ENDIAN;
             case type_utf8:
@@ -272,7 +275,7 @@ private static String getEClTypeDefinition(FieldDef field, HashMap<String, Strin
                 {
                     return "DATA" + field.getDataLen();
                 }
-                
+
                 return "DATA";
             }
             case BOOLEAN:
@@ -298,7 +301,7 @@ private static String getEClTypeDefinition(FieldDef field, HashMap<String, Strin
             {
                 if (field.isUnsigned() == false)
                 {
-                    throw new Exception("Error: Filepos must be unsigned"); 
+                    throw new Exception("Error: Filepos must be unsigned");
                 }
 
                 if (field.getDataLen() != 8)
@@ -518,6 +521,20 @@ private static int getTypeID(FieldDef field) throws Exception
                 {
                     typeID |= FLAG_UNSIGNED;
                 }
+
+                HpccSrcType srcType = field.getSourceType();
+                if (srcType == HpccSrcType.KEYED_INTEGER)
+                {
+                    typeID = type_keyedint;
+                }
+                else if (srcType == HpccSrcType.SWAPPED_INTEGER)
+                {
+                    typeID = type_swapint;
+                }
+                else if (srcType == HpccSrcType.BIAS_SWAPPED_INTEGER)
+                {
+                    typeID = type_biasedswapint;
+                }
                 break;
             }
             case DECIMAL:
@@ -743,6 +760,10 @@ private static int getJsonTypeDefinition(FieldDef field, HashMap<Integer, Intege
                     {
                         char delim = 0x0001;
                         childJson.put("xpath", childField.getFieldName() + delim + "Row");
+                    } else if (childField.getFieldType() == FieldType.SET)
+                    {
+                        char delim = 0x0001;
+                        childJson.put("xpath", childField.getFieldName() + delim + "Item");
                     }
 
                     fields.put(childJson);
diff --git a/commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/TestFieldDefinitions.java b/commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/TestFieldDefinitions.java
index d0adc0a08..fa89580fd 100644
--- a/commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/TestFieldDefinitions.java
+++ b/commons-hpcc/src/main/java/org/hpccsystems/commons/ecl/TestFieldDefinitions.java
@@ -46,6 +46,24 @@ public class TestFieldDefinitions
             + "\"type\": \"ty17\",\r\n   \"xpath\": \"childdataset\\u0001Row\",\r\n   \"flags\": 1044\r\n  },\r\n  {\r\n   \"name\": \"int2set\",\r\n   "
             + "\"type\": \"ty19\",\r\n   \"xpath\": \"int2set\\u0001Item\",\r\n   \"flags\": 1045\r\n  }\r\n ]\r\n}";
 
+    private static final String allTypesIndexRecordDefinitionStr = "{\r\n \"ty1\": {\r\n  \"fieldType\": 1,\r\n  \"length\": 8\r\n },\r\n \"ty2\":"
+            + " {\r\n  \"fieldType\": 10,\r\n  \"length\": 8,\r\n  \"child\": \"ty1\"\r\n },\r\n \"ty3\": {\r\n  \"fieldType\": 257,\r\n  \"length\":"
+            + " 8\r\n },\r\n \"ty4\": {\r\n  \"fieldType\": 266,\r\n  \"length\": 8,\r\n  \"child\": \"ty3\"\r\n },\r\n \"ty5\": {\r\n  \"fieldType\": "
+            + "1,\r\n  \"length\": 4\r\n },\r\n \"ty6\": {\r\n  \"fieldType\": 10,\r\n  \"length\": 4,\r\n  \"child\": \"ty5\"\r\n },\r\n \"ty7\": {\r\n "
+            + " \"fieldType\": 257,\r\n  \"length\": 4\r\n },\r\n \"ty8\": {\r\n  \"fieldType\": 266,\r\n  \"length\": 4,\r\n  \"child\": \"ty7\"\r\n },\r\n"
+            + " \"ty9\": {\r\n  \"fieldType\": 1,\r\n  \"length\": 2\r\n },\r\n \"ty10\": {\r\n  \"fieldType\": 10,\r\n  \"length\": 2,\r\n  \"child\": "
+            + "\"ty9\"\r\n },\r\n \"ty11\": {\r\n  \"fieldType\": 257,\r\n  \"length\": 2\r\n },\r\n \"ty12\": {\r\n  \"fieldType\": 266,\r\n  \"length\": "
+            + "2,\r\n  \"child\": \"ty11\"\r\n },\r\n \"ty13\": {\r\n  \"fieldType\": 259,\r\n  \"length\": 524304\r\n },\r\n \"ty14\": {\r\n  \"fieldType\":"
+            + " 4,\r\n  \"length\": 8\r\n },\r\n \"ty15\": {\r\n  \"fieldType\": 0,\r\n  \"length\": 1\r\n },\r\n \"ty16\": {\r\n  \"fieldType\": 285,\r\n  "
+            + "\"length\": 8\r\n },\r\n \"fieldType\": 13,\r\n \"length\": 61,\r\n \"fields\": [\r\n  {\r\n   \"name\": \"int8\",\r\n   \"type\": \"ty2\",\r\n"
+            + "   \"flags\": 10\r\n  },\r\n  {\r\n   \"name\": \"uint8\",\r\n   \"type\": \"ty4\",\r\n   \"flags\": 266\r\n  },\r\n  {\r\n   \"name\": \"int4\","
+            + "\r\n   \"type\": \"ty6\",\r\n   \"flags\": 10\r\n  },\r\n  {\r\n   \"name\": \"uint4\",\r\n   \"type\": \"ty8\",\r\n   \"flags\": 266\r\n  },\r\n"
+            + "  {\r\n   \"name\": \"int2\",\r\n   \"type\": \"ty10\",\r\n   \"flags\": 10\r\n  },\r\n  {\r\n   \"name\": \"uint2\",\r\n   \"type\": \"ty12\",\r\n"
+            + "   \"flags\": 266\r\n  },\r\n  {\r\n   \"name\": \"udec16\",\r\n   \"type\": \"ty13\",\r\n   \"flags\": 259\r\n  },\r\n  {\r\n   \"name\": \"fixstr8\",\r\n"
+            + "   \"type\": \"ty14\",\r\n   \"flags\": 4\r\n  },\r\n  {\r\n   \"name\": \"recptr\",\r\n   \"type\": \"ty4\",\r\n   \"flags\": 266\r\n  },\r\n  {\r\n"
+            + "   \"name\": \"isactive\",\r\n   \"type\": \"ty15\",\r\n   \"flags\": 65536\r\n  },\r\n  {\r\n   \"name\": \"__internal_fpos__\",\r\n   \"type\": \"ty16\",\r\n"
+            + "   \"flags\": 65821\r\n  }\r\n ]\r\n}";
+
     /**
      * Gets the complex record definition json.
      *
@@ -56,6 +74,11 @@ public static String getComplexRecordDefinitionJson()
         return complexRecordDefinitionStr;
     }
 
+    public static String getAllTypesIndexRecordDefinitionJson()
+    {
+        return allTypesIndexRecordDefinitionStr;
+    }
+
     /**
      * Gets the complex record definition.
      *
diff --git a/commons-hpcc/src/test/java/org/hpccsystems/commons/ecl/RecordDefinitionTests.java b/commons-hpcc/src/test/java/org/hpccsystems/commons/ecl/RecordDefinitionTests.java
index 53654cf5d..b0d17dcbf 100644
--- a/commons-hpcc/src/test/java/org/hpccsystems/commons/ecl/RecordDefinitionTests.java
+++ b/commons-hpcc/src/test/java/org/hpccsystems/commons/ecl/RecordDefinitionTests.java
@@ -38,30 +38,34 @@ public void setup()
     @Test
     public void testJsonRecordParsing() throws Exception
     {
-        String jsonRecDefStr = TestFieldDefinitions.getComplexRecordDefinitionJson();
-        try
+        String[] recordDefStrs = new String[] { TestFieldDefinitions.getComplexRecordDefinitionJson(),
+                                                 TestFieldDefinitions.getAllTypesIndexRecordDefinitionJson() };
+        for (String recordDefStr : recordDefStrs)
         {
-            JSONObject jsonRecDef = new JSONObject(jsonRecDefStr);
-             
-            FieldDef recDef = RecordDefinitionTranslator.parseJsonRecordDefinition(jsonRecDef);
+            try
+            {
+                JSONObject expectedRefDef = new JSONObject(recordDefStr);
 
-            JSONObject tJsonRecDef = RecordDefinitionTranslator.toJsonRecord(recDef);
+                FieldDef recDef = RecordDefinitionTranslator.parseJsonRecordDefinition(expectedRefDef);
+                JSONObject actualRecDef = RecordDefinitionTranslator.toJsonRecord(recDef);
 
-            if (tJsonRecDef.equals(jsonRecDef))
+                // Use similar instead of equals because the order of fields in the JSON object may differ
+                if (!actualRecDef.similar(expectedRefDef))
+                {
+                    System.out.println("Expected Record Def: " + expectedRefDef + "\n");
+                    System.out.println("Actual Record Def: " + actualRecDef + "\n");
+                    Assert.fail("Translated JSON record definition differs from original");
+                }
+            }
+            catch (UnparsableContentException e)
             {
-                System.out.println(jsonRecDef + "\n");
-                System.out.println(tJsonRecDef + "\n");
-                Assert.fail("Translated JSON record definition differs from original");
+                Assert.fail("Encountered invalid record definition: '" + recordDefStr + "'");
+            }
+            catch (Exception e)
+            {
+                System.out.println(e.getMessage());
+                Assert.fail("Exception occurred: '" + e.getMessage() + "'");
             }
-        }
-        catch (UnparsableContentException e)
-        {
-            Assert.fail("Encountered invalid record definition: '" + jsonRecDefStr + "'");
-        }
-        catch (Exception e)
-        {
-            System.out.println(e.getMessage());
-            Assert.fail("Exception occurred: '" + e.getMessage() + "'");
         }
 
     }