HPCC-32851 Add file concat tests to the performance suite

PerformanceTesting/TestSummary.rst

@@ -18,6 +18,7 @@ This file contains a list of all the tests within the test suite, and what parti
 | 11.     Smart join testing
 | 12.     Parallel execution
 | 13.     Splitters
+| 15.     SOAPCALL
 | 80.     Various miscellaneous tests picked up from other places.
 
 ToDo:
@@ -385,6 +386,14 @@ TBD:01h - Limits on index reads [class: indexread]
 | 13bd - unbalanced splitter, alternating one arm ahead of the other
 | 13be - unbalanced splitter, lookahead on rhs
 | 13bf - unbalanced splitter, lookahead on lhs, rhs
+| 13ca - balanced splitted
+
+14 Concat/Funnel
+++++++++++++++++
+| 14aa - Ordered concat/funnel, 2 inputs
+| 14ab - Ordered concat/funnel, 8 inputs
+| 14ba - Unordered concat/funnel, 2 inputs
+| 14bb - Unordered concat/funnel, 8 inputs
 
 15 SOAPCALL
 +++++++++++

PerformanceTesting/ecl/13aa_splitlarge.ecl

@@ -1,6 +1,9 @@
 //class=memory
 //class=sort
 
+//nohthor - hthor does not have splitters
+//noroxie - likely to run out of memory because of the very large records
+
 import $ as suite;
 import suite.perform.config;
 import suite.perform.format;

PerformanceTesting/ecl/13ab_splitvariable.ecl

@@ -1,6 +1,9 @@
 //class=memory
 //class=sort
 
+//nohthor - hthor does not have splitters
+//noroxie - also likely to exeed the roxie memory limits since all rows will be buffered
+
 import $ as suite;
 import suite.perform.config;
 import suite.perform.format;

PerformanceTesting/ecl/14aa_ofunnel2.ecl

@@ -0,0 +1,25 @@
+//class=memory
+//class=funnel
+
+//Test the performance of a ordered concat
+
+import ^ as root;
+import $ as suite;
+import suite.perform.config;
+import suite.perform.files;
+
+numRecords := 20000000;
+
+r := RECORD
+    UNSIGNED id1;
+END;
+
+r createSimple(unsigned8 c) := TRANSFORM
+    SELF.id1 := c;
+END;
+
+ds(unsigned i)  := DATASET(numRecords, createSimple(COUNTER+i), LOCAL);
+
+result := (+)(ds(0), ds(1), ordered);
+c := COUNT(NOFOLD(result));
+output(c - 2 * numRecords * CLUSTERSIZE);

PerformanceTesting/ecl/14ab_ofunnel8.ecl

@@ -0,0 +1,25 @@
+//class=memory
+//class=funnel
+
+//Test the performance of a ordered concat
+
+import ^ as root;
+import $ as suite;
+import suite.perform.config;
+import suite.perform.files;
+
+numRecords := 20000000;
+
+r := RECORD
+    UNSIGNED id1;
+END;
+
+r createSimple(unsigned8 c) := TRANSFORM
+    SELF.id1 := c;
+END;
+
+ds(unsigned i)  := DATASET(numRecords, createSimple(COUNTER+i), LOCAL);
+
+result := (+)(ds(0), ds(1), ds(2), ds(3), ds(4), ds(5), ds(6), ds(7), ordered);
+c := COUNT(NOFOLD(result));
+output(c - 8 * numRecords * CLUSTERSIZE);

PerformanceTesting/ecl/14ba_ufunnel2.ecl

@@ -0,0 +1,25 @@
+//class=memory
+//class=funnel
+
+//Test the performance of a ordered concat
+
+import ^ as root;
+import $ as suite;
+import suite.perform.config;
+import suite.perform.files;
+
+numRecords := 20000000;
+
+r := RECORD
+    UNSIGNED id1;
+END;
+
+r createSimple(unsigned8 c) := TRANSFORM
+    SELF.id1 := c;
+END;
+
+ds(unsigned i)  := DATASET(numRecords, createSimple(COUNTER+i), LOCAL);
+
+result := (+)(ds(0), ds(1), unordered);
+c := COUNT(NOFOLD(result));
+output(c - 2 * numRecords * CLUSTERSIZE);

PerformanceTesting/ecl/14bb_ufunnel8.ecl

@@ -0,0 +1,25 @@
+//class=memory
+//class=funnel
+
+//Test the performance of a ordered concat
+
+import ^ as root;
+import $ as suite;
+import suite.perform.config;
+import suite.perform.files;
+
+numRecords := 20000000;
+
+r := RECORD
+    UNSIGNED id1;
+END;
+
+r createSimple(unsigned8 c) := TRANSFORM
+    SELF.id1 := c;
+END;
+
+ds(unsigned i)  := DATASET(numRecords, createSimple(COUNTER+i), LOCAL);
+
+result := (+)(ds(0), ds(1), ds(2), ds(3), ds(4), ds(5), ds(6), ds(7), unordered);
+c := COUNT(NOFOLD(result));
+output(c - 8 * numRecords * CLUSTERSIZE);

PerformanceTesting/ecl/key/14aa_ofunnel2.xml

@@ -0,0 +1,3 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>0</Result_1></Row>
+</Dataset>

PerformanceTesting/ecl/key/14ab_ofunnel8.xml

@@ -0,0 +1,3 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>0</Result_1></Row>
+</Dataset>

PerformanceTesting/ecl/key/14ba_ufunnel2.xml

@@ -0,0 +1,3 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>0</Result_1></Row>
+</Dataset>

PerformanceTesting/ecl/key/14bb_ufunnel8.xml

@@ -0,0 +1,3 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>0</Result_1></Row>
+</Dataset>