Skip to content

Commit

Permalink
HPCC4J-569 Correct EOS character handling
Browse files Browse the repository at this point in the history
- Modified BinaryRecordWriter to only truncate strings at EOS in var strings
- Added unit test that verifies read / write all of Unicode chars and mid EOS

Signed-off-by: James McMullan [email protected]
  • Loading branch information
jpmcmu committed Jan 25, 2024
1 parent 5554f9f commit 8cdc53d
Show file tree
Hide file tree
Showing 2 changed files with 108 additions and 5 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -475,10 +475,13 @@ else if (fd.getDataLen() == 8)
case STRING:
{
String value = fieldValue != null ? (String) fieldValue : "";
int eosIdx = value.indexOf('\0');
if (eosIdx > -1)
if (fd.getFieldType() == FieldType.VAR_STRING)
{
value = value.substring(0,eosIdx);
int eosIdx = value.indexOf('\0');
if (eosIdx > -1)
{
value = value.substring(0,eosIdx);
}
}

byte[] data = new byte[0];
Expand Down Expand Up @@ -609,10 +612,23 @@ else if (fd.getSourceType() == HpccSrcType.QSTRING)
if (fd.getFieldType() == FieldType.VAR_STRING)
{
byte nullByte = '\0';
this.buffer.put(nullByte);
if (fd.getSourceType().isUTF16())
{
this.buffer.put(nullByte);
boolean needsNullAdded = data.length < 2
|| data[data.length - 1] != nullByte
|| data[data.length - 2] != nullByte;
if (needsNullAdded) {
this.buffer.put(nullByte);
this.buffer.put(nullByte);
}
}
else
{
boolean needsNullAdded = data.length < 1
|| data[data.length - 1] != nullByte;
if (needsNullAdded) {
this.buffer.put(nullByte);
}
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,93 @@ public void readWithForcedTimeoutTest() throws Exception
assertEquals("Not all records loaded",expectedCounts[0], records.size());
}

@Test
public void allCharsNullTest() throws Exception
{
// Unicode
{
FieldDef recordDef = null;
{
FieldDef[] fieldDefs = new FieldDef[2];
fieldDefs[0] = new FieldDef("uni", FieldType.STRING, "STRING", 100, false, false, HpccSrcType.UTF16LE, new FieldDef[0]);
fieldDefs[1] = new FieldDef("fixedUni", FieldType.STRING, "STRING", 100, true, false, HpccSrcType.UTF16LE, new FieldDef[0]);

recordDef = new FieldDef("RootRecord", FieldType.RECORD, "rec", 4, false, false, HpccSrcType.LITTLE_ENDIAN, fieldDefs);
}

List<HPCCRecord> records = new ArrayList<HPCCRecord>();
int maxUTF16BMPChar = Character.MAX_CODE_POINT;
for (int i = 0; i < maxUTF16BMPChar; i++) {
String strMidEOS = "";
for (int j = 0; j < 98; j++, i++) {
if (j == 50) {
strMidEOS += "\0";
}
strMidEOS += Character.toString((char) i);
}

Object[] fields = {strMidEOS, strMidEOS};
records.add(new HPCCRecord(fields, recordDef));
}

String fileName = "unicode::all_chars::test";
writeFile(records, fileName, recordDef, connTO);

HPCCFile file = new HPCCFile(fileName, connString , hpccUser, hpccPass);
List<HPCCRecord> readRecords = readFile(file, 10000, false, false, BinaryRecordReader.TRIM_STRINGS);

for (int i = 0; i < records.size(); i++) {
HPCCRecord record = records.get(i);
HPCCRecord readRecord = readRecords.get(i);
if (readRecord.equals(record) == false)
{
System.out.println("Record: " + i + " did not match\n" + record + "\n" + readRecord);
}
}
}

// SBC / ASCII
{
FieldDef recordDef = null;
{
FieldDef[] fieldDefs = new FieldDef[2];
fieldDefs[0] = new FieldDef("str", FieldType.STRING, "STRING", 10, false, false, HpccSrcType.SINGLE_BYTE_CHAR, new FieldDef[0]);
fieldDefs[1] = new FieldDef("fixedStr", FieldType.STRING, "STRING", 10, true, false, HpccSrcType.SINGLE_BYTE_CHAR, new FieldDef[0]);

recordDef = new FieldDef("RootRecord", FieldType.RECORD, "rec", 4, false, false, HpccSrcType.LITTLE_ENDIAN, fieldDefs);
}

List<HPCCRecord> records = new ArrayList<HPCCRecord>();
for (int i = 0; i < 255; i++) {
String strMidEOS = "";
for (int j = 0; j < 9; j++, i++) {
if (j == 5) {
strMidEOS += "\0";
}
strMidEOS += Character.toString((char) i);
}

Object[] fields = {strMidEOS, strMidEOS};
records.add(new HPCCRecord(fields, recordDef));
}

String fileName = "ascii::all_chars::test";
writeFile(records, fileName, recordDef, connTO);

HPCCFile file = new HPCCFile(fileName, connString , hpccUser, hpccPass);
List<HPCCRecord> readRecords = readFile(file, 10000, false, false, BinaryRecordReader.TRIM_STRINGS);

for (int i = 0; i < records.size(); i++) {
HPCCRecord record = records.get(i);
HPCCRecord readRecord = readRecords.get(i);
if (readRecord.equals(record) == false)
{
System.out.println("Record: " + i + " did not match\n" + record + "\n" + readRecord);
}
}
}
}

@Test
public void integrationReadWriteBackTest() throws Exception
{
Expand Down

0 comments on commit 8cdc53d

Please sign in to comment.