Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adds support for lexing the argument encoding bitmap (presence bits). #897

Merged
merged 2 commits into from
Jul 2, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 26 additions & 1 deletion src/main/java/com/amazon/ion/impl/IonCursorBinary.java
Original file line number Diff line number Diff line change
Expand Up @@ -3057,7 +3057,6 @@ public Event nextTaglessValue(PrimitiveType primitiveType) {
seekPastDelimitedContainer_1_1();
}
}
valueTid = null;
if (dataHandler != null) {
reportConsumedData();
}
Expand All @@ -3079,6 +3078,32 @@ public Event nextTaglessValue(PrimitiveType primitiveType) {
return event;
}

/**
* Fills the argument encoding bitmap (AEB) of the given byte width that is expected to occur at
* the cursor's current `peekIndex`. This method may return:
* <ul>
* <li>NEEDS_DATA, if not enough data is available in the stream</li>
* <li>NEEDS_INSTRUCTION, if the AEB was filled and the cursor is now positioned on the first byte of the
* macro invocation.</li>
* </ul>
* After return, `valueMarker` is set with the start and end indices of the AEB.
* @param numberOfBytes the byte width of the AEB.
* @return an Event conveying the result of the operation.
*/
public Event fillArgumentEncodingBitmap(int numberOfBytes) {
event = Event.NEEDS_DATA;
valueMarker.typeId = null;
valueMarker.startIndex = peekIndex;
valueMarker.endIndex = peekIndex + numberOfBytes;
if (isSlowMode && !fillAt(peekIndex, numberOfBytes)) {
return event;
}
peekIndex = valueMarker.endIndex;
setCheckpoint(CheckpointLocation.BEFORE_UNANNOTATED_TYPE_ID);
event = Event.NEEDS_INSTRUCTION;
return event;
}

@Override
public Event fillValue() {
event = Event.VALUE_READY;
Expand Down
87 changes: 86 additions & 1 deletion src/test/java/com/amazon/ion/impl/IonCursorBinaryTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -783,7 +783,11 @@ public void systemSymbolValue(InputType inputType) throws Exception {
*/
private static void assertValueMarker(IonCursorBinary cursor, IonType expectedType, int expectedStartIndex, int expectedEndIndex) {
Marker marker = cursor.getValueMarker();
assertEquals(expectedType, marker.typeId.type);
if (expectedType == null) {
assertTrue(marker.typeId == null || marker.typeId.type == null);
} else {
assertEquals(expectedType, marker.typeId.type);
}
assertEquals(expectedStartIndex, marker.startIndex);
assertEquals(expectedEndIndex, marker.endIndex);
}
Expand Down Expand Up @@ -899,6 +903,20 @@ private static ExpectationProvider<IonCursorBinary> nextTaglessValue(IonCursorBi
));
}

/**
* Provides Expectations that fill the argument encoding bitmap (AEB) at the cursor's current index and verify that
* the AEB has the given start and end indices.
*/
private static ExpectationProvider<IonCursorBinary> fillArgumentEncodingBitmap(int numberOfBytes, int expectedStartIndex, int expectedEndIndex) {
return consumer -> consumer.accept(new Expectation<>(
String.format("next %d-byte AEB", numberOfBytes),
cursor -> {
assertEquals(NEEDS_INSTRUCTION, cursor.fillArgumentEncodingBitmap(numberOfBytes));
assertValueMarker(cursor, null, expectedStartIndex, expectedEndIndex);
}
));
}

/**
* Provides Expectations that advance the reader to the next tagless value, fill the value, and verify that it has
* the given attributes.
Expand Down Expand Up @@ -1287,4 +1305,71 @@ public void readFlexSymsIncrementally() throws Exception {
);
executeIncrementally(data, instructions);
}

private static byte[] macroWithOneByteAEBThenIntZero() throws Exception {
return withIvm(1, hexStringToByteArray(cleanCommentedHexBytes(
"13 | Opcode 0x13 -> macro ID 0x13 \n" +
"00 | AEB 0x00 \n" +
"60 | int 0 \n"
)));
}

private static byte[] macroWithThreeByteAEBThenIntZero() throws Exception {
return withIvm(1, hexStringToByteArray(cleanCommentedHexBytes(
"13 | Opcode 0x13 -> macro ID 0x13 \n" +
"01 00 00 | AEB 0x01 0x00 0x00 \n" +
"60 | int 0 \n"
)));
}

private static void assertAEBThenIntZero(byte[] data, boolean constructFromBytes, int numberOfBytesInAEB) {
// The given data will always have a four-byte IVM followed by a 1-byte macro invocation opcode. Therefore,
// the AEB starts at index 5.
int expectedAEBEndIndex = 5 + numberOfBytesInAEB;
try (IonCursorBinary cursor = initializeCursor(STANDARD_BUFFER_CONFIGURATION, constructFromBytes, data)) {
assertSequence(
cursor,
nextMacroInvocation(0x13), valueMarker(null, 5, -1),
fillArgumentEncodingBitmap(numberOfBytesInAEB, 5, expectedAEBEndIndex),
nextTaggedValue(IonType.INT, expectedAEBEndIndex + 1, expectedAEBEndIndex + 1),
endStream()
);
}
}

private static void assertAEBThenIntZeroIncremental(byte[] data, int numberOfBytesInAEB) {
// The given data will always have a four-byte IVM followed by a 1-byte macro invocation opcode. Therefore,
// the AEB starts at index 5.
int expectedAEBEndIndex = 5 + numberOfBytesInAEB;
List<Instruction> instructions = Arrays.asList(
instruction(IonCursorBinary::nextValue, macroInvocation(0x13)),
instruction(cursor -> cursor.fillArgumentEncodingBitmap(numberOfBytesInAEB), valueMarker(null, 5, expectedAEBEndIndex)),
instruction(IonCursorBinary::nextValue, valueMarker(IonType.INT, expectedAEBEndIndex + 1, expectedAEBEndIndex + 1)),
// This is the end of the stream, so the response is not used.
instruction(IonCursorBinary::nextValue, null)
);
executeIncrementally(data, instructions);
}

@ParameterizedTest(name = "constructFromBytes={0}")
@ValueSource(booleans = {true, false})
public void macroInvocationWithIdInOpcodeAndOneByteAEB(boolean constructFromBytes) throws Exception {
assertAEBThenIntZero(macroWithOneByteAEBThenIntZero(), constructFromBytes, 1);
}

@Test
public void macroInvocationWithIdInOpcodeAndOneByteAEBIncremental() throws Exception {
assertAEBThenIntZeroIncremental(macroWithOneByteAEBThenIntZero(), 1);
}

@ParameterizedTest(name = "constructFromBytes={0}")
@ValueSource(booleans = {true, false})
public void macroInvocationWithIdInOpcodeAndMultiByteAEB(boolean constructFromBytes) throws Exception {
assertAEBThenIntZero(macroWithThreeByteAEBThenIntZero(), constructFromBytes, 3);
}

@Test
public void macroInvocationWithIdInOpcodeAndMultiByteAEBIncremental() throws Exception {
assertAEBThenIntZeroIncremental(macroWithThreeByteAEBThenIntZero(), 3);
}
}
Loading