Adds support for lexing the argument encoding bitmap (presence bits). (…

…#897)

src/main/java/com/amazon/ion/impl/IonCursorBinary.java

@@ -3068,7 +3068,6 @@ public Event nextTaglessValue(PrimitiveType primitiveType) {
                 seekPastDelimitedContainer_1_1();
             }
         }
-        valueTid = null;
         if (dataHandler != null) {
             reportConsumedData();
         }
@@ -3090,6 +3089,32 @@ public Event nextTaglessValue(PrimitiveType primitiveType) {
         return event;
     }
 
+    /**
+     * Fills the argument encoding bitmap (AEB) of the given byte width that is expected to occur at
+     * the cursor's current `peekIndex`. This method may return:
+     * <ul>
+     *     <li>NEEDS_DATA, if not enough data is available in the stream</li>
+     *     <li>NEEDS_INSTRUCTION, if the AEB was filled and the cursor is now positioned on the first byte of the
+     *     macro invocation.</li>
+     * </ul>
+     * After return, `valueMarker` is set with the start and end indices of the AEB.
+     * @param numberOfBytes the byte width of the AEB.
+     * @return an Event conveying the result of the operation.
+     */
+    public Event fillArgumentEncodingBitmap(int numberOfBytes) {
+        event = Event.NEEDS_DATA;
+        valueMarker.typeId = null;
+        valueMarker.startIndex = peekIndex;
+        valueMarker.endIndex = peekIndex + numberOfBytes;
+        if (isSlowMode && !fillAt(peekIndex, numberOfBytes)) {
+            return event;
+        }
+        peekIndex = valueMarker.endIndex;
+        setCheckpoint(CheckpointLocation.BEFORE_UNANNOTATED_TYPE_ID);
+        event = Event.NEEDS_INSTRUCTION;
+        return event;
+    }
+
     @Override
     public Event fillValue() {
         event = Event.VALUE_READY;

src/test/java/com/amazon/ion/impl/IonCursorBinaryTest.java

@@ -864,7 +864,11 @@ public void systemSymbolValue(InputType inputType) throws Exception {
      */
     private static void assertValueMarker(IonCursorBinary cursor, IonType expectedType, int expectedStartIndex, int expectedEndIndex) {
         Marker marker = cursor.getValueMarker();
-        assertEquals(expectedType, marker.typeId.type);
+        if (expectedType == null) {
+            assertTrue(marker.typeId == null || marker.typeId.type == null);
+        } else {
+            assertEquals(expectedType, marker.typeId.type);
+        }
         assertEquals(expectedStartIndex, marker.startIndex);
         assertEquals(expectedEndIndex, marker.endIndex);
     }
@@ -980,6 +984,20 @@ private static ExpectationProvider<IonCursorBinary> nextTaglessValue(IonCursorBi
         ));
     }
 
+    /**
+     * Provides Expectations that fill the argument encoding bitmap (AEB) at the cursor's current index and verify that
+     * the AEB has the given start and end indices.
+     */
+    private static ExpectationProvider<IonCursorBinary> fillArgumentEncodingBitmap(int numberOfBytes, int expectedStartIndex, int expectedEndIndex) {
+        return consumer -> consumer.accept(new Expectation<>(
+            String.format("next %d-byte AEB", numberOfBytes),
+            cursor -> {
+                assertEquals(NEEDS_INSTRUCTION, cursor.fillArgumentEncodingBitmap(numberOfBytes));
+                assertValueMarker(cursor, null, expectedStartIndex, expectedEndIndex);
+            }
+        ));
+    }
+
     /**
      * Provides Expectations that advance the reader to the next tagless value, fill the value, and verify that it has
      * the given attributes.
@@ -1368,4 +1386,71 @@ public void readFlexSymsIncrementally() throws Exception {
         );
         executeIncrementally(data, instructions);
     }
+
+    private static byte[] macroWithOneByteAEBThenIntZero() throws Exception {
+        return withIvm(1, hexStringToByteArray(cleanCommentedHexBytes(
+            "13  | Opcode 0x13 -> macro ID 0x13 \n" +
+            "00  | AEB 0x00 \n" +
+            "60  | int 0 \n"
+        )));
+    }
+
+    private static byte[] macroWithThreeByteAEBThenIntZero() throws Exception {
+        return withIvm(1, hexStringToByteArray(cleanCommentedHexBytes(
+            "13        | Opcode 0x13 -> macro ID 0x13 \n" +
+            "01 00 00  | AEB 0x01 0x00 0x00 \n" +
+            "60        | int 0 \n"
+        )));
+    }
+
+    private static void assertAEBThenIntZero(byte[] data, boolean constructFromBytes, int numberOfBytesInAEB) {
+        // The given data will always have a four-byte IVM followed by a 1-byte macro invocation opcode. Therefore,
+        // the AEB starts at index 5.
+        int expectedAEBEndIndex = 5 + numberOfBytesInAEB;
+        try (IonCursorBinary cursor = initializeCursor(STANDARD_BUFFER_CONFIGURATION, constructFromBytes, data)) {
+            assertSequence(
+                cursor,
+                nextMacroInvocation(0x13), valueMarker(null, 5, -1),
+                fillArgumentEncodingBitmap(numberOfBytesInAEB, 5, expectedAEBEndIndex),
+                nextTaggedValue(IonType.INT, expectedAEBEndIndex + 1, expectedAEBEndIndex + 1),
+                endStream()
+            );
+        }
+    }
+
+    private static void assertAEBThenIntZeroIncremental(byte[] data, int numberOfBytesInAEB) {
+        // The given data will always have a four-byte IVM followed by a 1-byte macro invocation opcode. Therefore,
+        // the AEB starts at index 5.
+        int expectedAEBEndIndex = 5 + numberOfBytesInAEB;
+        List<Instruction> instructions = Arrays.asList(
+            instruction(IonCursorBinary::nextValue, macroInvocation(0x13)),
+            instruction(cursor -> cursor.fillArgumentEncodingBitmap(numberOfBytesInAEB), valueMarker(null, 5, expectedAEBEndIndex)),
+            instruction(IonCursorBinary::nextValue, valueMarker(IonType.INT, expectedAEBEndIndex + 1, expectedAEBEndIndex + 1)),
+            // This is the end of the stream, so the response is not used.
+            instruction(IonCursorBinary::nextValue, null)
+        );
+        executeIncrementally(data, instructions);
+    }
+
+    @ParameterizedTest(name = "constructFromBytes={0}")
+    @ValueSource(booleans = {true, false})
+    public void macroInvocationWithIdInOpcodeAndOneByteAEB(boolean constructFromBytes) throws Exception {
+        assertAEBThenIntZero(macroWithOneByteAEBThenIntZero(), constructFromBytes, 1);
+    }
+
+    @Test
+    public void macroInvocationWithIdInOpcodeAndOneByteAEBIncremental() throws Exception {
+        assertAEBThenIntZeroIncremental(macroWithOneByteAEBThenIntZero(), 1);
+    }
+
+    @ParameterizedTest(name = "constructFromBytes={0}")
+    @ValueSource(booleans = {true, false})
+    public void macroInvocationWithIdInOpcodeAndMultiByteAEB(boolean constructFromBytes) throws Exception {
+        assertAEBThenIntZero(macroWithThreeByteAEBThenIntZero(), constructFromBytes, 3);
+    }
+
+    @Test
+    public void macroInvocationWithIdInOpcodeAndMultiByteAEBIncremental() throws Exception {
+        assertAEBThenIntZeroIncremental(macroWithThreeByteAEBThenIntZero(), 3);
+    }
 }