Adds support for reading text Ion 1.1 system macros and implicit rest…

… parameters.

src/main/java/com/amazon/ion/impl/IonReaderContinuableCoreBinary.java

@@ -1428,7 +1428,7 @@ private void readSingleExpression(Macro.Parameter parameter, List<Expression.EEx
             if (event == Event.NEEDS_DATA) {
                 throw new UnsupportedOperationException("TODO: support continuable parsing of macro arguments.");
             }
-            readValueAsExpression(expressions);
+            readValueAsExpression(false, expressions);
         }
 
         /**
@@ -1450,7 +1450,7 @@ private void readGroupExpression(Macro.Parameter parameter, List<Expression.EExp
             expressions.add(Expression.Placeholder.INSTANCE);
             boolean isSingleton = true;
             while (nextGroupedValue() != Event.NEEDS_INSTRUCTION) {
-                readValueAsExpression(expressions);
+                readValueAsExpression(false, expressions);
                 isSingleton = false;
             }
             if (requireSingleton && !isSingleton) {
@@ -1476,7 +1476,7 @@ private void addVoidExpression(List<Expression.EExpressionBodyExpression> expres
         }
 
         @Override
-        protected void readParameter(Macro.Parameter parameter, long parameterPresence, List<Expression.EExpressionBodyExpression> expressions) {
+        protected void readParameter(Macro.Parameter parameter, long parameterPresence, List<Expression.EExpressionBodyExpression> expressions, boolean isTrailing) {
             switch (parameter.getCardinality()) {
                 case ZeroOrOne:
                     if (parameterPresence == PresenceBitmap.EXPRESSION) {

src/main/java/com/amazon/ion/impl/IonReaderTextSystemX.java

@@ -35,6 +35,7 @@
 import com.amazon.ion.impl.macro.MacroRef;
 import com.amazon.ion.impl.macro.ReaderAdapter;
 import com.amazon.ion.impl.macro.ReaderAdapterIonReader;
+import com.amazon.ion.impl.macro.SystemMacro;
 
 import java.io.IOException;
 import java.math.BigDecimal;
@@ -62,11 +63,11 @@ class IonReaderTextSystemX
 
     SymbolTable _system_symtab;
 
-    // The IonReader-like MacroEvaluator that this core reader delegates to when evaluating a macro invocation.
-    protected MacroEvaluatorAsIonReader macroEvaluatorIonReader = null;
-
     // The core MacroEvaluator that this core reader delegates to when evaluating a macro invocation.
-    private MacroEvaluator macroEvaluator = null;
+    private final MacroEvaluator macroEvaluator = new MacroEvaluator();
+
+    // The IonReader-like MacroEvaluator that this core reader delegates to when evaluating a macro invocation.
+    protected final MacroEvaluatorAsIonReader macroEvaluatorIonReader = new MacroEvaluatorAsIonReader(macroEvaluator);
 
     // The encoding context (macro table) that is currently active.
     private EncodingContext encodingContext = null;
@@ -1184,8 +1185,6 @@ private void readEncodingDirective() {
             ));
         }
         encodingContext = new EncodingContext(encodingDirectiveReader.getNewMacros());
-        macroEvaluator = new MacroEvaluator();
-        macroEvaluatorIonReader = new MacroEvaluatorAsIonReader(macroEvaluator);
     }
 
 
@@ -1199,11 +1198,11 @@ private class TextEExpressionArgsReader extends EExpressionArgsReader {
         }
 
         @Override
-        protected void readParameter(Macro.Parameter parameter, long parameterPresence, List<Expression.EExpressionBodyExpression> expressions) {
+        protected void readParameter(Macro.Parameter parameter, long parameterPresence, List<Expression.EExpressionBodyExpression> expressions, boolean isTrailing) {
             if (IonReaderTextSystemX.this.nextRaw() == null) {
                 return;
             }
-            readValueAsExpression(expressions);
+            readValueAsExpression(isTrailing && parameter.getCardinality().canBeMulti, expressions);
         }
 
         @Override
@@ -1229,7 +1228,12 @@ protected Macro loadMacro() {
                 throw new IonException("E-expressions must begin with an address.");
             }
             Macro macro;
-            if (encodingContext == null || ((macro = encodingContext.getMacroTable().get(address)) == null)) {
+            if (encodingContext == null) {
+                macro = SystemMacro.get(address);
+                if (macro == null) {
+                    throw new IonException(String.format("Encountered an unknown macro address: %s.", address));
+                }
+            } else if ((macro = encodingContext.getMacroTable().get(address)) == null) {
                 throw new IonException(String.format("Encountered an unknown macro address: %s.", address));
             }
             return macro;

src/main/java/com/amazon/ion/impl/macro/EExpressionArgsReader.java

@@ -83,8 +83,9 @@ public EExpressionArgsReader(ReaderAdapter reader) {
      * @param parameter information about the parameter from the macro signature.
      * @param parameterPresence the presence bits dedicated to this parameter (unused in text).
      * @param expressions receives the expressions as they are materialized.
+     * @param isTrailing true if this parameter is the last one in the signature; otherwise, false (unused in binary).
      */
-    protected abstract void readParameter(Macro.Parameter parameter, long parameterPresence, List<Expression.EExpressionBodyExpression> expressions);
+    protected abstract void readParameter(Macro.Parameter parameter, long parameterPresence, List<Expression.EExpressionBodyExpression> expressions, boolean isTrailing);
 
     /**
      * Reads the macro's address and attempts to resolve that address to a Macro from the macro table.
@@ -180,7 +181,7 @@ private void readContainerValueAsExpression(
             if (type == IonType.STRUCT) {
                 expressions.add(new Expression.FieldName(reader.getFieldNameSymbol()));
             }
-            readValueAsExpression(expressions); // TODO avoid recursion
+            readValueAsExpression(false, expressions); // TODO avoid recursion
         }
         stepOutRaw();
         // Overwrite the placeholder with an expression representing the actual type of the container and the
@@ -207,19 +208,38 @@ private void readContainerValueAsExpression(
         expressions.set(startIndex, expression);
     }
 
+    /**
+     * Reads the rest of the stream into a single expression group.
+     * @param expressions receives the expressions as they are materialized.
+     */
+    private void readStreamAsExpressionGroup(
+        List<Expression.EExpressionBodyExpression> expressions
+    ) {
+        int startIndex = expressions.size();
+        expressions.add(Expression.Placeholder.INSTANCE);
+        do {
+            readValueAsExpression(false, expressions); // TODO avoid recursion
+        } while (nextRaw());
+        expressions.set(startIndex, new Expression.ExpressionGroup(startIndex, expressions.size()));
+    }
+
     /**
      * Reads a value from the stream into expression(s) that will eventually be passed to the MacroEvaluator
      * responsible for evaluating the e-expression to which this value belongs.
+     * @param isImplicitRest true if this is the final parameter in the signature, it is variadic, and the format
+     *                       supports implicit rest parameters (text only); otherwise, false.
      * @param expressions receives the expressions as they are materialized.
      */
-    protected void readValueAsExpression(List<Expression.EExpressionBodyExpression> expressions) {
+    protected void readValueAsExpression(boolean isImplicitRest, List<Expression.EExpressionBodyExpression> expressions) {
         if (isMacroInvocation()) {
             collectEExpressionArgs(expressions); // TODO avoid recursion
             return;
         }
         IonType type = reader.encodingType();
         List<SymbolToken> annotations = getAnnotations();
-        if (IonType.isContainer(type)) {
+        if (isImplicitRest) {
+            readStreamAsExpressionGroup(expressions);
+        } else if (IonType.isContainer(type)) {
             readContainerValueAsExpression(type, annotations, expressions);
         } else {
             readScalarValueAsExpression(type, annotations, expressions);
@@ -242,7 +262,12 @@ private void collectEExpressionArgs(List<Expression.EExpressionBodyExpression> e
         int numberOfParameters = signature.size();
         stepIntoEExpression();
         for (int i = 0; i < numberOfParameters; i++) {
-            readParameter(signature.get(i), presenceBitmap == null ? 0 : presenceBitmap.get(i), expressions);
+            readParameter(
+                signature.get(i),
+                presenceBitmap == null ? 0 : presenceBitmap.get(i),
+                expressions,
+                i == (numberOfParameters - 1)
+            );
         }
         stepOutOfEExpression();
         expressions.set(invocationStartIndex, new Expression.EExpression(macro, invocationStartIndex, expressions.size()));

src/main/java/com/amazon/ion/impl/macro/SystemMacro.kt

@@ -50,6 +50,14 @@ enum class SystemMacro(val id: Byte, val macroName: String, override val signatu
         @JvmStatic
         operator fun get(name: String): SystemMacro? = MACROS_BY_NAME[name]?.takeUnless { it.id < 0 }
 
+        @JvmStatic
+        operator fun get(address: MacroRef): SystemMacro? {
+            return when (address) {
+                is MacroRef.ById -> get(address.id)
+                is MacroRef.ByName -> get(address.name)
+            }
+        }
+
         /** Gets a [SystemMacro] by name, including those which are not in the system table (i.e. special forms) */
         @JvmStatic
         fun getMacroOrSpecialForm(name: String): SystemMacro? = MACROS_BY_NAME[name]

src/test/java/com/amazon/ion/impl/IonRawTextReaderTest_1_1.java

@@ -2,10 +2,12 @@
 // SPDX-License-Identifier: Apache-2.0
 package com.amazon.ion.impl;
 
+import com.amazon.ion.IonReader;
 import com.amazon.ion.IonType;
 import com.amazon.ion.system.SimpleCatalog;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.CsvSource;
 import org.junit.jupiter.params.provider.MethodSource;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -126,4 +128,31 @@ public void invalidExpressionSyntax(int minorVersion, String input, String first
             assertThrows(IonReaderTextRawX.IonReaderTextParsingException.class, reader::nextRaw);
         }
     }
+
+    @ParameterizedTest
+    @CsvSource({
+        "(:values 0) (:values 1)",
+        "(:values (: values 0)) 1",
+        "(:values) 0 (:values) 1",
+        "(:values) (:values 0) 1",
+        "(:values) (:values) (:values 0) 1",
+        "(:values (:: ) ) 0 1",
+        "(:values (:: 0 1))",
+        "(:values 0 1)",
+        "(:values (:: (:: 0) (:values (:: 1))))",
+        "(:values (:: 0) (:values (:: 1)))",
+        "(:values (:values (:: 0 1)))",
+        "(:values (:values 0 1))",
+        "(:1 (:1 0 1))",
+        "(:1 (:: (:: 0) (:1 (:: 1))))"
+    })
+    public void validValuesInvocations(String text) throws Exception {
+        try (IonReader reader = newTextReader("$ion_1_1 " + text)) {
+            assertEquals(IonType.INT, reader.next());
+            assertEquals(0, reader.intValue());
+            assertEquals(IonType.INT, reader.next());
+            assertEquals(1, reader.intValue());
+            assertNull(reader.next());
+        }
+    }
 }

src/test/java/com/amazon/ion/impl/IonReaderContinuableTopLevelBinaryTest.java

@@ -5961,7 +5961,7 @@ public void invokeValuesUsingSystemMacroOpcode(String bytes) throws Exception {
         "EF 01 02 07 60 61 01",
         // (:values 0 1) // using delimited expression group
         "EF 01 02 01 60 61 01 F0",
-        // (:values (:: 0) (:values (:: 1))
+        // (:values (:: 0) (:values (:: 1)))
         "EF 01 02 03 60 EF 01 02 05 61 01",
         // (:values (:values 0 1))
         "EF 01 01 EF 01 02 07 60 61 01",