Enables MacroCompiler to operate over an IonReader.

src/main/java/com/amazon/ion/impl/IonReaderContinuableCoreBinary.java

@@ -20,6 +20,7 @@
 import com.amazon.ion.impl.macro.Expression;
 import com.amazon.ion.impl.macro.Macro;
 import com.amazon.ion.impl.macro.MacroCompiler;
+import com.amazon.ion.impl.macro.MacroCompilerContinuable;
 import com.amazon.ion.impl.macro.MacroEvaluator;
 import com.amazon.ion.impl.macro.MacroEvaluatorAsIonReader;
 import com.amazon.ion.impl.macro.MacroRef;
@@ -1162,7 +1163,7 @@ private class EncodingDirectiveReader {
         boolean isSymbolTableAppend = false;
         List<String> newSymbols = new ArrayList<>(8);
         Map<MacroRef, Macro> newMacros = new HashMap<>();
-        MacroCompiler macroCompiler = new MacroCompiler(IonReaderContinuableCoreBinary.this, newMacros::get);
+        MacroCompiler macroCompiler = new MacroCompilerContinuable(IonReaderContinuableCoreBinary.this, newMacros::get);
 
         private boolean valueUnavailable() {
             Event event = fillValue();

src/main/java/com/amazon/ion/impl/macro/MacroCompiler.kt

@@ -6,20 +6,16 @@ import com.amazon.ion.*
 import com.amazon.ion.impl.*
 import com.amazon.ion.impl.macro.Expression.*
 import com.amazon.ion.util.confirm
+import java.math.BigDecimal
+import java.math.BigInteger
 
 /**
- * [MacroCompiler] wraps an [IonReader]. When directed to do so, it will take over advancing and getting values from the
+ * [MacroCompiler] wraps an Ion reader. When directed to do so, it will take over advancing and getting values from the
  * reader in order to read one [TemplateMacro].
- *
- * This is currently implemented using [IonReader], but it could be adapted to work with
- * [IonReaderContinuableCore][com.amazon.ion.impl.IonReaderContinuableCore].
  */
-// TODO determine a strategy for generalizing or templatizing this for use in both text and binary
-class MacroCompiler(
-    private val reader: IonReaderContinuableCore,
+abstract class MacroCompiler(
     private val getMacro: (MacroRef) -> Macro?,
 ) {
-    // TODO: Make sure that we can throw exceptions if there's an over-sized value.
 
     /** The name of the macro that was read. Returns `null` if no macro name is available. */
     var macroName: String? = null
@@ -31,32 +27,29 @@ class MacroCompiler(
     /**
      * Compiles a template macro definition from the reader. Caller is responsible for positioning the reader at—but not
      * stepped into—the macro template s-expression.
-     *
-     * TODO: if we switch the macro compiler to use a continuable reader, change the return type of this
-     *       to a compiler state enum, and add a separate function to get the compiled macro once it is ready.
      */
     fun compileMacro(): TemplateMacro {
         macroName = null
         signature.clear()
         expressions.clear()
 
-        confirm(reader.encodingType == IonType.SEXP) { "macro compilation expects a sexp starting with the keyword `macro`" }
-        reader.confirmNoAnnotations("a macro definition sexp")
-        reader.readContainer {
-            reader.nextValue()
-            confirm(reader.encodingType == IonType.SYMBOL && reader.stringValue() == "macro") { "macro compilation expects a sexp starting with the keyword `macro`" }
+        confirm(encodingType() == IonType.SEXP) { "macro compilation expects a sexp starting with the keyword `macro`" }
+        confirmNoAnnotations("a macro definition sexp")
+        readContainer {
+            nextValue()
+            confirm(encodingType() == IonType.SYMBOL && stringValue() == "macro") { "macro compilation expects a sexp starting with the keyword `macro`" }
 
             nextAndCheckType(IonType.SYMBOL, IonType.NULL, "macro name")
             confirmNoAnnotations("macro name")
-            if (encodingType != IonType.NULL) {
+            if (encodingType() != IonType.NULL) {
                 macroName = stringValue().also { confirm(isIdentifierSymbol(it)) { "invalid macro name: '$it'" } }
             }
             nextAndCheckType(IonType.SEXP, "macro signature")
             confirmNoAnnotations("macro signature")
             readSignature()
-            confirm(nextValue() != IonCursor.Event.END_CONTAINER) { "Macro definition is missing a template body expression." }
+            confirm(nextValue()) { "Macro definition is missing a template body expression." }
             compileTemplateBodyExpression(isQuoted = false)
-            confirm(nextValue() == IonCursor.Event.END_CONTAINER) { "Unexpected $type after template body expression." }
+            confirm(!nextValue()) { "Unexpected ${encodingType()} after template body expression." }
         }
         return TemplateMacro(signature.toList(), expressions.toList())
     }
@@ -68,8 +61,8 @@ class MacroCompiler(
     private fun readSignature() {
         var pendingParameter: Macro.Parameter? = null
 
-        reader.forEachInContainer {
-            if (encodingType != IonType.SYMBOL) throw IonException("parameter must be a symbol; found $type")
+        forEachInContainer {
+            if (encodingType() != IonType.SYMBOL) throw IonException("parameter must be a symbol; found ${encodingType()}")
 
             val symbolText = stringValue()
 
@@ -121,31 +114,31 @@ class MacroCompiler(
      */
     private fun compileTemplateBodyExpression(isQuoted: Boolean) {
         // NOTE: `toList()` does not allocate for an empty list.
-        val annotations: List<SymbolToken> = reader.getTypeAnnotationSymbols()
+        val annotations: List<SymbolToken> = getTypeAnnotationSymbols()
 
-        if (reader.isNullValue) {
-            expressions.add(NullValue(annotations, reader.encodingType))
-        } else when (reader.encodingType) {
-            IonType.BOOL -> expressions.add(BoolValue(annotations, reader.booleanValue()))
+        if (isNullValue()) {
+            expressions.add(NullValue(annotations, encodingType()!!))
+        } else when (encodingType()) {
+            IonType.BOOL -> expressions.add(BoolValue(annotations, booleanValue()))
             IonType.INT -> expressions.add(
-                when (reader.integerSize!!) {
+                when (integerSize()!!) {
                     IntegerSize.INT,
-                    IntegerSize.LONG -> LongIntValue(annotations, reader.longValue())
-                    IntegerSize.BIG_INTEGER -> BigIntValue(annotations, reader.bigIntegerValue())
+                    IntegerSize.LONG -> LongIntValue(annotations, longValue())
+                    IntegerSize.BIG_INTEGER -> BigIntValue(annotations, bigIntegerValue())
                 }
             )
-            IonType.FLOAT -> expressions.add(FloatValue(annotations, reader.doubleValue()))
-            IonType.DECIMAL -> expressions.add(DecimalValue(annotations, reader.decimalValue()))
-            IonType.TIMESTAMP -> expressions.add(TimestampValue(annotations, reader.timestampValue()))
-            IonType.STRING -> expressions.add(StringValue(annotations, reader.stringValue()))
-            IonType.BLOB -> expressions.add(BlobValue(annotations, reader.newBytes()))
-            IonType.CLOB -> expressions.add(ClobValue(annotations, reader.newBytes()))
+            IonType.FLOAT -> expressions.add(FloatValue(annotations, doubleValue()))
+            IonType.DECIMAL -> expressions.add(DecimalValue(annotations, decimalValue()))
+            IonType.TIMESTAMP -> expressions.add(TimestampValue(annotations, timestampValue()))
+            IonType.STRING -> expressions.add(StringValue(annotations, stringValue()))
+            IonType.BLOB -> expressions.add(BlobValue(annotations, newBytes()))
+            IonType.CLOB -> expressions.add(ClobValue(annotations, newBytes()))
             IonType.SYMBOL -> {
                 if (isQuoted) {
-                    expressions.add(SymbolValue(annotations, reader.symbolValue()))
+                    expressions.add(SymbolValue(annotations, symbolValue()))
                 } else {
-                    val name = reader.stringValue()
-                    reader.confirmNoAnnotations("on variable reference '$name'")
+                    val name = stringValue()
+                    confirmNoAnnotations("on variable reference '$name'")
                     val index = signature.indexOfFirst { it.variableName == name }
                     confirm(index >= 0) { "variable '$name' is not recognized" }
                     expressions.add(VariableRef(index))
@@ -156,13 +149,13 @@ class MacroCompiler(
                 if (isQuoted) {
                     compileSequence(isQuoted = true) { start, end -> SExpValue(annotations, start, end) }
                 } else {
-                    reader.confirmNoAnnotations(location = "a macro invocation")
+                    confirmNoAnnotations(location = "a macro invocation")
                     compileMacroInvocation()
                 }
             }
             IonType.STRUCT -> compileStruct(annotations, isQuoted)
             // IonType.NULL, IonType.DATAGRAM, null
-            else -> throw IllegalStateException("Found ${reader.encodingType}; this should be unreachable.")
+            else -> throw IllegalStateException("Found ${encodingType()}; this should be unreachable.")
         }
     }
 
@@ -176,8 +169,8 @@ class MacroCompiler(
         val start = expressions.size
         expressions.add(Placeholder)
         val templateStructIndex = mutableMapOf<String, ArrayList<Int>>()
-        reader.forEachInContainer {
-            val fieldName: SymbolToken = fieldNameSymbol
+        forEachInContainer {
+            val fieldName: SymbolToken = fieldNameSymbol()
             expressions.add(FieldName(fieldName))
             fieldName.text?.let {
                 val valueIndex = expressions.size
@@ -199,7 +192,7 @@ class MacroCompiler(
     private inline fun compileSequence(isQuoted: Boolean, newTemplateBodySequence: (Int, Int) -> TemplateBodyExpression) {
         val seqStart = expressions.size
         expressions.add(Placeholder)
-        reader.forEachInContainer { compileTemplateBodyExpression(isQuoted) }
+        forEachInContainer { compileTemplateBodyExpression(isQuoted) }
         val seqEnd = expressions.size
         expressions[seqStart] = newTemplateBodySequence(seqStart, seqEnd)
     }
@@ -211,44 +204,44 @@ class MacroCompiler(
      * Caller will need to call [IonReader.next] to get the next value.
      */
     private fun compileMacroInvocation() {
-        reader.stepIntoContainer()
-        reader.nextValue()
-        val macroRef = when (reader.encodingType) {
+        stepIntoContainer()
+        nextValue()
+        val macroRef = when (encodingType()) {
             IonType.SYMBOL -> {
-                val macroName = reader.stringValue()
+                val macroName = stringValue()
                 // TODO: Come up with a consistent strategy for handling special forms.
                 when (macroName) {
                     "literal" -> {
                         // It's the "literal" special form; skip compiling a macro invocation and just treat all contents as literals
-                        reader.forEachRemaining { compileTemplateBodyExpression(isQuoted = true) }
-                        reader.stepOutOfContainer()
+                        forEachRemaining { compileTemplateBodyExpression(isQuoted = true) }
+                        stepOutOfContainer()
                         return
                     }
                     ";" -> {
                         val macroStart = expressions.size
                         expressions.add(Placeholder)
-                        reader.forEachRemaining { compileTemplateBodyExpression(isQuoted = false) }
+                        forEachRemaining { compileTemplateBodyExpression(isQuoted = false) }
                         val macroEnd = expressions.size
                         expressions[macroStart] = ExpressionGroup(macroStart, macroEnd)
-                        reader.stepOutOfContainer()
+                        stepOutOfContainer()
                         return
                     }
                     else -> MacroRef.ByName(macroName)
                 }
             }
-            IonType.INT -> MacroRef.ById(reader.intValue())
-            else -> throw IonException("macro invocation must start with an id (int) or identifier (symbol); found ${reader.encodingType ?: "nothing"}\"")
+            IonType.INT -> MacroRef.ById(intValue())
+            else -> throw IonException("macro invocation must start with an id (int) or identifier (symbol); found ${encodingType() ?: "nothing"}\"")
         }
 
         val macro = getMacro(macroRef) ?: throw IonException("Unrecognized macro: $macroRef")
 
         val macroStart = expressions.size
         expressions.add(Placeholder)
-        reader.forEachRemaining { compileTemplateBodyExpression(isQuoted = false) }
+        forEachRemaining { compileTemplateBodyExpression(isQuoted = false) }
         val macroEnd = expressions.size
         expressions[macroStart] = MacroInvocation(macro, macroStart, macroEnd)
 
-        reader.stepOutOfContainer()
+        stepOutOfContainer()
     }
 
     // Helper functions
@@ -257,35 +250,48 @@ class MacroCompiler(
     private fun List<SymbolToken>.isEmptyOr(text: String): Boolean = isEmpty() || (size == 1 && this[0].assumeText() == text)
 
     /** Throws [IonException] if any annotations are on the current value in this [IonReader]. */
-    private fun IonReaderContinuableCore.confirmNoAnnotations(location: String) {
+    private fun confirmNoAnnotations(location: String) {
         confirm(!hasAnnotations()) { "found annotations on $location" }
     }
 
     /** Moves to the next type and throw [IonException] if it is not the `expected` [IonType]. */
-    private fun IonReaderContinuableCore.nextAndCheckType(expected: IonType, location: String) {
-        confirm(nextValue() != IonCursor.Event.NEEDS_DATA && encodingType == expected) { "$location must be a $expected; found ${encodingType ?: "nothing"}" }
+    private fun nextAndCheckType(expected: IonType, location: String) {
+        confirm(nextValue() && encodingType() == expected) { "$location must be a $expected; found ${encodingType() ?: "nothing"}" }
     }
 
     /** Moves to the next type and throw [IonException] if it is not the `expected` [IonType]. */
-    private fun IonReaderContinuableCore.nextAndCheckType(expected0: IonType, expected1: IonType, location: String) {
-        confirm(nextValue() != IonCursor.Event.NEEDS_DATA && (encodingType == expected0 || encodingType == expected1)) { "$location must be a $expected0 or $expected1; found ${encodingType ?: "nothing"}" }
+    private fun nextAndCheckType(expected0: IonType, expected1: IonType, location: String) {
+        confirm(nextValue() && (encodingType() == expected0 || encodingType() == expected1)) { "$location must be a $expected0 or $expected1; found ${encodingType() ?: "nothing"}" }
     }
 
     /** Steps into a container, executes [block], and steps out. */
-    private inline fun IonReaderContinuableCore.readContainer(block: IonReaderContinuableCore.() -> Unit) { stepIntoContainer(); block(); stepOutOfContainer() }
+    private inline fun readContainer(block: () -> Unit) { stepIntoContainer(); block(); stepOutOfContainer() }
 
     /** Executes [block] for each remaining value at the current reader depth. */
-    private inline fun IonReaderContinuableCore.forEachRemaining(block: IonReaderContinuableCore.(IonType) -> Unit) { while (nextValue() != IonCursor.Event.END_CONTAINER) { block(encodingType) } }
+    private inline fun forEachRemaining(block: (IonType) -> Unit) { while (nextValue()) { block(encodingType()!!) } }
 
     /** Steps into a container, executes [block] for each value at that reader depth, and steps out. */
-    private inline fun IonReaderContinuableCore.forEachInContainer(block: IonReaderContinuableCore.(IonType) -> Unit) = readContainer { forEachRemaining(block) }
-
-    private fun IonReaderContinuableCore.getTypeAnnotationSymbols(): List<SymbolToken> {
-        if (!hasAnnotations()) {
-            return emptyList()
-        }
-        val annotations = arrayListOf<SymbolToken>()
-        consumeAnnotationTokens { annotations += it }
-        return annotations
-    }
+    private inline fun forEachInContainer(block: (IonType) -> Unit) = readContainer { forEachRemaining(block) }
+
+    protected abstract fun hasAnnotations(): Boolean
+    protected abstract fun fieldNameSymbol(): SymbolToken
+    protected abstract fun encodingType(): IonType?
+
+    /** Returns true if positioned on a value; false if at container or stream end. */
+    protected abstract fun nextValue(): Boolean
+    protected abstract fun stringValue(): String
+    protected abstract fun intValue(): Int
+    protected abstract fun decimalValue(): BigDecimal
+    protected abstract fun doubleValue(): Double
+    protected abstract fun stepIntoContainer()
+    protected abstract fun stepOutOfContainer()
+    protected abstract fun getTypeAnnotationSymbols(): List<SymbolToken>
+    protected abstract fun integerSize(): IntegerSize?
+    protected abstract fun booleanValue(): Boolean
+    protected abstract fun isNullValue(): Boolean
+    protected abstract fun longValue(): Long
+    protected abstract fun bigIntegerValue(): BigInteger
+    protected abstract fun timestampValue(): Timestamp
+    protected abstract fun newBytes(): ByteArray
+    protected abstract fun symbolValue(): SymbolToken
 }

src/main/java/com/amazon/ion/impl/macro/MacroCompilerContinuable.kt

@@ -0,0 +1,71 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package com.amazon.ion.impl.macro
+
+import com.amazon.ion.*
+import com.amazon.ion.impl.IonReaderContinuableCore
+import java.math.BigDecimal
+import java.math.BigInteger
+
+/**
+ * A [MacroCompiler] that wraps an [IonReaderContinuableCore].
+ */
+class MacroCompilerContinuable(
+    val reader: IonReaderContinuableCore,
+    getMacro: (MacroRef) -> Macro?
+) : MacroCompiler(getMacro) {
+
+    // TODO: Make sure that we can throw exceptions if there's an over-sized value.
+
+    override fun hasAnnotations(): Boolean = reader.hasAnnotations()
+
+    override fun fieldNameSymbol(): SymbolToken = reader.fieldNameSymbol
+
+    override fun encodingType(): IonType? = reader.encodingType
+
+    override fun nextValue(): Boolean {
+        val event = reader.nextValue()
+        return event != IonCursor.Event.NEEDS_DATA && event != IonCursor.Event.END_CONTAINER
+    }
+
+    override fun stringValue(): String = reader.stringValue()
+
+    override fun intValue(): Int = reader.intValue()
+
+    override fun decimalValue(): BigDecimal = reader.decimalValue()
+
+    override fun doubleValue(): Double = reader.doubleValue()
+
+    override fun stepIntoContainer() {
+        reader.stepIntoContainer()
+    }
+
+    override fun stepOutOfContainer() {
+        reader.stepOutOfContainer()
+    }
+
+    override fun getTypeAnnotationSymbols(): List<SymbolToken> {
+        if (!reader.hasAnnotations()) {
+            return emptyList()
+        }
+        val annotations = arrayListOf<SymbolToken>()
+        reader.consumeAnnotationTokens { annotations += it }
+        return annotations
+    }
+
+    override fun symbolValue(): SymbolToken = reader.symbolValue()
+
+    override fun newBytes(): ByteArray = reader.newBytes()
+
+    override fun timestampValue(): Timestamp = reader.timestampValue()
+
+    override fun bigIntegerValue(): BigInteger = reader.bigIntegerValue()
+
+    override fun longValue(): Long = reader.longValue()
+
+    override fun isNullValue(): Boolean = reader.isNullValue
+
+    override fun booleanValue(): Boolean = reader.booleanValue()
+
+    override fun integerSize(): IntegerSize? = reader.integerSize
+}