Adds implementations of repeat, make_field, and make_blob (#972)

src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt

@@ -7,6 +7,9 @@ import com.amazon.ion.SymbolToken
 import com.amazon.ion.impl._Private_RecyclingStack
 import com.amazon.ion.impl._Private_Utils.newSymbolToken
 import com.amazon.ion.impl.macro.Expression.*
+import com.amazon.ion.util.*
+import java.io.ByteArrayOutputStream
+import java.lang.IllegalStateException
 import java.math.BigDecimal
 
 /**
@@ -88,6 +91,30 @@ class MacroEvaluator {
                 macroEvaluator.expansionStack.pop()
             }
         }
+
+        /**
+         * Reads the first expanded value from one argument.
+         *
+         * Does not perform any sort of cardinality check, and leaves the evaluator stepped into the level of the
+         * returned expression. Returns null if the argument expansion produces no values.
+         */
+        fun readFirstExpandedArgumentValue(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator): DataModelExpression? {
+            val i = expansionInfo.i
+            expansionInfo.nextSourceExpression()
+
+            macroEvaluator.pushExpansion(
+                expansionKind = ExpansionKind.Values,
+                argsStartInclusive = i,
+                // There can only be one top-level expression for an argument (it's either a value, macro, or
+                // expression group) so we can set the end to one more than the start.
+                argsEndExclusive = i + 1,
+                environment = expansionInfo.environment ?: Environment.EMPTY,
+                expressions = expansionInfo.expressions!!,
+            )
+
+            val depth = macroEvaluator.expansionStack.size()
+            return macroEvaluator.expandNext(depth)
+        }
     }
 
     private object SimpleExpander : Expander {
@@ -127,7 +154,6 @@ class MacroEvaluator {
                 when (it) {
                     is StringValue -> sb.append(it.value)
                     is SymbolValue -> sb.append(it.value.assumeText())
-                    is NullValue -> {}
                     is DataModelValue -> throw IonException("Invalid argument type for 'make_string': ${it.type}")
                     is FieldName -> TODO("Unreachable. We shouldn't be able to get here without first encountering a StructValue.")
                 }
@@ -144,7 +170,6 @@ class MacroEvaluator {
                 when (it) {
                     is StringValue -> sb.append(it.value)
                     is SymbolValue -> sb.append(it.value.assumeText())
-                    is NullValue -> {}
                     is DataModelValue -> throw IonException("Invalid argument type for 'make_symbol': ${it.type}")
                     is FieldName -> TODO("Unreachable. We shouldn't be able to get here without first encountering a StructValue.")
                 }
@@ -154,6 +179,23 @@ class MacroEvaluator {
         }
     }
 
+    private object MakeBlobExpander : Expander {
+        override fun nextExpression(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator): Expression {
+            // TODO: See if we can create a `ByteArrayView` or similar class based on the principles of a Persistent
+            //       Collection in order to minimize copying (and therefore allocation).
+            val baos = ByteArrayOutputStream()
+            readExpandedArgumentValues(expansionInfo, macroEvaluator) {
+                when (it) {
+                    is LobValue -> baos.write(it.value)
+                    is DataModelValue -> throw IonException("Invalid argument type for 'make_blob': ${it.type}")
+                    is FieldName -> TODO("Unreachable. We shouldn't be able to get here without first encountering a StructValue.")
+                }
+                true // continue expansion
+            }
+            return BlobValue(value = baos.toByteArray())
+        }
+    }
+
     private object MakeDecimalExpander : Expander {
         override fun nextExpression(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator): Expression {
             val coefficient = readExactlyOneExpandedArgumentValue(expansionInfo, macroEvaluator, SystemMacro.MakeDecimal.signature[0].variableName)
@@ -199,18 +241,90 @@ class MacroEvaluator {
         }
     }
 
+    private object RepeatExpander : Expander {
+        /**
+         * Initializes the counter of the number of iterations remaining.
+         * [ExpansionInfo.additionalState] is the number of iterations remaining. Once initialized, it is always `Int`.
+         */
+        private fun init(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator): Int {
+            val nExpression = readExactlyOneExpandedArgumentValue(expansionInfo, macroEvaluator, "n")
+            var iterationsRemaining = when (nExpression) {
+                is LongIntValue -> nExpression.value.toInt()
+                is BigIntValue -> {
+                    if (nExpression.value.bitLength() >= Int.SIZE_BITS) {
+                        throw IonException("ion-java does not support repeats of more than ${Int.MAX_VALUE}")
+                    }
+                    nExpression.value.intValueExact()
+                }
+                else -> throw IonException("The first argument of repeat must be a positive integer")
+            }
+            if (iterationsRemaining <= 0) {
+                // TODO: Confirm https://github.com/amazon-ion/ion-docs/issues/350
+                throw IonException("The first argument of repeat must be a positive integer")
+            }
+            // Decrement because we're starting the first iteration right away.
+            iterationsRemaining--
+            expansionInfo.additionalState = iterationsRemaining
+            return iterationsRemaining
+        }
+
+        override fun nextExpression(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator): Expression {
+            val repeatsRemaining = expansionInfo.additionalState as? Int
+                ?: init(expansionInfo, macroEvaluator)
+
+            val repeatedExpressionIndex = expansionInfo.i
+            val next = readFirstExpandedArgumentValue(expansionInfo, macroEvaluator)
+            next ?: throw IonException("repeat macro requires at least one value for value parameter")
+            if (repeatsRemaining > 0) {
+                expansionInfo.additionalState = repeatsRemaining - 1
+                expansionInfo.i = repeatedExpressionIndex
+            }
+            return next
+        }
+    }
+
+    private object MakeFieldExpander : Expander {
+        override fun nextExpression(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator): Expression {
+            /**
+             * Uses [ExpansionInfo.additionalState] to track whether the expansion is on the field name or value.
+             * If unset, reads the field name. If set to 0, reads the field value.
+             */
+            return when (expansionInfo.additionalState) {
+                // First time, get the field name
+                null -> {
+                    val fieldName = readExactlyOneExpandedArgumentValue(expansionInfo, macroEvaluator, "field_name")
+                    val fieldNameExpression = when (fieldName) {
+                        is SymbolValue -> FieldName(fieldName.value)
+                        else -> throw IonException("the first argument of make_field must expand to exactly one symbol value")
+                    }
+                    expansionInfo.additionalState = 0
+                    fieldNameExpression
+                }
+                0 -> {
+                    val value = readExactlyOneExpandedArgumentValue(expansionInfo, macroEvaluator, "value")
+                    expansionInfo.additionalState = 1
+                    value
+                }
+                else -> throw IllegalStateException("Unreachable")
+            }
+        }
+    }
+
     private enum class ExpansionKind(val expander: Expander) {
         Container(SimpleExpander),
         TemplateBody(SimpleExpander),
         Values(SimpleExpander),
         Annotate(AnnotateExpander),
         MakeString(MakeStringExpander),
         MakeSymbol(MakeSymbolExpander),
+        MakeBlob(MakeBlobExpander),
         MakeDecimal(MakeDecimalExpander),
+        MakeField(MakeFieldExpander),
         IfNone(IfExpander.IF_NONE),
         IfSome(IfExpander.IF_SOME),
         IfSingle(IfExpander.IF_SINGLE),
         IfMulti(IfExpander.IF_MULTI),
+        Repeat(RepeatExpander),
         ;
 
         companion object {
@@ -222,11 +336,14 @@ class MacroEvaluator {
                     SystemMacro.Annotate -> Annotate
                     SystemMacro.MakeString -> MakeString
                     SystemMacro.MakeSymbol -> MakeSymbol
+                    SystemMacro.MakeBlob -> MakeBlob
                     SystemMacro.MakeDecimal -> MakeDecimal
                     SystemMacro.IfNone -> IfNone
                     SystemMacro.IfSome -> IfSome
                     SystemMacro.IfSingle -> IfSingle
                     SystemMacro.IfMulti -> IfMulti
+                    SystemMacro.Repeat -> Repeat
+                    SystemMacro.MakeField -> MakeField
                 }
             }
         }
@@ -253,6 +370,17 @@ class MacroEvaluator {
         /** Current position within [expressions] of this expansion */
         @JvmField var i: Int = 0
 
+        /**
+         * Field for storing any additional state required in an expander.
+         *
+         * TODO: Once all system macros are implemented, see if we can make this an int instead
+         *
+         * There is currently some lost space in ExpansionInfo. We can add one more `additionalState` field without
+         * actually increasing the object size.
+         */
+        @JvmField
+        var additionalState: Any? = null
+
         /** Checks if this expansion can produce any more expressions */
         override fun hasNext(): Boolean = i < endExclusive
 
@@ -491,6 +619,7 @@ class MacroEvaluator {
             it.expressions = expressions
             it.i = argsStartInclusive
             it.endExclusive = argsEndExclusive
+            it.additionalState = null
         }
     }
 

src/main/java/com/amazon/ion/impl/macro/ParameterFactory.kt

@@ -11,6 +11,8 @@ object ParameterFactory {
     @JvmStatic
     fun zeroToManyTagged(name: String) = Parameter(name, ParameterEncoding.Tagged, ParameterCardinality.ZeroOrMore)
     @JvmStatic
+    fun oneToManyTagged(name: String) = Parameter(name, ParameterEncoding.Tagged, ParameterCardinality.OneOrMore)
+    @JvmStatic
     fun exactlyOneTagged(name: String) = Parameter(name, ParameterEncoding.Tagged, ParameterCardinality.ExactlyOne)
     @JvmStatic
     fun exactlyOneFlexInt(name: String) = Parameter(name, ParameterEncoding.CompactInt, ParameterCardinality.ExactlyOne)

src/main/java/com/amazon/ion/impl/macro/SystemMacro.kt

@@ -4,6 +4,7 @@ package com.amazon.ion.impl.macro
 
 import com.amazon.ion.impl.macro.ParameterFactory.exactlyOneFlexInt
 import com.amazon.ion.impl.macro.ParameterFactory.exactlyOneTagged
+import com.amazon.ion.impl.macro.ParameterFactory.oneToManyTagged
 import com.amazon.ion.impl.macro.ParameterFactory.zeroToManyTagged
 
 /**
@@ -15,8 +16,18 @@ enum class SystemMacro(val id: Byte, val macroName: String, override val signatu
     Annotate(2, "annotate", listOf(zeroToManyTagged("ann"), exactlyOneTagged("value"))),
     MakeString(3, "make_string", listOf(zeroToManyTagged("text"))),
     MakeSymbol(4, "make_symbol", listOf(zeroToManyTagged("text"))),
+    MakeBlob(5, "make_blob", listOf(zeroToManyTagged("bytes"))),
     MakeDecimal(6, "make_decimal", listOf(exactlyOneFlexInt("coefficient"), exactlyOneFlexInt("exponent"))),
 
+    Repeat(17, "repeat", listOf(exactlyOneTagged("n"), oneToManyTagged("value"))),
+
+    MakeField(
+        22, "make_field",
+        listOf(
+            Macro.Parameter("field_name", Macro.ParameterEncoding.CompactSymbol, Macro.ParameterCardinality.ExactlyOne), exactlyOneTagged("value")
+        )
+    ),
+
     // TODO: Other system macros
 
     // Technically not system macros, but special forms. However, it's easier to model them as if they are macros in TDL.