Adds expansions for make_symbol, make_decimal, none, and annotate (#955)

src/main/java/com/amazon/ion/impl/_Private_Utils.java

@@ -180,6 +180,14 @@ public static SymbolTokenImpl newSymbolToken(String text, int sid)
         return new SymbolTokenImpl(text, sid);
     }
 
+    /**
+     * @return not null
+     */
+    public static SymbolToken newSymbolToken(String text)
+    {
+        return new SymbolTokenImpl(text, UNKNOWN_SYMBOL_ID);
+    }
+
     /** Cached copy of $0 */
     public static final SymbolTokenImpl SYMBOL_0 = newSymbolToken((String) null, 0);
 

src/main/java/com/amazon/ion/impl/macro/Expression.kt

@@ -60,6 +60,8 @@ sealed interface Expression {
     sealed interface DataModelValue : DataModelExpression {
         val annotations: List<SymbolToken>
         val type: IonType
+
+        fun withAnnotations(annotations: List<SymbolToken>): DataModelValue
     }
 
     /** Expressions that represent Ion container types */
@@ -82,32 +84,44 @@ sealed interface Expression {
     data class ExpressionGroup(override val selfIndex: Int, override val endExclusive: Int) : EExpressionBodyExpression, TemplateBodyExpression, HasStartAndEnd
 
     // Scalars
-    data class NullValue(override val annotations: List<SymbolToken> = emptyList(), override val type: IonType) : DataModelValue
+    data class NullValue(override val annotations: List<SymbolToken> = emptyList(), override val type: IonType) : DataModelValue {
+        override fun withAnnotations(annotations: List<SymbolToken>) = copy(annotations = annotations)
+    }
 
     data class BoolValue(override val annotations: List<SymbolToken> = emptyList(), val value: Boolean) : DataModelValue {
         override val type: IonType get() = IonType.BOOL
+        override fun withAnnotations(annotations: List<SymbolToken>) = copy(annotations = annotations)
     }
 
-    sealed interface IntValue : DataModelValue
+    sealed interface IntValue : DataModelValue {
+        val bigIntegerValue: BigInteger
+    }
 
     data class LongIntValue(override val annotations: List<SymbolToken> = emptyList(), val value: Long) : IntValue {
         override val type: IonType get() = IonType.INT
+        override fun withAnnotations(annotations: List<SymbolToken>) = copy(annotations = annotations)
+        override val bigIntegerValue: BigInteger get() = BigInteger.valueOf(value)
     }
 
     data class BigIntValue(override val annotations: List<SymbolToken> = emptyList(), val value: BigInteger) : IntValue {
         override val type: IonType get() = IonType.INT
+        override fun withAnnotations(annotations: List<SymbolToken>) = copy(annotations = annotations)
+        override val bigIntegerValue: BigInteger get() = value
     }
 
     data class FloatValue(override val annotations: List<SymbolToken> = emptyList(), val value: Double) : DataModelValue {
         override val type: IonType get() = IonType.FLOAT
+        override fun withAnnotations(annotations: List<SymbolToken>) = copy(annotations = annotations)
     }
 
     data class DecimalValue(override val annotations: List<SymbolToken> = emptyList(), val value: BigDecimal) : DataModelValue {
         override val type: IonType get() = IonType.DECIMAL
+        override fun withAnnotations(annotations: List<SymbolToken>) = copy(annotations = annotations)
     }
 
     data class TimestampValue(override val annotations: List<SymbolToken> = emptyList(), val value: Timestamp) : DataModelValue {
         override val type: IonType get() = IonType.TIMESTAMP
+        override fun withAnnotations(annotations: List<SymbolToken>) = copy(annotations = annotations)
     }
 
     sealed interface TextValue : DataModelValue {
@@ -117,11 +131,13 @@ sealed interface Expression {
     data class StringValue(override val annotations: List<SymbolToken> = emptyList(), val value: String) : TextValue {
         override val type: IonType get() = IonType.STRING
         override val stringValue: String get() = value
+        override fun withAnnotations(annotations: List<SymbolToken>) = copy(annotations = annotations)
     }
 
     data class SymbolValue(override val annotations: List<SymbolToken> = emptyList(), val value: SymbolToken) : TextValue {
         override val type: IonType get() = IonType.SYMBOL
         override val stringValue: String get() = value.assumeText()
+        override fun withAnnotations(annotations: List<SymbolToken>) = copy(annotations = annotations)
     }
 
     sealed interface LobValue : DataModelValue {
@@ -133,6 +149,7 @@ sealed interface Expression {
     // We must override hashcode and equals in the lob types because `value` is a `byte[]`
     data class BlobValue(override val annotations: List<SymbolToken> = emptyList(), override val value: ByteArray) : LobValue {
         override val type: IonType get() = IonType.BLOB
+        override fun withAnnotations(annotations: List<SymbolToken>) = copy(annotations = annotations)
         override fun hashCode(): Int = annotations.hashCode() * 31 + value.contentHashCode()
         override fun equals(other: Any?): Boolean {
             if (this === other) return true
@@ -144,6 +161,7 @@ sealed interface Expression {
 
     data class ClobValue(override val annotations: List<SymbolToken> = emptyList(), override val value: ByteArray) : LobValue {
         override val type: IonType get() = IonType.CLOB
+        override fun withAnnotations(annotations: List<SymbolToken>) = copy(annotations = annotations)
         override fun hashCode(): Int = annotations.hashCode() * 31 + value.contentHashCode()
         override fun equals(other: Any?): Boolean {
             if (this === other) return true
@@ -165,6 +183,7 @@ sealed interface Expression {
         override val endExclusive: Int
     ) : DataModelContainer {
         override val type: IonType get() = IonType.LIST
+        override fun withAnnotations(annotations: List<SymbolToken>) = copy(annotations = annotations)
     }
 
     /**
@@ -176,6 +195,7 @@ sealed interface Expression {
         override val endExclusive: Int
     ) : DataModelContainer {
         override val type: IonType get() = IonType.SEXP
+        override fun withAnnotations(annotations: List<SymbolToken>) = copy(annotations = annotations)
     }
 
     /**
@@ -188,6 +208,7 @@ sealed interface Expression {
         val templateStructIndex: Map<String, List<Int>>
     ) : DataModelContainer {
         override val type: IonType get() = IonType.STRUCT
+        override fun withAnnotations(annotations: List<SymbolToken>) = copy(annotations = annotations)
     }
 
     data class FieldName(val value: SymbolToken) : DataModelExpression

src/main/java/com/amazon/ion/impl/macro/Macro.kt

@@ -3,6 +3,7 @@
 package com.amazon.ion.impl.macro
 
 import com.amazon.ion.impl.*
+import com.amazon.ion.impl.macro.Macro.Parameter.Companion.exactlyOneTagged
 import com.amazon.ion.impl.macro.Macro.Parameter.Companion.zeroToManyTagged
 
 /**
@@ -116,8 +117,19 @@ data class TemplateMacro(override val signature: List<Macro.Parameter>, val body
  * Macros that are built in, rather than being defined by a template.
  */
 enum class SystemMacro(val macroName: String, override val signature: List<Macro.Parameter>) : Macro {
+    None("none", emptyList()),
     Values("values", listOf(zeroToManyTagged("values"))),
+    Annotate("annotate", listOf(zeroToManyTagged("ann"), exactlyOneTagged("value"))),
     MakeString("make_string", listOf(zeroToManyTagged("text"))),
+    MakeSymbol("make_symbol", listOf(zeroToManyTagged("text"))),
+    MakeDecimal(
+        "make_decimal",
+        listOf(
+            Macro.Parameter("coefficient", Macro.ParameterEncoding.CompactInt, Macro.ParameterCardinality.ExactlyOne),
+            Macro.Parameter("exponent", Macro.ParameterEncoding.CompactInt, Macro.ParameterCardinality.ExactlyOne),
+        )
+    ),
+
     // TODO: Other system macros
     ;
 

src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt

@@ -2,9 +2,12 @@
 // SPDX-License-Identifier: Apache-2.0
 package com.amazon.ion.impl.macro
 
-import com.amazon.ion.*
-import com.amazon.ion.impl.*
+import com.amazon.ion.IonException
+import com.amazon.ion.SymbolToken
+import com.amazon.ion.impl._Private_RecyclingStack
+import com.amazon.ion.impl._Private_Utils.newSymbolToken
 import com.amazon.ion.impl.macro.Expression.*
+import java.math.BigDecimal
 
 /**
  * Evaluates an EExpression from a List of [EExpressionBodyExpression] and the [TemplateBodyExpression]s
@@ -26,6 +29,56 @@ class MacroEvaluator {
      */
     private fun interface Expander {
         fun nextExpression(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator): Expression
+
+        /**
+         * Read the expanded values from one argument, returning exactly one value.
+         * Throws an exception if there is not exactly one expanded value.
+         */
+        fun readExactlyOneExpandedArgumentValue(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator, argName: String): DataModelExpression {
+            return readZeroOrOneExpandedArgumentValues(expansionInfo, macroEvaluator, argName)
+                ?: throw IonException("Argument $argName expanded to nothing.")
+        }
+
+        /**
+         * Read the expanded values from one argument, returning zero or one values.
+         * Throws an exception if there is more than one expanded value.
+         */
+        fun readZeroOrOneExpandedArgumentValues(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator, argName: String): DataModelExpression? {
+            var value: DataModelExpression? = null
+            readExpandedArgumentValues(expansionInfo, macroEvaluator) {
+                if (value == null) {
+                    value = it
+                } else {
+                    throw IonException("Too many values for argument $argName")
+                }
+            }
+            return value
+        }
+
+        /**
+         * Reads the expanded values from one argument.
+         */
+        fun readExpandedArgumentValues(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator, callback: (DataModelExpression) -> Unit) {
+            val i = expansionInfo.i
+            expansionInfo.nextSourceExpression()
+
+            macroEvaluator.pushExpansion(
+                expansionKind = ExpansionKind.Values,
+                argsStartInclusive = i,
+                // There can only be one top-level expression for an argument (it's either a value, macro, or
+                // expression group) so we can set the end to one more than the start.
+                argsEndExclusive = i + 1,
+                environment = expansionInfo.environment ?: Environment.EMPTY,
+                expressions = expansionInfo.expressions!!,
+            )
+
+            val depth = macroEvaluator.expansionStack.size()
+            var expr = macroEvaluator.expandNext(depth)
+            while (expr != null) {
+                callback(expr)
+                expr = macroEvaluator.expandNext(depth)
+            }
+        }
     }
 
     private object SimpleExpander : Expander {
@@ -34,40 +87,97 @@ class MacroEvaluator {
         }
     }
 
+    private object AnnotateExpander : Expander {
+        // TODO: Handle edge cases mentioned in https://github.com/amazon-ion/ion-docs/issues/347
+        override fun nextExpression(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator): Expression {
+            val annotations = mutableListOf<SymbolToken>()
+
+            readExpandedArgumentValues(expansionInfo, macroEvaluator) {
+                when (it) {
+                    is StringValue -> annotations.add(newSymbolToken(it.value))
+                    is SymbolValue -> annotations.add(it.value)
+                    is DataModelValue -> throw IonException("Annotation arguments must be string or symbol; found: ${it.type}")
+                    is FieldName -> TODO("Unreachable. Must encounter a StructValue first.")
+                }
+            }
+
+            val valueToAnnotate = readExactlyOneExpandedArgumentValue(expansionInfo, macroEvaluator, SystemMacro.Annotate.signature[1].variableName)
+
+            // It cannot be a FieldName expression because we haven't stepped into a struct, so it must be DataModelValue
+            valueToAnnotate as DataModelValue
+            // Combine the annotations
+            annotations.addAll(valueToAnnotate.annotations)
+            return valueToAnnotate.withAnnotations(annotations)
+        }
+    }
+
     private object MakeStringExpander : Expander {
         override fun nextExpression(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator): Expression {
-            // Tell the macro evaluator to treat this as a values expansion...
-            macroEvaluator.expansionStack.peek().expansionKind = ExpansionKind.Values
-            val minDepth = macroEvaluator.expansionStack.size()
-            // ...But capture the output and turn it into a String
             val sb = StringBuilder()
-            while (true) {
-                when (val expr: DataModelExpression? = macroEvaluator.expandNext(minDepth)) {
-                    is StringValue -> sb.append(expr.value)
-                    is SymbolValue -> sb.append(expr.value.assumeText())
+            readExpandedArgumentValues(expansionInfo, macroEvaluator) {
+                when (it) {
+                    is StringValue -> sb.append(it.value)
+                    is SymbolValue -> sb.append(it.value.assumeText())
                     is NullValue -> {}
-                    null -> break
-                    is DataModelValue -> throw IonException("Invalid argument type for 'make_string': ${expr.type}")
+                    is DataModelValue -> throw IonException("Invalid argument type for 'make_string': ${it.type}")
                     is FieldName -> TODO("Unreachable. We shouldn't be able to get here without first encountering a StructValue.")
                 }
             }
             return StringValue(value = sb.toString())
         }
     }
 
+    private object MakeSymbolExpander : Expander {
+        override fun nextExpression(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator): Expression {
+            val sb = StringBuilder()
+            readExpandedArgumentValues(expansionInfo, macroEvaluator) {
+                when (it) {
+                    is StringValue -> sb.append(it.value)
+                    is SymbolValue -> sb.append(it.value.assumeText())
+                    is NullValue -> {}
+                    is DataModelValue -> throw IonException("Invalid argument type for 'make_symbol': ${it.type}")
+                    is FieldName -> TODO("Unreachable. We shouldn't be able to get here without first encountering a StructValue.")
+                }
+            }
+            return SymbolValue(value = newSymbolToken(sb.toString()))
+        }
+    }
+
+    private object MakeDecimalExpander : Expander {
+        override fun nextExpression(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator): Expression {
+            val coefficient = readExactlyOneExpandedArgumentValue(expansionInfo, macroEvaluator, SystemMacro.MakeDecimal.signature[0].variableName)
+                .let { it as? IntValue }
+                ?.bigIntegerValue
+                ?: throw IonException("Coefficient must be an integer")
+            val exponent = readExactlyOneExpandedArgumentValue(expansionInfo, macroEvaluator, SystemMacro.MakeDecimal.signature[1].variableName)
+                .let { it as? IntValue }
+                ?.bigIntegerValue
+                ?: throw IonException("Exponent must be an integer")
+
+            return DecimalValue(value = BigDecimal(coefficient, -1 * exponent.intValueExact()))
+        }
+    }
+
     private enum class ExpansionKind(val expander: Expander) {
         Container(SimpleExpander),
         TemplateBody(SimpleExpander),
         Values(SimpleExpander),
+        Annotate(AnnotateExpander),
         MakeString(MakeStringExpander),
+        MakeSymbol(MakeSymbolExpander),
+        MakeDecimal(MakeDecimalExpander),
         ;
 
         companion object {
             @JvmStatic
             fun forSystemMacro(macro: SystemMacro): ExpansionKind {
                 return when (macro) {
+                    SystemMacro.None -> Values // "none" takes no args, so we can treat it as an empty "values" expansion
                     SystemMacro.Values -> Values
+                    SystemMacro.Annotate -> Annotate
                     SystemMacro.MakeString -> MakeString
+                    SystemMacro.MakeSymbol -> MakeSymbol
+                    SystemMacro.MakeDecimal -> MakeDecimal
                 }
             }
         }