Implements encoding directive macros and comment macro (#973)

src/test/java/com/amazon/ion/impl/macro/ExpressionBuilderDsl.kt → src/main/java/com/amazon/ion/impl/macro/ExpressionBuilderDsl.kt

@@ -3,17 +3,24 @@
 package com.amazon.ion.impl.macro
 
 import com.amazon.ion.*
+import com.amazon.ion.impl.*
 import com.amazon.ion.impl.macro.Expression.*
 import java.math.BigInteger
 import kotlin.reflect.KFunction1
 
+/**
+ * Nothing in this file should be made public because it would expose the shaded kotlin std library in our public API.
+ */
+
 /** A marker annotation for a [type-safe builder](https://kotlinlang.org/docs/type-safe-builders.html). */
 @DslMarker
-annotation class ExpressionBuilderDslMarker
+internal annotation class ExpressionBuilderDslMarker
 
 /** Base DSL; functions are common for [DataModelExpression], [TemplateBodyExpression], and [EExpressionBodyExpression]. */
-interface ValuesDsl {
+internal interface ValuesDsl {
     fun <T> annotated(annotations: List<SymbolToken>, valueFn: KFunction1<T, Unit>, value: T)
+    fun <T> annotated(annotation: SystemSymbols_1_1, valueFn: KFunction1<T, Unit>, value: T) =
+        annotated(listOf(annotation.token), valueFn, value)
     fun nullValue(value: IonType = IonType.NULL)
     fun bool(value: Boolean)
     fun int(value: Long)
@@ -22,20 +29,22 @@ interface ValuesDsl {
     fun decimal(value: Decimal)
     fun timestamp(value: Timestamp)
     fun symbol(value: SymbolToken)
+    fun symbol(value: String) = symbol(_Private_Utils.newSymbolToken(value))
+    fun symbol(value: SystemSymbols_1_1) = symbol(value.token)
     fun string(value: String)
     fun clob(value: ByteArray)
     fun blob(value: ByteArray)
 
     /** Helper interface for use when building the content of a struct */
     interface Fields {
         fun fieldName(fieldName: SymbolToken)
-        fun fieldName(fieldName: String) = fieldName(FakeSymbolToken(fieldName, -1))
+        fun fieldName(fieldName: String) = fieldName(_Private_Utils.newSymbolToken(fieldName))
     }
 }
 
 /** DSL for building [DataModelExpression] lists. */
 @ExpressionBuilderDslMarker
-interface DataModelDsl : ValuesDsl {
+internal interface DataModelDsl : ValuesDsl {
     fun list(content: DataModelDsl.() -> Unit)
     fun sexp(content: DataModelDsl.() -> Unit)
     fun struct(content: Fields.() -> Unit)
@@ -46,7 +55,7 @@ interface DataModelDsl : ValuesDsl {
 
 /** DSL for building [TemplateBodyExpression] lists. */
 @ExpressionBuilderDslMarker
-interface TemplateDsl : ValuesDsl {
+internal interface TemplateDsl : ValuesDsl {
     fun macro(macro: Macro, arguments: InvocationBody.() -> Unit)
     fun variable(signatureIndex: Int)
     fun list(content: TemplateDsl.() -> Unit)
@@ -64,7 +73,7 @@ interface TemplateDsl : ValuesDsl {
 
 /** DSL for building [EExpressionBodyExpression] lists. */
 @ExpressionBuilderDslMarker
-interface EExpDsl : ValuesDsl {
+internal interface EExpDsl : ValuesDsl {
     fun eexp(macro: Macro, arguments: InvocationBody.() -> Unit)
     fun list(content: EExpDsl.() -> Unit)
     fun sexp(content: EExpDsl.() -> Unit)

src/main/java/com/amazon/ion/impl/macro/Macro.kt

@@ -9,6 +9,7 @@ import com.amazon.ion.impl.TaglessEncoding
  */
 sealed interface Macro {
     val signature: List<Parameter>
+    val body: List<Expression.TemplateBodyExpression>?
     val dependencies: Iterable<Macro>
 
     data class Parameter(val variableName: String, val type: ParameterEncoding, val cardinality: ParameterCardinality) {

src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt

@@ -7,9 +7,7 @@ import com.amazon.ion.SymbolToken
 import com.amazon.ion.impl._Private_RecyclingStack
 import com.amazon.ion.impl._Private_Utils.newSymbolToken
 import com.amazon.ion.impl.macro.Expression.*
-import com.amazon.ion.util.*
 import java.io.ByteArrayOutputStream
-import java.lang.IllegalStateException
 import java.math.BigDecimal
 
 /**
@@ -330,7 +328,9 @@ class MacroEvaluator {
         companion object {
             @JvmStatic
             fun forSystemMacro(macro: SystemMacro): ExpansionKind {
-                return when (macro) {
+                return if (macro.body != null) {
+                    TemplateBody
+                } else when (macro) {
                     SystemMacro.None -> Values // "none" takes no args, so we can treat it as an empty "values" expansion
                     SystemMacro.Values -> Values
                     SystemMacro.Annotate -> Annotate
@@ -344,6 +344,7 @@ class MacroEvaluator {
                     SystemMacro.IfMulti -> IfMulti
                     SystemMacro.Repeat -> Repeat
                     SystemMacro.MakeField -> MakeField
+                    else -> throw IllegalStateException("Unreachable. All other macros have a template body.")
                 }
             }
         }
@@ -585,21 +586,20 @@ class MacroEvaluator {
         encodingExpressions: List<Expression>,
     ) {
         val argIndices = calculateArgumentIndices(macro, encodingExpressions, argsStartInclusive, argsEndExclusive)
-
-        when (macro) {
-            is TemplateMacro -> pushExpansion(
+        val templateBody = macro.body
+        if (templateBody == null) {
+            // If there's no template body, it must be a system macro.
+            macro as SystemMacro
+            val kind = ExpansionKind.forSystemMacro(macro)
+            pushExpansion(kind, argsStartInclusive, argsEndExclusive, environment, encodingExpressions)
+        } else {
+            pushExpansion(
                 ExpansionKind.TemplateBody,
                 argsStartInclusive = 0,
-                argsEndExclusive = macro.body.size,
-                expressions = macro.body,
+                argsEndExclusive = templateBody.size,
+                expressions = templateBody,
                 environment = environment.createChild(encodingExpressions, argIndices)
             )
-            // TODO: Values and MakeString have the same code in their blocks. As we get further along, see
-            //       if this is generally applicable for all system macros.
-            is SystemMacro -> {
-                val kind = ExpansionKind.forSystemMacro(macro)
-                pushExpansion(kind, argsStartInclusive, argsEndExclusive, environment, encodingExpressions,)
-            }
         }
     }
 

src/main/java/com/amazon/ion/impl/macro/MacroEvaluatorAsIonReader.kt

@@ -93,9 +93,9 @@ class MacroEvaluatorAsIonReader(
     }
 
     override fun close() { /* Nothing to do (yet) */ }
-    override fun <T : Any?> asFacet(facetType: Class<T>?): Nothing = TODO("Not supported")
+    override fun <T : Any?> asFacet(facetType: Class<T>?): Nothing? = null
     override fun getDepth(): Int = containerStack.size()
-    override fun getSymbolTable(): SymbolTable = TODO("Not implemented in this abstraction")
+    override fun getSymbolTable(): SymbolTable? = null
 
     override fun getType(): IonType? = currentValueExpression?.type
 

src/main/java/com/amazon/ion/impl/macro/ParameterFactory.kt

@@ -11,6 +11,8 @@ object ParameterFactory {
     @JvmStatic
     fun zeroToManyTagged(name: String) = Parameter(name, ParameterEncoding.Tagged, ParameterCardinality.ZeroOrMore)
     @JvmStatic
+    fun zeroOrOneTagged(name: String) = Parameter(name, ParameterEncoding.Tagged, ParameterCardinality.ZeroOrOne)
+    @JvmStatic
     fun oneToManyTagged(name: String) = Parameter(name, ParameterEncoding.Tagged, ParameterCardinality.OneOrMore)
     @JvmStatic
     fun exactlyOneTagged(name: String) = Parameter(name, ParameterEncoding.Tagged, ParameterCardinality.ExactlyOne)

src/main/java/com/amazon/ion/impl/macro/SystemMacro.kt

@@ -2,15 +2,27 @@
 // SPDX-License-Identifier: Apache-2.0
 package com.amazon.ion.impl.macro
 
+import com.amazon.ion.impl.*
+import com.amazon.ion.impl.SystemSymbols_1_1.*
+import com.amazon.ion.impl.macro.ExpressionBuilderDsl.Companion.templateBody
 import com.amazon.ion.impl.macro.ParameterFactory.exactlyOneFlexInt
 import com.amazon.ion.impl.macro.ParameterFactory.exactlyOneTagged
 import com.amazon.ion.impl.macro.ParameterFactory.oneToManyTagged
+import com.amazon.ion.impl.macro.ParameterFactory.zeroOrOneTagged
 import com.amazon.ion.impl.macro.ParameterFactory.zeroToManyTagged
 
 /**
  * Macros that are built in, rather than being defined by a template.
  */
-enum class SystemMacro(val id: Byte, val macroName: String, override val signature: List<Macro.Parameter>) : Macro {
+enum class SystemMacro(val id: Byte, val macroName: String, override val signature: List<Macro.Parameter>, override val body: List<Expression.TemplateBodyExpression>? = null) : Macro {
+    // Technically not system macros, but special forms. However, it's easier to model them as if they are macros in TDL.
+    // We give them an ID of -1 to distinguish that they are not addressable outside TDL.
+    IfNone(-1, "if_none", listOf(zeroToManyTagged("stream"), zeroToManyTagged("true_branch"), zeroToManyTagged("false_branch"))),
+    IfSome(-1, "if_some", listOf(zeroToManyTagged("stream"), zeroToManyTagged("true_branch"), zeroToManyTagged("false_branch"))),
+    IfSingle(-1, "if_single", listOf(zeroToManyTagged("stream"), zeroToManyTagged("true_branch"), zeroToManyTagged("false_branch"))),
+    IfMulti(-1, "if_multi", listOf(zeroToManyTagged("stream"), zeroToManyTagged("true_branch"), zeroToManyTagged("false_branch"))),
+
+    // The real macros
     None(0, "none", emptyList()),
     Values(1, "values", listOf(zeroToManyTagged("values"))),
     Annotate(2, "annotate", listOf(zeroToManyTagged("ann"), exactlyOneTagged("value"))),
@@ -19,8 +31,150 @@ enum class SystemMacro(val id: Byte, val macroName: String, override val signatu
     MakeBlob(5, "make_blob", listOf(zeroToManyTagged("bytes"))),
     MakeDecimal(6, "make_decimal", listOf(exactlyOneFlexInt("coefficient"), exactlyOneFlexInt("exponent"))),
 
+    /**
+     * ```ion
+     * (macro set_symbols (symbols*)
+     *        $ion_encoding::(
+     *          (symbol_table [(%symbols)])
+     *          (macro_table $ion_encoding)
+     *        ))
+     * ```
+     */
+    SetSymbols(
+        11, "set_symbols", listOf(zeroToManyTagged("symbols")),
+        templateBody {
+            annotated(ION_ENCODING, ::sexp) {
+                sexp {
+                    symbol(SYMBOL_TABLE)
+                    list { variable(0) }
+                }
+                sexp {
+                    symbol(MACRO_TABLE)
+                    symbol(ION_ENCODING)
+                }
+            }
+        }
+    ),
+
+    /**
+     * ```ion
+     * (macro add_symbols (symbols*)
+     *        $ion_encoding::(
+     *          (symbol_table $ion_encoding [(%symbols)])
+     *          (macro_table $ion_encoding)
+     *        ))
+     * ```
+     */
+    AddSymbols(
+        12, "add_symbols", listOf(zeroToManyTagged("symbols")),
+        templateBody {
+            annotated(ION_ENCODING, ::sexp) {
+                sexp {
+                    symbol(SYMBOL_TABLE)
+                    symbol(ION_ENCODING)
+                    list { variable(0) }
+                }
+                sexp {
+                    symbol(MACRO_TABLE)
+                    symbol(ION_ENCODING)
+                }
+            }
+        }
+    ),
+
+    /**
+     * ```ion
+     * (macro set_macros (macros*)
+     *        $ion_encoding::(
+     *          (symbol_table $ion_encoding)
+     *          (macro_table (%macros))
+     *        ))
+     * ```
+     */
+    SetMacros(
+        13, "set_macros", listOf(zeroToManyTagged("macros")),
+        templateBody {
+            annotated(ION_ENCODING, ::sexp) {
+                sexp {
+                    symbol(SYMBOL_TABLE)
+                    symbol(ION_ENCODING)
+                }
+                sexp {
+                    symbol(MACRO_TABLE)
+                    variable(0)
+                }
+            }
+        }
+    ),
+
+    /**
+     * ```ion
+     * (macro add_macros (macros*)
+     *        $ion_encoding::(
+     *          (symbol_table $ion_encoding)
+     *          (macro_table $ion_encoding (%macros))
+     *        ))
+     * ```
+     */
+    AddMacros(
+        14, "add_macros", listOf(zeroToManyTagged("macros")),
+        templateBody {
+            annotated(ION_ENCODING, ::sexp) {
+                sexp {
+                    symbol(SYMBOL_TABLE)
+                    symbol(ION_ENCODING)
+                }
+                sexp {
+                    symbol(MACRO_TABLE)
+                    symbol(ION_ENCODING)
+                    variable(0)
+                }
+            }
+        }
+    ),
+
+    /**
+     * ```ion
+     * (macro use (catalog_key version?)
+     *        $ion_encoding::(
+     *          (import the_module (%catalog_key) (.if_none (%version) 1 (%version)))
+     *          (symbol_table $ion_encoding the_module)
+     *          (macro_table $ion_encoding the_module)
+     *        ))
+     * ```
+     */
+    Use(
+        15, "use", listOf(exactlyOneTagged("catalog_key"), zeroOrOneTagged("version")),
+        templateBody {
+            val theModule = _Private_Utils.newSymbolToken("the_module")
+            annotated(ION_ENCODING, ::sexp) {
+                sexp {
+                    symbol(IMPORT)
+                    symbol(theModule)
+                    variable(0)
+                    macro(IfNone) {
+                        variable(1)
+                        int(1)
+                        variable(1)
+                    }
+                }
+                sexp {
+                    symbol(SYMBOL_TABLE)
+                    symbol(ION_ENCODING)
+                    symbol(theModule)
+                }
+                sexp {
+                    symbol(MACRO_TABLE)
+                    symbol(ION_ENCODING)
+                    symbol(theModule)
+                }
+            }
+        }
+    ),
+
     Repeat(17, "repeat", listOf(exactlyOneTagged("n"), oneToManyTagged("value"))),
 
+    Comment(21, "comment", listOf(zeroToManyTagged("values")), templateBody { macro(None) {} }),
     MakeField(
         22, "make_field",
         listOf(
@@ -29,17 +183,14 @@ enum class SystemMacro(val id: Byte, val macroName: String, override val signatu
     ),
 
     // TODO: Other system macros
-
-    // Technically not system macros, but special forms. However, it's easier to model them as if they are macros in TDL.
-    // We give them an ID of -1 to distinguish that they are not addressable outside TDL.
-    IfNone(-1, "if_none", listOf(zeroToManyTagged("stream"), zeroToManyTagged("true_branch"), zeroToManyTagged("false_branch"))),
-    IfSome(-1, "if_some", listOf(zeroToManyTagged("stream"), zeroToManyTagged("true_branch"), zeroToManyTagged("false_branch"))),
-    IfSingle(-1, "if_single", listOf(zeroToManyTagged("stream"), zeroToManyTagged("true_branch"), zeroToManyTagged("false_branch"))),
-    IfMulti(-1, "if_multi", listOf(zeroToManyTagged("stream"), zeroToManyTagged("true_branch"), zeroToManyTagged("false_branch"))),
     ;
 
     override val dependencies: List<Macro>
-        get() = emptyList()
+        get() = body
+            ?.filterIsInstance<Expression.MacroInvocation>()
+            ?.map(Expression.MacroInvocation::macro)
+            ?.distinct()
+            ?: emptyList()
 
     companion object {
 

src/main/java/com/amazon/ion/impl/macro/TemplateMacro.kt

@@ -1,10 +1,12 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
 package com.amazon.ion.impl.macro
 
 /**
  * Represents a template macro. A template macro is defined by a signature, and a list of template expressions.
  * A template macro only gains a name and/or ID when it is added to a macro table.
  */
-data class TemplateMacro(override val signature: List<Macro.Parameter>, val body: List<Expression.TemplateBodyExpression>) :
+class TemplateMacro(override val signature: List<Macro.Parameter>, override val body: List<Expression.TemplateBodyExpression>) :
     Macro {
     // TODO: Consider rewriting the body of the macro if we discover that there are any macros invoked using only
     //       constants as arguments—either at compile time or lazily.