diff --git a/src/test/java/com/amazon/ion/impl/macro/ExpressionBuilderDsl.kt b/src/main/java/com/amazon/ion/impl/macro/ExpressionBuilderDsl.kt similarity index 91% rename from src/test/java/com/amazon/ion/impl/macro/ExpressionBuilderDsl.kt rename to src/main/java/com/amazon/ion/impl/macro/ExpressionBuilderDsl.kt index f1fe286fc..7a1756361 100644 --- a/src/test/java/com/amazon/ion/impl/macro/ExpressionBuilderDsl.kt +++ b/src/main/java/com/amazon/ion/impl/macro/ExpressionBuilderDsl.kt @@ -3,17 +3,24 @@ package com.amazon.ion.impl.macro import com.amazon.ion.* +import com.amazon.ion.impl.* import com.amazon.ion.impl.macro.Expression.* import java.math.BigInteger import kotlin.reflect.KFunction1 +/** + * Nothing in this file should be made public because it would expose the shaded kotlin std library in our public API. + */ + /** A marker annotation for a [type-safe builder](https://kotlinlang.org/docs/type-safe-builders.html). */ @DslMarker -annotation class ExpressionBuilderDslMarker +internal annotation class ExpressionBuilderDslMarker /** Base DSL; functions are common for [DataModelExpression], [TemplateBodyExpression], and [EExpressionBodyExpression]. */ -interface ValuesDsl { +internal interface ValuesDsl { fun annotated(annotations: List, valueFn: KFunction1, value: T) + fun annotated(annotation: SystemSymbols_1_1, valueFn: KFunction1, value: T) = + annotated(listOf(annotation.token), valueFn, value) fun nullValue(value: IonType = IonType.NULL) fun bool(value: Boolean) fun int(value: Long) @@ -22,6 +29,8 @@ interface ValuesDsl { fun decimal(value: Decimal) fun timestamp(value: Timestamp) fun symbol(value: SymbolToken) + fun symbol(value: String) = symbol(_Private_Utils.newSymbolToken(value)) + fun symbol(value: SystemSymbols_1_1) = symbol(value.token) fun string(value: String) fun clob(value: ByteArray) fun blob(value: ByteArray) @@ -29,13 +38,13 @@ interface ValuesDsl { /** Helper interface for use when building the content of a struct */ interface Fields { fun fieldName(fieldName: SymbolToken) - fun fieldName(fieldName: String) = fieldName(FakeSymbolToken(fieldName, -1)) + fun fieldName(fieldName: String) = fieldName(_Private_Utils.newSymbolToken(fieldName)) } } /** DSL for building [DataModelExpression] lists. */ @ExpressionBuilderDslMarker -interface DataModelDsl : ValuesDsl { +internal interface DataModelDsl : ValuesDsl { fun list(content: DataModelDsl.() -> Unit) fun sexp(content: DataModelDsl.() -> Unit) fun struct(content: Fields.() -> Unit) @@ -46,7 +55,7 @@ interface DataModelDsl : ValuesDsl { /** DSL for building [TemplateBodyExpression] lists. */ @ExpressionBuilderDslMarker -interface TemplateDsl : ValuesDsl { +internal interface TemplateDsl : ValuesDsl { fun macro(macro: Macro, arguments: InvocationBody.() -> Unit) fun variable(signatureIndex: Int) fun list(content: TemplateDsl.() -> Unit) @@ -64,7 +73,7 @@ interface TemplateDsl : ValuesDsl { /** DSL for building [EExpressionBodyExpression] lists. */ @ExpressionBuilderDslMarker -interface EExpDsl : ValuesDsl { +internal interface EExpDsl : ValuesDsl { fun eexp(macro: Macro, arguments: InvocationBody.() -> Unit) fun list(content: EExpDsl.() -> Unit) fun sexp(content: EExpDsl.() -> Unit) diff --git a/src/main/java/com/amazon/ion/impl/macro/Macro.kt b/src/main/java/com/amazon/ion/impl/macro/Macro.kt index bac9ad60e..5632413fc 100644 --- a/src/main/java/com/amazon/ion/impl/macro/Macro.kt +++ b/src/main/java/com/amazon/ion/impl/macro/Macro.kt @@ -9,6 +9,7 @@ import com.amazon.ion.impl.TaglessEncoding */ sealed interface Macro { val signature: List + val body: List? val dependencies: Iterable data class Parameter(val variableName: String, val type: ParameterEncoding, val cardinality: ParameterCardinality) { diff --git a/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt b/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt index 0383770d1..2a345cead 100644 --- a/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt +++ b/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt @@ -7,9 +7,7 @@ import com.amazon.ion.SymbolToken import com.amazon.ion.impl._Private_RecyclingStack import com.amazon.ion.impl._Private_Utils.newSymbolToken import com.amazon.ion.impl.macro.Expression.* -import com.amazon.ion.util.* import java.io.ByteArrayOutputStream -import java.lang.IllegalStateException import java.math.BigDecimal /** @@ -330,7 +328,9 @@ class MacroEvaluator { companion object { @JvmStatic fun forSystemMacro(macro: SystemMacro): ExpansionKind { - return when (macro) { + return if (macro.body != null) { + TemplateBody + } else when (macro) { SystemMacro.None -> Values // "none" takes no args, so we can treat it as an empty "values" expansion SystemMacro.Values -> Values SystemMacro.Annotate -> Annotate @@ -344,6 +344,7 @@ class MacroEvaluator { SystemMacro.IfMulti -> IfMulti SystemMacro.Repeat -> Repeat SystemMacro.MakeField -> MakeField + else -> throw IllegalStateException("Unreachable. All other macros have a template body.") } } } @@ -585,21 +586,20 @@ class MacroEvaluator { encodingExpressions: List, ) { val argIndices = calculateArgumentIndices(macro, encodingExpressions, argsStartInclusive, argsEndExclusive) - - when (macro) { - is TemplateMacro -> pushExpansion( + val templateBody = macro.body + if (templateBody == null) { + // If there's no template body, it must be a system macro. + macro as SystemMacro + val kind = ExpansionKind.forSystemMacro(macro) + pushExpansion(kind, argsStartInclusive, argsEndExclusive, environment, encodingExpressions) + } else { + pushExpansion( ExpansionKind.TemplateBody, argsStartInclusive = 0, - argsEndExclusive = macro.body.size, - expressions = macro.body, + argsEndExclusive = templateBody.size, + expressions = templateBody, environment = environment.createChild(encodingExpressions, argIndices) ) - // TODO: Values and MakeString have the same code in their blocks. As we get further along, see - // if this is generally applicable for all system macros. - is SystemMacro -> { - val kind = ExpansionKind.forSystemMacro(macro) - pushExpansion(kind, argsStartInclusive, argsEndExclusive, environment, encodingExpressions,) - } } } diff --git a/src/main/java/com/amazon/ion/impl/macro/MacroEvaluatorAsIonReader.kt b/src/main/java/com/amazon/ion/impl/macro/MacroEvaluatorAsIonReader.kt index fd0fd197b..399a95643 100644 --- a/src/main/java/com/amazon/ion/impl/macro/MacroEvaluatorAsIonReader.kt +++ b/src/main/java/com/amazon/ion/impl/macro/MacroEvaluatorAsIonReader.kt @@ -93,9 +93,9 @@ class MacroEvaluatorAsIonReader( } override fun close() { /* Nothing to do (yet) */ } - override fun asFacet(facetType: Class?): Nothing = TODO("Not supported") + override fun asFacet(facetType: Class?): Nothing? = null override fun getDepth(): Int = containerStack.size() - override fun getSymbolTable(): SymbolTable = TODO("Not implemented in this abstraction") + override fun getSymbolTable(): SymbolTable? = null override fun getType(): IonType? = currentValueExpression?.type diff --git a/src/main/java/com/amazon/ion/impl/macro/ParameterFactory.kt b/src/main/java/com/amazon/ion/impl/macro/ParameterFactory.kt index 14e79dcae..79bd5bb8f 100644 --- a/src/main/java/com/amazon/ion/impl/macro/ParameterFactory.kt +++ b/src/main/java/com/amazon/ion/impl/macro/ParameterFactory.kt @@ -11,6 +11,8 @@ object ParameterFactory { @JvmStatic fun zeroToManyTagged(name: String) = Parameter(name, ParameterEncoding.Tagged, ParameterCardinality.ZeroOrMore) @JvmStatic + fun zeroOrOneTagged(name: String) = Parameter(name, ParameterEncoding.Tagged, ParameterCardinality.ZeroOrOne) + @JvmStatic fun oneToManyTagged(name: String) = Parameter(name, ParameterEncoding.Tagged, ParameterCardinality.OneOrMore) @JvmStatic fun exactlyOneTagged(name: String) = Parameter(name, ParameterEncoding.Tagged, ParameterCardinality.ExactlyOne) diff --git a/src/main/java/com/amazon/ion/impl/macro/SystemMacro.kt b/src/main/java/com/amazon/ion/impl/macro/SystemMacro.kt index 7b1482ae6..3af9130be 100644 --- a/src/main/java/com/amazon/ion/impl/macro/SystemMacro.kt +++ b/src/main/java/com/amazon/ion/impl/macro/SystemMacro.kt @@ -2,15 +2,27 @@ // SPDX-License-Identifier: Apache-2.0 package com.amazon.ion.impl.macro +import com.amazon.ion.impl.* +import com.amazon.ion.impl.SystemSymbols_1_1.* +import com.amazon.ion.impl.macro.ExpressionBuilderDsl.Companion.templateBody import com.amazon.ion.impl.macro.ParameterFactory.exactlyOneFlexInt import com.amazon.ion.impl.macro.ParameterFactory.exactlyOneTagged import com.amazon.ion.impl.macro.ParameterFactory.oneToManyTagged +import com.amazon.ion.impl.macro.ParameterFactory.zeroOrOneTagged import com.amazon.ion.impl.macro.ParameterFactory.zeroToManyTagged /** * Macros that are built in, rather than being defined by a template. */ -enum class SystemMacro(val id: Byte, val macroName: String, override val signature: List) : Macro { +enum class SystemMacro(val id: Byte, val macroName: String, override val signature: List, override val body: List? = null) : Macro { + // Technically not system macros, but special forms. However, it's easier to model them as if they are macros in TDL. + // We give them an ID of -1 to distinguish that they are not addressable outside TDL. + IfNone(-1, "if_none", listOf(zeroToManyTagged("stream"), zeroToManyTagged("true_branch"), zeroToManyTagged("false_branch"))), + IfSome(-1, "if_some", listOf(zeroToManyTagged("stream"), zeroToManyTagged("true_branch"), zeroToManyTagged("false_branch"))), + IfSingle(-1, "if_single", listOf(zeroToManyTagged("stream"), zeroToManyTagged("true_branch"), zeroToManyTagged("false_branch"))), + IfMulti(-1, "if_multi", listOf(zeroToManyTagged("stream"), zeroToManyTagged("true_branch"), zeroToManyTagged("false_branch"))), + + // The real macros None(0, "none", emptyList()), Values(1, "values", listOf(zeroToManyTagged("values"))), Annotate(2, "annotate", listOf(zeroToManyTagged("ann"), exactlyOneTagged("value"))), @@ -19,8 +31,150 @@ enum class SystemMacro(val id: Byte, val macroName: String, override val signatu MakeBlob(5, "make_blob", listOf(zeroToManyTagged("bytes"))), MakeDecimal(6, "make_decimal", listOf(exactlyOneFlexInt("coefficient"), exactlyOneFlexInt("exponent"))), + /** + * ```ion + * (macro set_symbols (symbols*) + * $ion_encoding::( + * (symbol_table [(%symbols)]) + * (macro_table $ion_encoding) + * )) + * ``` + */ + SetSymbols( + 11, "set_symbols", listOf(zeroToManyTagged("symbols")), + templateBody { + annotated(ION_ENCODING, ::sexp) { + sexp { + symbol(SYMBOL_TABLE) + list { variable(0) } + } + sexp { + symbol(MACRO_TABLE) + symbol(ION_ENCODING) + } + } + } + ), + + /** + * ```ion + * (macro add_symbols (symbols*) + * $ion_encoding::( + * (symbol_table $ion_encoding [(%symbols)]) + * (macro_table $ion_encoding) + * )) + * ``` + */ + AddSymbols( + 12, "add_symbols", listOf(zeroToManyTagged("symbols")), + templateBody { + annotated(ION_ENCODING, ::sexp) { + sexp { + symbol(SYMBOL_TABLE) + symbol(ION_ENCODING) + list { variable(0) } + } + sexp { + symbol(MACRO_TABLE) + symbol(ION_ENCODING) + } + } + } + ), + + /** + * ```ion + * (macro set_macros (macros*) + * $ion_encoding::( + * (symbol_table $ion_encoding) + * (macro_table (%macros)) + * )) + * ``` + */ + SetMacros( + 13, "set_macros", listOf(zeroToManyTagged("macros")), + templateBody { + annotated(ION_ENCODING, ::sexp) { + sexp { + symbol(SYMBOL_TABLE) + symbol(ION_ENCODING) + } + sexp { + symbol(MACRO_TABLE) + variable(0) + } + } + } + ), + + /** + * ```ion + * (macro add_macros (macros*) + * $ion_encoding::( + * (symbol_table $ion_encoding) + * (macro_table $ion_encoding (%macros)) + * )) + * ``` + */ + AddMacros( + 14, "add_macros", listOf(zeroToManyTagged("macros")), + templateBody { + annotated(ION_ENCODING, ::sexp) { + sexp { + symbol(SYMBOL_TABLE) + symbol(ION_ENCODING) + } + sexp { + symbol(MACRO_TABLE) + symbol(ION_ENCODING) + variable(0) + } + } + } + ), + + /** + * ```ion + * (macro use (catalog_key version?) + * $ion_encoding::( + * (import the_module (%catalog_key) (.if_none (%version) 1 (%version))) + * (symbol_table $ion_encoding the_module) + * (macro_table $ion_encoding the_module) + * )) + * ``` + */ + Use( + 15, "use", listOf(exactlyOneTagged("catalog_key"), zeroOrOneTagged("version")), + templateBody { + val theModule = _Private_Utils.newSymbolToken("the_module") + annotated(ION_ENCODING, ::sexp) { + sexp { + symbol(IMPORT) + symbol(theModule) + variable(0) + macro(IfNone) { + variable(1) + int(1) + variable(1) + } + } + sexp { + symbol(SYMBOL_TABLE) + symbol(ION_ENCODING) + symbol(theModule) + } + sexp { + symbol(MACRO_TABLE) + symbol(ION_ENCODING) + symbol(theModule) + } + } + } + ), + Repeat(17, "repeat", listOf(exactlyOneTagged("n"), oneToManyTagged("value"))), + Comment(21, "comment", listOf(zeroToManyTagged("values")), templateBody { macro(None) {} }), MakeField( 22, "make_field", listOf( @@ -29,17 +183,14 @@ enum class SystemMacro(val id: Byte, val macroName: String, override val signatu ), // TODO: Other system macros - - // Technically not system macros, but special forms. However, it's easier to model them as if they are macros in TDL. - // We give them an ID of -1 to distinguish that they are not addressable outside TDL. - IfNone(-1, "if_none", listOf(zeroToManyTagged("stream"), zeroToManyTagged("true_branch"), zeroToManyTagged("false_branch"))), - IfSome(-1, "if_some", listOf(zeroToManyTagged("stream"), zeroToManyTagged("true_branch"), zeroToManyTagged("false_branch"))), - IfSingle(-1, "if_single", listOf(zeroToManyTagged("stream"), zeroToManyTagged("true_branch"), zeroToManyTagged("false_branch"))), - IfMulti(-1, "if_multi", listOf(zeroToManyTagged("stream"), zeroToManyTagged("true_branch"), zeroToManyTagged("false_branch"))), ; override val dependencies: List - get() = emptyList() + get() = body + ?.filterIsInstance() + ?.map(Expression.MacroInvocation::macro) + ?.distinct() + ?: emptyList() companion object { diff --git a/src/main/java/com/amazon/ion/impl/macro/TemplateMacro.kt b/src/main/java/com/amazon/ion/impl/macro/TemplateMacro.kt index 06c2be23e..bdcaaba01 100644 --- a/src/main/java/com/amazon/ion/impl/macro/TemplateMacro.kt +++ b/src/main/java/com/amazon/ion/impl/macro/TemplateMacro.kt @@ -1,10 +1,12 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ion.impl.macro /** * Represents a template macro. A template macro is defined by a signature, and a list of template expressions. * A template macro only gains a name and/or ID when it is added to a macro table. */ -data class TemplateMacro(override val signature: List, val body: List) : +class TemplateMacro(override val signature: List, override val body: List) : Macro { // TODO: Consider rewriting the body of the macro if we discover that there are any macros invoked using only // constants as arguments—either at compile time or lazily. diff --git a/src/test/java/com/amazon/ion/impl/macro/MacroEvaluatorTest.kt b/src/test/java/com/amazon/ion/impl/macro/MacroEvaluatorTest.kt index 37f0582dd..d449a995c 100644 --- a/src/test/java/com/amazon/ion/impl/macro/MacroEvaluatorTest.kt +++ b/src/test/java/com/amazon/ion/impl/macro/MacroEvaluatorTest.kt @@ -2,14 +2,16 @@ // SPDX-License-Identifier: Apache-2.0 package com.amazon.ion.impl.macro -import com.amazon.ion.FakeSymbolToken -import com.amazon.ion.IonException -import com.amazon.ion.IonType +import com.amazon.ion.* +import com.amazon.ion.impl.* +import com.amazon.ion.impl.SystemSymbols_1_1.* import com.amazon.ion.impl._Private_Utils.newSymbolToken +import com.amazon.ion.impl.bin.IonManagedWriter_1_1_Test.Companion.ion_encoding import com.amazon.ion.impl.macro.Expression.* import com.amazon.ion.impl.macro.ExpressionBuilderDsl.Companion.eExpBody import com.amazon.ion.impl.macro.ExpressionBuilderDsl.Companion.templateBody import com.amazon.ion.impl.macro.SystemMacro.* +import com.amazon.ion.system.IonSystemBuilder import java.math.BigDecimal import java.math.BigInteger import java.util.Base64 @@ -18,6 +20,7 @@ import kotlin.contracts.contract import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Assertions.assertArrayEquals import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows @@ -905,7 +908,7 @@ class MacroEvaluatorTest { companion object { /** Helper function to create template macros */ - fun template(vararg parameters: String, body: TemplateDsl.() -> Unit): Macro { + internal fun template(vararg parameters: String, body: TemplateDsl.() -> Unit): Macro { val signature = parameters.map { val cardinality = Macro.ParameterCardinality.fromSigil("${it.last()}") if (cardinality == null) { @@ -918,7 +921,7 @@ class MacroEvaluatorTest { } /** Helper function to use Expression DSL for evaluator inputs */ - fun MacroEvaluator.initExpansion(eExpression: EExpDsl.() -> Unit) = initExpansion(eExpBody(eExpression)) + internal fun MacroEvaluator.initExpansion(eExpression: EExpDsl.() -> Unit) = initExpansion(eExpBody(eExpression)) @OptIn(ExperimentalContracts::class) private inline fun assertIsInstance(value: Any?) { @@ -934,6 +937,18 @@ class MacroEvaluatorTest { Assertions.fail(message) } } + + /** + * Helper function for testing the output of macro invocations. + */ + fun MacroEvaluator.assertExpansion(expectedOutput: String) { + val ion = IonSystemBuilder.standard().build() as _Private_IonSystem + val actual = mutableListOf() + ion.systemIterate(MacroEvaluatorAsIonReader(this)).forEachRemaining(actual::add) + val expected = mutableListOf() + ion.systemIterate(expectedOutput).forEachRemaining(expected::add) + assertEquals(expected, actual) + } } @Test @@ -1089,4 +1104,162 @@ class MacroEvaluatorTest { assertThrows { evaluator.expandNext() } } + + @Test + fun `the comment macro expands to nothing`() { + // Given: + // When: + // (:comment 1 2 3) + // Then: + // + + evaluator.initExpansion { + eexp(Comment) { + expressionGroup { + int(1) + int(2) + int(3) + } + } + } + + assertNull(evaluator.expandNext()) + } + + @Test + fun `set_symbols expands to an encoding directive that replaces the symbol table and preserves macros`() { + evaluator.initExpansion { + // (:set_symbols a b c) + eexp(SetSymbols) { + expressionGroup { + symbol("a") + symbol("b") + symbol("c") + } + } + } + evaluator.assertExpansion( + """ + $ion_encoding::( + (symbol_table [a, b, c]) + (macro_table $ion_encoding) + ) + """ + ) + } + + @Test + fun `add_symbols expands to an encoding directive that appends to the symbol table and preserves macros`() { + evaluator.initExpansion { + // (:add_symbols a b c) + eexp(AddSymbols) { + expressionGroup { + symbol("a") + symbol("b") + symbol("c") + } + } + } + evaluator.assertExpansion( + """ + $ion_encoding::( + (symbol_table $ion_encoding [a, b, c]) + (macro_table $ion_encoding) + ) + """ + ) + } + + @Test + fun `set_macros expands to an encoding directive that preserves symbols and replaces the macro table`() { + evaluator.initExpansion { + // (:set_macros (macro answer () 42)) + eexp(SetMacros) { + expressionGroup { + sexp { + symbol(MACRO) + symbol("answer") + sexp { } + int(42) + } + } + } + } + evaluator.assertExpansion( + """ + $ion_encoding::( + (symbol_table $ion_encoding) + (macro_table + (macro answer () 42)) + ) + """ + ) + } + + @Test + fun `add_macros expands to an encoding directive that preserves symbols and appends to the macro table`() { + evaluator.initExpansion { + // (:add_macros (macro answer () 42)) + eexp(AddMacros) { + expressionGroup { + sexp { + symbol(MACRO) + symbol("answer") + sexp { } + int(42) + } + } + } + } + evaluator.assertExpansion( + """ + $ion_encoding::( + (symbol_table $ion_encoding) + (macro_table + $ion_encoding + (macro answer () 42)) + ) + """ + ) + } + + @Test + fun `use expands to an encoding directive that imports a module and appends it to the symbol and macro tables`() { + evaluator.initExpansion { + // (:use "com.amazon.Foo" 2) + eexp(Use) { + string("com.amazon.Foo") + int(2) + } + } + evaluator.assertExpansion( + """ + $ion_encoding::( + (import the_module "com.amazon.Foo" 2) + (symbol_table $ion_encoding the_module) + (macro_table $ion_encoding the_module) + ) + """ + ) + } + + @Test + fun `use defaults to version 1 if no version is provided`() { + evaluator.initExpansion { + // (:use "com.amazon.Foo") + eexp(Use) { + string("com.amazon.Foo") + expressionGroup { } + } + } + evaluator.assertExpansion( + """ + $ion_encoding::( + (import the_module "com.amazon.Foo" 1) + (symbol_table $ion_encoding the_module) + (macro_table $ion_encoding the_module) + ) + """ + ) + } }