From f9f72f017d118c4820f01a93806bbc5d7f1852d0 Mon Sep 17 00:00:00 2001 From: Matthew Pope <81593196+popematt@users.noreply.github.com> Date: Thu, 3 Oct 2024 17:56:10 -0700 Subject: [PATCH] Adds expansions for make_symbol, make_decimal, none, and annotate (#955) --- .../com/amazon/ion/impl/_Private_Utils.java | 8 + .../com/amazon/ion/impl/macro/Expression.kt | 25 +- .../java/com/amazon/ion/impl/macro/Macro.kt | 12 + .../amazon/ion/impl/macro/MacroEvaluator.kt | 134 +++++++- .../ion/impl/macro/MacroEvaluatorTest.kt | 305 ++++++++++++++++++ 5 files changed, 470 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/amazon/ion/impl/_Private_Utils.java b/src/main/java/com/amazon/ion/impl/_Private_Utils.java index 9de2e70ab..9c388683c 100644 --- a/src/main/java/com/amazon/ion/impl/_Private_Utils.java +++ b/src/main/java/com/amazon/ion/impl/_Private_Utils.java @@ -180,6 +180,14 @@ public static SymbolTokenImpl newSymbolToken(String text, int sid) return new SymbolTokenImpl(text, sid); } + /** + * @return not null + */ + public static SymbolToken newSymbolToken(String text) + { + return new SymbolTokenImpl(text, UNKNOWN_SYMBOL_ID); + } + /** Cached copy of $0 */ public static final SymbolTokenImpl SYMBOL_0 = newSymbolToken((String) null, 0); diff --git a/src/main/java/com/amazon/ion/impl/macro/Expression.kt b/src/main/java/com/amazon/ion/impl/macro/Expression.kt index 9bff69879..79263eff0 100644 --- a/src/main/java/com/amazon/ion/impl/macro/Expression.kt +++ b/src/main/java/com/amazon/ion/impl/macro/Expression.kt @@ -60,6 +60,8 @@ sealed interface Expression { sealed interface DataModelValue : DataModelExpression { val annotations: List val type: IonType + + fun withAnnotations(annotations: List): DataModelValue } /** Expressions that represent Ion container types */ @@ -82,32 +84,44 @@ sealed interface Expression { data class ExpressionGroup(override val selfIndex: Int, override val endExclusive: Int) : EExpressionBodyExpression, TemplateBodyExpression, HasStartAndEnd // Scalars - data class NullValue(override val annotations: List = emptyList(), override val type: IonType) : DataModelValue + data class NullValue(override val annotations: List = emptyList(), override val type: IonType) : DataModelValue { + override fun withAnnotations(annotations: List) = copy(annotations = annotations) + } data class BoolValue(override val annotations: List = emptyList(), val value: Boolean) : DataModelValue { override val type: IonType get() = IonType.BOOL + override fun withAnnotations(annotations: List) = copy(annotations = annotations) } - sealed interface IntValue : DataModelValue + sealed interface IntValue : DataModelValue { + val bigIntegerValue: BigInteger + } data class LongIntValue(override val annotations: List = emptyList(), val value: Long) : IntValue { override val type: IonType get() = IonType.INT + override fun withAnnotations(annotations: List) = copy(annotations = annotations) + override val bigIntegerValue: BigInteger get() = BigInteger.valueOf(value) } data class BigIntValue(override val annotations: List = emptyList(), val value: BigInteger) : IntValue { override val type: IonType get() = IonType.INT + override fun withAnnotations(annotations: List) = copy(annotations = annotations) + override val bigIntegerValue: BigInteger get() = value } data class FloatValue(override val annotations: List = emptyList(), val value: Double) : DataModelValue { override val type: IonType get() = IonType.FLOAT + override fun withAnnotations(annotations: List) = copy(annotations = annotations) } data class DecimalValue(override val annotations: List = emptyList(), val value: BigDecimal) : DataModelValue { override val type: IonType get() = IonType.DECIMAL + override fun withAnnotations(annotations: List) = copy(annotations = annotations) } data class TimestampValue(override val annotations: List = emptyList(), val value: Timestamp) : DataModelValue { override val type: IonType get() = IonType.TIMESTAMP + override fun withAnnotations(annotations: List) = copy(annotations = annotations) } sealed interface TextValue : DataModelValue { @@ -117,11 +131,13 @@ sealed interface Expression { data class StringValue(override val annotations: List = emptyList(), val value: String) : TextValue { override val type: IonType get() = IonType.STRING override val stringValue: String get() = value + override fun withAnnotations(annotations: List) = copy(annotations = annotations) } data class SymbolValue(override val annotations: List = emptyList(), val value: SymbolToken) : TextValue { override val type: IonType get() = IonType.SYMBOL override val stringValue: String get() = value.assumeText() + override fun withAnnotations(annotations: List) = copy(annotations = annotations) } sealed interface LobValue : DataModelValue { @@ -133,6 +149,7 @@ sealed interface Expression { // We must override hashcode and equals in the lob types because `value` is a `byte[]` data class BlobValue(override val annotations: List = emptyList(), override val value: ByteArray) : LobValue { override val type: IonType get() = IonType.BLOB + override fun withAnnotations(annotations: List) = copy(annotations = annotations) override fun hashCode(): Int = annotations.hashCode() * 31 + value.contentHashCode() override fun equals(other: Any?): Boolean { if (this === other) return true @@ -144,6 +161,7 @@ sealed interface Expression { data class ClobValue(override val annotations: List = emptyList(), override val value: ByteArray) : LobValue { override val type: IonType get() = IonType.CLOB + override fun withAnnotations(annotations: List) = copy(annotations = annotations) override fun hashCode(): Int = annotations.hashCode() * 31 + value.contentHashCode() override fun equals(other: Any?): Boolean { if (this === other) return true @@ -165,6 +183,7 @@ sealed interface Expression { override val endExclusive: Int ) : DataModelContainer { override val type: IonType get() = IonType.LIST + override fun withAnnotations(annotations: List) = copy(annotations = annotations) } /** @@ -176,6 +195,7 @@ sealed interface Expression { override val endExclusive: Int ) : DataModelContainer { override val type: IonType get() = IonType.SEXP + override fun withAnnotations(annotations: List) = copy(annotations = annotations) } /** @@ -188,6 +208,7 @@ sealed interface Expression { val templateStructIndex: Map> ) : DataModelContainer { override val type: IonType get() = IonType.STRUCT + override fun withAnnotations(annotations: List) = copy(annotations = annotations) } data class FieldName(val value: SymbolToken) : DataModelExpression diff --git a/src/main/java/com/amazon/ion/impl/macro/Macro.kt b/src/main/java/com/amazon/ion/impl/macro/Macro.kt index c4601f4c2..a22197541 100644 --- a/src/main/java/com/amazon/ion/impl/macro/Macro.kt +++ b/src/main/java/com/amazon/ion/impl/macro/Macro.kt @@ -3,6 +3,7 @@ package com.amazon.ion.impl.macro import com.amazon.ion.impl.* +import com.amazon.ion.impl.macro.Macro.Parameter.Companion.exactlyOneTagged import com.amazon.ion.impl.macro.Macro.Parameter.Companion.zeroToManyTagged /** @@ -116,8 +117,19 @@ data class TemplateMacro(override val signature: List, val body * Macros that are built in, rather than being defined by a template. */ enum class SystemMacro(val macroName: String, override val signature: List) : Macro { + None("none", emptyList()), Values("values", listOf(zeroToManyTagged("values"))), + Annotate("annotate", listOf(zeroToManyTagged("ann"), exactlyOneTagged("value"))), MakeString("make_string", listOf(zeroToManyTagged("text"))), + MakeSymbol("make_symbol", listOf(zeroToManyTagged("text"))), + MakeDecimal( + "make_decimal", + listOf( + Macro.Parameter("coefficient", Macro.ParameterEncoding.CompactInt, Macro.ParameterCardinality.ExactlyOne), + Macro.Parameter("exponent", Macro.ParameterEncoding.CompactInt, Macro.ParameterCardinality.ExactlyOne), + ) + ), + // TODO: Other system macros ; diff --git a/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt b/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt index f995ccaa6..45ad2d0d9 100644 --- a/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt +++ b/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt @@ -2,9 +2,12 @@ // SPDX-License-Identifier: Apache-2.0 package com.amazon.ion.impl.macro -import com.amazon.ion.* -import com.amazon.ion.impl.* +import com.amazon.ion.IonException +import com.amazon.ion.SymbolToken +import com.amazon.ion.impl._Private_RecyclingStack +import com.amazon.ion.impl._Private_Utils.newSymbolToken import com.amazon.ion.impl.macro.Expression.* +import java.math.BigDecimal /** * Evaluates an EExpression from a List of [EExpressionBodyExpression] and the [TemplateBodyExpression]s @@ -26,6 +29,56 @@ class MacroEvaluator { */ private fun interface Expander { fun nextExpression(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator): Expression + + /** + * Read the expanded values from one argument, returning exactly one value. + * Throws an exception if there is not exactly one expanded value. + */ + fun readExactlyOneExpandedArgumentValue(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator, argName: String): DataModelExpression { + return readZeroOrOneExpandedArgumentValues(expansionInfo, macroEvaluator, argName) + ?: throw IonException("Argument $argName expanded to nothing.") + } + + /** + * Read the expanded values from one argument, returning zero or one values. + * Throws an exception if there is more than one expanded value. + */ + fun readZeroOrOneExpandedArgumentValues(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator, argName: String): DataModelExpression? { + var value: DataModelExpression? = null + readExpandedArgumentValues(expansionInfo, macroEvaluator) { + if (value == null) { + value = it + } else { + throw IonException("Too many values for argument $argName") + } + } + return value + } + + /** + * Reads the expanded values from one argument. + */ + fun readExpandedArgumentValues(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator, callback: (DataModelExpression) -> Unit) { + val i = expansionInfo.i + expansionInfo.nextSourceExpression() + + macroEvaluator.pushExpansion( + expansionKind = ExpansionKind.Values, + argsStartInclusive = i, + // There can only be one top-level expression for an argument (it's either a value, macro, or + // expression group) so we can set the end to one more than the start. + argsEndExclusive = i + 1, + environment = expansionInfo.environment ?: Environment.EMPTY, + expressions = expansionInfo.expressions!!, + ) + + val depth = macroEvaluator.expansionStack.size() + var expr = macroEvaluator.expandNext(depth) + while (expr != null) { + callback(expr) + expr = macroEvaluator.expandNext(depth) + } + } } private object SimpleExpander : Expander { @@ -34,20 +87,39 @@ class MacroEvaluator { } } + private object AnnotateExpander : Expander { + // TODO: Handle edge cases mentioned in https://github.com/amazon-ion/ion-docs/issues/347 + override fun nextExpression(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator): Expression { + val annotations = mutableListOf() + + readExpandedArgumentValues(expansionInfo, macroEvaluator) { + when (it) { + is StringValue -> annotations.add(newSymbolToken(it.value)) + is SymbolValue -> annotations.add(it.value) + is DataModelValue -> throw IonException("Annotation arguments must be string or symbol; found: ${it.type}") + is FieldName -> TODO("Unreachable. Must encounter a StructValue first.") + } + } + + val valueToAnnotate = readExactlyOneExpandedArgumentValue(expansionInfo, macroEvaluator, SystemMacro.Annotate.signature[1].variableName) + + // It cannot be a FieldName expression because we haven't stepped into a struct, so it must be DataModelValue + valueToAnnotate as DataModelValue + // Combine the annotations + annotations.addAll(valueToAnnotate.annotations) + return valueToAnnotate.withAnnotations(annotations) + } + } + private object MakeStringExpander : Expander { override fun nextExpression(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator): Expression { - // Tell the macro evaluator to treat this as a values expansion... - macroEvaluator.expansionStack.peek().expansionKind = ExpansionKind.Values - val minDepth = macroEvaluator.expansionStack.size() - // ...But capture the output and turn it into a String val sb = StringBuilder() - while (true) { - when (val expr: DataModelExpression? = macroEvaluator.expandNext(minDepth)) { - is StringValue -> sb.append(expr.value) - is SymbolValue -> sb.append(expr.value.assumeText()) + readExpandedArgumentValues(expansionInfo, macroEvaluator) { + when (it) { + is StringValue -> sb.append(it.value) + is SymbolValue -> sb.append(it.value.assumeText()) is NullValue -> {} - null -> break - is DataModelValue -> throw IonException("Invalid argument type for 'make_string': ${expr.type}") + is DataModelValue -> throw IonException("Invalid argument type for 'make_string': ${it.type}") is FieldName -> TODO("Unreachable. We shouldn't be able to get here without first encountering a StructValue.") } } @@ -55,19 +127,57 @@ class MacroEvaluator { } } + private object MakeSymbolExpander : Expander { + override fun nextExpression(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator): Expression { + val sb = StringBuilder() + readExpandedArgumentValues(expansionInfo, macroEvaluator) { + when (it) { + is StringValue -> sb.append(it.value) + is SymbolValue -> sb.append(it.value.assumeText()) + is NullValue -> {} + is DataModelValue -> throw IonException("Invalid argument type for 'make_symbol': ${it.type}") + is FieldName -> TODO("Unreachable. We shouldn't be able to get here without first encountering a StructValue.") + } + } + return SymbolValue(value = newSymbolToken(sb.toString())) + } + } + + private object MakeDecimalExpander : Expander { + override fun nextExpression(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator): Expression { + val coefficient = readExactlyOneExpandedArgumentValue(expansionInfo, macroEvaluator, SystemMacro.MakeDecimal.signature[0].variableName) + .let { it as? IntValue } + ?.bigIntegerValue + ?: throw IonException("Coefficient must be an integer") + val exponent = readExactlyOneExpandedArgumentValue(expansionInfo, macroEvaluator, SystemMacro.MakeDecimal.signature[1].variableName) + .let { it as? IntValue } + ?.bigIntegerValue + ?: throw IonException("Exponent must be an integer") + + return DecimalValue(value = BigDecimal(coefficient, -1 * exponent.intValueExact())) + } + } + private enum class ExpansionKind(val expander: Expander) { Container(SimpleExpander), TemplateBody(SimpleExpander), Values(SimpleExpander), + Annotate(AnnotateExpander), MakeString(MakeStringExpander), + MakeSymbol(MakeSymbolExpander), + MakeDecimal(MakeDecimalExpander), ; companion object { @JvmStatic fun forSystemMacro(macro: SystemMacro): ExpansionKind { return when (macro) { + SystemMacro.None -> Values // "none" takes no args, so we can treat it as an empty "values" expansion SystemMacro.Values -> Values + SystemMacro.Annotate -> Annotate SystemMacro.MakeString -> MakeString + SystemMacro.MakeSymbol -> MakeSymbol + SystemMacro.MakeDecimal -> MakeDecimal } } } diff --git a/src/test/java/com/amazon/ion/impl/macro/MacroEvaluatorTest.kt b/src/test/java/com/amazon/ion/impl/macro/MacroEvaluatorTest.kt index 04c9c4b17..73bd18379 100644 --- a/src/test/java/com/amazon/ion/impl/macro/MacroEvaluatorTest.kt +++ b/src/test/java/com/amazon/ion/impl/macro/MacroEvaluatorTest.kt @@ -4,12 +4,18 @@ package com.amazon.ion.impl.macro import com.amazon.ion.FakeSymbolToken import com.amazon.ion.IonType +import com.amazon.ion.impl._Private_Utils.newSymbolToken import com.amazon.ion.impl.macro.Expression.* import com.amazon.ion.impl.macro.ExpressionBuilderDsl.Companion.eExpBody import com.amazon.ion.impl.macro.ExpressionBuilderDsl.Companion.templateBody import com.amazon.ion.impl.macro.SystemMacro.* +import java.math.BigDecimal +import java.math.BigInteger +import kotlin.contracts.ExperimentalContracts +import kotlin.contracts.contract import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test class MacroEvaluatorTest { @@ -39,6 +45,47 @@ class MacroEvaluatorTest { val evaluator = MacroEvaluator() + @Test + fun `the 'none' system macro`() { + // Given: + // When: + // (:none) + // Then: + // + + evaluator.initExpansion { + eexp(None) {} + } + + assertEquals(null, evaluator.expandNext()) + } + + @Test + fun `the 'none' system macro, invoked in TDL`() { + // Given: + // (macro blackhole (any*) (.none)) + // When: + // (:blackhole "abc" 123 true) + // Then: + // + + val blackholeMacro = template("any*") { + macro(None) {} + } + + evaluator.initExpansion { + eexp(blackholeMacro) { + expressionGroup { + string("abc") + int(123) + bool(true) + } + } + } + + assertEquals(null, evaluator.expandNext()) + } + @Test fun `a trivial constant macro evaluation`() { // Given: @@ -384,6 +431,262 @@ class MacroEvaluatorTest { assertEquals(null, evaluator.expandNext()) } + @Test + fun `simple make_symbol`() { + // Given: + // When: + // (:make_symbol "a" "b" "c") + // Then: + // abc + + evaluator.initExpansion { + eexp(MakeSymbol) { + expressionGroup { + string("a") + string("b") + string("c") + } + } + } + + val expr = evaluator.expandNext() + assertIsInstance(expr) + assertEquals("abc", expr.value.text) + assertEquals(null, evaluator.expandNext()) + } + + @Test + fun `simple make_decimal`() { + // Given: + // When: + // (:make_decimal 2 4) + // Then: + // 2d4 + + evaluator.initExpansion { + eexp(MakeDecimal) { + int(2) + int(4) + } + } + + val expr = evaluator.expandNext() + assertIsInstance(expr) + assertTrue(BigDecimal.valueOf(20000).compareTo(expr.value) == 0) + assertEquals(BigInteger.valueOf(2), expr.value.unscaledValue()) + assertEquals(-4, expr.value.scale()) + assertEquals(null, evaluator.expandNext()) + } + + @Test + fun `make_decimal from nested expressions`() { + // Given: + // (macro fixed_point (x) (.make_decimal x (.values -2))) + // When: + // (:fixed_point (:identity 123)) + // Then: + // 1.23 + + val fixedPointMacro = template("x") { + macro(MakeDecimal) { + variable(0) + macro(Values) { + expressionGroup { + int(-2) + } + } + } + } + + evaluator.initExpansion { + eexp(fixedPointMacro) { + eexp(IDENTITY_MACRO) { + int(123) + } + } + } + + val expr = evaluator.expandNext() + assertIsInstance(expr) + assertEquals(BigDecimal.valueOf(123, 2), expr.value) + assertEquals(null, evaluator.expandNext()) + } + + @Test + fun `simple annotate`() { + // Given: + // When: + // (:annotate (:: "a" "b" "c") 1) + // Then: + // a::b::c::1 + + evaluator.initExpansion { + eexp(Annotate) { + expressionGroup { + string("a") + string("b") + string("c") + } + int(1) + } + } + + val expr = evaluator.expandNext() + assertIsInstance(expr) + assertEquals(listOf("a", "b", "c"), expr.annotations.map { it.text }) + assertEquals(1, expr.value) + assertEquals(null, evaluator.expandNext()) + } + + @Test + fun `annotate a value that already has some annotations`() { + // Given: + // When: + // (:annotate (:: "a" "b") c::1) + // Then: + // a::b::c::1 + + evaluator.initExpansion { + eexp(Annotate) { + expressionGroup { + string("a") + string("b") + } + annotated(listOf(newSymbolToken("c")), ::int, 1) + } + } + + val expr = evaluator.expandNext() + assertIsInstance(expr) + assertEquals(listOf("a", "b", "c"), expr.annotations.map { it.text }) + assertEquals(1, expr.value) + assertEquals(null, evaluator.expandNext()) + } + + @Test + fun `annotate a container`() { + // Given: + // When: + // (:annotate (:: "a" "b" "c") [1]) + // Then: + // a::b::c::[1] + + evaluator.initExpansion { + eexp(Annotate) { + expressionGroup { + string("a") + string("b") + string("c") + } + list { + int(1) + } + } + } + + val expr = evaluator.expandNext() + assertIsInstance(expr) + assertEquals(listOf("a", "b", "c"), expr.annotations.map { it.text }) + evaluator.stepIn() + assertEquals(LongIntValue(emptyList(), 1), evaluator.expandNext()) + assertEquals(null, evaluator.expandNext()) + evaluator.stepOut() + assertEquals(null, evaluator.expandNext()) + } + + @Test + fun `annotate with nested make_string`() { + // Given: + // When: + // (:annotate (:make_string (:: "a" "b" "c")) 1) + // Then: + // abc::1 + + evaluator.initExpansion { + eexp(Annotate) { + eexp(MakeString) { + expressionGroup { + string("a") + string("b") + string("c") + } + } + int(1) + } + } + + val expr = evaluator.expandNext() + assertIsInstance(expr) + assertEquals(listOf("abc"), expr.annotations.map { it.text }) + assertEquals(1, expr.value) + assertEquals(null, evaluator.expandNext()) + } + + @Test + fun `annotate an e-expression result`() { + // Given: + // When: + // (:annotate (:: "a" "b" "c") (:make_string "d" "e" "f")) + // Then: + // a::b::c::"def" + + evaluator.initExpansion { + eexp(Annotate) { + expressionGroup { + string("a") + string("b") + string("c") + } + + eexp(MakeString) { + expressionGroup { + string("d") + string("e") + string("f") + } + } + } + } + + val expr = evaluator.expandNext() + assertIsInstance(expr) + assertEquals(listOf("a", "b", "c"), expr.annotations.map { it.text }) + assertEquals("def", expr.value) + assertEquals(null, evaluator.expandNext()) + } + + @Test + fun `annotate a TDL macro invocation result`() { + // Given: + // (macro pi () 3.14159) + // (macro annotate_pi (x) (.annotate (..x) (.pi))) + // When: + // (:annotate_pi "foo") + // Then: + // foo::3.14159 + + val annotatePi = template("x") { + macro(Annotate) { + expressionGroup { + variable(0) + } + macro(PI_MACRO) {} + } + } + + evaluator.initExpansion { + eexp(annotatePi) { + string("foo") + } + } + + val expr = evaluator.expandNext() + assertIsInstance(expr) + assertEquals(listOf("foo"), expr.annotations.map { it.text }) + assertEquals(3.14159, expr.value) + assertEquals(null, evaluator.expandNext()) + } + @Test fun `macro with a variable substitution in struct field position`() { // Given: @@ -501,7 +804,9 @@ class MacroEvaluatorTest { /** Helper function to use Expression DSL for evaluator inputs */ fun MacroEvaluator.initExpansion(eExpression: EExpDsl.() -> Unit) = initExpansion(eExpBody(eExpression)) + @OptIn(ExperimentalContracts::class) private inline fun assertIsInstance(value: Any?) { + contract { returns() implies (value is T) } if (value !is T) { val message = if (value == null) { "Expected instance of ${T::class.qualifiedName}; was null"