Skip to content

Commit

Permalink
Updates TDL syntax for macro invocations, expression groups, and vari…
Browse files Browse the repository at this point in the history 
…able expansions (#951)
  • Loading branch information
@popematt
popematt authored Oct 3, 2024
1 parent 76276cf commit fc8f49e
Show file tree
Hide file tree
Showing 9 changed files with 259 additions and 184 deletions.
35 changes: 34 additions & 1 deletion src/main/java/com/amazon/ion/impl/IonRawTextWriter_1_1.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -327,7 +327,11 @@ class IonRawTextWriter_1_1 internal constructor(
}

override fun stepInSExp(usingLengthPrefix: Boolean) {
openValue { output.appendAscii("(") }
startSexp { output.appendAscii("(") }
}

private inline fun startSexp(openingTokens: () -> Unit) {
openValue(openingTokens)
ancestorContainersStack.add(currentContainer)
currentContainer = SExp
currentContainerHasValues = false
Expand Down Expand Up @@ -404,4 +408,33 @@ class IonRawTextWriter_1_1 internal constructor(
override fun writeMacroParameterCardinality(cardinality: Macro.ParameterCardinality) {
output.appendAscii(cardinality.sigil)
}

override fun stepInTdlExpressionGroup() {
startSexp { output.appendAscii("(..") }
isPendingSeparator = true
}

override fun stepInTdlMacroInvocation(macroRef: Int) {
startSexp {
output.appendAscii("(.")
output.printInt(macroRef.toLong())
}
isPendingSeparator = true
}

override fun stepInTdlMacroInvocation(macroRef: String) {
startSexp {
output.appendAscii("(.")
output.appendAscii(macroRef)
}
isPendingSeparator = true
}

override fun writeTdlVariableExpansion(variableName: String) {
writeScalar {
output.appendAscii("(%")
output.appendAscii(variableName)
output.appendAscii(")")
}
}
}
20 changes: 20 additions & 0 deletions src/main/java/com/amazon/ion/impl/PrivateIonRawWriter_1_1.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,26 @@ internal interface PrivateIonRawWriter_1_1 : IonRawWriter_1_1 {
*/
fun writeMacroParameterCardinality(cardinality: Macro.ParameterCardinality)

/**
* Steps into a TDL Macro Invocation—an s-expression, followed by `.` and then the macro address
*/
fun stepInTdlMacroInvocation(macroRef: Int)

/**
* Steps into a TDL Macro Invocation—an s-expression, followed by `.` and then the macro name
*/
fun stepInTdlMacroInvocation(macroRef: String)

/**
* Steps in s-expression, writes `%` symbol, variable name, and then closes the s-expression.
*/
fun writeTdlVariableExpansion(variableName: String)

/**
* Steps in s-expression and writes `..` symbol.
*/
fun stepInTdlExpressionGroup()

/**
* Sets a flag that can override the newlines that are normally inserted by a pretty printer.
*
Expand Down
101 changes: 22 additions & 79 deletions src/main/java/com/amazon/ion/impl/bin/IonManagedWriter_1_1.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,6 @@ internal class IonManagedWriter_1_1(
companion object {
private val ION_VERSION_MARKER_REGEX = Regex("^\\\$ion_\\d+_\\d+$")

private const val TDL_EXPRESSION_GROUP_START = ";"

// These are chosen subjectively to be neither too big nor too small.
private const val MAX_PARAMETERS_IN_ONE_LINE_SIGNATURE = 4
private const val MAX_SYMBOLS_IN_SINGLE_LINE_SYMBOL_TABLE = 10
Expand Down Expand Up @@ -403,17 +401,13 @@ internal class IonManagedWriter_1_1(

when (expression) {
is Expression.DataModelValue -> {
// If we need to write a sexp, we must use the annotate macro.
// If we're writing a symbol, we can put them inside the "literal" special form.
if (expression.type != IonType.SEXP && expression.type != IonType.SYMBOL) {
expression.annotations.forEach {
if (it.text != null) {
// TODO: If it's already in the symbol table we could check the
// symbol-inline strategy and possibly write a SID.
writeAnnotations(it.text)
} else {
writeAnnotations(it.sid)
}
expression.annotations.forEach {
if (it.text != null) {
// TODO: If it's already in the symbol table we could check the
// symbol-inline strategy and possibly write a SID.
writeAnnotations(it.text)
} else {
writeAnnotations(it.sid)
}
}

Expand All @@ -432,25 +426,13 @@ internal class IonManagedWriter_1_1(
IonType.DECIMAL -> writeDecimal((expression as Expression.DecimalValue).value)
IonType.TIMESTAMP -> writeTimestamp((expression as Expression.TimestampValue).value)
IonType.SYMBOL -> {
writeSystemSexp {
writeSymbol(SystemSymbols_1_1.LITERAL)
expression.annotations.forEach {
if (it.text != null) {
// TODO: If it's already in the symbol table we could check the
// symbol-inline strategy and possibly write a SID.
writeAnnotations(it.text)
} else {
writeAnnotations(it.sid)
}
}
val symbolToken = (expression as Expression.SymbolValue).value
if (symbolToken.text != null) {
// TODO: If it's already in the symbol table we could check the
// symbol-inline strategy and possibly write a SID.
writeSymbol(symbolToken.text)
} else {
writeSymbol(symbolToken.sid)
}
val symbolToken = (expression as Expression.SymbolValue).value
if (symbolToken.text != null) {
// TODO: If it's already in the symbol table we could check the
// symbol-inline strategy and possibly write a SID.
writeSymbol(symbolToken.text)
} else {
writeSymbol(symbolToken.sid)
}
}
IonType.STRING -> writeString((expression as Expression.StringValue).value)
Expand All @@ -463,41 +445,8 @@ internal class IonManagedWriter_1_1(
}
IonType.SEXP -> {
expression as Expression.HasStartAndEnd
if (expression.annotations.isNotEmpty()) {
stepInSExp(usingLengthPrefix = false)
numberOfTimesToStepOut[expression.endExclusive]++
writeSymbol(SystemSymbols_1_1.ANNOTATE)

// Write the annotations as symbols within an expression group
writeSystemSexp {
writeSymbol(TDL_EXPRESSION_GROUP_START)
expression.annotations.forEach {
if (it.text != null) {
// TODO: If it's already in the symbol table we could check the
// symbol-inline strategy and possibly write a SID.

// Write the annotation as a string so that we don't have to use
// `literal` to prevent it from being interpreted as a variable
writeString(it.text)
} else {
// TODO: See if there is a less verbose way to use SIDs in TDL
writeSystemSexp {
writeSymbol(SystemSymbols_1_1.LITERAL)
writeSymbol(it.sid)
}
}
}
}
}
// Start a `(make_sexp [ ...` invocation
stepInSExp(usingLengthPrefix = false)
numberOfTimesToStepOut[expression.endExclusive]++
writeSymbol(SystemSymbols_1_1.MAKE_SEXP)

if (expression.startInclusive != expression.endExclusive) {
stepInList(usingLengthPrefix = false)
numberOfTimesToStepOut[expression.endExclusive]++
}
}
IonType.STRUCT -> {
expression as Expression.HasStartAndEnd
Expand All @@ -517,28 +466,22 @@ internal class IonManagedWriter_1_1(
}
}
is Expression.ExpressionGroup -> {
stepInSExp(usingLengthPrefix = false)
stepInTdlExpressionGroup()
numberOfTimesToStepOut[expression.endExclusive]++
writeSymbol(TDL_EXPRESSION_GROUP_START)
}
is Expression.MacroInvocation -> {
stepInSExp(usingLengthPrefix = false)
numberOfTimesToStepOut[expression.endExclusive]++

val invokedAddress = macroTable[expression.macro]
?: newMacros[expression.macro]
?: throw IllegalStateException("A macro in the macro table is missing a dependency")

if (options.invokeTdlMacrosByName) {
val invokedName = macroNames[invokedAddress]
if (invokedName != null) {
writeSymbol(invokedName)
return@forEachIndexed
}
val invokedName = macroNames[invokedAddress]
if (options.invokeTdlMacrosByName && invokedName != null) {
stepInTdlMacroInvocation(invokedName)
} else {
stepInTdlMacroInvocation(invokedAddress)
}
writeInt(invokedAddress.toLong())
numberOfTimesToStepOut[expression.endExclusive]++
}
is Expression.VariableRef -> writeSymbol(macro.signature[expression.signatureIndex].variableName)
is Expression.VariableRef -> writeTdlVariableExpansion(macro.signature[expression.signatureIndex].variableName)
else -> error("Unreachable")
}
}
Expand Down
25 changes: 25 additions & 0 deletions src/main/java/com/amazon/ion/impl/bin/IonRawBinaryWriter_1_1.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -917,4 +917,29 @@ class IonRawBinaryWriter_1_1 internal constructor(
// TODO: Write as a system symbol
writeSymbol(cardinality.sigil.toString())
}

override fun stepInTdlMacroInvocation(macroRef: Int) {
stepInSExp(usingLengthPrefix = false)
writeSymbol(".")
writeInt(macroRef.toLong())
}

override fun stepInTdlMacroInvocation(macroRef: String) {
stepInSExp(usingLengthPrefix = false)
writeSymbol(".")
writeSymbol(macroRef)
}

override fun writeTdlVariableExpansion(variableName: String) {
stepInSExp(usingLengthPrefix = false)
writeSymbol("%")
writeSymbol(variableName)
stepOut()
}

override fun stepInTdlExpressionGroup() {
stepInSExp(usingLengthPrefix = false)
// TODO: Write as a system symbol
writeSymbol("..")
}
}
4 changes: 4 additions & 0 deletions src/main/java/com/amazon/ion/impl/bin/Ion_1_1_Constants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,10 @@
public class Ion_1_1_Constants {
private Ion_1_1_Constants() {}

public static final String TDL_MACRO_INVOCATION_SIGIL = ".";
public static final String TDL_EXPRESSION_GROUP_SIGIL = "..";
public static final String TDL_VARIABLE_EXPANSION_SIGIL = "%";

// When writing system symbols (or $0) in a flex sym, the SID must be offset to
// avoid clashing with E-Expression op codes.
public static final int FLEX_SYM_SYSTEM_SYMBOL_OFFSET = 0x60;
Expand Down
Loading

0 comments on commit fc8f49e

Please sign in to comment.