diff --git a/src/main/java/com/amazon/ion/impl/IonRawTextWriter_1_1.kt b/src/main/java/com/amazon/ion/impl/IonRawTextWriter_1_1.kt index c6e205f05..040e53986 100644 --- a/src/main/java/com/amazon/ion/impl/IonRawTextWriter_1_1.kt +++ b/src/main/java/com/amazon/ion/impl/IonRawTextWriter_1_1.kt @@ -188,6 +188,8 @@ class IonRawTextWriter_1_1 internal constructor( numAnnotations += annotations.size } + override fun writeAnnotations(annotation0: SystemSymbols_1_1) = writeAnnotations(annotation0.text) + override fun writeAnnotations(annotation0: CharSequence) { ensureAnnotationSpace(numAnnotations + 1) annotationsTextBuffer[numAnnotations++] = annotation0 @@ -237,6 +239,8 @@ class IonRawTextWriter_1_1 internal constructor( hasFieldName = true } + override fun writeFieldName(symbol: SystemSymbols_1_1) = writeFieldName(symbol.text) + override fun writeNull() = writeScalar { output.appendAscii("null") } @@ -306,6 +310,8 @@ class IonRawTextWriter_1_1 internal constructor( } } + override fun writeSymbol(symbol: SystemSymbols_1_1) = writeSymbol(symbol.text) + override fun writeString(value: CharSequence) = writeScalar { output.printString(value) } override fun writeBlob(value: ByteArray, start: Int, length: Int) = writeScalar { output.printBlob(options, value, start, length) } diff --git a/src/main/java/com/amazon/ion/impl/IonRawWriter_1_1.kt b/src/main/java/com/amazon/ion/impl/IonRawWriter_1_1.kt index 7be485f80..e1b8b109a 100644 --- a/src/main/java/com/amazon/ion/impl/IonRawWriter_1_1.kt +++ b/src/main/java/com/amazon/ion/impl/IonRawWriter_1_1.kt @@ -65,6 +65,12 @@ interface IonRawWriter_1_1 { */ fun _private_hasFirstAnnotation(sid: Int, text: String?): Boolean + /** + * Writes one annotation for the next value. + * [writeAnnotations] may be called more than once to build up a list of annotations. + */ + fun writeAnnotations(annotation0: SystemSymbols_1_1) + /** * Writes one annotation for the next value. * [writeAnnotations] may be called more than once to build up a list of annotations. @@ -106,6 +112,12 @@ interface IonRawWriter_1_1 { */ fun _private_hasFieldName(): Boolean + /** + * Writes the field name for the next value. Must be called while in a struct and must be called before [writeAnnotations]. + * @throws com.amazon.ion.IonException if annotations are already written for the value or if not in a struct. + */ + fun writeFieldName(symbol: SystemSymbols_1_1) + /** * Writes the field name for the next value. Must be called while in a struct and must be called before [writeAnnotations]. * @throws com.amazon.ion.IonException if annotations are already written for the value or if not in a struct. @@ -195,6 +207,7 @@ interface IonRawWriter_1_1 { fun writeSymbol(id: Int) fun writeSymbol(text: CharSequence) + fun writeSymbol(symbol: SystemSymbols_1_1) fun writeString(value: CharSequence) diff --git a/src/main/java/com/amazon/ion/impl/bin/IonManagedWriter_1_1.kt b/src/main/java/com/amazon/ion/impl/bin/IonManagedWriter_1_1.kt index f029d3091..f13696cfb 100644 --- a/src/main/java/com/amazon/ion/impl/bin/IonManagedWriter_1_1.kt +++ b/src/main/java/com/amazon/ion/impl/bin/IonManagedWriter_1_1.kt @@ -40,17 +40,10 @@ internal class IonManagedWriter_1_1( } companion object { - private val SYSTEM_SYMBOL_TABLE_MAP = hashMapOf() - - init { - var id = 1 - Symbols.systemSymbolTable().iterateDeclaredSymbolNames().forEach { - SYSTEM_SYMBOL_TABLE_MAP[it] = id++ - } - } - private val ION_VERSION_MARKER_REGEX = Regex("^\\\$ion_\\d+_\\d+$") + private const val TDL_EXPRESSION_GROUP_START = ";" + // These are chosen subjectively to be neither too big nor too small. private const val MAX_PARAMETERS_IN_ONE_LINE_SIGNATURE = 4 private const val MAX_SYMBOLS_IN_SINGLE_LINE_SYMBOL_TABLE = 10 @@ -123,7 +116,7 @@ internal class IonManagedWriter_1_1( // plus a list of symbols added by the current encoding context. /** The symbol table for the prior encoding context */ - private var symbolTable: HashMap = HashMap(SYSTEM_SYMBOL_TABLE_MAP) + private var symbolTable: HashMap = HashMap() /** Symbols to be interned since the prior encoding context. */ private var newSymbols: HashMap = LinkedHashMap() // Preserves insertion order. @@ -264,7 +257,6 @@ internal class IonManagedWriter_1_1( // in order to avoid writing a data stream with leaky context. if (depth != 0) throw IllegalStateException("Cannot reset the encoding context while stepped in any value.") symbolTable.clear() - symbolTable.putAll(SYSTEM_SYMBOL_TABLE_MAP) macroNames.clear() macrosById.clear() macroTable.clear() @@ -286,7 +278,7 @@ internal class IonManagedWriter_1_1( private fun writeEncodingDirective() { if (newSymbols.isEmpty() && newMacros.isEmpty()) return - systemData.writeAnnotations(SystemSymbols.ION_ENCODING) + systemData.writeAnnotations(SystemSymbols_1_1.ION_ENCODING) writeSystemSexp { writeSymbolTableClause() writeMacroTableClause() @@ -307,17 +299,17 @@ internal class IonManagedWriter_1_1( */ private fun writeSymbolTableClause() { val hasSymbolsToAdd = newSymbols.isNotEmpty() - val hasSymbolsToRetain = symbolTable.size > SystemSymbols.ION_1_0_MAX_ID + val hasSymbolsToRetain = symbolTable.isNotEmpty() if (!hasSymbolsToAdd && !hasSymbolsToRetain) return writeSystemSexp { forceNoNewlines(true) - systemData.writeSymbol(SystemSymbols.SYMBOL_TABLE) + systemData.writeSymbol(SystemSymbols_1_1.SYMBOL_TABLE) // Add previous symbol table if (hasSymbolsToRetain) { if (newSymbols.size > 0) forceNoNewlines(false) - writeSymbol(SystemSymbols.ION_ENCODING) + writeSymbol(SystemSymbols_1_1.ION_ENCODING) } // Add new symbols @@ -344,10 +336,10 @@ internal class IonManagedWriter_1_1( writeSystemSexp { forceNoNewlines(true) - writeSymbol(SystemSymbols.MACRO_TABLE) + writeSymbol(SystemSymbols_1_1.MACRO_TABLE) if (newMacros.size > 0) forceNoNewlines(false) if (hasMacrosToRetain) { - writeSymbol(SystemSymbols.ION_ENCODING) + writeSymbol(SystemSymbols_1_1.ION_ENCODING) } forceNoNewlines(false) newMacros.forEach { (macro, address) -> @@ -366,8 +358,8 @@ internal class IonManagedWriter_1_1( // TODO: Support for aliases writeSystemSexp { forceNoNewlines(true) - writeSymbol(SystemSymbols.EXPORT) - writeAnnotations(SystemSymbols.ION) + writeSymbol(SystemSymbols_1_1.EXPORT) + writeAnnotations(SystemSymbols_1_1.ION) writeSymbol(macro.macroName) } systemData.forceNoNewlines(false) @@ -376,7 +368,7 @@ internal class IonManagedWriter_1_1( private fun writeMacroDefinition(name: String?, macro: TemplateMacro) { writeSystemSexp { forceNoNewlines(true) - writeSymbol(SystemSymbols.MACRO) + writeSymbol(SystemSymbols_1_1.MACRO) if (name != null) writeSymbol(name) else writeNull() if (macro.signature.size > MAX_PARAMETERS_IN_ONE_LINE_SIGNATURE) forceNoNewlines(false) @@ -441,7 +433,7 @@ internal class IonManagedWriter_1_1( IonType.TIMESTAMP -> writeTimestamp((expression as Expression.TimestampValue).value) IonType.SYMBOL -> { writeSystemSexp { - writeSymbol(SystemSymbols.LITERAL) + writeSymbol(SystemSymbols_1_1.LITERAL) expression.annotations.forEach { if (it.text != null) { // TODO: If it's already in the symbol table we could check the @@ -474,11 +466,11 @@ internal class IonManagedWriter_1_1( if (expression.annotations.isNotEmpty()) { stepInSExp(usingLengthPrefix = false) numberOfTimesToStepOut[expression.endExclusive]++ - writeSymbol(SystemSymbols.ANNOTATE) + writeSymbol(SystemSymbols_1_1.ANNOTATE) // Write the annotations as symbols within an expression group writeSystemSexp { - writeSymbol(SystemSymbols.TDL_EXPRESSION_GROUP) + writeSymbol(TDL_EXPRESSION_GROUP_START) expression.annotations.forEach { if (it.text != null) { // TODO: If it's already in the symbol table we could check the @@ -490,7 +482,7 @@ internal class IonManagedWriter_1_1( } else { // TODO: See if there is a less verbose way to use SIDs in TDL writeSystemSexp { - writeSymbol(SystemSymbols.LITERAL) + writeSymbol(SystemSymbols_1_1.LITERAL) writeSymbol(it.sid) } } @@ -500,7 +492,7 @@ internal class IonManagedWriter_1_1( // Start a `(make_sexp [ ...` invocation stepInSExp(usingLengthPrefix = false) numberOfTimesToStepOut[expression.endExclusive]++ - writeSymbol(SystemSymbols.MAKE_SEXP) + writeSymbol(SystemSymbols_1_1.MAKE_SEXP) if (expression.startInclusive != expression.endExclusive) { stepInList(usingLengthPrefix = false) @@ -527,7 +519,7 @@ internal class IonManagedWriter_1_1( is Expression.ExpressionGroup -> { stepInSExp(usingLengthPrefix = false) numberOfTimesToStepOut[expression.endExclusive]++ - writeSymbol(SystemSymbols.TDL_EXPRESSION_GROUP) + writeSymbol(TDL_EXPRESSION_GROUP_START) } is Expression.MacroInvocation -> { stepInSExp(usingLengthPrefix = false) @@ -598,7 +590,7 @@ internal class IonManagedWriter_1_1( IonType.LIST -> userData.stepInList(options.writeLengthPrefix(ContainerType.LIST, newDepth)) IonType.SEXP -> userData.stepInSExp(options.writeLengthPrefix(ContainerType.SEXP, newDepth)) IonType.STRUCT -> { - if (depth == 0 && userData._private_hasFirstAnnotation(SystemSymbols.ION_SYMBOL_TABLE_SID, SystemSymbols.ION_SYMBOL_TABLE)) { + if (depth == 0 && userData._private_hasFirstAnnotation(SystemSymbols_1_1.ION_SYMBOL_TABLE.id, SystemSymbols_1_1.ION_SYMBOL_TABLE.text)) { throw IonException("User-defined symbol tables not permitted by the Ion 1.1 managed writer.") } userData.stepInStruct(options.writeLengthPrefix(ContainerType.STRUCT, newDepth)) @@ -632,8 +624,8 @@ internal class IonManagedWriter_1_1( userData.writeNull(IonType.SYMBOL) } else { val text: String? = content.text - if (content.sid == SystemSymbols.ION_1_0_SID) throw IonException("Can't write a top-level symbol that is the same as the IVM.") - if (text == SystemSymbols.ION_1_0) throw IonException("Can't write a top-level symbol that is the same as the IVM.") + // TODO: Check to see if the SID refers to a user symbol with text that looks like an IVM + if (text == SystemSymbols_1_1.ION_1_0.text && depth == 0) throw IonException("Can't write a top-level symbol that is the same as the IVM.") handleSymbolToken(content.sid, content.text, SymbolKind.VALUE, userData) } } @@ -780,7 +772,11 @@ internal class IonManagedWriter_1_1( // from Ion 1.0, we will have to adjust any SIDs that we are writing. reader.typeAnnotationSymbols.forEach { - handleSymbolToken(it.sid, it.text, SymbolKind.ANNOTATION, userData, preserveEncoding = true) + if (it.text == SystemSymbols_1_1.ION_SYMBOL_TABLE.text) { + userData.writeAnnotations(SystemSymbols_1_1.ION_SYMBOL_TABLE) + } else { + handleSymbolToken(it.sid, it.text, SymbolKind.ANNOTATION, userData, preserveEncoding = true) + } } if (isInStruct) { // TODO: Can't use reader.fieldId, reader.fieldName because it will throw UnknownSymbolException. diff --git a/src/main/java/com/amazon/ion/impl/bin/IonRawBinaryWriter_1_1.kt b/src/main/java/com/amazon/ion/impl/bin/IonRawBinaryWriter_1_1.kt index 83d2484ac..b0b73d6ab 100644 --- a/src/main/java/com/amazon/ion/impl/bin/IonRawBinaryWriter_1_1.kt +++ b/src/main/java/com/amazon/ion/impl/bin/IonRawBinaryWriter_1_1.kt @@ -108,7 +108,7 @@ class IonRawBinaryWriter_1_1 internal constructor( private val utf8StringEncoder = Utf8StringEncoderPool.getInstance().getOrCreate() - private var annotationsTextBuffer = arrayOfNulls(8) + private var annotationsFlexSymBuffer = arrayOfNulls(8) private var annotationsIdBuffer = IntArray(8) private var numAnnotations = 0 /** @@ -195,16 +195,22 @@ class IonRawBinaryWriter_1_1 internal constructor( * are being added one by one. */ private inline fun ensureAnnotationSpace(n: Int) { - if (annotationsIdBuffer.size < n || annotationsTextBuffer.size < n) { + if (annotationsIdBuffer.size < n || annotationsFlexSymBuffer.size < n) { val oldIds = annotationsIdBuffer annotationsIdBuffer = IntArray(n + 8) oldIds.copyInto(annotationsIdBuffer) - val oldText = annotationsTextBuffer - annotationsTextBuffer = arrayOfNulls(n + 8) - oldText.copyInto(annotationsTextBuffer) + val oldText = annotationsFlexSymBuffer + annotationsFlexSymBuffer = arrayOfNulls(n + 8) + oldText.copyInto(annotationsFlexSymBuffer) } } + override fun writeAnnotations(annotation0: SystemSymbols_1_1) { + ensureAnnotationSpace(numAnnotations + 1) + annotationsFlexSymBuffer[numAnnotations++] = annotation0 + annotationFlexSymFlag = FLEX_SYMS_REQUIRED + } + override fun writeAnnotations(annotation0: Int) { confirm(annotation0 >= 0) { "Invalid SID: $annotation0" } ensureAnnotationSpace(numAnnotations + 1) @@ -227,21 +233,21 @@ class IonRawBinaryWriter_1_1 internal constructor( override fun writeAnnotations(annotation0: CharSequence) { ensureAnnotationSpace(numAnnotations + 1) - annotationsTextBuffer[numAnnotations++] = annotation0 + annotationsFlexSymBuffer[numAnnotations++] = annotation0 annotationFlexSymFlag = FLEX_SYMS_REQUIRED } override fun writeAnnotations(annotation0: CharSequence, annotation1: CharSequence) { ensureAnnotationSpace(numAnnotations + 2) - annotationsTextBuffer[numAnnotations++] = annotation0 - annotationsTextBuffer[numAnnotations++] = annotation1 + annotationsFlexSymBuffer[numAnnotations++] = annotation0 + annotationsFlexSymBuffer[numAnnotations++] = annotation1 annotationFlexSymFlag = FLEX_SYMS_REQUIRED } override fun writeAnnotations(annotations: Array) { if (annotations.isEmpty()) return ensureAnnotationSpace(numAnnotations + annotations.size) - annotations.copyInto(annotationsTextBuffer, numAnnotations) + annotations.copyInto(annotationsFlexSymBuffer, numAnnotations) numAnnotations += annotations.size annotationFlexSymFlag = FLEX_SYMS_REQUIRED } @@ -251,7 +257,7 @@ class IonRawBinaryWriter_1_1 internal constructor( annotationFlexSymFlag = 0 // erase the first entries to ensure old values don't leak into `_private_hasFirstAnnotation()` annotationsIdBuffer[0] = -1 - annotationsTextBuffer[0] = null + annotationsFlexSymBuffer[0] = null } override fun _private_hasFirstAnnotation(sid: Int, text: String?): Boolean { @@ -259,7 +265,7 @@ class IonRawBinaryWriter_1_1 internal constructor( if (sid >= 0 && annotationsIdBuffer[0] == sid) { return true } - if (text != null && annotationsTextBuffer[0] == text) { + if (text != null && annotationsFlexSymBuffer[0] == text) { return true } return false @@ -296,8 +302,8 @@ class IonRawBinaryWriter_1_1 internal constructor( // If there's only one annotation, and we know that at least one has text, we don't need to check // whether this is SID. buffer.writeByte(OpCodes.ANNOTATIONS_1_FLEX_SYM) - annotationsTotalLength += buffer.writeFlexSym(utf8StringEncoder.encode(annotationsTextBuffer[0].toString())) - annotationsTextBuffer[0] = null + annotationsTotalLength += writeFlexSymFromAnnotationsBuffer(0) + annotationsFlexSymBuffer[0] = null } -3 -> { buffer.writeByte(OpCodes.ANNOTATIONS_2_FLEX_SYM) @@ -318,10 +324,14 @@ class IonRawBinaryWriter_1_1 internal constructor( * Writes a FlexSym annotation for the specified position in the annotations buffers. */ private fun writeFlexSymFromAnnotationsBuffer(i: Int): Int { - val annotationText = annotationsTextBuffer[i] + val annotationText = annotationsFlexSymBuffer[i] return if (annotationText != null) { - annotationsTextBuffer[i] = null - buffer.writeFlexSym(utf8StringEncoder.encode(annotationText.toString())) + annotationsFlexSymBuffer[i] = null + if (annotationText is SystemSymbols_1_1) { + buffer.writeFlexSym(annotationText) + } else { + buffer.writeFlexSym(utf8StringEncoder.encode(annotationText.toString())) + } } else { buffer.writeFlexSym(annotationsIdBuffer[i]) } @@ -421,6 +431,13 @@ class IonRawBinaryWriter_1_1 internal constructor( hasFieldName = true } + override fun writeFieldName(symbol: SystemSymbols_1_1) { + confirm(currentContainer.type == STRUCT) { "Can only write a field name inside of a struct." } + if (!currentContainer.usesFlexSym) switchCurrentStructToFlexSym() + currentContainer.length += buffer.writeFlexSym(symbol) + hasFieldName = true + } + override fun _private_hasFieldName(): Boolean = hasFieldName private fun switchCurrentStructToFlexSym() { @@ -578,6 +595,12 @@ class IonRawBinaryWriter_1_1 internal constructor( } ) + override fun writeSymbol(symbol: SystemSymbols_1_1) = writeScalar { + buffer.writeByte(OpCodes.SYSTEM_SYMBOL) + buffer.writeByte(symbol.id.toByte()) + 2 + } + override fun writeString(value: CharSequence) = writeScalar { writeStringValue(buffer, utf8StringEncoder.encode(value.toString())) } override fun writeBlob(value: ByteArray, start: Int, length: Int) = writeScalar { writeBlobValue(buffer, value, start, length) } diff --git a/src/main/java/com/amazon/ion/impl/bin/Ion_1_1_Constants.java b/src/main/java/com/amazon/ion/impl/bin/Ion_1_1_Constants.java index da7b45a45..e5ca627a1 100644 --- a/src/main/java/com/amazon/ion/impl/bin/Ion_1_1_Constants.java +++ b/src/main/java/com/amazon/ion/impl/bin/Ion_1_1_Constants.java @@ -11,6 +11,10 @@ public class Ion_1_1_Constants { private Ion_1_1_Constants() {} + // When writing system symbols (or $0) in a flex sym, the SID must be offset to + // avoid clashing with E-Expression op codes. + public static final int FLEX_SYM_SYSTEM_SYMBOL_OFFSET = 0x60; + static final int FIRST_2_BYTE_SYMBOL_ADDRESS = 256; static final int FIRST_MANY_BYTE_SYMBOL_ADDRESS = 65792; diff --git a/src/main/java/com/amazon/ion/impl/bin/WriteBuffer.java b/src/main/java/com/amazon/ion/impl/bin/WriteBuffer.java index 29a824cb0..4049f49d6 100644 --- a/src/main/java/com/amazon/ion/impl/bin/WriteBuffer.java +++ b/src/main/java/com/amazon/ion/impl/bin/WriteBuffer.java @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 package com.amazon.ion.impl.bin; +import com.amazon.ion.impl.SystemSymbols_1_1; import com.amazon.ion.impl.bin.utf8.Utf8StringEncoder; import java.io.Closeable; @@ -11,6 +12,8 @@ import java.util.ArrayList; import java.util.List; +import static com.amazon.ion.impl.bin.Ion_1_1_Constants.FLEX_SYM_SYSTEM_SYMBOL_OFFSET; + /** * A facade over {@link Block} management and low-level Ion encoding concerns for the {@link IonRawBinaryWriter}. * @@ -1523,8 +1526,7 @@ public int writeFlexSym(int sid) { return writeFlexInt(sid); } else { writeByte(FlexInt.ZERO); - // The INLINE_SYMBOL_ZERO_LENGTH opcode is used for $0 in FlexSyms - writeByte(OpCodes.INLINE_SYMBOL_ZERO_LENGTH); + writeByte((byte) FLEX_SYM_SYSTEM_SYMBOL_OFFSET); return 2; } } @@ -1533,17 +1535,23 @@ public int writeFlexSym(int sid) { * Writes a FlexSym with inline text. */ public int writeFlexSym(Utf8StringEncoder.Result text) { - if (text.getEncodedLength() == 0) { + if (text.getEncodedLength() == 0) { + return writeFlexSym(SystemSymbols_1_1.THE_EMPTY_SYMBOL); + } else { + int numLengthBytes = writeFlexInt(-text.getEncodedLength()); + writeBytes(text.getBuffer(), 0, text.getEncodedLength()); + return numLengthBytes + text.getEncodedLength(); + } + } + + /** + * Writes a FlexSym with inline text. + */ + public int writeFlexSym(SystemSymbols_1_1 symbol) { writeByte(FlexInt.ZERO); - // The STRING_ZERO_LENGTH opcode is used for '' in FlexSyms - writeByte(OpCodes.STRING_ZERO_LENGTH); + writeByte((byte) (symbol.getId() + FLEX_SYM_SYSTEM_SYMBOL_OFFSET)); return 2; - } else { - int numLengthBytes = writeFlexInt(-text.getEncodedLength()); - writeBytes(text.getBuffer(), 0, text.getEncodedLength()); - return numLengthBytes + text.getEncodedLength(); } -} /** Write the entire buffer to output stream. */ public void writeTo(final OutputStream out) throws IOException diff --git a/src/test/java/com/amazon/ion/Ion_1_1_RoundTripTest.kt b/src/test/java/com/amazon/ion/Ion_1_1_RoundTripTest.kt index 47ed9798e..35fb41983 100644 --- a/src/test/java/com/amazon/ion/Ion_1_1_RoundTripTest.kt +++ b/src/test/java/com/amazon/ion/Ion_1_1_RoundTripTest.kt @@ -21,6 +21,7 @@ import org.junit.jupiter.params.provider.MethodSource /** * Suite of tests for running round trip tests on user and system values for various Ion 1.1 encodings. */ +@Disabled("IonCursorBinary has not been updated to read system symbols") class Ion_1_1_RoundTripTest { @Nested diff --git a/src/test/java/com/amazon/ion/impl/bin/IonManagedWriter_1_1_Test.kt b/src/test/java/com/amazon/ion/impl/bin/IonManagedWriter_1_1_Test.kt index eb2863b54..4d57999b5 100644 --- a/src/test/java/com/amazon/ion/impl/bin/IonManagedWriter_1_1_Test.kt +++ b/src/test/java/com/amazon/ion/impl/bin/IonManagedWriter_1_1_Test.kt @@ -13,6 +13,7 @@ import java.io.ByteArrayOutputStream import java.math.BigInteger import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Disabled import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows import org.junit.jupiter.params.ParameterizedTest @@ -134,6 +135,7 @@ internal class IonManagedWriter_1_1_Test { } @Test + @Disabled("IonCursorBinary has not been updated to read system symbols in FlexSyms") fun `use writeValues to transform symbol IDS`() { `transform symbol IDS` { reader -> writeValues(reader) { sid -> sid + 32 } @@ -141,6 +143,7 @@ internal class IonManagedWriter_1_1_Test { } @Test + @Disabled("IonCursorBinary has not been updated to read system symbols in FlexSyms") fun `use writeValue to transform symbol IDS`() { `transform symbol IDS` { reader -> while (reader.next() != null) { @@ -248,7 +251,7 @@ internal class IonManagedWriter_1_1_Test { val expected = """ $ion_1_1 $ion_encoding::((symbol_table ["foo"])) - $10 + $1 """.trimIndent() val actual = write(symbolInliningStrategy = SymbolInliningStrategy.NEVER_INLINE) { @@ -288,9 +291,9 @@ internal class IonManagedWriter_1_1_Test { val expected = """ $ion_1_1 $ion_encoding::((symbol_table ["foo"])) - $10 + $1 $ion_encoding::((symbol_table $ion_encoding ["bar"])) - $11 + $2 """.trimIndent() val actual = write(symbolInliningStrategy = SymbolInliningStrategy.NEVER_INLINE) { @@ -328,9 +331,9 @@ internal class IonManagedWriter_1_1_Test { val expected = """ $ion_1_1 $ion_encoding::((symbol_table ["foo"])) - $10 + $1 $ion_encoding::((symbol_table ["bar"])) - $10 + $1 """.trimIndent() val actual = write(symbolInliningStrategy = SymbolInliningStrategy.NEVER_INLINE) { @@ -347,7 +350,7 @@ internal class IonManagedWriter_1_1_Test { val expected = """ $ion_1_1 $ion_encoding::((symbol_table ["foo"])) - $10 + $1 $ion_encoding::((symbol_table $ion_encoding) (macro_table (macro null () "foo"))) """.trimIndent() @@ -366,7 +369,7 @@ internal class IonManagedWriter_1_1_Test { $ion_1_1 $ion_encoding::((macro_table (macro null () "foo"))) $ion_encoding::((symbol_table ["foo"]) (macro_table $ion_encoding)) - $10 + $1 (:0) """.trimIndent() @@ -666,9 +669,9 @@ internal class IonManagedWriter_1_1_Test { (macro null () "foo") (macro null () "bar")) ) - $10 - $11 - $12 + $1 + $2 + $3 (:0) (:1) $ion_encoding::( @@ -679,9 +682,9 @@ internal class IonManagedWriter_1_1_Test { $ion_encoding (macro null () "abc")) ) - $13 - $14 - $15 + $4 + $5 + $6 (:2) """.trimIndent() diff --git a/src/test/java/com/amazon/ion/impl/bin/IonRawBinaryWriterTest_1_1.kt b/src/test/java/com/amazon/ion/impl/bin/IonRawBinaryWriterTest_1_1.kt index 658d1ad66..03c5fb06b 100644 --- a/src/test/java/com/amazon/ion/impl/bin/IonRawBinaryWriterTest_1_1.kt +++ b/src/test/java/com/amazon/ion/impl/bin/IonRawBinaryWriterTest_1_1.kt @@ -480,6 +480,8 @@ class IonRawBinaryWriterTest_1_1 { 6E | true 02 01 | FlexSym SID 64 6E | true + 01 6F | System Symbol symbol_table + 6E | true 01 F0 | End delimited struct """ ) { @@ -490,6 +492,8 @@ class IonRawBinaryWriterTest_1_1 { writeBool(true) writeFieldName(64) writeBool(true) + writeFieldName(SystemSymbols_1_1.SYMBOL_TABLE) + writeBool(true) stepOut() } } @@ -584,7 +588,7 @@ class IonRawBinaryWriterTest_1_1 { FD | Variable length Struct 09 | Length = 4 01 | switch to FlexSym encoding - 01 A0 | FlexSym SID 0 + 01 60 | FlexSym SID 0 6E | true """ ) { @@ -604,11 +608,11 @@ class IonRawBinaryWriterTest_1_1 { 03 | SID 1 6E | true 01 | switch to FlexSym encoding - 01 A0 | FlexSym SID 0 + 01 60 | FlexSym SID 0 6E | true 05 | FlexSym SID 2 6E | true - 01 A0 | FlexSym SID 0 + 01 60 | FlexSym SID 0 6E | true """ ) { @@ -625,6 +629,24 @@ class IonRawBinaryWriterTest_1_1 { } } + @Test + fun `write prefixed struct with a system symbol as a field name`() { + assertWriterOutputEquals( + """ + FD | Variable length Struct + 09 | Length = 4 + 01 | switch to FlexSym encoding + 01 6F | FlexSym System Symbol 'symbol_table' + 6E | true + """ + ) { + stepInStruct(usingLengthPrefix = true) + writeFieldName(SystemSymbols_1_1.SYMBOL_TABLE) + writeBool(true) + stepOut() + } + } + @Test fun `writing a value in a struct with no field name should throw an exception`() { assertWriterThrows { @@ -800,7 +822,8 @@ class IonRawBinaryWriterTest_1_1 { @Test fun `write empty text and sid 0 annotations`() { - assertWriterOutputEquals("E8 01 A0 01 90 6E") { + // Empty text is a system symbol + assertWriterOutputEquals("E8 01 60 01 75 6E") { writeAnnotations(0) writeAnnotations("") writeBool(true) @@ -833,6 +856,52 @@ class IonRawBinaryWriterTest_1_1 { } } + @Test + fun `write one system symbol annotation`() { + val expectedBytes = "E7 01 64 6E" + assertWriterOutputEquals(expectedBytes) { + writeAnnotations(SystemSymbols_1_1.NAME) + writeBool(true) + } + assertWriterOutputEquals(expectedBytes) { + writeAnnotations(SystemSymbols_1_1.NAME) + writeAnnotations(intArrayOf()) + writeAnnotations(arrayOf()) + writeBool(true) + } + } + + @Test + fun `write two mixed sid and system annotations`() { + val expectedBytes = "E8 15 01 6A 6F" + assertWriterOutputEquals(expectedBytes) { + writeAnnotations(10) + writeAnnotations(SystemSymbols_1_1.ION_ENCODING) + writeBool(false) + } + } + + @Test + fun `write two mixed inline and system annotations`() { + val expectedBytes = "E8 FB 66 6F 6F 01 6A 6F" + assertWriterOutputEquals(expectedBytes) { + writeAnnotations("foo") + writeAnnotations(SystemSymbols_1_1.ION_ENCODING) + writeBool(false) + } + } + + @Test + fun `write three mixed sid, inline, and system annotations`() { + val expectedBytes = "E9 0F 15 FB 66 6F 6F 01 6A 6F" + assertWriterOutputEquals(expectedBytes) { + writeAnnotations(10) + writeAnnotations("foo") + writeAnnotations(SystemSymbols_1_1.ION_ENCODING) + writeBool(false) + } + } + @Test fun `write annotations that are long enough to need a patch point`() { val opCode = "E7" @@ -1009,12 +1078,16 @@ class IonRawBinaryWriterTest_1_1 { E1 01 E2 39 2F A3 66 6F 6F + EE 0B + EE 15 """ ) { writeSymbol(0) writeSymbol(1) writeSymbol(12345) writeSymbol("foo") + writeSymbol(SystemSymbols_1_1.ION_LITERAL) + writeSymbol(SystemSymbols_1_1.THE_EMPTY_SYMBOL) } } @@ -1804,13 +1877,13 @@ class IonRawBinaryWriterTest_1_1 { @ParameterizedTest @CsvSource( // SID - " 0, 01 A0", + " 0, 01 60", " 4, 09", " 246, DA 03", // Text " a, FF 61", " abc, FB 61 62 63", - " '', 01 90", + " '', 01 75", ) fun `write a tagless symbol`(value: String, expectedBytes: String) { val macro = dummyMacro(nArgs = 1, variadicParam(ParameterEncoding.CompactSymbol)) diff --git a/src/test/java/com/amazon/ion/impl/bin/WriteBufferTest.java b/src/test/java/com/amazon/ion/impl/bin/WriteBufferTest.java index b3d8b5691..9b5fc46f9 100644 --- a/src/test/java/com/amazon/ion/impl/bin/WriteBufferTest.java +++ b/src/test/java/com/amazon/ion/impl/bin/WriteBufferTest.java @@ -1778,7 +1778,7 @@ public void testWriteFixedIntOrUIntThrowsExceptionWhenNumBytesIsOutOfBounds() { @ParameterizedTest @CsvSource({ - " 0, 00000001 10100000", + " 0, 00000001 01100000", " 1, 00000011", " 2, 00000101", "63, 01111111", @@ -1793,7 +1793,7 @@ public void testWriteSidFlexSym(int value, String expectedBits) { @ParameterizedTest @CsvSource({ - "'', 00000001 10010000", + "'', 00000001 01110101", // 01110101 == SystemSymbols_1_1.THE_EMPTY_SYMBOL.getId() converted to binary "a, 11111111 01100001", "abc, 11111011 01100001 01100010 01100011", "this is a very very very very very long symbol, " +