From 41a666e5557797376b69bc2e117eaa050b22348d Mon Sep 17 00:00:00 2001 From: Tyler Gregg Date: Tue, 1 Oct 2024 16:01:00 -0700 Subject: [PATCH] Adds support for parsing Ion 1.1 encoding directives in the text reader. --- .../ion/impl/EncodingDirectiveReader.kt | 145 ++++++++++++++ .../amazon/ion/impl/IonRawTextWriter_1_1.kt | 16 ++ .../impl/IonReaderContinuableCoreBinary.java | 8 +- .../amazon/ion/impl/IonReaderTextSystemX.java | 133 +++++++++++-- .../amazon/ion/impl/IonReaderTextUserX.java | 15 +- .../EncodingDirectiveCompilationTest.java | 179 +++++++++++++++--- 6 files changed, 440 insertions(+), 56 deletions(-) create mode 100644 src/main/java/com/amazon/ion/impl/EncodingDirectiveReader.kt diff --git a/src/main/java/com/amazon/ion/impl/EncodingDirectiveReader.kt b/src/main/java/com/amazon/ion/impl/EncodingDirectiveReader.kt new file mode 100644 index 0000000000..88ae51d673 --- /dev/null +++ b/src/main/java/com/amazon/ion/impl/EncodingDirectiveReader.kt @@ -0,0 +1,145 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package com.amazon.ion.impl.macro + +import com.amazon.ion.* +import com.amazon.ion.impl.* +import com.amazon.ion.impl.macro.MacroRef.Companion.byId + +/** + * Reads encoding directives from the given [IonReader]. + */ +class EncodingDirectiveReader(private val reader: IonReader) { + + private var macroCompiler: MacroCompiler = MacroCompilerIonReader(reader) { key: Any? -> newMacros[key] } + private var localMacroMaxOffset: Int = -1 + private var state: State = State.READING_VALUE + + var isSymbolTableAppend = false + var newSymbols: MutableList = ArrayList(8) + var newMacros: MutableMap = HashMap() + + private enum class State { + IN_ION_ENCODING_SEXP, + IN_SYMBOL_TABLE_SEXP, + IN_SYMBOL_TABLE_LIST, + IN_MACRO_TABLE_SEXP, + COMPILING_MACRO, + READING_VALUE + } + + private fun classifySexpWithinEncodingDirective() { + val name: String = reader.stringValue() + state = if (SystemSymbols_1_1.SYMBOL_TABLE.text == name) { + State.IN_SYMBOL_TABLE_SEXP + } else if (SystemSymbols_1_1.MACRO_TABLE.text == name) { + State.IN_MACRO_TABLE_SEXP + } else { + throw IonException(String.format("\$ion_encoding expressions '%s' not supported.", name)) + } + } + + private fun classifySymbolTable() { + val type: IonType = reader.type + if (IonType.isText(type)) { + if (SystemSymbols.ION_ENCODING == reader.stringValue() && !isSymbolTableAppend) { + if (reader.next() == null || reader.type != IonType.LIST) { + throw IonException("symbol_table s-expression must begin with a list.") + } + isSymbolTableAppend = true + } else { + throw IonException("symbol_table s-expression must begin with either \$ion_encoding or a list.") + } + } else if (type != IonType.LIST) { + throw IonException("symbol_table s-expression must begin with either \$ion_encoding or a list.") + } + reader.stepIn() + state = State.IN_SYMBOL_TABLE_LIST + } + + /** + * Reads an encoding directive. After this method returns, the caller should access this class's properties to + * retrieve the symbols and macros declared within the directive. + */ + fun readEncodingDirective() { + reader.stepIn() + state = State.IN_ION_ENCODING_SEXP + while (true) { + when (state) { + + State.IN_ION_ENCODING_SEXP -> { + if (reader.next() == null) { + reader.stepOut() + state = State.READING_VALUE + return + } + if (reader.type != IonType.SEXP) { + throw IonException("Ion encoding directives must contain only s-expressions.") + } + reader.stepIn() + if (reader.next() == null || !IonType.isText(reader.type)) { + throw IonException("S-expressions within encoding directives must begin with a text token.") + } + classifySexpWithinEncodingDirective() + } + + State.IN_SYMBOL_TABLE_SEXP -> { + if (reader.next() == null) { + reader.stepOut() + state = State.IN_ION_ENCODING_SEXP + continue + } + classifySymbolTable() + } + + State.IN_SYMBOL_TABLE_LIST -> { + if (reader.next() == null) { + reader.stepOut() + state = State.IN_SYMBOL_TABLE_SEXP + continue + } + if (!IonType.isText(reader.type)) { + throw IonException("The symbol_table must contain text.") + } + newSymbols.add(reader.stringValue()) + } + + State.IN_MACRO_TABLE_SEXP -> { + if (reader.next() == null) { + reader.stepOut() + state = State.IN_ION_ENCODING_SEXP + continue + } + if (reader.type != IonType.SEXP) { + throw IonException("macro_table s-expression must contain s-expressions.") + } + state = State.COMPILING_MACRO + val newMacro: Macro = macroCompiler.compileMacro() + newMacros[byId(++localMacroMaxOffset)] = newMacro + state = State.IN_MACRO_TABLE_SEXP + } + + // TODO handle other legal encoding directive s-expression shapes. + // TODO add strict enforcement of the schema around e.g. repeats + + else -> throw IllegalStateException(state.toString()) + } + } + } + + /** + * @return true if the reader is currently being used by the [MacroCompiler]. + */ + fun isMacroCompilationInProgress(): Boolean { + return state == State.COMPILING_MACRO + } + + /** + * Prepares the EncodingDirectiveReader to read a new encoding directive. + */ + fun reset() { + isSymbolTableAppend = false + newSymbols.clear() + newMacros.clear() + } +} diff --git a/src/main/java/com/amazon/ion/impl/IonRawTextWriter_1_1.kt b/src/main/java/com/amazon/ion/impl/IonRawTextWriter_1_1.kt index bc3f1e4ae3..12aaed0f16 100644 --- a/src/main/java/com/amazon/ion/impl/IonRawTextWriter_1_1.kt +++ b/src/main/java/com/amazon/ion/impl/IonRawTextWriter_1_1.kt @@ -4,8 +4,12 @@ package com.amazon.ion.impl import com.amazon.ion.* import com.amazon.ion.impl.IonRawTextWriter_1_1.ContainerType.* +import com.amazon.ion.impl.IonRawTextWriter_1_1.ContainerType.List +import com.amazon.ion.impl.bin.* import com.amazon.ion.impl.macro.* +import com.amazon.ion.system.* import com.amazon.ion.util.* +import java.io.OutputStream import java.math.BigDecimal import java.math.BigInteger @@ -25,6 +29,18 @@ class IonRawTextWriter_1_1 internal constructor( companion object { const val IVM = "\$ion_1_1" + + @JvmStatic + fun from(output: OutputStream, blockSize: Int, options: IonTextWriterBuilder_1_1): IonRawTextWriter_1_1 { + val bufferedOutput = BufferedOutputStreamFastAppendable( + output, + BlockAllocatorProviders.basicProvider().vendAllocator(blockSize) + ) + return IonRawTextWriter_1_1( + options as _Private_IonTextWriterBuilder_1_1, + _Private_IonTextAppender.forFastAppendable(bufferedOutput, Charsets.UTF_8) + ) + } } enum class ContainerType { diff --git a/src/main/java/com/amazon/ion/impl/IonReaderContinuableCoreBinary.java b/src/main/java/com/amazon/ion/impl/IonReaderContinuableCoreBinary.java index 1e36c6d050..ca02978014 100644 --- a/src/main/java/com/amazon/ion/impl/IonReaderContinuableCoreBinary.java +++ b/src/main/java/com/amazon/ion/impl/IonReaderContinuableCoreBinary.java @@ -1233,7 +1233,9 @@ private void installMacros() { * Install any new symbols and macros, step out of the encoding directive, and resume reading raw values. */ private void finishEncodingDirective() { - resetSymbolTable(); // TODO handle appended symbols + if (!isSymbolTableAppend) { + resetSymbolTable(); + } installSymbols(newSymbols); installMacros(); stepOutOfContainer(); @@ -1361,10 +1363,6 @@ void readEncodingDirective() { newMacros.put(MacroRef.byId(++localMacroMaxOffset), newMacro); state = State.IN_MACRO_TABLE_SEXP; break; - case COMPILING_MACRO: - // This state can only be reached during compilation of a macro. Do nothing, as the reader must - // navigate normally while the macro is compiled. - break; default: throw new IllegalStateException(state.toString()); } diff --git a/src/main/java/com/amazon/ion/impl/IonReaderTextSystemX.java b/src/main/java/com/amazon/ion/impl/IonReaderTextSystemX.java index c187682d53..0f6c843009 100644 --- a/src/main/java/com/amazon/ion/impl/IonReaderTextSystemX.java +++ b/src/main/java/com/amazon/ion/impl/IonReaderTextSystemX.java @@ -1,18 +1,5 @@ -/* - * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ion.impl; import static com.amazon.ion.impl._Private_ScalarConversions.getValueTypeName; @@ -38,11 +25,17 @@ import com.amazon.ion.impl.IonTokenConstsX.CharacterSequence; import com.amazon.ion.impl._Private_ScalarConversions.AS_TYPE; import com.amazon.ion.impl._Private_ScalarConversions.CantConvertException; +import com.amazon.ion.impl.macro.EncodingContext; +import com.amazon.ion.impl.macro.EncodingDirectiveReader; +import com.amazon.ion.impl.macro.MacroEvaluator; +import com.amazon.ion.impl.macro.MacroEvaluatorAsIonReader; + import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; +import java.util.Collections; import java.util.Date; -import java.lang.Character; +import java.util.List; /** * This reader calls the {@link IonReaderTextRawX} for low level events. @@ -62,6 +55,18 @@ class IonReaderTextSystemX SymbolTable _system_symtab; + // The IonReader-like MacroEvaluator that this core reader delegates to when evaluating a macro invocation. + protected MacroEvaluatorAsIonReader macroEvaluatorIonReader = null; + + // The core MacroEvaluator that this core reader delegates to when evaluating a macro invocation. + private MacroEvaluator macroEvaluator = null; + + // The encoding context (macro table) that is currently active. + private EncodingContext encodingContext = null; + + // Reads encoding directives from the stream. + private EncodingDirectiveReader encodingDirectiveReader = null; + protected IonReaderTextSystemX(UnifiedInputStreamX iis) { _system_symtab = _Private_Utils.systemSymtab(1); // TODO check IVM to determine version: amazon-ion/ion-java/issues/19 @@ -1009,4 +1014,100 @@ public SymbolTable pop_passed_symbol_table() { return null; } + + /** + * Sets the active symbol table. + * @param symbolTable the symbol table to make active. + */ + protected void setSymbolTable(SymbolTable symbolTable) { + // System readers don't handle symbol tables. + } + + /** + * While reading an encoding directive, the reader allows itself to be controlled by the MacroCompiler during + * compilation of a macro. While this is happening, the reader should never attempt to read another encoding + * directive. + * @return true if the reader is not in the process of compiling a macro; false if it is. + */ + private boolean macroCompilationNotInProgress() { + return encodingDirectiveReader == null || !encodingDirectiveReader.isMacroCompilationInProgress(); + } + + /** + * @return true if current value has a sequence of annotations that begins with `$ion_encoding`; otherwise, false. + */ + boolean startsWithIonEncoding() { + // TODO also resolve symbol identifiers and compare against text that looks like $ion_encoding + return SystemSymbols_1_1.ION_ENCODING.getText().equals(_annotations[0].getText()); + } + + /** + * @return true if the reader is positioned on an encoding directive; otherwise, false. + */ + private boolean isPositionedOnEncodingDirective() { + return _annotation_count > 0 + && _value_type == IonType.SEXP + && !isNullValue() + && macroCompilationNotInProgress() + && startsWithIonEncoding(); + } + + /** + * Reads an encoding directive and installs any symbols and/or macros found within. Upon calling this method, + * the reader must be positioned on an s-expression annotated with `$ion_encoding`. + */ + private void readEncodingDirective() { + if (encodingDirectiveReader == null) { + encodingDirectiveReader = new EncodingDirectiveReader(this); + } + encodingDirectiveReader.reset(); + encodingDirectiveReader.readEncodingDirective(); + List newSymbols = encodingDirectiveReader.getNewSymbols(); + if (encodingDirectiveReader.isSymbolTableAppend()) { + LocalSymbolTable current = ((LocalSymbolTable) getSymbolTable()); + for (String appendedSymbol : newSymbols) { + current.putSymbol(appendedSymbol); + } + } else { + setSymbolTable(new LocalSymbolTable( + // TODO handle shared symbol table imports declared in the encoding directive + LocalSymbolTableImports.EMPTY, + newSymbols + )); + } + encodingContext = new EncodingContext(encodingDirectiveReader.getNewMacros()); + macroEvaluator = new MacroEvaluator(); + macroEvaluatorIonReader = new MacroEvaluatorAsIonReader(macroEvaluator); + } + + /** + * Advances the reader, if necessary and possible, to the next value, reading any Ion 1.1+ encoding directives + * found along the way. + * @return true if the reader is positioned on a value; otherwise, false. + */ + protected final boolean has_next_system_value() { + while (!_has_next_called && !_eof) { + has_next_raw_value(); + if (minorVersion > 0 && _value_type != null && IonType.DATAGRAM.equals(getContainerType()) && isPositionedOnEncodingDirective()) { + readEncodingDirective(); + _has_next_called = false; + continue; + } + break; + } + return !_eof; + } + + @Override + public boolean hasNext() + { + return has_next_system_value(); + } + + /** + * @return the {@link EncodingContext} currently active, or {@code null}. + */ + EncodingContext getEncodingContext() { + return encodingContext; + } } diff --git a/src/main/java/com/amazon/ion/impl/IonReaderTextUserX.java b/src/main/java/com/amazon/ion/impl/IonReaderTextUserX.java index fd91c4d7a5..d0984312f7 100644 --- a/src/main/java/com/amazon/ion/impl/IonReaderTextUserX.java +++ b/src/main/java/com/amazon/ion/impl/IonReaderTextUserX.java @@ -65,7 +65,7 @@ protected IonReaderTextUserX(IonCatalog catalog, int physicalStartOffset) { super(uis); - _symbols = _system_symtab; + setSymbolTable(_system_symtab); _physical_start_offset = physicalStartOffset; _catalog = catalog; _lstFactory = lstFactory; @@ -77,6 +77,11 @@ protected IonReaderTextUserX(IonCatalog catalog, this(catalog, lstFactory, uis, 0); } + @Override + protected void setSymbolTable(SymbolTable symbolTable) { + _symbols = symbolTable; + } + /** * this looks forward to see if there is an upcoming value * and if there is it returns true. It may have to clean up @@ -109,7 +114,7 @@ private final boolean has_next_user_value() { // first move to the next value regardless of whether // it's a system value or a user value - has_next_raw_value(); + has_next_system_value(); // system values are only at the datagram level // we don't care about them if they're buried @@ -120,9 +125,9 @@ private final boolean has_next_user_value() switch (_value_type) { case STRUCT: if (_annotation_count > 0 && (ION_SYMBOL_TABLE.equals(_annotations[0].getText()) || ION_SYMBOL_TABLE_SID == _annotations[0].getSid())) { - _symbols = _lstFactory.newLocalSymtab(_catalog, + setSymbolTable(_lstFactory.newLocalSymtab(_catalog, this, - true); + true)); push_symbol_table(_symbols); _has_next_called = false; } @@ -168,7 +173,7 @@ private final void symbol_table_reset() { IonType t = next(); assert( IonType.SYMBOL.equals(t) ); - _symbols = _system_symtab; + setSymbolTable(_system_symtab); return; } diff --git a/src/test/java/com/amazon/ion/impl/EncodingDirectiveCompilationTest.java b/src/test/java/com/amazon/ion/impl/EncodingDirectiveCompilationTest.java index 44c6c3fbed..1a8f09bf8b 100644 --- a/src/test/java/com/amazon/ion/impl/EncodingDirectiveCompilationTest.java +++ b/src/test/java/com/amazon/ion/impl/EncodingDirectiveCompilationTest.java @@ -4,24 +4,30 @@ import com.amazon.ion.FakeSymbolToken; import com.amazon.ion.IntegerSize; +import com.amazon.ion.IonEncodingVersion; import com.amazon.ion.IonReader; import com.amazon.ion.IonType; import com.amazon.ion.SystemSymbols; import com.amazon.ion.impl.bin.IonRawBinaryWriter_1_1; +import com.amazon.ion.impl.macro.EncodingContext; import com.amazon.ion.impl.macro.Expression; import com.amazon.ion.impl.macro.Macro; import com.amazon.ion.impl.macro.MacroRef; import com.amazon.ion.impl.macro.TemplateMacro; import com.amazon.ion.system.IonReaderBuilder; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import java.io.ByteArrayOutputStream; +import java.io.OutputStream; import java.math.BigDecimal; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Consumer; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -35,8 +41,8 @@ public class EncodingDirectiveCompilationTest { private static final int FIRST_LOCAL_SYMBOL_ID = 1; - private static void assertMacroTablesEqual(IonReader reader, Map expected) { - Map actual = ((IonReaderContinuableCoreBinary) reader).getEncodingContext().getMacroTable(); + private static void assertMacroTablesEqual(IonReader reader, StreamType streamType, Map expected) { + Map actual = streamType.getEncodingContext(reader).getMacroTable(); assertEquals(expected, actual); } @@ -58,9 +64,12 @@ private static void endEncodingDirective(IonRawWriter_1_1 writer) { writer.stepOut(); } - private static void writeEncodingDirectiveSymbolTable(IonRawWriter_1_1 writer, String... userSymbols) { + private static void writeEncodingDirectiveSymbolTable(IonRawWriter_1_1 writer, boolean append, String... userSymbols) { writer.stepInSExp(false); writer.writeSymbol(SystemSymbols.SYMBOL_TABLE); + if (append) { + writer.writeSymbol(SystemSymbols.ION_ENCODING); + } writer.stepInList(false); for (String userSymbol : userSymbols) { writer.writeString(userSymbol); @@ -69,6 +78,10 @@ private static void writeEncodingDirectiveSymbolTable(IonRawWriter_1_1 writer, S writer.stepOut(); } + private static void writeEncodingDirectiveSymbolTable(IonRawWriter_1_1 writer, String... userSymbols) { + writeEncodingDirectiveSymbolTable(writer, false, userSymbols); + } + private static Map initializeSymbolTable(IonRawWriter_1_1 writer, String... userSymbols) { startEncodingDirective(writer); writeEncodingDirectiveSymbolTable(writer, userSymbols); @@ -90,10 +103,28 @@ private static void endMacroTable(IonRawWriter_1_1 writer) { writer.stepOut(); } + private static void writeSymbolToken(Consumer tokenTextWriter, Consumer tokenSidWriter, Map symbols, String value) { + Integer sid = symbols.get(value); + if (sid == null) { + // There is no mapping; write as text + tokenTextWriter.accept(value); + } else { + tokenSidWriter.accept(sid); + } + } + + private static void writeSymbol(IonRawWriter_1_1 writer, Map symbols, String value) { + writeSymbolToken(writer::writeSymbol, writer::writeSymbol, symbols, value); + } + + private static void writeFieldName(IonRawWriter_1_1 writer, Map symbols, String name) { + writeSymbolToken(writer::writeFieldName, writer::writeFieldName, symbols, name); + } + private static void startMacro(IonRawWriter_1_1 writer, Map symbols, String name) { writer.stepInSExp(false); writer.writeSymbol(SystemSymbols_1_1.MACRO); - writer.writeSymbol(symbols.get(name)); + writeSymbol(writer, symbols, name); } private static void endMacro(IonRawWriter_1_1 writer) { @@ -103,15 +134,15 @@ private static void endMacro(IonRawWriter_1_1 writer) { private static void writeMacroSignature(IonRawWriter_1_1 writer, Map symbols, String... signature) { writer.stepInSExp(false); for (String parameter : signature) { - writer.writeSymbol(symbols.get(parameter)); + writeSymbol(writer, symbols, parameter); } writer.stepOut(); } - private static void writeVariableExpansion(IonRawWriter_1_1 writer, Integer variableNameSid) { + private static void writeVariableExpansion(IonRawWriter_1_1 writer, Map symbols, String variableName) { writer.stepInSExp(false); writer.writeSymbol("%"); - writer.writeSymbol(variableNameSid); + writeSymbol(writer, symbols, variableName); writer.stepOut(); } @@ -122,8 +153,8 @@ private static void stepInTdlMacroInvocation(IonRawWriter_1_1 writer, Integer ma } private static void writeVariableField(IonRawWriter_1_1 writer, Map symbols, String fieldName, String variableName) { - writer.writeFieldName(symbols.get(fieldName)); - writeVariableExpansion(writer, symbols.get(variableName)); + writeFieldName(writer, symbols, fieldName); + writeVariableExpansion(writer, symbols, variableName); } private static byte[] getBytes(IonRawWriter_1_1 writer, ByteArrayOutputStream out) { @@ -131,12 +162,99 @@ private static byte[] getBytes(IonRawWriter_1_1 writer, ByteArrayOutputStream ou return out.toByteArray(); } - @Test - public void structMacroWithOneOptional() throws Exception { + public enum StreamType { + BINARY { + @Override + IonRawWriter_1_1 newWriter(OutputStream out) { + return IonRawBinaryWriter_1_1.from(out, 256, 0); + } + + @Override + EncodingContext getEncodingContext(IonReader reader) { + return ((IonReaderContinuableCoreBinary) reader).getEncodingContext(); + } + }, + TEXT { + @Override + IonRawWriter_1_1 newWriter(OutputStream out) { + return IonRawTextWriter_1_1.from(out, 256, IonEncodingVersion.ION_1_1.textWriterBuilder()); + } + + @Override + EncodingContext getEncodingContext(IonReader reader) { + return ((IonReaderTextSystemX) reader).getEncodingContext(); + } + }; + + abstract IonRawWriter_1_1 newWriter(OutputStream out); + abstract EncodingContext getEncodingContext(IonReader reader); + } + + private static int getSymbolId(Map symbols, String value) { + Integer sid = symbols.get(value); + return sid == null ? -1 : sid; + } + + @ParameterizedTest + @EnumSource(StreamType.class) + public void symbolsOnly(StreamType streamType) throws Exception { ByteArrayOutputStream out = new ByteArrayOutputStream(); - IonRawWriter_1_1 writer = IonRawBinaryWriter_1_1.from(out, 256, 0); + IonRawWriter_1_1 writer = streamType.newWriter(out); writer.writeIVM(); - Map symbols = initializeSymbolTable(writer, "People", "ID", "Name", "Bald", "$ID", "$Name", "$Bald", "?"); + startEncodingDirective(writer); + writeEncodingDirectiveSymbolTable(writer, "foo", "bar"); + endEncodingDirective(writer); + writer.writeSymbol(FIRST_LOCAL_SYMBOL_ID); + writer.writeSymbol(FIRST_LOCAL_SYMBOL_ID + 1); + byte[] data = getBytes(writer, out); + try (IonReader reader = IonReaderBuilder.standard().build(data)) { + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("foo", reader.stringValue()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("bar", reader.stringValue()); + assertNull(reader.next()); + } + } + + @ParameterizedTest + @EnumSource(StreamType.class) + public void symbolAppendWithoutMacros(StreamType streamType) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + IonRawWriter_1_1 writer = streamType.newWriter(out); + writer.writeIVM(); + startEncodingDirective(writer); + writeEncodingDirectiveSymbolTable(writer, "foo", "bar"); + endEncodingDirective(writer); + startEncodingDirective(writer); + writeEncodingDirectiveSymbolTable(writer, true, "baz"); + endEncodingDirective(writer); + writer.writeSymbol(FIRST_LOCAL_SYMBOL_ID); + writer.writeSymbol(FIRST_LOCAL_SYMBOL_ID + 1); + writer.writeSymbol(FIRST_LOCAL_SYMBOL_ID + 2); + byte[] data = getBytes(writer, out); + try (IonReader reader = IonReaderBuilder.standard().build(data)) { + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("foo", reader.stringValue()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("bar", reader.stringValue()); + assertEquals(IonType.SYMBOL, reader.next()); + assertEquals("baz", reader.stringValue()); + assertNull(reader.next()); + } + } + + @ParameterizedTest + @EnumSource(StreamType.class) + public void structMacroWithOneOptional(StreamType streamType) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + IonRawWriter_1_1 writer = streamType.newWriter(out); + writer.writeIVM(); + Map symbols; + if (streamType == StreamType.BINARY) { + symbols = initializeSymbolTable(writer, "People", "ID", "Name", "Bald", "$ID", "$Name", "$Bald", "?"); + } else { + symbols = Collections.emptyMap(); + } startEncodingDirective(writer); startMacroTable(writer); startMacro(writer, symbols, "People"); @@ -165,25 +283,26 @@ public void structMacroWithOneOptional() throws Exception { put("Name", Collections.singletonList(4)); put("Bald", Collections.singletonList(6)); }}), - new Expression.FieldName(new FakeSymbolToken("ID", symbols.get("ID"))), + new Expression.FieldName(new FakeSymbolToken("ID", getSymbolId(symbols, "ID"))), new Expression.VariableRef(0), - new Expression.FieldName(new FakeSymbolToken("Name", symbols.get("Name"))), + new Expression.FieldName(new FakeSymbolToken("Name", getSymbolId(symbols, "Name"))), new Expression.VariableRef(1), - new Expression.FieldName(new FakeSymbolToken("Bald", symbols.get("Bald"))), + new Expression.FieldName(new FakeSymbolToken("Bald", getSymbolId(symbols, "Bald"))), new Expression.VariableRef(2) ) ); try (IonReader reader = IonReaderBuilder.standard().build(data)) { assertEquals(IonType.INT, reader.next()); - assertMacroTablesEqual(reader, newMacroTable(expectedMacro)); + assertMacroTablesEqual(reader, streamType, newMacroTable(expectedMacro)); } } - @Test - public void constantMacroWithUserSymbol() throws Exception { + @ParameterizedTest + @EnumSource(StreamType.class) + public void constantMacroWithUserSymbol(StreamType streamType) throws Exception { ByteArrayOutputStream out = new ByteArrayOutputStream(); - IonRawWriter_1_1 writer = IonRawBinaryWriter_1_1.from(out, 256, 0); + IonRawWriter_1_1 writer = streamType.newWriter(out); writer.writeIVM(); Map symbols = initializeSymbolTable(writer, "Pi"); startEncodingDirective(writer); @@ -205,7 +324,7 @@ public void constantMacroWithUserSymbol() throws Exception { try (IonReader reader = IonReaderBuilder.standard().build(data)) { assertEquals(IonType.SYMBOL, reader.next()); - assertMacroTablesEqual(reader, newMacroTable(expectedMacro)); + assertMacroTablesEqual(reader, streamType, newMacroTable(expectedMacro)); assertEquals("foo", reader.stringValue()); } } @@ -265,7 +384,7 @@ public void structMacroWithOneOptionalInvoked() throws Exception { try (IonReader reader = IonReaderBuilder.standard().build(data)) { assertEquals(IonType.STRUCT, reader.next()); - assertMacroTablesEqual(reader, newMacroTable(expectedMacro)); + assertMacroTablesEqual(reader, StreamType.BINARY, newMacroTable(expectedMacro)); reader.stepIn(); assertEquals(1, reader.getDepth()); assertEquals(IonType.INT, reader.next()); @@ -353,7 +472,7 @@ public void macroInvocationWithinStruct() throws Exception { try (IonReader reader = IonReaderBuilder.standard().build(data)) { assertEquals(IonType.STRUCT, reader.next()); - assertMacroTablesEqual(reader, newMacroTable(expectedMacro)); + assertMacroTablesEqual(reader, StreamType.BINARY, newMacroTable(expectedMacro)); reader.stepIn(); assertEquals(IonType.STRUCT, reader.next()); assertEquals("foo", reader.getFieldName()); @@ -428,7 +547,7 @@ public void macroInvocationWithOptionalSuppressedBeforeEndWithinStruct() throws try (IonReader reader = IonReaderBuilder.standard().build(data)) { assertEquals(IonType.STRUCT, reader.next()); - assertMacroTablesEqual(reader, newMacroTable(expectedMacro)); + assertMacroTablesEqual(reader, StreamType.BINARY, newMacroTable(expectedMacro)); reader.stepIn(); assertEquals(IonType.STRUCT, reader.next()); assertEquals("foo", reader.getFieldName()); @@ -475,7 +594,7 @@ public void constantMacroInvoked() throws Exception { try (IonReader reader = IonReaderBuilder.standard().build(data)) { assertEquals(IonType.DECIMAL, reader.next()); - assertMacroTablesEqual(reader, newMacroTable(expectedMacro)); + assertMacroTablesEqual(reader, StreamType.BINARY, newMacroTable(expectedMacro)); assertEquals(new BigDecimal("3.14159"), reader.decimalValue()); } } @@ -489,7 +608,7 @@ private Macro writeSimonSaysMacro(IonRawWriter_1_1 writer) { startMacro(writer, symbols, "SimonSays"); writeMacroSignature(writer, symbols, "anything"); // The body - writeVariableExpansion(writer, symbols.get("anything")); + writeVariableExpansion(writer, symbols, "anything"); endMacro(writer); endMacroTable(writer); endEncodingDirective(writer); @@ -517,7 +636,7 @@ public void structAsParameter() throws Exception { try (IonReader reader = IonReaderBuilder.standard().build(data)) { assertEquals(IonType.STRUCT, reader.next()); - assertMacroTablesEqual(reader, newMacroTable(expectedMacro)); + assertMacroTablesEqual(reader, StreamType.BINARY, newMacroTable(expectedMacro)); reader.stepIn(); assertEquals(IonType.INT, reader.next()); assertEquals("foo", reader.getFieldName()); @@ -658,10 +777,10 @@ public void twoArgumentGroups() throws Exception { writeMacroSignature(writer, symbols, "these", "*", "those", "+"); writer.stepInList(true); writer.stepInList(true); - writeVariableExpansion(writer, symbols.get("those")); + writeVariableExpansion(writer, symbols, "those"); writer.stepOut(); writer.stepInList(true); - writeVariableExpansion(writer, symbols.get("these")); + writeVariableExpansion(writer, symbols, "these"); writer.stepOut(); writer.stepOut(); endMacro(writer); @@ -728,7 +847,7 @@ public void macroInvocationInMacroDefinition() throws Exception { startMacroTable(writer); startMacro(writer, symbols, "SimonSays"); writeMacroSignature(writer, symbols, "anything"); - writeVariableExpansion(writer, symbols.get("anything")); // The body: a variable + writeVariableExpansion(writer, symbols, "anything"); // The body: a variable endMacro(writer); startMacro(writer, symbols, "Echo"); writeMacroSignature(writer, symbols); // empty signature