Adds support for parsing Ion 1.1 encoding directives in the text reader.

src/main/java/com/amazon/ion/impl/IonRawTextWriter_1_1.kt

@@ -4,8 +4,12 @@ package com.amazon.ion.impl
 
 import com.amazon.ion.*
 import com.amazon.ion.impl.IonRawTextWriter_1_1.ContainerType.*
+import com.amazon.ion.impl.IonRawTextWriter_1_1.ContainerType.List
+import com.amazon.ion.impl.bin.*
 import com.amazon.ion.impl.macro.*
+import com.amazon.ion.system.*
 import com.amazon.ion.util.*
+import java.io.OutputStream
 import java.math.BigDecimal
 import java.math.BigInteger
 
@@ -25,6 +29,18 @@ class IonRawTextWriter_1_1 internal constructor(
 
     companion object {
         const val IVM = "\$ion_1_1"
+
+        @JvmStatic
+        fun from(output: OutputStream, blockSize: Int, options: IonTextWriterBuilder_1_1): IonRawTextWriter_1_1 {
+            val bufferedOutput = BufferedOutputStreamFastAppendable(
+                output,
+                BlockAllocatorProviders.basicProvider().vendAllocator(blockSize)
+            )
+            return IonRawTextWriter_1_1(
+                options as _Private_IonTextWriterBuilder_1_1,
+                _Private_IonTextAppender.forFastAppendable(bufferedOutput, Charsets.UTF_8)
+            )
+        }
     }
 
     enum class ContainerType {

src/main/java/com/amazon/ion/impl/IonReaderContinuableCoreBinary.java

@@ -1233,7 +1233,9 @@ private void installMacros() {
          * Install any new symbols and macros, step out of the encoding directive, and resume reading raw values.
          */
         private void finishEncodingDirective() {
-            resetSymbolTable(); // TODO handle appended symbols
+            if (!isSymbolTableAppend) {
+                resetSymbolTable();
+            }
             installSymbols(newSymbols);
             installMacros();
             stepOutOfContainer();
@@ -1361,10 +1363,6 @@ void readEncodingDirective() {
                         newMacros.put(MacroRef.byId(++localMacroMaxOffset), newMacro);
                         state = State.IN_MACRO_TABLE_SEXP;
                         break;
-                    case COMPILING_MACRO:
-                        // This state can only be reached during compilation of a macro. Do nothing, as the reader must
-                        // navigate normally while the macro is compiled.
-                        break;
                     default:
                         throw new IllegalStateException(state.toString());
                 }

src/main/java/com/amazon/ion/impl/IonReaderTextSystemX.java

@@ -1,18 +1,5 @@
-/*
- * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
 package com.amazon.ion.impl;
 
 import static com.amazon.ion.impl._Private_ScalarConversions.getValueTypeName;
@@ -38,11 +25,17 @@
 import com.amazon.ion.impl.IonTokenConstsX.CharacterSequence;
 import com.amazon.ion.impl._Private_ScalarConversions.AS_TYPE;
 import com.amazon.ion.impl._Private_ScalarConversions.CantConvertException;
+import com.amazon.ion.impl.macro.EncodingContext;
+import com.amazon.ion.impl.macro.EncodingDirectiveReader;
+import com.amazon.ion.impl.macro.MacroEvaluator;
+import com.amazon.ion.impl.macro.MacroEvaluatorAsIonReader;
+
 import java.io.IOException;
 import java.math.BigDecimal;
 import java.math.BigInteger;
+import java.util.Collections;
 import java.util.Date;
-import java.lang.Character;
+import java.util.List;
 
 /**
  * This reader calls the {@link IonReaderTextRawX} for low level events.
@@ -62,6 +55,18 @@ class IonReaderTextSystemX
 
     SymbolTable _system_symtab;
 
+    // The IonReader-like MacroEvaluator that this core reader delegates to when evaluating a macro invocation.
+    protected MacroEvaluatorAsIonReader macroEvaluatorIonReader = null;
+
+    // The core MacroEvaluator that this core reader delegates to when evaluating a macro invocation.
+    private MacroEvaluator macroEvaluator = null;
+
+    // The encoding context (macro table) that is currently active.
+    private EncodingContext encodingContext = null;
+
+    // Reads encoding directives from the stream.
+    private EncodingDirectiveReader encodingDirectiveReader = null;
+
     protected IonReaderTextSystemX(UnifiedInputStreamX iis)
     {
         _system_symtab = _Private_Utils.systemSymtab(1); // TODO check IVM to determine version: amazon-ion/ion-java/issues/19
@@ -1009,4 +1014,100 @@ public SymbolTable pop_passed_symbol_table()
     {
         return null;
     }
+
+    /**
+     * Sets the active symbol table.
+     * @param symbolTable the symbol table to make active.
+     */
+    protected void setSymbolTable(SymbolTable symbolTable) {
+        // System readers don't handle symbol tables.
+    }
+
+    /**
+     * While reading an encoding directive, the reader allows itself to be controlled by the MacroCompiler during
+     * compilation of a macro. While this is happening, the reader should never attempt to read another encoding
+     * directive.
+     * @return true if the reader is not in the process of compiling a macro; false if it is.
+     */
+    private boolean macroCompilationNotInProgress() {
+        return encodingDirectiveReader == null || !encodingDirectiveReader.isMacroCompilationInProgress();
+    }
+
+    /**
+     * @return true if current value has a sequence of annotations that begins with `$ion_encoding`; otherwise, false.
+     */
+    boolean startsWithIonEncoding() {
+        // TODO also resolve symbol identifiers and compare against text that looks like $ion_encoding
+        return SystemSymbols_1_1.ION_ENCODING.getText().equals(_annotations[0].getText());
+    }
+
+    /**
+     * @return true if the reader is positioned on an encoding directive; otherwise, false.
+     */
+    private boolean isPositionedOnEncodingDirective() {
+        return _annotation_count > 0
+            && _value_type == IonType.SEXP
+            && !isNullValue()
+            && macroCompilationNotInProgress()
+            && startsWithIonEncoding();
+    }
+
+    /**
+     * Reads an encoding directive and installs any symbols and/or macros found within. Upon calling this method,
+     * the reader must be positioned on an s-expression annotated with `$ion_encoding`.
+     */
+    private void readEncodingDirective() {
+        if (encodingDirectiveReader == null) {
+            encodingDirectiveReader = new EncodingDirectiveReader(this);
+        }
+        encodingDirectiveReader.reset();
+        encodingDirectiveReader.readEncodingDirective();
+        List<String> newSymbols = encodingDirectiveReader.getNewSymbols();
+        if (encodingDirectiveReader.isSymbolTableAppend()) {
+            LocalSymbolTable current = ((LocalSymbolTable) getSymbolTable());
+            for (String appendedSymbol : newSymbols) {
+                current.putSymbol(appendedSymbol);
+            }
+        } else {
+            setSymbolTable(new LocalSymbolTable(
+                // TODO handle shared symbol table imports declared in the encoding directive
+                LocalSymbolTableImports.EMPTY,
+                newSymbols
+            ));
+        }
+        encodingContext = new EncodingContext(encodingDirectiveReader.getNewMacros());
+        macroEvaluator = new MacroEvaluator();
+        macroEvaluatorIonReader = new MacroEvaluatorAsIonReader(macroEvaluator);
+    }
+
+    /**
+     * Advances the reader, if necessary and possible, to the next value, reading any Ion 1.1+ encoding directives
+     * found along the way.
+     * @return true if the reader is positioned on a value; otherwise, false.
+     */
+    protected final boolean has_next_system_value() {
+        while (!_has_next_called && !_eof) {
+            has_next_raw_value();
+            if (minorVersion > 0 && _value_type != null && IonType.DATAGRAM.equals(getContainerType()) && isPositionedOnEncodingDirective()) {
+                readEncodingDirective();
+                _has_next_called = false;
+                continue;
+            }
+            break;
+        }
+        return !_eof;
+    }
+
+    @Override
+    public boolean hasNext()
+    {
+        return has_next_system_value();
+    }
+
+    /**
+     * @return the {@link EncodingContext} currently active, or {@code null}.
+     */
+    EncodingContext getEncodingContext() {
+        return encodingContext;
+    }
 }

src/main/java/com/amazon/ion/impl/IonReaderTextUserX.java

@@ -65,7 +65,7 @@ protected IonReaderTextUserX(IonCatalog catalog,
                                  int physicalStartOffset)
     {
         super(uis);
-        _symbols = _system_symtab;
+        setSymbolTable(_system_symtab);
         _physical_start_offset = physicalStartOffset;
         _catalog = catalog;
         _lstFactory = lstFactory;
@@ -77,6 +77,11 @@ protected IonReaderTextUserX(IonCatalog catalog,
         this(catalog, lstFactory, uis, 0);
     }
 
+    @Override
+    protected void setSymbolTable(SymbolTable symbolTable) {
+        _symbols = symbolTable;
+    }
+
     /**
      * this looks forward to see if there is an upcoming value
      * and if there is it returns true.  It may have to clean up
@@ -109,7 +114,7 @@ private final boolean has_next_user_value()
         {
             // first move to the next value regardless of whether
             // it's a system value or a user value
-            has_next_raw_value();
+            has_next_system_value();
 
             // system values are only at the datagram level
             // we don't care about them if they're buried
@@ -120,9 +125,9 @@ private final boolean has_next_user_value()
                 switch (_value_type) {
                 case STRUCT:
                     if (_annotation_count > 0 && (ION_SYMBOL_TABLE.equals(_annotations[0].getText()) || ION_SYMBOL_TABLE_SID == _annotations[0].getSid())) {
-                        _symbols = _lstFactory.newLocalSymtab(_catalog,
+                        setSymbolTable(_lstFactory.newLocalSymtab(_catalog,
                                                               this,
-                                                              true);
+                                                              true));
                         push_symbol_table(_symbols);
                         _has_next_called = false;
                     }
@@ -168,7 +173,7 @@ private final void symbol_table_reset()
     {
         IonType t = next();
         assert( IonType.SYMBOL.equals(t) );
-        _symbols = _system_symtab;
+        setSymbolTable(_system_symtab);
         return;
     }