Implements writeValues for IonManagedWriter_1_1 (#867)

src/main/java/com/amazon/ion/impl/IonRawTextWriter_1_1.kt

@@ -211,6 +211,17 @@ class IonRawTextWriter_1_1 internal constructor(
         numAnnotations = 0
     }
 
+    override fun _private_hasFirstAnnotation(sid: Int, text: String?): Boolean {
+        if (numAnnotations == 0) return false
+        if (sid >= 0 && annotationsIdBuffer[0] == sid) {
+            return true
+        }
+        if (text != null && annotationsTextBuffer[0] == text) {
+            return true
+        }
+        return false
+    }
+
     override fun _private_hasFieldName(): Boolean = hasFieldName
 
     override fun writeFieldName(sid: Int) {

src/main/java/com/amazon/ion/impl/IonRawWriter_1_1.kt

@@ -59,6 +59,12 @@ interface IonRawWriter_1_1 {
      */
     fun _private_clearAnnotations()
 
+    /**
+     * Returns true if the reader has at least one annotation set and the first annotation matches the
+     * given sid OR text.
+     */
+    fun _private_hasFirstAnnotation(sid: Int, text: String?): Boolean
+
     /**
      * Writes one annotation for the next value.
      * [writeAnnotations] may be called more than once to build up a list of annotations.

src/main/java/com/amazon/ion/impl/IonReaderContinuableApplicationBinary.java

@@ -891,7 +891,7 @@ void readSymbolTable() {
                         }
                         readImportMaxId();
                         break;
-                    default: throw new IllegalStateException();
+                    default: throw new IllegalStateException(state.toString());
                 }
             }
         }
@@ -924,14 +924,28 @@ private enum State {
      *  false.
      */
     boolean startsWithIonSymbolTable() {
-        if (minorVersion == 0 || annotationTokenMarkers.isEmpty()) {
+        if (minorVersion == 0 && annotationSequenceMarker.startIndex >= 0) {
             long savedPeekIndex = peekIndex;
             peekIndex = annotationSequenceMarker.startIndex;
             int sid = readVarUInt_1_0();
             peekIndex = savedPeekIndex;
             return ION_SYMBOL_TABLE_SID == sid;
         }
-        return ION_SYMBOL_TABLE_SID == annotationTokenMarkers.get(0).endIndex;
+        if (minorVersion == 1) {
+            Marker marker = annotationTokenMarkers.get(0);
+            if (marker.startIndex < 0) {
+                return marker.endIndex == ION_SYMBOL_TABLE_SID;
+            } else {
+                if (marker.endIndex - marker.startIndex != ION_SYMBOL_TABLE_UTF8.length) return false;
+                int start = (int) marker.startIndex;
+                boolean isIonSymbolTable = true;
+                for (int i = 0; i < ION_SYMBOL_TABLE_UTF8.length; i++) {
+                    isIonSymbolTable &= buffer[start + i] == ION_SYMBOL_TABLE_UTF8[i];
+                }
+                return isIonSymbolTable;
+            }
+        }
+        return false;
     }
 
     /**

src/main/java/com/amazon/ion/impl/IonReaderNonContinuableSystem.java

@@ -35,11 +35,34 @@
  */
 final class IonReaderNonContinuableSystem implements IonReader {
 
+    private static final SymbolToken IVM_1_0 = new SymbolTokenImpl(SystemSymbols.ION_1_0, SystemSymbols.ION_1_0_SID);
+    private static final SymbolToken IVM_1_1 = new SymbolTokenImpl("$ion_1_1", -1);
+
+    /**
+     * Represents an IVM that was read that has not yet been exposed as a Symbol value.
+     */
+    private enum PendingIvm {
+        ION_1_0(IVM_1_0),
+        ION_1_1(IVM_1_1);
+
+        private final SymbolToken token;
+        PendingIvm(SymbolToken symbolToken) {
+            token = symbolToken;
+        }
+
+        static PendingIvm pendingIvmForVersionOrNull(int major, int minor) {
+            if (major != 1) return null;
+            if (minor == 0) return ION_1_0;
+            if (minor == 1) return ION_1_1;
+            return null;
+        }
+    }
+
     private final IonReaderContinuableCore reader;
     private IonType type = null;
     private IonType typeAfterIvm = null;
-    private final Queue<Integer> pendingIvmSids = new ArrayDeque<>(1);
-    private int pendingIvmSid = -1;
+    private final Queue<PendingIvm> pendingIvms = new ArrayDeque<>(1);
+    private PendingIvm pendingIvm = null;
 
     /**
      * Constructs a new non-continuable system-level reader over the given continuable reader.
@@ -48,14 +71,11 @@ final class IonReaderNonContinuableSystem implements IonReader {
     IonReaderNonContinuableSystem(IonReaderContinuableCore reader) {
         this.reader = reader;
         reader.registerIvmNotificationConsumer((major, minor) -> {
-            if (major != 1 || minor > 1) {
+            PendingIvm ivm = PendingIvm.pendingIvmForVersionOrNull(major, minor);
+            if (ivm == null) {
                 throw new IllegalStateException("The parser should have already thrown upon encountering this illegal IVM.");
             }
-            if (minor == 0) {
-                pendingIvmSids.add(SystemSymbols.ION_1_0_SID);
-            } else {
-                // TODO how should this be handled for Ion 1.1?
-            }
+            pendingIvms.add(ivm);
         });
     }
 
@@ -70,20 +90,20 @@ public boolean hasNext() {
      * @return true if a value is ready to be presented to the user; otherwise, false.
      */
     private boolean handleIvm() {
-        Integer ivmSid = pendingIvmSids.poll();
-        if (ivmSid != null) {
+        PendingIvm nextPendingIvm = pendingIvms.poll();
+        if (nextPendingIvm != null) {
             // An IVM has been found between values.
             if (typeAfterIvm == null) {
                 // Only save the type of the next user value the first time an IVM is encountered before that value.
                 typeAfterIvm = type;
             }
             // For consistency with the legacy implementation, the system reader surfaces IVMs as symbol values.
             type = IonType.SYMBOL;
-            pendingIvmSid = ivmSid;
+            pendingIvm = nextPendingIvm;
             return true;
-        } else if (pendingIvmSid != -1) {
+        } else if (pendingIvm != null) {
             // All preceding IVMs have been surfaced. Restore the value that follows.
-            pendingIvmSid = -1;
+            pendingIvm = null;
             type = typeAfterIvm;
             typeAfterIvm = null;
             return true;
@@ -162,7 +182,7 @@ public IntegerSize getIntegerSize() {
 
     @Override
     public boolean isNullValue() {
-        return pendingIvmSid == -1 && reader.isNullValue();
+        return pendingIvm == null && reader.isNullValue();
     }
 
     @Override
@@ -226,8 +246,8 @@ public Timestamp timestampValue() {
 
     @Override
     public String stringValue() {
-        if (pendingIvmSid != -1) {
-            return getSymbolTable().findKnownSymbol(pendingIvmSid);
+        if (pendingIvm != null) {
+            return pendingIvm.token.getText();
         }
         prepareScalar();
         String value;
@@ -278,7 +298,7 @@ public SymbolTable getSymbolTable() {
 
     @Override
     public String[] getTypeAnnotations() {
-        if (pendingIvmSid != -1 || !reader.hasAnnotations()) {
+        if (pendingIvm != null || !reader.hasAnnotations()) {
             return _Private_Utils.EMPTY_STRING_ARRAY;
         }
         // Note: it is not expected that the system reader is used in performance-sensitive applications; hence,
@@ -301,7 +321,7 @@ public String[] getTypeAnnotations() {
 
     @Override
     public SymbolToken[] getTypeAnnotationSymbols() {
-        if (pendingIvmSid != -1 || !reader.hasAnnotations()) {
+        if (pendingIvm != null || !reader.hasAnnotations()) {
             return SymbolToken.EMPTY_ARRAY;
         }
         // Note: it is not expected that the system reader is used in performance-sensitive applications; hence,
@@ -322,7 +342,7 @@ public SymbolToken[] getTypeAnnotationSymbols() {
 
     @Override
     public Iterator<String> iterateTypeAnnotations() {
-        if (pendingIvmSid != -1 || !reader.hasAnnotations()) {
+        if (pendingIvm != null || !reader.hasAnnotations()) {
             return _Private_Utils.emptyIterator();
         }
         return _Private_Utils.stringIterator(getTypeAnnotations());
@@ -369,9 +389,8 @@ public SymbolToken getFieldNameSymbol() {
     public SymbolToken symbolValue() {
         String symbolText;
         int sid = -1;
-        if (pendingIvmSid != -1) {
-            sid = pendingIvmSid;
-            symbolText = getSymbolTable().findKnownSymbol(sid);
+        if (pendingIvm != null) {
+            return pendingIvm.token;
         } else {
             prepareScalar();
             if (reader.hasSymbolText()) {

src/main/java/com/amazon/ion/impl/IonReaderTextUserX.java

@@ -4,6 +4,7 @@
 
 import static com.amazon.ion.SystemSymbols.ION_1_0;
 import static com.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE;
+import static com.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE_SID;
 
 import com.amazon.ion.IonCatalog;
 import com.amazon.ion.IonType;
@@ -118,7 +119,7 @@ private final boolean has_next_user_value()
             if (_value_type != null && !isNullValue() && IonType.DATAGRAM.equals(getContainerType())) {
                 switch (_value_type) {
                 case STRUCT:
-                    if (_annotation_count > 0 && ION_SYMBOL_TABLE.equals(_annotations[0].getText())) {
+                    if (_annotation_count > 0 && (ION_SYMBOL_TABLE.equals(_annotations[0].getText()) || ION_SYMBOL_TABLE_SID == _annotations[0].getSid())) {
                         _symbols = _lstFactory.newLocalSymtab(_catalog,
                                                               this,
                                                               true);
@@ -158,7 +159,7 @@ private final boolean has_next_user_value()
         return (!_eof);
     }
 
-    private static boolean isIonVersionMarker(String text)
+    static boolean isIonVersionMarker(String text)
     {
         return text != null && ION_VERSION_MARKER_REGEX.matcher(text).matches();
     }

src/main/java/com/amazon/ion/impl/IonReaderTreeUserX.java

@@ -1,23 +1,12 @@
-/*
- * Copyright 2007-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
 package com.amazon.ion.impl;
 
 import static com.amazon.ion.SymbolTable.UNKNOWN_SYMBOL_ID;
 import static com.amazon.ion.SystemSymbols.ION_1_0_SID;
 import static com.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE;
+import static com.amazon.ion.SystemSymbols.ION_SYMBOL_TABLE_SID;
+import static com.amazon.ion.impl.IonReaderTextUserX.isIonVersionMarker;
 
 import com.amazon.ion.IonCatalog;
 import com.amazon.ion.IonDatagram;
@@ -29,6 +18,7 @@
 import com.amazon.ion.Span;
 import com.amazon.ion.SpanProvider;
 import com.amazon.ion.SymbolTable;
+import com.amazon.ion.SymbolToken;
 
 
 final class IonReaderTreeUserX
@@ -111,9 +101,9 @@ boolean next_helper_user()
                             sid = _system_symtab.findSymbol(name);
                         }
                     }
-                    if (sid == ION_1_0_SID
+                    if ((sid == ION_1_0_SID || isIonVersionMarker(sym.symbolValue().getText()))
                         && _next.getTypeAnnotationSymbols().length == 0) {
-                        // $ion_1_0 is read as an IVM only if it is not annotated
+                        // $ion_1_0 and other version markers are read as an IVM only if unannotated
                         SymbolTable symbols = _system_symtab;
                         _symbols = symbols;
                         push_symbol_table(symbols);
@@ -122,7 +112,7 @@ boolean next_helper_user()
                     }
                 }
                 else if (IonType.STRUCT.equals(next_type)
-                      && _next.findTypeAnnotation(ION_SYMBOL_TABLE) == 0
+                    && _next_has_ion_symbol_table_annotation()
                 ) {
                     assert(_next instanceof IonStruct);
                     // read a local symbol table
@@ -143,6 +133,14 @@ else if (IonType.STRUCT.equals(next_type)
         }
         return (next_type != null);
     }
+
+    private boolean _next_has_ion_symbol_table_annotation() {
+        SymbolToken[] annotations = _next.getTypeAnnotationSymbols();
+        if (annotations.length == 0) return false;
+        return annotations[0].getSid() == ION_SYMBOL_TABLE_SID
+            || annotations[0].getText() == ION_SYMBOL_TABLE;
+    }
+
     //
     //  This code handles the skipped symbol table
     //  support - it is cloned in IonReaderTextUserX