Adds runner for Ion conformance tests

build.gradle.kts

@@ -79,6 +79,11 @@ dependencies {
     testImplementation("org.hamcrest:hamcrest:2.2")
     testImplementation("pl.pragmatists:JUnitParams:1.1.1")
     testImplementation("com.google.code.tempus-fugit:tempus-fugit:1.1")
+
+    // Used for the conformance test runner, because IonValue is not suitable for it.
+    testImplementation("com.amazon.ion:ion-element:1.2.0")
+    // Force the tests to use the locally built version rather than a version transitively provided by `ion-element`.
+    testImplementation(project)
 }
 
 group = "com.amazon.ion"

src/main/java/com/amazon/ion/impl/_Private_IonTextWriterBuilder_1_1.java

@@ -78,12 +78,12 @@ public _Private_IonTextWriterBuilder_1_1 mutable()
 
     @Override
     public _Private_IonTextWriterBuilder_1_1 withCatalog(IonCatalog catalog) {
-        return (_Private_IonTextWriterBuilder_1_1) super.getCatalog();
+        return (_Private_IonTextWriterBuilder_1_1) super.withCatalog(catalog);
     }
 
     @Override
     public _Private_IonTextWriterBuilder_1_1 withImports(SymbolTable[] imports) {
-        return (_Private_IonTextWriterBuilder_1_1) super.getCatalog();
+        return (_Private_IonTextWriterBuilder_1_1) super.withImports(imports);
     }
 
     @Override

src/test/java/com/amazon/ion/FakeSymbolToken.java

@@ -2,11 +2,13 @@
 // SPDX-License-Identifier: Apache-2.0
 package com.amazon.ion;
 
+import com.amazon.ion.impl._Private_SymbolToken;
+
 /**
  * NOT SUITABLE FOR PUBLIC USE since it doesn't enforce correctness.
  */
 public class FakeSymbolToken
-    implements SymbolToken
+    implements SymbolToken, _Private_SymbolToken
 {
     private final String myText;
     private final int mySid;

src/test/java/com/amazon/ion/conformance/Config.kt

@@ -0,0 +1,20 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package com.amazon.ion.conformance
+
+import com.amazon.ion.system.IonReaderBuilder
+import java.io.File
+
+/** Top-level configuration for running the conformance tests */
+data class Config(
+    /** Controls whether debug printing should be turned on. */
+    val debugEnabled: Boolean = true,
+    /** If a NotImplementedError is encountered, should we fail the test or ignore it. */
+    val failUnimplemented: Boolean = false,
+    /** Use for a skip list, or for running only one or two tests. Return true to run the test. */
+    val testFilter: (File, String) -> Boolean = { _, _ -> true },
+    /** Named set of reader builders (i.e. different reader configurations) to use for all tests. */
+    val readerBuilders: Map<String, IonReaderBuilder>,
+) {
+    fun newCaseBuilder(file: File) = ConformanceTestBuilder(this, file)
+}

src/test/java/com/amazon/ion/conformance/ConformanceTestBuilder.kt

@@ -0,0 +1,105 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package com.amazon.ion.conformance
+
+import com.amazon.ion.*
+import com.amazon.ion.system.*
+import com.amazon.ionelement.api.IonElement
+import com.amazon.ionelement.api.SeqElement
+import com.amazon.ionelement.api.location
+import java.io.File
+import org.junit.jupiter.api.DynamicContainer.dynamicContainer
+import org.junit.jupiter.api.DynamicNode
+import org.junit.jupiter.api.DynamicTest.dynamicTest
+import org.opentest4j.AssertionFailedError
+import org.opentest4j.TestAbortedException
+
+data class ConformanceTestBuilder(
+    val config: Config,
+    /** File that the cases are bring created from. */
+    val file: File,
+    // Internal fields for building up state from which to create a test case
+    private val nameParts: List<String> = listOf(),
+    private val fragments: List<SeqElement> = listOf(),
+) {
+
+    /**
+     * Helper class that provides runtime support to the test cases.
+     */
+    class TestCaseSupport(private val testBuilder: ConformanceTestBuilder, private val readerBuilder: IonReaderBuilder) {
+
+        private val data: ByteArray by lazy { readFragments(testBuilder.fragments) }
+
+        /** Creates a new reader for this test case */
+        fun createFragmentReader(): IonReader = readerBuilder.build(data)
+
+        /** Logs a lazily-evaluated message, if debug is enabled. */
+        fun debug(message: () -> String) = testBuilder.debug(message)
+
+        /** Throws an exception for a syntax error in the tests */
+        fun reportSyntaxError(element: IonElement, details: String? = null): Nothing =
+            testBuilder.reportSyntaxError(element, details)
+
+        /** Creates a file URI for the given IonElement */
+        fun locationOf(element: IonElement) = "file://${testBuilder.file.absolutePath}:${element.metas.location}"
+
+        /** Creates a failure message that includes a file link to [element] */
+        fun createFailureMessage(element: IonElement, details: String? = null): String =
+            "${details ?: "Assertion failed"} at ${locationOf(element)}; $element"
+
+        /** Throws an [AssertionFailedError] to fail a test case */
+        fun fail(expectation: IonElement, details: String, t: Throwable? = null): Nothing =
+            throw AssertionFailedError(createFailureMessage(expectation, details), t)
+    }
+
+    // Leaf nodes need a full name or else the HTML report is incomprehensible.
+    private val fullName: String
+        get() = nameParts.joinToString(" ")
+
+    // TODO: this could be fullName or nameParts.last()
+    //       Both have drawbacks, but it only affects the display of the interior nodes of the test tree
+    val containerName: String
+        get() = fullName // nameParts.last()
+
+    /** Prints a debug message, if debug messages are enabled in the config. */
+    fun debug(message: () -> String) {
+        if (config.debugEnabled) println("[TEST: $fullName] ${message()}")
+    }
+
+    // Copy-on-write setters
+    fun plusName(name: String): ConformanceTestBuilder = copy(nameParts = nameParts + name)
+    fun plusFragment(fragment: SeqElement): ConformanceTestBuilder = copy(fragments = fragments + fragment)
+    fun plusFragments(newFragments: List<SeqElement>): ConformanceTestBuilder = copy(fragments = fragments + newFragments)
+    fun plus(name: String, fragment: SeqElement): ConformanceTestBuilder = copy(nameParts = nameParts + name, fragments = fragments + fragment)
+    fun plus(name: String, newFragments: List<SeqElement>): ConformanceTestBuilder = copy(nameParts = nameParts + name, fragments = fragments + newFragments)
+
+    fun build(executable: TestCaseSupport.() -> Unit): DynamicNode {
+        return config.readerBuilders.map { (readerName, readerBuilder) ->
+            val testName = "$fullName using $readerName"
+            val testCaseSupport = TestCaseSupport(this, readerBuilder)
+            dynamicTest(testName) {
+                if (!config.testFilter(file, testName)) throw TestAbortedException(testName)
+                debug { "Begin Test using $readerName" }
+                try {
+                    executable(testCaseSupport)
+                } catch (e: NotImplementedError) {
+                    if (config.failUnimplemented) throw e
+                    debug { "Ignored because ${e.message}" }
+                    throw TestAbortedException("$e")
+                }
+            }
+        }.let {
+            dynamicContainer(containerName, it)
+        }
+    }
+
+    /** Builds a [DynamicNode] container with the correct name */
+    fun buildContainer(children: Iterable<DynamicNode>): DynamicNode = dynamicContainer(containerName, children)
+
+    /** Builds a [DynamicNode] container with the correct name */
+    fun buildContainer(vararg children: DynamicNode): DynamicNode = dynamicContainer(containerName, children.toList())
+
+    /** Signals to the test builder that there is a syntax error */
+    fun reportSyntaxError(element: IonElement, details: String? = null): Nothing =
+        throw ConformanceTestInvalidSyntaxException(file, element, details)
+}

src/test/java/com/amazon/ion/conformance/ConformanceTestDslInterpreterTest.kt

@@ -0,0 +1,104 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package com.amazon.ion.conformance
+
+import com.amazon.ion.system.*
+import java.io.File
+import kotlin.streams.toList
+import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.DynamicContainer
+import org.junit.jupiter.api.DynamicNode
+import org.junit.jupiter.api.DynamicTest
+import org.junit.jupiter.params.ParameterizedTest
+import org.junit.jupiter.params.provider.MethodSource
+
+/** Some minimal tests for the DSL interpreter. */
+object ConformanceTestDslInterpreterTest {
+
+    private val CONFIG = Config(
+        debugEnabled = true,
+        failUnimplemented = false,
+        readerBuilders = mapOf("only reader" to IonReaderBuilder.standard()),
+    )
+
+    @JvmStatic
+    fun data(): Iterable<Pair<String, Int>> = listOf(
+        """
+        (document "a test using 'produces'"
+                  (produces))
+        """ to 1,
+        """
+        (ion_1_0 "a test using 'text'"
+                 (text ''' {a:1, b:2} "two" ''')
+                 (produces {b:2, a:1} "two"))
+        """ to 1,
+        """
+        (ion_1_0 "a test using 'signals'"
+                 (text ''' {a:1, b:2 "two" ''')
+                 (signals "struct missing closing delimiter"))
+        """ to 1,
+        """
+        (ion_1_1 "a test that uses binary"
+                  (bytes "6F 6E 60")
+                  (produces false true 0))
+        """ to 1,
+        """
+        (ion_1_0 "a test that uses denotes"
+                  (text "${'$'}4")
+                  (denotes (Symbol "name")))
+        """ to 1,
+        """
+        (ion_1_0 "a test using 'then'"
+                 (text ''' 1 ''')
+                 (then (text "2")
+                       (produces 1 2)))
+        """ to 1,
+        """
+        (ion_1_0 "a test using 'then' to create more than one test case"
+                 (text ''' 1 ''')
+                 (then "then 2"
+                       (text "2")
+                       (produces 1 2))
+                 (then "then 3"
+                       (text "3")
+                       (produces 1 3)))
+        """ to 2,
+        """
+        (ion_1_0 "a test using 'each' to create more than one test case"
+                 (text " 1 ")
+                 (each "unclosed container"
+                       (text " { ")
+                       (text " [ ")
+                       (text " ( ")
+                       (signals "something bad")))
+        """ to 3,
+        """
+        (ion_1_x "a test using 'ion_1_x' to create more than one test case"
+                 (text " 1 ")
+                 (produces 1))
+        """ to 2,
+        // TODO: Tests to check the demangling behavior, use different types of fragments
+    )
+
+    @MethodSource("data")
+    @ParameterizedTest
+    fun interpreterTests(testInput: Pair<String, Int>) {
+        val (dsl, expectedNumberOfTestCases) = testInput
+
+        val testBuilder = CONFIG.newCaseBuilder(File("fake-file"))
+        val testCases = testBuilder.readAllTests(ION.newReader(dsl)).flatten()
+
+        // It should have the correct number of test cases
+        assertEquals(expectedNumberOfTestCases, testCases.size)
+        // All the test case executables should run without throwing any exceptions (i.e. pass)
+        testCases.forEach { it.executable.execute() }
+    }
+
+    private fun DynamicNode.flatten(): List<DynamicTest> {
+        return when (this@flatten) {
+            is DynamicContainer -> children.toList().flatMap { it.flatten() }
+            is DynamicTest -> listOf(this)
+            else -> TODO("Unreachable")
+        }
+    }
+}

src/test/java/com/amazon/ion/conformance/ConformanceTestInvalidSyntaxException.kt

@@ -0,0 +1,21 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package com.amazon.ion.conformance
+
+import com.amazon.ionelement.api.IonElement
+import com.amazon.ionelement.api.location
+import java.io.File
+
+/** Exception for signalling invalid syntax in the conformance tests. */
+class ConformanceTestInvalidSyntaxException(
+    file: File,
+    element: IonElement,
+    description: String? = null,
+    cause: Throwable? = null
+) : Error(cause) {
+    override val message: String = """
+        Invalid conformance dsl syntax${ description?.let { "; $it" } ?: ""}
+          - at file://${file.absolutePath}:${element.metas.location}
+          - invalid clause was: $element
+    """.trimIndent()
+}

src/test/java/com/amazon/ion/conformance/ConformanceTestRunner.kt

@@ -0,0 +1,77 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+package com.amazon.ion.conformance
+
+import com.amazon.ion.system.*
+import java.io.File
+import org.junit.jupiter.api.DynamicNode
+import org.junit.jupiter.api.TestFactory
+
+object ConformanceTestRunner {
+    val DEFAULT_READER_BUILDER_CONFIGURATIONS = mapOf(
+        "default reader" to IonReaderBuilder.standard()
+            .withCatalog(ION_CONFORMANCE_TEST_CATALOG),
+        "incremental reader" to IonReaderBuilder.standard()
+            .withCatalog(ION_CONFORMANCE_TEST_CATALOG)
+            .withIncrementalReadingEnabled(true),
+        // TODO: Other reader configurations
+    )
+
+    private val DEFAULT_SKIP_FILTER: (File, String) -> Boolean = { file, name ->
+        when {
+            // IonElement can't load $0. TODO: Use IonValue for `produces`, I guess.
+            "$0" in name -> false
+            // For some reason, $ion_symbol_table::null.struct is not handled as expected
+            "IST structs are elided from app view" in name -> false
+            // IonWriter is making it difficult to write invalid data
+            "If no max_id, lack of exact-match must raise an error «then»" in name -> false
+            // IonCatalog's "best choice" logic is not spec compliant
+            // TODO—current test name has a typo. Update to correct spelling once ion-tests is fixed.
+            "When max_id is valid, pad/truncade mismatched or absent SSTs" in name -> false
+            // No support for reading `$ion_encoding` directives yet.
+            "conformance/ion_encoding/" in file.absolutePath -> false
+            // TODO: Seems like the Ion writer is removing "extra" fields from user-supplied symbol tables.
+            file.endsWith("local_symtab_imports.ion") -> when {
+                // If you inspect the debug output, the serialized data does not include the repeated fields.
+                // This implies that the writer is attempting to clean a user-supplied symbol table.
+                "Repeated fields" in name -> false
+                // Unfortunately, the writer also seems to remove "imports" field if and only if
+                "Importing the current symbol table" in name -> false
+                // For these tests, the writer is validating the max_id field, and failing before
+                // we have a chance to test the reader.
+                "If no max_id, lack of exact-match must raise an error" in name -> false
+                "If max_id not non-negative int, lack of exact-match must raise an error" in name -> false
+                else -> true
+            }
+            // Some of these are failing because
+            //  - Ion Java doesn't support the Ion 1.1 system symbol table yet
+            //  - The tokens `$ion_1_0` and `'$ion_1_0'` are never user values.
+            // TODO: Add test names once they are added to this file
+            file.endsWith("system_symbols.ion") -> false
+            // $ion_literal not supported yet
+            file.endsWith("ion_literal.ion") -> false
+            else -> true
+        }
+    }
+
+    private val CONFIG = Config(
+        debugEnabled = true,
+        failUnimplemented = false,
+        readerBuilders = DEFAULT_READER_BUILDER_CONFIGURATIONS,
+        testFilter = DEFAULT_SKIP_FILTER,
+    )
+
+    @TestFactory
+    fun `Conformance Tests`(): Iterable<DynamicNode> {
+        return ION_CONFORMANCE_DIR.walk()
+            .filter { it.isFile && it.extension == "ion" }
+            .map { file ->
+                with(CONFIG.newCaseBuilder(file)) {
+                    file.inputStream()
+                        .let(ION::newReader)
+                        .use { reader -> readAllTests(reader) }
+                }
+            }
+            .asIterable()
+    }
+}