From c873a7f8ecbca9c53ccf04813052fd39ff3fa13a Mon Sep 17 00:00:00 2001 From: Matthew Pope Date: Tue, 18 Jun 2024 19:24:43 -0700 Subject: [PATCH] Adds runner for Ion conformance tests --- build.gradle.kts | 5 + ion-tests | 2 +- .../_Private_IonTextWriterBuilder_1_1.java | 4 +- .../java/com/amazon/ion/FakeSymbolToken.java | 4 +- .../java/com/amazon/ion/conformance/Config.kt | 20 ++ .../ion/conformance/ConformanceTestBuilder.kt | 105 +++++++ .../ConformanceTestDslInterpreterTest.kt | 104 +++++++ .../ConformanceTestInvalidSyntaxException.kt | 21 ++ .../ion/conformance/ConformanceTestRunner.kt | 77 +++++ .../amazon/ion/conformance/expectations.kt | 249 ++++++++++++++++ .../com/amazon/ion/conformance/fragments.kt | 276 ++++++++++++++++++ .../com/amazon/ion/conformance/structure.kt | 184 ++++++++++++ .../java/com/amazon/ion/conformance/util.kt | 64 ++++ .../java/com/amazon/ion/util/formatting.kt | 12 + 14 files changed, 1123 insertions(+), 4 deletions(-) create mode 100644 src/test/java/com/amazon/ion/conformance/Config.kt create mode 100644 src/test/java/com/amazon/ion/conformance/ConformanceTestBuilder.kt create mode 100644 src/test/java/com/amazon/ion/conformance/ConformanceTestDslInterpreterTest.kt create mode 100644 src/test/java/com/amazon/ion/conformance/ConformanceTestInvalidSyntaxException.kt create mode 100644 src/test/java/com/amazon/ion/conformance/ConformanceTestRunner.kt create mode 100644 src/test/java/com/amazon/ion/conformance/expectations.kt create mode 100644 src/test/java/com/amazon/ion/conformance/fragments.kt create mode 100644 src/test/java/com/amazon/ion/conformance/structure.kt create mode 100644 src/test/java/com/amazon/ion/conformance/util.kt create mode 100644 src/test/java/com/amazon/ion/util/formatting.kt diff --git a/build.gradle.kts b/build.gradle.kts index 654377445c..4887b323da 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -79,6 +79,11 @@ dependencies { testImplementation("org.hamcrest:hamcrest:2.2") testImplementation("pl.pragmatists:JUnitParams:1.1.1") testImplementation("com.google.code.tempus-fugit:tempus-fugit:1.1") + + // Used for the conformance test runner, because IonValue is not suitable for it. + testImplementation("com.amazon.ion:ion-element:1.2.0") + // Force the tests to use the locally built version rather than a version transitively provided by `ion-element`. + testImplementation(project) } group = "com.amazon.ion" diff --git a/ion-tests b/ion-tests index ef0451a72a..7a1129a6b7 160000 --- a/ion-tests +++ b/ion-tests @@ -1 +1 @@ -Subproject commit ef0451a72a39f572175ad8e90b1a77110e9aec4c +Subproject commit 7a1129a6b79687383d2220f9b62125146194e868 diff --git a/src/main/java/com/amazon/ion/impl/_Private_IonTextWriterBuilder_1_1.java b/src/main/java/com/amazon/ion/impl/_Private_IonTextWriterBuilder_1_1.java index 35a867732d..42dc1bbed2 100644 --- a/src/main/java/com/amazon/ion/impl/_Private_IonTextWriterBuilder_1_1.java +++ b/src/main/java/com/amazon/ion/impl/_Private_IonTextWriterBuilder_1_1.java @@ -78,12 +78,12 @@ public _Private_IonTextWriterBuilder_1_1 mutable() @Override public _Private_IonTextWriterBuilder_1_1 withCatalog(IonCatalog catalog) { - return (_Private_IonTextWriterBuilder_1_1) super.getCatalog(); + return (_Private_IonTextWriterBuilder_1_1) super.withCatalog(catalog); } @Override public _Private_IonTextWriterBuilder_1_1 withImports(SymbolTable[] imports) { - return (_Private_IonTextWriterBuilder_1_1) super.getCatalog(); + return (_Private_IonTextWriterBuilder_1_1) super.withImports(imports); } @Override diff --git a/src/test/java/com/amazon/ion/FakeSymbolToken.java b/src/test/java/com/amazon/ion/FakeSymbolToken.java index d602d2ebc6..10d6528900 100644 --- a/src/test/java/com/amazon/ion/FakeSymbolToken.java +++ b/src/test/java/com/amazon/ion/FakeSymbolToken.java @@ -2,11 +2,13 @@ // SPDX-License-Identifier: Apache-2.0 package com.amazon.ion; +import com.amazon.ion.impl._Private_SymbolToken; + /** * NOT SUITABLE FOR PUBLIC USE since it doesn't enforce correctness. */ public class FakeSymbolToken - implements SymbolToken + implements SymbolToken, _Private_SymbolToken { private final String myText; private final int mySid; diff --git a/src/test/java/com/amazon/ion/conformance/Config.kt b/src/test/java/com/amazon/ion/conformance/Config.kt new file mode 100644 index 0000000000..43eee2435b --- /dev/null +++ b/src/test/java/com/amazon/ion/conformance/Config.kt @@ -0,0 +1,20 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package com.amazon.ion.conformance + +import com.amazon.ion.system.IonReaderBuilder +import java.io.File + +/** Top-level configuration for running the conformance tests */ +data class Config( + /** Controls whether debug printing should be turned on. */ + val debugEnabled: Boolean = true, + /** If a NotImplementedError is encountered, should we fail the test or ignore it. */ + val failUnimplemented: Boolean = false, + /** Use for a skip list, or for running only one or two tests. Return true to run the test. */ + val testFilter: (File, String) -> Boolean = { _, _ -> true }, + /** Named set of reader builders (i.e. different reader configurations) to use for all tests. */ + val readerBuilders: Map, +) { + fun newCaseBuilder(file: File) = ConformanceTestBuilder(this, file) +} diff --git a/src/test/java/com/amazon/ion/conformance/ConformanceTestBuilder.kt b/src/test/java/com/amazon/ion/conformance/ConformanceTestBuilder.kt new file mode 100644 index 0000000000..0e29a8dac0 --- /dev/null +++ b/src/test/java/com/amazon/ion/conformance/ConformanceTestBuilder.kt @@ -0,0 +1,105 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package com.amazon.ion.conformance + +import com.amazon.ion.* +import com.amazon.ion.system.* +import com.amazon.ionelement.api.IonElement +import com.amazon.ionelement.api.SeqElement +import com.amazon.ionelement.api.location +import java.io.File +import org.junit.jupiter.api.DynamicContainer.dynamicContainer +import org.junit.jupiter.api.DynamicNode +import org.junit.jupiter.api.DynamicTest.dynamicTest +import org.opentest4j.AssertionFailedError +import org.opentest4j.TestAbortedException + +data class ConformanceTestBuilder( + val config: Config, + /** File that the cases are bring created from. */ + val file: File, + // Internal fields for building up state from which to create a test case + private val nameParts: List = listOf(), + private val fragments: List = listOf(), +) { + + /** + * Helper class that provides runtime support to the test cases. + */ + class TestCaseSupport(private val testBuilder: ConformanceTestBuilder, private val readerBuilder: IonReaderBuilder) { + + private val data: ByteArray by lazy { readFragments(testBuilder.fragments) } + + /** Creates a new reader for this test case */ + fun createFragmentReader(): IonReader = readerBuilder.build(data) + + /** Logs a lazily-evaluated message, if debug is enabled. */ + fun debug(message: () -> String) = testBuilder.debug(message) + + /** Throws an exception for a syntax error in the tests */ + fun reportSyntaxError(element: IonElement, details: String? = null): Nothing = + testBuilder.reportSyntaxError(element, details) + + /** Creates a file URI for the given IonElement */ + fun locationOf(element: IonElement) = "file://${testBuilder.file.absolutePath}:${element.metas.location}" + + /** Creates a failure message that includes a file link to [element] */ + fun createFailureMessage(element: IonElement, details: String? = null): String = + "${details ?: "Assertion failed"} at ${locationOf(element)}; $element" + + /** Throws an [AssertionFailedError] to fail a test case */ + fun fail(expectation: IonElement, details: String, t: Throwable? = null): Nothing = + throw AssertionFailedError(createFailureMessage(expectation, details), t) + } + + // Leaf nodes need a full name or else the HTML report is incomprehensible. + private val fullName: String + get() = nameParts.joinToString(" ") + + // TODO: this could be fullName or nameParts.last() + // Both have drawbacks, but it only affects the display of the interior nodes of the test tree + val containerName: String + get() = fullName // nameParts.last() + + /** Prints a debug message, if debug messages are enabled in the config. */ + fun debug(message: () -> String) { + if (config.debugEnabled) println("[TEST: $fullName] ${message()}") + } + + // Copy-on-write setters + fun plusName(name: String): ConformanceTestBuilder = copy(nameParts = nameParts + name) + fun plusFragment(fragment: SeqElement): ConformanceTestBuilder = copy(fragments = fragments + fragment) + fun plusFragments(newFragments: List): ConformanceTestBuilder = copy(fragments = fragments + newFragments) + fun plus(name: String, fragment: SeqElement): ConformanceTestBuilder = copy(nameParts = nameParts + name, fragments = fragments + fragment) + fun plus(name: String, newFragments: List): ConformanceTestBuilder = copy(nameParts = nameParts + name, fragments = fragments + newFragments) + + fun build(executable: TestCaseSupport.() -> Unit): DynamicNode { + return config.readerBuilders.map { (readerName, readerBuilder) -> + val testName = "$fullName using $readerName" + val testCaseSupport = TestCaseSupport(this, readerBuilder) + dynamicTest(testName) { + if (!config.testFilter(file, testName)) throw TestAbortedException(testName) + debug { "Begin Test using $readerName" } + try { + executable(testCaseSupport) + } catch (e: NotImplementedError) { + if (config.failUnimplemented) throw e + debug { "Ignored because ${e.message}" } + throw TestAbortedException("$e") + } + } + }.let { + dynamicContainer(containerName, it) + } + } + + /** Builds a [DynamicNode] container with the correct name */ + fun buildContainer(children: Iterable): DynamicNode = dynamicContainer(containerName, children) + + /** Builds a [DynamicNode] container with the correct name */ + fun buildContainer(vararg children: DynamicNode): DynamicNode = dynamicContainer(containerName, children.toList()) + + /** Signals to the test builder that there is a syntax error */ + fun reportSyntaxError(element: IonElement, details: String? = null): Nothing = + throw ConformanceTestInvalidSyntaxException(file, element, details) +} diff --git a/src/test/java/com/amazon/ion/conformance/ConformanceTestDslInterpreterTest.kt b/src/test/java/com/amazon/ion/conformance/ConformanceTestDslInterpreterTest.kt new file mode 100644 index 0000000000..f75795529f --- /dev/null +++ b/src/test/java/com/amazon/ion/conformance/ConformanceTestDslInterpreterTest.kt @@ -0,0 +1,104 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package com.amazon.ion.conformance + +import com.amazon.ion.system.* +import java.io.File +import kotlin.streams.toList +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.DynamicContainer +import org.junit.jupiter.api.DynamicNode +import org.junit.jupiter.api.DynamicTest +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.MethodSource + +/** Some minimal tests for the DSL interpreter. */ +object ConformanceTestDslInterpreterTest { + + private val CONFIG = Config( + debugEnabled = true, + failUnimplemented = false, + readerBuilders = mapOf("only reader" to IonReaderBuilder.standard()), + ) + + @JvmStatic + fun data(): Iterable> = listOf( + """ + (document "a test using 'produces'" + (produces)) + """ to 1, + """ + (ion_1_0 "a test using 'text'" + (text ''' {a:1, b:2} "two" ''') + (produces {b:2, a:1} "two")) + """ to 1, + """ + (ion_1_0 "a test using 'signals'" + (text ''' {a:1, b:2 "two" ''') + (signals "struct missing closing delimiter")) + """ to 1, + """ + (ion_1_1 "a test that uses binary" + (bytes "6F 6E 60") + (produces false true 0)) + """ to 1, + """ + (ion_1_0 "a test that uses denotes" + (text "${'$'}4") + (denotes (Symbol "name"))) + """ to 1, + """ + (ion_1_0 "a test using 'then'" + (text ''' 1 ''') + (then (text "2") + (produces 1 2))) + """ to 1, + """ + (ion_1_0 "a test using 'then' to create more than one test case" + (text ''' 1 ''') + (then "then 2" + (text "2") + (produces 1 2)) + (then "then 3" + (text "3") + (produces 1 3))) + """ to 2, + """ + (ion_1_0 "a test using 'each' to create more than one test case" + (text " 1 ") + (each "unclosed container" + (text " { ") + (text " [ ") + (text " ( ") + (signals "something bad"))) + """ to 3, + """ + (ion_1_x "a test using 'ion_1_x' to create more than one test case" + (text " 1 ") + (produces 1)) + """ to 2, + // TODO: Tests to check the demangling behavior, use different types of fragments + ) + + @MethodSource("data") + @ParameterizedTest + fun interpreterTests(testInput: Pair) { + val (dsl, expectedNumberOfTestCases) = testInput + + val testBuilder = CONFIG.newCaseBuilder(File("fake-file")) + val testCases = testBuilder.readAllTests(ION.newReader(dsl)).flatten() + + // It should have the correct number of test cases + assertEquals(expectedNumberOfTestCases, testCases.size) + // All the test case executables should run without throwing any exceptions (i.e. pass) + testCases.forEach { it.executable.execute() } + } + + private fun DynamicNode.flatten(): List { + return when (this@flatten) { + is DynamicContainer -> children.toList().flatMap { it.flatten() } + is DynamicTest -> listOf(this) + else -> TODO("Unreachable") + } + } +} diff --git a/src/test/java/com/amazon/ion/conformance/ConformanceTestInvalidSyntaxException.kt b/src/test/java/com/amazon/ion/conformance/ConformanceTestInvalidSyntaxException.kt new file mode 100644 index 0000000000..b19fe05376 --- /dev/null +++ b/src/test/java/com/amazon/ion/conformance/ConformanceTestInvalidSyntaxException.kt @@ -0,0 +1,21 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package com.amazon.ion.conformance + +import com.amazon.ionelement.api.IonElement +import com.amazon.ionelement.api.location +import java.io.File + +/** Exception for signalling invalid syntax in the conformance tests. */ +class ConformanceTestInvalidSyntaxException( + file: File, + element: IonElement, + description: String? = null, + cause: Throwable? = null +) : Error(cause) { + override val message: String = """ + Invalid conformance dsl syntax${ description?.let { "; $it" } ?: ""} + - at file://${file.absolutePath}:${element.metas.location} + - invalid clause was: $element + """.trimIndent() +} diff --git a/src/test/java/com/amazon/ion/conformance/ConformanceTestRunner.kt b/src/test/java/com/amazon/ion/conformance/ConformanceTestRunner.kt new file mode 100644 index 0000000000..c6b34890b8 --- /dev/null +++ b/src/test/java/com/amazon/ion/conformance/ConformanceTestRunner.kt @@ -0,0 +1,77 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package com.amazon.ion.conformance + +import com.amazon.ion.system.* +import java.io.File +import org.junit.jupiter.api.DynamicNode +import org.junit.jupiter.api.TestFactory + +object ConformanceTestRunner { + val DEFAULT_READER_BUILDER_CONFIGURATIONS = mapOf( + "default reader" to IonReaderBuilder.standard() + .withCatalog(ION_CONFORMANCE_TEST_CATALOG), + "incremental reader" to IonReaderBuilder.standard() + .withCatalog(ION_CONFORMANCE_TEST_CATALOG) + .withIncrementalReadingEnabled(true), + // TODO: Other reader configurations + ) + + private val DEFAULT_SKIP_FILTER: (File, String) -> Boolean = { file, name -> + when { + // IonElement can't load $0. TODO: Use IonValue for `produces`, I guess. + "$0" in name -> false + // For some reason, $ion_symbol_table::null.struct is not handled as expected + "IST structs are elided from app view" in name -> false + // IonWriter is making it difficult to write invalid data + "If no max_id, lack of exact-match must raise an error «then»" in name -> false + // IonCatalog's "best choice" logic is not spec compliant + // TODO—current test name has a typo. Update to correct spelling once ion-tests is fixed. + "When max_id is valid, pad/truncade mismatched or absent SSTs" in name -> false + // No support for reading `$ion_encoding` directives yet. + "conformance/ion_encoding/" in file.absolutePath -> false + // TODO: Seems like the Ion writer is removing "extra" fields from user-supplied symbol tables. + file.endsWith("local_symtab_imports.ion") -> when { + // If you inspect the debug output, the serialized data does not include the repeated fields. + // This implies that the writer is attempting to clean a user-supplied symbol table. + "Repeated fields" in name -> false + // Unfortunately, the writer also seems to remove "imports" field if and only if + "Importing the current symbol table" in name -> false + // For these tests, the writer is validating the max_id field, and failing before + // we have a chance to test the reader. + "If no max_id, lack of exact-match must raise an error" in name -> false + "If max_id not non-negative int, lack of exact-match must raise an error" in name -> false + else -> true + } + // Some of these are failing because + // - Ion Java doesn't support the Ion 1.1 system symbol table yet + // - The tokens `$ion_1_0` and `'$ion_1_0'` are never user values. + // TODO: Add test names once they are added to this file + file.endsWith("system_symbols.ion") -> false + // $ion_literal not supported yet + file.endsWith("ion_literal.ion") -> false + else -> true + } + } + + private val CONFIG = Config( + debugEnabled = true, + failUnimplemented = false, + readerBuilders = DEFAULT_READER_BUILDER_CONFIGURATIONS, + testFilter = DEFAULT_SKIP_FILTER, + ) + + @TestFactory + fun `Conformance Tests`(): Iterable { + return ION_CONFORMANCE_DIR.walk() + .filter { it.isFile && it.extension == "ion" } + .map { file -> + with(CONFIG.newCaseBuilder(file)) { + file.inputStream() + .let(ION::newReader) + .use { reader -> readAllTests(reader) } + } + } + .asIterable() + } +} diff --git a/src/test/java/com/amazon/ion/conformance/expectations.kt b/src/test/java/com/amazon/ion/conformance/expectations.kt new file mode 100644 index 0000000000..1581e6c769 --- /dev/null +++ b/src/test/java/com/amazon/ion/conformance/expectations.kt @@ -0,0 +1,249 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package com.amazon.ion.conformance + +import com.amazon.ion.* +import com.amazon.ion.conformance.ConformanceTestBuilder.* +import com.amazon.ionelement.api.AnyElement +import com.amazon.ionelement.api.BoolElement +import com.amazon.ionelement.api.IntElement +import com.amazon.ionelement.api.IntElementSize +import com.amazon.ionelement.api.SeqElement +import com.amazon.ionelement.api.SexpElement +import com.amazon.ionelement.api.StringElement +import com.amazon.ionelement.api.TextElement +import kotlin.streams.toList +import org.junit.jupiter.api.Assertions.assertArrayEquals +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertNotEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertTrue + +/** + * Asserts that fully traversing the reader will result in an [IonException]. + * It's expected to be for the reason given in [sexp], but we don't have a way + * to check that right now because the `signals` message is non-normative. + */ +fun TestCaseSupport.assertSignals(sexp: SeqElement, r: IonReader) { + val signalDescription = sexp.tail.single().textValue + // The usual `assertThrows` doesn't give us the ability to add our own context to the failure message. + val events = try { + // Just walk the reader without materializing so that we can ensure that the error is raised + // specifically by the reader. + r.walk() + } catch (e: IonException) { + debug { "Expected an IonException because '$signalDescription'; found $e" } + // Test case passes + return + } catch (t: Throwable) { + fail(sexp, "Expected an IonException because '$signalDescription' but was ${t::class.simpleName}", t) + } + fail( + sexp, + "Expected an IonException because '$signalDescription'; " + + "successfully read: ${events.joinToString("\n")}" + ) +} + +/** + * Walks all data available from an IonReader. Records all data as a stream of events so that + * if an error is _not_ encountered, we have some useful information for debugging the test failure. + */ +private fun IonReader.walk(): List { + val events = mutableListOf("START") + fun recordEvent(eventType: String = type.toString(), value: Any? = "") { + events.add("[$eventType] $value") + } + + while (true) { + next() + val currentType = type + ?: try { + stepOut() + recordEvent("STEP-OUT") + continue + } catch (e: IllegalStateException) { + recordEvent("END") + return events + } + + if (isInStruct) recordEvent("FIELD-NAME", fieldNameSymbol) + typeAnnotationSymbols.forEach { recordEvent("ANNOTATION", it) } + + if (isNullValue) { + recordEvent("NULL", currentType) + } else when (currentType) { + IonType.BOOL -> recordEvent(value = booleanValue()) + IonType.INT -> recordEvent(value = bigIntegerValue()) + IonType.FLOAT -> recordEvent(value = doubleValue()) + IonType.DECIMAL -> recordEvent(value = decimalValue()) + IonType.TIMESTAMP -> recordEvent(value = timestampValue()) + IonType.SYMBOL -> recordEvent(value = symbolValue()) + IonType.STRING -> recordEvent(value = stringValue()) + IonType.CLOB, + IonType.BLOB -> recordEvent(value = newBytes()) + IonType.LIST, + IonType.SEXP, + IonType.STRUCT -> { + recordEvent("STEP-IN", type) + stepIn() + } + IonType.NULL, + IonType.DATAGRAM -> TODO("Unreachable") + } + } +} + +/** + * Entry point into `denotes` evaluation. Asserts that each top-level value on the reader + * matches its respective model-value, and that there are no extra, unexpected values. + * + * See https://github.com/amazon-ion/ion-tests/tree/master/conformance#modeling-outputs + */ +fun TestCaseSupport.assertDenotes(modelValues: List, reader: IonReader) { + modelValues.forEach { + reader.next() + denotesModelValue(it, reader) + } + // Assert no more elements in sequence + assertNull(reader.next(), "unexpected extra element(s) at end of stream") +} + +/** + * Assert that the data at the reader's current position matches a particular Ion value. + */ +private fun TestCaseSupport.denotesModelValue(expectation: AnyElement, reader: IonReader) { + if (reader.type == null) fail(expectation, "no more values; expected $expectation") + if (expectation is SexpElement && expectation.head == "annot") { + val actualAnnotations = reader.typeAnnotationSymbols + expectation.tailFrom(2) + .forEachIndexed { i, it -> denotesSymtok(it, actualAnnotations[i]) } + denotesModelContent(expectation.tail.first(), reader) + } else { + assertEquals(SymbolToken.EMPTY_ARRAY, reader.typeAnnotationSymbols, createFailureMessage(expectation, "expected no annotations")) + denotesModelContent(expectation, reader) + } +} + +private fun TestCaseSupport.denotesModelContent(modelContent: AnyElement, reader: IonReader) { + when (modelContent) { + is IntElement -> denotesInt(modelContent, reader) + is BoolElement -> denotesBool(modelContent, reader) + is StringElement -> { + val failureContext = createFailureMessage(modelContent) + assertEquals(IonType.STRING, reader.type, failureContext) + assertEquals(modelContent.stringValue, reader.stringValue(), failureContext) + } + is SexpElement -> when (modelContent.head) { + "Null" -> denotesNull(modelContent, reader) + "Bool" -> denotesBool(modelContent, reader) + "Int" -> denotesInt(modelContent, reader) + "Float" -> TODO("denotes float") + "Decimal" -> TODO("denotes decimal") + "Timestamp" -> TODO("denotes timestamp") + "Symbol" -> denotesSymtok(modelContent.tail.single(), reader.symbolValue()) + "String" -> denotesCodepoints(modelContent, reader.stringValue()) + "Blob" -> denotesLob(IonType.BLOB, modelContent, reader) + "Clob" -> denotesLob(IonType.CLOB, modelContent, reader) + "List" -> denotesSeq(IonType.LIST, modelContent, reader) + "Sexp" -> denotesSeq(IonType.SEXP, modelContent, reader) + "Struct" -> TODO("denotes struct") + else -> reportSyntaxError(modelContent, "model-content") + } + else -> reportSyntaxError(modelContent, "model-content") + } +} + +private fun TestCaseSupport.denotesNull(expectation: SeqElement, reader: IonReader) { + val expectedType = expectation.tail.single().textValue.uppercase().let(IonType::valueOf) + val actualType = reader.next() + assertTrue(reader.isNullValue, createFailureMessage(expectation)) + assertEquals(expectedType, actualType) +} + +private fun TestCaseSupport.denotesBool(modelBoolean: AnyElement, reader: IonReader) { + val expected = when (modelBoolean) { + is BoolElement -> modelBoolean.booleanValue + is SexpElement -> modelBoolean.tail.single().booleanValue + else -> reportSyntaxError(modelBoolean, "model-boolean") + } + assertEquals(IonType.BOOL, reader.type, createFailureMessage(modelBoolean)) + assertEquals(expected, reader.booleanValue(), createFailureMessage(modelBoolean)) +} + +private fun TestCaseSupport.denotesInt(expectation: AnyElement, reader: IonReader) { + val expectedValue = when (expectation) { + is SexpElement -> expectation.tail.single().asInt() + is IntElement -> expectation + else -> reportSyntaxError(expectation, "model-integer") + } + assertEquals(IonType.INT, reader.type, createFailureMessage(expectation)) + assertFalse(reader.isNullValue, createFailureMessage(expectation)) + when (expectedValue.integerSize) { + IntElementSize.LONG -> { + assertNotEquals(IntegerSize.BIG_INTEGER, reader.integerSize, createFailureMessage(expectation)) + assertEquals(expectedValue.longValue, reader.longValue(), createFailureMessage(expectation)) + } + IntElementSize.BIG_INTEGER -> { + assertEquals(IntegerSize.BIG_INTEGER, reader.integerSize, createFailureMessage(expectation)) + } + } + assertEquals(expectedValue.bigIntegerValue, reader.bigIntegerValue(), createFailureMessage(expectation)) +} + +private fun TestCaseSupport.denotesSeq(type: IonType, expectation: SeqElement, reader: IonReader) { + assertFalse(reader.isNullValue, createFailureMessage(expectation)) + assertEquals(type, reader.type, createFailureMessage(expectation)) + reader.stepIn() + expectation.tail.forEach { + reader.next() + denotesModelValue(it, reader) + } + // Assert no more elements in sequence + assertNull(reader.next(), "unexpected extra element(s) at end of $type") + reader.stepOut() +} + +private fun TestCaseSupport.denotesSymtok(expectation: AnyElement, actual: SymbolToken) { + when (expectation) { + is TextElement -> assertEquals(expectation.textValue, actual.text, createFailureMessage(expectation)) + is IntElement -> assertEquals(expectation.longValue, actual.sid, createFailureMessage(expectation)) + is SeqElement -> when (expectation.head) { + "absent" -> { + if (actual.text != null) fail(expectation, "Expected unknown text; was '${actual.text}'") + // TODO: Calculate offset, Symtab name? + } + "text" -> + actual.text + ?.let { denotesCodepoints(expectation, it) } + ?: fail(expectation, "Expected known text; none present in $actual") + else -> reportSyntaxError(expectation, "model-symtok") + } + } +} + +private fun TestCaseSupport.denotesCodepoints(expectation: SeqElement, actual: String) { + val expectedCodePoints = expectation.tail.map { it.longValue } + val actualCodePoints = actual.codePoints().toList() + assertEquals(expectedCodePoints, actualCodePoints, createFailureMessage(expectation)) +} + +private fun TestCaseSupport.denotesLob(type: IonType, expectation: SeqElement, reader: IonReader) { + val expectedBytes = readBytes(expectation) + assertEquals(type, reader.type, createFailureMessage(expectation)) + assertEquals(expectedBytes.size, reader.byteSize(), createFailureMessage(expectation)) + + // bufferSize is intentionally small but >1 so that we can test reading chunks of a lob. + val bufferSize = 3 + val buffer = ByteArray(bufferSize) + expectedBytes.toList().chunked(bufferSize).forEachIndexed { i, chunk -> + val bytesRead = reader.getBytes(buffer, i * 3, 3) + if (bytesRead == bufferSize) { + assertArrayEquals(chunk.toByteArray(), buffer, createFailureMessage(expectation)) + } else { + chunk.forEachIndexed { j, byte -> assertEquals(byte, buffer[j], createFailureMessage(expectation)) } + } + } + assertArrayEquals(expectedBytes, reader.newBytes(), createFailureMessage(expectation)) +} diff --git a/src/test/java/com/amazon/ion/conformance/fragments.kt b/src/test/java/com/amazon/ion/conformance/fragments.kt new file mode 100644 index 0000000000..fb60007657 --- /dev/null +++ b/src/test/java/com/amazon/ion/conformance/fragments.kt @@ -0,0 +1,276 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package com.amazon.ion.conformance + +import com.amazon.ion.* +import com.amazon.ion.IonEncodingVersion.* +import com.amazon.ion.TestUtils.* +import com.amazon.ion.conformance.ConformanceTestBuilder.* +import com.amazon.ion.conformance.Encoding.* +import com.amazon.ion.impl.* +import com.amazon.ion.impl.bin.SymbolInliningStrategy +import com.amazon.ion.system.* +import com.amazon.ion.util.* +import com.amazon.ionelement.api.AnyElement +import com.amazon.ionelement.api.ElementType +import com.amazon.ionelement.api.IntElement +import com.amazon.ionelement.api.SeqElement +import com.amazon.ionelement.api.StringElement +import com.amazon.ionelement.api.SymbolElement +import com.amazon.ionelement.api.TextElement +import com.amazon.ionelement.api.ionInt +import com.amazon.ionelement.api.ionSexpOf +import com.amazon.ionelement.api.ionSymbol +import java.io.ByteArrayOutputStream + +/** Helper function for creating ivm fragments for the `ion_1_*` keywords */ +fun ivm(sexp: SeqElement, major: Int, minor: Int): SeqElement { + return ionSexpOf(listOf(ionSymbol("ivm"), ionInt(major.toLong()), ionInt(minor.toLong())), metas = sexp.metas) +} + +// All known fragment keywords +val FRAGMENT_KEYWORDS = setOf("ivm", "text", "bytes", "toplevel", "encoding", "mactab") +// Insert this between every fragment when transcoding to text +val SERIALIZED_TEXT_FRAGMENT_SEPARATOR = "\n".toByteArray(Charsets.UTF_8) + +// TODO: Update these so that they provide raw writers. That will resolve some of the issues +// such as not being able to write system values to the Ion 1.1 managed writer and not +// being able to write invalid imports. +private sealed interface Encoding { + val writerBuilder: IonWriterBuilder + + sealed interface Binary : Encoding + sealed interface Text : Encoding + sealed interface `1,0` : Encoding + sealed interface `1,1` : Encoding + + object Binary10 : Binary, `1,0` { + override val writerBuilder: IonWriterBuilder = + ION_1_0.binaryWriterBuilder().withCatalog(ION_CONFORMANCE_TEST_CATALOG) + } + + object Binary11 : Binary, `1,1` { + override val writerBuilder: IonWriterBuilder = + ION_1_1.binaryWriterBuilder().withCatalog(ION_CONFORMANCE_TEST_CATALOG) as IonWriterBuilder + } + + object Text10 : Text, `1,0` { + override val writerBuilder: IonWriterBuilder = + (ION_1_0.textWriterBuilder() as _Private_IonTextWriterBuilder<*>) + .withInvalidSidsAllowed(true) + .withWriteTopLevelValuesOnNewLines(true) + .withCatalog(ION_CONFORMANCE_TEST_CATALOG) + .withInitialIvmHandling(IonWriterBuilder.InitialIvmHandling.SUPPRESS) + } + + object Text11 : Text, `1,1` { + override val writerBuilder: IonWriterBuilder = + (ION_1_1.textWriterBuilder() as _Private_IonTextWriterBuilder<*>) + .withInvalidSidsAllowed(true) + .withWriteTopLevelValuesOnNewLines(true) + .withInitialIvmHandling(IonWriterBuilder.InitialIvmHandling.SUPPRESS) + .withCatalog(ION_CONFORMANCE_TEST_CATALOG) + .also { + it as _Private_IonTextWriterBuilder_1_1 + it.withSymbolInliningStrategy(SymbolInliningStrategy.ALWAYS_INLINE) + } + } +} + +/** + * If we have an invalid version, don't complain here. Bad fragments should propagate through + * and be detected by the test expectations. + */ +private fun Encoding.getEncodingVersion(minor: Int): Encoding = when (minor) { + 0 -> if (this is Text) Text10 else Binary10 + 1 -> if (this is Text) Text11 else Binary11 + // Unknown version -- just return this. + else -> this +} + +private val Encoding.ivmBytes: ByteArray + get() = when (this) { + Binary10 -> byteArrayOf(0xE0.toByte(), 1, 0, 0xEA.toByte()) + Binary11 -> byteArrayOf(0xE0.toByte(), 1, 1, 0xEA.toByte()) + Text10 -> "\$ion_1_0".toByteArray(Charsets.UTF_8) + Text11 -> "\$ion_1_1".toByteArray(Charsets.UTF_8) + } + +/** + * Read all fragments, transcoding and combining the data into Ion binary or Ion text UTF-8 encoded bytes. + */ +fun TestCaseSupport.readFragments(fragments: List): ByteArray { + debug { "Initializing Input Data..." } + // TODO: Detect versions and switch accordingly. + val encodeToBinary = 0 < fragments.count { + debug { "Inspecting (${it.head} ...) at ${locationOf(it)}" } + it.head == "bytes" + } + + val encoding: Encoding = if (encodeToBinary) Binary10 else Text10 + + fun debugString(i: Int, bytes: ByteArray): String = + with(bytes) { if (encodeToBinary) toPrettyHexString() else toString(Charsets.UTF_8) } + .replaceIndent(" | ") + .let { "Fragment $i\n$it" } + + val serializedFragments = mutableListOf() + + // All documents start as Ion 1.0, but we must explicitly ensure that the IVM is present if + // transcoding fragments to binary. + if (encodeToBinary) serializedFragments.add(byteArrayOf(0xE0.toByte(), 0x01, 0x00, 0xEA.toByte())) + + fragments.foldIndexed(encoding) { i, encodingVersion, fragment -> + val (bytes, continueWithVersion) = readFragment(fragment, encodingVersion) + serializedFragments.add(bytes) + debug { debugString(i, bytes) } + // If it's text, we need to ensure there is whitespace between fragments + if (encodingVersion is Text) serializedFragments.add(SERIALIZED_TEXT_FRAGMENT_SEPARATOR) + continueWithVersion + } + return serializedFragments.joinToByteArray() +} + +/** Reads a single fragment */ +private fun TestCaseSupport.readFragment(fragment: SeqElement, encoding: Encoding): Pair { + return when (fragment.head) { + "ivm" -> readIvmFragment(fragment, encoding) + "text" -> readTextFragment(fragment, encoding) + "bytes" -> readBytesFragment(fragment, encoding) + "toplevel" -> readTopLevelFragment(fragment, encoding) + "mactab" -> TODO("mactab") + "encoding" -> TODO("encoding") + else -> reportSyntaxError(fragment, "not a valid fragment") + } +} + +/** Reads an `IVM` fragment and returns a byte array with an IVM for the given [encoding]. */ +private fun TestCaseSupport.readIvmFragment(fragment: SeqElement, encoding: Encoding): Pair { + require(fragment.values[1].longValue == 1L) + val minor = fragment.values[2].longValue + val newEncodingVersion = encoding.getEncodingVersion(minor.toInt()) + return newEncodingVersion.ivmBytes to newEncodingVersion +} + +/** + * Reads a `text` fragment. Does not transcode, but (to-do) keeps track of whether an IVM is encountered, + * and returns the text as a UTF-8 [ByteArray] along with the current encoding version at the end of the fragment. + */ +private fun TestCaseSupport.readTextFragment(fragment: SeqElement, encoding: Encoding): Pair { + require(encoding is Text) + val text = fragment.tail.joinToString("\n") { + // TODO: Detect and update the encoding if there's an IVM midstream + (it as? StringElement)?.textValue + ?: reportSyntaxError(it, "text fragment may only contain strings") + } + return text.toByteArray(Charsets.UTF_8) to encoding +} + +/** + * Reads a `bytes` fragment. Does not transcode, but (to-do) keeps track of whether an IVM is encountered, + * and returns bytes and the current encoding version at the end of the fragment. + */ +private fun TestCaseSupport.readBytesFragment(fragment: SeqElement, encoding: Encoding): Pair { + require(encoding is Binary) + // TODO: Detect and update the encoding if there's an IVM midstream + return readBytes(fragment) to encoding +} + +/** + * Reads a `bytes` clause, returning a [ByteArray]. + */ +fun TestCaseSupport.readBytes(sexp: SeqElement): ByteArray { + val bytes = mutableListOf() + sexp.tail.forEach { + when (it) { + is StringElement -> hexStringToByteArray(cleanCommentedHexBytes(it.stringValue)) + is IntElement -> byteArrayOf(it.longValue.toByte()) + else -> reportSyntaxError(it, "Not a valid element in a bytes clause") + }.let(bytes::add) + } + return bytes.joinToByteArray() +} + +/** + * Reads a `toplevel` clause, transcoding it to the requested [encoding]. + */ +private fun TestCaseSupport.readTopLevelFragment(fragment: SeqElement, encoding: Encoding): Pair { + val baos = ByteArrayOutputStream() + var currentEncoding = encoding + var currentWriter = encoding.writerBuilder.build(baos) + + fragment.tail.forEach { + // TODO: Check for IVMs and update `currentEncoding` and `currentWriter` accordingly + // Alternately, we could check for IVMs and split into multiple fragments so that + // each fragment can be written separately. + if (it is SymbolElement && it.textValue.matches(Regex("#?\\\$ion_\\d+_\\d+"))) { + TODO("change Ion version while in in toplevel fragment") + } + it.asAnyElement().demangledWriteTo(currentWriter) + } + currentWriter.close() + val bytes = baos.toByteArray() + // Drop the initial IVM + .let { if (encoding is Binary) it.drop(4).toByteArray() else it } + .let { if (encoding is Text11) it.drop("\$ion_1_1".length).toByteArray() else it } + return bytes to currentEncoding +} + +/** + * Writes this [AnyElement] to an [IonWriter], applying the de-mangling logic described at + * [Conformance – Abstract Syntax Forms](https://github.com/amazon-ion/ion-tests/tree/master/conformance#abstract-syntax-forms). + */ +private fun AnyElement.demangledWriteTo(writer: IonWriter) { + writer.setTypeAnnotationSymbols(*annotations.map(::demangleSymbolToken).toTypedArray()) + if (isNull) { + writer.writeNull(type.toIonType()) + } else when (type) { + ElementType.BOOL -> writer.writeBool(booleanValue) + ElementType.INT -> writer.writeInt(bigIntegerValue) + ElementType.FLOAT -> writer.writeFloat(doubleValue) + ElementType.DECIMAL -> writer.writeDecimal(decimalValue) + ElementType.TIMESTAMP -> writer.writeTimestamp(timestampValue) + ElementType.SYMBOL -> writer.writeSymbolToken(demangleSymbolToken(symbolValue)) + ElementType.STRING -> writer.writeString(stringValue) + ElementType.CLOB -> writer.writeClob(bytesValue.copyOfBytes()) + ElementType.BLOB -> writer.writeBlob(bytesValue.copyOfBytes()) + ElementType.LIST -> { + writer.stepIn(IonType.LIST) + listValues.forEach { it.demangledWriteTo(writer) } + writer.stepOut() + } + ElementType.SEXP -> { + if (sexpValues.firstOrNull().let { it is TextElement && it.textValue.startsWith("#$:") }) { + TODO("demangled e-expressions") + } + writer.stepIn(IonType.SEXP) + sexpValues.forEach { it.demangledWriteTo(writer) } + writer.stepOut() + } + ElementType.STRUCT -> { + writer.stepIn(IonType.STRUCT) + structFields.forEach { (k, v) -> + writer.setFieldNameSymbol(demangleSymbolToken(k)) + v.demangledWriteTo(writer) + } + writer.stepOut() + } + ElementType.NULL -> TODO("Unreachable") + } +} + +private fun demangleSymbolToken(text: String): SymbolToken { + return if (text.startsWith("#\$ion_")) { + // Escaped IVM or system symbol + FakeSymbolToken(text.drop(1), -1) + } else if (text.startsWith("#$:")) { + // E-Expression macro id + TODO("demangled e-expressions - $text") + } else if (text.startsWith("#$")) { + // Escaped SID + val id = text.drop(2).toInt() + FakeSymbolToken(null, id) + } else { + FakeSymbolToken(text, -1) + } +} diff --git a/src/test/java/com/amazon/ion/conformance/structure.kt b/src/test/java/com/amazon/ion/conformance/structure.kt new file mode 100644 index 0000000000..dafe0616b5 --- /dev/null +++ b/src/test/java/com/amazon/ion/conformance/structure.kt @@ -0,0 +1,184 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package com.amazon.ion.conformance + +import com.amazon.ion.* +import com.amazon.ionelement.api.AnyElement +import com.amazon.ionelement.api.ElementType +import com.amazon.ionelement.api.SeqElement +import com.amazon.ionelement.api.StringElement +import com.amazon.ionelement.api.loadAllElements +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assumptions.assumeTrue +import org.junit.jupiter.api.DynamicContainer +import org.junit.jupiter.api.DynamicNode +import org.junit.jupiter.api.DynamicTest + +// There are three distinct parts to this DSL +// 1. The structure clauses (document, then, each, etc.) +// 2. The input (fragment) clauses +// 3. The expectation clauses +// +// The structure is eagerly evaluated. The other clauses are lazily evaluated in the actual test cases. + +/** + * Tuple of a [ConformanceTestBuilder], the current s-expression, and the current position in the s-expression. + * + * This is immutable. Branches in the `read` functions require creating a new updated copy of [ParserState]. + */ +private data class ParserState(val builder: ConformanceTestBuilder, val sexp: SeqElement, val pos: Int = 0) + +private fun ParserState.updateState(pos: Int = this.pos, builderUpdate: ConformanceTestBuilder.() -> ConformanceTestBuilder = { this }): ParserState = + copy(pos = pos, builder = builderUpdate(builder)) + +/** + * Entry point to reading the test structure. + */ +fun ConformanceTestBuilder.readAllTests(reader: IonReader): DynamicNode { + return loadAllElements(reader, ELEMENT_LOADER_OPTIONS) + .mapIndexed { i, it -> + try { + readTest(it) + } catch (e: ConformanceTestInvalidSyntaxException) { + // If there's a syntax error in this test tree, we'll create a test case to represent it + // and rethrow the error in there. This will allow other tests to run even if some malformed + // tests exist. + DynamicTest.dynamicTest("$file[$i]") { throw e } + } catch (e: NotImplementedError) { + // Hack to report something useful if we can't read the test case because we + // haven't implemented something yet. This creates a test case that always skips. + DynamicTest.dynamicTest("$file[$i] - ${e.message}") { assumeTrue(false) } + } + } + .let { DynamicContainer.dynamicContainer(file.path, it) } +} + +/** Reads a top-level test clause. */ +fun ConformanceTestBuilder.readTest(element: AnyElement): DynamicNode { + val sexp = element as? SeqElement ?: reportSyntaxError(element, "test-case") + val parserState = ParserState(this, sexp, 1) + + return when (sexp.head) { + "document" -> + parserState.readDescription() + .readFragments { updateState { plusFragments(it) } } + .readContinuation() + + "ion_1_0" -> + parserState.updateState { plusFragment(ivm(sexp, 1, 0)) } + .readDescription() + .readFragments { updateState { plusFragments(it) } } + .readContinuation() + + "ion_1_1" -> + parserState.updateState { plusFragment(ivm(sexp, 1, 1)) } + .readDescription() + .readFragments { updateState { plusFragments(it) } } + .readContinuation() + + "ion_1_x" -> { + parserState.readDescription() + .let { p -> + val ion10Branch = p.updateState { plus("In Ion 1.0", ivm(sexp, 1, 0)) } + val ion11Branch = p.updateState { plus("In Ion 1.1", ivm(sexp, 1, 1)) } + p.builder.buildContainer( + ion10Branch + .readFragments { updateState { plusFragments(it) } } + .readContinuation(), + ion11Branch + .readFragments { updateState { plusFragments(it) } } + .readContinuation(), + ) + } + } + else -> reportSyntaxError(sexp) + } +} + +/** + * Reads 0 or more fragments from an s-expression starting from the position + * given in [ParserState]. Returns a [ParserState] with an updated position + * and a list of any fragment expressions that were found. + */ +private fun ParserState.readFragments(useFragments: ParserState.(List) -> T): T { + val fragments = sexp.tailFrom(pos) + .takeWhile { it is SeqElement && it.head in FRAGMENT_KEYWORDS } as List + return this.updateState(pos = pos + fragments.size).useFragments(fragments) +} + +/** + * Reads an optional description, returning an updated [ParserState]. + * This function always adds _some_ description to the [ParserState]. + * If the clause contains no description, it uses the clause keyword as a description. + */ +private fun ParserState.readDescription(): ParserState { + return sexp.values[pos].let { + // If it's a string (even null), update position + val newPos = pos + if (it.type == ElementType.STRING) 1 else 0 + // If there is no description, or the description is null, use the clause name instead. + val text = (it as? StringElement)?.textValue ?: "«${sexp.head}»" + updateState(newPos) { plusName(text) } + } +} + +/** Reads a `then` clause, starting _after_ the `then` keyword. */ +private fun ParserState.readThen(): List { + return readDescription() + .readFragments { frags -> updateState { plusFragments(frags) } } + .readContinuation() + .let(::listOf) +} + +/** Reads an `each` clause, starting _after_ the `each` keyword. */ +private fun ParserState.readEach(): List { + // TODO: Handle case where 0 fragments + return readDescription() + .readFragments { + it.mapIndexed { i, frag -> + updateState { plus(name = "[$i]", frag) }.readContinuation() + } + } +} + +/** Reads an extension, returning a list of test case nodes constructed from those extensions. */ +private fun ParserState.readExtension(): List { + return when (sexp.head) { + "each" -> updateState(pos = 1).readEach() + "then" -> updateState(pos = 1).readThen() + else -> builder.reportSyntaxError(sexp, "unknown extension") + } +} + +/** Reads a continuation—a single expectation or one-to-many extensions. */ +private fun ParserState.readContinuation(): DynamicNode { + val continuation = sexp.tailFrom(pos) + + val firstExpression = continuation.first() + firstExpression as? SeqElement ?: builder.reportSyntaxError(firstExpression, "continuation") + + return continuation.flatMap { + it as? SeqElement ?: builder.reportSyntaxError(it, "extension") + with(ParserState(builder, it)) { + readExpectation()?.let { expectation -> return expectation } + readExtension() + } + }.let(builder::buildContainer) +} + +/** + * Reads an optional expectation clause. If the current clause is not an expectation, + * returns null. + */ +private fun ParserState.readExpectation(): DynamicNode? { + return when (sexp.head) { + "and" -> TODO("'and' not implemented") + "not" -> TODO("'not' not implemented") + "produces" -> builder.build { + val actual = loadAllElements(createFragmentReader()).toList() + assertEquals(sexp.tail, actual, createFailureMessage(sexp)) + } + "signals" -> builder.build { assertSignals(sexp, createFragmentReader()) } + "denotes" -> builder.build { assertDenotes(sexp.tail, createFragmentReader()) } + else -> null + } +} diff --git a/src/test/java/com/amazon/ion/conformance/util.kt b/src/test/java/com/amazon/ion/conformance/util.kt new file mode 100644 index 0000000000..a59adc9e7e --- /dev/null +++ b/src/test/java/com/amazon/ion/conformance/util.kt @@ -0,0 +1,64 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package com.amazon.ion.conformance + +import com.amazon.ion.* +import com.amazon.ion.system.* +import com.amazon.ionelement.api.AnyElement +import com.amazon.ionelement.api.IonElementLoaderOptions +import com.amazon.ionelement.api.SeqElement +import java.io.File + +val ION: IonSystem = IonSystemBuilder.standard().build() + +val ELEMENT_LOADER_OPTIONS = IonElementLoaderOptions(includeLocationMeta = true) + +val ION_CONFORMANCE_DIR = File("ion-tests/conformance") + +val TEST_CATALOG_DIR = File("ion-tests/catalog") + +/** + * Catalog for conformance tests. + */ +val ION_CONFORMANCE_TEST_CATALOG = SimpleCatalog().apply { + TEST_CATALOG_DIR.walk() + .filter { it.isFile && it.extension == "ion" } + .onEach { println(it.absolutePath) } + .forEach { file -> + file.inputStream() + .let(ION::newReader) + .use { r -> while (r.next() != null) putTable(ION.newSharedSymbolTable(r, true)) } + } +} + +/** + * Gets the first value of a [SeqElement]. + * Throws an exception if the first value is not text. + */ +val SeqElement.head: String + get() = values.first().textValue + +/** + * Gets all elements of a [SeqElement], except for [head]. + */ +val SeqElement.tail: List + get() = tailFrom(1) + +/** + * Gets the tail of a [SeqElement], starting with position [i]. + */ +fun SeqElement.tailFrom(i: Int) = values.subList(i, size) + +/** + * Join a list of [ByteArray] into a single [ByteArray] + */ +fun List.joinToByteArray(): ByteArray { + val size = sumOf { it.size } + var offset = 0 + val combined = ByteArray(size) + forEach { + it.copyInto(combined, offset) + offset += it.size + } + return combined +} diff --git a/src/test/java/com/amazon/ion/util/formatting.kt b/src/test/java/com/amazon/ion/util/formatting.kt new file mode 100644 index 0000000000..7a5df69ef1 --- /dev/null +++ b/src/test/java/com/amazon/ion/util/formatting.kt @@ -0,0 +1,12 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +@file:JvmName("Formatting") +package com.amazon.ion.util + +@OptIn(ExperimentalStdlibApi::class) +fun ByteArray.toPrettyHexString(bytesPerWord: Int = 4, wordsPerLine: Int = 8): String { + return map { it.toHexString(HexFormat.UpperCase) } + .windowed(bytesPerWord, bytesPerWord, partialWindows = true) + .windowed(wordsPerLine, wordsPerLine, partialWindows = true) + .joinToString("\n") { it.joinToString(" ") { it.joinToString(" ") } } +}