Skip to content

Commit

Permalink
Adds Ion Schema 2.0 support for imports
Browse files Browse the repository at this point in the history
  • Loading branch information
@popematt
popematt committed Nov 3, 2022
1 parent 87677cf commit b55e380
Show file tree
Hide file tree
Showing 7 changed files with 442 additions and 78 deletions.
2 changes: 1 addition & 1 deletion ion-schema-tests
Submodule ion-schema-tests updated from 0add43 to 8a32c9
124 changes: 63 additions & 61 deletions ion-schema/src/main/kotlin/com/amazon/ionschema/internal/SchemaImpl_2_0.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,12 @@ import com.amazon.ionschema.Type
import com.amazon.ionschema.Violations
import com.amazon.ionschema.internal.util.IonSchema_2_0
import com.amazon.ionschema.internal.util.getFields
import com.amazon.ionschema.internal.util.getIslOptionalField
import com.amazon.ionschema.internal.util.getIslRequiredField
import com.amazon.ionschema.internal.util.islRequire
import com.amazon.ionschema.internal.util.islRequireElementType
import com.amazon.ionschema.internal.util.islRequireIonTypeNotNull
import com.amazon.ionschema.internal.util.islRequireOnlyExpectedFieldNames
import com.amazon.ionschema.internal.util.islRequireZeroOrOneElements
import com.amazon.ionschema.internal.util.markReadOnly
import kotlin.contracts.ExperimentalContracts
Expand All @@ -47,12 +50,12 @@ internal class SchemaImpl_2_0 private constructor(
override val schemaId: String?,
preloadedImports: Map<String, Import>,
preloadedUserReservedFields: UserReservedFields,
/*
* [types] is declared as a MutableMap in order to be populated DURING
* INITIALIZATION ONLY. This enables type B to find its already-loaded
* dependency type A. After initialization, [types] is expected to
* be treated as immutable as required by the Schema interface.
*/
/*
* [types] is declared as a MutableMap in order to be populated DURING
* INITIALIZATION ONLY. This enables type B to find its already-loaded
* dependency type A. After initialization, [types] is expected to
* be treated as immutable as required by the Schema interface.
*/
private val types: MutableMap<String, Type>,
) : SchemaImpl {

Expand Down Expand Up @@ -131,9 +134,7 @@ internal class SchemaImpl_2_0 private constructor(
}
isType(it) -> {
islRequire(!foundFooter) { "Types may not occur after the schema footer." }
islRequire(it.count { it.fieldName == "name" } == 1) { "Top-level types must have exactly one name." }
val name = islRequireIonTypeNotNull<IonSymbol>(it["name"]) { "Type names must be a non-null symbol." }
islRequire(name.typeAnnotations.isEmpty()) { "Type names must not have annotations." }
it.getIslRequiredField<IonSymbol>("name")
val newType = TypeImpl(it, this)
islRequire(newType.name !in types.keys) { "Invalid duplicate type name: '${newType.name}'" }
addType(types, newType)
Expand Down Expand Up @@ -242,17 +243,11 @@ internal class SchemaImpl_2_0 private constructor(
* Gets the list of field names that the user would like to reserve for a particular Ion Schema structure.
*/
private fun loadUserReservedFieldsSubfield(userContent: IonStruct, fieldName: String): Set<String> {
return userContent.getFields(fieldName)
.islRequireZeroOrOneElements { "'$fieldName' field may only appear 0 or 1 times in the 'user_reserved_fields' struct" }
?.let { list ->
islRequireIonTypeNotNull<IonList>(list) { "'$fieldName' field in user_reserved_fields struct must be a non-null Ion list" }
islRequire(list.typeAnnotations.isEmpty()) { "'$fieldName' list in user_reserved_fields struct may not have any annotations" }
islRequireElementType<IonSymbol>(list) { "'$fieldName' list in user_reserved_fields struct must only contain non-null Ion symbols" }
.onEach { islRequire(it.typeAnnotations.isEmpty()) { "symbols in the user_reserved_fields '$fieldName' list may not have any annotations" } }
.map { it.stringValue() }
.onEach { islRequire(it !in IonSchema_2_0.KEYWORDS) { "Ion Schema 2.0 keyword '$it' may not be declared as a user reserved field: $userContent" } }
.toSet()
}
return userContent.getIslOptionalField<IonList>(fieldName)
?.islRequireElementType<IonSymbol>("list of user reserved symbols for $fieldName")
?.map { it.stringValue() }
?.onEach { islRequire(it !in IonSchema_2_0.KEYWORDS) { "Ion Schema 2.0 keyword '$it' may not be declared as a user reserved field: $userContent" } }
?.toSet()
?: emptySet()
}

Expand Down Expand Up @@ -290,55 +285,62 @@ internal class SchemaImpl_2_0 private constructor(
val importsMap = mutableMapOf<String, SchemaAndTypeImports>()
val importSet: MutableSet<String> = schemaSystem.getSchemaImportSet()

(header.get("imports") as? IonList)
?.filterIsInstance<IonStruct>()
?.forEach {
val childImportId = it["id"] as IonText
val alias = it["as"] as? IonSymbol
// if importSet has an import with this id then do not load schema again to break the cycle.
if (!importSet.contains(childImportId.stringValue())) {
var parentImportId = schemaId ?: ""

// if Schema is importing itself then throw error
if (parentImportId.equals(childImportId.stringValue())) {
throw InvalidSchemaException("Schema can not import itself.")
}
val imports = header.getIslOptionalField<IonList>("imports")
?.islRequireElementType<IonStruct>(containerDescription = "imports list")
// If there's no imports field, then there's nothing to do
?: return emptyMap()

// add parent and current schema to importSet and continue loading current schema
importSet.add(parentImportId)
importSet.add(childImportId.stringValue())
val importedSchema = schemaSystem.loadSchema(childImportId.stringValue())
importSet.remove(childImportId.stringValue())
importSet.remove(parentImportId)
imports.forEach {
it.islRequireOnlyExpectedFieldNames(IonSchema_2_0.IMPORT_KEYWORDS)

val schemaAndTypes = importsMap.getOrPut(childImportId.stringValue()) {
SchemaAndTypeImports(childImportId.stringValue(), importedSchema)
}
val idField = it.getIslRequiredField<IonText>("id")
val typeField = it.getIslOptionalField<IonSymbol>("type")
val asField = it.getIslOptionalField<IonSymbol>("as")

val typeName = (it["type"] as? IonSymbol)?.stringValue()
if (typeName != null) {
var importedType = importedSchema.getDeclaredType(typeName)
?.toImportedType(childImportId.stringValue())
typeField ?: islRequire(asField == null) { "'as' only allowed when 'type' is present: $it" }

importedType ?: throw InvalidSchemaException("Schema $childImportId doesn't contain a type named '$typeName'")
val importedSchemaId = idField.stringValue()
// if Schema is importing itself then throw error
if (schemaId == importedSchemaId) {
throw InvalidSchemaException("Schema can not import itself: $it")
}
// if importSet has an import with this id then do not load schema again to break the cycle.
if (!importSet.contains(importedSchemaId)) {

if (alias != null) {
importedType = TypeAliased(alias, importedType)
}
addType(typeMap, importedType)
schemaAndTypes.addType(alias?.stringValue() ?: typeName, importedType)
} else {
val typesToAdd = importedSchema.getDeclaredTypes()

typesToAdd.asSequence()
.map { type -> type.toImportedType(childImportId.stringValue()) }
.forEach { type ->
addType(typeMap, type)
schemaAndTypes.addType(type.name, type)
}
// add current schema to importSet and continue loading current schema
importSet.add(importedSchemaId)
val importedSchema = runCatching { schemaSystem.loadSchema(importedSchemaId) }
.getOrElse { e -> throw InvalidSchemaException("Unable to load schema '$importedSchemaId'; ${e.message}") }
importSet.remove(importedSchemaId)

val schemaAndTypes = importsMap.getOrPut(importedSchemaId) {
SchemaAndTypeImports(importedSchemaId, importedSchema)
}

val typeName = typeField?.stringValue()
if (typeName != null) {
var importedType = importedSchema.getDeclaredType(typeName)
?.toImportedType(importedSchemaId)

importedType ?: throw InvalidSchemaException("Schema $importedSchemaId doesn't contain a type named '$typeName'")

if (asField != null) {
importedType = TypeAliased(asField, importedType)
}
addType(typeMap, importedType)
schemaAndTypes.addType(asField?.stringValue() ?: typeName, importedType)
} else {
val typesToAdd = importedSchema.getDeclaredTypes()

typesToAdd.asSequence()
.map { type -> type.toImportedType(importedSchemaId) }
.forEach { type ->
addType(typeMap, type)
schemaAndTypes.addType(type.name, type)
}
}
}
}
return importsMap.mapValues {
ImportImpl(it.value.id, it.value.schema, it.value.types)
}
Expand Down
18 changes: 14 additions & 4 deletions ion-schema/src/main/kotlin/com/amazon/ionschema/internal/TypeReference.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,11 @@ import com.amazon.ionschema.InvalidSchemaException
import com.amazon.ionschema.IonSchemaVersion
import com.amazon.ionschema.Schema
import com.amazon.ionschema.Violations
import com.amazon.ionschema.internal.util.IonSchema_2_0
import com.amazon.ionschema.internal.util.getIslOptionalField
import com.amazon.ionschema.internal.util.getIslRequiredField
import com.amazon.ionschema.internal.util.islRequire
import com.amazon.ionschema.internal.util.islRequireOnlyExpectedFieldNames
import com.amazon.ionschema.internal.util.markReadOnly

/**
Expand Down Expand Up @@ -77,13 +81,19 @@ internal class TypeReference private constructor() {
}

private fun handleStruct(ion: IonStruct, schema: Schema, isField: Boolean): () -> TypeInternal {
val id = ion["id"] as? IonText
val id = ion.getIslOptionalField<IonText>("id")
val type = when {
id != null -> {
// import
val newSchema = schema.getSchemaSystem().loadSchema(id.stringValue())
val typeName = ion.get("type") as IonSymbol
newSchema.getType(typeName.stringValue()) as? TypeInternal
if (schema.ionSchemaLanguageVersion >= IonSchemaVersion.v2_0) {
ion.islRequireOnlyExpectedFieldNames(IonSchema_2_0.INLINE_IMPORT_KEYWORDS)
val thisSchemaId = (schema as? SchemaImpl)?.schemaId
islRequire(id.stringValue() != thisSchemaId) { "A schema may not directly import itself: $ion" }
}
val typeName = ion.getIslRequiredField<IonSymbol>("type")
val importedSchema = runCatching { schema.getSchemaSystem().loadSchema(id.stringValue()) }
.getOrElse { e -> throw InvalidSchemaException("Unable to load schema '${id.stringValue()}'; ${e.message}") }
importedSchema.getType(typeName.stringValue()) as? TypeInternal
}
isField -> TypeImpl(ion, schema)
ion.size() == 1 && ion["type"] != null -> {
Expand Down
5 changes: 5 additions & 0 deletions ion-schema/src/main/kotlin/com/amazon/ionschema/internal/util/IonSchema_2_0.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,11 @@ internal object IonSchema_2_0 {
*/
val IMPORT_KEYWORDS = setOf("id", "type", "as")

/**
* Keywords that are valid in an inline import.
*/
val INLINE_IMPORT_KEYWORDS = setOf("id", "type")

/**
* Keywords that are valid as annotations on top-level types.
*/
Expand Down
134 changes: 127 additions & 7 deletions ion-schema/src/main/kotlin/com/amazon/ionschema/internal/util/Preconditions.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@

package com.amazon.ionschema.internal.util

import com.amazon.ion.IonContainer
import com.amazon.ion.IonStruct
import com.amazon.ion.IonValue
import com.amazon.ionschema.InvalidSchemaException
import kotlin.contracts.ExperimentalContracts
Expand Down Expand Up @@ -83,12 +83,132 @@ internal inline fun <T : Any> Collection<T>.islRequireZeroOrOneElements(lazyMess
}

/**
* Validates that the given [IonContainer] has homogeneous elements of Ion type [T].
* Validates that all elements of an [Iterable] are of Ion type [T].
* If [allowAnnotations] is false, validates that all elements have no annotations.
* If [allowIonNulls] is false, validates that all elements are not an Ion null value.
*
* @throws InvalidSchemaException if this [IonContainer] does not meet the above condition.
* @return the elements of [container] as `List<T>`
* @throws InvalidSchemaException if any element in the collection does not meet the above conditions.
* @return the elements of [this] as `Iterable<T>`
*/
internal inline fun <reified T : IonValue> islRequireElementType(container: IonContainer, allowNulls: Boolean = false, lazyMessage: () -> Any): List<T> {
islRequire(container.all { it is T && (allowNulls || !it.isNullValue) }, lazyMessage)
return container.filterIsInstance<T>()
internal inline fun <reified T : IonValue> Iterable<IonValue>.islRequireElementType(
containerDescription: String,
allowIonNulls: Boolean = false,
allowAnnotations: Boolean = false,
): Iterable<T> {
this.onEach {
if (allowIonNulls) {
islRequire(it is T) { "$containerDescription elements must be a ${ionTypeDescription<T>()}: $this" }
} else {
islRequireIonTypeNotNull<T>(it) { "$containerDescription elements must be a non-null ${ionTypeDescription<T>()}: $this" }
}
if (!allowAnnotations) {
islRequire(it.typeAnnotations.isEmpty()) { "$containerDescription elements may not have annotations: $this" }
}
}
@Suppress("UNCHECKED_CAST")
return this as Iterable<T>
}

/**
* Validates that all elements of a [List] are of Ion type [T].
* If [allowAnnotations] is false, validates that all elements have no annotations.
* If [allowIonNulls] is false, validates that all elements are not an Ion null value.
*
* @throws InvalidSchemaException if any element in the collection does not meet the above conditions.
* @return the elements of [this] as `List<T>`
*/
internal inline fun <reified T : IonValue> List<IonValue>.islRequireElementType(
containerDescription: String,
allowIonNulls: Boolean = false,
allowAnnotations: Boolean = false,
): List<T> = (this as Iterable<IonValue>).islRequireElementType<T>(
containerDescription,
allowIonNulls,
allowAnnotations,
) as List<T>

/**
* Gets a required field from an IonStruct, validating for type, nullability, and presence of annotations.
*
* Validates that a given field name is present in the struct exactly once, and that the corresponding field value is
* of type [T].
* If [allowAnnotations] is false, validates that the field value has no annotations.
* If [allowIonNulls] is false, validates that the field value is not an Ion null value.
*
* @throws InvalidSchemaException if this [IonStruct] does not contain exactly one element with the given field name
* or if the element does not meet the required conditions.
* @return the value for the given field name
*/
internal inline fun <reified T : IonValue> IonStruct.getIslRequiredField(
fieldName: String,
allowIonNulls: Boolean = false,
allowAnnotations: Boolean = false,
): T {
val theValue = getIslOptionalField<T>(fieldName, allowIonNulls, allowAnnotations)
return islRequireNotNull(theValue) { "missing required field '$fieldName': $this" }
}

/**
* Gets an optional field from an IonStruct, validating for type, nullability, and presence of annotations.
*
* Validates that a given field name is present in the struct zero or one times, and that the corresponding field value
* (if it exists) is of type [T].
* If [allowAnnotations] is false, validates that the field value has no annotations.
* If [allowIonNulls] is false, validates that the field value is not an Ion null value.
*
* If the field name does not occur in the struct, returns (Kotlin) `null`.
*
* @throws InvalidSchemaException if this [IonStruct] does not contain zero or one element with the given field name
* or if the element does not meet the required conditions.
* @return the value for the given field name or null if no value has the given field name
*/
internal inline fun <reified T : IonValue> IonStruct.getIslOptionalField(
fieldName: String,
allowIonNulls: Boolean = false,
allowAnnotations: Boolean = false,
): T? {
if (!this.containsKey(fieldName)) return null
val theValue = this.getFields(fieldName)
.also { islRequire(it.size == 1) { "field '$fieldName' may not occur more than once: $this" } }
.single()
if (allowIonNulls) {
islRequire(theValue is T) { "field must be a ${ionTypeDescription<T>()}; '$fieldName': $theValue" }
} else {
islRequireIonTypeNotNull<T>(theValue) { "field must be a non-null ${ionTypeDescription<T>()}; '$fieldName': $theValue" }
}
if (!allowAnnotations) {
islRequire(theValue.typeAnnotations.isEmpty()) { "field may not have annotations; '$fieldName': $theValue" }
}
return theValue
}

/**
* Converts an interface in the [IonValue] hierarchy into a prose-friendly name.
*/
private inline fun <reified T : IonValue> ionTypeDescription(): String {
return when (val name = T::class.simpleName?.removePrefix("Ion")?.toLowerCase()) {
null -> TODO("Unreachable")
"number" -> "int, float, or decimal"
"text" -> "string or symbol"
"sequence" -> "list or sexp"
"container" -> "struct, list, or sexp"
"lob" -> "blob or clob"
else -> name
}
}

/**
* Validates that an [IonStruct] contains no unexpected fields.
* @return [this]
* @throws InvalidSchemaException if any fields have a name not in [expectedFieldNames].
*/
internal fun IonStruct.islRequireOnlyExpectedFieldNames(
expectedFieldNames: Collection<String>,
): IonStruct {
val unknownFields = this.filterNot { it.fieldName in expectedFieldNames }
if (unknownFields.isNotEmpty()) {
throw InvalidSchemaException("Unknown fields ${unknownFields.map { it.fieldName }} in struct: $this")
} else {
return this
}
}
5 changes: 2 additions & 3 deletions ion-schema/src/test/kotlin/com/amazon/ionschema/IonSchemaTestsRunner.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,8 @@ class IonSchemaTests_1_0 : TestFactory by IonSchemaTestsRunner(v1_0)
class IonSchemaTests_2_0 : TestFactory by IonSchemaTestsRunner(
islVersion = v2_0,
additionalFileFilter = {
it.path.contains("ion_schema_2_0/schema/") ||
it.path.contains("ion_schema_2_0/constraints/") ||
it.path.contains("ion_schema_2_0/open_content/")
// Pending fix for https://github.com/amzn/ion-schema-kotlin/issues/209
!it.path.contains("cycles/")
}
)

Expand Down
Loading

0 comments on commit b55e380

Please sign in to comment.