Deprecates absent types and removes associated planner logic

CHANGELOG.md

@@ -28,8 +28,25 @@ Thank you to all who have contributed!
 ### Added
 
 ### Changed
+- **Behavioral change**: The planner now does NOT support the NullType and MissingType variants of StaticType. The logic
+is that the null and missing values are part of *all* data types. Therefore, one must assume that the types returned by
+the planner allow for NULL and MISSING values. Similarly, the testFixtures Ion-encoded test resources
+representing the catalog do not use "null" or "missing".
 
 ### Deprecated
+- We have deprecated `org.partiql.type.NullType` and `org.partiql.type.MissingType`. Please see the corresponding
+information in the "Changed" section. In relation to the deprecation of the above, the following APIs have also
+been deprecated:
+  - `org.partiql.type.StaticType.MISSING`
+  - `org.partiql.type.StaticType.NULL`
+  - `org.partiql.type.StaticType.NULL_OR_MISSING`
+  - `org.partiql.type.StaticType.asNullable()`
+  - `org.partiql.type.StaticType.isNullable()`
+  - `org.partiql.type.StaticType.isMissable()`
+  - `org.partiql.type.StaticType.asOptional()`
+  - `org.partiql.type.AnyOfType()`
+  - `org.partiql.value.PartiQLValueType.NULL`
+  - `org.partiql.value.PartiQLValueType.MISSING`
 
 ### Fixed
 

partiql-planner/src/main/kotlin/org/partiql/planner/internal/transforms/PlanTransform.kt

@@ -365,7 +365,7 @@ internal object PlanTransform : PlanBaseVisitor<PlanNode, ProblemCallback>() {
                     org.partiql.plan.Agg(
                         FunctionSignature.Aggregation(
                             "UNKNOWN_AGG::$name",
-                            returns = PartiQLValueType.MISSING,
+                            returns = PartiQLValueType.ANY,
                             parameters = emptyList()
                         )
                     )

partiql-planner/src/main/kotlin/org/partiql/planner/internal/transforms/RexConverter.kt

@@ -42,7 +42,6 @@ import org.partiql.planner.internal.ir.rexOpStructField
 import org.partiql.planner.internal.ir.rexOpSubquery
 import org.partiql.planner.internal.ir.rexOpTupleUnion
 import org.partiql.planner.internal.ir.rexOpVarUnresolved
-import org.partiql.planner.internal.typer.toNonNullStaticType
 import org.partiql.planner.internal.typer.toStaticType
 import org.partiql.types.StaticType
 import org.partiql.types.TimeType
@@ -70,10 +69,7 @@ internal object RexConverter {
             throw IllegalArgumentException("unsupported rex $node")
 
         override fun visitExprLit(node: Expr.Lit, context: Env): Rex {
-            val type = when (node.value.isNull) {
-                true -> node.value.type.toStaticType()
-                else -> node.value.type.toNonNullStaticType()
-            }
+            val type = node.value.type.toStaticType()
             val op = rexOpLit(node.value)
             return rex(type, op)
         }
@@ -82,10 +78,7 @@ internal object RexConverter {
             val value =
                 PartiQLValueIonReaderBuilder
                     .standard().build(node.value).read()
-            val type = when (value.isNull) {
-                true -> value.type.toStaticType()
-                else -> value.type.toNonNullStaticType()
-            }
+            val type = value.type.toStaticType()
             return rex(type, rexOpLit(value))
         }
 
@@ -287,7 +280,7 @@ internal object RexConverter {
             }.toMutableList()
 
             val defaultRex = when (val default = node.default) {
-                null -> rex(type = StaticType.NULL, op = rexOpLit(value = nullValue()))
+                null -> rex(type = StaticType.ANY, op = rexOpLit(value = nullValue()))
                 else -> visitExprCoerce(default, context)
             }
             val op = rexOpCase(branches = branches, default = defaultRex)
@@ -528,8 +521,8 @@ internal object RexConverter {
             val type = node.asType
             val arg0 = visitExprCoerce(node.value, ctx)
             return when (type) {
-                is Type.NullType -> rex(StaticType.NULL, call("cast_null", arg0))
-                is Type.Missing -> rex(StaticType.MISSING, call("cast_missing", arg0))
+                is Type.NullType -> error("Cannot cast any value to NULL")
+                is Type.Missing -> error("Cannot cast any value to MISSING")
                 is Type.Bool -> rex(StaticType.BOOL, call("cast_bool", arg0))
                 is Type.Tinyint -> TODO("Static Type does not have TINYINT type")
                 is Type.Smallint, is Type.Int2 -> rex(StaticType.INT2, call("cast_int16", arg0))

partiql-planner/src/main/kotlin/org/partiql/planner/internal/typer/DynamicTyper.kt

@@ -2,9 +2,10 @@
 
 package org.partiql.planner.internal.typer
 
-import org.partiql.types.MissingType
-import org.partiql.types.NullType
+import org.partiql.planner.internal.ir.Rex
 import org.partiql.types.StaticType
+import org.partiql.value.MissingValue
+import org.partiql.value.NullValue
 import org.partiql.value.PartiQLValueExperimental
 import org.partiql.value.PartiQLValueType
 import org.partiql.value.PartiQLValueType.ANY
@@ -27,8 +28,6 @@ import org.partiql.value.PartiQLValueType.INT64
 import org.partiql.value.PartiQLValueType.INT8
 import org.partiql.value.PartiQLValueType.INTERVAL
 import org.partiql.value.PartiQLValueType.LIST
-import org.partiql.value.PartiQLValueType.MISSING
-import org.partiql.value.PartiQLValueType.NULL
 import org.partiql.value.PartiQLValueType.SEXP
 import org.partiql.value.PartiQLValueType.STRING
 import org.partiql.value.PartiQLValueType.STRUCT
@@ -57,59 +56,63 @@ internal class DynamicTyper {
     private var supertype: PartiQLValueType? = null
     private var args = mutableListOf<PartiQLValueType>()
 
-    private var nullable = false
-    private var missable = false
     private val types = mutableSetOf<StaticType>()
 
     /**
-     * This primarily unpacks a StaticType because of NULL, MISSING.
-     *
-     *  - T
-     *  - NULL
-     *  - MISSING
-     *  - (NULL)
-     *  - (MISSING)
-     *  - (T..)
-     *  - (T..|NULL)
-     *  - (T..|MISSING)
-     *  - (T..|NULL|MISSING)
-     *  - (NULL|MISSING)
-     *
+     * Adds the [rex]'s [Rex.type] to the typing accumulator (if the [rex] is not a literal NULL/MISSING).
+     */
+    fun accumulate(rex: Rex) {
+        when (rex.isLiteralAbsent()) {
+            true -> accumulateUnknown()
+            false -> accumulate(rex.type)
+        }
+    }
+
+    /**
+     * Checks for literal NULL or MISSING.
+     */
+    private fun Rex.isLiteralAbsent(): Boolean {
+        val op = this.op
+        return op is Rex.Op.Lit && (op.value is MissingValue || op.value is NullValue)
+    }
+
+    /**
+     * When a literal null or missing value is present in the query, its type is unknown. Therefore, its type must be
+     * inferred. This function ignores literal null/missing values, yet adds their indices to know how to return the
+     * mapping.
+     */
+    private fun accumulateUnknown() {
+        args.add(ANY)
+    }
+
+    /**
+     * This adds non-unknown types (aka not NULL / MISSING literals) to the typing accumulator.
      * @param type
      */
-    fun accumulate(type: StaticType) {
-        val nonAbsentTypes = mutableSetOf<StaticType>()
+    private fun accumulate(type: StaticType) {
         val flatType = type.flatten()
         if (flatType == StaticType.ANY) {
-            // Use ANY runtime; do not expand ANY
             types.add(flatType)
             args.add(ANY)
             calculate(ANY)
             return
         }
-        for (t in flatType.allTypes) {
-            when (t) {
-                is NullType -> nullable = true
-                is MissingType -> missable = true
-                else -> nonAbsentTypes.add(t)
-            }
-        }
-        when (nonAbsentTypes.size) {
+        val allTypes = flatType.allTypes
+        when (allTypes.size) {
             0 -> {
-                // Ignore in calculating supertype.
-                args.add(NULL)
+                error("This should not have happened.")
             }
             1 -> {
                 // Had single type
-                val single = nonAbsentTypes.first()
+                val single = allTypes.first()
                 val singleRuntime = single.toRuntimeType()
                 types.add(single)
                 args.add(singleRuntime)
                 calculate(singleRuntime)
             }
             else -> {
                 // Had a union; use ANY runtime
-                types.addAll(nonAbsentTypes)
+                types.addAll(allTypes)
                 args.add(ANY)
                 calculate(ANY)
             }
@@ -124,31 +127,20 @@ internal class DynamicTyper {
      * @return
      */
     fun mapping(): Pair<StaticType, List<Pair<PartiQLValueType, PartiQLValueType>>?> {
-        val modifiers = mutableSetOf<StaticType>()
-        if (nullable) modifiers.add(StaticType.NULL)
-        if (missable) modifiers.add(StaticType.MISSING)
         // If at top supertype, then return union of all accumulated types
         if (supertype == ANY) {
-            return StaticType.unionOf(types + modifiers).flatten() to null
+            return StaticType.unionOf(types).flatten() to null
         }
         // If a collection, then return union of all accumulated types as these coercion rules are not defined by SQL.
         if (supertype == STRUCT || supertype == BAG || supertype == LIST || supertype == SEXP) {
-            return StaticType.unionOf(types + modifiers) to null
+            return StaticType.unionOf(types) to null
         }
         // If not initialized, then return null, missing, or null|missing.
-        val s = supertype
-        if (s == null) {
-            val t = if (modifiers.isEmpty()) StaticType.MISSING else StaticType.unionOf(modifiers).flatten()
-            return t to null
-        }
+        val s = supertype ?: return StaticType.ANY to null
         // Otherwise, return the supertype along with the coercion mapping
-        val type = s.toNonNullStaticType()
+        val type = s.toStaticType()
         val mapping = args.map { it to s }
-        return if (modifiers.isEmpty()) {
-            type to mapping
-        } else {
-            StaticType.unionOf(setOf(type) + modifiers).flatten() to mapping
-        }
+        return type to mapping
     }
 
     private fun calculate(type: PartiQLValueType) {
@@ -163,7 +155,7 @@ internal class DynamicTyper {
         // Lookup and set the new minimum common supertype
         supertype = when {
             type == ANY -> type
-            type == NULL || type == MISSING || s == type -> return // skip
+            s == type -> return // skip
             else -> graph[s][type] ?: ANY // lookup, if missing then go to top.
         }
     }
@@ -206,8 +198,6 @@ internal class DynamicTyper {
                 graph[type] = arrayOfNulls(N)
             }
             graph[ANY] = edges()
-            graph[NULL] = edges()
-            graph[MISSING] = edges()
             graph[BOOL] = edges(
                 BOOL to BOOL
             )

partiql-planner/src/main/kotlin/org/partiql/planner/internal/typer/FnResolver.kt

@@ -5,9 +5,6 @@ import org.partiql.planner.internal.ir.Agg
 import org.partiql.planner.internal.ir.Fn
 import org.partiql.planner.internal.ir.Identifier
 import org.partiql.planner.internal.ir.Rex
-import org.partiql.planner.internal.typer.FnResolver.Companion.compareTo
-import org.partiql.types.AnyOfType
-import org.partiql.types.NullType
 import org.partiql.types.StaticType
 import org.partiql.types.function.FunctionParameter
 import org.partiql.types.function.FunctionSignature
@@ -33,8 +30,6 @@ import org.partiql.value.PartiQLValueType.INT64
 import org.partiql.value.PartiQLValueType.INT8
 import org.partiql.value.PartiQLValueType.INTERVAL
 import org.partiql.value.PartiQLValueType.LIST
-import org.partiql.value.PartiQLValueType.MISSING
-import org.partiql.value.PartiQLValueType.NULL
 import org.partiql.value.PartiQLValueType.SEXP
 import org.partiql.value.PartiQLValueType.STRING
 import org.partiql.value.PartiQLValueType.STRUCT
@@ -76,27 +71,19 @@ internal sealed class FnMatch<T : FunctionSignature> {
      *
      * @property signature
      * @property mapping
-     * @property isMissable TRUE when anyone of the arguments _could_ be MISSING. We *always* propagate MISSING.
      */
     public data class Ok<T : FunctionSignature>(
         public val signature: T,
         public val mapping: Mapping,
-        public val isMissable: Boolean,
     ) : FnMatch<T>()
 
     /**
      * This represents dynamic dispatch.
      *
      * @property candidates an ordered list of potentially applicable functions to dispatch dynamically.
-     * @property isMissable TRUE when the argument permutations may not definitively invoke one of the candidates. You
-     * can think of [isMissable] as being the same as "not exhaustive". For example, if we have ABS(INT | STRING), then
-     * this function call [isMissable] because there isn't an `ABS(STRING)` function signature AKA we haven't exhausted
-     * all the arguments. On the other hand, take an "exhaustive" scenario: ABS(INT | DEC). In this case, [isMissable]
-     * is false because we have functions for each potential argument AKA we have exhausted the arguments.
      */
     public data class Dynamic<T : FunctionSignature>(
-        public val candidates: List<Ok<T>>,
-        public val isMissable: Boolean
+        public val candidates: List<Ok<T>>
     ) : FnMatch<T>()
 
     public data class Error<T : FunctionSignature>(
@@ -162,12 +149,8 @@ internal class FnResolver(private val header: Header) {
      */
     public fun resolveFn(fn: Fn.Unresolved, args: List<Rex>): FnMatch<FunctionSignature.Scalar> {
         val candidates = lookup(fn)
-        var canReturnMissing = false
         val parameterPermutations = buildArgumentPermutations(args.map { it.type }).mapNotNull { argList ->
             argList.mapIndexed { i, arg ->
-                if (arg.isMissable()) {
-                    canReturnMissing = true
-                }
                 // Skip over if we cannot convert type to runtime type.
                 val argType = arg.toRuntimeTypeOrNull() ?: return@mapNotNull null
                 FunctionParameter("arg-$i", argType)
@@ -176,12 +159,10 @@ internal class FnResolver(private val header: Header) {
         val potentialFunctions = parameterPermutations.mapNotNull { parameters ->
             when (val match = match(candidates, parameters)) {
                 null -> {
-                    canReturnMissing = true
                     null
                 }
                 else -> {
-                    val isMissable = canReturnMissing || isUnsafeCast(match.signature.specific)
-                    FnMatch.Ok(match.signature, match.mapping, isMissable)
+                    FnMatch.Ok(match.signature, match.mapping)
                 }
             }
         }
@@ -190,18 +171,12 @@ internal class FnResolver(private val header: Header) {
         return when (orderedUniqueFunctions.size) {
             0 -> FnMatch.Error(fn.identifier, args, candidates)
             1 -> orderedUniqueFunctions.first()
-            else -> FnMatch.Dynamic(orderedUniqueFunctions, canReturnMissing)
+            else -> FnMatch.Dynamic(orderedUniqueFunctions)
         }
     }
 
     private fun buildArgumentPermutations(args: List<StaticType>): List<List<StaticType>> {
-        val flattenedArgs = args.map {
-            if (it is AnyOfType) {
-                it.flatten().allTypes.filter { it !is NullType }
-            } else {
-                it.flatten().allTypes
-            }
-        }
+        val flattenedArgs = args.map { it.flatten().allTypes }
         return buildArgumentPermutations(flattenedArgs, accumulator = emptyList())
     }
 
@@ -229,19 +204,13 @@ internal class FnResolver(private val header: Header) {
      */
     public fun resolveAgg(agg: Agg.Unresolved, args: List<Rex>): FnMatch<FunctionSignature.Aggregation> {
         val candidates = lookup(agg)
-        var hadMissingArg = false
         val parameters = args.mapIndexed { i, arg ->
-            if (!hadMissingArg && arg.type.isMissable()) {
-                hadMissingArg = true
-            }
             FunctionParameter("arg-$i", arg.type.toRuntimeType())
         }
-        val match = match(candidates, parameters)
-        return when (match) {
+        return when (val match = match(candidates, parameters)) {
             null -> FnMatch.Error(agg.identifier, args, candidates)
             else -> {
-                val isMissable = hadMissingArg || isUnsafeCast(match.signature.specific)
-                FnMatch.Ok(match.signature, match.mapping, isMissable)
+                FnMatch.Ok(match.signature, match.mapping)
             }
         }
     }
@@ -290,9 +259,7 @@ internal class FnResolver(private val header: Header) {
                 a.type == p.type -> mapping.add(null)
                 // 2. Match ANY, no coercion needed
                 p.type == ANY -> mapping.add(null)
-                // 3. Match NULL argument
-                a.type == NULL -> mapping.add(null)
-                // 4. Check for a coercion
+                // 3. Check for a coercion
                 else -> {
                     val coercion = lookupCoercion(a.type, p.type)
                     when (coercion) {
@@ -440,8 +407,10 @@ internal class FnResolver(private val header: Header) {
         // This is not explicitly defined in the PartiQL Specification!!
         // This does not imply the ability to CAST; this defines function resolution behavior.
         private val precedence: Map<PartiQLValueType, Int> = listOf(
-            NULL,
-            MISSING,
+            @Suppress("DEPRECATION")
+            PartiQLValueType.NULL, // TODO: Remove once functions no longer specify parameter/return types with the NULL type.
+            @Suppress("DEPRECATION")
+            PartiQLValueType.MISSING, // TODO: Remove once functions no longer specify parameter/return types with the MISSING type.
             BOOL,
             INT8,
             INT16,