Skip to content

Commit

Permalink
struct subfield and list element type (#1449)
Browse files Browse the repository at this point in the history 
* struct subfield and list element type

---------

Co-authored-by: Alan Cai <[email protected]>
  • Loading branch information
@yliuuuu @alancai98
yliuuuu and alancai98 authored May 1, 2024
1 parent 607c4c0 commit 23f6fee
Show file tree
Hide file tree
Showing 14 changed files with 762 additions and 79 deletions.
105 changes: 95 additions & 10 deletions partiql-ast/api/partiql-ast.api
View file

Large diffs are not rendered by default.

13 changes: 12 additions & 1 deletion partiql-ast/src/main/kotlin/org/partiql/ast/helpers/ToLegacyAst.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1346,11 +1346,22 @@ private class AstTranslator(val metas: Map<String, MetaContainer>) : AstBaseVisi

override fun visitTypeList(node: Type.List, ctx: Ctx) = translate(node) { metas -> listType(metas) }

override fun visitTypeArray(node: Type.Array, ctx: Ctx) = translate(node) { metas ->
if (node.type != null) {
error("The legacy AST does not support element type declaration for list")
}
listType(metas)
}
override fun visitTypeSexp(node: Type.Sexp, ctx: Ctx) = translate(node) { metas -> sexpType(metas) }

override fun visitTypeTuple(node: Type.Tuple, ctx: Ctx) = translate(node) { metas -> tupleType(metas) }

override fun visitTypeStruct(node: Type.Struct, ctx: Ctx) = translate(node) { metas -> structType(metas) }
override fun visitTypeStruct(node: Type.Struct, ctx: Ctx) = translate(node) { metas ->
if (node.fields.isNotEmpty()) {
error("The legacy AST does not support field declaration in struct type")
}
structType(metas)
}

override fun visitTypeAny(node: Type.Any, ctx: Ctx) = translate(node) { metas -> anyType(metas) }

Expand Down
3 changes: 3 additions & 0 deletions partiql-ast/src/main/kotlin/org/partiql/ast/sql/internal/InternalSqlDialect.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -201,10 +201,13 @@ internal abstract class InternalSqlDialect : AstBaseVisitor<InternalSqlBlock, In

override fun visitTypeList(node: Type.List, tail: InternalSqlBlock): InternalSqlBlock = tail concat "LIST"

// TODO: Support List Element
override fun visitTypeArray(node: Type.Array, tail: InternalSqlBlock): InternalSqlBlock = tail concat "ARRAY"
override fun visitTypeSexp(node: Type.Sexp, tail: InternalSqlBlock): InternalSqlBlock = tail concat "SEXP"

override fun visitTypeTuple(node: Type.Tuple, tail: InternalSqlBlock): InternalSqlBlock = tail concat "TUPLE"

// TODO: Support Struct Field
override fun visitTypeStruct(node: Type.Struct, tail: InternalSqlBlock): InternalSqlBlock = tail concat "STRUCT"

override fun visitTypeAny(node: Type.Any, tail: InternalSqlBlock): InternalSqlBlock = tail concat "ANY"
Expand Down
38 changes: 37 additions & 1 deletion partiql-ast/src/main/resources/partiql_ast.ion
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,10 +196,46 @@ type::[
timestamp_with_tz::{ precision: optional::int }, // TIMESTAMP [(<int>)] WITH TIMEZONE
interval::{ precision: optional::int }, // INTERVAL
bag::{}, // BAG
// Potential migration strategy:
// v0.14.x -- the current state in `main`
// typeList()
//
// v0.15.x -- the next release
// @Deprecated // recommend using `typeArray()`
// typeList()
// typeArray(...)
//
// v1.0.x
// typeArray(...) // `typeList` now deleted
list::{}, // LIST
array::{ type: optional::type }, // ARRAY [<type>]
sexp::{}, // SEXP
// Potential migration strategy:
// v0.14.x -- the current state in `main`
// typeTuple()
// typeStruct()
//
// v0.15.x -- the next release
// @Deprecated // recommend using `typeStruct`
// typeTuple()
// typeStruct(fields = ...)
//
// v1.0.x
// typeStruct(fields = ...) // `typeTuple` now deleted
tuple::{}, // TUPLE
struct::{}, // STRUCT
struct::{
fields: list::[field],
_ : [
field :: {
name: '.identifier.symbol',
type: '.type',
// This could be a boolean flag since we only support NOT NULL constraint
// for struct subfield. But modeling this to be a list of constraints
// to prevent future breaking changes.
constraints: list::[constraint],
}
],
}, // STRUCT <fields>
any::{}, // ANY
custom::{ name: string }, // <symbol>
]
Expand Down
16 changes: 15 additions & 1 deletion partiql-ast/src/test/kotlin/org/partiql/ast/helpers/ToLegacyAstTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -407,15 +407,29 @@ class ToLegacyAstTest {
expect("(struct_type)") { typeStruct() },
expect("(tuple_type)") { typeTuple() },
expect("(list_type)") { typeList() },
expect("(list_type)") { typeArray(null) },
expect("(sexp_type)") { typeSexp() },
expect("(bag_type)") { typeBag() },
expect("(any_type)") { typeAny() },
// Other (??)
expect("(integer4_type)") { typeInt4() },
expect("(integer8_type)") { typeInt8() },
expect("(custom_type dog)") { typeCustom("dog") }
expect("(custom_type dog)") { typeCustom("dog") },
// LEGACY AST does not have TIMESTAMP or INTERVAL
// LEGACY AST does not have parameterized blob/clob
// LEGACY AST does not support struct with field declaration
fail("The legacy AST does not support field declaration in struct type") {
typeStruct {
fields += org.partiql.ast.typeStructField(
org.partiql.ast.identifierSymbol("a", Identifier.CaseSensitivity.INSENSITIVE),
typeInt2(),
emptyList()
)
}
},
fail("The legacy AST does not support element type declaration for list") {
typeArray(typeInt2())
},
)

@JvmStatic
Expand Down
62 changes: 48 additions & 14 deletions partiql-lang/src/main/kotlin/org/partiql/lang/syntax/impl/PartiQLPigVisitor.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1089,20 +1089,20 @@ internal class PartiQLPigVisitor(
* SIMPLE EXPRESSIONS
*/

override fun visitOr(ctx: PartiQLParser.OrContext) = visitBinaryOperation(ctx.lhs, ctx.rhs, ctx.OR().symbol, null)
override fun visitOr(ctx: PartiQLParser.OrContext) = visitBinaryOperation(ctx.lhs, ctx.rhs, listOf(ctx.OR().symbol), null)

override fun visitAnd(ctx: PartiQLParser.AndContext) = visitBinaryOperation(ctx.lhs, ctx.rhs, ctx.op, null)
override fun visitAnd(ctx: PartiQLParser.AndContext) = visitBinaryOperation(ctx.lhs, ctx.rhs, listOf(ctx.op), null)

override fun visitNot(ctx: PartiQLParser.NotContext) = visitUnaryOperation(ctx.rhs, ctx.op, null)

override fun visitMathOp00(ctx: PartiQLParser.MathOp00Context): PartiqlAst.PartiqlAstNode =
visitBinaryOperation(ctx.lhs, ctx.rhs, ctx.op, ctx.parent)
visitBinaryOperation(ctx.lhs, ctx.rhs, listOf(ctx.op), ctx.parent)

override fun visitMathOp01(ctx: PartiQLParser.MathOp01Context): PartiqlAst.PartiqlAstNode =
visitBinaryOperation(ctx.lhs, ctx.rhs, ctx.op, ctx.parent)
visitBinaryOperation(ctx.lhs, ctx.rhs, listOf(ctx.op), ctx.parent)

override fun visitMathOp02(ctx: PartiQLParser.MathOp02Context): PartiqlAst.PartiqlAstNode =
visitBinaryOperation(ctx.lhs, ctx.rhs, ctx.op, ctx.parent)
visitBinaryOperation(ctx.lhs, ctx.rhs, listOf(ctx.op), ctx.parent)

override fun visitValueExpr(ctx: PartiQLParser.ValueExprContext) =
visitUnaryOperation(ctx.rhs, ctx.sign, ctx.parent)
Expand All @@ -1114,7 +1114,7 @@ internal class PartiQLPigVisitor(
*/

override fun visitPredicateComparison(ctx: PartiQLParser.PredicateComparisonContext) =
visitBinaryOperation(ctx.lhs, ctx.rhs, ctx.op)
visitBinaryOperation(ctx.lhs, ctx.rhs, ctx.op.children.map { (it as TerminalNode).symbol })

/**
* Note: This predicate can take a wrapped expression on the RHS, and it will wrap it in a LIST. However, if the
Expand Down Expand Up @@ -1474,7 +1474,7 @@ internal class PartiQLPigVisitor(

override fun visitBag(ctx: PartiQLParser.BagContext) = PartiqlAst.build {
val exprList = ctx.expr().map { visitExpr(it) }
bag(exprList, ctx.ANGLE_DOUBLE_LEFT().getSourceMetaContainer())
bag(exprList, ctx.ANGLE_LEFT(0).getSourceMetaContainer())
}

override fun visitLiteralDecimal(ctx: PartiQLParser.LiteralDecimalContext) = PartiqlAst.build {
Expand Down Expand Up @@ -1609,9 +1609,6 @@ internal class PartiQLPigVisitor(
PartiQLParser.BLOB -> blobType(metas)
PartiQLParser.CLOB -> clobType(metas)
PartiQLParser.DATE -> dateType(metas)
PartiQLParser.STRUCT -> structType(metas)
PartiQLParser.TUPLE -> tupleType(metas)
PartiQLParser.LIST -> listType(metas)
PartiQLParser.BAG -> bagType(metas)
PartiQLParser.SEXP -> sexpType(metas)
PartiQLParser.ANY -> anyType(metas)
Expand Down Expand Up @@ -1677,6 +1674,32 @@ internal class PartiQLPigVisitor(
customType_(SymbolPrimitive(customName, metas), metas)
}

override fun visitTypeComplexUnparameterized(ctx: PartiQLParser.TypeComplexUnparameterizedContext) = PartiqlAst.build {
val metas = ctx.datatype.getSourceMetaContainer()
when (ctx.datatype.type) {
PartiQLParser.TUPLE -> tupleType(metas)
PartiQLParser.STRUCT -> structType(metas)
PartiQLParser.ARRAY, PartiQLParser.LIST -> listType(metas)
else -> throw ParserException("Unknown datatype", ErrorCode.PARSE_UNEXPECTED_TOKEN, PropertyValueMap())
}
}

override fun visitTypeStruct(ctx: PartiQLParser.TypeStructContext) = PartiqlAst.build {
throw ParserException(
"PIG Parser does not support struct type with field declaration",
ErrorCode.PARSE_UNEXPECTED_TOKEN,
PropertyValueMap()
)
}

override fun visitTypeList(ctx: PartiQLParser.TypeListContext): PartiqlAst.PartiqlAstNode {
throw ParserException(
"PIG Parser does not support element type declaration for list",
ErrorCode.PARSE_UNEXPECTED_TOKEN,
PropertyValueMap()
)
}

/**
*
* HELPER METHODS
Expand All @@ -1695,6 +1718,14 @@ internal class PartiQLPigVisitor(
return com.amazon.ionelement.api.metaContainerOf(Pair(metas.tag, metas))
}

private fun List<Token>.getSourceMetaContainer(): MetaContainer {
val base = this.firstOrNull() ?: return emptyMetaContainer()
val length = this.fold(0) { acc, token ->
acc + token.stopIndex - token.startIndex + 1
}
return metaContainerOf(SourceLocationMeta(base.line.toLong(), base.charPositionInLine.toLong() + 1, length.toLong()))
}

private fun TerminalNode.getSourceMetas(): SourceLocationMeta = this.symbol.getSourceMetas()

private fun Token.getSourceMetas(): SourceLocationMeta {
Expand All @@ -1705,13 +1736,13 @@ internal class PartiQLPigVisitor(
private fun visitBinaryOperation(
lhs: ParserRuleContext?,
rhs: ParserRuleContext?,
op: Token?,
op: List<Token>,
parent: ParserRuleContext? = null,
) = PartiqlAst.build {
if (parent != null) return@build visit(parent) as PartiqlAst.Expr
val args = listOf(lhs!!, rhs!!).map { visit(it) as PartiqlAst.Expr }
val metas = op.getSourceMetaContainer()
when (op!!.type) {
when (op.first().type) {
PartiQLParser.AND -> and(args, metas)
PartiQLParser.OR -> or(args, metas)
PartiQLParser.ASTERISK -> times(args, metas)
Expand All @@ -1720,11 +1751,14 @@ internal class PartiQLPigVisitor(
PartiQLParser.MINUS -> minus(args, metas)
PartiQLParser.PERCENT -> modulo(args, metas)
PartiQLParser.CONCAT -> concat(args, metas)
PartiQLParser.ANGLE_LEFT -> lt(args, metas)
PartiQLParser.ANGLE_LEFT -> {
if (op.size > 1) ne(args, metas)
else lt(args, metas)
}
PartiQLParser.LT_EQ -> lte(args, metas)
PartiQLParser.ANGLE_RIGHT -> gt(args, metas)
PartiQLParser.GT_EQ -> gte(args, metas)
PartiQLParser.NEQ -> ne(args, metas)
PartiQLParser.BANG -> ne(args, metas)
PartiQLParser.EQ -> eq(args, metas)
PartiQLParser.AMPERSAND -> bitwiseAnd(args, metas)
else -> throw ParserException("Unknown binary operator", ErrorCode.PARSE_INVALID_QUERY)
Expand Down
54 changes: 54 additions & 0 deletions partiql-lang/src/test/kotlin/org/partiql/lang/syntax/PartiQLParserDDLTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,60 @@ internal class PartiQLParserDDLTest : PartiQLParserTestBase() {
code = ErrorCode.PARSE_UNEXPECTED_TOKEN,
context = mapOf(),
),
ParserErrorTestCase(
description = "PIG Parser does not support Struct Type with field declaration",
query = """
CREATE TABLE tbl (
a STRUCT<b : INT2>
)
""".trimIndent(),
code = ErrorCode.PARSE_UNEXPECTED_TOKEN,
context = mapOf(),
),
ParserErrorTestCase(
description = "PIG Parser does not support element type declaration in Array Type",
query = """
CREATE TABLE tbl (
a ARRAY<INT2>
)
""".trimIndent(),
code = ErrorCode.PARSE_UNEXPECTED_TOKEN,
context = mapOf(),
),

// Putting those tests here are they are impacted by DDL implementation
ParserErrorTestCase(
description = "PIG Parser does not support Struct Type with field declaration -- IS operator",
query = """
a IS STRUCT<b : INT2>
""".trimIndent(),
code = ErrorCode.PARSE_UNEXPECTED_TOKEN,
context = mapOf(),
),
ParserErrorTestCase(
description = "PIG Parser does not support Struct Type with field declaration -- CAST operator",
query = """
CAST(a AS STRUCT<b : INT2>)
""".trimIndent(),
code = ErrorCode.PARSE_UNEXPECTED_TOKEN,
context = mapOf(),
),
ParserErrorTestCase(
description = "PIG Parser does not support element type declaration in Array Type -- IS operator",
query = """
a IS ARRAY<INT2>
""".trimIndent(),
code = ErrorCode.PARSE_UNEXPECTED_TOKEN,
context = mapOf(),
),
ParserErrorTestCase(
description = "PIG Parser does not support element type declaration in Array Type -- cast operator",
query = """
CAST(a AS ARRAY<INT2>)
""".trimIndent(),
code = ErrorCode.PARSE_UNEXPECTED_TOKEN,
context = mapOf(),
),
)
}
}
Loading

0 comments on commit 23f6fee

Please sign in to comment.