Lots of fixes around MySQL (#53)

Big thing is that insert behavior is deferred to the Grammar so that MySQL can do it custom. By default, MySQL now runs two queries, one to insert and one to pull the recent item. `insertAll()` also is now separate, consecutive queries in MySQL. The user can opt out of this (single query for insert and insertAll) by passing `returnItems: false` to `QueryBuilder.insert`. 

This also...

Adds
- `ON DELETE` and `ON UPDATE` to migration builders
- bigInt type and `UNSIGNED` modifier to migration builders
- Revamped column constraint building to add most constraints separately, at the end of a `CREATE TABLE` statement since the shortcuts I was using weren't all compatible across Postgres and MySQL

Fixes
- Issue with SQL columns not being populated if `.select` wasn't called in query builder (Postgres accepted it with no *, MySQL did not. That's why it didn't get caught)
- Adds some missing date time types for parsing from MySQL
- Fix constraint issues that were borked on MySQL. 

Tested all the Quickstart endpoints on Postgres/MySQL to confirm nothing else is missing.

Also updated the quickstart migrations to use `.unsigned` and `.bigInt` for some columns by default, since even tho it works fine with Postgres without, MySQL requires them. 

Co-authored-by: Chris Anderson <[email protected]>

Quickstarts/Backend/Sources/Backend/Migrations/_20210107155059CreateUsers.swift

@@ -16,7 +16,7 @@ struct _20210107155059CreateUsers: Migration {
             $0.increments("id").primary()
             $0.string("value").notNull()
             $0.date("created_at").notNull()
-            $0.int("user_id").references("id", on: "users").notNull()
+            $0.bigInt("user_id").unsigned().references("id", on: "users").notNull()
         }
     }
 

Quickstarts/Backend/Sources/Backend/Migrations/_20210107155107CreateTodos.swift

@@ -8,22 +8,22 @@ struct _20210107155107CreateTodos: Migration {
             $0.increments("id").primary()
             $0.string("name").notNull()
             $0.bool("is_complete").notNull().default(val: false)
-            $0.int("user_id").references("id", on: "users").notNull()
+            $0.bigInt("user_id").unsigned().references("id", on: "users").notNull()
         }
 
         // Create a table backing `Tag`.
         schema.create(table: "tags") {
             $0.increments("id").primary()
             $0.string("name").notNull()
             $0.int("color").notNull()
-            $0.int("user_id").references("id", on: "users").notNull()
+            $0.bigInt("user_id").unsigned().references("id", on: "users").notNull()
         }
 
         // Create a table backing `TodoTag`.
         schema.create(table: "todo_tags") {
             $0.increments("id").primary()
-            $0.int("todo_id").references("id", on: "todos").notNull()
-            $0.int("tag_id").references("id", on: "tags").notNull()
+            $0.bigInt("todo_id").unsigned().references("id", on: "todos").notNull()
+            $0.bigInt("tag_id").unsigned().references("id", on: "tags").notNull()
         }
     }
 

Quickstarts/Fullstack/Backend/Sources/Backend/Migrations/_20210107155059CreateUsers.swift

@@ -16,7 +16,7 @@ struct _20210107155059CreateUsers: Migration {
             $0.increments("id").primary()
             $0.string("value").notNull()
             $0.date("created_at").notNull()
-            $0.int("user_id").references("id", on: "users").notNull()
+            $0.bigInt("user_id").unsigned().references("id", on: "users").notNull()
         }
     }
 

Quickstarts/Fullstack/Backend/Sources/Backend/Migrations/_20210107155107CreateTodos.swift

@@ -8,22 +8,22 @@ struct _20210107155107CreateTodos: Migration {
             $0.increments("id").primary()
             $0.string("name").notNull()
             $0.bool("is_complete").notNull().default(val: false)
-            $0.int("user_id").references("id", on: "users").notNull()
+            $0.bigInt("user_id").unsigned().references("id", on: "users").notNull()
         }
 
         // Create a table backing `Tag`.
         schema.create(table: "tags") {
             $0.increments("id").primary()
             $0.string("name").notNull()
             $0.int("color").notNull()
-            $0.int("user_id").references("id", on: "users").notNull()
+            $0.bigInt("user_id").unsigned().references("id", on: "users").notNull()
         }
 
         // Create a table backing `TodoTag`.
         schema.create(table: "todo_tags") {
             $0.increments("id").primary()
-            $0.int("todo_id").references("id", on: "todos").notNull()
-            $0.int("tag_id").references("id", on: "tags").notNull()
+            $0.bigInt("todo_id").unsigned().references("id", on: "todos").notNull()
+            $0.bigInt("tag_id").unsigned().references("id", on: "tags").notNull()
         }
     }
 

Sources/Alchemy/SQL/Database/MySQL/MySQL+Database.swift

@@ -49,6 +49,33 @@ public final class MySQLDatabase: Database {
         }
     }
 
+    /// MySQL doesn't have a way to return a row after inserting. This
+    /// runs a query and if MySQL metadata contains a `lastInsertID`,
+    /// fetches the row with that id from the given table.
+    ///
+    /// - Parameters:
+    ///   - sql: The SQL to run.
+    ///   - table: The table from which `lastInsertID` should be
+    ///     fetched.
+    ///   - values: Any bindings for the query.
+    /// - Returns: A future containing the result of fetching the last
+    ///   inserted id, or the result of the original query.
+    func runAndReturnLastInsertedItem(_ sql: String, table: String, values: [DatabaseValue]) -> EventLoopFuture<[DatabaseRow]> {
+        self.pool.withConnection(logger: Log.logger, on: Services.eventLoop) { conn in
+            var lastInsertId: Int?
+            return conn
+                .query(sql, values.map(MySQLData.init), onMetadata: { lastInsertId = $0.lastInsertID.map(Int.init) })
+                .flatMap { rows -> EventLoopFuture<[MySQLRow]> in
+                    if let lastInsertId = lastInsertId {
+                        return conn.query("select * from \(table) where id = ?;", [MySQLData(.int(lastInsertId))])
+                    } else {
+                        return .new(rows)
+                    }
+                }
+                .map { $0 }
+        }
+    }
+
     public func shutdown() throws {
         try self.pool.syncShutdownGracefully()
     }

Sources/Alchemy/SQL/Database/MySQL/MySQL+DatabaseRow.swift

@@ -75,7 +75,7 @@ extension MySQLData {
         case .varchar, .string, .varString:
             let value = DatabaseValue.string(try validateNil(self.string))
             return DatabaseField(column: column, value: value)
-        case .date, .timestamp, .timestamp2:
+        case .date, .timestamp, .timestamp2, .datetime, .datetime2:
             let value = DatabaseValue.date(try validateNil(self.time?.date))
             return DatabaseField(column: column, value: value)
         case .time:

Sources/Alchemy/SQL/Database/MySQL/MySQL+Grammar.swift

@@ -1,12 +1,8 @@
+import NIO
+
 /// A MySQL specific Grammar for compiling QueryBuilder statements
 /// into SQL strings.
 final class MySQLGrammar: Grammar {
-    override func compileInsert(_ query: Query, values: [OrderedDictionary<String, Parameter>]) throws -> SQL {
-        var initial = try super.compileInsert(query, values: values)
-        initial.query.append("; select * from table where Id=LAST_INSERT_ID();")
-        return initial
-    }
-
     override func compileDropIndex(table: String, indexName: String) -> SQL {
         SQL("DROP INDEX \(indexName) ON \(table)")
     }
@@ -20,9 +16,11 @@ final class MySQLGrammar: Grammar {
         case .double:
             return "double"
         case .increments:
-            return "SERIAL"
+            return "serial"
         case .int:
             return "int"
+        case .bigInt:
+            return "bigint"
         case .json:
             return "json"
         case .string(let length):
@@ -33,12 +31,37 @@ final class MySQLGrammar: Grammar {
                 return "varchar(\(characters))"
             }
         case .uuid:
-            // There isn't a MySQL UUID type; store UUIDs as a 36 length varchar.
+            // There isn't a MySQL UUID type; store UUIDs as a 36
+            // length varchar.
             return "varchar(36)"
         }
     }
 
     override func jsonLiteral(from jsonString: String) -> String {
         "('\(jsonString)')"
     }
+
+    override func allowsUnsigned() -> Bool {
+        true
+    }
+
+    // MySQL needs custom insert behavior, since bulk inserting and
+    // returning is not supported.
+    override func insert(_ values: [OrderedDictionary<String, Parameter>], query: Query, returnItems: Bool) -> EventLoopFuture<[DatabaseRow]> {
+        catchError {
+            guard
+                returnItems,
+                let table = query.from,
+                let database = query.database as? MySQLDatabase
+            else {
+                return super.insert(values, query: query, returnItems: returnItems)
+            }
+
+            return try values
+                .map { try self.compileInsert(query, values: [$0]) }
+                .map { database.runAndReturnLastInsertedItem($0.query, table: table, values: $0.bindings) }
+                .flatten(on: Services.eventLoop)
+                .map { $0.flatMap { $0 } }
+        }
+    }
 }

Sources/Alchemy/SQL/Migrations/Builders/CreateColumnBuilder.swift

@@ -4,6 +4,41 @@ protocol ColumnBuilderErased {
     func toCreate() -> CreateColumn
 }
 
+/// Options for an `onDelete` or `onUpdate` reference constraint.
+public enum ReferenceOption: String {
+    /// RESTRICT
+    case restrict = "RESTRICT"
+    /// CASCADE
+    case cascade = "CASCADE"
+    /// SET NULL
+    case setNull = "SET NULL"
+    /// NO ACTION
+    case noAction = "NO ACTION"
+    /// SET DEFAULT
+    case setDefault = "SET DEFAULT"
+}
+
+/// Various constraints for columns.
+enum ColumnConstraint {
+    /// This column shouldn't be null.
+    case notNull
+    /// The default value for this column.
+    case `default`(String)
+    /// This column is the primary key of it's table.
+    case primaryKey
+    /// This column is unique on this table.
+    case unique
+    /// This column references a `column` on another `table`.
+    case foreignKey(
+            column: String,
+            table: String,
+            onDelete: ReferenceOption? = nil,
+            onUpdate: ReferenceOption? = nil
+         )
+    /// This int column is unsigned.
+    case unsigned
+}
+
 /// A builder for creating columns on a table in a relational database.
 ///
 /// `Default` is a Swift type that can be used to add a default value
@@ -19,7 +54,7 @@ public final class CreateColumnBuilder<Default: Sequelizable>: ColumnBuilderEras
     private let type: ColumnType
 
     /// Any modifiers of the column.
-    private var modifiers: [String]
+    private var constraints: [ColumnConstraint]
 
     /// Create with a name, a type, and a modifier array.
     ///
@@ -29,11 +64,11 @@ public final class CreateColumnBuilder<Default: Sequelizable>: ColumnBuilderEras
     ///   - name: The name of the column to create.
     ///   - type: The type of the column to create.
     ///   - modifiers: Any modifiers of the column.
-    init(grammar: Grammar, name: String, type: ColumnType, modifiers: [String] = []) {
+    init(grammar: Grammar, name: String, type: ColumnType, constraints: [ColumnConstraint] = []) {
         self.grammar = grammar
         self.name = name
         self.type = type
-        self.modifiers = modifiers
+        self.constraints = constraints
     }
 
     /// Adds an expression as the default value of this column.
@@ -42,28 +77,28 @@ public final class CreateColumnBuilder<Default: Sequelizable>: ColumnBuilderEras
     ///   default value of this column.
     /// - Returns: This column builder.
     @discardableResult public func `default`(expression: String) -> Self {
-        self.appending(modifier: "DEFAULT \(expression)")
+        self.adding(constraint: .default(expression))
     }
 
     /// Adds a value as the default for this column.
     ///
     /// - Parameter expression: A default value for this column.
     /// - Returns: This column builder.
     @discardableResult public func `default`(val: Default) -> Self {
-        // Janky, but MySQL requires parenthases around text (but not
+        // Janky, but MySQL requires parentheses around text (but not
         // varchar...) literals.
         if case .string(.unlimited) = self.type, self.grammar is MySQLGrammar {
-            return self.appending(modifier: "DEFAULT (\(val.toSQL().query))")
+            return self.adding(constraint: .default("(\(val.toSQL().query))"))
         }
 
-        return self.appending(modifier: "DEFAULT \(val.toSQL().query)")
+        return self.adding(constraint: .default(val.toSQL().query))
     }
 
     /// Define this column as not nullable.
     ///
     /// - Returns: This column builder.
     @discardableResult public func notNull() -> Self {
-        self.appending(modifier: "NOT NULL")
+        self.adding(constraint: .notNull)
     }
 
     /// Defines this column as a reference to another column on a
@@ -72,38 +107,58 @@ public final class CreateColumnBuilder<Default: Sequelizable>: ColumnBuilderEras
     /// - Parameters:
     ///   - column: The column name this column references.
     ///   - table: The table of the column this column references.
+    ///   - onDelete: The `ON DELETE` reference option for this
+    ///     column. Defaults to nil.
+    ///   - onUpdate: The `ON UPDATE` reference option for this
+    ///     column. Defaults to nil.
     /// - Returns: This column builder.
-    @discardableResult public func references(_ column: String, on table: String) -> Self {
-        self.appending(modifier: "REFERENCES \(table)(\(column))")
+    @discardableResult public func references(
+        _ column: String,
+        on table: String,
+        onDelete: ReferenceOption? = nil,
+        onUpdate: ReferenceOption? = nil
+    ) -> Self {
+        self.adding(constraint: .foreignKey(column: column, table: table, onDelete: onDelete, onUpdate: onUpdate))
     }
 
     /// Defines this column as a primary key.
     ///
     /// - Returns: This column builder.
     @discardableResult public func primary() -> Self {
-        self.appending(modifier: "PRIMARY KEY")
+        self.adding(constraint: .primaryKey)
     }
 
     /// Defines this column as unique.
     ///
     /// - Returns: This column builder.
     @discardableResult public func unique() -> Self {
-        self.appending(modifier: "UNIQUE")
+        self.adding(constraint: .unique)
     }
 
     /// Adds a modifier to `self.modifiers` and then returns `self`.
     ///
     /// - Parameter modifier: The modifier to add.
     /// - Returns: This column builder.
-    private func appending(modifier: String) -> Self {
-        self.modifiers.append(modifier)
+    private func adding(constraint: ColumnConstraint) -> Self {
+        self.constraints.append(constraint)
         return self
     }
 
     // MARK: ColumnBuilderErased
 
     func toCreate() -> CreateColumn {
-        CreateColumn(column: self.name, type: self.type, constraints: self.modifiers)
+        CreateColumn(column: self.name, type: self.type, constraints: self.constraints)
+    }
+}
+
+extension CreateColumnBuilder where Default == Int {
+    /// Defines this integer column as unsigned.
+    ///
+    /// - Note: Ignored if the backing Database is `PostgresDatabase`.
+    ///
+    /// - Returns: This column builder.
+    @discardableResult public func unsigned() -> Self {
+        self.adding(constraint: .unsigned)
     }
 }
 
@@ -115,7 +170,7 @@ extension CreateColumnBuilder where Default == SQLJSON {
     ///   for this column.
     /// - Returns: This column builder.
     @discardableResult public func `default`(jsonString: String) -> Self {
-        self.appending(modifier: "DEFAULT \(self.grammar.jsonLiteral(from: jsonString))")
+        self.adding(constraint: .default(self.grammar.jsonLiteral(from: jsonString)))
     }
 
     /// Adds an `Encodable` as the default for this column.
@@ -135,7 +190,7 @@ extension CreateColumnBuilder where Default == SQLJSON {
         }
 
         let jsonString = String(decoding: jsonData, as: UTF8.self)
-        return self.appending(modifier: "DEFAULT \(self.grammar.jsonLiteral(from: jsonString))")
+        return self.adding(constraint: .default(self.grammar.jsonLiteral(from: jsonString)))
     }
 }
 

Sources/Alchemy/SQL/Migrations/Builders/CreateTableBuilder.swift

@@ -55,6 +55,14 @@ public class CreateTableBuilder {
         self.appendAndReturn(builder: CreateColumnBuilder(grammar: self.grammar, name: column, type: .int))
     }
 
+    /// Adds a big int column.
+    ///
+    /// - Parameter column: The name of the column to add.
+    /// - Returns: A builder for adding modifiers to the column.
+    @discardableResult public func bigInt(_ column: String) -> CreateColumnBuilder<Int> {
+        self.appendAndReturn(builder: CreateColumnBuilder(grammar: self.grammar, name: column, type: .bigInt))
+    }
+
     /// Adds a `Double` column.
     ///
     /// - Parameter column: The name of the column to add.
@@ -172,17 +180,5 @@ public struct CreateColumn {
     let type: ColumnType
 
     /// Any constraints.
-    let constraints: [String]
-
-    /// Convert this `CreateColumn` to a `String` for inserting into
-    /// an SQL statement.
-    ///
-    /// - Returns: The SQL `String` describing this column.
-    func toSQL(with grammar: Grammar) -> String {
-        var baseSQL = "\(self.column) \(grammar.typeString(for: self.type))"
-        if !self.constraints.isEmpty {
-            baseSQL.append(" \(self.constraints.joined(separator: " "))")
-        }
-        return baseSQL
-    }
+    let constraints: [ColumnConstraint]
 }