Merge pull request #307 from Shopify/update-go-mysql

Update go mysql to address a very rare data corruption problem

binlog_streamer.go

@@ -9,8 +9,8 @@ import (
 
 	sql "github.com/Shopify/ghostferry/sqlwrapper"
 
-	"github.com/siddontang/go-mysql/mysql"
-	"github.com/siddontang/go-mysql/replication"
+	"github.com/go-mysql-org/go-mysql/mysql"
+	"github.com/go-mysql-org/go-mysql/replication"
 	"github.com/sirupsen/logrus"
 )
 
@@ -459,7 +459,6 @@ func idsOnServer(db *sql.DB) ([]uint32, error) {
 
 		columns, err = rows.Columns()
 
-
 		// SHOW SLAVE HOSTS has a different return value for different implementations
 		// i.e MySQL/Percona have 5 columns as it includes slave_uuid for MariaDB slave_uuid is omitted
 		// since all other values are not used check for the amount of columns and gather only what is possible

compression_verifier.go

@@ -6,13 +6,14 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
-	sql "github.com/Shopify/ghostferry/sqlwrapper"
 	"strconv"
 	"strings"
 
+	sql "github.com/Shopify/ghostferry/sqlwrapper"
+
 	sq "github.com/Masterminds/squirrel"
+	"github.com/go-mysql-org/go-mysql/schema"
 	"github.com/golang/snappy"
-	"github.com/siddontang/go-mysql/schema"
 	"github.com/sirupsen/logrus"
 )
 

copydb/test/filter_test.go

@@ -9,7 +9,7 @@ import (
 	"github.com/Shopify/ghostferry/testhelpers"
 	"github.com/stretchr/testify/suite"
 
-	sqlSchema "github.com/siddontang/go-mysql/schema"
+	sqlSchema "github.com/go-mysql-org/go-mysql/schema"
 )
 
 type FilterTestSuite struct {

cursor.go

@@ -8,7 +8,7 @@ import (
 	sql "github.com/Shopify/ghostferry/sqlwrapper"
 
 	"github.com/Masterminds/squirrel"
-	"github.com/siddontang/go-mysql/schema"
+	"github.com/go-mysql-org/go-mysql/schema"
 	"github.com/sirupsen/logrus"
 )
 

dml_events.go

@@ -10,9 +10,9 @@ import (
 
 	"github.com/shopspring/decimal"
 
-	"github.com/siddontang/go-mysql/mysql"
-	"github.com/siddontang/go-mysql/replication"
-	"github.com/siddontang/go-mysql/schema"
+	"github.com/go-mysql-org/go-mysql/mysql"
+	"github.com/go-mysql-org/go-mysql/replication"
+	"github.com/go-mysql-org/go-mysql/schema"
 )
 
 var annotationRegex = regexp.MustCompile(`^/\*(.*?)\*/`)
@@ -36,17 +36,17 @@ type RowData []interface{}
 //
 // At some point, this code was refactored into this function, such that the
 // BinlogStreamer also uses the same code to decode integers. The binlog data is
-// given to us by siddontang/go-mysql. The siddontang/go-mysql library should
+// given to us by go-mysql-org/go-mysql. The go-mysql-org/go-mysql library should
 // not be giving us awkward byte slices. Instead, it should properly gives us
 // uint64. This code thus panics when it encounters such case. See
 // https://github.com/Shopify/ghostferry/issues/165.
 //
 // In summary:
 // - This code receives values from both go-sql-driver/mysql and
-//   siddontang/go-mysql.
+//   go-mysql-org/go-mysql.
 // - go-sql-driver/mysql gives us int64 for signed integer, and uint64 in a byte
 //   slice for unsigned integer.
-// - siddontang/go-mysql gives us int64 for signed integer, and uint64 for
+// - go-mysql-org/go-mysql gives us int64 for signed integer, and uint64 for
 //   unsigned integer.
 // - We currently make this function deal with both cases. In the future we can
 //   investigate alternative solutions.

examples/copydb/conf.json

@@ -37,12 +37,13 @@
 
   "DumpStateOnSignal": true,
 
-  "VerifierType": "ChecksumTable",
+  "VerifierType": "Inline",
 
   "ControlServerCustomScripts": {
     "Custom Script 1": ["examples/copydb/s1"],
     "Custom Script 2": ["examples/copydb/s2"]
   },
 
-  "DumpStateToStdoutOnError": true
+  "DumpStateToStdoutOnError": true,
+  "SkipTargetVerification": true
 }

ferry.go

@@ -17,9 +17,9 @@ import (
 
 	sql "github.com/Shopify/ghostferry/sqlwrapper"
 
+	siddontangmysql "github.com/go-mysql-org/go-mysql/mysql"
 	"github.com/go-sql-driver/mysql"
 	siddontanglog "github.com/siddontang/go-log/log"
-	siddontangmysql "github.com/siddontang/go-mysql/mysql"
 	"github.com/sirupsen/logrus"
 )
 
@@ -337,7 +337,7 @@ func (f *Ferry) Initialize() (err error) {
 		}
 	}
 
-	// Suppress siddontang/go-mysql logging as we already log the equivalents.
+	// Suppress go-mysql-org/go-mysql logging as we already log the equivalents.
 	// It also by defaults logs to stdout, which is different from Ghostferry
 	// logging, which all goes to stderr. stdout in Ghostferry is reserved for
 	// dumping states due to an abort.

go.mod

@@ -5,11 +5,11 @@ go 1.14
 require (
 	github.com/Masterminds/squirrel v0.0.0-20180620232226-b127ed9be034
 	github.com/Shopify/go-dogstatsd v0.0.0-20150311181532-76f9377ffde2
-	github.com/go-sql-driver/mysql v1.4.1
+	github.com/go-mysql-org/go-mysql v1.3.0
+	github.com/go-sql-driver/mysql v1.5.0
 	github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db
 	github.com/gorilla/context v1.1.1 // indirect
 	github.com/gorilla/mux v1.6.1
-	github.com/jmoiron/sqlx v1.2.0 // indirect
 	github.com/lann/builder v0.0.0-20180216234317-1b87b36280d0 // indirect
 	github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
 	github.com/lib/pq v1.3.0 // indirect
@@ -19,11 +19,9 @@ require (
 	github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712 // indirect
 	github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24
 	github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07
-	github.com/siddontang/go-mysql v0.0.0-20200424072754-803944a6e4ea
 	github.com/sirupsen/logrus v1.8.1
 	github.com/stretchr/testify v1.4.0
 	golang.org/x/sys v0.0.0-20210426230700-d19ff857e887 // indirect
-	google.golang.org/appengine v1.6.5 // indirect
 	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
 	gopkg.in/yaml.v2 v2.2.4 // indirect
 )