Merge pull request #207 from zendesk/ben/ucs2

support UCS2 character encoding
zendesk · Dec 16, 2015 · 5610a4f · 5610a4f
2 parents b5764bb + f517253
commit 5610a4f
Show file tree

Hide file tree

Showing 4 changed files with 29 additions and 6 deletions.
diff --git a/src/main/java/com/zendesk/maxwell/schema/columndef/StringColumnDef.java b/src/main/java/com/zendesk/maxwell/schema/columndef/StringColumnDef.java
@@ -8,7 +8,11 @@
 
 import com.google.code.or.common.util.MySQLConstants;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 public class StringColumnDef extends ColumnDef {
+	static final Logger LOGGER = LoggerFactory.getLogger(StringColumnDef.class);
 	public StringColumnDef(String tableName, String name, String type, int pos, String encoding) {
 		super(tableName, name, type, pos);
 		this.encoding = encoding;
@@ -39,12 +43,15 @@ public String toSQL(Object value) {
 
 	// this could obviously be more complete.
 	private Charset charsetForEncoding() {
-		switch(encoding) {
+		switch(encoding.toLowerCase()) {
 		case "utf8": case "utf8mb4":
 			return Charset.forName("UTF-8");
 		case "latin1": case "ascii":
 			return Charset.forName("ISO-8859-1");
+		case "ucs2":
+			return Charset.forName("UTF-16");
 		default:
+			LOGGER.warn("warning: unhandled character set '" + encoding + "'");
 			return null;
 		}
 	}

diff --git a/src/test/java/com/zendesk/maxwell/MaxwellIntegrationTest.java b/src/test/java/com/zendesk/maxwell/MaxwellIntegrationTest.java
@@ -353,4 +353,9 @@ public void testTime() throws Exception {
 		if ( server.getVersion().equals("5.6") )
 			runJSONTestFile(getSQLDir() + "/json/test_time");
 	}
+
+	@Test
+	public void testUCS2() throws Exception {
+		runJSONTestFile(getSQLDir() + "/json/test_ucs2");
+	}
 }
diff --git a/src/test/java/com/zendesk/maxwell/MysqlIsolatedServer.java b/src/test/java/com/zendesk/maxwell/MysqlIsolatedServer.java
@@ -8,15 +8,12 @@
 
 import java.io.BufferedReader;
 import java.io.IOException;
-import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.SQLException;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
-import java.util.Map;
 
 public class MysqlIsolatedServer {
 	public static final Long SERVER_ID = 123123L;
@@ -25,11 +22,20 @@ public class MysqlIsolatedServer {
 	private int serverPid;
 
 	static final Logger LOGGER = LoggerFactory.getLogger(MysqlIsolatedServer.class);
+
 	public void boot() throws IOException, SQLException, InterruptedException {
         final String dir = System.getProperty("user.dir");
 
-		ProcessBuilder pb = new ProcessBuilder(dir + "/src/test/onetimeserver", "--mysql-version=" + this.getVersion(),
-												"--log-bin=master", "--binlog_format=row", "--innodb_flush_log_at_trx_commit=1", "--server_id=" + SERVER_ID);
+		ProcessBuilder pb = new ProcessBuilder(
+				dir + "/src/test/onetimeserver",
+				"--mysql-version=" + this.getVersion(),
+				"--log-bin=master",
+				"--binlog_format=row",
+				"--innodb_flush_log_at_trx_commit=1",
+				"--server_id=" + SERVER_ID,
+				"--character-set-server=utf8"
+		);
+
 		LOGGER.debug("booting onetimeserver: " + StringUtils.join(pb.command(), " "));
 		Process p = pb.start();
 		BufferedReader reader = new BufferedReader(new InputStreamReader(p.getInputStream()));

diff --git a/src/test/resources/sql/json/test_ucs2 b/src/test/resources/sql/json/test_ucs2
@@ -0,0 +1,5 @@
+CREATE DATABASE `test_ucs2` default charset 'ucs2'
+create table `test_ucs2`.`ucs2` ( textcol text )
+insert into `test_ucs2`.`ucs2` set `textcol` = '∆˚ø¨´˚∆ƒ¬ß∂µ≤˜π∑ˆ¨´øßå¬∂˚∆ƒ…¬å˚∆ß';
+-> {"database": "test_ucs2", "table": "ucs2", "type": "insert", "data": {"textcol": "∆˚ø¨´˚∆ƒ¬ß∂µ≤˜π∑ˆ¨´øßå¬∂˚∆ƒ…¬å˚∆ß"} }
+