fixed issue #71, upgrate mysql driver to 5.1.35

alibaba · Jun 30, 2015 · c227cbd · c227cbd
1 parent 716dffd
commit c227cbd
Show file tree

Hide file tree

Showing 8 changed files with 64 additions and 15 deletions.
diff --git a/manager/biz/src/main/java/com/alibaba/otter/manager/biz/common/DataSourceCreator.java b/manager/biz/src/main/java/com/alibaba/otter/manager/biz/common/DataSourceCreator.java
@@ -17,6 +17,7 @@
 package com.alibaba.otter.manager.biz.common;
 
 import java.sql.SQLException;
+import java.util.Arrays;
 import java.util.List;
 
 import javax.sql.DataSource;
@@ -71,8 +72,12 @@ public DataSource createDataSource(DataMediaSource dataMediaSource) {
             return customDataSource;
         }
 
-        return createDataSource(dbMediaSource.getUrl(), dbMediaSource.getUsername(), dbMediaSource.getPassword(),
-                                dbMediaSource.getDriver(), dbMediaSource.getType(), dbMediaSource.getEncode());
+        return createDataSource(dbMediaSource.getUrl(),
+            dbMediaSource.getUsername(),
+            dbMediaSource.getPassword(),
+            dbMediaSource.getDriver(),
+            dbMediaSource.getType(),
+            dbMediaSource.getEncode());
     }
 
     public void destroyDataSource(DataSource dataSource) {
@@ -166,9 +171,17 @@ private DataSource createDataSource(String url, String userName, String password
         } else if (dataMediaType.isMysql()) {
             // open the batch mode for mysql since 5.1.8
             dbcpDs.addConnectionProperty("useServerPrepStmts", "false");
-            dbcpDs.addConnectionProperty("rewriteBatchedStatements", "false");
+            dbcpDs.addConnectionProperty("rewriteBatchedStatements", "true");
+            dbcpDs.addConnectionProperty("zeroDateTimeBehavior", "convertToNull");// 将0000-00-00的时间类型返回null
+            dbcpDs.addConnectionProperty("yearIsDateType", "false");// 直接返回字符串，不做year转换date处理
+            dbcpDs.addConnectionProperty("noDatetimeStringSync", "true");// 返回时间类型的字符串,不做时区处理
             if (StringUtils.isNotEmpty(encoding)) {
-                dbcpDs.addConnectionProperty("characterEncoding", encoding);
+                if (StringUtils.equalsIgnoreCase(encoding, "utf8mb4")) {
+                    dbcpDs.addConnectionProperty("characterEncoding", "utf8");
+                    dbcpDs.setConnectionInitSqls(Arrays.asList("set names utf8mb4"));
+                } else {
+                    dbcpDs.addConnectionProperty("characterEncoding", encoding);
+                }
             }
             // dbcpDs.setValidationQuery("select 1");
         } else {

diff --git a/manager/biz/src/main/java/com/alibaba/otter/manager/biz/utils/DataSourceChecker.java b/manager/biz/src/main/java/com/alibaba/otter/manager/biz/utils/DataSourceChecker.java
@@ -147,6 +147,7 @@ public String check(String url, String username, String password, String encode,
             try {
                 conn = dataSource.getConnection();
             } catch (Exception e) {
+                logger.error("check error!", e);
             }
 
             if (null == conn) {

diff --git a/manager/deployer/src/main/bin/stop.sh b/manager/deployer/src/main/bin/stop.sh
@@ -1,10 +1,14 @@
 #!/bin/bash
 
 cygwin=false;
+linux=false;
 case "`uname`" in
     CYGWIN*)
         cygwin=true
         ;;
+    Linux*)
+    	linux=true
+    	;;
 esac
 
 get_pid() {	
@@ -15,11 +19,19 @@ get_pid() {
         JAVA_CMD=`cygpath --path --unix $JAVA_CMD`
         JAVA_PID=`ps |grep $JAVA_CMD |awk '{print $1}'`
     else
-        if [ ! -z "$PID" ]; then
-        	JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep "$PID"|grep -v grep|awk '{print $2}'`
-	    else 
-	        JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep -v grep|awk '{print $2}'`
-        fi
+        if $linux; then
+	        if [ ! -z "$PID" ]; then
+	        	JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep "$PID"|grep -v grep|awk '{print $2}'`
+		    else 
+		        JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep -v grep|awk '{print $2}'`
+	        fi
+	    else
+	    	if [ ! -z "$PID" ]; then
+	        	JAVA_PID=`ps aux |grep "$STR"|grep "$PID"|grep -v grep|awk '{print $2}'`
+		    else 
+		        JAVA_PID=`ps aux |grep "$STR"|grep -v grep|awk '{print $2}'`
+	        fi
+	    fi
     fi
     echo $JAVA_PID;
 }

diff --git a/manager/deployer/src/main/resources/webapp/templates/home/screen/addDataSource.vm b/manager/deployer/src/main/resources/webapp/templates/home/screen/addDataSource.vm
@@ -84,6 +84,7 @@ $control.setTemplate("home:navigation.vm")
 			<select id="sourceEncode" name="$dataMediaSourceGroup.encode.key"  style="width:200px;" > 
 				<option value="GBK" #if($!dataMediaSourceGroup.encode.value == 'GBK') selected  #end>GBK</option>
 				<option value="UTF8" #if($!dataMediaSourceGroup.encode.value == 'UTF8') selected  #end>UTF8</option>
+				<option value="UTF8MB4" #if($!dataMediaSourceGroup.encode.value == 'UTF8MB4') selected  #end>UTF8MB4</option>
 				<option value="ISO-8859-1" #if($!dataMediaSourceGroup.encode.value == 'ISO-8859-1') selected  #end>ISO-8859-1</option>
             </select><span class="red">*</span>
 		</td>

diff --git a/manager/deployer/src/main/resources/webapp/templates/home/screen/editDataSource.vm b/manager/deployer/src/main/resources/webapp/templates/home/screen/editDataSource.vm
@@ -88,6 +88,7 @@ $control.setTemplate("home:navigation.vm")
 			<select id="sourceEncode" name="$dataMediaSourceGroup.encode.key"  style="width:200px;" > 
 				<option value="GBK" #if($!source.encode == 'GBK') selected  #end>GBK</option>
 				<option value="UTF8" #if($!source.encode == 'UTF8') selected  #end>UTF8</option>
+				<option value="UTF8MB4" #if($!source.encode == 'UTF8MB4') selected  #end>UTF8MB4</option>
 				<option value="ISO-8859-1" #if($!source.encode == 'ISO-8859-1') selected  #end>ISO-8859-1</option>
             </select><span class="red">*</span>
 		</td>

diff --git a/node/deployer/src/main/bin/stop.sh b/node/deployer/src/main/bin/stop.sh
@@ -1,10 +1,14 @@
 #!/bin/bash
 
 cygwin=false;
+linux=false;
 case "`uname`" in
     CYGWIN*)
         cygwin=true
         ;;
+    Linux*)
+    	linux=true
+    	;;
 esac
 
 get_pid() {	
@@ -15,11 +19,19 @@ get_pid() {
         JAVA_CMD=`cygpath --path --unix $JAVA_CMD`
         JAVA_PID=`ps |grep $JAVA_CMD |awk '{print $1}'`
     else
-        if [ ! -z "$PID" ]; then
-        	JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep "$PID"|grep -v grep|awk '{print $2}'`
-	    else 
-	        JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep -v grep|awk '{print $2}'`
-        fi
+        if $linux; then
+	        if [ ! -z "$PID" ]; then
+	        	JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep "$PID"|grep -v grep|awk '{print $2}'`
+		    else 
+		        JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep -v grep|awk '{print $2}'`
+	        fi
+	    else
+	    	if [ ! -z "$PID" ]; then
+	        	JAVA_PID=`ps aux |grep "$STR"|grep "$PID"|grep -v grep|awk '{print $2}'`
+		    else 
+		        JAVA_PID=`ps aux |grep "$STR"|grep -v grep|awk '{print $2}'`
+	        fi
+	    fi
     fi
     echo $JAVA_PID;
 }

diff --git a/.../src/main/java/com/alibaba/otter/node/etl/common/datasource/impl/DBDataSourceService.java b/.../src/main/java/com/alibaba/otter/node/etl/common/datasource/impl/DBDataSourceService.java
@@ -17,6 +17,7 @@
 package com.alibaba.otter.node.etl.common.datasource.impl;
 
 import java.sql.SQLException;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 
@@ -218,7 +219,12 @@ private DataSource createDataSource(String url, String userName, String password
             dbcpDs.addConnectionProperty("yearIsDateType", "false");// 直接返回字符串，不做year转换date处理
             dbcpDs.addConnectionProperty("noDatetimeStringSync", "true");// 返回时间类型的字符串,不做时区处理
             if (StringUtils.isNotEmpty(encoding)) {
-                dbcpDs.addConnectionProperty("characterEncoding", encoding);
+                if (StringUtils.equalsIgnoreCase(encoding, "utf8mb4")) {
+                    dbcpDs.addConnectionProperty("characterEncoding", "utf8");
+                    dbcpDs.setConnectionInitSqls(Arrays.asList("set names utf8mb4"));
+                } else {
+                    dbcpDs.addConnectionProperty("characterEncoding", encoding);
+                }
             }
             dbcpDs.setValidationQuery("select 1");
         } else {

diff --git a/node/etl/src/main/java/com/alibaba/otter/node/etl/select/selector/MessageParser.java b/node/etl/src/main/java/com/alibaba/otter/node/etl/select/selector/MessageParser.java
@@ -474,6 +474,9 @@ private EventData internParse(Pipeline pipeline, Entry entry, RowChange rowChang
             for (Column column : beforeColumns) {
                 if (isKey(tableHolder, tableName, column)) {
                     oldKeyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
+                    // 同时记录一下new
+                    // key,因为mysql5.6之后出现了minimal模式,after里会没有主键信息,需要在before记录中找
+                    keyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
                 } else {
                     if (needAllColumns && entry.getHeader().getSourceType() == CanalEntry.Type.ORACLE) {
                         // 针对行记录同步时，针对oracle记录一下非主键的字段，因为update时针对未变更的字段在aftercolume里没有