From c227cbd986f63d08046b8af8a5ac397a47e3ea55 Mon Sep 17 00:00:00 2001 From: agapple Date: Tue, 30 Jun 2015 14:29:27 +0800 Subject: [PATCH] fixed issue #71, upgrate mysql driver to 5.1.35 --- .../manager/biz/common/DataSourceCreator.java | 21 ++++++++++++++---- .../manager/biz/utils/DataSourceChecker.java | 1 + manager/deployer/src/main/bin/stop.sh | 22 ++++++++++++++----- .../templates/home/screen/addDataSource.vm | 1 + .../templates/home/screen/editDataSource.vm | 1 + node/deployer/src/main/bin/stop.sh | 22 ++++++++++++++----- .../datasource/impl/DBDataSourceService.java | 8 ++++++- .../etl/select/selector/MessageParser.java | 3 +++ 8 files changed, 64 insertions(+), 15 deletions(-) diff --git a/manager/biz/src/main/java/com/alibaba/otter/manager/biz/common/DataSourceCreator.java b/manager/biz/src/main/java/com/alibaba/otter/manager/biz/common/DataSourceCreator.java index aecf976f..2ff9b3a5 100644 --- a/manager/biz/src/main/java/com/alibaba/otter/manager/biz/common/DataSourceCreator.java +++ b/manager/biz/src/main/java/com/alibaba/otter/manager/biz/common/DataSourceCreator.java @@ -17,6 +17,7 @@ package com.alibaba.otter.manager.biz.common; import java.sql.SQLException; +import java.util.Arrays; import java.util.List; import javax.sql.DataSource; @@ -71,8 +72,12 @@ public DataSource createDataSource(DataMediaSource dataMediaSource) { return customDataSource; } - return createDataSource(dbMediaSource.getUrl(), dbMediaSource.getUsername(), dbMediaSource.getPassword(), - dbMediaSource.getDriver(), dbMediaSource.getType(), dbMediaSource.getEncode()); + return createDataSource(dbMediaSource.getUrl(), + dbMediaSource.getUsername(), + dbMediaSource.getPassword(), + dbMediaSource.getDriver(), + dbMediaSource.getType(), + dbMediaSource.getEncode()); } public void destroyDataSource(DataSource dataSource) { @@ -166,9 +171,17 @@ private DataSource createDataSource(String url, String userName, String password } else if (dataMediaType.isMysql()) { // open the batch mode for mysql since 5.1.8 dbcpDs.addConnectionProperty("useServerPrepStmts", "false"); - dbcpDs.addConnectionProperty("rewriteBatchedStatements", "false"); + dbcpDs.addConnectionProperty("rewriteBatchedStatements", "true"); + dbcpDs.addConnectionProperty("zeroDateTimeBehavior", "convertToNull");// 将0000-00-00的时间类型返回null + dbcpDs.addConnectionProperty("yearIsDateType", "false");// 直接返回字符串，不做year转换date处理 + dbcpDs.addConnectionProperty("noDatetimeStringSync", "true");// 返回时间类型的字符串,不做时区处理 if (StringUtils.isNotEmpty(encoding)) { - dbcpDs.addConnectionProperty("characterEncoding", encoding); + if (StringUtils.equalsIgnoreCase(encoding, "utf8mb4")) { + dbcpDs.addConnectionProperty("characterEncoding", "utf8"); + dbcpDs.setConnectionInitSqls(Arrays.asList("set names utf8mb4")); + } else { + dbcpDs.addConnectionProperty("characterEncoding", encoding); + } } // dbcpDs.setValidationQuery("select 1"); } else { diff --git a/manager/biz/src/main/java/com/alibaba/otter/manager/biz/utils/DataSourceChecker.java b/manager/biz/src/main/java/com/alibaba/otter/manager/biz/utils/DataSourceChecker.java index 0e6cc0a5..f6d090c6 100644 --- a/manager/biz/src/main/java/com/alibaba/otter/manager/biz/utils/DataSourceChecker.java +++ b/manager/biz/src/main/java/com/alibaba/otter/manager/biz/utils/DataSourceChecker.java @@ -147,6 +147,7 @@ public String check(String url, String username, String password, String encode, try { conn = dataSource.getConnection(); } catch (Exception e) { + logger.error("check error!", e); } if (null == conn) { diff --git a/manager/deployer/src/main/bin/stop.sh b/manager/deployer/src/main/bin/stop.sh index 082116ac..a97547f6 100644 --- a/manager/deployer/src/main/bin/stop.sh +++ b/manager/deployer/src/main/bin/stop.sh @@ -1,10 +1,14 @@ #!/bin/bash cygwin=false; +linux=false; case "`uname`" in CYGWIN*) cygwin=true ;; + Linux*) + linux=true + ;; esac get_pid() { @@ -15,11 +19,19 @@ get_pid() { JAVA_CMD=`cygpath --path --unix $JAVA_CMD` JAVA_PID=`ps |grep $JAVA_CMD |awk '{print $1}'` else - if [ ! -z "$PID" ]; then - JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep "$PID"|grep -v grep|awk '{print $2}'` - else - JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep -v grep|awk '{print $2}'` - fi + if $linux; then + if [ ! -z "$PID" ]; then + JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep "$PID"|grep -v grep|awk '{print $2}'` + else + JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep -v grep|awk '{print $2}'` + fi + else + if [ ! -z "$PID" ]; then + JAVA_PID=`ps aux |grep "$STR"|grep "$PID"|grep -v grep|awk '{print $2}'` + else + JAVA_PID=`ps aux |grep "$STR"|grep -v grep|awk '{print $2}'` + fi + fi fi echo $JAVA_PID; } diff --git a/manager/deployer/src/main/resources/webapp/templates/home/screen/addDataSource.vm b/manager/deployer/src/main/resources/webapp/templates/home/screen/addDataSource.vm index 22ec06a0..b51a59ab 100644 --- a/manager/deployer/src/main/resources/webapp/templates/home/screen/addDataSource.vm +++ b/manager/deployer/src/main/resources/webapp/templates/home/screen/addDataSource.vm @@ -84,6 +84,7 @@ $control.setTemplate("home:navigation.vm") * diff --git a/manager/deployer/src/main/resources/webapp/templates/home/screen/editDataSource.vm b/manager/deployer/src/main/resources/webapp/templates/home/screen/editDataSource.vm index c4c977b2..f4ef9808 100644 --- a/manager/deployer/src/main/resources/webapp/templates/home/screen/editDataSource.vm +++ b/manager/deployer/src/main/resources/webapp/templates/home/screen/editDataSource.vm @@ -88,6 +88,7 @@ $control.setTemplate("home:navigation.vm") * diff --git a/node/deployer/src/main/bin/stop.sh b/node/deployer/src/main/bin/stop.sh index 1882e56d..538c7fc0 100644 --- a/node/deployer/src/main/bin/stop.sh +++ b/node/deployer/src/main/bin/stop.sh @@ -1,10 +1,14 @@ #!/bin/bash cygwin=false; +linux=false; case "`uname`" in CYGWIN*) cygwin=true ;; + Linux*) + linux=true + ;; esac get_pid() { @@ -15,11 +19,19 @@ get_pid() { JAVA_CMD=`cygpath --path --unix $JAVA_CMD` JAVA_PID=`ps |grep $JAVA_CMD |awk '{print $1}'` else - if [ ! -z "$PID" ]; then - JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep "$PID"|grep -v grep|awk '{print $2}'` - else - JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep -v grep|awk '{print $2}'` - fi + if $linux; then + if [ ! -z "$PID" ]; then + JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep "$PID"|grep -v grep|awk '{print $2}'` + else + JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep -v grep|awk '{print $2}'` + fi + else + if [ ! -z "$PID" ]; then + JAVA_PID=`ps aux |grep "$STR"|grep "$PID"|grep -v grep|awk '{print $2}'` + else + JAVA_PID=`ps aux |grep "$STR"|grep -v grep|awk '{print $2}'` + fi + fi fi echo $JAVA_PID; } diff --git a/node/etl/src/main/java/com/alibaba/otter/node/etl/common/datasource/impl/DBDataSourceService.java b/node/etl/src/main/java/com/alibaba/otter/node/etl/common/datasource/impl/DBDataSourceService.java index 92785d1f..1686a0cd 100644 --- a/node/etl/src/main/java/com/alibaba/otter/node/etl/common/datasource/impl/DBDataSourceService.java +++ b/node/etl/src/main/java/com/alibaba/otter/node/etl/common/datasource/impl/DBDataSourceService.java @@ -17,6 +17,7 @@ package com.alibaba.otter.node.etl.common.datasource.impl; import java.sql.SQLException; +import java.util.Arrays; import java.util.List; import java.util.Map; @@ -218,7 +219,12 @@ private DataSource createDataSource(String url, String userName, String password dbcpDs.addConnectionProperty("yearIsDateType", "false");// 直接返回字符串，不做year转换date处理 dbcpDs.addConnectionProperty("noDatetimeStringSync", "true");// 返回时间类型的字符串,不做时区处理 if (StringUtils.isNotEmpty(encoding)) { - dbcpDs.addConnectionProperty("characterEncoding", encoding); + if (StringUtils.equalsIgnoreCase(encoding, "utf8mb4")) { + dbcpDs.addConnectionProperty("characterEncoding", "utf8"); + dbcpDs.setConnectionInitSqls(Arrays.asList("set names utf8mb4")); + } else { + dbcpDs.addConnectionProperty("characterEncoding", encoding); + } } dbcpDs.setValidationQuery("select 1"); } else { diff --git a/node/etl/src/main/java/com/alibaba/otter/node/etl/select/selector/MessageParser.java b/node/etl/src/main/java/com/alibaba/otter/node/etl/select/selector/MessageParser.java index eb512156..c8a24176 100644 --- a/node/etl/src/main/java/com/alibaba/otter/node/etl/select/selector/MessageParser.java +++ b/node/etl/src/main/java/com/alibaba/otter/node/etl/select/selector/MessageParser.java @@ -474,6 +474,9 @@ private EventData internParse(Pipeline pipeline, Entry entry, RowChange rowChang for (Column column : beforeColumns) { if (isKey(tableHolder, tableName, column)) { oldKeyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder)); + // 同时记录一下new + // key,因为mysql5.6之后出现了minimal模式,after里会没有主键信息,需要在before记录中找 + keyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder)); } else { if (needAllColumns && entry.getHeader().getSourceType() == CanalEntry.Type.ORACLE) { // 针对行记录同步时，针对oracle记录一下非主键的字段，因为update时针对未变更的字段在aftercolume里没有