模块化mq connector (#2562)

* add connector module, add kafka producer connector module for server

* add rocketMQ producer connector module for server

* add rabbitMQ producer connector module for server

* optimize code, add annotation

* add canal message consumer connector for canal-adapter module

client-adapter/common/pom.xml

@@ -1,5 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
         <artifactId>canal.client-adapter</artifactId>
         <groupId>com.alibaba.otter</groupId>
@@ -17,6 +18,11 @@
             <artifactId>canal.protocol</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>connector.core</artifactId>
+            <version>${project.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.springframework</groupId>
             <artifactId>spring-core</artifactId>
@@ -49,7 +55,7 @@
         <dependency>
             <groupId>com.alibaba</groupId>
             <artifactId>druid</artifactId>
-        </dependency>        
+        </dependency>
         <dependency>
             <groupId>org.yaml</groupId>
             <artifactId>snakeyaml</artifactId>

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/CanalClientConfig.java

@@ -3,6 +3,7 @@ package com.alibaba.otter.canal.client.adapter.support;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Properties;
 
 /**
  * 配置信息类
@@ -27,7 +28,7 @@ public class CanalClientConfig {
     // 重试次数
     private Integer            retries;
     // 消费超时时间
-    private Long               timeout;
+    private Long               timeout       = 500L;
     // 模式 tcp kafka rocketMQ
     private String             mode          = "tcp";
     // aliyun ak/sk
@@ -39,6 +40,8 @@ public class CanalClientConfig {
     // rabbitmq vhost
     private String             vhost         = "/";
 
+    private Properties         consumerProperties;
+
     private Long               resourceOwnerId;
     // 是否启用消息轨迹
     private boolean            enableMessageTrace;
@@ -163,6 +166,14 @@ public class CanalClientConfig {
         this.vhost = vhost;
     }
 
+    public Properties getConsumerProperties() {
+        return consumerProperties;
+    }
+
+    public void setConsumerProperties(Properties consumerProperties) {
+        this.consumerProperties = consumerProperties;
+    }
+
     public Long getResourceOwnerId() {
         return resourceOwnerId;
     }

client-adapter/common/src/main/java/com/alibaba/otter/canal/client/adapter/support/MessageUtil.java

@@ -2,6 +2,7 @@ package com.alibaba.otter.canal.client.adapter.support;
 
 import java.util.*;
 
+import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
 import com.alibaba.otter.canal.protocol.CanalEntry;
 import com.alibaba.otter.canal.protocol.FlatMessage;
 import com.alibaba.otter.canal.protocol.Message;
@@ -131,10 +132,13 @@ public class MessageUtil {
         return dmls;
     }
 
-    public static List<Dml> flatMessage2Dml(String destination, String groupId, List<FlatMessage> flatMessages) {
-        List<Dml> dmls = new ArrayList<Dml>(flatMessages.size());
-        for (FlatMessage flatMessage : flatMessages) {
-            Dml dml = flatMessage2Dml(destination, groupId, flatMessage);
+    public static List<Dml> flatMessage2Dml(String destination, String groupId, List<CommonMessage> commonMessages) {
+        if (commonMessages == null) {
+            return new ArrayList<>();
+        }
+        List<Dml> dmls = new ArrayList<Dml>(commonMessages.size());
+        for (CommonMessage commonMessage : commonMessages) {
+            Dml dml = flatMessage2Dml(destination, groupId, commonMessage);
             if (dml != null) {
                 dmls.add(dml);
             }
@@ -143,31 +147,31 @@ public class MessageUtil {
         return dmls;
     }
 
-    public static Dml flatMessage2Dml(String destination, String groupId, FlatMessage flatMessage) {
-        if (flatMessage == null) {
+    public static Dml flatMessage2Dml(String destination, String groupId, CommonMessage commonMessage) {
+        if (commonMessage == null) {
             return null;
         }
         Dml dml = new Dml();
         dml.setDestination(destination);
         dml.setGroupId(groupId);
-        dml.setDatabase(flatMessage.getDatabase());
-        dml.setTable(flatMessage.getTable());
-        dml.setPkNames(flatMessage.getPkNames());
-        dml.setIsDdl(flatMessage.getIsDdl());
-        dml.setType(flatMessage.getType());
-        dml.setTs(flatMessage.getTs());
-        dml.setEs(flatMessage.getEs());
-        dml.setSql(flatMessage.getSql());
+        dml.setDatabase(commonMessage.getDatabase());
+        dml.setTable(commonMessage.getTable());
+        dml.setPkNames(commonMessage.getPkNames());
+        dml.setIsDdl(commonMessage.getIsDdl());
+        dml.setType(commonMessage.getType());
+        dml.setTs(commonMessage.getTs());
+        dml.setEs(commonMessage.getEs());
+        dml.setSql(commonMessage.getSql());
         // if (flatMessage.getSqlType() == null || flatMessage.getMysqlType() == null) {
         // throw new RuntimeException("SqlType or mysqlType is null");
         // }
-        List<Map<String, String>> data = flatMessage.getData();
+        List<Map<String, Object>> data = commonMessage.getData();
         if (data != null) {
-            dml.setData(changeRows(dml.getTable(), data, flatMessage.getSqlType(), flatMessage.getMysqlType()));
+            dml.setData(data);
         }
-        List<Map<String, String>> old = flatMessage.getOld();
+        List<Map<String, Object>> old = commonMessage.getOld();
         if (old != null) {
-            dml.setOld(changeRows(dml.getTable(), old, flatMessage.getSqlType(), flatMessage.getMysqlType()));
+            dml.setOld(old);
         }
         return dml;
     }

client-adapter/launcher/pom.xml

@@ -30,11 +30,6 @@
             <artifactId>client-adapter.common</artifactId>
             <version>${project.version}</version>
         </dependency>
-        <dependency>
-            <groupId>com.alibaba.otter</groupId>
-            <artifactId>canal.client</artifactId>
-            <version>${project.version}</version>
-        </dependency>
         <dependency>
             <groupId>org.yaml</groupId>
             <artifactId>snakeyaml</artifactId>
@@ -56,25 +51,6 @@
             <groupId>org.apache.curator</groupId>
             <artifactId>curator-recipes</artifactId>
         </dependency>
-        <!-- 单独引入rocketmq依赖 -->
-        <dependency>
-            <groupId>org.apache.rocketmq</groupId>
-            <artifactId>rocketmq-client</artifactId>
-        </dependency>
-        <!-- 单独引入kafka依赖 -->
-        <dependency>
-            <groupId>org.apache.kafka</groupId>
-            <artifactId>kafka-clients</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>com.rabbitmq</groupId>
-            <artifactId>amqp-client</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>com.alibaba.mq-amqp</groupId>
-            <artifactId>mq-amqp-client</artifactId>
-        </dependency>
         <!-- jdbc -->
         <dependency>
             <groupId>mysql</groupId>
@@ -159,6 +135,59 @@
             <classifier>jar-with-dependencies</classifier>
             <scope>provided</scope>
         </dependency>
+        <!-- connector plugin -->
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>connector.tcp</artifactId>
+            <version>${project.version}</version>
+            <exclusions>
+                <exclusion>
+                    <artifactId>*</artifactId>
+                    <groupId>*</groupId>
+                </exclusion>
+            </exclusions>
+            <classifier>jar-with-dependencies</classifier>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>connector.kafka</artifactId>
+            <version>${project.version}</version>
+            <exclusions>
+                <exclusion>
+                    <artifactId>*</artifactId>
+                    <groupId>*</groupId>
+                </exclusion>
+            </exclusions>
+            <classifier>jar-with-dependencies</classifier>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>connector.rocketmq</artifactId>
+            <version>${project.version}</version>
+            <exclusions>
+                <exclusion>
+                    <artifactId>*</artifactId>
+                    <groupId>*</groupId>
+                </exclusion>
+            </exclusions>
+            <classifier>jar-with-dependencies</classifier>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>connector.rabbitmq</artifactId>
+            <version>${project.version}</version>
+            <exclusions>
+                <exclusion>
+                    <artifactId>*</artifactId>
+                    <groupId>*</groupId>
+                </exclusion>
+            </exclusions>
+            <classifier>jar-with-dependencies</classifier>
+            <scope>provided</scope>
+        </dependency>
     </dependencies>
 
     <build>

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AbstractCanalAdapterWorker.java

@@ -1,263 +0,0 @@
-package com.alibaba.otter.canal.adapter.launcher.loader;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.*;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.alibaba.otter.canal.adapter.launcher.common.SyncSwitch;
-import com.alibaba.otter.canal.adapter.launcher.config.SpringContext;
-import com.alibaba.otter.canal.client.CanalMQConnector;
-import com.alibaba.otter.canal.client.adapter.OuterAdapter;
-import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
-import com.alibaba.otter.canal.client.adapter.support.Dml;
-import com.alibaba.otter.canal.client.adapter.support.MessageUtil;
-import com.alibaba.otter.canal.client.adapter.support.Util;
-import com.alibaba.otter.canal.protocol.FlatMessage;
-import com.alibaba.otter.canal.protocol.Message;
-
-/**
- * 适配器工作线程抽象类
- *
- * @author rewerma 2018-8-19 下午11:30:49
- * @version 1.0.0
- */
-public abstract class AbstractCanalAdapterWorker {
-
-    protected final Logger                    logger  = LoggerFactory.getLogger(this.getClass());
-
-    protected String                          canalDestination;                                                // canal实例
-    protected String                          groupId = null;                                                  // groupId
-    protected List<List<OuterAdapter>>        canalOuterAdapters;                                              // 外部适配器
-    protected CanalClientConfig               canalClientConfig;                                               // 配置
-    protected ExecutorService                 groupInnerExecutorService;                                       // 组内工作线程池
-    protected volatile boolean                running = false;                                                 // 是否运行中
-    protected Thread                          thread  = null;
-    protected Thread.UncaughtExceptionHandler handler = (t, e) -> logger.error("parse events has an error", e);
-
-    protected SyncSwitch                      syncSwitch;
-
-    public AbstractCanalAdapterWorker(List<List<OuterAdapter>> canalOuterAdapters){
-        this.canalOuterAdapters = canalOuterAdapters;
-        this.groupInnerExecutorService = Util.newFixedThreadPool(canalOuterAdapters.size(), 5000L);
-        syncSwitch = (SyncSwitch) SpringContext.getBean(SyncSwitch.class);
-    }
-
-    protected void writeOut(final Message message) {
-        List<Future<Boolean>> futures = new ArrayList<>();
-        // 组间适配器并行运行
-        canalOuterAdapters.forEach(outerAdapters -> {
-            final List<OuterAdapter> adapters = outerAdapters;
-            futures.add(groupInnerExecutorService.submit(() -> {
-                try {
-                    // 组内适配器穿行运行，尽量不要配置组内适配器
-                    adapters.forEach(adapter -> {
-                        long begin = System.currentTimeMillis();
-                        List<Dml> dmls = MessageUtil.parse4Dml(canalDestination, groupId, message);
-                        if (dmls != null) {
-                            batchSync(dmls, adapter);
-
-                            if (logger.isDebugEnabled()) {
-                                logger.debug("{} elapsed time: {}",
-                                    adapter.getClass().getName(),
-                                    (System.currentTimeMillis() - begin));
-                            }
-                        }
-                    });
-                    return true;
-                } catch (Exception e) {
-                    logger.error(e.getMessage(), e);
-                    return false;
-                }
-            }));
-
-            // 等待所有适配器写入完成
-            // 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成
-            RuntimeException exception = null;
-            for (Future<Boolean> future : futures) {
-                try {
-                    if (!future.get()) {
-                        exception = new RuntimeException("Outer adapter sync failed! ");
-                    }
-                } catch (Exception e) {
-                    exception = new RuntimeException(e);
-                }
-            }
-            if (exception != null) {
-                throw exception;
-            }
-        });
-    }
-
-    protected void writeOut(final List<FlatMessage> flatMessages) {
-        List<Future<Boolean>> futures = new ArrayList<>();
-        // 组间适配器并行运行
-        canalOuterAdapters.forEach(outerAdapters -> {
-            futures.add(groupInnerExecutorService.submit(() -> {
-                try {
-                    // 组内适配器穿行运行，尽量不要配置组内适配器
-                    outerAdapters.forEach(adapter -> {
-                        long begin = System.currentTimeMillis();
-                        List<Dml> dmls = MessageUtil.flatMessage2Dml(canalDestination, groupId, flatMessages);
-                        batchSync(dmls, adapter);
-
-                        if (logger.isDebugEnabled()) {
-                            logger.debug("{} elapsed time: {}",
-                                adapter.getClass().getName(),
-                                (System.currentTimeMillis() - begin));
-                        }
-                    });
-                    return true;
-                } catch (Exception e) {
-                    logger.error(e.getMessage(), e);
-                    return false;
-                }
-            }));
-
-            // 等待所有适配器写入完成
-            // 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成
-            RuntimeException exception = null;
-            for (Future<Boolean> future : futures) {
-                try {
-                    if (!future.get()) {
-                        exception = new RuntimeException("Outer adapter sync failed! ");
-                    }
-                } catch (Exception e) {
-                    exception = new RuntimeException(e);
-                }
-            }
-            if (exception != null) {
-                throw exception;
-            }
-        });
-    }
-
-    @SuppressWarnings("unchecked")
-    protected boolean mqWriteOutData(int retry, long timeout, int i, final boolean flatMessage,
-                                     CanalMQConnector connector) {
-        ExecutorService workerExecutor =  Util.newSingleDaemonThreadExecutor(5000);
-        try {
-            List<?> messages;
-            if (!flatMessage) {
-                messages = connector.getListWithoutAck(100L, TimeUnit.MILLISECONDS);
-            } else {
-                messages = connector.getFlatListWithoutAck(100L, TimeUnit.MILLISECONDS);
-            }
-            if (messages != null && !messages.isEmpty()) {
-                Future<Boolean> future = workerExecutor.submit(() -> {
-                    if (flatMessage) {
-                        // batch write
-                        writeOut((List<FlatMessage>) messages);
-                        // FIXME xxx
-                        // messages.forEach((message ->
-                        // System.out.println(JSON.toJSONString(message))));
-                    } else {
-                        for (final Object message : messages) {
-                            writeOut((Message) message);
-                        }
-                    }
-                    return true;
-                });
-
-                try {
-                    future.get(timeout, TimeUnit.MILLISECONDS);
-                } catch (Exception e) {
-                    future.cancel(true);
-                    throw e;
-                }
-                connector.ack();
-            }
-            return true;
-        } catch (Throwable e) {
-            if (i == retry - 1) {
-                connector.ack();
-                logger.error(e.getMessage() + " Error sync but ACK!");
-                return true;
-            } else {
-                connector.rollback();
-                logger.error(e.getMessage() + " Error sync and rollback, execute times: " + (i + 1));
-            }
-            try {
-                Thread.sleep(500);
-            } catch (InterruptedException e1) {
-                // ignore
-            }
-        } finally {
-            workerExecutor.shutdown();
-        }
-        return false;
-    }
-
-    /**
-     * 分批同步
-     *
-     * @param dmls
-     * @param adapter
-     */
-    private void batchSync(List<Dml> dmls, OuterAdapter adapter) {
-        // 分批同步
-        if (dmls.size() <= canalClientConfig.getSyncBatchSize()) {
-            adapter.sync(dmls);
-        } else {
-            int len = 0;
-            List<Dml> dmlsBatch = new ArrayList<>();
-            for (Dml dml : dmls) {
-                dmlsBatch.add(dml);
-                if (dml.getData() == null || dml.getData().isEmpty()) {
-                    len += 1;
-                } else {
-                    len += dml.getData().size();
-                }
-                if (len >= canalClientConfig.getSyncBatchSize()) {
-                    adapter.sync(dmlsBatch);
-                    dmlsBatch.clear();
-                    len = 0;
-                }
-            }
-            if (!dmlsBatch.isEmpty()) {
-                adapter.sync(dmlsBatch);
-            }
-        }
-    }
-
-    public void start() {
-        if (!running) {
-            thread = new Thread(this::process);
-            thread.setUncaughtExceptionHandler(handler);
-            thread.start();
-            running = true;
-        }
-    }
-
-    protected abstract void process();
-
-    public void stop() {
-        try {
-            if (!running) {
-                return;
-            }
-
-            running = false;
-
-            syncSwitch.release(canalDestination);
-
-            logger.info("destination {} is waiting for adapters' worker thread die!", canalDestination);
-            if (thread != null) {
-                try {
-                    thread.join();
-                } catch (InterruptedException e) {
-                    // ignore
-                }
-            }
-            groupInnerExecutorService.shutdown();
-            logger.info("destination {} adapters worker thread dead!", canalDestination);
-            canalOuterAdapters.forEach(outerAdapters -> outerAdapters.forEach(OuterAdapter::destroy));
-            logger.info("destination {} all adapters destroyed!", canalDestination);
-        } catch (Exception e) {
-            logger.error(e.getMessage(), e);
-        }
-    }
-
-}

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/AdapterProcessor.java

@@ -0,0 +1,268 @@
+package com.alibaba.otter.canal.adapter.launcher.loader;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import com.alibaba.otter.canal.adapter.launcher.config.SpringContext;
+import com.alibaba.otter.canal.client.adapter.support.Util;
+import com.alibaba.otter.canal.connector.core.config.CanalConstants;
+import com.alibaba.otter.canal.connector.core.spi.ExtensionLoader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.adapter.launcher.common.SyncSwitch;
+import com.alibaba.otter.canal.client.adapter.OuterAdapter;
+import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
+import com.alibaba.otter.canal.client.adapter.support.Dml;
+import com.alibaba.otter.canal.client.adapter.support.MessageUtil;
+import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
+import com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer;
+
+/**
+ * 适配处理器
+ * 
+ * @author rewerma 2020-02-01
+ * @version 1.0.0
+ */
+public class AdapterProcessor {
+
+    private static final Logger             logger                    = LoggerFactory.getLogger(AdapterProcessor.class);
+
+    private static final String             CONNECTOR_SPI_DIR         = "/plugin";
+    private static final String             CONNECTOR_STANDBY_SPI_DIR = "/canal-adapter/plugin";
+
+    private CanalMsgConsumer                canalMsgConsumer;
+
+    private String                          canalDestination;                                                           // canal实例
+    private String                          groupId                   = null;                                           // groupId
+    private List<List<OuterAdapter>>        canalOuterAdapters;                                                         // 外部适配器
+    private CanalClientConfig               canalClientConfig;                                                          // 配置
+    private ExecutorService                 groupInnerExecutorService;                                                  // 组内工作线程池
+    private volatile boolean                running                   = false;                                          // 是否运行中
+    private Thread                          thread                    = null;
+    private Thread.UncaughtExceptionHandler handler                   = (t, e) -> logger
+        .error("parse events has an error", e);
+
+    private SyncSwitch                      syncSwitch;
+
+    public AdapterProcessor(CanalClientConfig canalClientConfig, String destination, String groupId,
+                            List<List<OuterAdapter>> canalOuterAdapters){
+        this.canalClientConfig = canalClientConfig;
+        this.canalDestination = destination;
+        this.groupId = groupId;
+        this.canalOuterAdapters = canalOuterAdapters;
+
+        this.groupInnerExecutorService = Util.newFixedThreadPool(canalOuterAdapters.size(), 5000L);
+        syncSwitch = (SyncSwitch) SpringContext.getBean(SyncSwitch.class);
+
+        // load connector consumer
+        ExtensionLoader<CanalMsgConsumer> loader = new ExtensionLoader<>(CanalMsgConsumer.class);
+        canalMsgConsumer = loader
+            .getExtension(canalClientConfig.getMode().toLowerCase(), CONNECTOR_SPI_DIR, CONNECTOR_STANDBY_SPI_DIR);
+
+        Properties properties = canalClientConfig.getConsumerProperties();
+        properties.put(CanalConstants.CANAL_MQ_FLAT_MESSAGE, canalClientConfig.getFlatMessage());
+        properties.put(CanalConstants.CANAL_ALIYUN_ACCESS_KEY, canalClientConfig.getAccessKey());
+        properties.put(CanalConstants.CANAL_ALIYUN_SECRET_KEY, canalClientConfig.getSecretKey());
+        ClassLoader cl = Thread.currentThread().getContextClassLoader();
+        Thread.currentThread().setContextClassLoader(canalMsgConsumer.getClass().getClassLoader());
+        canalMsgConsumer.init(properties, canalDestination, groupId);
+        Thread.currentThread().setContextClassLoader(cl);
+    }
+
+    public void start() {
+        if (!running) {
+            thread = new Thread(this::process);
+            thread.setUncaughtExceptionHandler(handler);
+            thread.start();
+            running = true;
+        }
+    }
+
+    public void writeOut(final List<CommonMessage> commonMessages) {
+        List<Future<Boolean>> futures = new ArrayList<>();
+        // 组间适配器并行运行
+        canalOuterAdapters.forEach(outerAdapters -> {
+            futures.add(groupInnerExecutorService.submit(() -> {
+                try {
+                    // 组内适配器穿行运行，尽量不要配置组内适配器
+                    outerAdapters.forEach(adapter -> {
+                        long begin = System.currentTimeMillis();
+                        List<Dml> dmls = MessageUtil.flatMessage2Dml(canalDestination, groupId, commonMessages);
+                        batchSync(dmls, adapter);
+
+                        if (logger.isDebugEnabled()) {
+                            logger.debug("{} elapsed time: {}",
+                                adapter.getClass().getName(),
+                                (System.currentTimeMillis() - begin));
+                        }
+                    });
+                    return true;
+                } catch (Exception e) {
+                    logger.error(e.getMessage(), e);
+                    return false;
+                }
+            }));
+
+            // 等待所有适配器写入完成
+            // 由于是组间并发操作，所以将阻塞直到耗时最久的工作组操作完成
+            RuntimeException exception = null;
+            for (Future<Boolean> future : futures) {
+                try {
+                    if (!future.get()) {
+                        exception = new RuntimeException("Outer adapter sync failed! ");
+                    }
+                } catch (Exception e) {
+                    exception = new RuntimeException(e);
+                }
+            }
+            if (exception != null) {
+                throw exception;
+            }
+        });
+    }
+
+    /**
+     * 分批同步
+     *
+     * @param dmls
+     * @param adapter
+     */
+    private void batchSync(List<Dml> dmls, OuterAdapter adapter) {
+        // 分批同步
+        if (dmls.size() <= canalClientConfig.getSyncBatchSize()) {
+            adapter.sync(dmls);
+        } else {
+            int len = 0;
+            List<Dml> dmlsBatch = new ArrayList<>();
+            for (Dml dml : dmls) {
+                dmlsBatch.add(dml);
+                if (dml.getData() == null || dml.getData().isEmpty()) {
+                    len += 1;
+                } else {
+                    len += dml.getData().size();
+                }
+                if (len >= canalClientConfig.getSyncBatchSize()) {
+                    adapter.sync(dmlsBatch);
+                    dmlsBatch.clear();
+                    len = 0;
+                }
+            }
+            if (!dmlsBatch.isEmpty()) {
+                adapter.sync(dmlsBatch);
+            }
+        }
+    }
+
+    private void process() {
+        while (!running) { // waiting until running == true
+            while (!running) {
+                try {
+                    Thread.sleep(1000);
+                } catch (InterruptedException e) {
+                }
+            }
+        }
+
+        int retry = canalClientConfig.getRetries() == null
+                    || canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
+        if (retry == -1) {
+            // 重试次数-1代表异常时一直阻塞重试
+            retry = Integer.MAX_VALUE;
+        }
+
+        while (running) {
+            try {
+                syncSwitch.get(canalDestination);
+
+                logger.info("=============> Start to connect destination: {} <=============", this.canalDestination);
+                canalMsgConsumer.connect();
+                logger.info("=============> Subscribe destination: {} succeed <=============", this.canalDestination);
+                while (running) {
+                    try {
+                        syncSwitch.get(canalDestination, 1L, TimeUnit.MINUTES);
+                    } catch (TimeoutException e) {
+                        break;
+                    }
+                    if (!running) {
+                        break;
+                    }
+
+                    for (int i = 0; i < retry; i++) {
+                        if (!running) {
+                            break;
+                        }
+                        try {
+                            if (logger.isDebugEnabled()) {
+                                logger.debug("destination: {} ", canalDestination);
+                            }
+                            long begin = System.currentTimeMillis();
+                            List<CommonMessage> commonMessages = canalMsgConsumer
+                                .getMessage(this.canalClientConfig.getTimeout(), TimeUnit.MILLISECONDS);
+                            writeOut(commonMessages);
+                            canalMsgConsumer.ack();
+                            if (logger.isDebugEnabled()) {
+                                logger.debug("destination: {} elapsed time: {} ms",
+                                    canalDestination,
+                                    System.currentTimeMillis() - begin);
+                            }
+                        } catch (Exception e) {
+                            if (i != retry - 1) {
+                                canalMsgConsumer.rollback(); // 处理失败, 回滚数据
+                                logger.error(e.getMessage() + " Error sync and rollback, execute times: " + (i + 1));
+                            } else {
+                                canalMsgConsumer.ack();
+                                logger.error(e.getMessage() + " Error sync but ACK!");
+                            }
+                            Thread.sleep(500);
+                        }
+                    }
+                }
+
+                canalMsgConsumer.disconnect();
+            } catch (Throwable e) {
+                logger.error("process error!", e);
+            }
+
+            if (running) { // is reconnect
+                try {
+                    Thread.sleep(1000);
+                } catch (InterruptedException e) {
+                    // ignore
+                }
+            }
+        }
+    }
+
+    public void stop() {
+        try {
+            if (!running) {
+                return;
+            }
+
+            running = false;
+
+            syncSwitch.release(canalDestination);
+
+            logger.info("destination {} is waiting for adapters' worker thread die!", canalDestination);
+            if (thread != null) {
+                try {
+                    thread.join();
+                } catch (InterruptedException e) {
+                    // ignore
+                }
+            }
+            groupInnerExecutorService.shutdown();
+            logger.info("destination {} adapters worker thread dead!", canalDestination);
+            canalOuterAdapters.forEach(outerAdapters -> outerAdapters.forEach(OuterAdapter::destroy));
+            logger.info("destination {} all adapters destroyed!", canalDestination);
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        }
+    }
+}

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterKafkaWorker.java

@@ -1,96 +0,0 @@
-package com.alibaba.otter.canal.adapter.launcher.loader;
-
-import java.util.List;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.kafka.common.errors.WakeupException;
-
-import com.alibaba.otter.canal.client.adapter.OuterAdapter;
-import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
-import com.alibaba.otter.canal.client.adapter.support.Util;
-import com.alibaba.otter.canal.client.kafka.KafkaCanalConnector;
-
-/**
- * kafka对应的client适配器工作线程
- *
- * @author rewerma 2018-8-19 下午11:30:49
- * @version 1.0.0
- */
-public class CanalAdapterKafkaWorker extends AbstractCanalAdapterWorker {
-
-    private KafkaCanalConnector connector;
-    private String              topic;
-    private boolean             flatMessage;
-
-    public CanalAdapterKafkaWorker(CanalClientConfig canalClientConfig, String bootstrapServers, String topic,
-                                   String groupId, List<List<OuterAdapter>> canalOuterAdapters, boolean flatMessage){
-        super(canalOuterAdapters);
-        this.canalClientConfig = canalClientConfig;
-        this.topic = topic;
-        super.canalDestination = topic;
-        super.groupId = groupId;
-        this.flatMessage = flatMessage;
-        this.connector = new KafkaCanalConnector(bootstrapServers,
-            topic,
-            null,
-            groupId,
-            canalClientConfig.getBatchSize(),
-            flatMessage);
-        connector.setSessionTimeout(30L, TimeUnit.SECONDS);
-    }
-
-    @Override
-    protected void process() {
-        while (!running) {
-            try {
-                Thread.sleep(500);
-            } catch (InterruptedException e) {
-                // ignore
-            }
-        }
-        int retry = canalClientConfig.getRetries() == null
-                    || canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
-        long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒
-
-        while (running) {
-            try {
-                syncSwitch.get(canalDestination);
-                logger.info("=============> Start to connect topic: {} <=============", this.topic);
-                connector.connect();
-                logger.info("=============> Start to subscribe topic: {} <=============", this.topic);
-                connector.subscribe();
-                logger.info("=============> Subscribe topic: {} succeed <=============", this.topic);
-                while (running) {
-                    boolean status = syncSwitch.status(canalDestination);
-                    if (!status) {
-                        connector.disconnect();
-                        break;
-                    }
-                    if (retry == -1) {
-                        retry = Integer.MAX_VALUE;
-                    }
-                    for (int i = 0; i < retry; i++) {
-                        if (!running) {
-                            break;
-                        }
-                        if (mqWriteOutData(retry, timeout, i, flatMessage, connector)) {
-                            break;
-                        }
-                    }
-                }
-            } catch (Exception e) {
-                logger.error(e.getMessage(), e);
-            }
-        }
-
-
-        try {
-            connector.unsubscribe();
-        } catch (WakeupException e) {
-            // No-op. Continue process
-        }
-        connector.disconnect();
-        logger.info("=============> Disconnect topic: {} <=============", this.topic);
-    }
-}

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterLoader.java

@@ -37,9 +37,7 @@ public class CanalAdapterLoader {
 
     private CanalClientConfig                       canalClientConfig;
 
-    private Map<String, CanalAdapterWorker>         canalWorkers  = new HashMap<>();
-
-    private Map<String, AbstractCanalAdapterWorker> canalMQWorker = new HashMap<>();
+    private Map<String, AdapterProcessor>           canalAdapterProcessors = new HashMap<>();
 
     private ExtensionLoader<OuterAdapter>           loader;
 
@@ -53,120 +51,132 @@ public class CanalAdapterLoader {
     public void init() {
         loader = ExtensionLoader.getExtensionLoader(OuterAdapter.class);
 
-        String canalServerHost = this.canalClientConfig.getCanalServerHost();
-        SocketAddress sa = null;
-        if (canalServerHost != null) {
-            String[] ipPort = canalServerHost.split(":");
-            sa = new InetSocketAddress(ipPort[0], Integer.parseInt(ipPort[1]));
-        }
-        String zkHosts = this.canalClientConfig.getZookeeperHosts();
-
-        if ("tcp".equalsIgnoreCase(canalClientConfig.getMode())) {
-            // 初始化canal-client的适配器
-            for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
+        for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
+            for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
                 List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>();
-
-                for (CanalClientConfig.Group connectorGroup : canalAdapter.getGroups()) {
-                    List<OuterAdapter> canalOutConnectors = new CopyOnWriteArrayList<>();
-                    for (OuterAdapterConfig c : connectorGroup.getOuterAdapters()) {
-                        loadAdapter(c, canalOutConnectors);
-                    }
-                    canalOuterAdapterGroups.add(canalOutConnectors);
-                }
-                CanalAdapterWorker worker;
-                if (sa != null) {
-                    worker = new CanalAdapterWorker(canalClientConfig,
-                        canalAdapter.getInstance(),
-                        sa,
-                        canalOuterAdapterGroups);
-                } else if (zkHosts != null) {
-                    worker = new CanalAdapterWorker(canalClientConfig,
-                        canalAdapter.getInstance(),
-                        zkHosts,
-                        canalOuterAdapterGroups);
-                } else {
-                    throw new RuntimeException("No canal server connector found");
+                List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
+                for (OuterAdapterConfig config : group.getOuterAdapters()) {
+                    loadAdapter(config, canalOuterAdapters);
                 }
-                canalWorkers.put(canalAdapter.getInstance(), worker);
-                worker.start();
-                logger.info("Start adapter for canal instance: {} succeed", canalAdapter.getInstance());
-            }
-        } else if ("kafka".equalsIgnoreCase(canalClientConfig.getMode())) {
-            // 初始化canal-client-kafka的适配器
-            for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
-                for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
-                    List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>();
-                    List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
-                    for (OuterAdapterConfig config : group.getOuterAdapters()) {
-                        loadAdapter(config, canalOuterAdapters);
-                    }
-                    canalOuterAdapterGroups.add(canalOuterAdapters);
-
-                    CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(canalClientConfig,
-                        canalClientConfig.getMqServers(),
-                        canalAdapter.getInstance(),
-                        group.getGroupId(),
-                        canalOuterAdapterGroups,
-                        canalClientConfig.getFlatMessage());
-                    canalMQWorker.put(canalAdapter.getInstance() + "-kafka-" + group.getGroupId(), canalKafkaWorker);
-                    canalKafkaWorker.start();
-                    logger.info("Start adapter for canal-client mq topic: {} succeed",
-                        canalAdapter.getInstance() + "-" + group.getGroupId());
-                }
-            }
-        } else if ("rocketMQ".equalsIgnoreCase(canalClientConfig.getMode())) {
-            // 初始化canal-client-rocketMQ的适配器
-            for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
-                for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
-                    List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>();
-                    List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
-                    for (OuterAdapterConfig config : group.getOuterAdapters()) {
-                        loadAdapter(config, canalOuterAdapters);
-                    }
-                    canalOuterAdapterGroups.add(canalOuterAdapters);
-                    CanalAdapterRocketMQWorker rocketMQWorker = new CanalAdapterRocketMQWorker(canalClientConfig,
-                        canalClientConfig.getMqServers(),
-                        canalAdapter.getInstance(),
-                        group.getGroupId(),
-                        canalOuterAdapterGroups,
-                        canalClientConfig.getAccessKey(),
-                        canalClientConfig.getSecretKey(),
-                        canalClientConfig.getFlatMessage(),
-                        canalClientConfig.isEnableMessageTrace(),
-                        canalClientConfig.getCustomizedTraceTopic(),
-                        canalClientConfig.getAccessChannel(),
-                        canalClientConfig.getNamespace());
-                    canalMQWorker.put(canalAdapter.getInstance() + "-rocketmq-" + group.getGroupId(), rocketMQWorker);
-                    rocketMQWorker.start();
-
-                    logger.info("Start adapter for canal-client mq topic: {} succeed",
-                        canalAdapter.getInstance() + "-" + group.getGroupId());
-                }
-            }
-        } else if ("rabbitMQ".equalsIgnoreCase(canalClientConfig.getMode())) {
-            // 初始化canal-client-rabbitMQ的适配器
-            for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
-                for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
-                    List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>();
-                    List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
-                    for (OuterAdapterConfig config : group.getOuterAdapters()) {
-                        loadAdapter(config, canalOuterAdapters);
-                    }
-                    canalOuterAdapterGroups.add(canalOuterAdapters);
-                    CanalAdapterRabbitMQWorker rabbitMQWork = new CanalAdapterRabbitMQWorker(canalClientConfig,
-                        canalOuterAdapterGroups,
-                        canalAdapter.getInstance(),
-                        group.getGroupId(),
-                        canalClientConfig.getFlatMessage());
-                    canalMQWorker.put(canalAdapter.getInstance() + "-rabbitmq-" + group.getGroupId(), rabbitMQWork);
-                    rabbitMQWork.start();
-
-                    logger.info("Start adapter for canal-client mq topic: {} succeed",
+                canalOuterAdapterGroups.add(canalOuterAdapters);
+
+                AdapterProcessor adapterProcessor = canalAdapterProcessors.computeIfAbsent(canalAdapter.getInstance()+"|"+StringUtils.trimToEmpty(group.getGroupId()),
+                        f-> new AdapterProcessor(canalClientConfig,canalAdapter.getInstance(),
+                                group.getGroupId(), canalOuterAdapterGroups));
+                adapterProcessor.start();
+
+                logger.info("Start adapter for canal-client mq topic: {} succeed",
                         canalAdapter.getInstance() + "-" + group.getGroupId());
-                }
             }
-            // CanalAdapterRabbitMQWork
         }
+
+//        if ("tcp".equalsIgnoreCase(canalClientConfig.getMode())) {
+//            // 初始化canal-client的适配器
+//            for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
+//                List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>();
+//
+//                for (CanalClientConfig.Group connectorGroup : canalAdapter.getGroups()) {
+//                    List<OuterAdapter> canalOutConnectors = new CopyOnWriteArrayList<>();
+//                    for (OuterAdapterConfig c : connectorGroup.getOuterAdapters()) {
+//                        loadAdapter(c, canalOutConnectors);
+//                    }
+//                    canalOuterAdapterGroups.add(canalOutConnectors);
+//                }
+//                CanalAdapterWorker worker;
+//                if (StringUtils.isNotEmpty(canalServerHost)) {
+//                    worker = new CanalAdapterWorker(canalClientConfig,
+//                        canalAdapter.getInstance(),
+//                            canalServerHost,
+//                            zkHosts,
+//                        canalOuterAdapterGroups);
+//                } else if (zkHosts != null) {
+//                    worker = new CanalAdapterWorker(canalClientConfig,
+//                        canalAdapter.getInstance(),
+//                        zkHosts,
+//                        canalOuterAdapterGroups);
+//                } else {
+//                    throw new RuntimeException("No canal server connector found");
+//                }
+//                canalWorkers.put(canalAdapter.getInstance(), worker);
+//                worker.start();
+//                logger.info("Start adapter for canal instance: {} succeed", canalAdapter.getInstance());
+//            }
+//        } else if ("kafka".equalsIgnoreCase(canalClientConfig.getMode())) {
+//            // 初始化canal-client-kafka的适配器
+//            for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
+//                for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
+//                    List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>();
+//                    List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
+//                    for (OuterAdapterConfig config : group.getOuterAdapters()) {
+//                        loadAdapter(config, canalOuterAdapters);
+//                    }
+//                    canalOuterAdapterGroups.add(canalOuterAdapters);
+//
+//                    CanalAdapterKafkaWorker canalKafkaWorker = new CanalAdapterKafkaWorker(canalClientConfig,
+//                        canalClientConfig.getMqServers(),
+//                        canalAdapter.getInstance(),
+//                        group.getGroupId(),
+//                        canalOuterAdapterGroups,
+//                        canalClientConfig.getFlatMessage());
+//                    canalMQWorker.put(canalAdapter.getInstance() + "-kafka-" + group.getGroupId(), canalKafkaWorker);
+//                    canalKafkaWorker.start();
+//                    logger.info("Start adapter for canal-client mq topic: {} succeed",
+//                        canalAdapter.getInstance() + "-" + group.getGroupId());
+//                }
+//            }
+//        } else if ("rocketMQ".equalsIgnoreCase(canalClientConfig.getMode())) {
+//            // 初始化canal-client-rocketMQ的适配器
+//            for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
+//                for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
+//                    List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>();
+//                    List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
+//                    for (OuterAdapterConfig config : group.getOuterAdapters()) {
+//                        loadAdapter(config, canalOuterAdapters);
+//                    }
+//                    canalOuterAdapterGroups.add(canalOuterAdapters);
+//                    CanalAdapterRocketMQWorker rocketMQWorker = new CanalAdapterRocketMQWorker(canalClientConfig,
+//                        canalClientConfig.getMqServers(),
+//                        canalAdapter.getInstance(),
+//                        group.getGroupId(),
+//                        canalOuterAdapterGroups,
+//                        canalClientConfig.getAccessKey(),
+//                        canalClientConfig.getSecretKey(),
+//                        canalClientConfig.getFlatMessage(),
+//                        canalClientConfig.isEnableMessageTrace(),
+//                        canalClientConfig.getCustomizedTraceTopic(),
+//                        canalClientConfig.getAccessChannel(),
+//                        canalClientConfig.getNamespace());
+//                    canalMQWorker.put(canalAdapter.getInstance() + "-rocketmq-" + group.getGroupId(), rocketMQWorker);
+//                    rocketMQWorker.start();
+//
+//                    logger.info("Start adapter for canal-client mq topic: {} succeed",
+//                        canalAdapter.getInstance() + "-" + group.getGroupId());
+//                }
+//            }
+//        } else if ("rabbitMQ".equalsIgnoreCase(canalClientConfig.getMode())) {
+//            // 初始化canal-client-rabbitMQ的适配器
+//            for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
+//                for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
+//                    List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>();
+//                    List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
+//                    for (OuterAdapterConfig config : group.getOuterAdapters()) {
+//                        loadAdapter(config, canalOuterAdapters);
+//                    }
+//                    canalOuterAdapterGroups.add(canalOuterAdapters);
+//                    CanalAdapterRabbitMQWorker rabbitMQWork = new CanalAdapterRabbitMQWorker(canalClientConfig,
+//                        canalOuterAdapterGroups,
+//                        canalAdapter.getInstance(),
+//                        group.getGroupId(),
+//                        canalClientConfig.getFlatMessage());
+//                    canalMQWorker.put(canalAdapter.getInstance() + "-rabbitmq-" + group.getGroupId(), rabbitMQWork);
+//                    rabbitMQWork.start();
+//
+//                    logger.info("Start adapter for canal-client mq topic: {} succeed",
+//                        canalAdapter.getInstance() + "-" + group.getGroupId());
+//                }
+//            }
+//            // CanalAdapterRabbitMQWork
+//        }
     }
 
     private void loadAdapter(OuterAdapterConfig config, List<OuterAdapter> canalOutConnectors) {
@@ -206,10 +216,10 @@ public class CanalAdapterLoader {
      * 销毁所有适配器 为防止canal实例太多造成销毁阻塞, 并行销毁
      */
     public void destroy() {
-        if (!canalWorkers.isEmpty()) {
-            ExecutorService stopExecutorService = Executors.newFixedThreadPool(canalWorkers.size());
-            for (CanalAdapterWorker canalAdapterWorker : canalWorkers.values()) {
-                stopExecutorService.execute(canalAdapterWorker::stop);
+        if (!canalAdapterProcessors.isEmpty()) {
+            ExecutorService stopExecutorService = Executors.newFixedThreadPool(canalAdapterProcessors.size());
+            for (AdapterProcessor adapterProcessor : canalAdapterProcessors.values()) {
+                stopExecutorService.execute(adapterProcessor::stop);
             }
             stopExecutorService.shutdown();
             try {
@@ -220,21 +230,6 @@ public class CanalAdapterLoader {
                 // ignore
             }
         }
-
-        if (!canalMQWorker.isEmpty()) {
-            ExecutorService stopMQWorkerService = Executors.newFixedThreadPool(canalMQWorker.size());
-            for (AbstractCanalAdapterWorker canalAdapterMQWorker : canalMQWorker.values()) {
-                stopMQWorkerService.execute(canalAdapterMQWorker::stop);
-            }
-            stopMQWorkerService.shutdown();
-            try {
-                while (!stopMQWorkerService.awaitTermination(1, TimeUnit.SECONDS)) {
-                    // ignore
-                }
-            } catch (InterruptedException e) {
-                // ignore
-            }
-        }
         logger.info("All canal adapters destroyed");
     }
 }

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRabbitMQWorker.java

@@ -1,90 +0,0 @@
-package com.alibaba.otter.canal.adapter.launcher.loader;
-
-import com.alibaba.otter.canal.client.adapter.OuterAdapter;
-import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
-import com.alibaba.otter.canal.client.adapter.support.Util;
-import com.alibaba.otter.canal.client.rabbitmq.RabbitMQCanalConnector;
-import org.apache.kafka.common.errors.WakeupException;
-
-import java.util.List;
-import java.util.concurrent.ExecutorService;
-
-public class CanalAdapterRabbitMQWorker extends AbstractCanalAdapterWorker {
-
-    private RabbitMQCanalConnector connector;
-    private String                 topic;
-    private boolean                flatMessage;
-
-    public CanalAdapterRabbitMQWorker(CanalClientConfig canalClientConfig, List<List<OuterAdapter>> canalOuterAdapters,
-                                      String topic, String groupId, boolean flatMessage){
-        super(canalOuterAdapters);
-        this.canalClientConfig = canalClientConfig;
-        this.topic = topic;
-        this.flatMessage = flatMessage;
-        this.canalDestination = topic;
-        this.groupId = groupId;
-        this.connector = new RabbitMQCanalConnector(canalClientConfig.getMqServers(),
-            canalClientConfig.getVhost(),
-            topic,
-            canalClientConfig.getAccessKey(),
-            canalClientConfig.getSecretKey(),
-            canalClientConfig.getUsername(),
-            canalClientConfig.getPassword(),
-            canalClientConfig.getResourceOwnerId(),
-            flatMessage);
-    }
-
-    @Override
-    protected void process() {
-        while (!running) {
-            try {
-                Thread.sleep(1000);
-            } catch (InterruptedException ignored) {
-            }
-        }
-
-        int retry = canalClientConfig.getRetries() == null
-                    || canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
-        long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒
-        while (running) {
-            try {
-                syncSwitch.get(canalDestination);
-                logger.info("=============> Start to connect topic: {} <=============", this.topic);
-                connector.connect();
-                logger.info("=============> Start to subscribe topic: {}<=============", this.topic);
-                connector.subscribe();
-                logger.info("=============> Subscribe topic: {} succeed<=============", this.topic);
-                while (running) {
-                    boolean status = syncSwitch.status(canalDestination);
-                    if (!status) {
-                        connector.disconnect();
-                        break;
-                    }
-                    if (retry == -1) {
-                        retry = Integer.MAX_VALUE;
-                    }
-                    for (int i = 0; i < retry; i++) {
-                        if (!running) {
-                            break;
-                        }
-                        if (mqWriteOutData(retry, timeout, i, flatMessage, connector)) {
-                            break;
-                        }
-                    }
-                }
-            } catch (Exception e) {
-                logger.error(e.getMessage(), e);
-            }
-        }
-
-
-        try {
-            connector.unsubscribe();
-        } catch (WakeupException e) {
-            // No-op. Continue process
-        }
-        connector.disconnect();
-        logger.info("=============> Disconnect topic: {} <=============", this.topic);
-
-    }
-}

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterRocketMQWorker.java

@@ -1,119 +0,0 @@
-package com.alibaba.otter.canal.adapter.launcher.loader;
-
-import java.util.List;
-import java.util.concurrent.ExecutorService;
-
-import org.apache.kafka.common.errors.WakeupException;
-
-import com.alibaba.otter.canal.client.adapter.OuterAdapter;
-import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
-import com.alibaba.otter.canal.client.adapter.support.Util;
-import com.alibaba.otter.canal.client.rocketmq.RocketMQCanalConnector;
-
-/**
- * rocketmq对应的client适配器工作线程
- *
- * @version 1.0.0
- */
-public class CanalAdapterRocketMQWorker extends AbstractCanalAdapterWorker {
-
-    private RocketMQCanalConnector connector;
-    private String                 topic;
-    private boolean                flatMessage;
-
-    public CanalAdapterRocketMQWorker(CanalClientConfig canalClientConfig, String nameServers, String topic,
-                                      String groupId, List<List<OuterAdapter>> canalOuterAdapters, String accessKey,
-                                      String secretKey, boolean flatMessage){
-        super(canalOuterAdapters);
-        this.canalClientConfig = canalClientConfig;
-        this.topic = topic;
-        this.flatMessage = flatMessage;
-        super.canalDestination = topic;
-        super.groupId = groupId;
-        this.connector = new RocketMQCanalConnector(nameServers,
-            topic,
-            groupId,
-            accessKey,
-            secretKey,
-            canalClientConfig.getBatchSize(),
-            flatMessage);
-        logger.info("RocketMQ consumer config topic:{}, nameServer:{}, groupId:{}", topic, nameServers, groupId);
-    }
-
-    public CanalAdapterRocketMQWorker(CanalClientConfig canalClientConfig, String nameServers, String topic,
-        String groupId, List<List<OuterAdapter>> canalOuterAdapters, String accessKey,
-        String secretKey, boolean flatMessage, boolean enableMessageTrace,
-        String customizedTraceTopic, String accessChannel, String namespace) {
-        super(canalOuterAdapters);
-        this.canalClientConfig = canalClientConfig;
-        this.topic = topic;
-        this.flatMessage = flatMessage;
-        super.canalDestination = topic;
-        super.groupId = groupId;
-        this.connector = new RocketMQCanalConnector(nameServers,
-            topic,
-            groupId,
-            accessKey,
-            secretKey,
-            canalClientConfig.getBatchSize(),
-            flatMessage,
-            enableMessageTrace,
-            customizedTraceTopic,
-            accessChannel,
-            namespace);
-        logger.info("RocketMQ consumer config topic:{}, nameServer:{}, groupId:{}", topic, nameServers, groupId);
-    }
-
-    @Override
-    protected void process() {
-        while (!running) {
-            try {
-                Thread.sleep(1000);
-            } catch (InterruptedException e) {
-            }
-        }
-
-        int retry = canalClientConfig.getRetries() == null
-                    || canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
-        long timeout = canalClientConfig.getTimeout() == null ? 30000 : canalClientConfig.getTimeout(); // 默认超时30秒
-
-        while (running) {
-            try {
-                syncSwitch.get(canalDestination);
-                logger.info("=============> Start to connect topic: {} <=============", this.topic);
-                connector.connect();
-                logger.info("=============> Start to subscribe topic: {}<=============", this.topic);
-                connector.subscribe();
-                logger.info("=============> Subscribe topic: {} succeed<=============", this.topic);
-                while (running) {
-                    boolean status = syncSwitch.status(canalDestination);
-                    if (!status) {
-                        connector.disconnect();
-                        break;
-                    }
-                    if (retry == -1) {
-                        retry = Integer.MAX_VALUE;
-                    }
-                    for (int i = 0; i < retry; i++) {
-                        if (!running) {
-                            break;
-                        }
-                        if (mqWriteOutData(retry, timeout, i, flatMessage, connector)) {
-                            break;
-                        }
-                    }
-                }
-            } catch (Exception e) {
-                logger.error(e.getMessage(), e);
-            }
-        }
-
-        try {
-            connector.unsubscribe();
-        } catch (WakeupException e) {
-            // No-op. Continue process
-        }
-        connector.disconnect();
-        logger.info("=============> Disconnect topic: {} <=============", this.topic);
-    }
-}

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/loader/CanalAdapterWorker.java

@@ -1,193 +0,0 @@
-package com.alibaba.otter.canal.adapter.launcher.loader;
-
-import java.net.SocketAddress;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-import com.alibaba.otter.canal.client.CanalConnector;
-import com.alibaba.otter.canal.client.CanalConnectors;
-import com.alibaba.otter.canal.client.adapter.OuterAdapter;
-import com.alibaba.otter.canal.client.adapter.support.CanalClientConfig;
-import com.alibaba.otter.canal.client.impl.ClusterCanalConnector;
-import com.alibaba.otter.canal.client.impl.SimpleCanalConnector;
-import com.alibaba.otter.canal.protocol.Message;
-
-/**
- * 原生canal-server对应的client适配器工作线程
- *
- * @author rewrema 2018-8-19 下午11:30:49
- * @version 1.0.0
- */
-public class CanalAdapterWorker extends AbstractCanalAdapterWorker {
-
-    private static final int BATCH_SIZE = 50;
-    private static final int SO_TIMEOUT = 0;
-
-    private CanalConnector   connector;
-
-    /**
-     * 单台client适配器worker的构造方法
-     *
-     * @param canalDestination canal实例名
-     * @param address canal-server地址
-     * @param canalOuterAdapters 外部适配器组
-     */
-    public CanalAdapterWorker(CanalClientConfig canalClientConfig, String canalDestination, SocketAddress address,
-                              List<List<OuterAdapter>> canalOuterAdapters){
-        super(canalOuterAdapters);
-        this.canalClientConfig = canalClientConfig;
-        this.canalDestination = canalDestination;
-        connector = CanalConnectors.newSingleConnector(address, canalDestination, "", "");
-    }
-
-    /**
-     * HA模式下client适配器worker的构造方法
-     *
-     * @param canalDestination canal实例名
-     * @param zookeeperHosts zookeeper地址
-     * @param canalOuterAdapters 外部适配器组
-     */
-    public CanalAdapterWorker(CanalClientConfig canalClientConfig, String canalDestination, String zookeeperHosts,
-                              List<List<OuterAdapter>> canalOuterAdapters){
-        super(canalOuterAdapters);
-        this.canalDestination = canalDestination;
-        this.canalClientConfig = canalClientConfig;
-        connector = CanalConnectors.newClusterConnector(zookeeperHosts, canalDestination, "", "");
-        ((ClusterCanalConnector) connector).setSoTimeout(SO_TIMEOUT);
-    }
-
-    @Override
-    protected void process() {
-        while (!running) { // waiting until running == true
-            while (!running) {
-                try {
-                    Thread.sleep(1000);
-                } catch (InterruptedException e) {
-                }
-            }
-        }
-
-        int retry = canalClientConfig.getRetries() == null || canalClientConfig.getRetries() == 0 ? 1 : canalClientConfig.getRetries();
-        if (retry == -1) {
-            // 重试次数-1代表异常时一直阻塞重试
-            retry = Integer.MAX_VALUE;
-        }
-        // long timeout = canalClientConfig.getTimeout() == null ? 300000 :
-        // canalClientConfig.getTimeout(); // 默认超时5分钟
-        Integer batchSize = canalClientConfig.getBatchSize();
-        if (batchSize == null) {
-            batchSize = BATCH_SIZE;
-        }
-
-        while (running) {
-            try {
-                syncSwitch.get(canalDestination);
-
-                logger.info("=============> Start to connect destination: {} <=============", this.canalDestination);
-                connector.connect();
-                logger.info("=============> Start to subscribe destination: {} <=============", this.canalDestination);
-                connector.subscribe();
-                logger.info("=============> Subscribe destination: {} succeed <=============", this.canalDestination);
-                while (running) {
-                    try {
-                        syncSwitch.get(canalDestination, 1L, TimeUnit.MINUTES);
-                    } catch (TimeoutException e) {
-                        break;
-                    }
-                    if (!running) {
-                        break;
-                    }
-
-                    for (int i = 0; i < retry; i++) {
-                        if (!running) {
-                            break;
-                        }
-                        Message message = connector.getWithoutAck(batchSize); // 获取指定数量的数据
-                        long batchId = message.getId();
-                        try {
-                            int size = message.getEntries().size();
-                            if (batchId == -1 || size == 0) {
-                                Thread.sleep(500);
-                            } else {
-                                if (logger.isDebugEnabled()) {
-                                    logger.debug("destination: {} batchId: {} batchSize: {} ",
-                                        canalDestination,
-                                        batchId,
-                                        size);
-                                }
-                                long begin = System.currentTimeMillis();
-                                writeOut(message);
-                                if (logger.isDebugEnabled()) {
-                                    logger.debug("destination: {} batchId: {} elapsed time: {} ms",
-                                        canalDestination,
-                                        batchId,
-                                        System.currentTimeMillis() - begin);
-                                }
-                            }
-                            connector.ack(batchId); // 提交确认
-                            break;
-                        } catch (Exception e) {
-                            if (i != retry - 1) {
-                                connector.rollback(batchId); // 处理失败, 回滚数据
-                                logger.error(e.getMessage() + " Error sync and rollback, execute times: " + (i + 1));
-                            } else {
-                                connector.ack(batchId);
-                                logger.error(e.getMessage() + " Error sync but ACK!");
-                            }
-                            Thread.sleep(500);
-                        }
-                    }
-                }
-
-            } catch (Throwable e) {
-                logger.error("process error!", e);
-            } finally {
-                connector.disconnect();
-                logger.info("=============> Disconnect destination: {} <=============", this.canalDestination);
-            }
-
-            if (running) { // is reconnect
-                try {
-                    Thread.sleep(1000);
-                } catch (InterruptedException e) {
-                    // ignore
-                }
-            }
-        }
-    }
-
-    @Override
-    public void stop() {
-        try {
-            if (!running) {
-                return;
-            }
-
-            if (connector instanceof ClusterCanalConnector) {
-                ((ClusterCanalConnector) connector).stopRunning();
-            } else if (connector instanceof SimpleCanalConnector) {
-                ((SimpleCanalConnector) connector).stopRunning();
-            }
-
-            running = false;
-
-            syncSwitch.release(canalDestination);
-
-            logger.info("destination {} is waiting for adapters' worker thread die!", canalDestination);
-            if (thread != null) {
-                try {
-                    thread.join();
-                } catch (InterruptedException e) {
-                    // ignore
-                }
-            }
-            groupInnerExecutorService.shutdown();
-            logger.info("destination {} adapters worker thread dead!", canalDestination);
-            canalOuterAdapters.forEach(outerAdapters -> outerAdapters.forEach(OuterAdapter::destroy));
-            logger.info("destination {} all adapters destroyed!", canalDestination);
-        } catch (Exception e) {
-            logger.error(e.getMessage(), e);
-        }
-    }
-}

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/DbRemoteConfigLoader.java

@@ -15,14 +15,14 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 
+import com.alibaba.otter.canal.common.utils.CommonUtils;
+import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
 import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.alibaba.druid.pool.DruidDataSource;
 import com.alibaba.otter.canal.adapter.launcher.config.AdapterConfigHolder;
-import com.alibaba.otter.canal.common.utils.CommonUtils;
-import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
 import com.google.common.base.Joiner;
 
 /**

client-adapter/launcher/src/main/java/com/alibaba/otter/canal/adapter/launcher/monitor/remote/RemoteAdapterMonitorImpl.java

@@ -5,11 +5,10 @@ import java.io.FileOutputStream;
 import java.io.OutputStreamWriter;
 import java.nio.charset.StandardCharsets;
 
+import com.alibaba.otter.canal.common.utils.CommonUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.alibaba.otter.canal.common.utils.CommonUtils;
-
 /**
  * 远程配置监听器实现
  *

client-adapter/launcher/src/main/resources/application.yml

@@ -7,20 +7,45 @@ spring:
     default-property-inclusion: non_null
 
 canal.conf:
-  mode: tcp # kafka rocketMQ
-  canalServerHost: 127.0.0.1:11111
-#  zookeeperHosts: slave1:2181
-#  mqServers: 127.0.0.1:9092 #or rocketmq
-#  flatMessage: true
-  batchSize: 500
+  mode: tcp #tcp kafka rocketMQ rabbitMQ
+  flatMessage: true
+  zookeeperHosts:
   syncBatchSize: 1000
   retries: 0
   timeout:
   accessKey:
   secretKey:
-  username:
-  password:
-  vhost:
+  consumerProperties:
+    # canal tcp consumer
+    canal.tcp.server.host: 127.0.0.1:11111
+    canal.tcp.zookeeper.hosts:
+    canal.tcp.batch.size: 500
+    canal.tcp.username:
+    canal.tcp.password:
+    # kafka consumer
+    kafka.bootstrap.servers: 127.0.0.1:9092
+    kafka.enable.auto.commit: false
+    kafka.auto.commit.interval.ms: 1000
+    kafka.auto.offset.reset: latest
+    kafka.request.timeout.ms: 40000
+    kafka.session.timeout.ms: 30000
+    kafka.isolation.level: read_committed
+    kafka.max.poll.records: 1000
+    # rocketMQ consumer
+    rocketmq.namespace:
+    rocketmq.namesrv.addr: 127.0.0.1:9876
+    rocketmq.batch.size: 1000
+    rocketmq.enable.message.trace: false
+    rocketmq.customized.trace.topic:
+    rocketmq.access.channel:
+    rocketmq.subscribe.filter:
+    # rabbitMQ consumer
+    rabbitmq.host:
+    rabbitmq.virtual.host:
+    rabbitmq.username:
+    rabbitmq.password:
+    rabbitmq.resource.ownerId:
+
 #  srcDataSources:
 #    defaultDS:
 #      url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true

client/pom.xml

@@ -101,23 +101,31 @@
 			<version>${spring_version}</version>
 			<scope>test</scope>
 		</dependency>
+		<!-- 客户端要使用请单独引入mq-clients依赖 -->
 		<dependency>
 			<groupId>org.apache.rocketmq</groupId>
 			<artifactId>rocketmq-client</artifactId>
+			<version>4.5.2</version>
+			<scope>provided</scope>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.rocketmq</groupId>
 			<artifactId>rocketmq-acl</artifactId>
+			<version>4.5.2</version>
+			<scope>provided</scope>
 		</dependency>
 		<dependency>
 			<groupId>com.rabbitmq</groupId>
 			<artifactId>amqp-client</artifactId>
+			<version>5.5.0</version>
+			<scope>provided</scope>
 		</dependency>
 		<dependency>
 			<groupId>com.alibaba.mq-amqp</groupId>
 			<artifactId>mq-amqp-client</artifactId>
+			<version>1.0.3</version>
+			<scope>provided</scope>
 		</dependency>
-		<!-- 客户端要使用请单独引入kafka-clients依赖 -->
 		<dependency>
 			<groupId>org.apache.kafka</groupId>
 			<artifactId>kafka-clients</artifactId>
@@ -143,7 +151,7 @@
 					<createDependencyReducedPom>false</createDependencyReducedPom>
 					<artifactSet>
 						<includes>
-							<include>com.alibaba.otter:*</include>
+							<include>com.alibaba.otter:canal.client</include>
 							<include>com.google.guava:*</include>
 							<include>org.joda:*</include>
 							<include>com.google.common:*</include>

connector/core/pom.xml

@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>canal.connector</artifactId>
+        <groupId>com.alibaba.otter</groupId>
+        <version>1.1.5-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>com.alibaba.otter</groupId>
+    <artifactId>connector.core</artifactId>
+    <packaging>jar</packaging>
+    <name>canal connector core module for otter ${project.version}</name>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>canal.protocol</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.googlecode.aviator</groupId>
+            <artifactId>aviator</artifactId>
+            <version>2.2.1</version>
+        </dependency>
+        <dependency>
+            <groupId>oro</groupId>
+            <artifactId>oro</artifactId>
+            <version>2.0.8</version>
+        </dependency>
+        <dependency>
+            <groupId>joda-time</groupId>
+            <artifactId>joda-time</artifactId>
+            <version>2.9.4</version>
+        </dependency>
+    </dependencies>
+
+</project>

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/config/CanalConstants.java

@@ -0,0 +1,26 @@
+package com.alibaba.otter.canal.connector.core.config;
+
+/**
+ * MQ配置常量
+ *
+ * @author rewerma 2020-01-27
+ * @version 1.0.0
+ */
+public class CanalConstants {
+
+    public static final String ROOT                           = "canal";
+
+    public static final String CANAL_FILTER_TRANSACTION_ENTRY = ROOT + "." + "instance.filter.transaction.entry";
+
+    public static final String CANAL_MQ_FLAT_MESSAGE          = ROOT + "." + "mq.flat.message";
+    public static final String CANAL_MQ_DATABASE_HASH         = ROOT + "." + "mq.database.hash";
+    public static final String CANAL_MQ_PARALLEL_THREAD_SIZE  = ROOT + "." + "mq.parallel.thread.size";
+    public static final String CANAL_MQ_CANAL_BATCH_SIZE      = ROOT + "." + "mq.canal.batch.size";
+    public static final String CANAL_MQ_CANAL_FETCH_TIMEOUT   = ROOT + "." + "mq.canal.fetch.timeout";
+    public static final String CANAL_MQ_ACCESS_CHANNEL        = ROOT + "." + "mq.access.channel";
+
+    public static final String CANAL_ALIYUN_ACCESS_KEY        = ROOT + "." + "aliyun.accessKey";
+    public static final String CANAL_ALIYUN_SECRET_KEY        = ROOT + "." + "aliyun.secretKey";
+    public static final String CANAL_ALIYUN_UID               = ROOT + "." + "aliyun.uid";
+
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/config/MQProperties.java

@@ -0,0 +1,102 @@
+package com.alibaba.otter.canal.connector.core.config;
+
+/**
+ * MQ配置类
+ *
+ * @author rewerma 2020-01-27
+ * @version 1.0.0
+ */
+public class MQProperties {
+
+    private boolean flatMessage            = true;
+    private boolean databaseHash           = true;
+    private boolean filterTransactionEntry = true;
+    private Integer parallelThreadSize     = 8;
+    private Integer fetchTimeout           = 100;
+    private Integer batchSize              = 50;
+    private String  accessChannel          = "local";
+
+    private String  aliyunAccessKey        = "";
+    private String  aliyunSecretKey        = "";
+    private int     aliyunUid              = 0;
+
+    public boolean isFlatMessage() {
+        return flatMessage;
+    }
+
+    public void setFlatMessage(boolean flatMessage) {
+        this.flatMessage = flatMessage;
+    }
+
+    public boolean isDatabaseHash() {
+        return databaseHash;
+    }
+
+    public void setDatabaseHash(boolean databaseHash) {
+        this.databaseHash = databaseHash;
+    }
+
+    public boolean isFilterTransactionEntry() {
+        return filterTransactionEntry;
+    }
+
+    public void setFilterTransactionEntry(boolean filterTransactionEntry) {
+        this.filterTransactionEntry = filterTransactionEntry;
+    }
+
+    public Integer getParallelThreadSize() {
+        return parallelThreadSize;
+    }
+
+    public void setParallelThreadSize(Integer parallelThreadSize) {
+        this.parallelThreadSize = parallelThreadSize;
+    }
+
+    public Integer getFetchTimeout() {
+        return fetchTimeout;
+    }
+
+    public void setFetchTimeout(Integer fetchTimeout) {
+        this.fetchTimeout = fetchTimeout;
+    }
+
+    public Integer getBatchSize() {
+        return batchSize;
+    }
+
+    public void setBatchSize(Integer batchSize) {
+        this.batchSize = batchSize;
+    }
+
+    public String getAccessChannel() {
+        return accessChannel;
+    }
+
+    public void setAccessChannel(String accessChannel) {
+        this.accessChannel = accessChannel;
+    }
+
+    public String getAliyunAccessKey() {
+        return aliyunAccessKey;
+    }
+
+    public void setAliyunAccessKey(String aliyunAccessKey) {
+        this.aliyunAccessKey = aliyunAccessKey;
+    }
+
+    public String getAliyunSecretKey() {
+        return aliyunSecretKey;
+    }
+
+    public void setAliyunSecretKey(String aliyunSecretKey) {
+        this.aliyunSecretKey = aliyunSecretKey;
+    }
+
+    public int getAliyunUid() {
+        return aliyunUid;
+    }
+
+    public void setAliyunUid(int aliyunUid) {
+        this.aliyunUid = aliyunUid;
+    }
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/consumer/CommonMessage.java

@@ -0,0 +1,131 @@
+package com.alibaba.otter.canal.connector.core.consumer;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+
+public class CommonMessage implements Serializable {
+
+    private static final long serialVersionUID = 2611556444074013268L;
+
+    private String database;                               // 数据库或schema
+    private String table;                                  // 表名
+    private List<String> pkNames;
+    private Boolean isDdl;
+    private String type;                                   // 类型: INSERT UPDATE DELETE
+    // binlog executeTime
+    private Long es;                                       // 执行耗时
+    // dml build timeStamp
+    private Long ts;                                       // 同步时间
+    private String sql;                                    // 执行的sql, dml sql为空
+    private List<Map<String, Object>> data;                // 数据列表
+    private List<Map<String, Object>> old;                 // 旧数据列表, 用于update, size和data的size一一对应
+
+
+    public String getDatabase() {
+        return database;
+    }
+
+    public void setDatabase(String database) {
+        this.database = database;
+    }
+
+    public String getTable() {
+        return table;
+    }
+
+    public void setTable(String table) {
+        this.table = table;
+    }
+
+    public List<String> getPkNames() {
+        return pkNames;
+    }
+
+    public void setPkNames(List<String> pkNames) {
+        this.pkNames = pkNames;
+    }
+
+    public Boolean getIsDdl() {
+        return isDdl;
+    }
+
+    public void setIsDdl(Boolean isDdl) {
+        this.isDdl = isDdl;
+    }
+
+    public String getType() {
+        return type;
+    }
+
+    public void setType(String type) {
+        this.type = type;
+    }
+
+    public Long getTs() {
+        return ts;
+    }
+
+    public void setTs(Long ts) {
+        this.ts = ts;
+    }
+
+    public String getSql() {
+        return sql;
+    }
+
+    public void setSql(String sql) {
+        this.sql = sql;
+    }
+
+    public List<Map<String, Object>> getData() {
+        return data;
+    }
+
+    public void setData(List<Map<String, Object>> data) {
+        this.data = data;
+    }
+
+    public List<Map<String, Object>> getOld() {
+        return old;
+    }
+
+    public void setOld(List<Map<String, Object>> old) {
+        this.old = old;
+    }
+
+    public Long getEs() {
+        return es;
+    }
+
+    public void setEs(Long es) {
+        this.es = es;
+    }
+
+    public void clear() {
+        database = null;
+        table = null;
+        type = null;
+        ts = null;
+        es = null;
+        data = null;
+        old = null;
+        sql = null;
+    }
+
+    @Override
+    public String toString() {
+        return "CommonMessage{" +
+                "database='" + database + '\'' +
+                ", table='" + table + '\'' +
+                ", pkNames=" + pkNames +
+                ", isDdl=" + isDdl +
+                ", type='" + type + '\'' +
+                ", es=" + es +
+                ", ts=" + ts +
+                ", sql='" + sql + '\'' +
+                ", data=" + data +
+                ", old=" + old +
+                '}';
+    }
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/filter/AviaterRegexFilter.java

@@ -0,0 +1,125 @@
+package com.alibaba.otter.canal.connector.core.filter;
+
+import java.util.*;
+
+import org.apache.commons.lang.StringUtils;
+
+import com.googlecode.aviator.AviatorEvaluator;
+import com.googlecode.aviator.Expression;
+
+/**
+ * 基于aviater进行tableName正则匹配的过滤算法
+ *
+ * @author jianghang 2012-7-20 下午06:01:34
+ */
+public class AviaterRegexFilter {
+
+    private static final String        SPLIT             = ",";
+    private static final String        PATTERN_SPLIT     = "|";
+    private static final String        FILTER_EXPRESSION = "regex(pattern,target)";
+    private static final RegexFunction regexFunction     = new RegexFunction();
+    private final Expression           exp               = AviatorEvaluator.compile(FILTER_EXPRESSION, true);
+    static {
+        AviatorEvaluator.addFunction(regexFunction);
+    }
+
+    private static final Comparator<String> COMPARATOR = new StringComparator();
+
+    final private String                    pattern;
+    final private boolean                   defaultEmptyValue;
+
+    public AviaterRegexFilter(String pattern){
+        this(pattern, true);
+    }
+
+    public AviaterRegexFilter(String pattern, boolean defaultEmptyValue){
+        this.defaultEmptyValue = defaultEmptyValue;
+        List<String> list = null;
+        if (StringUtils.isEmpty(pattern)) {
+            list = new ArrayList<String>();
+        } else {
+            String[] ss = StringUtils.split(pattern, SPLIT);
+            list = Arrays.asList(ss);
+        }
+
+        // 对pattern按照从长到短的排序
+        // 因为 foo|foot 匹配 foot 会出错，原因是 foot 匹配了 foo 之后，会返回 foo，但是 foo 的长度和 foot
+        // 的长度不一样
+        Collections.sort(list, COMPARATOR);
+        // 对pattern进行头尾完全匹配
+        list = completionPattern(list);
+        this.pattern = StringUtils.join(list, PATTERN_SPLIT);
+    }
+
+    public boolean filter(String filtered) {
+        if (StringUtils.isEmpty(pattern)) {
+            return defaultEmptyValue;
+        }
+
+        if (StringUtils.isEmpty(filtered)) {
+            return defaultEmptyValue;
+        }
+
+        Map<String, Object> env = new HashMap<String, Object>();
+        env.put("pattern", pattern);
+        env.put("target", filtered.toLowerCase());
+        return (Boolean) exp.execute(env);
+    }
+
+    /**
+     * 修复正则表达式匹配的问题，因为使用了 oro 的 matches，会出现：
+     *
+     * <pre>
+     * foo|foot 匹配 foot 出错，原因是 foot 匹配了 foo 之后，会返回 foo，但是 foo 的长度和 foot 的长度不一样
+     * </pre>
+     *
+     * 因此此类对正则表达式进行了从长到短的排序
+     *
+     * @author zebin.xuzb 2012-10-22 下午2:02:26
+     * @version 1.0.0
+     */
+    private static class StringComparator implements Comparator<String> {
+
+        @Override
+        public int compare(String str1, String str2) {
+            if (str1.length() > str2.length()) {
+                return -1;
+            } else if (str1.length() < str2.length()) {
+                return 1;
+            } else {
+                return 0;
+            }
+        }
+    }
+
+    /**
+     * 修复正则表达式匹配的问题，即使按照长度递减排序，还是会出现以下问题：
+     *
+     * <pre>
+     * foooo|f.*t 匹配 fooooot 出错，原因是 fooooot 匹配了 foooo 之后，会将 fooo 和数据进行匹配，但是 foooo 的长度和 fooooot 的长度不一样
+     * </pre>
+     *
+     * 因此此类对正则表达式进行头尾完全匹配
+     *
+     * @author simon
+     * @version 1.0.0
+     */
+
+    private List<String> completionPattern(List<String> patterns) {
+        List<String> result = new ArrayList<String>();
+        for (String pattern : patterns) {
+            StringBuffer stringBuffer = new StringBuffer();
+            stringBuffer.append("^");
+            stringBuffer.append(pattern);
+            stringBuffer.append("$");
+            result.add(stringBuffer.toString());
+        }
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        return pattern;
+    }
+
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/filter/PatternUtils.java

@@ -0,0 +1,33 @@
+package com.alibaba.otter.canal.connector.core.filter;
+
+import java.util.Map;
+
+import org.apache.oro.text.regex.MalformedPatternException;
+import org.apache.oro.text.regex.Pattern;
+import org.apache.oro.text.regex.PatternCompiler;
+import org.apache.oro.text.regex.Perl5Compiler;
+
+import com.google.common.collect.MapMaker;
+import com.google.common.collect.MigrateMap;
+
+public class PatternUtils {
+
+    @SuppressWarnings("deprecation")
+    private static Map<String, Pattern> patterns = MigrateMap.makeComputingMap(new MapMaker().softValues(), pattern -> {
+        try {
+            PatternCompiler pc = new Perl5Compiler();
+            return pc.compile(pattern,
+                Perl5Compiler.CASE_INSENSITIVE_MASK | Perl5Compiler.READ_ONLY_MASK | Perl5Compiler.SINGLELINE_MASK);
+        } catch (MalformedPatternException e) {
+            throw new RuntimeException(e);
+        }
+    });
+
+    public static Pattern getPattern(String pattern) {
+        return patterns.get(pattern);
+    }
+
+    public static void clear() {
+        patterns.clear();
+    }
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/filter/RegexFunction.java

@@ -0,0 +1,31 @@
+package com.alibaba.otter.canal.connector.core.filter;
+
+import java.util.Map;
+
+import org.apache.oro.text.regex.Perl5Matcher;
+
+import com.googlecode.aviator.runtime.function.AbstractFunction;
+import com.googlecode.aviator.runtime.function.FunctionUtils;
+import com.googlecode.aviator.runtime.type.AviatorBoolean;
+import com.googlecode.aviator.runtime.type.AviatorObject;
+
+/**
+ * 提供aviator regex的代码扩展
+ *
+ * @author jianghang 2012-7-23 上午10:29:23
+ */
+public class RegexFunction extends AbstractFunction {
+
+    public AviatorObject call(Map<String, Object> env, AviatorObject arg1, AviatorObject arg2) {
+        String pattern = FunctionUtils.getStringValue(arg1, env);
+        String text = FunctionUtils.getStringValue(arg2, env);
+        Perl5Matcher matcher = new Perl5Matcher();
+        boolean isMatch = matcher.matches(text, PatternUtils.getPattern(pattern));
+        return AviatorBoolean.valueOf(isMatch);
+    }
+
+    public String getName() {
+        return "regex";
+    }
+
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/producer/AbstractMQProducer.java

@@ -0,0 +1,110 @@
+package com.alibaba.otter.canal.connector.core.producer;
+
+import java.util.Properties;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import com.alibaba.otter.canal.connector.core.config.CanalConstants;
+import org.apache.commons.lang.StringUtils;
+
+import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
+import com.alibaba.otter.canal.connector.core.config.MQProperties;
+import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;
+
+/**
+ * MQ producer 抽象类
+ *
+ * @author rewerma 2020-01-27
+ * @version 1.0.0
+ */
+public abstract class AbstractMQProducer implements CanalMQProducer {
+
+    protected MQProperties       mqProperties;
+
+    protected ThreadPoolExecutor executor;
+
+    @Override
+    public void init(Properties properties) {
+        // parse canal mq properties
+        loadCanalMqProperties(properties);
+
+        int parallelThreadSize = mqProperties.getParallelThreadSize();
+        executor = new ThreadPoolExecutor(parallelThreadSize,
+            parallelThreadSize,
+            0,
+            TimeUnit.SECONDS,
+            new ArrayBlockingQueue<Runnable>(parallelThreadSize * 2),
+            new NamedThreadFactory("MQParallel"),
+            new ThreadPoolExecutor.CallerRunsPolicy());
+    }
+
+    @Override
+    public MQProperties getMqProperties() {
+        return this.mqProperties;
+    }
+
+    @Override
+    public void stop() {
+        if (executor != null) {
+            executor.shutdownNow();
+        }
+    }
+
+    /**
+     * 初始化配置
+     * <p>
+     * canal.mq.flat.message = true <br/>
+     * canal.mq.database.hash = true <br/>
+     * canal.mq.filter.transaction.entry = true <br/>
+     * canal.mq.parallel.thread.size = 8 <br/>
+     * canal.mq.batch.size = 50 <br/>
+     * canal.mq.timeout = 100 <br/>
+     * canal.mq.access.channel = local <br/>
+     * </p>
+     * 
+     * @param properties 总配置对象
+     */
+    private void loadCanalMqProperties(Properties properties) {
+        String flatMessage = properties.getProperty(CanalConstants.CANAL_MQ_FLAT_MESSAGE);
+        if (!StringUtils.isEmpty(flatMessage)) {
+            mqProperties.setFlatMessage(Boolean.parseBoolean(flatMessage));
+        }
+        String databaseHash = properties.getProperty(CanalConstants.CANAL_MQ_DATABASE_HASH);
+        if (!StringUtils.isEmpty(databaseHash)) {
+            mqProperties.setDatabaseHash(Boolean.parseBoolean(databaseHash));
+        }
+        String filterTranEntry = properties.getProperty(CanalConstants.CANAL_FILTER_TRANSACTION_ENTRY);
+        if (!StringUtils.isEmpty(filterTranEntry)) {
+            mqProperties.setFilterTransactionEntry(Boolean.parseBoolean(filterTranEntry));
+        }
+        String parallelThreadSize = properties.getProperty(CanalConstants.CANAL_MQ_PARALLEL_THREAD_SIZE);
+        if (!StringUtils.isEmpty(parallelThreadSize)) {
+            mqProperties.setParallelThreadSize(Integer.parseInt(parallelThreadSize));
+        }
+        String batchSize = properties.getProperty(CanalConstants.CANAL_MQ_CANAL_BATCH_SIZE);
+        if (!StringUtils.isEmpty(batchSize)) {
+            mqProperties.setBatchSize(Integer.parseInt(batchSize));
+        }
+        String timeOut = properties.getProperty(CanalConstants.CANAL_MQ_CANAL_FETCH_TIMEOUT);
+        if (!StringUtils.isEmpty(timeOut)) {
+            mqProperties.setFetchTimeout(Integer.parseInt(timeOut));
+        }
+        String accessChannel = properties.getProperty(CanalConstants.CANAL_MQ_ACCESS_CHANNEL);
+        if (!StringUtils.isEmpty(accessChannel)) {
+            mqProperties.setAccessChannel(accessChannel);
+        }
+        String aliyunAccessKey = properties.getProperty(CanalConstants.CANAL_ALIYUN_ACCESS_KEY);
+        if (!StringUtils.isEmpty(aliyunAccessKey)) {
+            mqProperties.setAliyunAccessKey(aliyunAccessKey);
+        }
+        String aliyunSecretKey = properties.getProperty(CanalConstants.CANAL_ALIYUN_SECRET_KEY);
+        if (!StringUtils.isEmpty(aliyunSecretKey)) {
+            mqProperties.setAliyunAccessKey(aliyunSecretKey);
+        }
+        String aliyunUid = properties.getProperty(CanalConstants.CANAL_ALIYUN_UID);
+        if (!StringUtils.isEmpty(aliyunUid)) {
+            mqProperties.setAliyunUid(Integer.parseInt(aliyunUid));
+        }
+    }
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/producer/MQDestination.java

@@ -0,0 +1,64 @@
+package com.alibaba.otter.canal.connector.core.producer;
+
+/**
+ * MQ producer destination
+ *
+ * @author rewerma 2020-01-27
+ * @version 1.0.0
+ */
+public class MQDestination {
+    private String  canalDestination;
+    private String  topic;
+    private Integer partition;
+    private Integer partitionsNum;
+    private String  partitionHash;
+    private String  dynamicTopic;
+
+    public String getCanalDestination() {
+        return canalDestination;
+    }
+
+    public void setCanalDestination(String canalDestination) {
+        this.canalDestination = canalDestination;
+    }
+
+    public String getTopic() {
+        return topic;
+    }
+
+    public void setTopic(String topic) {
+        this.topic = topic;
+    }
+
+    public Integer getPartition() {
+        return partition;
+    }
+
+    public void setPartition(Integer partition) {
+        this.partition = partition;
+    }
+
+    public Integer getPartitionsNum() {
+        return partitionsNum;
+    }
+
+    public void setPartitionsNum(Integer partitionsNum) {
+        this.partitionsNum = partitionsNum;
+    }
+
+    public String getPartitionHash() {
+        return partitionHash;
+    }
+
+    public void setPartitionHash(String partitionHash) {
+        this.partitionHash = partitionHash;
+    }
+
+    public String getDynamicTopic() {
+        return dynamicTopic;
+    }
+
+    public void setDynamicTopic(String dynamicTopic) {
+        this.dynamicTopic = dynamicTopic;
+    }
+}

server/src/main/java/com/alibaba/otter/canal/common/MQMessageUtils.java → connector/core/src/main/java/com/alibaba/otter/canal/connector/core/producer/MQMessageUtils.java

@@ -1,25 +1,18 @@
-package com.alibaba.otter.canal.common;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+package com.alibaba.otter.canal.connector.core.producer;
+
+import java.util.*;
 import java.util.concurrent.ThreadPoolExecutor;
 
+import com.google.common.base.Function;
 import org.apache.commons.lang.StringUtils;
 
 import com.alibaba.otter.canal.common.utils.ExecutorTemplate;
-import com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter;
+import com.alibaba.otter.canal.connector.core.filter.AviaterRegexFilter;
 import com.alibaba.otter.canal.protocol.CanalEntry;
 import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
 import com.alibaba.otter.canal.protocol.CanalEntry.RowChange;
 import com.alibaba.otter.canal.protocol.FlatMessage;
 import com.alibaba.otter.canal.protocol.Message;
-import com.google.common.base.Function;
 import com.google.common.collect.Lists;
 import com.google.common.collect.MapMaker;
 import com.google.common.collect.MigrateMap;
@@ -29,77 +22,88 @@ import com.google.protobuf.InvalidProtocolBufferException;
 /**
  * process MQ Message utils
  *
- * @author agapple 2018年12月11日 下午1:28:32
- */
-/**
  * @author agapple 2019年9月29日 下午12:36:26
  * @since 5.0.0
  */
 public class MQMessageUtils {
 
     @SuppressWarnings("deprecation")
-    private static Map<String, List<PartitionData>>    partitionDatas    = MigrateMap.makeComputingMap(new MapMaker().softValues(),
-                                                                             new Function<String, List<PartitionData>>() {
-
-                                                                                 public List<PartitionData> apply(String pkHashConfigs) {
-                                                                                     List<PartitionData> datas = Lists.newArrayList();
-
-                                                                                     String[] pkHashConfigArray = StringUtils.split(StringUtils.replace(pkHashConfigs,
-                                                                                         ",",
-                                                                                         ";"),
-                                                                                         ";");
-                                                                                     // schema.table:id^name
-                                                                                     for (String pkHashConfig : pkHashConfigArray) {
-                                                                                         PartitionData data = new PartitionData();
-                                                                                         int i = pkHashConfig.lastIndexOf(":");
-                                                                                         if (i > 0) {
-                                                                                             String pkStr = pkHashConfig.substring(i + 1);
-                                                                                             if (pkStr.equalsIgnoreCase("$pk$")) {
-                                                                                                 data.hashMode.autoPkHash = true;
-                                                                                             } else {
-                                                                                                 data.hashMode.pkNames = Lists.newArrayList(StringUtils.split(pkStr,
-                                                                                                     '^'));
-                                                                                             }
+    private static Map<String, List<PartitionData>>    partitionDatas    = MigrateMap
+        .makeComputingMap(new MapMaker().softValues(), pkHashConfigs -> {
+            List<PartitionData> datas = Lists
+                .newArrayList();
+
+            String[] pkHashConfigArray = StringUtils
+                .split(StringUtils.replace(
+                    pkHashConfigs,
+                    ",",
+                    ";"), ";");
+            // schema.table:id^name
+            for (String pkHashConfig : pkHashConfigArray) {
+                PartitionData data = new PartitionData();
+                int i = pkHashConfig
+                    .lastIndexOf(":");
+                if (i > 0) {
+                    String pkStr = pkHashConfig
+                        .substring(i + 1);
+                    if (pkStr.equalsIgnoreCase(
+                        "$pk$")) {
+                        data.hashMode.autoPkHash = true;
+                    } else {
+                        data.hashMode.pkNames = Lists
+                            .newArrayList(
+                                StringUtils
+                                    .split(
+                                        pkStr,
+                                        '^'));
+                    }
 
-                                                                                             pkHashConfig = pkHashConfig.substring(0,
-                                                                                                 i);
-                                                                                         } else {
-                                                                                             data.hashMode.tableHash = true;
-                                                                                         }
+                    pkHashConfig = pkHashConfig
+                        .substring(0, i);
+                } else {
+                    data.hashMode.tableHash = true;
+                }
 
-                                                                                         if (!isWildCard(pkHashConfig)) {
-                                                                                             data.simpleName = pkHashConfig;
-                                                                                         } else {
-                                                                                             data.regexFilter = new AviaterRegexFilter(pkHashConfig);
-                                                                                         }
-                                                                                         datas.add(data);
-                                                                                     }
+                if (!isWildCard(
+                    pkHashConfig)) {
+                    data.simpleName = pkHashConfig;
+                } else {
+                    data.regexFilter = new AviaterRegexFilter(
+                        pkHashConfig);
+                }
+                datas.add(data);
+            }
 
-                                                                                     return datas;
-                                                                                 }
-                                                                             });
+            return datas;
+        });
 
     @SuppressWarnings("deprecation")
-    private static Map<String, List<DynamicTopicData>> dynamicTopicDatas = MigrateMap.makeComputingMap(new MapMaker().softValues(),
-                                                                             new Function<String, List<DynamicTopicData>>() {
+    private static Map<String, List<DynamicTopicData>> dynamicTopicDatas = MigrateMap
+        .makeComputingMap(new MapMaker().softValues(), new Function<String, List<DynamicTopicData>>() {
 
                                                                                  public List<DynamicTopicData> apply(String pkHashConfigs) {
-                                                                                     List<DynamicTopicData> datas = Lists.newArrayList();
-                                                                                     String[] dynamicTopicArray = StringUtils.split(StringUtils.replace(pkHashConfigs,
-                                                                                         ",",
-                                                                                         ";"),
-                                                                                         ";");
+                                                                                     List<DynamicTopicData> datas = Lists
+                                                                                         .newArrayList();
+                                                                                     String[] dynamicTopicArray = StringUtils
+                                                                                         .split(StringUtils.replace(
+                                                                                             pkHashConfigs,
+                                                                                             ",",
+                                                                                             ";"), ";");
                                                                                      // schema.table
                                                                                      for (String dynamicTopic : dynamicTopicArray) {
                                                                                          DynamicTopicData data = new DynamicTopicData();
 
-                                                                                         if (!isWildCard(dynamicTopic)) {
+                                                                                         if (!isWildCard(
+                                                                                             dynamicTopic)) {
                                                                                              data.simpleName = dynamicTopic;
                                                                                          } else {
-                                                                                             if (dynamicTopic.contains("\\.")) {
-                                                                                                 data.tableRegexFilter = new AviaterRegexFilter(dynamicTopic);
+                                                                                             if (dynamicTopic
+                                                                                                 .contains("\\.")) {
+                                                                                                 data.tableRegexFilter = new AviaterRegexFilter(
+                                                                                                     dynamicTopic);
                                                                                              } else {
-                                                                                                 data.schemaRegexFilter = new AviaterRegexFilter(dynamicTopic);
+                                                                                                 data.schemaRegexFilter = new AviaterRegexFilter(
+                                                                                                     dynamicTopic);
                                                                                              }
                                                                                          }
                                                                                          datas.add(data);
@@ -238,8 +242,8 @@ public class MQMessageUtils {
      * @return 分区message数组
      */
     @SuppressWarnings("unchecked")
-    public static Message[] messagePartition(EntryRowData[] datas, long id, Integer partitionsNum,
-                                             String pkHashConfigs, boolean databaseHash) {
+    public static Message[] messagePartition(EntryRowData[] datas, long id, Integer partitionsNum, String pkHashConfigs,
+                                             boolean databaseHash) {
         if (partitionsNum == null) {
             partitionsNum = 1;
         }
@@ -338,8 +342,8 @@ public class MQMessageUtils {
     /**
      * 将Message转换为FlatMessage
      *
-     * @param message 原生message
      * @return FlatMessage列表
+     * @author agapple 2018年12月11日 下午1:28:32
      */
     public static List<FlatMessage> messageConverter(EntryRowData[] datas, long id) {
         List<FlatMessage> flatMessages = new ArrayList<>();
@@ -627,8 +631,8 @@ public class MQMessageUtils {
 
     private static boolean isWildCard(String value) {
         // not contaiins '.' ?
-        return StringUtils.containsAny(value, new char[] { '*', '?', '+', '|', '(', ')', '{', '}', '[', ']', '\\', '$',
-                '^' });
+        return StringUtils.containsAny(value,
+            new char[] { '*', '?', '+', '|', '(', ')', '{', '}', '[', ']', '\\', '$', '^' });
     }
 
     private static void put2MapMessage(Map<String, Message> messageMap, Long messageId, String topicName,

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/CanalMQProducer.java

@@ -0,0 +1,43 @@
+package com.alibaba.otter.canal.connector.core.spi;
+
+import java.util.Properties;
+
+import com.alibaba.otter.canal.connector.core.config.MQProperties;
+import com.alibaba.otter.canal.connector.core.util.Callback;
+import com.alibaba.otter.canal.connector.core.producer.MQDestination;
+import com.alibaba.otter.canal.protocol.Message;
+
+/**
+ * MQ producer SPI 接口
+ *
+ * @author rewerma 2020-01-27
+ * @version 1.0.0
+ */
+@SPI("kafka")
+public interface CanalMQProducer {
+
+    /**
+     * Init producer.
+     */
+    void init(Properties properties);
+
+    /**
+     * Get base mq properties
+     * 
+     * @return MQProperties
+     */
+    MQProperties getMqProperties();
+
+    /**
+     * Send canal message to related topic
+     *
+     * @param canalDestination canal mq destination
+     * @param message canal message
+     */
+    void send(MQDestination canalDestination, Message message, Callback callback);
+
+    /**
+     * Stop MQ producer service
+     */
+    void stop();
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/CanalMsgConsumer.java

@@ -0,0 +1,55 @@
+package com.alibaba.otter.canal.connector.core.spi;
+
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+
+import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
+
+/**
+ * Canal/MQ consumer SPI 接口
+ *
+ * @author rewerma @ 2020-02-01
+ * @version 1.0.0
+ */
+@SPI("kafka")
+public interface CanalMsgConsumer {
+
+    /**
+     * 初始化
+     * 
+     * @param properties consumer properties
+     * @param topic topic/destination
+     * @param groupId mq group id
+     */
+    void init(Properties properties, String topic, String groupId);
+
+    /**
+     * 连接Canal/MQ
+     */
+    void connect();
+
+    /**
+     * 批量拉取数据
+     * 
+     * @param timeout 超时时间
+     * @param unit 时间单位
+     * @return Message列表
+     */
+    List<CommonMessage> getMessage(Long timeout, TimeUnit unit);
+
+    /**
+     * 提交
+     */
+    void ack();
+
+    /**
+     * 回滚
+     */
+    void rollback();
+
+    /**
+     * 断开连接
+     */
+    void disconnect();
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/ExtensionLoader.java

@@ -0,0 +1,435 @@
+package com.alibaba.otter.canal.connector.core.spi;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.InputStreamReader;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.regex.Pattern;
+
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * SPI 类加载器
+ *
+ * @author rewerma 2018-8-19 下午11:30:49
+ * @version 1.0.0
+ */
+public class ExtensionLoader<T> {
+
+    private static final Logger                                      logger                     = LoggerFactory
+        .getLogger(ExtensionLoader.class);
+
+    private static final String                                      SERVICES_DIRECTORY         = "META-INF/services/";
+
+    private static final String                                      CANAL_DIRECTORY            = "META-INF/canal/";
+
+    private static final String                                      DEFAULT_CLASSLOADER_POLICY = "internal";
+
+    private static final Pattern                                     NAME_SEPARATOR             = Pattern
+        .compile("\\s*[,]+\\s*");
+
+    private static final ConcurrentMap<Class<?>, ExtensionLoader<?>> EXTENSION_LOADERS          = new ConcurrentHashMap<>();
+
+    private static final ConcurrentMap<Class<?>, Object>             EXTENSION_INSTANCES        = new ConcurrentHashMap<>();
+
+    private static final ConcurrentMap<String, Object>               EXTENSION_KEY_INSTANCE     = new ConcurrentHashMap<>();
+
+    private final Class<?>                                           type;
+
+    private final String                                             classLoaderPolicy;
+
+    private final ConcurrentMap<Class<?>, String>                    cachedNames                = new ConcurrentHashMap<>();
+
+    private final Holder<Map<String, Class<?>>>                      cachedClasses              = new Holder<>();
+
+    private final ConcurrentMap<String, Holder<Object>>              cachedInstances            = new ConcurrentHashMap<>();
+
+    private String                                                   cachedDefaultName;
+
+    private ConcurrentHashMap<String, IllegalStateException>         exceptions                 = new ConcurrentHashMap<>();
+
+    private static <T> boolean withExtensionAnnotation(Class<T> type) {
+        return type.isAnnotationPresent(SPI.class);
+    }
+
+    public static <T> ExtensionLoader<T> getExtensionLoader(Class<T> type) {
+        return getExtensionLoader(type, DEFAULT_CLASSLOADER_POLICY);
+    }
+
+    @SuppressWarnings("unchecked")
+    public static <T> ExtensionLoader<T> getExtensionLoader(Class<T> type, String classLoaderPolicy) {
+        if (type == null) throw new IllegalArgumentException("Extension type == null");
+        if (!type.isInterface()) {
+            throw new IllegalArgumentException("Extension type(" + type + ") is not interface!");
+        }
+        if (!withExtensionAnnotation(type)) {
+            throw new IllegalArgumentException("Extension type(" + type + ") is not extension, because WITHOUT @"
+                                               + SPI.class.getSimpleName() + " Annotation!");
+        }
+
+        ExtensionLoader<T> loader = (ExtensionLoader<T>) EXTENSION_LOADERS.get(type);
+        if (loader == null) {
+            EXTENSION_LOADERS.putIfAbsent(type, new ExtensionLoader<T>(type, classLoaderPolicy));
+            loader = (ExtensionLoader<T>) EXTENSION_LOADERS.get(type);
+        }
+        return loader;
+    }
+
+    public ExtensionLoader(Class<?> type){
+        this.type = type;
+        this.classLoaderPolicy = DEFAULT_CLASSLOADER_POLICY;
+    }
+
+    public ExtensionLoader(Class<?> type, String classLoaderPolicy){
+        this.type = type;
+        this.classLoaderPolicy = classLoaderPolicy;
+    }
+
+    /**
+     * 返回指定名字的扩展
+     *
+     * @param name
+     * @return
+     */
+    @SuppressWarnings("unchecked")
+    public T getExtension(String name, String spiDir, String standbyDir) {
+        if (name == null || name.length() == 0) throw new IllegalArgumentException("Extension name == null");
+        if ("true".equals(name)) {
+            return getDefaultExtension(spiDir, standbyDir);
+        }
+        Holder<Object> holder = cachedInstances.get(name);
+        if (holder == null) {
+            cachedInstances.putIfAbsent(name, new Holder<>());
+            holder = cachedInstances.get(name);
+        }
+        Object instance = holder.get();
+        if (instance == null) {
+            synchronized (holder) {
+                instance = holder.get();
+                if (instance == null) {
+                    instance = createExtension(name, spiDir, standbyDir);
+                    holder.set(instance);
+                }
+            }
+        }
+        return (T) instance;
+    }
+
+    @SuppressWarnings("unchecked")
+    public T getExtension(String name, String key, String spiDir, String standbyDir) {
+        if (name == null || name.length() == 0) throw new IllegalArgumentException("Extension name == null");
+        if ("true".equals(name)) {
+            return getDefaultExtension(spiDir, standbyDir);
+        }
+        String extKey = name + "-" + StringUtils.trimToEmpty(key);
+        Holder<Object> holder = cachedInstances.get(extKey);
+        if (holder == null) {
+            cachedInstances.putIfAbsent(extKey, new Holder<>());
+            holder = cachedInstances.get(extKey);
+        }
+        Object instance = holder.get();
+        if (instance == null) {
+            synchronized (holder) {
+                instance = holder.get();
+                if (instance == null) {
+                    instance = createExtension(name, key, spiDir, standbyDir);
+                    holder.set(instance);
+                }
+            }
+        }
+        return (T) instance;
+    }
+
+    /**
+     * 返回缺省的扩展，如果没有设置则返回<code>null</code>
+     */
+    public T getDefaultExtension(String spiDir, String standbyDir) {
+        getExtensionClasses(spiDir, standbyDir);
+        if (null == cachedDefaultName || cachedDefaultName.length() == 0 || "true".equals(cachedDefaultName)) {
+            return null;
+        }
+        return getExtension(cachedDefaultName, spiDir, standbyDir);
+    }
+
+    @SuppressWarnings("unchecked")
+    public T createExtension(String name, String spiDir, String standbyDir) {
+        Class<?> clazz = getExtensionClasses(spiDir, standbyDir).get(name);
+        if (clazz == null) {
+            throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type
+                                            + ")  could not be instantiated: class could not be found");
+        }
+        try {
+            T instance = (T) EXTENSION_INSTANCES.get(clazz);
+            if (instance == null) {
+                EXTENSION_INSTANCES.putIfAbsent(clazz, (T) clazz.newInstance());
+                instance = (T) EXTENSION_INSTANCES.get(clazz);
+            }
+            return instance;
+        } catch (Throwable t) {
+            throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type
+                                            + ")  could not be instantiated: " + t.getMessage(),
+                t);
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private T createExtension(String name, String key, String spiDir, String standbyDir) {
+        Class<?> clazz = getExtensionClasses(spiDir, standbyDir).get(name);
+        if (clazz == null) {
+            throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type
+                                            + ")  could not be instantiated: class could not be found");
+        }
+        try {
+            T instance = (T) EXTENSION_KEY_INSTANCE.get(name + "-" + key);
+            if (instance == null) {
+                EXTENSION_KEY_INSTANCE.putIfAbsent(name + "-" + key, clazz.newInstance());
+                instance = (T) EXTENSION_KEY_INSTANCE.get(name + "-" + key);
+            }
+            return instance;
+        } catch (Throwable t) {
+            throw new IllegalStateException("Extension instance(name: " + name + ", class: " + type
+                                            + ")  could not be instantiated: " + t.getMessage(),
+                t);
+        }
+    }
+
+    private Map<String, Class<?>> getExtensionClasses(String spiDir, String standbyDir) {
+        Map<String, Class<?>> classes = cachedClasses.get();
+        if (classes == null) {
+            synchronized (cachedClasses) {
+                classes = cachedClasses.get();
+                if (classes == null) {
+                    classes = loadExtensionClasses(spiDir, standbyDir);
+                    cachedClasses.set(classes);
+                }
+            }
+        }
+
+        return classes;
+    }
+
+    private String getJarDirectoryPath() {
+        URL url = Thread.currentThread().getContextClassLoader().getResource("");
+        String dirtyPath;
+        if (url != null) {
+            dirtyPath = url.toString();
+        } else {
+            File file = new File("");
+            dirtyPath = file.getAbsolutePath();
+        }
+        String jarPath = dirtyPath.replaceAll("^.*file:/", ""); // removes
+                                                                // file:/ and
+                                                                // everything
+                                                                // before it
+        jarPath = jarPath.replaceAll("jar!.*", "jar"); // removes everything
+                                                       // after .jar, if .jar
+                                                       // exists in dirtyPath
+        jarPath = jarPath.replaceAll("%20", " "); // necessary if path has
+                                                  // spaces within
+        if (!jarPath.endsWith(".jar")) { // this is needed if you plan to run
+                                         // the app using Spring Tools Suit play
+                                         // button.
+            jarPath = jarPath.replaceAll("/classes/.*", "/classes/");
+        }
+        Path path = Paths.get(jarPath).getParent(); // Paths - from java 8
+        if (path != null) {
+            return path.toString();
+        }
+        return null;
+    }
+
+    private Map<String, Class<?>> loadExtensionClasses(String spiDir, String standbyDir) {
+        final SPI defaultAnnotation = type.getAnnotation(SPI.class);
+        if (defaultAnnotation != null) {
+            String value = defaultAnnotation.value();
+            if ((value = value.trim()).length() > 0) {
+                String[] names = NAME_SEPARATOR.split(value);
+                if (names.length > 1) {
+                    throw new IllegalStateException("more than 1 default extension name on extension " + type.getName()
+                                                    + ": " + Arrays.toString(names));
+                }
+                if (names.length == 1) cachedDefaultName = names[0];
+            }
+        }
+
+        Map<String, Class<?>> extensionClasses = new HashMap<>();
+
+        if (spiDir != null && standbyDir != null) {
+            // 1. plugin folder，customized extension classLoader （jar_dir/plugin）
+            String dir = File.separator + this.getJarDirectoryPath() + spiDir; // + "plugin";
+
+            File externalLibDir = new File(dir);
+            if (!externalLibDir.exists()) {
+                externalLibDir = new File(File.separator + this.getJarDirectoryPath() + standbyDir);
+            }
+            logger.info("extension classpath dir: " + externalLibDir.getAbsolutePath());
+            if (externalLibDir.exists()) {
+                File[] files = externalLibDir.listFiles((dir1, name) -> name.endsWith(".jar"));
+                if (files != null) {
+                    for (File f : files) {
+                        URL url;
+                        try {
+                            url = f.toURI().toURL();
+                        } catch (MalformedURLException e) {
+                            throw new RuntimeException("load extension jar failed!", e);
+                        }
+
+                        ClassLoader parent = Thread.currentThread().getContextClassLoader();
+                        URLClassLoader localClassLoader;
+                        if (classLoaderPolicy == null || "".equals(classLoaderPolicy)
+                            || DEFAULT_CLASSLOADER_POLICY.equalsIgnoreCase(classLoaderPolicy)) {
+                            localClassLoader = new URLClassExtensionLoader(new URL[] { url });
+                        } else {
+                            localClassLoader = new URLClassLoader(new URL[] { url }, parent);
+                        }
+
+                        loadFile(extensionClasses, CANAL_DIRECTORY, localClassLoader);
+                        loadFile(extensionClasses, SERVICES_DIRECTORY, localClassLoader);
+                    }
+                }
+            }
+        }
+
+        // 2. load inner extension class with default classLoader
+        ClassLoader classLoader = findClassLoader();
+        loadFile(extensionClasses, CANAL_DIRECTORY, classLoader);
+        loadFile(extensionClasses, SERVICES_DIRECTORY, classLoader);
+
+        return extensionClasses;
+    }
+
+    private void loadFile(Map<String, Class<?>> extensionClasses, String dir, ClassLoader classLoader) {
+        String fileName = dir + type.getName();
+        try {
+            Enumeration<URL> urls;
+            if (classLoader != null) {
+                urls = classLoader.getResources(fileName);
+            } else {
+                urls = ClassLoader.getSystemResources(fileName);
+            }
+            if (urls != null) {
+                while (urls.hasMoreElements()) {
+                    URL url = urls.nextElement();
+                    try {
+                        BufferedReader reader = null;
+                        try {
+                            reader = new BufferedReader(
+                                new InputStreamReader(url.openStream(), StandardCharsets.UTF_8));
+                            String line = null;
+                            while ((line = reader.readLine()) != null) {
+                                final int ci = line.indexOf('#');
+                                if (ci >= 0) line = line.substring(0, ci);
+                                line = line.trim();
+                                if (line.length() > 0) {
+                                    try {
+                                        String name = null;
+                                        int i = line.indexOf('=');
+                                        if (i > 0) {
+                                            name = line.substring(0, i).trim();
+                                            line = line.substring(i + 1).trim();
+                                        }
+                                        if (line.length() > 0) {
+                                            Class<?> clazz = classLoader.loadClass(line);
+                                            // Class<?> clazz =
+                                            // Class.forName(line, true,
+                                            // classLoader);
+                                            if (!type.isAssignableFrom(clazz)) {
+                                                throw new IllegalStateException(
+                                                    "Error when load extension class(interface: " + type
+                                                                                + ", class line: " + clazz.getName()
+                                                                                + "), class " + clazz.getName()
+                                                                                + "is not subtype of interface.");
+                                            } else {
+                                                try {
+                                                    clazz.getConstructor(type);
+                                                } catch (NoSuchMethodException e) {
+                                                    clazz.getConstructor();
+                                                    String[] names = NAME_SEPARATOR.split(name);
+                                                    if (names != null && names.length > 0) {
+                                                        for (String n : names) {
+                                                            if (!cachedNames.containsKey(clazz)) {
+                                                                cachedNames.put(clazz, n);
+                                                            }
+                                                            Class<?> c = extensionClasses.get(n);
+                                                            if (c == null) {
+                                                                extensionClasses.put(n, clazz);
+                                                            } else if (c != clazz) {
+                                                                cachedNames.remove(clazz);
+                                                                throw new IllegalStateException(
+                                                                    "Duplicate extension " + type.getName() + " name "
+                                                                                                + n + " on "
+                                                                                                + c.getName() + " and "
+                                                                                                + clazz.getName());
+                                                            }
+                                                        }
+                                                    }
+                                                }
+                                            }
+                                        }
+                                    } catch (Throwable t) {
+                                        IllegalStateException e = new IllegalStateException(
+                                            "Failed to load extension class(interface: " + type + ", class line: "
+                                                                                            + line + ") in " + url
+                                                                                            + ", cause: "
+                                                                                            + t.getMessage(),
+                                            t);
+                                        exceptions.put(line, e);
+                                    }
+                                }
+                            } // end of while read lines
+                        } finally {
+                            if (reader != null) {
+                                reader.close();
+                            }
+                        }
+                    } catch (Throwable t) {
+                        logger.error("Exception when load extension class(interface: " + type + ", class file: " + url
+                                     + ") in " + url,
+                            t);
+                    }
+                } // end of while urls
+            }
+        } catch (Throwable t) {
+            logger.error(
+                "Exception when load extension class(interface: " + type + ", description file: " + fileName + ").",
+                t);
+        }
+    }
+
+    @SuppressWarnings("unused")
+    private static ClassLoader findClassLoader() {
+        return ExtensionLoader.class.getClassLoader();
+    }
+
+    @Override
+    public String toString() {
+        return this.getClass().getName() + "[" + type.getName() + "]";
+    }
+
+    private static class Holder<T> {
+
+        private volatile T value;
+
+        private void set(T value) {
+            this.value = value;
+        }
+
+        private T get() {
+            return value;
+        }
+
+    }
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/SPI.java

@@ -0,0 +1,18 @@
+package com.alibaba.otter.canal.connector.core.spi;
+
+import java.lang.annotation.*;
+
+/**
+ * SPI装载器注解
+ *
+ * @author rewerma @ 2018-10-20
+ * @version 1.0.0
+ */
+@Documented
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ ElementType.TYPE })
+public @interface SPI {
+
+    // Default SPI name
+    String value() default "";
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/spi/URLClassExtensionLoader.java

@@ -0,0 +1,88 @@
+package com.alibaba.otter.canal.connector.core.spi;
+
+import java.io.IOException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.Enumeration;
+import java.util.NoSuchElementException;
+
+public class URLClassExtensionLoader extends URLClassLoader {
+    public URLClassExtensionLoader(URL[] urls) {
+        super(urls);
+    }
+
+    @Override
+    public Class<?> loadClass(String name) throws ClassNotFoundException {
+        Class<?> c = findLoadedClass(name);
+        if (c != null) {
+            return c;
+        }
+
+        if (name.startsWith("java.") || name.startsWith("org.slf4j.")
+                || name.startsWith("org.apache.logging")
+                || name.startsWith("org.apache.commons.logging.")) {
+            // || name.startsWith("org.apache.hadoop."))
+            // {
+            c = super.loadClass(name);
+        }
+        if (c != null) return c;
+
+        try {
+            // 先加载jar内的class，可避免jar冲突
+            c = findClass(name);
+        } catch (ClassNotFoundException e) {
+            c = null;
+        }
+        if (c != null) {
+            return c;
+        }
+
+        return super.loadClass(name);
+    }
+
+    @Override
+    public Enumeration<URL> getResources(String name) throws IOException {
+        @SuppressWarnings("unchecked")
+        Enumeration<URL>[] tmp = (Enumeration<URL>[]) new Enumeration<?>[2];
+
+        tmp[0] = findResources(name); // local class
+        // path first
+        // tmp[1] = super.getResources(name);
+
+        return new CompoundEnumeration<>(tmp);
+    }
+
+    private static class CompoundEnumeration<E> implements Enumeration<E> {
+
+        private Enumeration<E>[] enums;
+        private int              index = 0;
+
+        public CompoundEnumeration(Enumeration<E>[] enums){
+            this.enums = enums;
+        }
+
+        private boolean next() {
+            while (this.index < this.enums.length) {
+                if (this.enums[this.index] != null && this.enums[this.index].hasMoreElements()) {
+                    return true;
+                }
+
+                ++this.index;
+            }
+
+            return false;
+        }
+
+        public boolean hasMoreElements() {
+            return this.next();
+        }
+
+        public E nextElement() {
+            if (!this.next()) {
+                throw new NoSuchElementException();
+            } else {
+                return this.enums[this.index].nextElement();
+            }
+        }
+    }
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/Callback.java

@@ -0,0 +1,13 @@
+package com.alibaba.otter.canal.connector.core.util;
+
+/**
+ * MQ 回调类
+ *
+ * @author rewerma 2020-01-27
+ * @version 1.0.0
+ */
+public interface Callback {
+    void commit();
+
+    void rollback();
+}

server/src/main/java/com/alibaba/otter/canal/common/CanalMessageSerializer.java → connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/CanalMessageSerializerUtil.java

@@ -1,20 +1,25 @@
-package com.alibaba.otter.canal.common;
+package com.alibaba.otter.canal.connector.core.util;
 
 import java.util.List;
 
-import org.apache.kafka.common.errors.SerializationException;
+import com.alibaba.otter.canal.protocol.exception.CanalClientException;
 import org.springframework.util.CollectionUtils;
 
 import com.alibaba.otter.canal.protocol.CanalEntry;
 import com.alibaba.otter.canal.protocol.CanalPacket;
 import com.alibaba.otter.canal.protocol.CanalPacket.PacketType;
 import com.alibaba.otter.canal.protocol.Message;
-import com.alibaba.otter.canal.server.netty.NettyUtils;
 import com.google.protobuf.ByteString;
 import com.google.protobuf.CodedOutputStream;
 import com.google.protobuf.WireFormat;
 
-public class CanalMessageSerializer {
+/**
+ * Canal message 序列化工具类
+ *
+ * @author rewerma 2020-01-27
+ * @version 1.0.0
+ */
+public class CanalMessageSerializerUtil {
 
     @SuppressWarnings("deprecation")
     public static byte[] serializer(Message data, boolean filterTransactionEntry) {
@@ -58,7 +63,8 @@ public class CanalMessageSerializer {
                         CanalPacket.Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder();
                         for (CanalEntry.Entry entry : data.getEntries()) {
                             if (filterTransactionEntry
-                                && (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND)) {
+                                && (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN
+                                    || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND)) {
                                 continue;
                             }
 
@@ -67,15 +73,60 @@ public class CanalMessageSerializer {
 
                         CanalPacket.Packet.Builder packetBuilder = CanalPacket.Packet.newBuilder();
                         packetBuilder.setType(PacketType.MESSAGES);
-                        packetBuilder.setVersion(NettyUtils.VERSION);
+                        packetBuilder.setVersion(1);
                         packetBuilder.setBody(messageBuilder.build().toByteString());
                         return packetBuilder.build().toByteArray();
                     }
                 }
             }
         } catch (Exception e) {
-            throw new SerializationException("Error when serializing message to byte[] ");
+            throw new RuntimeException("Error when serializing message to byte[] ");
         }
         return null;
     }
+
+    public static Message deserializer(byte[] data) {
+        return deserializer(data, false);
+    }
+
+    public static Message deserializer(byte[] data, boolean lazyParseEntry) {
+        try {
+            if (data == null) {
+                return null;
+            } else {
+                CanalPacket.Packet p = CanalPacket.Packet.parseFrom(data);
+                switch (p.getType()) {
+                    case MESSAGES: {
+                        if (!p.getCompression().equals(CanalPacket.Compression.NONE)
+                                && !p.getCompression().equals(CanalPacket.Compression.COMPRESSIONCOMPATIBLEPROTO2)) {
+                            throw new CanalClientException("compression is not supported in this connector");
+                        }
+
+                        CanalPacket.Messages messages = CanalPacket.Messages.parseFrom(p.getBody());
+                        Message result = new Message(messages.getBatchId());
+                        if (lazyParseEntry) {
+                            // byteString
+                            result.setRawEntries(messages.getMessagesList());
+                            result.setRaw(true);
+                        } else {
+                            for (ByteString byteString : messages.getMessagesList()) {
+                                result.addEntry(CanalEntry.Entry.parseFrom(byteString));
+                            }
+                            result.setRaw(false);
+                        }
+                        return result;
+                    }
+                    case ACK: {
+                        CanalPacket.Ack ack = CanalPacket.Ack.parseFrom(p.getBody());
+                        throw new CanalClientException("something goes wrong with reason: " + ack.getErrorMessage());
+                    }
+                    default: {
+                        throw new CanalClientException("unexpected packet type: " + p.getType());
+                    }
+                }
+            }
+        } catch (Exception e) {
+            throw new CanalClientException("deserializer failed", e);
+        }
+    }
 }

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/DateUtil.java

@@ -0,0 +1,46 @@
+package com.alibaba.otter.canal.connector.core.util;
+
+import org.apache.commons.lang.StringUtils;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+
+import java.util.Date;
+
+public class DateUtil {
+    private static DateTimeZone dateTimeZone;
+
+    static {
+        dateTimeZone = DateTimeZone.forID(TimeZone.LOCATION_TIME_ZONE);
+    }
+
+    /**
+     * 通用日期时间字符解析
+     *
+     * @param datetimeStr 日期时间字符串
+     * @return Date
+     */
+    public static Date parseDate(String datetimeStr) {
+        if (StringUtils.isEmpty(datetimeStr)) {
+            return null;
+        }
+        datetimeStr = datetimeStr.trim().replace('/', '-');
+        if (datetimeStr.contains("-")) {
+            if (datetimeStr.contains(":")) {
+                datetimeStr = datetimeStr.replace(" ", "T");
+            }
+        } else if (datetimeStr.contains(":")) {
+            datetimeStr = "T" + datetimeStr;
+        } else {
+            if (datetimeStr.length() == 8) {
+                String year = datetimeStr.substring(0, 4);
+                String month = datetimeStr.substring(4, 6);
+                String day = datetimeStr.substring(6, 8);
+                datetimeStr = year + "-" + month + "-" + day;
+            }
+        }
+
+        DateTime dateTime = new DateTime(datetimeStr, dateTimeZone);
+
+        return dateTime.toDate();
+    }
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/JdbcTypeUtil.java

@@ -0,0 +1,165 @@
+package com.alibaba.otter.canal.connector.core.util;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.sql.*;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * 类型转换工具类
+ *
+ * @author rewerma 2018-8-19 下午06:14:23
+ * @version 1.0.0
+ */
+public class JdbcTypeUtil {
+
+    private static Logger logger = LoggerFactory.getLogger(JdbcTypeUtil.class);
+
+    public static Object getRSData(ResultSet rs, String columnName, int jdbcType) throws SQLException {
+        if (jdbcType == Types.BIT || jdbcType == Types.BOOLEAN) {
+            return rs.getByte(columnName);
+        } else {
+            return rs.getObject(columnName);
+        }
+    }
+
+    public static Class<?> jdbcType2javaType(int jdbcType) {
+        switch (jdbcType) {
+            case Types.BIT:
+            case Types.BOOLEAN:
+                // return Boolean.class;
+            case Types.TINYINT:
+                return Byte.TYPE;
+            case Types.SMALLINT:
+                return Short.class;
+            case Types.INTEGER:
+                return Integer.class;
+            case Types.BIGINT:
+                return Long.class;
+            case Types.DECIMAL:
+            case Types.NUMERIC:
+                return BigDecimal.class;
+            case Types.REAL:
+                return Float.class;
+            case Types.FLOAT:
+            case Types.DOUBLE:
+                return Double.class;
+            case Types.CHAR:
+            case Types.VARCHAR:
+            case Types.LONGVARCHAR:
+                return String.class;
+            case Types.BINARY:
+            case Types.VARBINARY:
+            case Types.LONGVARBINARY:
+            case Types.BLOB:
+                return byte[].class;
+            case Types.DATE:
+                return Date.class;
+            case Types.TIME:
+                return Time.class;
+            case Types.TIMESTAMP:
+                return Timestamp.class;
+            default:
+                return String.class;
+        }
+    }
+
+    private static boolean isText(String columnType) {
+        return "LONGTEXT".equalsIgnoreCase(columnType) || "MEDIUMTEXT".equalsIgnoreCase(columnType)
+               || "TEXT".equalsIgnoreCase(columnType) || "TINYTEXT".equalsIgnoreCase(columnType);
+    }
+
+    public static Object typeConvert(String tableName, String columnName, String value, int sqlType, String mysqlType) {
+        if (value == null || (value.equals("") && !(isText(mysqlType) || sqlType == Types.CHAR
+                                                    || sqlType == Types.VARCHAR || sqlType == Types.LONGVARCHAR))) {
+            return null;
+        }
+
+        try {
+            Object res;
+            switch (sqlType) {
+                case Types.INTEGER:
+                    res = Integer.parseInt(value);
+                    break;
+                case Types.SMALLINT:
+                    res = Short.parseShort(value);
+                    break;
+                case Types.BIT:
+                case Types.TINYINT:
+                    res = Byte.parseByte(value);
+                    break;
+                case Types.BIGINT:
+                    if (mysqlType != null && mysqlType.startsWith("bigint") && mysqlType.endsWith("unsigned")) {
+                        res = new BigInteger(value);
+                    } else {
+                        res = Long.parseLong(value);
+                    }
+                    break;
+                // case Types.BIT:
+                case Types.BOOLEAN:
+                    res = !"0".equals(value);
+                    break;
+                case Types.DOUBLE:
+                case Types.FLOAT:
+                    res = Double.parseDouble(value);
+                    break;
+                case Types.REAL:
+                    res = Float.parseFloat(value);
+                    break;
+                case Types.DECIMAL:
+                case Types.NUMERIC:
+                    res = new BigDecimal(value);
+                    break;
+                case Types.BINARY:
+                case Types.VARBINARY:
+                case Types.LONGVARBINARY:
+                case Types.BLOB:
+                    res = value.getBytes("ISO-8859-1");
+                    break;
+                case Types.DATE:
+                    if (!value.startsWith("0000-00-00")) {
+                        java.util.Date date = DateUtil.parseDate(value);
+                        if (date != null) {
+                            res = new Date(date.getTime());
+                        } else {
+                            res = null;
+                        }
+                    } else {
+                        res = null;
+                    }
+                    break;
+                case Types.TIME: {
+                    java.util.Date date = DateUtil.parseDate(value);
+                    if (date != null) {
+                        res = new Time(date.getTime());
+                    } else {
+                        res = null;
+                    }
+                    break;
+                }
+                case Types.TIMESTAMP:
+                    if (!value.startsWith("0000-00-00")) {
+                        java.util.Date date = DateUtil.parseDate(value);
+                        if (date != null) {
+                            res = new Timestamp(date.getTime());
+                        } else {
+                            res = null;
+                        }
+                    } else {
+                        res = null;
+                    }
+                    break;
+                case Types.CLOB:
+                default:
+                    res = value;
+                    break;
+            }
+            return res;
+        } catch (Exception e) {
+            logger.error("table: {} column: {}, failed convert type {} to {}", tableName, columnName, value, sqlType);
+            return value;
+        }
+    }
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/MessageUtil.java

@@ -0,0 +1,131 @@
+package com.alibaba.otter.canal.connector.core.util;
+
+import java.util.*;
+
+import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
+import com.alibaba.otter.canal.protocol.CanalEntry;
+import com.alibaba.otter.canal.protocol.Message;
+
+/**
+ * Message对象解析工具类
+ *
+ * @author rewerma 2018-8-19 下午06:14:23
+ * @version 1.0.0
+ */
+public class MessageUtil {
+
+    public static List<CommonMessage> convert(Message message) {
+        if (message == null) {
+            return null;
+        }
+        List<CanalEntry.Entry> entries = message.getEntries();
+        List<CommonMessage> msgs = new ArrayList<>(entries.size());
+        for (CanalEntry.Entry entry : entries) {
+            if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN
+                    || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {
+                continue;
+            }
+
+            CanalEntry.RowChange rowChange;
+            try {
+                rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
+            } catch (Exception e) {
+                throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:" + entry.toString(),
+                        e);
+            }
+
+            CanalEntry.EventType eventType = rowChange.getEventType();
+
+            final CommonMessage msg = new CommonMessage();
+            msg.setIsDdl(rowChange.getIsDdl());
+            msg.setDatabase(entry.getHeader().getSchemaName());
+            msg.setTable(entry.getHeader().getTableName());
+            msg.setType(eventType.toString());
+            msg.setEs(entry.getHeader().getExecuteTime());
+            msg.setIsDdl(rowChange.getIsDdl());
+            msg.setTs(System.currentTimeMillis());
+            msg.setSql(rowChange.getSql());
+            msgs.add(msg);
+            List<Map<String, Object>> data = new ArrayList<>();
+            List<Map<String, Object>> old = new ArrayList<>();
+
+            if (!rowChange.getIsDdl()) {
+                Set<String> updateSet = new HashSet<>();
+                msg.setPkNames(new ArrayList<>());
+                int i = 0;
+                for (CanalEntry.RowData rowData : rowChange.getRowDatasList()) {
+                    if (eventType != CanalEntry.EventType.INSERT && eventType != CanalEntry.EventType.UPDATE
+                            && eventType != CanalEntry.EventType.DELETE) {
+                        continue;
+                    }
+
+                    Map<String, Object> row = new LinkedHashMap<>();
+                    List<CanalEntry.Column> columns;
+
+                    if (eventType == CanalEntry.EventType.DELETE) {
+                        columns = rowData.getBeforeColumnsList();
+                    } else {
+                        columns = rowData.getAfterColumnsList();
+                    }
+
+                    for (CanalEntry.Column column : columns) {
+                        if (i == 0) {
+                            if (column.getIsKey()) {
+                                msg.getPkNames().add(column.getName());
+                            }
+                        }
+                        if (column.getIsNull()) {
+                            row.put(column.getName(), null);
+                        } else {
+                            row.put(column.getName(),
+                                    JdbcTypeUtil.typeConvert(msg.getTable(),
+                                            column.getName(),
+                                            column.getValue(),
+                                            column.getSqlType(),
+                                            column.getMysqlType()));
+                        }
+                        // 获取update为true的字段
+                        if (column.getUpdated()) {
+                            updateSet.add(column.getName());
+                        }
+                    }
+                    if (!row.isEmpty()) {
+                        data.add(row);
+                    }
+
+                    if (eventType == CanalEntry.EventType.UPDATE) {
+                        Map<String, Object> rowOld = new LinkedHashMap<>();
+                        for (CanalEntry.Column column : rowData.getBeforeColumnsList()) {
+                            if (updateSet.contains(column.getName())) {
+                                if (column.getIsNull()) {
+                                    rowOld.put(column.getName(), null);
+                                } else {
+                                    rowOld.put(column.getName(),
+                                            JdbcTypeUtil.typeConvert(msg.getTable(),
+                                                    column.getName(),
+                                                    column.getValue(),
+                                                    column.getSqlType(),
+                                                    column.getMysqlType()));
+                                }
+                            }
+                        }
+                        // update操作将记录修改前的值
+                        if (!rowOld.isEmpty()) {
+                            old.add(rowOld);
+                        }
+                    }
+
+                    i++;
+                }
+                if (!data.isEmpty()) {
+                    msg.setData(data);
+                }
+                if (!old.isEmpty()) {
+                    msg.setOld(old);
+                }
+            }
+        }
+
+        return msgs;
+    }
+}

connector/core/src/main/java/com/alibaba/otter/canal/connector/core/util/TimeZone.java

@@ -0,0 +1,22 @@
+package com.alibaba.otter.canal.connector.core.util;
+
+
+public class TimeZone {
+    public final static String LOCATION_TIME_ZONE;    // 当前时区
+
+    static {
+        java.util.TimeZone localTimeZone = java.util.TimeZone.getDefault();
+        int rawOffset = localTimeZone.getRawOffset();
+        String symbol = "+";
+        if (rawOffset < 0) {
+            symbol = "-";
+        }
+        rawOffset = Math.abs(rawOffset);
+        int offsetHour = rawOffset / 3600000;
+        int offsetMinute = rawOffset % 3600000 / 60000;
+        String hour = String.format("%1$02d", offsetHour);
+        String minute = String.format("%1$02d", offsetMinute);
+        LOCATION_TIME_ZONE = symbol + hour + ":" + minute;
+        java.util.TimeZone.setDefault(java.util.TimeZone.getTimeZone("GMT" + LOCATION_TIME_ZONE));
+    }
+}

connector/kafka-connector/pom.xml

@@ -0,0 +1,94 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>canal.connector</artifactId>
+        <groupId>com.alibaba.otter</groupId>
+        <version>1.1.5-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>com.alibaba.otter</groupId>
+    <artifactId>connector.kafka</artifactId>
+    <packaging>jar</packaging>
+    <name>canal connector kafka module for otter ${project.version}</name>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>canal.protocol</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>connector.core</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.kafka</groupId>
+            <artifactId>kafka_2.11</artifactId>
+            <version>1.1.1</version>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>2.4</version>
+                <configuration>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>make-assembly</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-antrun-plugin</artifactId>
+                <version>1.8</version>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>run</goal>
+                        </goals>
+                        <configuration>
+                            <tasks>
+                                <copy todir="${project.basedir}/../../deployer/target/canal/plugin" overwrite="true">
+                                    <fileset dir="${project.basedir}/target" erroronmissingdir="true">
+                                        <include name="connector.kafka-${project.version}-jar-with-dependencies.jar" />
+                                    </fileset>
+                                </copy>
+                                <copy todir="${project.basedir}/../../client-adapter/launcher/target/canal-adapter/plugin" overwrite="true">
+                                    <fileset dir="${project.basedir}/target" erroronmissingdir="true">
+                                        <include name="connector.kafka-${project.version}-jar-with-dependencies.jar" />
+                                    </fileset>
+                                </copy>
+                            </tasks>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/config/KafkaConstants.java

@@ -0,0 +1,16 @@
+package com.alibaba.otter.canal.connector.kafka.config;
+
+/**
+ * Kafka 配置常量类
+ *
+ * @author rewerma 2020-01-27
+ * @version 1.0.0
+ */
+public class KafkaConstants {
+
+    public static final String ROOT                              = "canal";
+
+    public static final String CANAL_MQ_KAFKA_KERBEROS_ENABLE    = ROOT + "." + "mq.kafka.kerberos.enable";
+    public static final String CANAL_MQ_KAFKA_KERBEROS_KRB5_FILE = ROOT + "." + "mq.kafka.kerberos.krb5.file";
+    public static final String CANAL_MQ_KAFKA_KERBEROS_JAAS_FILE = ROOT + "." + "mq.kafka.kerberos.jaas.file";
+}

connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/config/KafkaProducerConfig.java

@@ -0,0 +1,53 @@
+package com.alibaba.otter.canal.connector.kafka.config;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import com.alibaba.otter.canal.connector.core.config.MQProperties;
+
+/**
+ * Kafka producer 配置类
+ *
+ * @author rewerma 2020-01-27
+ * @version 1.0.0
+ */
+public class KafkaProducerConfig extends MQProperties {
+
+    private Map<String, Object> kafkaProperties = new LinkedHashMap<>();
+
+    private boolean             kerberosEnabled         = false;
+    private String              krb5File;
+    private String              jaasFile;
+
+    public Map<String, Object> getKafkaProperties() {
+        return kafkaProperties;
+    }
+
+    public void setKafkaProperties(Map<String, Object> kafkaProperties) {
+        this.kafkaProperties = kafkaProperties;
+    }
+
+    public boolean isKerberosEnabled() {
+        return kerberosEnabled;
+    }
+
+    public void setKerberosEnabled(boolean kerberosEnabled) {
+        this.kerberosEnabled = kerberosEnabled;
+    }
+
+    public String getKrb5File() {
+        return krb5File;
+    }
+
+    public void setKrb5File(String krb5File) {
+        this.krb5File = krb5File;
+    }
+
+    public String getJaasFile() {
+        return jaasFile;
+    }
+
+    public void setJaasFile(String jaasFile) {
+        this.jaasFile = jaasFile;
+    }
+}

connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/consumer/CanalKafkaConsumer.java

@@ -0,0 +1,137 @@
+package com.alibaba.otter.canal.connector.kafka.consumer;
+
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.otter.canal.connector.core.config.CanalConstants;
+import com.alibaba.otter.canal.connector.core.spi.SPI;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.serialization.StringDeserializer;
+
+import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
+import com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer;
+import com.alibaba.otter.canal.connector.core.util.MessageUtil;
+import com.alibaba.otter.canal.protocol.Message;
+
+/**
+ * Kafka consumer SPI 实现
+ *
+ * @author rewerma @ 2020-02-01
+ * @version 1.0.0
+ */
+@SPI("kafka")
+public class CanalKafkaConsumer implements CanalMsgConsumer {
+
+    private static final String      PREFIX_KAFKA_CONFIG = "kafka.";
+
+    private KafkaConsumer<String, ?> kafkaConsumer;
+    private boolean                  flatMessage         = true;
+    private String                   topic;
+
+    private Map<Integer, Long>       currentOffsets      = new ConcurrentHashMap<>();
+    private Properties               kafkaProperties     = new Properties();
+
+    @Override
+    public void init(Properties properties, String topic, String groupId) {
+        this.topic = topic;
+
+        Boolean flatMessage = (Boolean) properties.get(CanalConstants.CANAL_MQ_FLAT_MESSAGE);
+        if (flatMessage != null) {
+            this.flatMessage = flatMessage;
+        }
+        for (Map.Entry<Object, Object> entry : properties.entrySet()) {
+            String k = (String) entry.getKey();
+            Object v = entry.getValue();
+            if (k.startsWith(PREFIX_KAFKA_CONFIG) && v != null) {
+                kafkaProperties.put(k.substring(PREFIX_KAFKA_CONFIG.length()), v);
+            }
+        }
+        kafkaProperties.put("group.id", groupId);
+        kafkaProperties.put("key.deserializer", StringDeserializer.class);
+        kafkaProperties.put("client.id", UUID.randomUUID().toString().substring(0, 6));
+    }
+
+    @Override
+    public void connect() {
+        if (this.flatMessage) {
+            kafkaProperties.put("value.deserializer", StringDeserializer.class);
+            this.kafkaConsumer = new KafkaConsumer<String, String>(kafkaProperties);
+        } else {
+            kafkaProperties.put("value.deserializer", KafkaMessageDeserializer.class);
+            this.kafkaConsumer = new KafkaConsumer<String, Message>(kafkaProperties);
+        }
+        kafkaConsumer.subscribe(Collections.singletonList(topic));
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public List<CommonMessage> getMessage(Long timeout, TimeUnit unit) {
+        if (!flatMessage) {
+            ConsumerRecords<String, Message> records = (ConsumerRecords<String, Message>) kafkaConsumer
+                .poll(unit.toMillis(timeout));
+            if (!records.isEmpty()) {
+                currentOffsets.clear();
+                List<CommonMessage> messages = new ArrayList<>();
+                for (ConsumerRecord<String, Message> record : records) {
+                    if (currentOffsets.get(record.partition()) == null) {
+                        currentOffsets.put(record.partition(), record.offset());
+                    }
+                    messages.addAll(MessageUtil.convert(record.value()));
+                }
+                return messages;
+            }
+        } else {
+            ConsumerRecords<String, String> records = (ConsumerRecords<String, String>) kafkaConsumer
+                .poll(unit.toMillis(timeout));
+
+            if (!records.isEmpty()) {
+                List<CommonMessage> messages = new ArrayList<>();
+                currentOffsets.clear();
+                for (ConsumerRecord<String, String> record : records) {
+                    if (currentOffsets.get(record.partition()) == null) {
+                        currentOffsets.put(record.partition(), record.offset());
+                    }
+                    String flatMessageJson = record.value();
+                    CommonMessage flatMessages = JSON.parseObject(flatMessageJson, CommonMessage.class);
+                    messages.add(flatMessages);
+                }
+                return messages;
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public void rollback() {
+        // 回滚所有分区
+        if (kafkaConsumer != null) {
+            for (Map.Entry<Integer, Long> entry : currentOffsets.entrySet()) {
+                kafkaConsumer.seek(new TopicPartition(topic, entry.getKey()), currentOffsets.get(entry.getKey()));
+                kafkaConsumer.commitSync();
+            }
+        }
+    }
+
+    @Override
+    public void ack() {
+        if (kafkaConsumer != null) {
+            kafkaConsumer.commitSync();
+        }
+    }
+
+    @Override
+    public void disconnect() {
+        if (kafkaConsumer != null) {
+            kafkaConsumer.unsubscribe();
+        }
+        if (kafkaConsumer != null) {
+            kafkaConsumer.close();
+            kafkaConsumer = null;
+        }
+    }
+}

connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/consumer/KafkaMessageDeserializer.java

@@ -0,0 +1,31 @@
+package com.alibaba.otter.canal.connector.kafka.consumer;
+
+import java.util.Map;
+
+import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil;
+import org.apache.kafka.common.serialization.Deserializer;
+
+import com.alibaba.otter.canal.protocol.Message;
+
+/**
+ * Kafka Message类的反序列化
+ *
+ * @author rewerma @ 2018-6-12
+ * @version 1.0.0
+ */
+public class KafkaMessageDeserializer implements Deserializer<Message> {
+
+    @Override
+    public void configure(Map<String, ?> configs, boolean isKey) {
+    }
+
+    @Override
+    public Message deserialize(String topic1, byte[] data) {
+        return CanalMessageSerializerUtil.deserializer(data);
+    }
+
+    @Override
+    public void close() {
+        // nothing to do
+    }
+}

server/src/main/java/com/alibaba/otter/canal/kafka/CanalKafkaProducer.java → connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/producer/CanalKafkaProducer.java

@@ -1,14 +1,12 @@
-package com.alibaba.otter.canal.kafka;
+package com.alibaba.otter.canal.connector.kafka.producer;
 
 import java.io.File;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
+import java.util.*;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
 
+import com.alibaba.otter.canal.connector.kafka.config.KafkaConstants;
 import org.apache.commons.lang.StringUtils;
 import org.apache.kafka.clients.producer.KafkaProducer;
 import org.apache.kafka.clients.producer.Producer;
@@ -19,68 +17,90 @@ import org.slf4j.LoggerFactory;
 
 import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.serializer.SerializerFeature;
-import com.alibaba.otter.canal.common.AbstractMQProducer;
-import com.alibaba.otter.canal.common.CanalMessageSerializer;
-import com.alibaba.otter.canal.common.MQMessageUtils;
-import com.alibaba.otter.canal.common.MQMessageUtils.EntryRowData;
-import com.alibaba.otter.canal.common.MQProperties;
 import com.alibaba.otter.canal.common.utils.ExecutorTemplate;
+import com.alibaba.otter.canal.connector.core.producer.AbstractMQProducer;
+import com.alibaba.otter.canal.connector.core.util.Callback;
+import com.alibaba.otter.canal.connector.core.producer.MQDestination;
+import com.alibaba.otter.canal.connector.core.producer.MQMessageUtils;
+import com.alibaba.otter.canal.connector.core.producer.MQMessageUtils.EntryRowData;
+import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;
+import com.alibaba.otter.canal.connector.core.spi.SPI;
+import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil;
+import com.alibaba.otter.canal.connector.kafka.config.KafkaProducerConfig;
 import com.alibaba.otter.canal.protocol.FlatMessage;
 import com.alibaba.otter.canal.protocol.Message;
-import com.alibaba.otter.canal.spi.CanalMQProducer;
 
 /**
- * kafka producer 主操作类
+ * kafka producer SPI 实现
  *
- * @author machengyuan 2018-6-11 下午05:30:49
+ * @author rewerma 2018-6-11 下午05:30:49
  * @version 1.0.0
  */
+@SuppressWarnings({ "rawtypes", "unchecked" })
+@SPI("kafka")
 public class CanalKafkaProducer extends AbstractMQProducer implements CanalMQProducer {
 
-    private static final Logger      logger = LoggerFactory.getLogger(CanalKafkaProducer.class);
+    private static final Logger      logger              = LoggerFactory.getLogger(CanalKafkaProducer.class);
+
+    private static final String      PREFIX_KAFKA_CONFIG = "kafka.";
 
     private Producer<String, byte[]> producer;
-    private MQProperties             kafkaProperties;
 
     @Override
-    public void init(MQProperties kafkaProperties) {
-        super.init(kafkaProperties);
-
-        this.kafkaProperties = kafkaProperties;
-        Properties properties = new Properties();
-        properties.put("bootstrap.servers", kafkaProperties.getServers());
-        properties.put("acks", kafkaProperties.getAcks());
-        properties.put("compression.type", kafkaProperties.getCompressionType());
-        properties.put("batch.size", kafkaProperties.getBatchSize());
-        properties.put("linger.ms", kafkaProperties.getLingerMs());
-        properties.put("max.request.size", kafkaProperties.getMaxRequestSize());
-        properties.put("buffer.memory", kafkaProperties.getBufferMemory());
-        properties.put("key.serializer", StringSerializer.class.getName());
-        properties.put("max.in.flight.requests.per.connection", 1);
-
-        if (!kafkaProperties.getProperties().isEmpty()) {
-            properties.putAll(kafkaProperties.getProperties());
-        }
-        properties.put("retries", kafkaProperties.getRetries());
-        if (kafkaProperties.isKerberosEnable()) {
-            File krb5File = new File(kafkaProperties.getKerberosKrb5FilePath());
-            File jaasFile = new File(kafkaProperties.getKerberosJaasFilePath());
+    public void init(Properties properties) {
+        KafkaProducerConfig kafkaProducerConfig = new KafkaProducerConfig();
+        this.mqProperties = kafkaProducerConfig;
+        super.init(properties);
+        // load properties
+        this.loadKafkaProperties(properties);
+
+        Properties kafkaProperties = new Properties();
+        kafkaProperties.putAll(kafkaProducerConfig.getKafkaProperties());
+        kafkaProperties.put("key.serializer", StringSerializer.class);
+        if (kafkaProducerConfig.isKerberosEnabled()) {
+            File krb5File = new File(kafkaProducerConfig.getKrb5File());
+            File jaasFile = new File(kafkaProducerConfig.getJaasFile());
             if (krb5File.exists() && jaasFile.exists()) {
                 // 配置kerberos认证，需要使用绝对路径
                 System.setProperty("java.security.krb5.conf", krb5File.getAbsolutePath());
                 System.setProperty("java.security.auth.login.config", jaasFile.getAbsolutePath());
                 System.setProperty("javax.security.auth.useSubjectCredsOnly", "false");
-                properties.put("security.protocol", "SASL_PLAINTEXT");
-                properties.put("sasl.kerberos.service.name", "kafka");
+                kafkaProperties.put("security.protocol", "SASL_PLAINTEXT");
+                kafkaProperties.put("sasl.kerberos.service.name", "kafka");
             } else {
                 String errorMsg = "ERROR # The kafka kerberos configuration file does not exist! please check it";
                 logger.error(errorMsg);
                 throw new RuntimeException(errorMsg);
             }
         }
+        kafkaProperties.put("value.serializer", KafkaMessageSerializer.class);
+        producer = new KafkaProducer<>(kafkaProperties);
+    }
+
+    private void loadKafkaProperties(Properties properties) {
+        KafkaProducerConfig kafkaProducerConfig = (KafkaProducerConfig) this.mqProperties;
+        Map<String, Object> kafkaProperties = kafkaProducerConfig.getKafkaProperties();
+        for (Map.Entry<Object, Object> entry : properties.entrySet()) {
+            String key = (String) entry.getKey();
+            Object value = entry.getValue();
+            if (key.startsWith(PREFIX_KAFKA_CONFIG) && value != null) {
+                key = key.substring(PREFIX_KAFKA_CONFIG.length());
+                kafkaProperties.put(key, value);
+            }
+        }
 
-        properties.put("value.serializer", KafkaMessageSerializer.class.getName());
-        producer = new KafkaProducer<String, byte[]>(properties);
+        String kerberosEnabled = properties.getProperty(KafkaConstants.CANAL_MQ_KAFKA_KERBEROS_ENABLE);
+        if (!StringUtils.isEmpty(kerberosEnabled)) {
+            kafkaProducerConfig.setKerberosEnabled(Boolean.parseBoolean(kerberosEnabled));
+        }
+        String krb5File = properties.getProperty(KafkaConstants.CANAL_MQ_KAFKA_KERBEROS_KRB5_FILE);
+        if (!StringUtils.isEmpty(krb5File)) {
+            kafkaProducerConfig.setKrb5File(krb5File);
+        }
+        String jaasFile = properties.getProperty(KafkaConstants.CANAL_MQ_KAFKA_KERBEROS_JAAS_FILE);
+        if (!StringUtils.isEmpty(jaasFile)) {
+            kafkaProducerConfig.setJaasFile(jaasFile);
+        }
     }
 
     @Override
@@ -90,41 +110,34 @@ public class CanalKafkaProducer extends AbstractMQProducer implements CanalMQPro
             if (producer != null) {
                 producer.close();
             }
+            super.stop();
         } catch (Throwable e) {
             logger.warn("##something goes wrong when stopping kafka producer:", e);
         } finally {
             logger.info("## kafka producer is down.");
         }
-
-        super.stop();
     }
 
     @Override
-    public void send(MQProperties.CanalDestination canalDestination, Message message, Callback callback) {
+    public void send(MQDestination mqDestination, Message message, Callback callback) {
         ExecutorTemplate template = new ExecutorTemplate(executor);
-        boolean flat = kafkaProperties.getFlatMessage();
 
         try {
-            List result = null;
-            if (!StringUtils.isEmpty(canalDestination.getDynamicTopic())) {
+            List result;
+            if (!StringUtils.isEmpty(mqDestination.getDynamicTopic())) {
                 // 动态topic路由计算,只是基于schema/table,不涉及proto数据反序列化
-                Map<String, Message> messageMap = MQMessageUtils.messageTopics(message,
-                    canalDestination.getTopic(),
-                    canalDestination.getDynamicTopic());
+                Map<String, Message> messageMap = MQMessageUtils
+                    .messageTopics(message, mqDestination.getTopic(), mqDestination.getDynamicTopic());
 
                 // 针对不同的topic,引入多线程提升效率
                 for (Map.Entry<String, Message> entry : messageMap.entrySet()) {
                     final String topicName = entry.getKey().replace('.', '_');
                     final Message messageSub = entry.getValue();
-                    template.submit(new Callable() {
-
-                        @Override
-                        public List<Future> call() throws Exception {
-                            try {
-                                return send(canalDestination, topicName, messageSub, flat);
-                            } catch (Exception e) {
-                                throw new RuntimeException(e);
-                            }
+                    template.submit((Callable) () -> {
+                        try {
+                            return send(mqDestination, topicName, messageSub, mqProperties.isFlatMessage());
+                        } catch (Exception e) {
+                            throw new RuntimeException(e);
                         }
                     });
                 }
@@ -132,7 +145,10 @@ public class CanalKafkaProducer extends AbstractMQProducer implements CanalMQPro
                 result = template.waitForResult();
             } else {
                 result = new ArrayList();
-                List<Future> futures = send(canalDestination, canalDestination.getTopic(), message, flat);
+                List<Future> futures = send(mqDestination,
+                    mqDestination.getTopic(),
+                    message,
+                    mqProperties.isFlatMessage());
                 result.add(futures);
             }
 
@@ -161,36 +177,34 @@ public class CanalKafkaProducer extends AbstractMQProducer implements CanalMQPro
         }
     }
 
-    private List<Future> send(MQProperties.CanalDestination canalDestination, String topicName, Message message,
-                              boolean flat) throws Exception {
-        List<ProducerRecord<String, byte[]>> records = new ArrayList<ProducerRecord<String, byte[]>>();
+    private List<Future> send(MQDestination mqDestination, String topicName, Message message, boolean flat) {
+        List<ProducerRecord<String, byte[]>> records = new ArrayList<>();
         if (!flat) {
-            if (canalDestination.getPartitionHash() != null && !canalDestination.getPartitionHash().isEmpty()) {
+            if (mqDestination.getPartitionHash() != null && !mqDestination.getPartitionHash().isEmpty()) {
                 // 并发构造
                 EntryRowData[] datas = MQMessageUtils.buildMessageData(message, executor);
                 // 串行分区
                 Message[] messages = MQMessageUtils.messagePartition(datas,
                     message.getId(),
-                    canalDestination.getPartitionsNum(),
-                    canalDestination.getPartitionHash(),
-                    kafkaProperties.getDatabaseHash());
+                    mqDestination.getPartitionsNum(),
+                    mqDestination.getPartitionHash(),
+                    this.mqProperties.isDatabaseHash());
                 int length = messages.length;
                 for (int i = 0; i < length; i++) {
                     Message messagePartition = messages[i];
                     if (messagePartition != null) {
-                        records.add(new ProducerRecord<String, byte[]>(topicName,
+                        records.add(new ProducerRecord<>(topicName,
                             i,
                             null,
-                            CanalMessageSerializer.serializer(messagePartition,
-                                kafkaProperties.isFilterTransactionEntry())));
+                            CanalMessageSerializerUtil.serializer(messagePartition, true)));
                     }
                 }
             } else {
-                final int partition = canalDestination.getPartition() != null ? canalDestination.getPartition() : 0;
-                records.add(new ProducerRecord<String, byte[]>(topicName,
+                final int partition = mqDestination.getPartition() != null ? mqDestination.getPartition() : 0;
+                records.add(new ProducerRecord<>(topicName,
                     partition,
                     null,
-                    CanalMessageSerializer.serializer(message, kafkaProperties.isFilterTransactionEntry())));
+                    CanalMessageSerializerUtil.serializer(message, true)));
             }
         } else {
             // 发送扁平数据json
@@ -199,24 +213,24 @@ public class CanalKafkaProducer extends AbstractMQProducer implements CanalMQPro
             // 串行分区
             List<FlatMessage> flatMessages = MQMessageUtils.messageConverter(datas, message.getId());
             for (FlatMessage flatMessage : flatMessages) {
-                if (canalDestination.getPartitionHash() != null && !canalDestination.getPartitionHash().isEmpty()) {
+                if (mqDestination.getPartitionHash() != null && !mqDestination.getPartitionHash().isEmpty()) {
                     FlatMessage[] partitionFlatMessage = MQMessageUtils.messagePartition(flatMessage,
-                        canalDestination.getPartitionsNum(),
-                        canalDestination.getPartitionHash(),
-                        kafkaProperties.getDatabaseHash());
+                        mqDestination.getPartitionsNum(),
+                        mqDestination.getPartitionHash(),
+                        this.mqProperties.isDatabaseHash());
                     int length = partitionFlatMessage.length;
                     for (int i = 0; i < length; i++) {
                         FlatMessage flatMessagePart = partitionFlatMessage[i];
                         if (flatMessagePart != null) {
-                            records.add(new ProducerRecord<String, byte[]>(topicName,
+                            records.add(new ProducerRecord<>(topicName,
                                 i,
                                 null,
                                 JSON.toJSONBytes(flatMessagePart, SerializerFeature.WriteMapNullValue)));
                         }
                     }
                 } else {
-                    final int partition = canalDestination.getPartition() != null ? canalDestination.getPartition() : 0;
-                    records.add(new ProducerRecord<String, byte[]>(topicName,
+                    final int partition = mqDestination.getPartition() != null ? mqDestination.getPartition() : 0;
+                    records.add(new ProducerRecord<>(topicName,
                         partition,
                         null,
                         JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue)));
@@ -224,11 +238,11 @@ public class CanalKafkaProducer extends AbstractMQProducer implements CanalMQPro
             }
         }
 
-        return produce(topicName, records, flat);
+        return produce(records);
     }
 
-    private List<Future> produce(String topicName, List<ProducerRecord<String, byte[]>> records, boolean flatMessage) {
-        List<Future> futures = new ArrayList<Future>();
+    private List<Future> produce(List<ProducerRecord<String, byte[]>> records) {
+        List<Future> futures = new ArrayList<>();
         // 异步发送，因为在partition hash的时候已经按照每个分区合并了消息，走到这一步不需要考虑单个分区内的顺序问题
         for (ProducerRecord record : records) {
             futures.add(producer.send(record));

server/src/main/java/com/alibaba/otter/canal/kafka/KafkaMessageSerializer.java → connector/kafka-connector/src/main/java/com/alibaba/otter/canal/connector/kafka/producer/KafkaMessageSerializer.java

@@ -1,13 +1,13 @@
-package com.alibaba.otter.canal.kafka;
-
-import java.util.Map;
+package com.alibaba.otter.canal.connector.kafka.producer;
 
 import org.apache.kafka.common.serialization.Serializer;
 
+import java.util.Map;
+
 /**
  * Kafka Message类的序列化
  *
- * @author machengyuan 2018-6-11 下午05:30:49
+ * @author rewerma 2018-6-11 下午05:30:49
  * @version 1.0.0
  */
 public class KafkaMessageSerializer implements Serializer<byte[]> {

connector/kafka-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMQProducer

@@ -0,0 +1 @@
+kafka=com.alibaba.otter.canal.connector.kafka.producer.CanalKafkaProducer

connector/kafka-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer

@@ -0,0 +1 @@
+kafka=com.alibaba.otter.canal.connector.kafka.consumer.CanalKafkaConsumer

connector/kafka-connector/src/test/java/com/alibaba/otter/canal/connector/kafka/test/CanalKafkaProducerTest.java

@@ -0,0 +1,29 @@
+package com.alibaba.otter.canal.connector.kafka.test;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.Properties;
+
+import org.junit.Ignore;
+import org.junit.Test;
+
+import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;
+import com.alibaba.otter.canal.connector.core.spi.ExtensionLoader;
+
+@Ignore
+public class CanalKafkaProducerTest {
+
+    @Test
+    public void testLoadKafkaProducer() throws IOException {
+        Properties pro = new Properties();
+        FileInputStream in = new FileInputStream("../../deployer/src/main/resources/canal.properties");
+        pro.load(in);
+
+        ExtensionLoader<CanalMQProducer> loader = ExtensionLoader.getExtensionLoader(CanalMQProducer.class);
+        CanalMQProducer canalMQProducer = loader
+            .getExtension("kafka", "/../../deployer/target/canal/plugin", "/../../deployer/target/canal/plugin");
+        canalMQProducer.init(pro);
+
+        in.close();
+    }
+}

connector/pom.xml

@@ -0,0 +1,131 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>com.alibaba.otter</groupId>
+        <artifactId>canal</artifactId>
+        <version>1.1.5-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+
+    <groupId>com.alibaba.otter</groupId>
+    <artifactId>canal.connector</artifactId>
+    <version>1.1.5-SNAPSHOT</version>
+    <packaging>pom</packaging>
+    <name>canal connector module for otter ${project.version}</name>
+
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <maven.test.skip>true</maven.test.skip>
+        <downloadSources>true</downloadSources>
+        <java_source_version>1.8</java_source_version>
+        <java_target_version>1.8</java_target_version>
+        <file_encoding>UTF-8</file_encoding>
+    </properties>
+
+    <licenses>
+        <license>
+            <name>Apache License, Version 2.0</name>
+            <url>http://www.apache.org/licenses/LICENSE-2.0</url>
+        </license>
+    </licenses>
+
+    <scm>
+        <url>git@github.com:alibaba/canal.git</url>
+        <connection>scm:git:git@github.com:alibaba/canal.git</connection>
+        <developerConnection>scm:git:git@github.com:alibaba/canal.git</developerConnection>
+    </scm>
+
+    <repositories>
+        <repository>
+            <id>central</id>
+            <url>http://repo1.maven.org/maven2</url>
+            <releases>
+                <enabled>true</enabled>
+            </releases>
+            <snapshots>
+                <enabled>false</enabled>
+            </snapshots>
+        </repository>
+        <repository>
+            <id>java.net</id>
+            <url>http://download.java.net/maven/2/</url>
+            <releases>
+                <enabled>true</enabled>
+            </releases>
+            <snapshots>
+                <enabled>false</enabled>
+            </snapshots>
+        </repository>
+        <repository>
+            <id>aliyun</id>
+            <url>http://maven.aliyun.com/nexus/content/groups/public/</url>
+            <releases>
+                <enabled>true</enabled>
+            </releases>
+            <snapshots>
+                <enabled>false</enabled>
+            </snapshots>
+        </repository>
+        <repository>
+            <id>sonatype</id>
+            <name>sonatype</name>
+            <url>https://oss.sonatype.org/content/repositories/snapshots</url>
+            <releases>
+                <enabled>false</enabled>
+            </releases>
+            <snapshots>
+                <enabled>true</enabled>
+            </snapshots>
+        </repository>
+        <repository>
+            <id>sonatype-release</id>
+            <name>sonatype-release</name>
+            <url>https://oss.sonatype.org/service/local/repositories/releases/content</url>
+            <releases>
+                <enabled>false</enabled>
+            </releases>
+            <snapshots>
+                <enabled>true</enabled>
+            </snapshots>
+        </repository>
+    </repositories>
+
+    <modules>
+        <module>kafka-connector</module>
+        <module>core</module>
+        <module>rocketmq-connector</module>
+        <module>rabbitmq-connector</module>
+        <module>tcp-connector</module>
+    </modules>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>3.7.0</version>
+                <configuration>
+                    <source>${java_source_version}</source>
+                    <target>${java_target_version}</target>
+                    <encoding>${file_encoding}</encoding>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+
+    <distributionManagement>
+        <snapshotRepository>
+            <id>sonatype-nexus-snapshots</id>
+            <name>Sonatype Nexus Snapshots</name>
+            <url>https://oss.sonatype.org/content/repositories/snapshots/</url>
+        </snapshotRepository>
+        <repository>
+            <id>sonatype-nexus-staging</id>
+            <name>Nexus Release Repository</name>
+            <url>https://oss.sonatype.org/service/local/staging/deploy/maven2/</url>
+        </repository>
+    </distributionManagement>
+</project>

connector/rabbitmq-connector/pom.xml

@@ -0,0 +1,99 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>canal.connector</artifactId>
+        <groupId>com.alibaba.otter</groupId>
+        <version>1.1.5-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>com.alibaba.otter</groupId>
+    <artifactId>connector.rabbitmq</artifactId>
+    <packaging>jar</packaging>
+    <name>canal connector rabbitmq module for otter ${project.version}</name>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>canal.protocol</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>connector.core</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.rabbitmq</groupId>
+            <artifactId>amqp-client</artifactId>
+            <version>5.5.0</version>
+        </dependency>
+        <dependency>
+            <groupId>com.alibaba.mq-amqp</groupId>
+            <artifactId>mq-amqp-client</artifactId>
+            <version>1.0.3</version>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>2.4</version>
+                <configuration>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>make-assembly</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-antrun-plugin</artifactId>
+                <version>1.8</version>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>run</goal>
+                        </goals>
+                        <configuration>
+                            <tasks>
+                                <copy todir="${project.basedir}/../../deployer/target/canal/plugin" overwrite="true">
+                                    <fileset dir="${project.basedir}/target" erroronmissingdir="true">
+                                        <include name="connector.rabbitmq-${project.version}-jar-with-dependencies.jar" />
+                                    </fileset>
+                                </copy>
+                                <copy todir="${project.basedir}/../../client-adapter/launcher/target/canal-adapter/plugin" overwrite="true">
+                                    <fileset dir="${project.basedir}/target" erroronmissingdir="true">
+                                        <include name="connector.rabbitmq-${project.version}-jar-with-dependencies.jar" />
+                                    </fileset>
+                                </copy>
+                            </tasks>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/config/RabbitMQConstants.java

@@ -0,0 +1,20 @@
+package com.alibaba.otter.canal.connector.rabbitmq.config;
+
+/**
+ * RabbitMQ 配置常量类
+ *
+ * @author rewerma 2020-01-27
+ * @version 1.0.0
+ */
+public class RabbitMQConstants {
+
+    public static final String ROOT                      = "rabbitmq";
+
+    public static final String RABBITMQ_HOST             = ROOT + "." + "host";
+    public static final String RABBITMQ_EXCHANGE         = ROOT + "." + "exchange";
+    public static final String RABBITMQ_VIRTUAL_HOST     = ROOT + "." + "virtual.host";
+    public static final String RABBITMQ_USERNAME         = ROOT + "." + "username";
+    public static final String RABBITMQ_PASSWORD         = ROOT + "." + "password";
+
+    public static final String RABBITMQ_RESOURCE_OWNERID = ROOT + "." + "rabbitmq.resource.ownerId";
+}

connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/config/RabbitMQProducerConfig.java

@@ -0,0 +1,58 @@
+package com.alibaba.otter.canal.connector.rabbitmq.config;
+
+import com.alibaba.otter.canal.connector.core.config.MQProperties;
+
+/**
+ * RabbitMQ 配置类
+ *
+ * @author rewerma 2020-01-27
+ * @version 1.0.0
+ */
+public class RabbitMQProducerConfig extends MQProperties {
+
+    private String host;
+    private String virtualHost;
+    private String exchange;
+    private String username;
+    private String password;
+
+    public String getHost() {
+        return host;
+    }
+
+    public void setHost(String host) {
+        this.host = host;
+    }
+
+    public String getVirtualHost() {
+        return virtualHost;
+    }
+
+    public void setVirtualHost(String virtualHost) {
+        this.virtualHost = virtualHost;
+    }
+
+    public String getExchange() {
+        return exchange;
+    }
+
+    public void setExchange(String exchange) {
+        this.exchange = exchange;
+    }
+
+    public String getUsername() {
+        return username;
+    }
+
+    public void setUsername(String username) {
+        this.username = username;
+    }
+
+    public String getPassword() {
+        return password;
+    }
+
+    public void setPassword(String password) {
+        this.password = password;
+    }
+}

connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/consumer/CanalRabbitMQConsumer.java

@@ -0,0 +1,214 @@
+package com.alibaba.otter.canal.connector.rabbitmq.consumer;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import com.alibaba.otter.canal.connector.core.spi.SPI;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.otter.canal.connector.core.config.CanalConstants;
+import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
+import com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer;
+import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil;
+import com.alibaba.otter.canal.connector.core.util.MessageUtil;
+import com.alibaba.otter.canal.connector.rabbitmq.config.RabbitMQConstants;
+import com.alibaba.otter.canal.connector.rabbitmq.producer.AliyunCredentialsProvider;
+import com.alibaba.otter.canal.protocol.Message;
+import com.alibaba.otter.canal.protocol.exception.CanalClientException;
+import com.google.common.collect.Lists;
+import com.rabbitmq.client.*;
+
+/**
+ * RabbitMQ consumer SPI 实现
+ *
+ * @author rewerma 2020-02-01
+ * @version 1.0.0
+ */
+@SPI("rabbitmq")
+public class CanalRabbitMQConsumer implements CanalMsgConsumer {
+
+    private static final Logger                                logger              = LoggerFactory
+        .getLogger(CanalRabbitMQConsumer.class);
+
+    // 链接地址
+    private String                                             nameServer;
+    // 主机名
+    private String                                             vhost;
+    private String                                             queueName;
+
+    // 一些鉴权信息
+    private String                                             accessKey;
+    private String                                             secretKey;
+    private Long                                               resourceOwnerId;
+    private String                                             username;
+    private String                                             password;
+
+    private boolean                                            flatMessage;
+
+    private Connection                                         connect;
+    private Channel                                            channel;
+
+    private long                                               batchProcessTimeout = 60 * 1000;
+    private BlockingQueue<ConsumerBatchMessage<CommonMessage>> messageBlockingQueue;
+    private volatile ConsumerBatchMessage<CommonMessage>       lastGetBatchMessage = null;
+
+    @Override
+    public void init(Properties properties, String topic, String groupId) {
+        this.nameServer = properties.getProperty("rabbitmq.host");
+        this.vhost = properties.getProperty("rabbitmq.virtual.host");
+        this.queueName = topic;
+        this.accessKey = properties.getProperty(CanalConstants.CANAL_ALIYUN_ACCESS_KEY);
+        this.secretKey = properties.getProperty(CanalConstants.CANAL_ALIYUN_SECRET_KEY);
+        this.username = properties.getProperty(RabbitMQConstants.RABBITMQ_USERNAME);
+        this.password = properties.getProperty(RabbitMQConstants.RABBITMQ_PASSWORD);
+        Long resourceOwnerIdPro = (Long) properties.get(RabbitMQConstants.RABBITMQ_RESOURCE_OWNERID);
+        if (resourceOwnerIdPro != null) {
+            this.resourceOwnerId = resourceOwnerIdPro;
+        }
+        this.flatMessage = (Boolean) properties.get(CanalConstants.CANAL_MQ_FLAT_MESSAGE);
+        this.messageBlockingQueue = new LinkedBlockingQueue<>(1024);
+    }
+
+    @Override
+    public void connect() {
+        ConnectionFactory factory = new ConnectionFactory();
+        if (accessKey.length() > 0 && secretKey.length() > 0) {
+            factory.setCredentialsProvider(new AliyunCredentialsProvider(accessKey, secretKey, resourceOwnerId));
+        } else {
+            factory.setUsername(username);
+            factory.setPassword(password);
+        }
+        factory.setHost(nameServer);
+        factory.setAutomaticRecoveryEnabled(true);
+        factory.setNetworkRecoveryInterval(5000);
+        factory.setVirtualHost(vhost);
+        try {
+            connect = factory.newConnection();
+            channel = connect.createChannel();
+        } catch (IOException | TimeoutException e) {
+            throw new CanalClientException("Start RabbitMQ producer error", e);
+        }
+
+        // 不存在连接 则重新连接
+        if (connect == null) {
+            this.connect();
+        }
+
+        Consumer consumer = new DefaultConsumer(channel) {
+
+            @Override
+            public void handleDelivery(String consumerTag, Envelope envelope, AMQP.BasicProperties properties,
+                                       byte[] body) throws IOException {
+
+                if (body != null) {
+                    channel.basicAck(envelope.getDeliveryTag(), process(body));
+                }
+            }
+        };
+        try {
+            channel.basicConsume(queueName, false, consumer);
+        } catch (IOException e) {
+            throw new CanalClientException("error", e);
+        }
+    }
+
+    private boolean process(byte[] messageData) {
+        if (logger.isDebugEnabled()) {
+            logger.debug("Get Message: {}", new String(messageData));
+        }
+        List<CommonMessage> messageList = Lists.newArrayList();
+        if (!flatMessage) {
+            Message message = CanalMessageSerializerUtil.deserializer(messageData);
+            messageList.addAll(MessageUtil.convert(message));
+        } else {
+            CommonMessage commonMessage = JSON.parseObject(messageData, CommonMessage.class);
+            messageList.add(commonMessage);
+        }
+        ConsumerBatchMessage<CommonMessage>  batchMessage = new ConsumerBatchMessage<>(messageList);
+        try {
+            messageBlockingQueue.put(batchMessage);
+        } catch (InterruptedException e) {
+            logger.error("Put message to queue error", e);
+            throw new RuntimeException(e);
+        }
+        boolean isCompleted;
+        try {
+            isCompleted = batchMessage.waitFinish(batchProcessTimeout);
+        } catch (InterruptedException e) {
+            logger.error("Interrupted when waiting messages to be finished.", e);
+            throw new RuntimeException(e);
+        }
+        boolean isSuccess = batchMessage.isSuccess();
+        return isCompleted && isSuccess;
+    }
+
+    @Override
+    public List<CommonMessage> getMessage(Long timeout, TimeUnit unit) {
+        try {
+            if (this.lastGetBatchMessage != null) {
+                throw new CanalClientException("mq get/ack not support concurrent & async ack");
+            }
+
+            ConsumerBatchMessage<CommonMessage> batchMessage = messageBlockingQueue.poll(timeout, unit);
+            if (batchMessage != null) {
+                this.lastGetBatchMessage = batchMessage;
+                return batchMessage.getData();
+            }
+        } catch (InterruptedException ex) {
+            logger.warn("Get message timeout", ex);
+            throw new CanalClientException("Failed to fetch the data after: " + timeout);
+        }
+        return null;
+    }
+
+    @Override
+    public void rollback() {
+        try {
+            if (this.lastGetBatchMessage != null) {
+                this.lastGetBatchMessage.fail();
+            }
+        } finally {
+            this.lastGetBatchMessage = null;
+        }
+    }
+
+    @Override
+    public void ack() {
+        try {
+            if (this.lastGetBatchMessage != null) {
+                this.lastGetBatchMessage.ack();
+            }
+        } catch (Throwable e) {
+            if (this.lastGetBatchMessage != null) {
+                this.lastGetBatchMessage.fail();
+            }
+        } finally {
+            this.lastGetBatchMessage = null;
+        }
+    }
+
+    @Override
+    public void disconnect() {
+        if (connect != null) {
+            try {
+                connect.close();
+            } catch (IOException e) {
+                throw new CanalClientException("stop connect error", e);
+            }
+        }
+        if (channel != null) {
+            try {
+                channel.close();
+            } catch (IOException | TimeoutException e) {
+                throw new CanalClientException("stop channel error", e);
+            }
+        }
+    }
+}

connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/consumer/ConsumerBatchMessage.java

@@ -0,0 +1,48 @@
+package com.alibaba.otter.canal.connector.rabbitmq.consumer;
+
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+public class ConsumerBatchMessage<T> {
+
+    private final List<T>  data;
+    private CountDownLatch         latch;
+    private boolean                hasFailure = false;
+
+    public ConsumerBatchMessage(List<T> data){
+        this.data = data;
+        latch = new CountDownLatch(1);
+    }
+
+    public boolean waitFinish(long timeout) throws InterruptedException {
+        return latch.await(timeout, TimeUnit.MILLISECONDS);
+    }
+
+    public boolean isSuccess() {
+        return !hasFailure;
+    }
+
+    public List<T> getData() {
+        return data;
+    }
+
+    /**
+     * Countdown if the sub message is successful.
+     */
+    public void ack() {
+        latch.countDown();
+    }
+
+    /**
+     * Countdown and fail-fast if the sub message is failed.
+     */
+    public void fail() {
+        hasFailure = true;
+        // fail fast
+        long count = latch.getCount();
+        for (int i = 0; i < count; i++) {
+            latch.countDown();
+        }
+    }
+}

server/src/main/java/com/alibaba/otter/canal/rabbitmq/AliyunCredentialsProvider.java → connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/producer/AliyunCredentialsProvider.java

@@ -1,8 +1,4 @@
-/**
- * aliyun amqp协议 账号类
- * 暂不支持STS授权情况
- */
-package com.alibaba.otter.canal.rabbitmq;
+package com.alibaba.otter.canal.connector.rabbitmq.producer;
 
 import java.security.InvalidKeyException;
 import java.security.NoSuchAlgorithmException;
@@ -10,6 +6,12 @@ import java.security.NoSuchAlgorithmException;
 import com.alibaba.mq.amqp.utils.UserUtils;
 import com.rabbitmq.client.impl.CredentialsProvider;
 
+/**
+ * aliyun amqp协议 账号类
+ * 暂不支持STS授权情况
+ *
+ * @version 1.0.0
+ */
 public class AliyunCredentialsProvider implements CredentialsProvider {
 
     /**

connector/rabbitmq-connector/src/main/java/com/alibaba/otter/canal/connector/rabbitmq/producer/CanalRabbitMQProducer.java

@@ -0,0 +1,180 @@
+package com.alibaba.otter.canal.connector.rabbitmq.producer;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.serializer.SerializerFeature;
+import com.alibaba.otter.canal.common.CanalException;
+import com.alibaba.otter.canal.common.utils.ExecutorTemplate;
+import com.alibaba.otter.canal.connector.core.producer.AbstractMQProducer;
+import com.alibaba.otter.canal.connector.core.util.Callback;
+import com.alibaba.otter.canal.connector.core.producer.MQDestination;
+import com.alibaba.otter.canal.connector.core.producer.MQMessageUtils;
+import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;
+import com.alibaba.otter.canal.connector.core.spi.SPI;
+import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil;
+import com.alibaba.otter.canal.connector.rabbitmq.config.RabbitMQConstants;
+import com.alibaba.otter.canal.connector.rabbitmq.config.RabbitMQProducerConfig;
+import com.alibaba.otter.canal.protocol.FlatMessage;
+import com.alibaba.otter.canal.protocol.Message;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+
+/**
+ * RabbitMQ Producer SPI 实现
+ *
+ * @author rewerma 2020-01-27
+ * @version 1.0.0
+ */
+@SPI("rabbitmq")
+public class CanalRabbitMQProducer extends AbstractMQProducer implements CanalMQProducer {
+
+    private static final Logger logger = LoggerFactory.getLogger(CanalRabbitMQProducer.class);
+
+    private Connection          connect;
+    private Channel             channel;
+
+    @Override
+    public void init(Properties properties) {
+        RabbitMQProducerConfig rabbitMQProperties = new RabbitMQProducerConfig();
+        this.mqProperties = rabbitMQProperties;
+        super.init(properties);
+        loadRabbitMQProperties(properties);
+
+        ConnectionFactory factory = new ConnectionFactory();
+        factory.setHost(rabbitMQProperties.getHost());
+        if (mqProperties.getAliyunAccessKey().length() > 0 && mqProperties.getAliyunSecretKey().length() > 0
+            && mqProperties.getAliyunUid() > 0) {
+            factory.setCredentialsProvider(new AliyunCredentialsProvider(mqProperties.getAliyunAccessKey(),
+                mqProperties.getAliyunSecretKey(),
+                mqProperties.getAliyunUid()));
+        } else {
+            factory.setUsername(rabbitMQProperties.getUsername());
+            factory.setPassword(rabbitMQProperties.getPassword());
+        }
+        factory.setVirtualHost(rabbitMQProperties.getVirtualHost());
+        try {
+            connect = factory.newConnection();
+            channel = connect.createChannel();
+            // channel.exchangeDeclare(mqProperties.getExchange(), "topic");
+        } catch (IOException | TimeoutException ex) {
+            throw new CanalException("Start RabbitMQ producer error", ex);
+        }
+    }
+
+    private void loadRabbitMQProperties(Properties properties) {
+        RabbitMQProducerConfig rabbitMQProperties = (RabbitMQProducerConfig) this.mqProperties;
+
+        String host = properties.getProperty(RabbitMQConstants.RABBITMQ_HOST);
+        if (!StringUtils.isEmpty(host)) {
+            rabbitMQProperties.setHost(host);
+        }
+        String vhost = properties.getProperty(RabbitMQConstants.RABBITMQ_VIRTUAL_HOST);
+        if (!StringUtils.isEmpty(vhost)) {
+            rabbitMQProperties.setVirtualHost(vhost);
+        }
+        String exchange = properties.getProperty(RabbitMQConstants.RABBITMQ_EXCHANGE);
+        if (!StringUtils.isEmpty(exchange)) {
+            rabbitMQProperties.setExchange(exchange);
+        }
+        String username = properties.getProperty(RabbitMQConstants.RABBITMQ_USERNAME);
+        if (!StringUtils.isEmpty(username)) {
+            rabbitMQProperties.setUsername(username);
+        }
+        String password = properties.getProperty(RabbitMQConstants.RABBITMQ_PASSWORD);
+        if (!StringUtils.isEmpty(password)) {
+            rabbitMQProperties.setPassword(password);
+        }
+    }
+
+    @Override
+    public void send(final MQDestination destination, Message message, Callback callback) {
+        ExecutorTemplate template = new ExecutorTemplate(executor);
+        try {
+            if (!StringUtils.isEmpty(destination.getDynamicTopic())) {
+                // 动态topic
+                Map<String, Message> messageMap = MQMessageUtils
+                    .messageTopics(message, destination.getTopic(), destination.getDynamicTopic());
+
+                for (Map.Entry<String, com.alibaba.otter.canal.protocol.Message> entry : messageMap.entrySet()) {
+                    final String topicName = entry.getKey().replace('.', '_');
+                    final com.alibaba.otter.canal.protocol.Message messageSub = entry.getValue();
+
+                    template.submit(new Runnable() {
+
+                        @Override
+                        public void run() {
+                            send(destination, topicName, messageSub);
+                        }
+                    });
+                }
+
+                template.waitForResult();
+            } else {
+                send(destination, destination.getTopic(), message);
+            }
+            callback.commit();
+        } catch (Throwable e) {
+            logger.error(e.getMessage(), e);
+            callback.rollback();
+        } finally {
+            template.clear();
+        }
+    }
+
+    private void send(MQDestination canalDestination, String topicName, Message messageSub) {
+        if (!mqProperties.isFlatMessage()) {
+            byte[] message = CanalMessageSerializerUtil.serializer(messageSub, mqProperties.isFilterTransactionEntry());
+            if (logger.isDebugEnabled()) {
+                logger.debug("send message:{} to destination:{}", message, canalDestination.getCanalDestination());
+            }
+            sendMessage(topicName, message);
+        } else {
+            // 并发构造
+            MQMessageUtils.EntryRowData[] datas = MQMessageUtils.buildMessageData(messageSub, executor);
+            // 串行分区
+            List<FlatMessage> flatMessages = MQMessageUtils.messageConverter(datas, messageSub.getId());
+            for (FlatMessage flatMessage : flatMessages) {
+                byte[] message = JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue);
+                if (logger.isDebugEnabled()) {
+                    logger.debug("send message:{} to destination:{}", message, canalDestination.getCanalDestination());
+                }
+                sendMessage(topicName, message);
+            }
+        }
+
+    }
+
+    private void sendMessage(String queueName, byte[] message) {
+        // tips: 目前逻辑中暂不处理对exchange处理，请在Console后台绑定 才可使用routekey
+        try {
+            RabbitMQProducerConfig rabbitMQProperties = (RabbitMQProducerConfig) this.mqProperties;
+            channel.basicPublish(rabbitMQProperties.getExchange(), queueName, null, message);
+        } catch (Throwable e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public void stop() {
+        logger.info("## Stop RabbitMQ producer##");
+        try {
+            this.connect.close();
+            this.channel.close();
+            super.stop();
+        } catch (IOException | TimeoutException ex) {
+            throw new CanalException("Stop RabbitMQ producer error", ex);
+        }
+
+        super.stop();
+    }
+}

connector/rabbitmq-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMQProducer

@@ -0,0 +1 @@
+rabbitmq=com.alibaba.otter.canal.connector.rabbitmq.producer.CanalRabbitMQProducer

connector/rabbitmq-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer

@@ -0,0 +1 @@
+rabbitmq=com.alibaba.otter.canal.connector.rabbitmq.consumer.CanalRabbitMQConsumer

connector/rocketmq-connector/pom.xml

@@ -0,0 +1,104 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>canal.connector</artifactId>
+        <groupId>com.alibaba.otter</groupId>
+        <version>1.1.5-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>com.alibaba.otter</groupId>
+    <artifactId>connector.rocketmq</artifactId>
+    <packaging>jar</packaging>
+    <name>canal connector rocketMQ module for otter ${project.version}</name>
+
+    <properties>
+        <rocketmq_version>4.5.2</rocketmq_version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>canal.protocol</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>connector.core</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.rocketmq</groupId>
+            <artifactId>rocketmq-client</artifactId>
+            <version>${rocketmq_version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.rocketmq</groupId>
+            <artifactId>rocketmq-acl</artifactId>
+            <version>${rocketmq_version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>2.4</version>
+                <configuration>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>make-assembly</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-antrun-plugin</artifactId>
+                <version>1.8</version>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>run</goal>
+                        </goals>
+                        <configuration>
+                            <tasks>
+                                <copy todir="${project.basedir}/../../deployer/target/canal/plugin" overwrite="true">
+                                    <fileset dir="${project.basedir}/target" erroronmissingdir="true">
+                                        <include name="connector.rocketmq-${project.version}-jar-with-dependencies.jar"/>
+                                    </fileset>
+                                </copy>
+                                <copy todir="${project.basedir}/../../client-adapter/launcher/target/canal-adapter/plugin" overwrite="true">
+                                    <fileset dir="${project.basedir}/target" erroronmissingdir="true">
+                                        <include name="connector.rocketmq-${project.version}-jar-with-dependencies.jar" />
+                                    </fileset>
+                                </copy>
+                            </tasks>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/config/RocketMQConstants.java

@@ -0,0 +1,25 @@
+package com.alibaba.otter.canal.connector.rocketmq.config;
+
+/**
+ * RocketMQ 配置常量类
+ *
+ * @author rewerma 2020-01-27
+ * @version 1.0.0
+ */
+public class RocketMQConstants {
+
+    public static final String ROOT                                  = "rocketmq";
+
+    public static final String ROCKETMQ_PRODUCER_GROUP               = ROOT + "." + "producer.group";
+    public static final String ROCKETMQ_ENABLE_MESSAGE_TRACE         = ROOT + "." + "enable.message.trace";
+    public static final String ROCKETMQ_CUSTOMIZED_TRACE_TOPIC       = ROOT + "." + "customized.trace.topic";
+    public static final String ROCKETMQ_NAMESPACE                    = ROOT + "." + "namespace";
+    public static final String ROCKETMQ_NAMESRV_ADDR                 = ROOT + "." + "namesrv.addr";
+    public static final String ROCKETMQ_RETRY_TIMES_WHEN_SEND_FAILED = ROOT + "." + "retry.times.when.send.failed";
+    public static final String ROCKETMQ_VIP_CHANNEL_ENABLED          = ROOT + "." + "vip.channel.enabled";
+
+    public static final String ROCKETMQ_ACCESS_CHANNEL               = ROOT + "." + "access.channel";
+    public static final String ROCKETMQ_BATCH_SIZE                   = ROOT + "." + "batch.size";
+    public static final String ROCKETMQ_SUBSCRIBE_FILTER             = ROOT + "." + "subscribe.filter";
+
+}

connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/config/RocketMQProducerConfig.java

@@ -0,0 +1,76 @@
+package com.alibaba.otter.canal.connector.rocketmq.config;
+
+import com.alibaba.otter.canal.connector.core.config.MQProperties;
+
+/**
+ * RocketMQ 配置类
+ *
+ * @author rewerma 2020-01-27
+ * @version 1.0.0
+ */
+public class RocketMQProducerConfig extends MQProperties {
+
+    private String  producerGroup;
+    private boolean enableMessageTrace       = false;
+    private String  customizedTraceTopic;
+    private String  namespace;
+    private String  namesrvAddr;
+    private int     retryTimesWhenSendFailed = 0;
+    private boolean vipChannelEnabled        = false;
+
+    public String getProducerGroup() {
+        return producerGroup;
+    }
+
+    public void setProducerGroup(String producerGroup) {
+        this.producerGroup = producerGroup;
+    }
+
+    public boolean isEnableMessageTrace() {
+        return enableMessageTrace;
+    }
+
+    public void setEnableMessageTrace(boolean enableMessageTrace) {
+        this.enableMessageTrace = enableMessageTrace;
+    }
+
+    public String getCustomizedTraceTopic() {
+        return customizedTraceTopic;
+    }
+
+    public void setCustomizedTraceTopic(String customizedTraceTopic) {
+        this.customizedTraceTopic = customizedTraceTopic;
+    }
+
+    public String getNamespace() {
+        return namespace;
+    }
+
+    public void setNamespace(String namespace) {
+        this.namespace = namespace;
+    }
+
+    public String getNamesrvAddr() {
+        return namesrvAddr;
+    }
+
+    public void setNamesrvAddr(String namesrvAddr) {
+        this.namesrvAddr = namesrvAddr;
+    }
+
+    public int getRetryTimesWhenSendFailed() {
+        return retryTimesWhenSendFailed;
+    }
+
+    public void setRetryTimesWhenSendFailed(int retryTimesWhenSendFailed) {
+        this.retryTimesWhenSendFailed = retryTimesWhenSendFailed;
+    }
+
+    public boolean isVipChannelEnabled() {
+        return vipChannelEnabled;
+    }
+
+    public void setVipChannelEnabled(boolean vipChannelEnabled) {
+        this.vipChannelEnabled = vipChannelEnabled;
+    }
+}

connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/consumer/CanalRocketMQConsumer.java

@@ -0,0 +1,237 @@
+package com.alibaba.otter.canal.connector.rocketmq.consumer;
+
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.rocketmq.acl.common.AclClientRPCHook;
+import org.apache.rocketmq.acl.common.SessionCredentials;
+import org.apache.rocketmq.client.AccessChannel;
+import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
+import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus;
+import org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly;
+import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.remoting.RPCHook;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.otter.canal.connector.core.config.CanalConstants;
+import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
+import com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer;
+import com.alibaba.otter.canal.connector.core.spi.SPI;
+import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil;
+import com.alibaba.otter.canal.connector.core.util.MessageUtil;
+import com.alibaba.otter.canal.connector.rocketmq.config.RocketMQConstants;
+import com.alibaba.otter.canal.protocol.Message;
+import com.alibaba.otter.canal.protocol.exception.CanalClientException;
+import com.google.common.collect.Lists;
+
+/**
+ * RocketMQ consumer SPI 实现
+ *
+ * @author rewerma 2020-02-01
+ * @version 1.0.0
+ */
+@SPI("rocketmq")
+public class CanalRocketMQConsumer implements CanalMsgConsumer {
+
+    private static final Logger                                logger               = LoggerFactory
+        .getLogger(CanalRocketMQConsumer.class);
+    private static final String                                CLOUD_ACCESS_CHANNEL = "cloud";
+
+    private String                                             nameServer;
+    private String                                             topic;
+    private String                                             groupName;
+    private volatile boolean                                   connected            = false;
+    private DefaultMQPushConsumer                              rocketMQConsumer;
+    private BlockingQueue<ConsumerBatchMessage<CommonMessage>> messageBlockingQueue;
+    private int                                                batchSize            = -1;
+    private long                                               batchProcessTimeout  = 60 * 1000;
+    private boolean                                            flatMessage;
+    private volatile ConsumerBatchMessage<CommonMessage>       lastGetBatchMessage  = null;
+    private String                                             accessKey;
+    private String                                             secretKey;
+    private String                                             customizedTraceTopic;
+    private boolean                                            enableMessageTrace   = false;
+    private String                                             accessChannel;
+    private String                                             namespace;
+    private String                                             filter               = "*";
+
+    @Override
+    public void init(Properties properties, String topic, String groupName) {
+        this.topic = topic;
+        this.groupName = groupName;
+        this.flatMessage = (Boolean) properties.get(CanalConstants.CANAL_MQ_FLAT_MESSAGE);
+        this.messageBlockingQueue = new LinkedBlockingQueue<>(1024);
+        this.accessKey = properties.getProperty(CanalConstants.CANAL_ALIYUN_ACCESS_KEY);
+        this.secretKey = properties.getProperty(CanalConstants.CANAL_ALIYUN_SECRET_KEY);
+        String enableMessageTrace = properties.getProperty(RocketMQConstants.ROCKETMQ_ENABLE_MESSAGE_TRACE);
+        if (StringUtils.isNotEmpty(enableMessageTrace)) {
+            this.enableMessageTrace = Boolean.parseBoolean(enableMessageTrace);
+        }
+        this.customizedTraceTopic = properties.getProperty(RocketMQConstants.ROCKETMQ_CUSTOMIZED_TRACE_TOPIC);
+        this.accessChannel = properties.getProperty(RocketMQConstants.ROCKETMQ_ACCESS_CHANNEL);
+        this.namespace = properties.getProperty(RocketMQConstants.ROCKETMQ_NAMESPACE);
+        this.nameServer = properties.getProperty(RocketMQConstants.ROCKETMQ_NAMESRV_ADDR);
+        String batchSize = properties.getProperty(RocketMQConstants.ROCKETMQ_BATCH_SIZE);
+        if (StringUtils.isNotEmpty(batchSize)) {
+            this.batchSize = Integer.parseInt(batchSize);
+        }
+        String subscribeFilter = properties.getProperty(RocketMQConstants.ROCKETMQ_SUBSCRIBE_FILTER);
+        if (StringUtils.isNotEmpty(subscribeFilter)) {
+            this.filter = subscribeFilter;
+        }
+    }
+
+    @Override
+    public void connect() {
+        RPCHook rpcHook = null;
+        if (null != accessKey && accessKey.length() > 0 && null != secretKey && secretKey.length() > 0) {
+            SessionCredentials sessionCredentials = new SessionCredentials();
+            sessionCredentials.setAccessKey(accessKey);
+            sessionCredentials.setSecretKey(secretKey);
+            rpcHook = new AclClientRPCHook(sessionCredentials);
+        }
+
+        rocketMQConsumer = new DefaultMQPushConsumer(groupName,
+            rpcHook,
+            new AllocateMessageQueueAveragely(),
+            enableMessageTrace,
+            customizedTraceTopic);
+        rocketMQConsumer.setVipChannelEnabled(false);
+        if (CLOUD_ACCESS_CHANNEL.equals(this.accessChannel)) {
+            rocketMQConsumer.setAccessChannel(AccessChannel.CLOUD);
+        }
+
+        if (!StringUtils.isEmpty(this.namespace)) {
+            rocketMQConsumer.setNamespace(this.namespace);
+        }
+
+        if (!StringUtils.isBlank(nameServer)) {
+            rocketMQConsumer.setNamesrvAddr(nameServer);
+        }
+        if (batchSize != -1) {
+            rocketMQConsumer.setConsumeMessageBatchMaxSize(batchSize);
+        }
+
+        try {
+            if (rocketMQConsumer == null) {
+                this.connect();
+            }
+            rocketMQConsumer.subscribe(this.topic, this.filter);
+            rocketMQConsumer.registerMessageListener((MessageListenerOrderly) (messageExts, context) -> {
+                context.setAutoCommit(true);
+                boolean isSuccess = process(messageExts);
+                if (isSuccess) {
+                    return ConsumeOrderlyStatus.SUCCESS;
+                } else {
+                    return ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT;
+                }
+            });
+            rocketMQConsumer.start();
+        } catch (MQClientException ex) {
+            connected = false;
+            logger.error("Start RocketMQ consumer error", ex);
+        }
+    }
+
+    private boolean process(List<MessageExt> messageExts) {
+        if (logger.isDebugEnabled()) {
+            logger.debug("Get Message: {}", messageExts);
+        }
+        List<CommonMessage> messageList = Lists.newArrayList();
+        for (MessageExt messageExt : messageExts) {
+            byte[] data = messageExt.getBody();
+            if (data != null) {
+                try {
+                    if (!flatMessage) {
+                        Message message = CanalMessageSerializerUtil.deserializer(data);
+                        messageList.addAll(MessageUtil.convert(message));
+                    } else {
+                        CommonMessage commonMessage = JSON.parseObject(data, CommonMessage.class);
+                        messageList.add(commonMessage);
+                    }
+                } catch (Exception ex) {
+                    logger.error("Add message error", ex);
+                    throw new CanalClientException(ex);
+                }
+            } else {
+                logger.warn("Received message data is null");
+            }
+        }
+        ConsumerBatchMessage<CommonMessage> batchMessage = new ConsumerBatchMessage<>(messageList);
+        try {
+            messageBlockingQueue.put(batchMessage);
+        } catch (InterruptedException e) {
+            logger.error("Put message to queue error", e);
+            throw new RuntimeException(e);
+        }
+        boolean isCompleted;
+        try {
+            isCompleted = batchMessage.waitFinish(batchProcessTimeout);
+        } catch (InterruptedException e) {
+            logger.error("Interrupted when waiting messages to be finished.", e);
+            throw new RuntimeException(e);
+        }
+        boolean isSuccess = batchMessage.isSuccess();
+        return isCompleted && isSuccess;
+    }
+
+    @Override
+    public List<CommonMessage> getMessage(Long timeout, TimeUnit unit) {
+        try {
+            if (this.lastGetBatchMessage != null) {
+                throw new CanalClientException("mq get/ack not support concurrent & async ack");
+            }
+
+            ConsumerBatchMessage<CommonMessage> batchMessage = messageBlockingQueue.poll(timeout, unit);
+            if (batchMessage != null) {
+                this.lastGetBatchMessage = batchMessage;
+                return batchMessage.getData();
+            }
+        } catch (InterruptedException ex) {
+            logger.warn("Get message timeout", ex);
+            throw new CanalClientException("Failed to fetch the data after: " + timeout);
+        }
+        return null;
+    }
+
+    @Override
+    public void rollback() {
+        try {
+            if (this.lastGetBatchMessage != null) {
+                this.lastGetBatchMessage.fail();
+            }
+        } finally {
+            this.lastGetBatchMessage = null;
+        }
+    }
+
+    @Override
+    public void ack() {
+        try {
+            if (this.lastGetBatchMessage != null) {
+                this.lastGetBatchMessage.ack();
+            }
+        } catch (Throwable e) {
+            if (this.lastGetBatchMessage != null) {
+                this.lastGetBatchMessage.fail();
+            }
+        } finally {
+            this.lastGetBatchMessage = null;
+        }
+    }
+
+    @Override
+    public void disconnect() {
+        rocketMQConsumer.unsubscribe(topic);
+        rocketMQConsumer.shutdown();
+        connected = false;
+    }
+}

connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/consumer/ConsumerBatchMessage.java

@@ -0,0 +1,48 @@
+package com.alibaba.otter.canal.connector.rocketmq.consumer;
+
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+public class ConsumerBatchMessage<T> {
+
+    private final List<T>  data;
+    private CountDownLatch         latch;
+    private boolean                hasFailure = false;
+
+    public ConsumerBatchMessage(List<T> data){
+        this.data = data;
+        latch = new CountDownLatch(1);
+    }
+
+    public boolean waitFinish(long timeout) throws InterruptedException {
+        return latch.await(timeout, TimeUnit.MILLISECONDS);
+    }
+
+    public boolean isSuccess() {
+        return !hasFailure;
+    }
+
+    public List<T> getData() {
+        return data;
+    }
+
+    /**
+     * Countdown if the sub message is successful.
+     */
+    public void ack() {
+        latch.countDown();
+    }
+
+    /**
+     * Countdown and fail-fast if the sub message is failed.
+     */
+    public void fail() {
+        hasFailure = true;
+        // fail fast
+        long count = latch.getCount();
+        for (int i = 0; i < count; i++) {
+            latch.countDown();
+        }
+    }
+}

connector/rocketmq-connector/src/main/java/com/alibaba/otter/canal/connector/rocketmq/producer/CanalRocketMQProducer.java

@@ -0,0 +1,312 @@
+package com.alibaba.otter.canal.connector.rocketmq.producer;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.stream.Collectors;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.rocketmq.acl.common.AclClientRPCHook;
+import org.apache.rocketmq.acl.common.SessionCredentials;
+import org.apache.rocketmq.client.AccessChannel;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;
+import org.apache.rocketmq.client.impl.producer.TopicPublishInfo;
+import org.apache.rocketmq.client.producer.DefaultMQProducer;
+import org.apache.rocketmq.client.producer.SendResult;
+import org.apache.rocketmq.common.message.Message;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.remoting.RPCHook;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.serializer.SerializerFeature;
+import com.alibaba.otter.canal.common.CanalException;
+import com.alibaba.otter.canal.common.utils.ExecutorTemplate;
+import com.alibaba.otter.canal.connector.core.producer.AbstractMQProducer;
+import com.alibaba.otter.canal.connector.core.util.Callback;
+import com.alibaba.otter.canal.connector.core.producer.MQDestination;
+import com.alibaba.otter.canal.connector.core.producer.MQMessageUtils;
+import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;
+import com.alibaba.otter.canal.connector.core.spi.SPI;
+import com.alibaba.otter.canal.connector.core.util.CanalMessageSerializerUtil;
+import com.alibaba.otter.canal.connector.rocketmq.config.RocketMQConstants;
+import com.alibaba.otter.canal.connector.rocketmq.config.RocketMQProducerConfig;
+import com.alibaba.otter.canal.protocol.FlatMessage;
+
+/**
+ * RocketMQ Producer SPI 实现
+ *
+ * @author rewerma 2020-01-27
+ * @version 1.0.0
+ */
+@SPI("rocketmq")
+public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQProducer {
+
+    private static final Logger logger               = LoggerFactory.getLogger(CanalRocketMQProducer.class);
+
+    private DefaultMQProducer   defaultMQProducer;
+    private static final String CLOUD_ACCESS_CHANNEL = "cloud";
+
+    @Override
+    public void init(Properties properties) {
+        RocketMQProducerConfig rocketMQProperties = new RocketMQProducerConfig();
+        this.mqProperties = rocketMQProperties;
+        super.init(properties);
+        loadRocketMQProperties(properties);
+
+        RPCHook rpcHook = null;
+        if (mqProperties.getAliyunAccessKey().length() > 0
+            && mqProperties.getAliyunSecretKey().length() > 0) {
+            SessionCredentials sessionCredentials = new SessionCredentials();
+            sessionCredentials.setAccessKey(mqProperties.getAliyunAccessKey());
+            sessionCredentials.setSecretKey(mqProperties.getAliyunSecretKey());
+            rpcHook = new AclClientRPCHook(sessionCredentials);
+        }
+
+        defaultMQProducer = new DefaultMQProducer(rocketMQProperties.getProducerGroup(),
+            rpcHook,
+            rocketMQProperties.isEnableMessageTrace(),
+            rocketMQProperties.getCustomizedTraceTopic());
+        if (CLOUD_ACCESS_CHANNEL.equals(rocketMQProperties.getAccessChannel())) {
+            defaultMQProducer.setAccessChannel(AccessChannel.CLOUD);
+        }
+        if (!StringUtils.isEmpty(rocketMQProperties.getNamespace())) {
+            defaultMQProducer.setNamespace(rocketMQProperties.getNamespace());
+        }
+        defaultMQProducer.setNamesrvAddr(rocketMQProperties.getNamesrvAddr());
+        defaultMQProducer.setRetryTimesWhenSendFailed(rocketMQProperties.getRetryTimesWhenSendFailed());
+        defaultMQProducer.setVipChannelEnabled(rocketMQProperties.isVipChannelEnabled());
+        logger.info("##Start RocketMQ producer##");
+        try {
+            defaultMQProducer.start();
+        } catch (MQClientException ex) {
+            throw new CanalException("Start RocketMQ producer error", ex);
+        }
+    }
+
+    private void loadRocketMQProperties(Properties properties) {
+        RocketMQProducerConfig rocketMQProperties = (RocketMQProducerConfig) this.mqProperties;
+
+        String producerGroup = properties.getProperty(RocketMQConstants.ROCKETMQ_PRODUCER_GROUP);
+        if (!StringUtils.isEmpty(producerGroup)) {
+            rocketMQProperties.setProducerGroup(producerGroup);
+        }
+        String enableMessageTrace = properties.getProperty(RocketMQConstants.ROCKETMQ_ENABLE_MESSAGE_TRACE);
+        if (!StringUtils.isEmpty(enableMessageTrace)) {
+            rocketMQProperties.setEnableMessageTrace(Boolean.parseBoolean(enableMessageTrace));
+        }
+        String customizedTraceTopic = properties.getProperty(RocketMQConstants.ROCKETMQ_CUSTOMIZED_TRACE_TOPIC);
+        if (!StringUtils.isEmpty(customizedTraceTopic)) {
+            rocketMQProperties.setCustomizedTraceTopic(customizedTraceTopic);
+        }
+        String namespace = properties.getProperty(RocketMQConstants.ROCKETMQ_NAMESPACE);
+        if (!StringUtils.isEmpty(namespace)) {
+            rocketMQProperties.setNamespace(namespace);
+        }
+        String namesrvAddr = properties.getProperty(RocketMQConstants.ROCKETMQ_NAMESRV_ADDR);
+        if (!StringUtils.isEmpty(namesrvAddr)) {
+            rocketMQProperties.setNamesrvAddr(namesrvAddr);
+        }
+        String retry = properties.getProperty(RocketMQConstants.ROCKETMQ_RETRY_TIMES_WHEN_SEND_FAILED);
+        if (!StringUtils.isEmpty(retry)) {
+            rocketMQProperties.setRetryTimesWhenSendFailed(Integer.parseInt(retry));
+        }
+        String vipChannelEnabled = properties.getProperty(RocketMQConstants.ROCKETMQ_VIP_CHANNEL_ENABLED);
+        if (!StringUtils.isEmpty(vipChannelEnabled)) {
+            rocketMQProperties.setVipChannelEnabled(Boolean.parseBoolean(vipChannelEnabled));
+        }
+    }
+
+    @Override
+    public void send(MQDestination destination, com.alibaba.otter.canal.protocol.Message message, Callback callback) {
+        ExecutorTemplate template = new ExecutorTemplate(executor);
+        try {
+            if (!StringUtils.isEmpty(destination.getDynamicTopic())) {
+                // 动态topic
+                Map<String, com.alibaba.otter.canal.protocol.Message> messageMap = MQMessageUtils
+                    .messageTopics(message, destination.getTopic(), destination.getDynamicTopic());
+
+                for (Map.Entry<String, com.alibaba.otter.canal.protocol.Message> entry : messageMap.entrySet()) {
+                    String topicName = entry.getKey().replace('.', '_');
+                    com.alibaba.otter.canal.protocol.Message messageSub = entry.getValue();
+                    template.submit(() -> {
+                        try {
+                            send(destination, topicName, messageSub);
+                        } catch (Exception e) {
+                            throw new RuntimeException(e);
+                        }
+                    });
+                }
+
+                template.waitForResult();
+            } else {
+                send(destination, destination.getTopic(), message);
+            }
+
+            callback.commit();
+        } catch (Throwable e) {
+            logger.error(e.getMessage(), e);
+            callback.rollback();
+        } finally {
+            template.clear();
+        }
+    }
+
+    public void send(final MQDestination destination, String topicName,
+                     com.alibaba.otter.canal.protocol.Message message) {
+        if (!mqProperties.isFlatMessage()) {
+            if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) {
+                // 并发构造
+                MQMessageUtils.EntryRowData[] datas = MQMessageUtils.buildMessageData(message, executor);
+                // 串行分区
+                com.alibaba.otter.canal.protocol.Message[] messages = MQMessageUtils.messagePartition(datas,
+                    message.getId(),
+                    destination.getPartitionsNum(),
+                    destination.getPartitionHash(),
+                    mqProperties.isDatabaseHash());
+                int length = messages.length;
+
+                ExecutorTemplate template = new ExecutorTemplate(executor);
+                for (int i = 0; i < length; i++) {
+                    com.alibaba.otter.canal.protocol.Message dataPartition = messages[i];
+                    if (dataPartition != null) {
+                        final int index = i;
+                        template.submit(() -> {
+                            Message data = new Message(topicName,
+                                CanalMessageSerializerUtil.serializer(dataPartition,
+                                    mqProperties.isFilterTransactionEntry()));
+                            sendMessage(data, index);
+                        });
+                    }
+                }
+                // 等所有分片发送完毕
+                template.waitForResult();
+            } else {
+                final int partition = destination.getPartition() != null ? destination.getPartition() : 0;
+                Message data = new Message(topicName,
+                    CanalMessageSerializerUtil.serializer(message, mqProperties.isFilterTransactionEntry()));
+                sendMessage(data, partition);
+            }
+        } else {
+            // 并发构造
+            MQMessageUtils.EntryRowData[] datas = MQMessageUtils.buildMessageData(message, executor);
+            // 串行分区
+            List<FlatMessage> flatMessages = MQMessageUtils.messageConverter(datas, message.getId());
+            // 初始化分区合并队列
+            if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) {
+                List<List<FlatMessage>> partitionFlatMessages = new ArrayList<>();
+                for (int i = 0; i < destination.getPartitionsNum(); i++) {
+                    partitionFlatMessages.add(new ArrayList<>());
+                }
+
+                for (FlatMessage flatMessage : flatMessages) {
+                    FlatMessage[] partitionFlatMessage = MQMessageUtils.messagePartition(flatMessage,
+                        destination.getPartitionsNum(),
+                        destination.getPartitionHash(),
+                        mqProperties.isDatabaseHash());
+                    int length = partitionFlatMessage.length;
+                    for (int i = 0; i < length; i++) {
+                        partitionFlatMessages.get(i).add(partitionFlatMessage[i]);
+                    }
+                }
+
+                ExecutorTemplate template = new ExecutorTemplate(executor);
+                for (int i = 0; i < partitionFlatMessages.size(); i++) {
+                    final List<FlatMessage> flatMessagePart = partitionFlatMessages.get(i);
+                    if (flatMessagePart != null) {
+                        final int index = i;
+                        template.submit(() -> {
+                            List<Message> messages = flatMessagePart.stream()
+                                .map(flatMessage -> new Message(topicName,
+                                    JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue)))
+                                .collect(Collectors.toList());
+                            // 批量发送
+                            sendMessage(messages, index);
+                        });
+                    }
+                }
+
+                // 批量等所有分区的结果
+                template.waitForResult();
+            } else {
+                final int partition = destination.getPartition() != null ? destination.getPartition() : 0;
+                List<Message> messages = flatMessages.stream()
+                    .map(flatMessage -> new Message(topicName,
+                        JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue)))
+                    .collect(Collectors.toList());
+                // 批量发送
+                sendMessage(messages, partition);
+            }
+        }
+    }
+
+    private void sendMessage(Message message, int partition) {
+        try {
+            SendResult sendResult = this.defaultMQProducer.send(message, (mqs, msg, arg) -> {
+                if (partition > mqs.size()) {
+                    return mqs.get(partition % mqs.size());
+                } else {
+                    return mqs.get(partition);
+                }
+            }, null);
+
+            if (logger.isDebugEnabled()) {
+                logger.debug("Send Message Result: {}", sendResult);
+            }
+        } catch (Throwable e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @SuppressWarnings("deprecation")
+    private void sendMessage(List<Message> messages, int partition) {
+        if (messages.isEmpty()) {
+            return;
+        }
+
+        // 获取一下messageQueue
+        DefaultMQProducerImpl innerProducer = this.defaultMQProducer.getDefaultMQProducerImpl();
+        TopicPublishInfo topicInfo = innerProducer.getTopicPublishInfoTable().get(messages.get(0).getTopic());
+        if (topicInfo == null) {
+            for (Message message : messages) {
+                sendMessage(message, partition);
+            }
+        } else {
+            // 批量发送
+            List<MessageQueue> queues = topicInfo.getMessageQueueList();
+            int size = queues.size();
+            if (size <= 0) {
+                // 可能是第一次创建
+                for (Message message : messages) {
+                    sendMessage(message, partition);
+                }
+            } else {
+                MessageQueue queue;
+                if (partition > size) {
+                    queue = queues.get(partition % size);
+                } else {
+                    queue = queues.get(partition);
+                }
+
+                try {
+                    SendResult sendResult = this.defaultMQProducer.send(messages, queue);
+                    if (logger.isDebugEnabled()) {
+                        logger.debug("Send Message Result: {}", sendResult);
+                    }
+                } catch (Throwable e) {
+                    throw new RuntimeException(e);
+                }
+            }
+        }
+    }
+
+    @Override
+    public void stop() {
+        logger.info("## Stop RocketMQ producer##");
+        this.defaultMQProducer.shutdown();
+        super.stop();
+    }
+}

connector/rocketmq-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMQProducer

@@ -0,0 +1 @@
+rocketmq=com.alibaba.otter.canal.connector.rocketmq.producer.CanalRocketMQProducer

connector/rocketmq-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer

@@ -0,0 +1 @@
+rocketmq=com.alibaba.otter.canal.connector.rocketmq.consumer.CanalRocketMQConsumer

connector/tcp-connector/pom.xml

@@ -0,0 +1,124 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>canal.connector</artifactId>
+        <groupId>com.alibaba.otter</groupId>
+        <version>1.1.5-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>com.alibaba.otter</groupId>
+    <artifactId>connector.tcp</artifactId>
+    <packaging>jar</packaging>
+    <name>canal connector tcp module for otter ${project.version}</name>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>connector.core</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>canal.protocol</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>canal.client</artifactId>
+            <version>${project.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.google.protobuf</groupId>
+                    <artifactId>protobuf-java</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.springframework</groupId>
+                    <artifactId>spring-orm</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.springframework</groupId>
+                    <artifactId>spring-jdbc</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.springframework</groupId>
+                    <artifactId>spring-aop</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.springframework</groupId>
+                    <artifactId>spring-core</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.springframework</groupId>
+                    <artifactId>spring-context</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.rocketmq</groupId>
+                    <artifactId>rocketmq-client</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.aliyun.openservices</groupId>
+                    <artifactId>aliware-apache-rocketmq-cloud</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>2.4</version>
+                <configuration>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>make-assembly</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-antrun-plugin</artifactId>
+                <version>1.8</version>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>run</goal>
+                        </goals>
+                        <configuration>
+                            <tasks>
+                                <copy todir="${project.basedir}/../../client-adapter/launcher/target/canal-adapter/plugin"
+                                      overwrite="true">
+                                    <fileset dir="${project.basedir}/target" erroronmissingdir="true">
+                                        <include name="connector.tcp-${project.version}-jar-with-dependencies.jar"/>
+                                    </fileset>
+                                </copy>
+                            </tasks>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

connector/tcp-connector/src/main/java/com/alibaba/otter/canal/connector/tcp/config/TCPConstants.java

@@ -0,0 +1,12 @@
+package com.alibaba.otter.canal.connector.tcp.config;
+
+public class TCPConstants {
+
+    public static final String ROOT                 = "canal";
+
+    public static final String CANAL_TCP_HOST       = ROOT + "." + "tcp.server.host";
+    public static final String CANAL_TCP_ZK_HOSTS   = ROOT + "." + "tcp.zookeeper.hosts";
+    public static final String CANAL_TCP_USERNAME   = ROOT + "." + "tcp.username";
+    public static final String CANAL_TCP_PASSWORD   = ROOT + "." + "tcp.password";
+    public static final String CANAL_TCP_BATCH_SIZE = ROOT + "." + "tcp.batch.size";
+}

connector/tcp-connector/src/main/java/com/alibaba/otter/canal/connector/tcp/consumer/CanalTCPConsumer.java

@@ -0,0 +1,106 @@
+package com.alibaba.otter.canal.connector.tcp.consumer;
+
+import com.alibaba.otter.canal.client.CanalConnector;
+import com.alibaba.otter.canal.client.impl.ClusterCanalConnector;
+import com.alibaba.otter.canal.client.impl.ClusterNodeAccessStrategy;
+import com.alibaba.otter.canal.client.impl.SimpleCanalConnector;
+import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
+import com.alibaba.otter.canal.connector.core.consumer.CommonMessage;
+import com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer;
+import com.alibaba.otter.canal.connector.core.spi.SPI;
+import com.alibaba.otter.canal.connector.core.util.MessageUtil;
+import com.alibaba.otter.canal.connector.tcp.config.TCPConstants;
+import com.alibaba.otter.canal.protocol.Message;
+
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * TCP 消费者连接器, 一个destination对应一个SPI实例
+ * 
+ * @author rewerma 2020-01-30
+ * @version 1.0.0
+ */
+@SPI("tcp")
+public class CanalTCPConsumer implements CanalMsgConsumer {
+
+    private static final String PREFIX_TCP_CONFIG = "canal.tcp.";
+
+    private Long                currentBatchId    = null;
+
+    private CanalConnector      canalConnector;
+
+    private int                 batchSize         = 500;
+
+    @Override
+    public void init(Properties properties, String destination, String groupId) {
+        // load config
+        String host = properties.getProperty(TCPConstants.CANAL_TCP_HOST);
+        String username = properties.getProperty(TCPConstants.CANAL_TCP_USERNAME);
+        String password = properties.getProperty(TCPConstants.CANAL_TCP_PASSWORD);
+        String zkHosts = properties.getProperty(TCPConstants.CANAL_TCP_ZK_HOSTS);
+        String batchSizePro = properties.getProperty(TCPConstants.CANAL_TCP_BATCH_SIZE);
+        if (batchSizePro != null) {
+            batchSize = Integer.parseInt(batchSizePro);
+        }
+        if (host != null) {
+            String[] ipPort = host.split(":");
+            SocketAddress sa = new InetSocketAddress(ipPort[0], Integer.parseInt(ipPort[1]));
+            this.canalConnector = new SimpleCanalConnector(sa, username, password, destination);
+        } else {
+            this.canalConnector = new ClusterCanalConnector(username,
+                password,
+                destination,
+                new ClusterNodeAccessStrategy(destination, ZkClientx.getZkClient(zkHosts)));
+        }
+    }
+
+    @Override
+    public void connect() {
+        canalConnector.connect();
+        canalConnector.subscribe();
+    }
+
+    @Override
+    public List<CommonMessage> getMessage(Long timeout, TimeUnit unit) {
+        try {
+            Message message = canalConnector.getWithoutAck(batchSize, timeout, unit);
+            long batchId = message.getId();
+            currentBatchId = batchId;
+            int size = message.getEntries().size();
+            if (batchId == -1 || size == 0) {
+                return null;
+            } else {
+                return MessageUtil.convert(message);
+            }
+        } catch (Throwable e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public void rollback() {
+        if (currentBatchId != null && currentBatchId != -1) {
+            canalConnector.rollback(currentBatchId);
+            currentBatchId = null;
+        }
+    }
+
+    @Override
+    public void ack() {
+        if (currentBatchId != null) {
+            canalConnector.ack(currentBatchId);
+            currentBatchId = null;
+        }
+    }
+
+    @Override
+    public void disconnect() {
+        canalConnector.unsubscribe();
+        canalConnector.disconnect();
+    }
+}

connector/tcp-connector/src/main/resources/META-INF/canal/com.alibaba.otter.canal.connector.core.spi.CanalMsgConsumer

@@ -0,0 +1 @@
+tcp=com.alibaba.otter.canal.connector.tcp.consumer.CanalTCPConsumer

deployer/pom.xml

@@ -1,127 +1,185 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-	<modelVersion>4.0.0</modelVersion>
-	<parent>
-		<groupId>com.alibaba.otter</groupId>
-		<artifactId>canal</artifactId>
-		<version>1.1.5-SNAPSHOT</version>
-		<relativePath>../pom.xml</relativePath>
-	</parent>
-	<groupId>com.alibaba.otter</groupId>
-	<artifactId>canal.deployer</artifactId>
-	<packaging>jar</packaging>
-	<name>canal deployer module for otter ${project.version}</name>
-	<dependencies>
-		<dependency>
-			<groupId>com.alibaba.otter</groupId>
-			<artifactId>canal.server</artifactId>
-			<version>${project.version}</version>
-		</dependency>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>com.alibaba.otter</groupId>
+        <artifactId>canal</artifactId>
+        <version>1.1.5-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+    <groupId>com.alibaba.otter</groupId>
+    <artifactId>canal.deployer</artifactId>
+    <packaging>jar</packaging>
+    <name>canal deployer module for otter ${project.version}</name>
+    <dependencies>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>canal.server</artifactId>
+            <version>${project.version}</version>
+        </dependency>
 
-		<!-- 这里指定runtime的metrics provider-->
-		<dependency>
-			<groupId>com.alibaba.otter</groupId>
-			<artifactId>canal.prometheus</artifactId>
-			<version>${project.version}</version>
-			<scope>runtime</scope>
-		</dependency>
-	</dependencies>
-	
-	<build>
-		<plugins>
-			<!-- deploy模块的packaging通常是jar，如果项目中没有java 源代码或资源文件，加上这一段配置使项目能通过构建 -->
-			<plugin>
-				<artifactId>maven-jar-plugin</artifactId>
-				<configuration>
-					<archive>
-						<addMavenDescriptor>true</addMavenDescriptor>
-					</archive>
-					<excludes>
-						<exclude>**/logback.xml</exclude>
-						<exclude>**/canal.properties</exclude>
-						<exclude>**/spring/**</exclude>
-						<exclude>**/example/**</exclude>
-						<exclude>**/mq.yml</exclude>
-					</excludes>
-				</configuration>
-			</plugin>
+        <!-- 这里指定runtime的metrics provider-->
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>canal.prometheus</artifactId>
+            <version>${project.version}</version>
+            <scope>runtime</scope>
+        </dependency>
 
-			<plugin>
-				<groupId>org.apache.maven.plugins</groupId>
-				<artifactId>maven-assembly-plugin</artifactId>
-				<!-- 这是最新版本，推荐使用这个版本 -->
-				<version>2.2.1</version>
-				<executions>
-					<execution>
-						<id>assemble</id>
-						<goals>
-							<goal>single</goal>
-						</goals>
-						<phase>package</phase>
-					</execution>
-				</executions>
-				<configuration>
-					<appendAssemblyId>false</appendAssemblyId>
-					<attach>false</attach>
-				</configuration>
-			</plugin>
-		</plugins>
-	</build>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>connector.kafka</artifactId>
+            <version>${project.version}</version>
+            <exclusions>
+                <exclusion>
+                    <artifactId>*</artifactId>
+                    <groupId>*</groupId>
+                </exclusion>
+            </exclusions>
+            <classifier>jar-with-dependencies</classifier>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>connector.rocketmq</artifactId>
+            <version>${project.version}</version>
+            <exclusions>
+                <exclusion>
+                    <artifactId>*</artifactId>
+                    <groupId>*</groupId>
+                </exclusion>
+            </exclusions>
+            <classifier>jar-with-dependencies</classifier>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.alibaba.otter</groupId>
+            <artifactId>connector.rabbitmq</artifactId>
+            <version>${project.version}</version>
+            <exclusions>
+                <exclusion>
+                    <artifactId>*</artifactId>
+                    <groupId>*</groupId>
+                </exclusion>
+            </exclusions>
+            <classifier>jar-with-dependencies</classifier>
+            <scope>provided</scope>
+        </dependency>
+    </dependencies>
 
-	<profiles>
-		<profile>
-			<id>dev</id>
-			<activation>
-				<activeByDefault>true</activeByDefault>
-				<property>
-					<name>env</name>
-					<value>!release</value>
-				</property>
-			</activation>
+    <build>
+        <plugins>
+            <plugin>
+                <artifactId>maven-jar-plugin</artifactId>
+                <configuration>
+                    <archive>
+                        <addMavenDescriptor>true</addMavenDescriptor>
+                    </archive>
+                    <excludes>
+                        <exclude>**/logback.xml</exclude>
+                        <exclude>**/canal.properties</exclude>
+                        <exclude>**/spring/**</exclude>
+                        <exclude>**/example/**</exclude>
+                        <exclude>**/mq.yml</exclude>
+                    </excludes>
+                </configuration>
+            </plugin>
 
-			<build>
-				<plugins>
-					<plugin>
-						<artifactId>maven-assembly-plugin</artifactId>
-						<configuration>
-							<!-- maven assembly插件需要一个描述文件 来告诉插件包的结构以及打包所需的文件来自哪里 -->
-							<descriptors>
-								<descriptor>${basedir}/src/main/assembly/dev.xml</descriptor>
-							</descriptors>
-							<finalName>canal</finalName>
-							<outputDirectory>${project.build.directory}</outputDirectory>
-						</configuration>
-					</plugin>
-				</plugins>
-			</build>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <!-- 这是最新版本，推荐使用这个版本 -->
+                <version>2.2.1</version>
+                <executions>
+                    <execution>
+                        <id>assemble</id>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                        <phase>package</phase>
+                    </execution>
+                </executions>
+                <configuration>
+                    <appendAssemblyId>false</appendAssemblyId>
+                    <attach>false</attach>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-dependency-plugin</artifactId>
+                <version>2.10</version>
+                <executions>
+                    <execution>
+                        <id>copy-dependencies-to-canal-client-service</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>copy-dependencies</goal>
+                        </goals>
+                        <configuration>
+                            <includeClassifiers>jar-with-dependencies</includeClassifiers>
+                            <outputDirectory>${project.basedir}/target/canal/plugin</outputDirectory>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
 
-		</profile>
+    <profiles>
+        <profile>
+            <id>dev</id>
+            <activation>
+                <activeByDefault>true</activeByDefault>
+                <property>
+                    <name>env</name>
+                    <value>!release</value>
+                </property>
+            </activation>
 
-		<profile>
-			<id>release</id>
-			<activation>
-				<property>
-					<name>env</name>
-					<value>release</value>
-				</property>
-			</activation>
+            <build>
+                <plugins>
+                    <plugin>
+                        <artifactId>maven-assembly-plugin</artifactId>
+                        <configuration>
+                            <!-- maven assembly插件需要一个描述文件 来告诉插件包的结构以及打包所需的文件来自哪里 -->
+                            <descriptors>
+                                <descriptor>${basedir}/src/main/assembly/dev.xml</descriptor>
+                            </descriptors>
+                            <finalName>canal</finalName>
+                            <outputDirectory>${project.build.directory}</outputDirectory>
+                        </configuration>
+                    </plugin>
+                </plugins>
+            </build>
 
-			<build>
-				<plugins>
-					<plugin>
-						<artifactId>maven-assembly-plugin</artifactId>
-						<configuration>
-							<!-- 发布模式使用的maven assembly插件描述文件 -->
-							<descriptors>
-								<descriptor>${basedir}/src/main/assembly/release.xml</descriptor>
-							</descriptors>
-							<!-- 如果一个应用的包含多个deploy模块，如果使用同样的包名， 如果把它们复制的一个目录中可能会失败，所以包名加了 artifactId以示区分 -->
-							<finalName>${project.artifactId}-${project.version}</finalName>
-							<!-- scm 要求 release 模式打出的包放到顶级目录下的target子目录中 -->
-							<outputDirectory>${project.parent.build.directory}</outputDirectory>
-						</configuration>
-					</plugin>
-				</plugins>
-			</build>
-		</profile>
-	</profiles>
+        </profile>
+
+        <profile>
+            <id>release</id>
+            <activation>
+                <property>
+                    <name>env</name>
+                    <value>release</value>
+                </property>
+            </activation>
+
+            <build>
+                <plugins>
+                    <plugin>
+                        <artifactId>maven-assembly-plugin</artifactId>
+                        <configuration>
+                            <!-- 发布模式使用的maven assembly插件描述文件 -->
+                            <descriptors>
+                                <descriptor>${basedir}/src/main/assembly/release.xml</descriptor>
+                            </descriptors>
+                            <!-- 如果一个应用的包含多个deploy模块，如果使用同样的包名， 如果把它们复制的一个目录中可能会失败，所以包名加了 artifactId以示区分 -->
+                            <finalName>${project.artifactId}-${project.version}</finalName>
+                            <!-- scm 要求 release 模式打出的包放到顶级目录下的target子目录中 -->
+                            <outputDirectory>${project.parent.build.directory}</outputDirectory>
+                        </configuration>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+    </profiles>
 </project>

deployer/src/main/assembly/release.xml

@@ -42,6 +42,10 @@
 				<exclude>**/*</exclude>
 			</excludes>
 		</fileSet>
+		<fileSet>
+			<directory>${project.basedir}/target/canal/plugin</directory>
+			<outputDirectory>/plugin/</outputDirectory>
+		</fileSet>
 	</fileSets>
 	<dependencySets>
 		<dependencySet>

deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalConstants.java

@@ -46,36 +46,37 @@ public class CanalConstants {
 
     public static final String CANAL_SOCKETCHANNEL                  = ROOT + "." + "socketChannel";
 
-    public static final String CANAL_MQ_SERVERS                     = ROOT + "." + "mq.servers";
-    public static final String CANAL_MQ_RETRIES                     = ROOT + "." + "mq.retries";
-    public static final String CANAL_MQ_BATCHSIZE                   = ROOT + "." + "mq.batchSize";
-    public static final String CANAL_MQ_LINGERMS                    = ROOT + "." + "mq.lingerMs";
-    public static final String CANAL_MQ_MAXREQUESTSIZE              = ROOT + "." + "mq.maxRequestSize";
-    public static final String CANAL_MQ_BUFFERMEMORY                = ROOT + "." + "mq.bufferMemory";
-    public static final String CANAL_MQ_CANALBATCHSIZE              = ROOT + "." + "mq.canalBatchSize";
-    public static final String CANAL_MQ_CANALGETTIMEOUT             = ROOT + "." + "mq.canalGetTimeout";
-    public static final String CANAL_MQ_FLATMESSAGE                 = ROOT + "." + "mq.flatMessage";
-    public static final String CANAL_MQ_PARALLELTHREADSIZE          = ROOT + "." + "mq.parallelThreadSize";
-    public static final String CANAL_MQ_COMPRESSION_TYPE            = ROOT + "." + "mq.compressionType";
-    public static final String CANAL_MQ_ACKS                        = ROOT + "." + "mq.acks";
-    public static final String CANAL_MQ_TRANSACTION                 = ROOT + "." + "mq.transaction";
-    public static final String CANAL_MQ_PRODUCERGROUP               = ROOT + "." + "mq.producerGroup";
     public static final String CANAL_ALIYUN_ACCESSKEY               = ROOT + "." + "aliyun.accessKey";
     public static final String CANAL_ALIYUN_SECRETKEY               = ROOT + "." + "aliyun.secretKey";
-    public static final String CANAL_MQ_PROPERTIES                  = ROOT + "." + "mq.properties";
-    public static final String CANAL_MQ_ENABLE_MESSAGE_TRACE        = ROOT + "." + "mq.enableMessageTrace";
-    public static final String CANAL_MQ_ACCESS_CHANNEL              = ROOT + "." + "mq.accessChannel";
-    public static final String CANAL_MQ_CUSTOMIZED_TRACE_TOPIC      = ROOT + "." + "mq.customizedTraceTopic";
-    public static final String CANAL_MQ_NAMESPACE                   = ROOT + "." + "mq.namespace";
-    public static final String CANAL_MQ_KAFKA_KERBEROS_ENABLE       = ROOT + "." + "mq.kafka.kerberos.enable";
-    public static final String CANAL_MQ_KAFKA_KERBEROS_KRB5FILEPATH = ROOT + "." + "mq.kafka.kerberos.krb5FilePath";
-    public static final String CANAL_MQ_KAFKA_KERBEROS_JAASFILEPATH = ROOT + "." + "mq.kafka.kerberos.jaasFilePath";
-    public static final String CANAL_MQ_USERNAME                    = ROOT + "." + "mq.username";
-    public static final String CANAL_MQ_PASSWORD                    = ROOT + "." + "mq.password";
-    public static final String CANAL_MQ_VHOST                       = ROOT + "." + "mq.vhost";
-    public static final String CANAL_MQ_ALIYUN_UID                  = ROOT + "." + "mq.aliyunuid";
-    public static final String CANAL_MQ_EXCHANGE                    = ROOT + "." + "mq.exchange";
-    public static final String CANAL_MQ_DATABASE_HASH               = ROOT + "." + "mq.database.hash";
+
+//    public static final String CANAL_MQ_SERVERS                     = ROOT + "." + "mq.servers";
+//    public static final String CANAL_MQ_RETRIES                     = ROOT + "." + "mq.retries";
+//    public static final String CANAL_MQ_BATCHSIZE                   = ROOT + "." + "mq.batchSize";
+//    public static final String CANAL_MQ_LINGERMS                    = ROOT + "." + "mq.lingerMs";
+//    public static final String CANAL_MQ_MAXREQUESTSIZE              = ROOT + "." + "mq.maxRequestSize";
+//    public static final String CANAL_MQ_BUFFERMEMORY                = ROOT + "." + "mq.bufferMemory";
+//    public static final String CANAL_MQ_CANALBATCHSIZE              = ROOT + "." + "mq.canalBatchSize";
+//    public static final String CANAL_MQ_CANALGETTIMEOUT             = ROOT + "." + "mq.canalGetTimeout";
+//    public static final String CANAL_MQ_FLATMESSAGE                 = ROOT + "." + "mq.flatMessage";
+//    public static final String CANAL_MQ_PARALLELTHREADSIZE          = ROOT + "." + "mq.parallelThreadSize";
+//    public static final String CANAL_MQ_COMPRESSION_TYPE            = ROOT + "." + "mq.compressionType";
+//    public static final String CANAL_MQ_ACKS                        = ROOT + "." + "mq.acks";
+//    public static final String CANAL_MQ_TRANSACTION                 = ROOT + "." + "mq.transaction";
+//    public static final String CANAL_MQ_PRODUCERGROUP               = ROOT + "." + "mq.producerGroup";
+//    public static final String CANAL_MQ_PROPERTIES                  = ROOT + "." + "mq.properties";
+//    public static final String CANAL_MQ_ENABLE_MESSAGE_TRACE        = ROOT + "." + "mq.enableMessageTrace";
+//    public static final String CANAL_MQ_ACCESS_CHANNEL              = ROOT + "." + "mq.accessChannel";
+//    public static final String CANAL_MQ_CUSTOMIZED_TRACE_TOPIC      = ROOT + "." + "mq.customizedTraceTopic";
+//    public static final String CANAL_MQ_NAMESPACE                   = ROOT + "." + "mq.namespace";
+//    public static final String CANAL_MQ_KAFKA_KERBEROS_ENABLE       = ROOT + "." + "mq.kafka.kerberos.enable";
+//    public static final String CANAL_MQ_KAFKA_KERBEROS_KRB5FILEPATH = ROOT + "." + "mq.kafka.kerberos.krb5FilePath";
+//    public static final String CANAL_MQ_KAFKA_KERBEROS_JAASFILEPATH = ROOT + "." + "mq.kafka.kerberos.jaasFilePath";
+//    public static final String CANAL_MQ_USERNAME                    = ROOT + "." + "mq.username";
+//    public static final String CANAL_MQ_PASSWORD                    = ROOT + "." + "mq.password";
+//    public static final String CANAL_MQ_VHOST                       = ROOT + "." + "mq.vhost";
+//    public static final String CANAL_MQ_ALIYUN_UID                  = ROOT + "." + "mq.aliyunuid";
+//    public static final String CANAL_MQ_EXCHANGE                    = ROOT + "." + "mq.exchange";
+//    public static final String CANAL_MQ_DATABASE_HASH               = ROOT + "." + "mq.database.hash";
 
     public static String getInstanceModeKey(String destination) {
         return MessageFormat.format(INSTANCE_MODE_TEMPLATE, destination);

deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalStarter.java

@@ -2,35 +2,36 @@ package com.alibaba.otter.canal.deployer;
 
 import java.util.Properties;
 
+import com.alibaba.otter.canal.connector.core.config.MQProperties;
 import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.alibaba.otter.canal.admin.netty.CanalAdminWithNetty;
-import com.alibaba.otter.canal.common.MQProperties;
+import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;
+import com.alibaba.otter.canal.connector.core.spi.ExtensionLoader;
 import com.alibaba.otter.canal.deployer.admin.CanalAdminController;
-import com.alibaba.otter.canal.kafka.CanalKafkaProducer;
-import com.alibaba.otter.canal.rabbitmq.CanalRabbitMQProducer;
-import com.alibaba.otter.canal.rocketmq.CanalRocketMQProducer;
 import com.alibaba.otter.canal.server.CanalMQStarter;
-import com.alibaba.otter.canal.spi.CanalMQProducer;
 
 /**
  * Canal server 启动类
  *
- * @author rewerma 2018-12-30 下午05:12:16
- * @version 1.0.1
+ * @author rewerma 2020-01-27
+ * @version 1.0.2
  */
 public class CanalStarter {
 
-    private static final Logger logger          = LoggerFactory.getLogger(CanalStarter.class);
+    private static final Logger logger                    = LoggerFactory.getLogger(CanalStarter.class);
 
-    private CanalController     controller      = null;
-    private CanalMQProducer     canalMQProducer = null;
-    private Thread              shutdownThread  = null;
-    private CanalMQStarter      canalMQStarter  = null;
+    private static final String CONNECTOR_SPI_DIR         = "/plugin";
+    private static final String CONNECTOR_STANDBY_SPI_DIR = "/canal/plugin";
+
+    private CanalController     controller                = null;
+    private CanalMQProducer     canalMQProducer           = null;
+    private Thread              shutdownThread            = null;
+    private CanalMQStarter      canalMQStarter            = null;
     private volatile Properties properties;
-    private volatile boolean    running         = false;
+    private volatile boolean    running                   = false;
 
     private CanalAdminWithNetty canalAdmin;
 
@@ -61,20 +62,23 @@ public class CanalStarter {
      */
     public synchronized void start() throws Throwable {
         String serverMode = CanalController.getProperty(properties, CanalConstants.CANAL_SERVER_MODE);
-        if (serverMode.equalsIgnoreCase("kafka")) {
-            canalMQProducer = new CanalKafkaProducer();
-        } else if (serverMode.equalsIgnoreCase("rocketmq")) {
-            canalMQProducer = new CanalRocketMQProducer();
-        } else if (serverMode.equalsIgnoreCase("rabbitmq")) {
-            canalMQProducer = new CanalRabbitMQProducer();
+        if (!"tcp".equalsIgnoreCase(serverMode)) {
+            ExtensionLoader<CanalMQProducer> loader = ExtensionLoader.getExtensionLoader(CanalMQProducer.class);
+            canalMQProducer = loader
+                .getExtension(serverMode.toLowerCase(), CONNECTOR_SPI_DIR, CONNECTOR_STANDBY_SPI_DIR);
+            if (canalMQProducer != null) {
+                ClassLoader cl = Thread.currentThread().getContextClassLoader();
+                Thread.currentThread().setContextClassLoader(canalMQProducer.getClass().getClassLoader());
+                canalMQProducer.init(properties);
+                Thread.currentThread().setContextClassLoader(cl);
+            }
         }
 
-        MQProperties mqProperties = null;
         if (canalMQProducer != null) {
-            mqProperties = buildMQProperties(properties);
+            MQProperties mqProperties = canalMQProducer.getMqProperties();
             // disable netty
             System.setProperty(CanalConstants.CANAL_WITHOUT_NETTY, "true");
-            if (mqProperties.getFlatMessage()) {
+            if (mqProperties.isFlatMessage()) {
                 // 设置为raw避免ByteString->Entry的二次解析
                 System.setProperty("canal.instance.memory.rawEntry", "false");
             }
@@ -104,7 +108,7 @@ public class CanalStarter {
         if (canalMQProducer != null) {
             canalMQStarter = new CanalMQStarter(canalMQProducer);
             String destinations = CanalController.getProperty(properties, CanalConstants.CANAL_DESTINATIONS);
-            canalMQStarter.start(mqProperties, destinations);
+            canalMQStarter.start(destinations);
             controller.setCanalMQStarter(canalMQStarter);
         }
 
@@ -119,11 +123,15 @@ public class CanalStarter {
 
             String ip = CanalController.getProperty(properties, CanalConstants.CANAL_IP);
 
-            logger.debug("canal admin port:{}, canal admin user:{}, canal admin password: {}, canal ip:{}", port, user, passwd, ip);
+            logger.debug("canal admin port:{}, canal admin user:{}, canal admin password: {}, canal ip:{}",
+                port,
+                user,
+                passwd,
+                ip);
 
             CanalAdminWithNetty canalAdminWithNetty = CanalAdminWithNetty.instance();
             canalAdminWithNetty.setCanalAdmin(canalAdmin);
-            canalAdminWithNetty.setPort(Integer.valueOf(port));
+            canalAdminWithNetty.setPort(Integer.parseInt(port));
             canalAdminWithNetty.setIp(ip);
             canalAdminWithNetty.start();
             this.canalAdmin = canalAdminWithNetty;
@@ -162,156 +170,4 @@ public class CanalStarter {
         }
         running = false;
     }
-
-    /**
-     * 构造MQ对应的配置
-     *
-     * @param properties canal.properties 配置
-     * @return
-     */
-    private static MQProperties buildMQProperties(Properties properties) {
-        MQProperties mqProperties = new MQProperties();
-        String servers = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_SERVERS);
-        if (!StringUtils.isEmpty(servers)) {
-            mqProperties.setServers(servers);
-        }
-        String retires = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_RETRIES);
-        if (!StringUtils.isEmpty(retires)) {
-            mqProperties.setRetries(Integer.valueOf(retires));
-        }
-        String batchSize = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_BATCHSIZE);
-        if (!StringUtils.isEmpty(batchSize)) {
-            mqProperties.setBatchSize(Integer.valueOf(batchSize));
-        }
-        String lingerMs = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_LINGERMS);
-        if (!StringUtils.isEmpty(lingerMs)) {
-            mqProperties.setLingerMs(Integer.valueOf(lingerMs));
-        }
-        String maxRequestSize = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_MAXREQUESTSIZE);
-        if (!StringUtils.isEmpty(maxRequestSize)) {
-            mqProperties.setMaxRequestSize(Integer.valueOf(maxRequestSize));
-        }
-        String bufferMemory = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_BUFFERMEMORY);
-        if (!StringUtils.isEmpty(bufferMemory)) {
-            mqProperties.setBufferMemory(Long.valueOf(bufferMemory));
-        }
-        String canalBatchSize = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_CANALBATCHSIZE);
-        if (!StringUtils.isEmpty(canalBatchSize)) {
-            mqProperties.setCanalBatchSize(Integer.valueOf(canalBatchSize));
-        }
-        String canalGetTimeout = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_CANALGETTIMEOUT);
-        if (!StringUtils.isEmpty(canalGetTimeout)) {
-            mqProperties.setCanalGetTimeout(Long.valueOf(canalGetTimeout));
-        }
-        String flatMessage = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_FLATMESSAGE);
-        if (!StringUtils.isEmpty(flatMessage)) {
-            mqProperties.setFlatMessage(Boolean.valueOf(flatMessage));
-        }
-        String parallelThreadSize = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_PARALLELTHREADSIZE);
-        if (!StringUtils.isEmpty(parallelThreadSize)) {
-            mqProperties.setParallelThreadSize(Integer.valueOf(parallelThreadSize));
-        }
-        String compressionType = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_COMPRESSION_TYPE);
-        if (!StringUtils.isEmpty(compressionType)) {
-            mqProperties.setCompressionType(compressionType);
-        }
-        String acks = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_ACKS);
-        if (!StringUtils.isEmpty(acks)) {
-            mqProperties.setAcks(acks);
-        }
-        String aliyunAccessKey = CanalController.getProperty(properties, CanalConstants.CANAL_ALIYUN_ACCESSKEY);
-        if (!StringUtils.isEmpty(aliyunAccessKey)) {
-            mqProperties.setAliyunAccessKey(aliyunAccessKey);
-        }
-        String aliyunSecretKey = CanalController.getProperty(properties, CanalConstants.CANAL_ALIYUN_SECRETKEY);
-        if (!StringUtils.isEmpty(aliyunSecretKey)) {
-            mqProperties.setAliyunSecretKey(aliyunSecretKey);
-        }
-
-        String producerGroup = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_PRODUCERGROUP);
-        if (!StringUtils.isEmpty(producerGroup)) {
-            mqProperties.setProducerGroup(producerGroup);
-        }
-
-        String enableMessageTrace = CanalController.getProperty(properties,
-            CanalConstants.CANAL_MQ_ENABLE_MESSAGE_TRACE);
-        if (!StringUtils.isEmpty(enableMessageTrace)) {
-            mqProperties.setEnableMessageTrace(Boolean.valueOf(enableMessageTrace));
-        }
-
-        String accessChannel = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_ACCESS_CHANNEL);
-        if (!StringUtils.isEmpty(accessChannel)) {
-            mqProperties.setAccessChannel(accessChannel);
-        }
-
-        String customizedTraceTopic = CanalController.getProperty(properties,
-            CanalConstants.CANAL_MQ_CUSTOMIZED_TRACE_TOPIC);
-        if (!StringUtils.isEmpty(customizedTraceTopic)) {
-            mqProperties.setCustomizedTraceTopic(customizedTraceTopic);
-        }
-
-        String namespace = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_NAMESPACE);
-        if (!StringUtils.isEmpty(namespace)) {
-            mqProperties.setNamespace(namespace);
-        }
-
-        String kafkaKerberosEnable = CanalController.getProperty(properties,
-            CanalConstants.CANAL_MQ_KAFKA_KERBEROS_ENABLE);
-        if (!StringUtils.isEmpty(kafkaKerberosEnable)) {
-            mqProperties.setKerberosEnable(Boolean.valueOf(kafkaKerberosEnable));
-        }
-
-        String kafkaKerberosKrb5Filepath = CanalController.getProperty(properties,
-            CanalConstants.CANAL_MQ_KAFKA_KERBEROS_KRB5FILEPATH);
-        if (!StringUtils.isEmpty(kafkaKerberosKrb5Filepath)) {
-            mqProperties.setKerberosKrb5FilePath(kafkaKerberosKrb5Filepath);
-        }
-
-        String kafkaKerberosJaasFilepath = CanalController.getProperty(properties,
-            CanalConstants.CANAL_MQ_KAFKA_KERBEROS_JAASFILEPATH);
-        if (!StringUtils.isEmpty(kafkaKerberosJaasFilepath)) {
-            mqProperties.setKerberosJaasFilePath(kafkaKerberosJaasFilepath);
-        }
-
-        String vhost = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_VHOST);
-        if (!StringUtils.isEmpty(vhost)) {
-            mqProperties.setVhost(vhost);
-        }
-
-        String username = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_USERNAME);
-        if (!StringUtils.isEmpty(username)) {
-            mqProperties.setUsername(username);
-        }
-
-        String password = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_PASSWORD);
-        if (!StringUtils.isEmpty(password)) {
-            mqProperties.setPassword(password);
-        }
-
-        String aliyunUID = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_ALIYUN_UID);
-        if (!StringUtils.isEmpty(aliyunUID)) {
-            mqProperties.setAliyunUID(Long.valueOf(aliyunUID));
-        }
-
-        String exchange = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_EXCHANGE);
-        if (!StringUtils.isEmpty(exchange)) {
-            mqProperties.setExchange(exchange);
-        }
-
-        String databaseHash = CanalController.getProperty(properties, CanalConstants.CANAL_MQ_DATABASE_HASH);
-        if (!StringUtils.isEmpty(databaseHash)){
-            mqProperties.setDatabaseHash(Boolean.valueOf(databaseHash));
-        }
-
-        for (Object key : properties.keySet()) {
-            key = StringUtils.trim(key.toString());
-            if (((String) key).startsWith(CanalConstants.CANAL_MQ_PROPERTIES)) {
-                String value = CanalController.getProperty(properties, (String) key);
-                String subKey = ((String) key).substring(CanalConstants.CANAL_MQ_PROPERTIES.length() + 1);
-                mqProperties.getProperties().put(subKey, value);
-            }
-        }
-
-        return mqProperties;
-    }
 }

deployer/src/main/resources/canal.properties

@@ -21,7 +21,7 @@ canal.zkServers =
 # flush data to zk
 canal.zookeeper.flush.period = 1000
 canal.withoutNetty = false
-# tcp, kafka, RocketMQ
+# tcp, kafka, rocketMQ, rabbitMQ
 canal.serverMode = tcp
 # flush meta cursor/parse position to file
 canal.file.data.dir = ${canal.conf.dir}
@@ -86,10 +86,6 @@ canal.instance.tsdb.snapshot.interval = 24
 # purge snapshot expire , default 360 hour(15 days)
 canal.instance.tsdb.snapshot.expire = 360
 
-# aliyun ak/sk , support rds/mq
-canal.aliyun.accessKey =
-canal.aliyun.secretKey =
-
 #################################################
 ######### 		destinations		#############
 #################################################
@@ -111,35 +107,54 @@ canal.instance.global.spring.xml = classpath:spring/file-instance.xml
 #canal.instance.global.spring.xml = classpath:spring/default-instance.xml
 
 ##################################################
-######### 		     MQ 		     #############
+######### 	      MQ Properties      #############
 ##################################################
-canal.mq.servers = 127.0.0.1:6667
-canal.mq.retries = 0
-canal.mq.batchSize = 16384
-canal.mq.maxRequestSize = 1048576
-canal.mq.lingerMs = 100
-canal.mq.bufferMemory = 33554432
-canal.mq.canalBatchSize = 50
-canal.mq.canalGetTimeout = 100
-canal.mq.parallelThreadSize = 8
-canal.mq.flatMessage = true
-canal.mq.compressionType = none
-canal.mq.acks = all
-#canal.mq.properties. =
-canal.mq.producerGroup = test
-# Set this value to "cloud", if you want open message trace feature in aliyun.
-canal.mq.accessChannel = local
-# aliyun mq namespace
-#canal.mq.namespace =
-canal.mq.vhost=
-canal.mq.exchange=
-canal.mq.username=
-canal.mq.password=
-canal.mq.aliyunuid=
+canal.mq.flat.message = true
 canal.mq.database.hash = true
+canal.mq.parallel.thread.size = 8
+canal.mq.canal.batch.size = 50
+canal.mq.canal.fetch.timeout = 100
+# Set this value to "cloud", if you want open message trace feature in aliyun.
+canal.mq.access.channel = local
+
+# aliyun ak/sk , support rds/mq
+canal.aliyun.accessKey =
+canal.aliyun.secretKey =
+canal.aliyun.uid=
+
 ##################################################
-#########     Kafka Kerberos Info    #############
+######### 		     Kafka 		     #############
 ##################################################
+kafka.bootstrap.servers = 127.0.0.1:9092
+kafka.acks = all
+kafka.compression.type = none
+kafka.batch.size = 16384
+kafka.linger.ms = 1
+kafka.max.request.size = 1048576
+kafka.buffer.memory = 33554432
+kafka.max.in.flight.requests.per.connection = 1
+kafka.retries = 0
+
 canal.mq.kafka.kerberos.enable = false
-canal.mq.kafka.kerberos.krb5FilePath = "../conf/kerberos/krb5.conf"
-canal.mq.kafka.kerberos.jaasFilePath = "../conf/kerberos/jaas.conf"
+canal.mq.kafka.kerberos.krb5.file = "../conf/kerberos/krb5.conf"
+canal.mq.kafka.kerberos.jaas.file = "../conf/kerberos/jaas.conf"
+
+##################################################
+######### 		    RocketMQ	     #############
+##################################################
+rocketmq.producer.group = test
+rocketmq.enable.message.trace = false
+rocketmq.customized.trace.topic =
+rocketmq.namespace =
+rocketmq.namesrv.addr = 127.0.0.1:9876
+rocketmq.retry.times.when.send.failed = 0
+rocketmq.vip.channel.enabled = false
+
+##################################################
+######### 		    RabbitMQ	     #############
+##################################################
+rabbitmq.host =
+rabbitmq.virtual.host =
+rabbitmq.exchange =
+rabbitmq.username =
+rabbitmq.password =

deployer/src/main/resources/logback.xml

@@ -73,11 +73,15 @@
 		<level value="INFO" />
 		<appender-ref ref="CANAL-META" />
 	</logger>
-	<logger name="com.alibaba.otter.canal.kafka" additivity="false">
+	<logger name="com.alibaba.otter.canal.connector.kafka" additivity="false">
 		<level value="INFO" />
 		<appender-ref ref="CANAL-ROOT" />
 	</logger>
-	<logger name="com.alibaba.otter.canal.rabbitmq" additivity="false">
+	<logger name="com.alibaba.otter.canal.connector.rocketmq" additivity="false">
+		<level value="INFO" />
+		<appender-ref ref="CANAL-ROOT" />
+	</logger>
+	<logger name="com.alibaba.otter.canal.connector.rabbitmq" additivity="false">
 		<level value="INFO" />
 		<appender-ref ref="CANAL-ROOT" />
 	</logger>

pom.xml

@@ -100,8 +100,6 @@
         <file_encoding>UTF-8</file_encoding>
         <javadoc_skip>true</javadoc_skip>
         <spring_version>3.2.18.RELEASE</spring_version>
-        <rocketmq_version>4.5.2</rocketmq_version>
-        <rabbitmq_version>5.5.0</rabbitmq_version>
         <maven-jacoco-plugin.version>0.8.3</maven-jacoco-plugin.version>
         <maven-surefire.version>2.22.1</maven-surefire.version>
         <argline>-server -Xms512m -Xmx1024m -Dfile.encoding=UTF-8
@@ -129,6 +127,7 @@
         <module>prometheus</module>
         <module>admin</module>
         <module>client-adapter</module>
+        <module>connector</module>
     </modules>
 
     <dependencyManagement>
@@ -233,6 +232,11 @@
                 <artifactId>aviator</artifactId>
                 <version>2.2.1</version>
             </dependency>
+            <dependency>
+                <groupId>com.alibaba</groupId>
+                <artifactId>fastjson</artifactId>
+                <version>1.2.58.sec06</version>
+            </dependency>
             <dependency>
                 <groupId>oro</groupId>
                 <artifactId>oro</artifactId>
@@ -336,26 +340,6 @@
                 <artifactId>jsr305</artifactId>
                 <version>3.0.2</version>
             </dependency>
-            <dependency>
-                <groupId>org.apache.rocketmq</groupId>
-                <artifactId>rocketmq-client</artifactId>
-                <version>${rocketmq_version}</version>
-            </dependency>
-            <dependency>
-                <groupId>org.apache.rocketmq</groupId>
-                <artifactId>rocketmq-acl</artifactId>
-                <version>${rocketmq_version}</version>
-            </dependency>
-            <dependency>
-                <groupId>com.rabbitmq</groupId>
-                <artifactId>amqp-client</artifactId>
-                <version>${rabbitmq_version}</version>
-            </dependency>
-            <dependency>
-                <groupId>com.alibaba.mq-amqp</groupId>
-                <artifactId>mq-amqp-client</artifactId>
-                <version>1.0.3</version>
-            </dependency>
             <dependency>
                 <groupId>javax.annotation</groupId>
                 <artifactId>javax.annotation-api</artifactId>

server/pom.xml

@@ -25,39 +25,15 @@
 			<artifactId>canal.instance.manager</artifactId>
 			<version>${project.version}</version>
 		</dependency>
-		<!-- kafka -->
 		<dependency>
-			<groupId>org.apache.kafka</groupId>
-			<artifactId>kafka_2.11</artifactId>
-			<version>1.1.1</version>
-			<exclusions>
-				<exclusion>
-					<groupId>org.slf4j</groupId>
-					<artifactId>slf4j-log4j12</artifactId>
-				</exclusion>
-			</exclusions>
-		</dependency>
-		<dependency>
-			<groupId>org.apache.rocketmq</groupId>
-			<artifactId>rocketmq-client</artifactId>
-		</dependency>
-		<dependency>
-			<groupId>org.apache.rocketmq</groupId>
-			<artifactId>rocketmq-acl</artifactId>
-		</dependency>
-		<dependency>
-			<groupId>com.rabbitmq</groupId>
-			<artifactId>amqp-client</artifactId>
-		</dependency>
-		<dependency>
-			<groupId>com.alibaba.mq-amqp</groupId>
-			<artifactId>mq-amqp-client</artifactId>
+			<groupId>com.alibaba.otter</groupId>
+			<artifactId>connector.core</artifactId>
+			<version>${project.version}</version>
 		</dependency>
-		<!--kafka_2.11_1.1.1 exclusion掉了netty 的依赖，但CanalServerWithNetty 依赖 netty3,升级kafka至 1.1.1 需要显示加入,否则会启动失败 -->
 		<dependency>
-			<groupId>org.jboss.netty</groupId>
-			<artifactId>netty</artifactId>
-			<version>3.2.2.Final</version>
+			<groupId>org.apache.commons</groupId>
+			<artifactId>commons-lang3</artifactId>
+			<version>3.7</version>
 		</dependency>
 		
 		<!-- test dependency -->

server/src/main/java/com/alibaba/otter/canal/common/AbstractMQProducer.java

@@ -1,37 +0,0 @@
-package com.alibaba.otter.canal.common;
-
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-import com.alibaba.otter.canal.common.utils.NamedThreadFactory;
-import com.alibaba.otter.canal.spi.CanalMQProducer;
-
-/**
- * @author agapple 2019年9月29日 上午11:17:11
- * @since 1.1.5
- */
-public abstract class AbstractMQProducer implements CanalMQProducer {
-
-    protected ThreadPoolExecutor executor;
-
-    @Override
-    public void init(MQProperties mqProperties) {
-        int parallelThreadSize = mqProperties.getParallelThreadSize();
-        executor = new ThreadPoolExecutor(parallelThreadSize,
-            parallelThreadSize,
-            0,
-            TimeUnit.SECONDS,
-            new ArrayBlockingQueue<Runnable>(parallelThreadSize * 2),
-            new NamedThreadFactory("MQParallel"),
-            new ThreadPoolExecutor.CallerRunsPolicy());
-
-    }
-
-    @Override
-    public void stop() {
-        executor.shutdownNow();
-        executor = null;
-    }
-
-}

server/src/main/java/com/alibaba/otter/canal/common/MQProperties.java

@@ -1,368 +0,0 @@
-package com.alibaba.otter.canal.common;
-
-import java.util.Properties;
-
-/**
- * MQ 配置项
- *
- * @author machengyuan 2018-6-11 下午05:30:49
- * @version 1.0.0
- */
-public class MQProperties {
-
-    private String     servers                = "127.0.0.1:6667";
-    private int        retries                = 0;
-    private int        batchSize              = 16384;
-    private int        lingerMs               = 1;
-    private int        maxRequestSize         = 1048576;
-    private long       bufferMemory           = 33554432L;
-    private boolean    filterTransactionEntry = true;
-    private String     producerGroup          = "Canal-Producer";
-    private int        canalBatchSize         = 50;
-    private Long       canalGetTimeout        = 100L;
-    private boolean    flatMessage            = true;
-    private String     compressionType        = "none";
-    private String     acks                   = "all";
-    private String     aliyunAccessKey        = "";
-    private String     aliyunSecretKey        = "";
-    private Properties properties             = new Properties();
-    private boolean    enableMessageTrace     = false;
-    private String     accessChannel          = null;
-    private String     customizedTraceTopic   = null;
-    private String     namespace              = "";
-    // kafka集群是否启动Kerberos认证
-    private boolean    kerberosEnable         = false;
-    // 启动Kerberos认证时配置为krb5.conf文件的路径
-    private String     kerberosKrb5FilePath   = "";
-    // 启动Kerberos认证时配置为jaas.conf文件的路径
-    private String     kerberosJaasFilePath   = "";
-    // rabbitmq 账号
-    private String     username               = "";
-    // rabbitmq 密码
-    private String     password               = "";
-    // rabbitmq 密码
-    private String     vhost                  = "";
-    // aliyun 用户ID rabbitmq 阿里云需要使用
-    private long       aliyunUID              = 0;
-    // rabbitmq 交换机
-    private String     exchange               = "";
-    // 消息发送的并行度
-    private int        parallelThreadSize     = 8;
-    // 是否取消根据database进行hash
-    private boolean    databaseHash           = true;
-
-    public static class CanalDestination {
-
-        private String  canalDestination;
-        private String  topic;
-        private Integer partition;
-        private Integer partitionsNum;
-        private String  partitionHash;
-        private String  dynamicTopic;
-
-        public String getCanalDestination() {
-            return canalDestination;
-        }
-
-        public void setCanalDestination(String canalDestination) {
-            this.canalDestination = canalDestination;
-        }
-
-        public String getTopic() {
-            return topic;
-        }
-
-        public void setTopic(String topic) {
-            this.topic = topic;
-        }
-
-        public Integer getPartition() {
-            return partition;
-        }
-
-        public void setPartition(Integer partition) {
-            this.partition = partition;
-        }
-
-        public Integer getPartitionsNum() {
-            return partitionsNum;
-        }
-
-        public void setPartitionsNum(Integer partitionsNum) {
-            this.partitionsNum = partitionsNum;
-        }
-
-        public String getPartitionHash() {
-            return partitionHash;
-        }
-
-        public void setPartitionHash(String partitionHash) {
-            this.partitionHash = partitionHash;
-        }
-
-        public String getDynamicTopic() {
-            return dynamicTopic;
-        }
-
-        public void setDynamicTopic(String dynamicTopic) {
-            this.dynamicTopic = dynamicTopic;
-        }
-    }
-
-    public String getServers() {
-        return servers;
-    }
-
-    public void setServers(String servers) {
-        this.servers = servers;
-    }
-
-    public int getRetries() {
-        return retries;
-    }
-
-    public void setRetries(int retries) {
-        this.retries = retries;
-    }
-
-    public int getBatchSize() {
-        return batchSize;
-    }
-
-    public void setBatchSize(int batchSize) {
-        this.batchSize = batchSize;
-    }
-
-    public int getLingerMs() {
-        return lingerMs;
-    }
-
-    public void setLingerMs(int lingerMs) {
-        this.lingerMs = lingerMs;
-    }
-
-    public long getBufferMemory() {
-        return bufferMemory;
-    }
-
-    public void setBufferMemory(long bufferMemory) {
-        this.bufferMemory = bufferMemory;
-    }
-
-    public int getCanalBatchSize() {
-        return canalBatchSize;
-    }
-
-    public void setCanalBatchSize(int canalBatchSize) {
-        this.canalBatchSize = canalBatchSize;
-    }
-
-    public Long getCanalGetTimeout() {
-        return canalGetTimeout;
-    }
-
-    public void setCanalGetTimeout(Long canalGetTimeout) {
-        this.canalGetTimeout = canalGetTimeout;
-    }
-
-    public boolean getFlatMessage() {
-        return flatMessage;
-    }
-
-    public void setFlatMessage(boolean flatMessage) {
-        this.flatMessage = flatMessage;
-    }
-
-    public boolean isFilterTransactionEntry() {
-        return filterTransactionEntry;
-    }
-
-    public void setFilterTransactionEntry(boolean filterTransactionEntry) {
-        this.filterTransactionEntry = filterTransactionEntry;
-    }
-
-    public String getProducerGroup() {
-        return producerGroup;
-    }
-
-    public void setProducerGroup(String producerGroup) {
-        this.producerGroup = producerGroup;
-    }
-
-    public String getCompressionType() {
-        return compressionType;
-    }
-
-    public void setCompressionType(String compressionType) {
-        this.compressionType = compressionType;
-    }
-
-    public String getAcks() {
-        return acks;
-    }
-
-    public void setAcks(String acks) {
-        this.acks = acks;
-    }
-
-    public String getAliyunAccessKey() {
-        return aliyunAccessKey;
-    }
-
-    public void setAliyunAccessKey(String aliyunAccessKey) {
-        this.aliyunAccessKey = aliyunAccessKey;
-    }
-
-    public String getAliyunSecretKey() {
-        return aliyunSecretKey;
-    }
-
-    public void setAliyunSecretKey(String aliyunSecretKey) {
-        this.aliyunSecretKey = aliyunSecretKey;
-    }
-
-    public int getMaxRequestSize() {
-        return maxRequestSize;
-    }
-
-    public void setMaxRequestSize(int maxRequestSize) {
-        this.maxRequestSize = maxRequestSize;
-    }
-
-    public Properties getProperties() {
-        return properties;
-    }
-
-    public void setProperties(Properties properties) {
-        this.properties = properties;
-    }
-
-    public boolean isEnableMessageTrace() {
-        return enableMessageTrace;
-    }
-
-    public void setEnableMessageTrace(boolean enableMessageTrace) {
-        this.enableMessageTrace = enableMessageTrace;
-    }
-
-    public String getAccessChannel() {
-        return accessChannel;
-    }
-
-    public void setAccessChannel(String accessChannel) {
-        this.accessChannel = accessChannel;
-    }
-
-    public String getCustomizedTraceTopic() {
-        return customizedTraceTopic;
-    }
-
-    public void setCustomizedTraceTopic(String customizedTraceTopic) {
-        this.customizedTraceTopic = customizedTraceTopic;
-    }
-
-    public String getNamespace() {
-        return namespace;
-    }
-
-    public void setNamespace(String namespace) {
-        this.namespace = namespace;
-    }
-
-    public boolean isKerberosEnable() {
-        return kerberosEnable;
-    }
-
-    public void setKerberosEnable(boolean kerberosEnable) {
-        this.kerberosEnable = kerberosEnable;
-    }
-
-    public String getKerberosKrb5FilePath() {
-        return kerberosKrb5FilePath;
-    }
-
-    public void setKerberosKrb5FilePath(String kerberosKrb5FilePath) {
-        this.kerberosKrb5FilePath = kerberosKrb5FilePath;
-    }
-
-    public String getKerberosJaasFilePath() {
-        return kerberosJaasFilePath;
-    }
-
-    public void setKerberosJaasFilePath(String kerberosJaasFilePath) {
-        this.kerberosJaasFilePath = kerberosJaasFilePath;
-    }
-
-    public void setPassword(String password) {
-        this.password = password;
-    }
-
-    public String getPassword() {
-        return password;
-    }
-
-    public void setUsername(String username) {
-        this.username = username;
-    }
-
-    public String getUsername() {
-        return username;
-    }
-
-    public void setVhost(String vhost) {
-        this.vhost = vhost;
-    }
-
-    public String getVhost() {
-        return vhost;
-    }
-
-    public long getAliyunUID() {
-        return aliyunUID;
-    }
-
-    public void setAliyunUID(long aliyunUID) {
-        this.aliyunUID = aliyunUID;
-    }
-
-    public String getExchange() {
-        return exchange;
-    }
-
-    public void setExchange(String exchange) {
-        this.exchange = exchange;
-    }
-
-    public int getParallelThreadSize() {
-        return parallelThreadSize;
-    }
-
-    public void setParallelThreadSize(int parallelThreadSize) {
-        this.parallelThreadSize = parallelThreadSize;
-    }
-
-    public boolean getDatabaseHash() {
-        return databaseHash;
-    }
-
-    public void setDatabaseHash(boolean databaseHash) {
-        this.databaseHash = databaseHash;
-    }
-
-    @Override
-    public String toString() {
-        return "MQProperties [servers=" + servers + ", retries=" + retries + ", batchSize=" + batchSize + ", lingerMs="
-               + lingerMs + ", maxRequestSize=" + maxRequestSize + ", bufferMemory=" + bufferMemory
-               + ", filterTransactionEntry=" + filterTransactionEntry + ", producerGroup=" + producerGroup
-               + ", canalBatchSize=" + canalBatchSize + ", canalGetTimeout=" + canalGetTimeout + ", flatMessage="
-               + flatMessage + ", compressionType=" + compressionType + ", acks=" + acks + ", aliyunAccessKey="
-               + aliyunAccessKey + ", aliyunSecretKey=" + aliyunSecretKey + ", properties=" + properties
-               + ", enableMessageTrace=" + enableMessageTrace + ", accessChannel=" + accessChannel
-               + ", customizedTraceTopic=" + customizedTraceTopic + ", namespace=" + namespace + ", kerberosEnable="
-               + kerberosEnable + ", kerberosKrb5FilePath=" + kerberosKrb5FilePath + ", kerberosJaasFilePath="
-               + kerberosJaasFilePath + ", username=" + username + ", password=" + password + ", vhost=" + vhost
-               + ", aliyunUID=" + aliyunUID + ", exchange=" + exchange + ", parallelThreadSize=" + parallelThreadSize
-               + ",databaseHash=" + databaseHash + "]";
-    }
-
-}

server/src/main/java/com/alibaba/otter/canal/rabbitmq/CanalRabbitMQProducer.java

@@ -1,145 +0,0 @@
-package com.alibaba.otter.canal.rabbitmq;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeoutException;
-
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.alibaba.fastjson.JSON;
-import com.alibaba.fastjson.serializer.SerializerFeature;
-import com.alibaba.otter.canal.common.AbstractMQProducer;
-import com.alibaba.otter.canal.common.CanalMessageSerializer;
-import com.alibaba.otter.canal.common.MQMessageUtils;
-import com.alibaba.otter.canal.common.MQMessageUtils.EntryRowData;
-import com.alibaba.otter.canal.common.MQProperties;
-import com.alibaba.otter.canal.common.utils.ExecutorTemplate;
-import com.alibaba.otter.canal.protocol.FlatMessage;
-import com.alibaba.otter.canal.protocol.Message;
-import com.alibaba.otter.canal.server.exception.CanalServerException;
-import com.alibaba.otter.canal.spi.CanalMQProducer;
-import com.rabbitmq.client.Channel;
-import com.rabbitmq.client.Connection;
-import com.rabbitmq.client.ConnectionFactory;
-
-public class CanalRabbitMQProducer extends AbstractMQProducer implements CanalMQProducer {
-
-    private static final Logger logger = LoggerFactory.getLogger(CanalRabbitMQProducer.class);
-    private MQProperties        mqProperties;
-    private Connection          connect;
-    private Channel             channel;
-
-    @Override
-    public void init(MQProperties mqProperties) {
-        super.init(mqProperties);
-        this.mqProperties = mqProperties;
-        ConnectionFactory factory = new ConnectionFactory();
-        factory.setHost(mqProperties.getServers());
-        if (mqProperties.getAliyunAccessKey().length() > 0 && mqProperties.getAliyunSecretKey().length() > 0
-            && mqProperties.getAliyunUID() > 0) {
-            factory.setCredentialsProvider(new AliyunCredentialsProvider(mqProperties.getAliyunAccessKey(),
-                mqProperties.getAliyunSecretKey(),
-                mqProperties.getAliyunUID()));
-        } else {
-            factory.setUsername(mqProperties.getUsername());
-            factory.setPassword(mqProperties.getPassword());
-        }
-        factory.setVirtualHost(mqProperties.getVhost());
-        try {
-            connect = factory.newConnection();
-            channel = connect.createChannel();
-            // channel.exchangeDeclare(mqProperties.getExchange(), "topic");
-        } catch (IOException | TimeoutException ex) {
-            throw new CanalServerException("Start RabbitMQ producer error", ex);
-        }
-    }
-
-    @Override
-    public void send(final MQProperties.CanalDestination canalDestination, Message message, Callback callback)
-                                                                                                              throws IOException {
-        ExecutorTemplate template = new ExecutorTemplate(executor);
-        try {
-            if (!StringUtils.isEmpty(canalDestination.getDynamicTopic())) {
-                // 动态topic
-                Map<String, com.alibaba.otter.canal.protocol.Message> messageMap = MQMessageUtils.messageTopics(message,
-                    canalDestination.getTopic(),
-                    canalDestination.getDynamicTopic());
-
-                for (Map.Entry<String, com.alibaba.otter.canal.protocol.Message> entry : messageMap.entrySet()) {
-                    final String topicName = entry.getKey().replace('.', '_');
-                    final com.alibaba.otter.canal.protocol.Message messageSub = entry.getValue();
-
-                    template.submit(new Runnable() {
-
-                        @Override
-                        public void run() {
-                            send(canalDestination, topicName, messageSub);
-                        }
-                    });
-                }
-
-                template.waitForResult();
-            } else {
-                send(canalDestination, canalDestination.getTopic(), message);
-            }
-            callback.commit();
-        } catch (Throwable e) {
-            logger.error(e.getMessage(), e);
-            callback.rollback();
-        } finally {
-            template.clear();
-        }
-    }
-
-    private void send(MQProperties.CanalDestination canalDestination, String topicName, Message messageSub) {
-        if (!mqProperties.getFlatMessage()) {
-            byte[] message = CanalMessageSerializer.serializer(messageSub, mqProperties.isFilterTransactionEntry());
-            if (logger.isDebugEnabled()) {
-                logger.debug("send message:{} to destination:{}", message, canalDestination.getCanalDestination());
-            }
-            sendMessage(topicName, message);
-        } else {
-            // 并发构造
-            EntryRowData[] datas = MQMessageUtils.buildMessageData(messageSub, executor);
-            // 串行分区
-            List<FlatMessage> flatMessages = MQMessageUtils.messageConverter(datas, messageSub.getId());
-            if (flatMessages != null) {
-                for (FlatMessage flatMessage : flatMessages) {
-                    byte[] message = JSON.toJSONBytes(flatMessage, SerializerFeature.WriteMapNullValue);
-                    if (logger.isDebugEnabled()) {
-                        logger.debug("send message:{} to destination:{}",
-                            message,
-                            canalDestination.getCanalDestination());
-                    }
-                    sendMessage(topicName, message);
-                }
-            }
-        }
-
-    }
-
-    private void sendMessage(String queueName, byte[] message) {
-        // tips: 目前逻辑中暂不处理对exchange处理，请在Console后台绑定 才可使用routekey
-        try {
-            channel.basicPublish(mqProperties.getExchange(), queueName, null, message);
-        } catch (Throwable e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    @Override
-    public void stop() {
-        logger.info("## Stop RabbitMQ producer##");
-        try {
-            this.connect.close();
-            this.channel.close();
-        } catch (IOException | TimeoutException ex) {
-            throw new CanalServerException("Stop RabbitMQ producer error", ex);
-        }
-
-        super.stop();
-    }
-}

server/src/main/java/com/alibaba/otter/canal/rocketmq/CanalRocketMQProducer.java

@@ -1,285 +0,0 @@
-package com.alibaba.otter.canal.rocketmq;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.rocketmq.acl.common.AclClientRPCHook;
-import org.apache.rocketmq.acl.common.SessionCredentials;
-import org.apache.rocketmq.client.AccessChannel;
-import org.apache.rocketmq.client.exception.MQClientException;
-import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;
-import org.apache.rocketmq.client.impl.producer.TopicPublishInfo;
-import org.apache.rocketmq.client.producer.DefaultMQProducer;
-import org.apache.rocketmq.client.producer.MessageQueueSelector;
-import org.apache.rocketmq.client.producer.SendResult;
-import org.apache.rocketmq.common.message.Message;
-import org.apache.rocketmq.common.message.MessageQueue;
-import org.apache.rocketmq.remoting.RPCHook;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.alibaba.fastjson.JSON;
-import com.alibaba.fastjson.serializer.SerializerFeature;
-import com.alibaba.otter.canal.common.AbstractMQProducer;
-import com.alibaba.otter.canal.common.CanalMessageSerializer;
-import com.alibaba.otter.canal.common.MQMessageUtils;
-import com.alibaba.otter.canal.common.MQMessageUtils.EntryRowData;
-import com.alibaba.otter.canal.common.MQProperties;
-import com.alibaba.otter.canal.common.utils.ExecutorTemplate;
-import com.alibaba.otter.canal.protocol.FlatMessage;
-import com.alibaba.otter.canal.server.exception.CanalServerException;
-import com.alibaba.otter.canal.spi.CanalMQProducer;
-
-public class CanalRocketMQProducer extends AbstractMQProducer implements CanalMQProducer {
-
-    private static final Logger logger               = LoggerFactory.getLogger(CanalRocketMQProducer.class);
-    private DefaultMQProducer   defaultMQProducer;
-    private MQProperties        mqProperties;
-    private static final String CLOUD_ACCESS_CHANNEL = "cloud";
-
-    @Override
-    public void init(MQProperties rocketMQProperties) {
-        super.init(rocketMQProperties);
-        this.mqProperties = rocketMQProperties;
-        RPCHook rpcHook = null;
-        if (rocketMQProperties.getAliyunAccessKey().length() > 0
-            && rocketMQProperties.getAliyunSecretKey().length() > 0) {
-            SessionCredentials sessionCredentials = new SessionCredentials();
-            sessionCredentials.setAccessKey(rocketMQProperties.getAliyunAccessKey());
-            sessionCredentials.setSecretKey(rocketMQProperties.getAliyunSecretKey());
-            rpcHook = new AclClientRPCHook(sessionCredentials);
-        }
-
-        defaultMQProducer = new DefaultMQProducer(rocketMQProperties.getProducerGroup(),
-            rpcHook,
-            mqProperties.isEnableMessageTrace(),
-            mqProperties.getCustomizedTraceTopic());
-        if (CLOUD_ACCESS_CHANNEL.equals(rocketMQProperties.getAccessChannel())) {
-            defaultMQProducer.setAccessChannel(AccessChannel.CLOUD);
-        }
-        if (!StringUtils.isEmpty(mqProperties.getNamespace())) {
-            defaultMQProducer.setNamespace(mqProperties.getNamespace());
-        }
-        defaultMQProducer.setNamesrvAddr(rocketMQProperties.getServers());
-        defaultMQProducer.setRetryTimesWhenSendFailed(rocketMQProperties.getRetries());
-        defaultMQProducer.setVipChannelEnabled(false);
-        logger.info("##Start RocketMQ producer##");
-        try {
-            defaultMQProducer.start();
-        } catch (MQClientException ex) {
-            throw new CanalServerException("Start RocketMQ producer error", ex);
-        }
-    }
-
-    @Override
-    public void send(final MQProperties.CanalDestination destination, com.alibaba.otter.canal.protocol.Message data,
-                     Callback callback) {
-        ExecutorTemplate template = new ExecutorTemplate(executor);
-        try {
-            if (!StringUtils.isEmpty(destination.getDynamicTopic())) {
-                // 动态topic
-                Map<String, com.alibaba.otter.canal.protocol.Message> messageMap = MQMessageUtils.messageTopics(data,
-                    destination.getTopic(),
-                    destination.getDynamicTopic());
-
-                for (Map.Entry<String, com.alibaba.otter.canal.protocol.Message> entry : messageMap.entrySet()) {
-                    String topicName = entry.getKey().replace('.', '_');
-                    com.alibaba.otter.canal.protocol.Message messageSub = entry.getValue();
-                    template.submit(new Runnable() {
-
-                        @Override
-                        public void run() {
-                            try {
-                                send(destination, topicName, messageSub);
-                            } catch (Exception e) {
-                                throw new RuntimeException(e);
-                            }
-                        }
-                    });
-                }
-
-                template.waitForResult();
-            } else {
-                send(destination, destination.getTopic(), data);
-            }
-
-            callback.commit();
-        } catch (Throwable e) {
-            logger.error(e.getMessage(), e);
-            callback.rollback();
-        } finally {
-            template.clear();
-        }
-    }
-
-    public void send(final MQProperties.CanalDestination destination, String topicName,
-                     com.alibaba.otter.canal.protocol.Message message) throws Exception {
-        if (!mqProperties.getFlatMessage()) {
-            if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) {
-                // 并发构造
-                EntryRowData[] datas = MQMessageUtils.buildMessageData(message, executor);
-                // 串行分区
-                com.alibaba.otter.canal.protocol.Message[] messages = MQMessageUtils.messagePartition(datas,
-                    message.getId(),
-                    destination.getPartitionsNum(),
-                    destination.getPartitionHash(),
-                    mqProperties.getDatabaseHash());
-                int length = messages.length;
-
-                ExecutorTemplate template = new ExecutorTemplate(executor);
-                for (int i = 0; i < length; i++) {
-                    com.alibaba.otter.canal.protocol.Message dataPartition = messages[i];
-                    if (dataPartition != null) {
-                        final int index = i;
-                        template.submit(new Runnable() {
-
-                            @Override
-                            public void run() {
-                                Message data = new Message(topicName, CanalMessageSerializer.serializer(dataPartition,
-                                    mqProperties.isFilterTransactionEntry()));
-                                sendMessage(data, index);
-                            }
-                        });
-                    }
-                }
-                // 等所有分片发送完毕
-                template.waitForResult();
-            } else {
-                final int partition = destination.getPartition() != null ? destination.getPartition() : 0;
-                Message data = new Message(topicName, CanalMessageSerializer.serializer(message,
-                    mqProperties.isFilterTransactionEntry()));
-                sendMessage(data, partition);
-            }
-        } else {
-            // 并发构造
-            EntryRowData[] datas = MQMessageUtils.buildMessageData(message, executor);
-            // 串行分区
-            List<FlatMessage> flatMessages = MQMessageUtils.messageConverter(datas, message.getId());
-            if (flatMessages != null) {
-                // 初始化分区合并队列
-                if (destination.getPartitionHash() != null && !destination.getPartitionHash().isEmpty()) {
-                    List<List<FlatMessage>> partitionFlatMessages = new ArrayList<List<FlatMessage>>();
-                    for (int i = 0; i < destination.getPartitionsNum(); i++) {
-                        partitionFlatMessages.add(new ArrayList<FlatMessage>());
-                    }
-
-                    for (FlatMessage flatMessage : flatMessages) {
-                        FlatMessage[] partitionFlatMessage = MQMessageUtils.messagePartition(flatMessage,
-                            destination.getPartitionsNum(),
-                            destination.getPartitionHash(),
-                            mqProperties.getDatabaseHash());
-                        int length = partitionFlatMessage.length;
-                        for (int i = 0; i < length; i++) {
-                            partitionFlatMessages.get(i).add(partitionFlatMessage[i]);
-                        }
-                    }
-
-                    ExecutorTemplate template = new ExecutorTemplate(executor);
-                    for (int i = 0; i < partitionFlatMessages.size(); i++) {
-                        final List<FlatMessage> flatMessagePart = partitionFlatMessages.get(i);
-                        if (flatMessagePart != null) {
-                            final int index = i;
-                            template.submit(new Runnable() {
-
-                                @Override
-                                public void run() {
-                                    List<Message> messages = flatMessagePart.stream()
-                                        .map(flatMessage -> new Message(topicName, JSON.toJSONBytes(flatMessage,
-                                            SerializerFeature.WriteMapNullValue)))
-                                        .collect(Collectors.toList());
-                                    // 批量发送
-                                    sendMessage(messages, index);
-                                }
-                            });
-                        }
-                    }
-
-                    // 批量等所有分区的结果
-                    template.waitForResult();
-                } else {
-                    final int partition = destination.getPartition() != null ? destination.getPartition() : 0;
-                    List<Message> messages = flatMessages.stream()
-                        .map(flatMessage -> new Message(topicName, JSON.toJSONBytes(flatMessage,
-                            SerializerFeature.WriteMapNullValue)))
-                        .collect(Collectors.toList());
-                    // 批量发送
-                    sendMessage(messages, partition);
-                }
-            }
-        }
-    }
-
-    private void sendMessage(Message message, int partition) {
-        try {
-            SendResult sendResult = this.defaultMQProducer.send(message, new MessageQueueSelector() {
-
-                @Override
-                public MessageQueue select(List<MessageQueue> mqs, Message msg, Object arg) {
-                    if (partition > mqs.size()) {
-                        return mqs.get(partition % mqs.size());
-                    } else {
-                        return mqs.get(partition);
-                    }
-                }
-            }, null);
-
-            if (logger.isDebugEnabled()) {
-                logger.debug("Send Message Result: {}", sendResult);
-            }
-        } catch (Throwable e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    @SuppressWarnings("deprecation")
-    private void sendMessage(List<Message> messages, int partition) {
-        if (messages.isEmpty()) {
-            return;
-        }
-
-        // 获取一下messageQueue
-        DefaultMQProducerImpl innerProducer = this.defaultMQProducer.getDefaultMQProducerImpl();
-        TopicPublishInfo topicInfo = innerProducer.getTopicPublishInfoTable().get(messages.get(0).getTopic());
-        if (topicInfo == null) {
-            for (Message message : messages) {
-                sendMessage(message, partition);
-            }
-        } else {
-            // 批量发送
-            List<MessageQueue> queues = topicInfo.getMessageQueueList();
-            int size = queues.size();
-            if (size <= 0) {
-                // 可能是第一次创建
-                for (Message message : messages) {
-                    sendMessage(message, partition);
-                }
-            } else {
-                MessageQueue queue = null;
-                if (partition > size) {
-                    queue = queues.get(partition % size);
-                } else {
-                    queue = queues.get(partition);
-                }
-
-                try {
-                    SendResult sendResult = this.defaultMQProducer.send(messages, queue);
-                    if (logger.isDebugEnabled()) {
-                        logger.debug("Send Message Result: {}", sendResult);
-                    }
-                } catch (Throwable e) {
-                    throw new RuntimeException(e);
-                }
-            }
-        }
-    }
-
-    @Override
-    public void stop() {
-        logger.info("## Stop RocketMQ producer##");
-        this.defaultMQProducer.shutdown();
-        super.stop();
-    }
-}

server/src/main/java/com/alibaba/otter/canal/server/CanalMQStarter.java

@@ -7,18 +7,21 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import com.alibaba.otter.canal.connector.core.util.Callback;
+import com.alibaba.otter.canal.connector.core.producer.MQDestination;
 import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.slf4j.MDC;
 
-import com.alibaba.otter.canal.common.MQProperties;
+import com.alibaba.otter.canal.connector.core.spi.CanalMQProducer;
 import com.alibaba.otter.canal.instance.core.CanalInstance;
 import com.alibaba.otter.canal.instance.core.CanalMQConfig;
 import com.alibaba.otter.canal.protocol.ClientIdentity;
 import com.alibaba.otter.canal.protocol.Message;
 import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded;
-import com.alibaba.otter.canal.spi.CanalMQProducer;
+
+import com.alibaba.otter.canal.connector.core.config.MQProperties;
 
 public class CanalMQStarter {
 
@@ -30,7 +33,7 @@ public class CanalMQStarter {
 
     private CanalMQProducer              canalMQProducer;
 
-    private MQProperties                 properties;
+    private MQProperties                 mqProperties;
 
     private CanalServerWithEmbedded      canalServer;
 
@@ -42,15 +45,14 @@ public class CanalMQStarter {
         this.canalMQProducer = canalMQProducer;
     }
 
-    public synchronized void start(MQProperties properties, String destinations) {
+    public synchronized void start(String destinations) {
         try {
             if (running) {
                 return;
             }
-            this.properties = properties;
-            canalMQProducer.init(properties);
+            mqProperties = canalMQProducer.getMqProperties();
             // set filterTransactionEntry
-            if (properties.isFilterTransactionEntry()) {
+            if (mqProperties.isFilterTransactionEntry()) {
                 System.setProperty("canal.instance.filter.transaction.entry", "true");
             }
 
@@ -151,7 +153,7 @@ public class CanalMQStarter {
                     }
                     continue;
                 }
-                MQProperties.CanalDestination canalDestination = new MQProperties.CanalDestination();
+                MQDestination canalDestination = new MQDestination();
                 canalDestination.setCanalDestination(destination);
                 CanalMQConfig mqConfig = canalInstance.getMqConfig();
                 canalDestination.setTopic(mqConfig.getTopic());
@@ -163,15 +165,13 @@ public class CanalMQStarter {
                 canalServer.subscribe(clientIdentity);
                 logger.info("## the MQ producer: {} is running now ......", destination);
 
-                Long getTimeout = properties.getCanalGetTimeout();
-                int getBatchSize = properties.getCanalBatchSize();
+                Integer getTimeout = mqProperties.getFetchTimeout();
+                Integer getBatchSize = mqProperties.getBatchSize();
                 while (running && destinationRunning.get()) {
                     Message message;
                     if (getTimeout != null && getTimeout > 0) {
-                        message = canalServer.getWithoutAck(clientIdentity,
-                            getBatchSize,
-                            getTimeout,
-                            TimeUnit.MILLISECONDS);
+                        message = canalServer
+                            .getWithoutAck(clientIdentity, getBatchSize, getTimeout.longValue(), TimeUnit.MILLISECONDS);
                     } else {
                         message = canalServer.getWithoutAck(clientIdentity, getBatchSize);
                     }
@@ -180,7 +180,7 @@ public class CanalMQStarter {
                     try {
                         int size = message.isRaw() ? message.getRawEntries().size() : message.getEntries().size();
                         if (batchId != -1 && size != 0) {
-                            canalMQProducer.send(canalDestination, message, new CanalMQProducer.Callback() {
+                            canalMQProducer.send(canalDestination, message, new Callback() {
 
                                 @Override
                                 public void commit() {

server/src/main/java/com/alibaba/otter/canal/spi/CanalMQProducer.java

@@ -1,37 +0,0 @@
-package com.alibaba.otter.canal.spi;
-
-import java.io.IOException;
-
-import com.alibaba.otter.canal.common.MQProperties;
-import com.alibaba.otter.canal.protocol.Message;
-
-public interface CanalMQProducer {
-
-    /**
-     * Init producer.
-     *
-     * @param mqProperties MQ config
-     */
-    void init(MQProperties mqProperties);
-
-    /**
-     * Send canal message to related topic
-     *
-     * @param canalDestination canal mq destination
-     * @param message canal message
-     * @throws IOException
-     */
-    void send(MQProperties.CanalDestination canalDestination, Message message, Callback callback) throws IOException;
-
-    /**
-     * Stop MQ producer service
-     */
-    void stop();
-
-    interface Callback {
-
-        void commit();
-
-        void rollback();
-    }
-}