init

.gitignore

@@ -0,0 +1,15 @@
+.svn/
+target/
+test-output/
+*.class
+.classpath
+.project
+.settings/
+tmp
+temp
+*.log
+antx.properties
+otter.properties
+jtester.properties
+.idea/
+*.iml

LICENSE.txt

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -0,0 +1,71 @@
+<div class="blog_content">
+    <div class="iteye-blog-content-contain">
+
+<h3>最新更新</h3>
+<ol>
+<li>canal QQ讨论群已经建立，群号：161559791 ，欢迎加入进行技术讨论。</li>
+<li>canal消费端项目开源: Otter(分布式数据库同步系统)，地址：<a href="https://github.com/alibaba/otter">https://github.com/alibaba/otter</a></li>
+</ol>
+
+<h1>背景</h1>
+<p style="font-size: 14px;">   早期，阿里巴巴B2B公司因为存在杭州和美国双机房部署，存在跨机房同步的业务需求。不过早期的数据库同步业务，主要是基于trigger的方式获取增量变更，不过从2010年开始，阿里系公司开始逐步的尝试基于数据库的日志解析，获取增量变更进行同步，由此衍生出了增量订阅&amp;消费的业务，从此开启了一段新纪元。</p>
+<p style="font-size: 14px;">   ps. 目前内部版本已经支持mysql和oracle部分版本的日志解析，当前的canal开源版本支持5.6及以下的版本(阿里内部mysql 5.6.10, mysql 5.5.18和5.1.40/48)</p>
+<p style="font-size: 14px;"> </p>
+<p style="font-size: 14px;">基于日志增量订阅&amp;消费支持的业务：</p>
+<ol style="font-size: 14px;">
+<li>数据库镜像</li>
+<li>数据库实时备份</li>
+<li>多级索引 (卖家和买家各自分库索引)</li>
+<li>search build</li>
+<li>业务cache刷新</li>
+<li>价格变化等重要业务消息</li>
+</ol>
+<h1>项目介绍</h1>
+<p style="font-size: 14px;">   名称：canal [kə'næl]</p>
+<p style="font-size: 14px;">   译意： 水道/管道/沟渠 </p>
+<p style="font-size: 14px;">   语言： 纯java开发</p>
+<p style="font-size: 14px;">   定位： 基于数据库增量日志解析，提供增量数据订阅&amp;消费，目前主要支持了mysql</p>
+<p style="font-size: 14px;">   关键词： mysql binlog parser / real-time / queue&topic </p>
+<p style="font-size: 14px;"> </p>
+<h2>工作原理</h2>
+<h3 style="font-size: 14px;">mysql主备复制实现</h3>
+<p><img src="http://dl.iteye.com/upload/attachment/0080/3086/468c1a14-e7ad-3290-9d3d-44ac501a7227.jpg" alt=""><br> 从上层来看，复制分成三步：
+<ol>
+<li>master将改变记录到二进制日志(binary log)中（这些记录叫做二进制日志事件，binary log events，可以通过show binlog events进行查看）；</li>
+<li>slave将master的binary log events拷贝到它的中继日志(relay log)；</li>
+<li>slave重做中继日志中的事件，将改变反映它自己的数据。</li>
+</ol>
+<h3>canal的工作原理：</h3>
+<p><img width="590" src="http://dl.iteye.com/upload/attachment/0080/3107/c87b67ba-394c-3086-9577-9db05be04c95.jpg" alt="" height="273">
+<p>原理相对比较简单：</p>
+<ol>
+<li>canal模拟mysql slave的交互协议，伪装自己为mysql slave，向mysql master发送dump协议</li>
+<li>mysql master收到dump请求，开始推送binary log给slave(也就是canal)</li>
+<li>canal解析binary log对象(原始为byte流)</li>
+</ol>
+
+<h1>相关文档</h1>
+
+See the wiki page for : <a href="https://github.com/alibaba/canal/wiki" >wiki文档</a>
+
+<h3><a name="table-of-contents" class="anchor" href="#table-of-contents"><span class="mini-icon mini-icon-link"></span></a>wiki文档列表</h3>
+<ul>
+<li><a class="internal present" href="https://github.com/alibaba/canal/wiki/Home">Home</a></li>
+<li><a class="internal present" href="https://github.com/alibaba/canal/wiki/Introduction">Introduction</a></li>
+<li><a class="internal present" href="https://github.com/alibaba/canal/wiki/QuickStart">QuickStart</a></li>
+<li><a class="internal present" href="https://github.com/alibaba/canal/wiki/ClientExample">ClientExample</a></li>
+<li><a class="internal present" href="https://github.com/alibaba/canal/wiki/AdminGuide">AdminGuide</a></li>
+<li><a class="internal present" href="https://github.com/alibaba/canal/wiki/ClientAPI">ClientAPI</a></li>
+<li><a class="internal present" href="https://github.com/alibaba/canal/wiki/DevGuide">DevGuide</a></li>
+<li><a class="internal present" href="https://github.com/alibaba/canal/wiki/BinlogChange%28mysql5.6%29">BinlogChange(Mysql5.6)</a></li>
+<li><a href="http://alibaba.github.com/canal/release.html">ReleaseNotes</a></li>
+<li><a href="https://github.com/alibaba/canal/releases">Download</a></li>
+</ul>
+
+<h1>问题反馈</h1>
+<ol>
+<li>qq交流群： 161559791 </li>
+<li>邮件交流： jianghang115@gmail.com </li>
+<li>新浪微博： agapple0002 </li>
+<li>报告issue：<a href="https://github.com/alibaba/canal/issues">issues</a></li>
+</ol>

RELEASE.txt

@@ -0,0 +1,3 @@
+release link : https://alibaba.github.com/canal/release.html
+
+download link : https://github.com/alibaba/canal/releases

client/pom.xml

@@ -0,0 +1,101 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:p="http://www.springframework.org/schema/p" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation=" ">
+	<modelVersion>4.0.0</modelVersion>
+	<parent>
+		<groupId>com.alibaba.otter</groupId>
+		<artifactId>canal</artifactId>
+		<version>1.0.19-SNAPSHOT</version>
+		<relativePath>../pom.xml</relativePath>
+	</parent>
+	<groupId>com.alibaba.otter</groupId>
+	<artifactId>canal.client</artifactId>
+	<packaging>jar</packaging>
+	<name>canal client module for otter ${project.version}</name>
+	<dependencies>
+		<dependency>
+			<groupId>com.alibaba.otter</groupId>
+			<artifactId>canal.protocol</artifactId>
+			<version>${project.version}</version>
+		</dependency>
+
+		<!-- junit -->
+		<dependency>
+			<groupId>junit</groupId>
+			<artifactId>junit</artifactId>
+		</dependency>
+	</dependencies>
+	
+	<profiles>
+		<profile>
+			<id>dev</id>
+			<activation>
+				<activeByDefault>true</activeByDefault>
+				<property>
+					<name>env</name>
+					<value>!javadoc</value>
+				</property>
+			</activation>
+		</profile>
+
+		<profile>
+			<id>javadoc</id>
+			<activation>
+				<property>
+					<name>env</name>
+					<value>javadoc</value>
+				</property>
+			</activation>
+			<build>
+				<plugins>
+					<plugin>
+						<groupId>org.apache.maven.plugins</groupId>
+						<artifactId>maven-javadoc-plugin</artifactId>
+						<version>2.9.1</version>
+						<executions>
+							<execution>
+								<id>attach-javadocs</id>
+								<phase>package</phase>
+								<goals>
+									<goal>jar</goal>
+								</goals>
+							</execution>
+						</executions>
+						<configuration>
+							<aggregate>true</aggregate>
+							<show>public</show>
+							<nohelp>true</nohelp>
+							<header>${project.artifactId}-${project.version}</header>
+							<footer>${project.artifactId}-${project.version}</footer>
+							<doctitle>${project.artifactId}-${project.version}</doctitle>
+							<links>
+								<link>https://github.com/alibaba/canal</link>
+							</links>
+							<outputDirectory>${project.build.directory}/apidocs/apidocs/${pom.version}</outputDirectory>
+						</configuration>
+					</plugin>
+					<plugin>
+						<groupId>org.apache.maven.plugins</groupId>
+						<artifactId>maven-scm-publish-plugin</artifactId>
+						<version>1.0-beta-2</version>
+						<executions>
+							<execution>
+								<id>attach-javadocs</id>
+								<phase>package</phase>
+								<goals>
+									<goal>publish-scm</goal>
+								</goals>
+							</execution>
+						</executions>
+						<configuration>
+							<checkoutDirectory>${project.build.directory}/scmpublish</checkoutDirectory>
+							<checkinComment>Publishing javadoc for ${project.artifactId}:${project.version}</checkinComment>
+							<content>${project.build.directory}/apidocs</content>
+							<skipDeletedFiles>true</skipDeletedFiles>
+							<pubScmUrl>scm:git:git@github.com:alibaba/canal.git</pubScmUrl>
+							<scmBranch>gh-pages</scmBranch> 
+						</configuration>
+					</plugin>
+				</plugins>
+			</build>
+		</profile>
+	</profiles>
+</project>

client/src/main/java/com/alibaba/otter/canal/client/CanalConnector.java

@@ -0,0 +1,156 @@
+package com.alibaba.otter.canal.client;
+
+import java.util.concurrent.TimeUnit;
+
+import com.alibaba.otter.canal.protocol.Message;
+import com.alibaba.otter.canal.protocol.exception.CanalClientException;
+
+/**
+ * canal数据操作客户端
+ * 
+ * @author zebin.xuzb @ 2012-6-19
+ * @author jianghang
+ * @version 1.0.0
+ */
+public interface CanalConnector {
+
+    /**
+     * 链接对应的canal server
+     * 
+     * @throws CanalClientException
+     */
+    public void connect() throws CanalClientException;
+
+    /**
+     * 释放链接
+     * 
+     * @throws CanalClientException
+     */
+    public void disconnect() throws CanalClientException;
+
+    /**
+     * 检查下链接是否合法
+     * 
+     * <pre>
+     * 几种case下链接不合法:
+     * 1. 链接canal server失败，一直没有一个可用的链接，返回false
+     * 2. 当前客户端在进行running抢占的时候，做为备份节点存在，非处于工作节点，返回false
+     * 
+     * 说明：
+     * a. 当前客户端一旦做为备份节点存在，当前所有的对{@linkplain CanalConnector}的操作都会处于阻塞状态，直到转为工作节点
+     * b. 所以业务方最好定时调用checkValid()方法用，比如调用CanalConnector所在线程的interrupt，直接退出CanalConnector，并根据自己的需要退出自己的资源
+     * </pre>
+     * 
+     * @throws CanalClientException
+     */
+    public boolean checkValid() throws CanalClientException;
+
+    /**
+     * 客户端订阅，重复订阅时会更新对应的filter信息
+     * 
+     * <pre>
+     * 说明：
+     * a. 如果本次订阅中filter信息为空，则直接使用canal server服务端配置的filter信息
+     * b. 如果本次订阅中filter信息不为空，目前会直接替换canal server服务端配置的filter信息，以本次提交的为准
+     * 
+     * TODO: 后续可以考虑，如果本次提交的filter不为空，在执行过滤时，是对canal server filter + 本次filter的交集处理，达到只取1份binlog数据，多个客户端消费不同的表
+     * </pre>
+     * 
+     * @param clientIdentity
+     * @throws CanalClientException
+     */
+    void subscribe(String filter) throws CanalClientException;
+
+    /**
+     * 客户端订阅，不提交客户端filter，以服务端的filter为准
+     * 
+     * @param clientIdentity
+     * @throws CanalClientException
+     */
+    void subscribe() throws CanalClientException;
+
+    /**
+     * 取消订阅
+     * 
+     * @param clientIdentity
+     * @throws CanalClientException
+     */
+    void unsubscribe() throws CanalClientException;
+
+    /**
+     * 获取数据，自动进行确认，该方法返回的条件：尝试拿batchSize条记录，有多少取多少，不会阻塞等待
+     * 
+     * @param batchSize
+     * @return
+     * @throws CanalClientException
+     */
+    Message get(int batchSize) throws CanalClientException;
+
+    /**
+     * 获取数据，自动进行确认
+     * 
+     * <pre>
+     * 该方法返回的条件：
+     *  a. 拿够batchSize条记录或者超过timeout时间
+     *  b. 如果timeout=0，则阻塞至拿到batchSize记录才返回
+     * </pre>
+     * 
+     * @param batchSize
+     * @return
+     * @throws CanalClientException
+     */
+    Message get(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException;
+
+    /**
+     * 不指定 position 获取事件，该方法返回的条件: 尝试拿batchSize条记录，有多少取多少，不会阻塞等待<br/>
+     * canal 会记住此 client 最新的position。 <br/>
+     * 如果是第一次 fetch，则会从 canal 中保存的最老一条数据开始输出。
+     * 
+     * @param batchSize
+     * @throws CanalClientException
+     */
+    Message getWithoutAck(int batchSize) throws CanalClientException;
+
+    /**
+     * 不指定 position 获取事件.
+     * 
+     * <pre>
+     * 该方法返回的条件：
+     *  a. 拿够batchSize条记录或者超过timeout时间
+     *  b. 如果timeout=0，则阻塞至拿到batchSize记录才返回
+     * </pre>
+     * 
+     * canal 会记住此 client 最新的position。 <br/>
+     * 如果是第一次 fetch，则会从 canal 中保存的最老一条数据开始输出。
+     * 
+     * @param batchSize
+     * @param timeout
+     * @param unit
+     * @return
+     * @throws CanalClientException
+     */
+    Message getWithoutAck(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException;
+
+    /**
+     * 进行 batch id 的确认。确认之后，小于等于此 batchId 的 Message 都会被确认。
+     * 
+     * @param batchId
+     * @throws CanalClientException
+     */
+    void ack(long batchId) throws CanalClientException;
+
+    /**
+     * 回滚到未进行 {@link ack} 的地方，指定回滚具体的batchId
+     * 
+     * @throws CanalClientException
+     */
+    void rollback(long batchId) throws CanalClientException;
+
+    /**
+     * 回滚到未进行 {@link ack} 的地方，下次fetch的时候，可以从最后一个没有 {@link ack} 的地方开始拿
+     * 
+     * @throws CanalClientException
+     */
+    void rollback() throws CanalClientException;
+
+}

client/src/main/java/com/alibaba/otter/canal/client/CanalConnectors.java

@@ -0,0 +1,70 @@
+package com.alibaba.otter.canal.client;
+
+import java.net.SocketAddress;
+import java.util.List;
+
+import com.alibaba.otter.canal.client.impl.ClusterCanalConnector;
+import com.alibaba.otter.canal.client.impl.ClusterNodeAccessStrategy;
+import com.alibaba.otter.canal.client.impl.SimpleCanalConnector;
+import com.alibaba.otter.canal.client.impl.SimpleNodeAccessStrategy;
+import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
+
+/**
+ * canal connectors创建工具类
+ * 
+ * @author jianghang 2012-10-29 下午11:18:50
+ * @version 1.0.0
+ */
+public class CanalConnectors {
+
+    /**
+     * 创建单链接的客户端链接
+     * 
+     * @param address
+     * @param username
+     * @param password
+     * @return
+     */
+    public static CanalConnector newSingleConnector(SocketAddress address, String destination, String username,
+                                                    String password) {
+        SimpleCanalConnector canalConnector = new SimpleCanalConnector(address, username, password, destination);
+        canalConnector.setSoTimeout(30 * 1000);
+        return canalConnector;
+    }
+
+    /**
+     * 创建带cluster模式的客户端链接，自动完成failover切换
+     * 
+     * @param addresses
+     * @param username
+     * @param password
+     * @return
+     */
+    public static CanalConnector newClusterConnector(List<? extends SocketAddress> addresses, String destination,
+                                                     String username, String password) {
+        ClusterCanalConnector canalConnector = new ClusterCanalConnector(username, password, destination,
+                                                                         new SimpleNodeAccessStrategy(addresses));
+        canalConnector.setSoTimeout(30 * 1000);
+        return canalConnector;
+    }
+
+    /**
+     * 创建带cluster模式的客户端链接，自动完成failover切换，服务器列表自动扫描
+     * 
+     * @param username
+     * @param password
+     * @return
+     */
+    public static CanalConnector newClusterConnector(String zkServers, String destination, String username,
+                                                     String password) {
+        ClusterCanalConnector canalConnector = new ClusterCanalConnector(
+                                                                         username,
+                                                                         password,
+                                                                         destination,
+                                                                         new ClusterNodeAccessStrategy(
+                                                                                                       destination,
+                                                                                                       ZkClientx.getZkClient(zkServers)));
+        canalConnector.setSoTimeout(30 * 1000);
+        return canalConnector;
+    }
+}

client/src/main/java/com/alibaba/otter/canal/client/CanalNodeAccessStrategy.java

@@ -0,0 +1,14 @@
+package com.alibaba.otter.canal.client;
+
+import java.net.SocketAddress;
+
+/**
+ * 集群节点访问控制接口
+ * 
+ * @author jianghang 2012-10-29 下午07:55:41
+ * @version 1.0.0
+ */
+public interface CanalNodeAccessStrategy {
+
+    SocketAddress nextNode();
+}

client/src/main/java/com/alibaba/otter/canal/client/impl/ClusterCanalConnector.java

@@ -0,0 +1,327 @@
+package com.alibaba.otter.canal.client.impl;
+
+import java.net.SocketAddress;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.client.CanalConnector;
+import com.alibaba.otter.canal.client.CanalNodeAccessStrategy;
+import com.alibaba.otter.canal.protocol.Message;
+import com.alibaba.otter.canal.protocol.exception.CanalClientException;
+
+/**
+ * 集群版本connector实现，自带了failover功能<br/>
+ * 
+ * @author jianghang 2012-10-29 下午08:04:06
+ * @version 1.0.0
+ */
+public class ClusterCanalConnector implements CanalConnector {
+
+    private final Logger            logger        = LoggerFactory.getLogger(this.getClass());
+    private String                  username;
+    private String                  password;
+    private int                     soTimeout     = 10000;
+    private int                     retryTimes    = 3;
+    private int                     retryInterval = 5000;                                    // 重试的时间间隔，默认5秒
+    private CanalNodeAccessStrategy accessStrategy;
+    private SimpleCanalConnector    currentConnector;
+    private String                  destination;
+    private String                  filter;                                                  // 记录上一次的filter提交值,便于自动重试时提交
+
+    public ClusterCanalConnector(String username, String password, String destination,
+                                 CanalNodeAccessStrategy accessStrategy){
+        this.username = username;
+        this.password = password;
+        this.destination = destination;
+        this.accessStrategy = accessStrategy;
+    }
+
+    public void connect() throws CanalClientException {
+        while (currentConnector == null) {
+            SocketAddress nextAddress = this.accessStrategy.nextNode();
+            int times = 0;
+            while (true) {
+                try {
+                    currentConnector = new SimpleCanalConnector(nextAddress, username, password, destination);
+                    currentConnector.setSoTimeout(soTimeout);
+                    if (filter != null) {
+                        currentConnector.setFilter(filter);
+                    }
+                    if (accessStrategy instanceof ClusterNodeAccessStrategy) {
+                        currentConnector.setZkClientx(((ClusterNodeAccessStrategy) accessStrategy).getZkClient());
+                    }
+
+                    currentConnector.connect();
+                    break;
+                } catch (Exception e) {
+                    logger.warn("failed to connect to:{} after retry {} times", nextAddress, times);
+                    currentConnector.disconnect();
+                    currentConnector = null;
+                    // retry for #retryTimes for each node when trying to
+                    // connect to it.
+                    times = times + 1;
+                    if (times >= retryTimes) {
+                        throw new CanalClientException(e);
+                    } else {
+                        // fixed issue #55，增加sleep控制，避免重试connect时cpu使用过高
+                        try {
+                            Thread.sleep(retryInterval);
+                        } catch (InterruptedException e1) {
+                            throw new CanalClientException(e1);
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    public boolean checkValid() {
+        return currentConnector != null && currentConnector.checkValid();
+    }
+
+    public void disconnect() throws CanalClientException {
+        if (currentConnector != null) {
+            currentConnector.disconnect();
+            currentConnector = null;
+        }
+    }
+
+    public void subscribe() throws CanalClientException {
+        subscribe(""); // 传递空字符即可
+    }
+
+    public void subscribe(String filter) throws CanalClientException {
+        int times = 0;
+        while (times < retryTimes) {
+            try {
+                currentConnector.subscribe(filter);
+                this.filter = filter;
+                return;
+            } catch (Throwable t) {
+                logger.warn("something goes wrong when subscribing from server:{}\n{}",
+                    currentConnector.getAddress(),
+                    ExceptionUtils.getFullStackTrace(t));
+                times++;
+                restart();
+                logger.info("restart the connector for next round retry.");
+            }
+        }
+
+        throw new CanalClientException("failed to subscribe after " + times + " times retry.");
+    }
+
+    public void unsubscribe() throws CanalClientException {
+        int times = 0;
+        while (times < retryTimes) {
+            try {
+                currentConnector.unsubscribe();
+                return;
+            } catch (Throwable t) {
+                logger.warn("something goes wrong when unsubscribing from server:{}\n{}",
+                    currentConnector.getAddress(),
+                    ExceptionUtils.getFullStackTrace(t));
+                times++;
+                restart();
+                logger.info("restart the connector for next round retry.");
+            }
+        }
+        throw new CanalClientException("failed to unsubscribe after " + times + " times retry.");
+    }
+
+    public Message get(int batchSize) throws CanalClientException {
+        int times = 0;
+        while (times < retryTimes) {
+            try {
+                Message msg = currentConnector.get(batchSize);
+                return msg;
+            } catch (Throwable t) {
+                logger.warn("something goes wrong when getting data from server:{}\n{}",
+                    currentConnector.getAddress(),
+                    ExceptionUtils.getFullStackTrace(t));
+                times++;
+                restart();
+                logger.info("restart the connector for next round retry.");
+            }
+        }
+        throw new CanalClientException("failed to fetch the data after " + times + " times retry");
+    }
+
+    public Message get(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException {
+        int times = 0;
+        while (times < retryTimes) {
+            try {
+                Message msg = currentConnector.get(batchSize, timeout, unit);
+                return msg;
+            } catch (Throwable t) {
+                logger.warn("something goes wrong when getting data from server:{}\n{}",
+                    currentConnector.getAddress(),
+                    ExceptionUtils.getFullStackTrace(t));
+                times++;
+                restart();
+                logger.info("restart the connector for next round retry.");
+            }
+        }
+        throw new CanalClientException("failed to fetch the data after " + times + " times retry");
+    }
+
+    public Message getWithoutAck(int batchSize) throws CanalClientException {
+        int times = 0;
+        while (times < retryTimes) {
+            try {
+                Message msg = currentConnector.getWithoutAck(batchSize);
+                return msg;
+            } catch (Throwable t) {
+                logger.warn("something goes wrong when getWithoutAck data from server:{}\n{}",
+                    currentConnector.getAddress(),
+                    ExceptionUtils.getFullStackTrace(t));
+                times++;
+                restart();
+                logger.info("restart the connector for next round retry.");
+            }
+        }
+        throw new CanalClientException("failed to fetch the data after " + times + " times retry");
+    }
+
+    public Message getWithoutAck(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException {
+        int times = 0;
+        while (times < retryTimes) {
+            try {
+                Message msg = currentConnector.getWithoutAck(batchSize, timeout, unit);
+                return msg;
+            } catch (Throwable t) {
+                logger.warn("something goes wrong when getWithoutAck data from server:{}\n{}",
+                    currentConnector.getAddress(),
+                    ExceptionUtils.getFullStackTrace(t));
+                times++;
+                restart();
+                logger.info("restart the connector for next round retry.");
+            }
+        }
+        throw new CanalClientException("failed to fetch the data after " + times + " times retry");
+    }
+
+    public void rollback(long batchId) throws CanalClientException {
+        int times = 0;
+        while (times < retryTimes) {
+            try {
+                currentConnector.rollback(batchId);
+                return;
+            } catch (Throwable t) {
+                logger.warn("something goes wrong when rollbacking data from server:{}\n{}",
+                    currentConnector.getAddress(),
+                    ExceptionUtils.getFullStackTrace(t));
+                times++;
+                restart();
+                logger.info("restart the connector for next round retry.");
+            }
+        }
+        throw new CanalClientException("failed to rollback after " + times + " times retry");
+    }
+
+    public void rollback() throws CanalClientException {
+        int times = 0;
+        while (times < retryTimes) {
+            try {
+                currentConnector.rollback();
+                return;
+            } catch (Throwable t) {
+                logger.warn("something goes wrong when rollbacking data from server:{}\n{}",
+                    currentConnector.getAddress(),
+                    ExceptionUtils.getFullStackTrace(t));
+                times++;
+                restart();
+                logger.info("restart the connector for next round retry.");
+            }
+        }
+
+        throw new CanalClientException("failed to rollback after " + times + " times retry");
+    }
+
+    public void ack(long batchId) throws CanalClientException {
+        int times = 0;
+        while (times < retryTimes) {
+            try {
+                currentConnector.ack(batchId);
+                return;
+            } catch (Throwable t) {
+                logger.warn("something goes wrong when acking data from server:{}\n{}",
+                    currentConnector.getAddress(),
+                    ExceptionUtils.getFullStackTrace(t));
+                times++;
+                restart();
+                logger.info("restart the connector for next round retry.");
+            }
+        }
+
+        throw new CanalClientException("failed to ack after " + times + " times retry");
+    }
+
+    private void restart() throws CanalClientException {
+        disconnect();
+        try {
+            Thread.sleep(retryInterval);
+        } catch (InterruptedException e) {
+            throw new CanalClientException(e);
+        }
+        connect();
+    }
+
+    // ============================= setter / getter
+    // ============================
+
+    public String getUsername() {
+        return username;
+    }
+
+    public void setUsername(String username) {
+        this.username = username;
+    }
+
+    public String getPassword() {
+        return password;
+    }
+
+    public void setPassword(String password) {
+        this.password = password;
+    }
+
+    public int getSoTimeout() {
+        return soTimeout;
+    }
+
+    public void setSoTimeout(int soTimeout) {
+        this.soTimeout = soTimeout;
+    }
+
+    public int getRetryTimes() {
+        return retryTimes;
+    }
+
+    public void setRetryTimes(int retryTimes) {
+        this.retryTimes = retryTimes;
+    }
+
+    public int getRetryInterval() {
+        return retryInterval;
+    }
+
+    public void setRetryInterval(int retryInterval) {
+        this.retryInterval = retryInterval;
+    }
+
+    public CanalNodeAccessStrategy getAccessStrategy() {
+        return accessStrategy;
+    }
+
+    public void setAccessStrategy(CanalNodeAccessStrategy accessStrategy) {
+        this.accessStrategy = accessStrategy;
+    }
+
+    public SimpleCanalConnector getCurrentConnector() {
+        return currentConnector;
+    }
+
+}

client/src/main/java/com/alibaba/otter/canal/client/impl/ClusterNodeAccessStrategy.java

@@ -0,0 +1,112 @@
+package com.alibaba.otter.canal.client.impl;
+
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.I0Itec.zkclient.IZkChildListener;
+import org.I0Itec.zkclient.IZkDataListener;
+import org.apache.commons.lang.StringUtils;
+
+import com.alibaba.otter.canal.client.CanalNodeAccessStrategy;
+import com.alibaba.otter.canal.common.utils.JsonUtils;
+import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
+import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
+import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningData;
+import com.alibaba.otter.canal.protocol.exception.CanalClientException;
+
+/**
+ * 集群模式的调度策略
+ * 
+ * @author jianghang 2012-12-3 下午10:01:04
+ * @version 1.0.0
+ */
+public class ClusterNodeAccessStrategy implements CanalNodeAccessStrategy {
+
+    private IZkChildListener                 childListener;                                      // 监听所有的服务器列表
+    private IZkDataListener                  dataListener;                                       // 监听当前的工作节点
+    private ZkClientx                        zkClient;
+    private volatile List<InetSocketAddress> currentAddress = new ArrayList<InetSocketAddress>();
+    private volatile InetSocketAddress       runningAddress = null;
+
+    public ClusterNodeAccessStrategy(String destination, ZkClientx zkClient){
+        this.zkClient = zkClient;
+        childListener = new IZkChildListener() {
+
+            public void handleChildChange(String parentPath, List<String> currentChilds) throws Exception {
+                initClusters(currentChilds);
+            }
+
+        };
+
+        dataListener = new IZkDataListener() {
+
+            public void handleDataDeleted(String dataPath) throws Exception {
+                runningAddress = null;
+            }
+
+            public void handleDataChange(String dataPath, Object data) throws Exception {
+                initRunning(data);
+            }
+
+        };
+
+        String clusterPath = ZookeeperPathUtils.getDestinationClusterRoot(destination);
+        this.zkClient.subscribeChildChanges(clusterPath, childListener);
+        initClusters(this.zkClient.getChildren(clusterPath));
+
+        String runningPath = ZookeeperPathUtils.getDestinationServerRunning(destination);
+        this.zkClient.subscribeDataChanges(runningPath, dataListener);
+        initRunning(this.zkClient.readData(runningPath, true));
+    }
+
+    public SocketAddress nextNode() {
+        if (runningAddress != null) {// 如果服务已经启动，直接选择当前正在工作的节点
+            return runningAddress;
+        } else if (!currentAddress.isEmpty()) { // 如果不存在已经启动的服务，可能服务是一种lazy启动，随机选择一台触发服务器进行启动
+            return currentAddress.get(0);// 默认返回第一个节点，之前已经做过shuffle
+        } else {
+            throw new CanalClientException("no alive canal server");
+        }
+    }
+
+    private void initClusters(List<String> currentChilds) {
+        if (currentChilds == null || currentChilds.isEmpty()) {
+            currentAddress = new ArrayList<InetSocketAddress>();
+        } else {
+            List<InetSocketAddress> addresses = new ArrayList<InetSocketAddress>();
+            for (String address : currentChilds) {
+                String[] strs = StringUtils.split(address, ":");
+                if (strs != null && strs.length == 2) {
+                    addresses.add(new InetSocketAddress(strs[0], Integer.valueOf(strs[1])));
+                }
+            }
+
+            Collections.shuffle(addresses);
+            currentAddress = addresses;// 直接切换引用
+        }
+    }
+
+    private void initRunning(Object data) {
+        if (data == null) {
+            return;
+        }
+
+        ServerRunningData runningData = JsonUtils.unmarshalFromByte((byte[]) data, ServerRunningData.class);
+        String[] strs = StringUtils.split(runningData.getAddress(), ':');
+        if (strs.length == 2) {
+            runningAddress = new InetSocketAddress(strs[0], Integer.valueOf(strs[1]));
+        }
+    }
+
+    public void setZkClient(ZkClientx zkClient) {
+        this.zkClient = zkClient;
+    }
+
+    public ZkClientx getZkClient() {
+        return zkClient;
+    }
+
+}

client/src/main/java/com/alibaba/otter/canal/client/impl/SimpleCanalConnector.java

@@ -0,0 +1,475 @@
+package com.alibaba.otter.canal.client.impl;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.channels.SocketChannel;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.client.CanalConnector;
+import com.alibaba.otter.canal.client.impl.running.ClientRunningData;
+import com.alibaba.otter.canal.client.impl.running.ClientRunningListener;
+import com.alibaba.otter.canal.client.impl.running.ClientRunningMonitor;
+import com.alibaba.otter.canal.common.utils.AddressUtils;
+import com.alibaba.otter.canal.common.utils.BooleanMutex;
+import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
+import com.alibaba.otter.canal.protocol.CanalEntry.Entry;
+import com.alibaba.otter.canal.protocol.CanalPacket.Ack;
+import com.alibaba.otter.canal.protocol.CanalPacket.ClientAck;
+import com.alibaba.otter.canal.protocol.CanalPacket.ClientAuth;
+import com.alibaba.otter.canal.protocol.CanalPacket.ClientRollback;
+import com.alibaba.otter.canal.protocol.CanalPacket.Compression;
+import com.alibaba.otter.canal.protocol.CanalPacket.Get;
+import com.alibaba.otter.canal.protocol.CanalPacket.Handshake;
+import com.alibaba.otter.canal.protocol.CanalPacket.Messages;
+import com.alibaba.otter.canal.protocol.CanalPacket.Packet;
+import com.alibaba.otter.canal.protocol.CanalPacket.PacketType;
+import com.alibaba.otter.canal.protocol.CanalPacket.Sub;
+import com.alibaba.otter.canal.protocol.CanalPacket.Unsub;
+import com.alibaba.otter.canal.protocol.ClientIdentity;
+import com.alibaba.otter.canal.protocol.Message;
+import com.alibaba.otter.canal.protocol.exception.CanalClientException;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.InvalidProtocolBufferException;
+
+/**
+ * 基于{@linkplain CanalServerWithNetty}定义的网络协议接口，对于canal数据进行get/rollback/ack等操作
+ * 
+ * @author jianghang 2012-10-24 下午05:37:20
+ * @version 1.0.0
+ */
+public class SimpleCanalConnector implements CanalConnector {
+
+    private static final Logger  logger                = LoggerFactory.getLogger(SimpleCanalConnector.class);
+    private SocketAddress        address;
+    private String               username;
+    private String               password;
+    private int                  soTimeout             = 60000;                                              // milliseconds
+    private String               filter;                                                                     // 记录上一次的filter提交值,便于自动重试时提交
+
+    private final ByteBuffer     readHeader            = ByteBuffer.allocate(4).order(ByteOrder.BIG_ENDIAN);
+    private final ByteBuffer     writeHeader           = ByteBuffer.allocate(4).order(ByteOrder.BIG_ENDIAN);
+    private SocketChannel        channel;
+    private List<Compression>    supportedCompressions = new ArrayList<Compression>();
+    private ClientIdentity       clientIdentity;
+    private ClientRunningMonitor runningMonitor;                                                             // 运行控制
+    private ZkClientx            zkClientx;
+    private BooleanMutex         mutex                 = new BooleanMutex(false);
+    private volatile boolean     connected             = false;                                              // 代表connected是否已正常执行，因为有HA，不代表在工作中
+    private boolean              rollbackOnConnect     = true;                                               // 是否在connect链接成功后，自动执行rollback操作
+    private boolean              rollbackOnDisConnect  = false;                                              // 是否在connect链接成功后，自动执行rollback操作
+
+    // 读写数据分别使用不同的锁进行控制，减小锁粒度,读也需要排他锁，并发度容易造成数据包混乱，反序列化失败
+    private Object               readDataLock          = new Object();
+    private Object               writeDataLock         = new Object();
+
+    public SimpleCanalConnector(SocketAddress address, String username, String password, String destination){
+        this(address, username, password, destination, 60000);
+    }
+
+    public SimpleCanalConnector(SocketAddress address, String username, String password, String destination,
+                                int soTimeout){
+        this.address = address;
+        this.username = username;
+        this.password = password;
+        this.soTimeout = soTimeout;
+        this.clientIdentity = new ClientIdentity(destination, (short) 1001);
+    }
+
+    public void connect() throws CanalClientException {
+        if (connected) {
+            return;
+        }
+
+        if (runningMonitor != null) {
+            if (!runningMonitor.isStart()) {
+                runningMonitor.start();
+            }
+        } else {
+            waitClientRunning();
+            doConnect();
+            if (filter != null) { // 如果存在条件，说明是自动切换，基于上一次的条件订阅一次
+                subscribe(filter);
+            }
+            if (rollbackOnConnect) {
+                rollback();
+            }
+        }
+        
+        connected = true;
+    }
+
+    public void disconnect() throws CanalClientException {
+        if (rollbackOnDisConnect && channel.isConnected()) {
+            rollback();
+        }
+
+        connected = false;
+        if (runningMonitor != null) {
+            if (runningMonitor.isStart()) {
+                runningMonitor.stop();
+            }
+        } else {
+            doDisconnnect();
+        }
+    }
+
+    private InetSocketAddress doConnect() throws CanalClientException {
+        try {
+            channel = SocketChannel.open();
+            channel.socket().setSoTimeout(soTimeout);
+            channel.connect(address);
+            Packet p = Packet.parseFrom(readNextPacket(channel));
+            if (p.getVersion() != 1) {
+                throw new CanalClientException("unsupported version at this client.");
+            }
+
+            if (p.getType() != PacketType.HANDSHAKE) {
+                throw new CanalClientException("expect handshake but found other type.");
+            }
+            //
+            Handshake handshake = Handshake.parseFrom(p.getBody());
+            supportedCompressions.addAll(handshake.getSupportedCompressionsList());
+            //
+            ClientAuth ca = ClientAuth.newBuilder()
+                .setUsername(username != null ? username : "")
+                .setNetReadTimeout(soTimeout)
+                .setNetWriteTimeout(soTimeout)
+                .build();
+            writeWithHeader(channel,
+                Packet.newBuilder()
+                    .setType(PacketType.CLIENTAUTHENTICATION)
+                    .setBody(ca.toByteString())
+                    .build()
+                    .toByteArray());
+            //
+            Packet ack = Packet.parseFrom(readNextPacket(channel));
+            if (ack.getType() != PacketType.ACK) {
+                throw new CanalClientException("unexpected packet type when ack is expected");
+            }
+
+            Ack ackBody = Ack.parseFrom(ack.getBody());
+            if (ackBody.getErrorCode() > 0) {
+                throw new CanalClientException("something goes wrong when doing authentication: "
+                                               + ackBody.getErrorMessage());
+            }
+
+            connected = true;
+            return new InetSocketAddress(channel.socket().getLocalAddress(), channel.socket().getLocalPort());
+        } catch (IOException e) {
+            throw new CanalClientException(e);
+        }
+    }
+
+    private void doDisconnnect() throws CanalClientException {
+        if (channel != null) {
+            try {
+                channel.close();
+            } catch (IOException e) {
+                logger.warn("exception on closing channel:{} \n {}", channel, e);
+            }
+            channel = null;
+        }
+    }
+
+    public void subscribe() throws CanalClientException {
+        subscribe(""); // 传递空字符即可
+    }
+
+    public void subscribe(String filter) throws CanalClientException {
+        waitClientRunning();
+        try {
+            writeWithHeader(channel,
+                Packet.newBuilder()
+                    .setType(PacketType.SUBSCRIPTION)
+                    .setBody(Sub.newBuilder()
+                        .setDestination(clientIdentity.getDestination())
+                        .setClientId(String.valueOf(clientIdentity.getClientId()))
+                        .setFilter(filter != null ? filter : "")
+                        .build()
+                        .toByteString())
+                    .build()
+                    .toByteArray());
+            //
+            Packet p = Packet.parseFrom(readNextPacket(channel));
+            Ack ack = Ack.parseFrom(p.getBody());
+            if (ack.getErrorCode() > 0) {
+                throw new CanalClientException("failed to subscribe with reason: " + ack.getErrorMessage());
+            }
+
+            clientIdentity.setFilter(filter);
+        } catch (IOException e) {
+            throw new CanalClientException(e);
+        }
+    }
+
+    public void unsubscribe() throws CanalClientException {
+        waitClientRunning();
+        try {
+            writeWithHeader(channel,
+                Packet.newBuilder()
+                    .setType(PacketType.UNSUBSCRIPTION)
+                    .setBody(Unsub.newBuilder()
+                        .setDestination(clientIdentity.getDestination())
+                        .setClientId(String.valueOf(clientIdentity.getClientId()))
+                        .build()
+                        .toByteString())
+                    .build()
+                    .toByteArray());
+            //
+            Packet p = Packet.parseFrom(readNextPacket(channel));
+            Ack ack = Ack.parseFrom(p.getBody());
+            if (ack.getErrorCode() > 0) {
+                throw new CanalClientException("failed to unSubscribe with reason: " + ack.getErrorMessage());
+            }
+        } catch (IOException e) {
+            throw new CanalClientException(e);
+        }
+    }
+
+    public Message get(int batchSize) throws CanalClientException {
+        return get(batchSize, null, null);
+    }
+
+    public Message get(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException {
+        Message message = getWithoutAck(batchSize, timeout, unit);
+        ack(message.getId());
+        return message;
+    }
+
+    public Message getWithoutAck(int batchSize) throws CanalClientException {
+        return getWithoutAck(batchSize, null, null);
+    }
+
+    public Message getWithoutAck(int batchSize, Long timeout, TimeUnit unit) throws CanalClientException {
+        waitClientRunning();
+        try {
+            int size = (batchSize <= 0) ? 1000 : batchSize;
+            long time = (timeout == null || timeout < 0) ? -1 : timeout; // -1代表不做timeout控制
+            if (unit == null) {
+                unit = TimeUnit.MILLISECONDS;
+            }
+
+            writeWithHeader(channel,
+                Packet.newBuilder()
+                    .setType(PacketType.GET)
+                    .setBody(Get.newBuilder()
+                        .setAutoAck(false)
+                        .setDestination(clientIdentity.getDestination())
+                        .setClientId(String.valueOf(clientIdentity.getClientId()))
+                        .setFetchSize(size)
+                        .setTimeout(time)
+                        .setUnit(unit.ordinal())
+                        .build()
+                        .toByteString())
+                    .build()
+                    .toByteArray());
+
+            return receiveMessages();
+        } catch (IOException e) {
+            throw new CanalClientException(e);
+        }
+    }
+
+    private Message receiveMessages() throws InvalidProtocolBufferException, IOException {
+        Packet p = Packet.parseFrom(readNextPacket(channel));
+        switch (p.getType()) {
+            case MESSAGES: {
+                if (!p.getCompression().equals(Compression.NONE)) {
+                    throw new CanalClientException("compression is not supported in this connector");
+                }
+
+                Messages messages = Messages.parseFrom(p.getBody());
+                Message result = new Message(messages.getBatchId());
+                for (ByteString byteString : messages.getMessagesList()) {
+                    result.addEntry(Entry.parseFrom(byteString));
+                }
+                return result;
+            }
+            case ACK: {
+                Ack ack = Ack.parseFrom(p.getBody());
+                throw new CanalClientException("something goes wrong with reason: " + ack.getErrorMessage());
+            }
+            default: {
+                throw new CanalClientException("unexpected packet type: " + p.getType());
+            }
+        }
+    }
+
+    public void ack(long batchId) throws CanalClientException {
+        waitClientRunning();
+        ClientAck ca = ClientAck.newBuilder()
+            .setDestination(clientIdentity.getDestination())
+            .setClientId(String.valueOf(clientIdentity.getClientId()))
+            .setBatchId(batchId)
+            .build();
+        try {
+            writeWithHeader(channel, Packet.newBuilder()
+                .setType(PacketType.CLIENTACK)
+                .setBody(ca.toByteString())
+                .build()
+                .toByteArray());
+        } catch (IOException e) {
+            throw new CanalClientException(e);
+        }
+    }
+
+    public void rollback(long batchId) throws CanalClientException {
+        waitClientRunning();
+        ClientRollback ca = ClientRollback.newBuilder()
+            .setDestination(clientIdentity.getDestination())
+            .setClientId(String.valueOf(clientIdentity.getClientId()))
+            .setBatchId(batchId)
+            .build();
+        try {
+            writeWithHeader(channel, Packet.newBuilder()
+                .setType(PacketType.CLIENTROLLBACK)
+                .setBody(ca.toByteString())
+                .build()
+                .toByteArray());
+        } catch (IOException e) {
+            throw new CanalClientException(e);
+        }
+    }
+
+    public void rollback() throws CanalClientException {
+        waitClientRunning();
+        rollback(0);// 0代笔未设置
+    }
+
+    // ==================== helper method ====================
+
+    private void writeWithHeader(SocketChannel channel, byte[] body) throws IOException {
+        synchronized (writeDataLock) {
+            writeHeader.clear();
+            writeHeader.putInt(body.length);
+            writeHeader.flip();
+            channel.write(writeHeader);
+            channel.write(ByteBuffer.wrap(body));
+        }
+    }
+
+    private byte[] readNextPacket(SocketChannel channel) throws IOException {
+        synchronized (readDataLock) {
+            readHeader.clear();
+            read(channel, readHeader);
+            int bodyLen = readHeader.getInt(0);
+            ByteBuffer bodyBuf = ByteBuffer.allocate(bodyLen).order(ByteOrder.BIG_ENDIAN);
+            read(channel, bodyBuf);
+            return bodyBuf.array();
+        }
+    }
+
+    private void read(SocketChannel channel, ByteBuffer buffer) throws IOException {
+        while (buffer.hasRemaining()) {
+            int r = channel.read(buffer);
+            if (r == -1) {
+                throw new IOException("end of stream when reading header");
+            }
+        }
+    }
+
+    private synchronized void initClientRunningMonitor(ClientIdentity clientIdentity) {
+        if (zkClientx != null && clientIdentity != null && runningMonitor == null) {
+            ClientRunningData clientData = new ClientRunningData();
+            clientData.setClientId(clientIdentity.getClientId());
+            clientData.setAddress(AddressUtils.getHostIp());
+
+            runningMonitor = new ClientRunningMonitor();
+            runningMonitor.setDestination(clientIdentity.getDestination());
+            runningMonitor.setZkClient(zkClientx);
+            runningMonitor.setClientData(clientData);
+            runningMonitor.setListener(new ClientRunningListener() {
+
+                public InetSocketAddress processActiveEnter() {
+                    InetSocketAddress address = doConnect();
+                    mutex.set(true);
+                    if (filter != null) { // 如果存在条件，说明是自动切换，基于上一次的条件订阅一次
+                        subscribe(filter);
+                    }
+
+                    if (rollbackOnConnect) {
+                        rollback();
+                    }
+
+                    return address;
+                }
+
+                public void processActiveExit() {
+                    mutex.set(false);
+                    doDisconnnect();
+                }
+
+            });
+        }
+    }
+
+
+    private void waitClientRunning() {
+        try {
+            if (zkClientx != null) {
+                if (!connected) {// 未调用connect
+                    throw new CanalClientException("should connect first");
+                }
+
+                mutex.get();// 阻塞等待
+            }
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            throw new CanalClientException(e);
+        }
+    }
+
+    public boolean checkValid() {
+        if (zkClientx != null) {
+            return mutex.state();
+        } else {
+            return true;// 默认都放过
+        }
+    }
+
+    public SocketAddress getAddress() {
+        return address;
+    }
+
+    public String getUsername() {
+        return username;
+    }
+
+    public String getPassword() {
+        return password;
+    }
+
+    public int getSoTimeout() {
+        return soTimeout;
+    }
+
+    public void setSoTimeout(int soTimeout) {
+        this.soTimeout = soTimeout;
+    }
+
+    public void setZkClientx(ZkClientx zkClientx) {
+        this.zkClientx = zkClientx;
+        initClientRunningMonitor(this.clientIdentity);
+    }
+
+    public void setRollbackOnConnect(boolean rollbackOnConnect) {
+        this.rollbackOnConnect = rollbackOnConnect;
+    }
+
+    public void setRollbackOnDisConnect(boolean rollbackOnDisConnect) {
+        this.rollbackOnDisConnect = rollbackOnDisConnect;
+    }
+
+    public void setFilter(String filter) {
+        this.filter = filter;
+    }
+
+}

client/src/main/java/com/alibaba/otter/canal/client/impl/SimpleNodeAccessStrategy.java

@@ -0,0 +1,35 @@
+package com.alibaba.otter.canal.client.impl;
+
+import java.net.SocketAddress;
+import java.util.ArrayList;
+import java.util.List;
+
+import com.alibaba.otter.canal.client.CanalNodeAccessStrategy;
+
+/**
+ * 简单版本的node访问实现
+ * 
+ * @author jianghang 2012-10-29 下午08:00:23
+ * @version 1.0.0
+ */
+public class SimpleNodeAccessStrategy implements CanalNodeAccessStrategy {
+
+    private List<SocketAddress> nodes = new ArrayList<SocketAddress>();
+    private int                 index = 0;
+
+    public SimpleNodeAccessStrategy(List<? extends SocketAddress> nodes){
+        if (nodes == null || nodes.size() < 1) {
+            throw new IllegalArgumentException("at least 1 node required.");
+        }
+        this.nodes.addAll(nodes);
+    }
+
+    public SocketAddress nextNode() {
+        try {
+            return nodes.get(index);
+        } finally {
+            index = (index + 1) % nodes.size();
+        }
+    }
+
+}

client/src/main/java/com/alibaba/otter/canal/client/impl/running/ClientRunningData.java

@@ -0,0 +1,39 @@
+package com.alibaba.otter.canal.client.impl.running;
+
+/**
+ * client running状态信息
+ * 
+ * @author jianghang 2012-11-22 下午03:41:50
+ * @version 1.0.0
+ */
+public class ClientRunningData {
+
+    private short   clientId;
+    private String  address;
+    private boolean active = true;
+
+    public short getClientId() {
+        return clientId;
+    }
+
+    public void setClientId(short clientId) {
+        this.clientId = clientId;
+    }
+
+    public String getAddress() {
+        return address;
+    }
+
+    public void setAddress(String address) {
+        this.address = address;
+    }
+
+    public boolean isActive() {
+        return active;
+    }
+
+    public void setActive(boolean active) {
+        this.active = active;
+    }
+
+}

client/src/main/java/com/alibaba/otter/canal/client/impl/running/ClientRunningListener.java

@@ -0,0 +1,23 @@
+package com.alibaba.otter.canal.client.impl.running;
+
+import java.net.InetSocketAddress;
+
+/**
+ * 触发一下mainstem发生切换
+ * 
+ * @author jianghang 2012-9-11 下午02:26:03
+ * @version 1.0.0
+ */
+public interface ClientRunningListener {
+
+    /**
+     * 触发现在轮到自己做为active，需要载入上一个active的上下文数据
+     */
+    public InetSocketAddress processActiveEnter();
+
+    /**
+     * 触发一下当前active模式失败
+     */
+    public void processActiveExit();
+
+}

client/src/main/java/com/alibaba/otter/canal/client/impl/running/ClientRunningMonitor.java

@@ -0,0 +1,228 @@
+package com.alibaba.otter.canal.client.impl.running;
+
+import java.net.InetSocketAddress;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.I0Itec.zkclient.IZkDataListener;
+import org.I0Itec.zkclient.exception.ZkException;
+import org.I0Itec.zkclient.exception.ZkInterruptedException;
+import org.I0Itec.zkclient.exception.ZkNoNodeException;
+import org.I0Itec.zkclient.exception.ZkNodeExistsException;
+import org.apache.zookeeper.CreateMode;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
+
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+import com.alibaba.otter.canal.common.utils.BooleanMutex;
+import com.alibaba.otter.canal.common.utils.JsonUtils;
+import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
+import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
+
+/**
+ * clinet running控制
+ * 
+ * @author jianghang 2012-11-22 下午03:43:01
+ * @version 1.0.0
+ */
+public class ClientRunningMonitor extends AbstractCanalLifeCycle {
+
+    private static final Logger        logger       = LoggerFactory.getLogger(ClientRunningMonitor.class);
+    private ZkClientx                  zkClient;
+    private String                     destination;
+    private ClientRunningData          clientData;
+    private IZkDataListener            dataListener;
+    private BooleanMutex               mutex        = new BooleanMutex(false);
+    private volatile boolean           release      = false;
+    private volatile ClientRunningData activeData;
+    private ScheduledExecutorService   delayExector = Executors.newScheduledThreadPool(1);
+    private ClientRunningListener      listener;
+    private int                        delayTime    = 5;
+
+    public ClientRunningMonitor(){
+        dataListener = new IZkDataListener() {
+
+            public void handleDataChange(String dataPath, Object data) throws Exception {
+                MDC.put("destination", destination);
+                ClientRunningData runningData = JsonUtils.unmarshalFromByte((byte[]) data, ClientRunningData.class);
+                if (!isMine(runningData.getAddress())) {
+                    mutex.set(false);
+                }
+
+                if (!runningData.isActive() && isMine(runningData.getAddress())) { // 说明出现了主动释放的操作，并且本机之前是active
+                    release = true;
+                    releaseRunning();// 彻底释放mainstem
+                }
+
+                activeData = (ClientRunningData) runningData;
+            }
+
+            public void handleDataDeleted(String dataPath) throws Exception {
+                MDC.put("destination", destination);
+                mutex.set(false);
+                if (!release && activeData != null && isMine(activeData.getAddress())) {
+                    // 如果上一次active的状态就是本机，则即时触发一下active抢占
+                    initRunning();
+                } else {
+                    // 否则就是等待delayTime，避免因网络瞬端或者zk异常，导致出现频繁的切换操作
+                    delayExector.schedule(new Runnable() {
+
+                        public void run() {
+                            initRunning();
+                        }
+                    }, delayTime, TimeUnit.SECONDS);
+                }
+            }
+
+        };
+
+    }
+
+    public void start() {
+        super.start();
+
+        String path = ZookeeperPathUtils.getDestinationClientRunning(this.destination, clientData.getClientId());
+        zkClient.subscribeDataChanges(path, dataListener);
+        initRunning();
+    }
+
+    public void stop() {
+        super.stop();
+
+        String path = ZookeeperPathUtils.getDestinationClientRunning(this.destination, clientData.getClientId());
+        zkClient.unsubscribeDataChanges(path, dataListener);
+        releaseRunning(); // 尝试一下release
+    }
+
+    public void initRunning() {
+        if (!isStart()) {
+            return;
+        }
+
+        String path = ZookeeperPathUtils.getDestinationClientRunning(this.destination, clientData.getClientId());
+        // 序列化
+        byte[] bytes = JsonUtils.marshalToByte(clientData);
+        try {
+            mutex.set(false);
+            zkClient.create(path, bytes, CreateMode.EPHEMERAL);
+            processActiveEnter();// 触发一下事件
+            activeData = clientData;
+            mutex.set(true);
+        } catch (ZkNodeExistsException e) {
+            bytes = zkClient.readData(path, true);
+            if (bytes == null) {// 如果不存在节点，立即尝试一次
+                initRunning();
+            } else {
+                activeData = JsonUtils.unmarshalFromByte(bytes, ClientRunningData.class);
+            }
+        } catch (ZkNoNodeException e) {
+            zkClient.createPersistent(ZookeeperPathUtils.getClientIdNodePath(this.destination, clientData.getClientId()),
+                true); // 尝试创建父节点
+            initRunning();
+        }
+    }
+
+    /**
+     * 阻塞等待自己成为active，如果自己成为active，立马返回
+     * 
+     * @throws InterruptedException
+     */
+    public void waitForActive() throws InterruptedException {
+        initRunning();
+        mutex.get();
+    }
+
+    /**
+     * 检查当前的状态
+     */
+    public boolean check() {
+        String path = ZookeeperPathUtils.getDestinationClientRunning(this.destination, clientData.getClientId());
+        try {
+            byte[] bytes = zkClient.readData(path);
+            ClientRunningData eventData = JsonUtils.unmarshalFromByte(bytes, ClientRunningData.class);
+            activeData = eventData;// 更新下为最新值
+            // 检查下nid是否为自己
+            boolean result = isMine(activeData.getAddress());
+            if (!result) {
+                logger.warn("canal is running in [{}] , but not in [{}]",
+                    activeData.getAddress(),
+                    clientData.getAddress());
+            }
+            return result;
+        } catch (ZkNoNodeException e) {
+            logger.warn("canal is not run any in node");
+            return false;
+        } catch (ZkInterruptedException e) {
+            logger.warn("canal check is interrupt");
+            Thread.interrupted();// 清除interrupt标记
+            return check();
+        } catch (ZkException e) {
+            logger.warn("canal check is failed");
+            return false;
+        }
+    }
+
+    public boolean releaseRunning() {
+        if (check()) {
+            String path = ZookeeperPathUtils.getDestinationClientRunning(this.destination, clientData.getClientId());
+            zkClient.delete(path);
+            mutex.set(false);
+            processActiveExit();
+            return true;
+        }
+
+        return false;
+    }
+
+    // ====================== helper method ======================
+
+    private boolean isMine(String address) {
+        return address.equals(clientData.getAddress());
+    }
+
+    private void processActiveEnter() {
+        if (listener != null) {
+            // 触发回调，建立与server的socket链接
+            InetSocketAddress connectAddress = listener.processActiveEnter();
+            String address = connectAddress.getAddress().getHostAddress() + ":" + connectAddress.getPort();
+            this.clientData.setAddress(address);
+
+            String path = ZookeeperPathUtils.getDestinationClientRunning(this.destination,
+                this.clientData.getClientId());
+            // 序列化
+            byte[] bytes = JsonUtils.marshalToByte(clientData);
+            zkClient.writeData(path, bytes);
+        }
+    }
+
+    private void processActiveExit() {
+        if (listener != null) {
+            listener.processActiveExit();
+        }
+    }
+
+    public void setListener(ClientRunningListener listener) {
+        this.listener = listener;
+    }
+
+    // ===================== setter / getter =======================
+
+    public void setDestination(String destination) {
+        this.destination = destination;
+    }
+
+    public void setClientData(ClientRunningData clientData) {
+        this.clientData = clientData;
+    }
+
+    public void setDelayTime(int delayTime) {
+        this.delayTime = delayTime;
+    }
+
+    public void setZkClient(ZkClientx zkClient) {
+        this.zkClient = zkClient;
+    }
+
+}

client/src/test/java/com/alibaba/otter/canal/client/running/AbstractZkTest.java

@@ -0,0 +1,18 @@
+package com.alibaba.otter.canal.client.running;
+
+import org.junit.Assert;
+
+public class AbstractZkTest {
+
+    protected String destination = "ljhtest1";
+    protected String cluster1    = "127.0.0.1:2188";
+    protected String cluster2    = "127.0.0.1:2188,127.0.0.1:2188";
+
+    public void sleep(long time) {
+        try {
+            Thread.sleep(time);
+        } catch (InterruptedException e) {
+            Assert.fail(e.getMessage());
+        }
+    }
+}

client/src/test/java/com/alibaba/otter/canal/client/running/ClientRunningTest.java

@@ -0,0 +1,136 @@
+package com.alibaba.otter.canal.client.running;
+
+import java.net.InetSocketAddress;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import org.apache.commons.lang.math.RandomUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alibaba.otter.canal.client.impl.running.ClientRunningData;
+import com.alibaba.otter.canal.client.impl.running.ClientRunningListener;
+import com.alibaba.otter.canal.client.impl.running.ClientRunningMonitor;
+import com.alibaba.otter.canal.common.utils.AddressUtils;
+import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
+import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
+
+public class ClientRunningTest extends AbstractZkTest {
+
+    private ZkClientx zkclientx = new ZkClientx(cluster1 + ";" + cluster2);
+    private short     clientId  = 1001;
+
+    @Before
+    public void setUp() {
+        String path = ZookeeperPathUtils.getDestinationPath(destination);
+        zkclientx.deleteRecursive(path);
+
+        zkclientx.createPersistent(ZookeeperPathUtils.getClientIdNodePath(this.destination, clientId), true);
+    }
+
+    @After
+    public void tearDown() {
+        String path = ZookeeperPathUtils.getDestinationPath(destination);
+        zkclientx.deleteRecursive(path);
+    }
+
+    @Test
+    public void testOneServer() {
+        final CountDownLatch countLatch = new CountDownLatch(2);
+        ClientRunningMonitor runningMonitor = buildClientRunning(countLatch, clientId, 2088);
+        runningMonitor.start();
+        sleep(2000L);
+        runningMonitor.stop();
+        sleep(2000L);
+
+        if (countLatch.getCount() != 0) {
+            Assert.fail();
+        }
+    }
+
+    @Test
+    public void testMultiServer() {
+        final CountDownLatch countLatch = new CountDownLatch(30);
+        final ClientRunningMonitor runningMonitor1 = buildClientRunning(countLatch, clientId, 2088);
+        final ClientRunningMonitor runningMonitor2 = buildClientRunning(countLatch, clientId, 2089);
+        final ClientRunningMonitor runningMonitor3 = buildClientRunning(countLatch, clientId, 2090);
+        final ExecutorService executor = Executors.newFixedThreadPool(3);
+        executor.submit(new Runnable() {
+
+            public void run() {
+                for (int i = 0; i < 10; i++) {
+                    if (!runningMonitor1.isStart()) {
+                        runningMonitor1.start();
+                    }
+                    sleep(2000L + RandomUtils.nextInt(500));
+                    runningMonitor1.stop();
+                    sleep(2000L + RandomUtils.nextInt(500));
+                }
+            }
+
+        });
+
+        executor.submit(new Runnable() {
+
+            public void run() {
+                for (int i = 0; i < 10; i++) {
+                    if (!runningMonitor2.isStart()) {
+                        runningMonitor2.start();
+                    }
+                    sleep(2000L + RandomUtils.nextInt(500));
+                    runningMonitor2.stop();
+                    sleep(2000L + RandomUtils.nextInt(500));
+                }
+            }
+
+        });
+
+        executor.submit(new Runnable() {
+
+            public void run() {
+                for (int i = 0; i < 10; i++) {
+                    if (!runningMonitor3.isStart()) {
+                        runningMonitor3.start();
+                    }
+                    sleep(2000L + RandomUtils.nextInt(500));
+                    runningMonitor3.stop();
+                    sleep(2000L + RandomUtils.nextInt(500));
+                }
+            }
+
+        });
+
+        sleep(30000L);
+    }
+
+    private ClientRunningMonitor buildClientRunning(final CountDownLatch countLatch, final short clientId,
+                                                    final int port) {
+        ClientRunningData clientData = new ClientRunningData();
+        clientData.setClientId(clientId);
+        clientData.setAddress(AddressUtils.getHostIp());
+
+        ClientRunningMonitor runningMonitor = new ClientRunningMonitor();
+        runningMonitor.setDestination(destination);
+        runningMonitor.setZkClient(zkclientx);
+        runningMonitor.setClientData(clientData);
+        runningMonitor.setListener(new ClientRunningListener() {
+
+            public InetSocketAddress processActiveEnter() {
+                System.out.println(String.format("clientId:%s port:%s has start", clientId, port));
+                countLatch.countDown();
+                return new InetSocketAddress(AddressUtils.getHostIp(), port);
+            }
+
+            public void processActiveExit() {
+                countLatch.countDown();
+                System.out.println(String.format("clientId:%s port:%s has stop", clientId, port));
+            }
+
+        });
+        runningMonitor.setDelayTime(1);
+        return runningMonitor;
+    }
+}

client/src/test/java/logback.xml

@@ -0,0 +1,14 @@
+<configuration scan="true" scanPeriod=" 5 seconds">
+
+	<jmxConfigurator />
+	<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+		<encoder>
+			<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n
+			</pattern>
+		</encoder>
+	</appender>
+	
+	<root level="WARN">
+		<appender-ref ref="STDOUT"/>
+	</root>
+</configuration>

common/pom.xml

@@ -0,0 +1,69 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<parent>
+		<groupId>com.alibaba.otter</groupId>
+		<artifactId>canal</artifactId>
+		<version>1.0.19-SNAPSHOT</version>
+		<relativePath>../pom.xml</relativePath>
+	</parent>
+	<groupId>com.alibaba.otter</groupId>
+	<artifactId>canal.common</artifactId>
+	<packaging>jar</packaging>
+	<name>canal common module for otter ${project.version}</name>
+	<url>http://b2b-doc.alibaba-inc.com/display/opentech/Otter</url>
+	<dependencies>
+		<!-- zk -->
+		<dependency>
+			<groupId>org.apache.zookeeper</groupId>
+			<artifactId>zookeeper</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>com.github.sgroschupf</groupId>
+			<artifactId>zkclient</artifactId>
+		</dependency>
+		<!-- external -->
+		<dependency>
+			<groupId>commons-io</groupId>
+			<artifactId>commons-io</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>commons-lang</groupId>
+			<artifactId>commons-lang</artifactId>
+			<version>2.6</version>
+		</dependency>
+		<dependency>
+			<groupId>org.springframework</groupId>
+			<artifactId>spring</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>com.alibaba</groupId>
+			<artifactId>fastjson</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>com.google.guava</groupId>
+			<artifactId>guava</artifactId>
+		</dependency>
+		<!-- log -->
+		<dependency>
+			<groupId>ch.qos.logback</groupId>
+			<artifactId>logback-core</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>ch.qos.logback</groupId>
+			<artifactId>logback-classic</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.slf4j</groupId>
+			<artifactId>jcl-over-slf4j</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.slf4j</groupId>
+			<artifactId>slf4j-api</artifactId>
+		</dependency>
+		<!-- junit -->
+		<dependency>
+		  <groupId>junit</groupId>
+		  <artifactId>junit</artifactId>
+		</dependency>
+  </dependencies>
+</project>

common/src/main/java/com/alibaba/otter/canal/common/AbstractCanalLifeCycle.java

@@ -0,0 +1,33 @@
+package com.alibaba.otter.canal.common;
+
+/**
+ * 基本实现
+ * 
+ * @author jianghang 2012-7-12 上午10:11:07
+ * @version 1.0.0
+ */
+public abstract class AbstractCanalLifeCycle implements CanalLifeCycle {
+
+    protected volatile boolean running = false; // 是否处于运行中
+
+    public boolean isStart() {
+        return running;
+    }
+
+    public void start() {
+        if (running) {
+            throw new CanalException(this.getClass().getName() + " has startup , don't repeat start");
+        }
+
+        running = true;
+    }
+
+    public void stop() {
+        if (!running) {
+            throw new CanalException(this.getClass().getName() + " isn't start , please check");
+        }
+
+        running = false;
+    }
+
+}

common/src/main/java/com/alibaba/otter/canal/common/CanalException.java

@@ -0,0 +1,37 @@
+package com.alibaba.otter.canal.common;
+
+import org.apache.commons.lang.exception.NestableRuntimeException;
+
+/**
+ * @author jianghang 2012-7-12 上午10:10:31
+ * @version 1.0.0
+ */
+public class CanalException extends NestableRuntimeException {
+
+    private static final long serialVersionUID = -654893533794556357L;
+
+    public CanalException(String errorCode){
+        super(errorCode);
+    }
+
+    public CanalException(String errorCode, Throwable cause){
+        super(errorCode, cause);
+    }
+
+    public CanalException(String errorCode, String errorDesc){
+        super(errorCode + ":" + errorDesc);
+    }
+
+    public CanalException(String errorCode, String errorDesc, Throwable cause){
+        super(errorCode + ":" + errorDesc, cause);
+    }
+
+    public CanalException(Throwable cause){
+        super(cause);
+    }
+
+    public Throwable fillInStackTrace() {
+        return this;
+    }
+
+}

common/src/main/java/com/alibaba/otter/canal/common/CanalLifeCycle.java

@@ -0,0 +1,14 @@
+package com.alibaba.otter.canal.common;
+
+/**
+ * @author jianghang 2012-7-12 上午09:39:33
+ * @version 1.0.0
+ */
+public interface CanalLifeCycle {
+
+    public void start();
+
+    public void stop();
+
+    public boolean isStart();
+}

common/src/main/java/com/alibaba/otter/canal/common/alarm/CanalAlarmHandler.java

@@ -0,0 +1,21 @@
+package com.alibaba.otter.canal.common.alarm;
+
+import com.alibaba.otter.canal.common.CanalLifeCycle;
+
+/**
+ * canal报警处理机制
+ * 
+ * @author jianghang 2012-8-22 下午10:08:56
+ * @version 1.0.0
+ */
+public interface CanalAlarmHandler extends CanalLifeCycle {
+
+    /**
+     * 发送对应destination的报警
+     * 
+     * @param destination
+     * @param title
+     * @param msg
+     */
+    public void sendAlarm(String destination, String msg);
+}

common/src/main/java/com/alibaba/otter/canal/common/alarm/LogAlarmHandler.java

@@ -0,0 +1,22 @@
+package com.alibaba.otter.canal.common.alarm;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+
+/**
+ * 基于log的alarm机制实现
+ * 
+ * @author jianghang 2012-8-22 下午10:12:35
+ * @version 1.0.0
+ */
+public class LogAlarmHandler extends AbstractCanalLifeCycle implements CanalAlarmHandler {
+
+    private static final Logger logger = LoggerFactory.getLogger(LogAlarmHandler.class);
+
+    public void sendAlarm(String destination, String msg) {
+        logger.error("destination:{}[{}]", new Object[] { destination, msg });
+    }
+
+}

common/src/main/java/com/alibaba/otter/canal/common/utils/AddressUtils.java

@@ -0,0 +1,95 @@
+package com.alibaba.otter.canal.common.utils;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.NetworkInterface;
+import java.net.ServerSocket;
+import java.util.Enumeration;
+import java.util.regex.Pattern;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class AddressUtils {
+
+    private static final Logger  logger       = LoggerFactory.getLogger(AddressUtils.class);
+    private static final String  LOCALHOST_IP = "127.0.0.1";
+    private static final String  EMPTY_IP     = "0.0.0.0";
+    private static final Pattern IP_PATTERN   = Pattern.compile("[0-9]{1,3}(\\.[0-9]{1,3}){3,}");
+
+    public static boolean isAvailablePort(int port) {
+        ServerSocket ss = null;
+        try {
+            ss = new ServerSocket(port);
+            ss.bind(null);
+            return true;
+        } catch (IOException e) {
+            return false;
+        } finally {
+            if (ss != null) {
+                try {
+                    ss.close();
+                } catch (IOException e) {
+                }
+            }
+        }
+    }
+
+    private static boolean isValidHostAddress(InetAddress address) {
+        if (address == null || address.isLoopbackAddress()) return false;
+        String name = address.getHostAddress();
+        return (name != null && !EMPTY_IP.equals(name) && !LOCALHOST_IP.equals(name) && IP_PATTERN.matcher(name).matches());
+    }
+
+    public static String getHostIp() {
+        InetAddress address = getHostAddress();
+        return address == null ? null : address.getHostAddress();
+    }
+
+    public static String getHostName() {
+        InetAddress address = getHostAddress();
+        return address == null ? null : address.getHostName();
+    }
+
+    public static InetAddress getHostAddress() {
+        InetAddress localAddress = null;
+        try {
+            localAddress = InetAddress.getLocalHost();
+            if (isValidHostAddress(localAddress)) {
+                return localAddress;
+            }
+        } catch (Throwable e) {
+            logger.warn("Failed to retriving local host ip address, try scan network card ip address. cause: "
+                        + e.getMessage());
+        }
+        try {
+            Enumeration<NetworkInterface> interfaces = NetworkInterface.getNetworkInterfaces();
+            if (interfaces != null) {
+                while (interfaces.hasMoreElements()) {
+                    try {
+                        NetworkInterface network = interfaces.nextElement();
+                        Enumeration<InetAddress> addresses = network.getInetAddresses();
+                        if (addresses != null) {
+                            while (addresses.hasMoreElements()) {
+                                try {
+                                    InetAddress address = addresses.nextElement();
+                                    if (isValidHostAddress(address)) {
+                                        return address;
+                                    }
+                                } catch (Throwable e) {
+                                    logger.warn("Failed to retriving network card ip address. cause:" + e.getMessage());
+                                }
+                            }
+                        }
+                    } catch (Throwable e) {
+                        logger.warn("Failed to retriving network card ip address. cause:" + e.getMessage());
+                    }
+                }
+            }
+        } catch (Throwable e) {
+            logger.warn("Failed to retriving network card ip address. cause:" + e.getMessage());
+        }
+        logger.error("Could not get local host ip address, will use 127.0.0.1 instead.");
+        return localAddress;
+    }
+}

common/src/main/java/com/alibaba/otter/canal/common/utils/BooleanMutex.java

@@ -0,0 +1,157 @@
+package com.alibaba.otter.canal.common.utils;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.locks.AbstractQueuedSynchronizer;
+
+/**
+ * 实现一个互斥实现，基于Cocurrent中的{@linkplain AbstractQueuedSynchronizer}实现了自己的sync <br/>
+ * 应用场景：系统初始化/授权控制，没权限时阻塞等待。有权限时所有线程都可以快速通过
+ * 
+ * <pre>
+ * false : 代表需要被阻塞挂起，等待mutex变为true被唤醒
+ * true : 唤醒被阻塞在false状态下的thread
+ * 
+ * BooleanMutex mutex = new BooleanMutex(true);
+ * try {
+ *     mutex.get(); //当前状态为true, 不会被阻塞
+ * } catch (InterruptedException e) {
+ *     // do something
+ * }
+ * 
+ * mutex.set(false);
+ * try {
+ *     mutex.get(); //当前状态为false, 会被阻塞直到另一个线程调用mutex.set(true);
+ * } catch (InterruptedException e) {
+ *     // do something
+ * }
+ * </pre>
+ * 
+ * @author jianghang 2011-9-23 上午09:58:03
+ * @version 1.0.0
+ */
+public class BooleanMutex {
+
+    private Sync sync;
+
+    public BooleanMutex(){
+        sync = new Sync();
+        set(false);
+    }
+
+    public BooleanMutex(Boolean mutex){
+        sync = new Sync();
+        set(mutex);
+    }
+
+    /**
+     * 阻塞等待Boolean为true
+     * 
+     * @throws InterruptedException
+     */
+    public void get() throws InterruptedException {
+        sync.innerGet();
+    }
+
+    /**
+     * 阻塞等待Boolean为true,允许设置超时时间
+     * 
+     * @param timeout
+     * @param unit
+     * @throws InterruptedException
+     * @throws TimeoutException
+     */
+    public void get(long timeout, TimeUnit unit) throws InterruptedException, TimeoutException {
+        sync.innerGet(unit.toNanos(timeout));
+    }
+
+    /**
+     * 重新设置对应的Boolean mutex
+     * 
+     * @param mutex
+     */
+    public void set(Boolean mutex) {
+        if (mutex) {
+            sync.innerSetTrue();
+        } else {
+            sync.innerSetFalse();
+        }
+    }
+
+    public boolean state() {
+        return sync.innerState();
+    }
+
+    /**
+     * Synchronization control for BooleanMutex. Uses AQS sync state to
+     * represent run status
+     */
+    private final class Sync extends AbstractQueuedSynchronizer {
+
+        private static final long serialVersionUID = 2559471934544126329L;
+        /** State value representing that TRUE */
+        private static final int  TRUE             = 1;
+        /** State value representing that FALSE */
+        private static final int  FALSE            = 2;
+
+        private boolean isTrue(int state) {
+            return (state & TRUE) != 0;
+        }
+
+        /**
+         * 实现AQS的接口，获取共享锁的判断
+         */
+        protected int tryAcquireShared(int state) {
+            // 如果为true，直接允许获取锁对象
+            // 如果为false，进入阻塞队列，等待被唤醒
+            return isTrue(getState()) ? 1 : -1;
+        }
+
+        /**
+         * 实现AQS的接口，释放共享锁的判断
+         */
+        protected boolean tryReleaseShared(int ignore) {
+            // 始终返回true，代表可以release
+            return true;
+        }
+
+        boolean innerState() {
+            return isTrue(getState());
+        }
+
+        void innerGet() throws InterruptedException {
+            acquireSharedInterruptibly(0);
+        }
+
+        void innerGet(long nanosTimeout) throws InterruptedException, TimeoutException {
+            if (!tryAcquireSharedNanos(0, nanosTimeout)) throw new TimeoutException();
+        }
+
+        void innerSetTrue() {
+            for (;;) {
+                int s = getState();
+                if (s == TRUE) {
+                    return; // 直接退出
+                }
+                if (compareAndSetState(s, TRUE)) {// cas更新状态，避免并发更新true操作
+                    releaseShared(0);// 释放一下锁对象，唤醒一下阻塞的Thread
+                    return;
+                }
+            }
+        }
+
+        void innerSetFalse() {
+            for (;;) {
+                int s = getState();
+                if (s == FALSE) {
+                    return; // 直接退出
+                }
+                if (compareAndSetState(s, FALSE)) {// cas更新状态，避免并发更新false操作
+                    return;
+                }
+            }
+        }
+
+    }
+
+}

common/src/main/java/com/alibaba/otter/canal/common/utils/CanalToStringStyle.java

@@ -0,0 +1,80 @@
+package com.alibaba.otter.canal.common.utils;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+import org.apache.commons.lang.builder.ToStringStyle;
+
+/**
+ * Otter项目内部使用的ToStringStyle
+ * 
+ * <pre>
+ * 默认Style输出格式：
+ * Person[name=John Doe,age=33,smoker=false ,time=2010-04-01 00:00:00]
+ * </pre>
+ * 
+ * @author jianghang 2010-6-18 上午11:35:27
+ */
+public class CanalToStringStyle extends ToStringStyle {
+
+    private static final long         serialVersionUID = -6568177374288222145L;
+
+    private static final String       DEFAULT_TIME     = "yyyy-MM-dd HH:mm:ss";
+    private static final String       DEFAULT_DAY      = "yyyy-MM-dd";
+
+    /**
+     * <pre>
+     * 输出格式：
+     * Person[name=John Doe,age=33,smoker=false ,time=2010-04-01 00:00:00]
+     * </pre>
+     */
+    public static final ToStringStyle TIME_STYLE       = new OtterDateStyle(DEFAULT_TIME);
+
+    /**
+     * <pre>
+     * 输出格式：
+     * Person[name=John Doe,age=33,smoker=false ,day=2010-04-01]
+     * </pre>
+     */
+    public static final ToStringStyle DAY_STYLE        = new OtterDateStyle(DEFAULT_DAY);
+
+    /**
+     * <pre>
+     * 输出格式：
+     * Person[name=John Doe,age=33,smoker=false ,time=2010-04-01 00:00:00]
+     * </pre>
+     */
+    public static final ToStringStyle DEFAULT_STYLE    = CanalToStringStyle.TIME_STYLE;
+
+    // =========================== 自定义style =============================
+
+    /**
+     * 支持日期格式化的ToStringStyle
+     * 
+     * @author li.jinl
+     */
+    private static class OtterDateStyle extends ToStringStyle {
+
+        private static final long serialVersionUID = 5208917932254652886L;
+
+        // 日期format格式
+        private String            pattern;
+
+        public OtterDateStyle(String pattern){
+            super();
+            this.setUseShortClassName(true);
+            this.setUseIdentityHashCode(false);
+            // 设置日期format格式
+            this.pattern = pattern;
+        }
+
+        protected void appendDetail(StringBuffer buffer, String fieldName, Object value) {
+            // 增加自定义的date对象处理
+            if (value instanceof Date) {
+                value = new SimpleDateFormat(pattern).format(value);
+            }
+            // 后续可以增加其他自定义对象处理
+            buffer.append(value);
+        }
+    }
+}

common/src/main/java/com/alibaba/otter/canal/common/utils/JsonUtils.java

@@ -0,0 +1,73 @@
+package com.alibaba.otter.canal.common.utils;
+
+import java.util.Arrays;
+import java.util.List;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.TypeReference;
+import com.alibaba.fastjson.serializer.JSONSerializer;
+import com.alibaba.fastjson.serializer.PropertyFilter;
+import com.alibaba.fastjson.serializer.SerializeWriter;
+import com.alibaba.fastjson.serializer.SerializerFeature;
+
+/**
+ * 字节处理相关工具类
+ * 
+ * @author jianghang
+ */
+public class JsonUtils {
+
+    public static <T> T unmarshalFromByte(byte[] bytes, Class<T> targetClass) {
+        return (T) JSON.parseObject(bytes, targetClass);// 默认为UTF-8
+    }
+
+    public static <T> T unmarshalFromByte(byte[] bytes, TypeReference<T> type) {
+        return (T) JSON.parseObject(bytes, type.getType());
+    }
+
+    public static byte[] marshalToByte(Object obj) {
+        return JSON.toJSONBytes(obj); // 默认为UTF-8
+    }
+
+    public static byte[] marshalToByte(Object obj, SerializerFeature... features) {
+        return JSON.toJSONBytes(obj, features); // 默认为UTF-8
+    }
+
+    public static <T> T unmarshalFromString(String json, Class<T> targetClass) {
+        return (T) JSON.parseObject(json, targetClass);// 默认为UTF-8
+    }
+
+    public static <T> T unmarshalFromString(String json, TypeReference<T> type) {
+        return (T) JSON.parseObject(json, type);// 默认为UTF-8
+    }
+
+    public static String marshalToString(Object obj) {
+        return JSON.toJSONString(obj); // 默认为UTF-8
+    }
+
+    public static String marshalToString(Object obj, SerializerFeature... features) {
+        return JSON.toJSONString(obj, features); // 默认为UTF-8
+    }
+
+    /**
+     * 可以允许指定一些过滤字段进行生成json对象
+     */
+    public static String marshalToString(Object obj, String... fliterFields) {
+        final List<String> propertyFliters = Arrays.asList(fliterFields);
+        SerializeWriter out = new SerializeWriter();
+        try {
+            JSONSerializer serializer = new JSONSerializer(out);
+            serializer.getPropertyFilters().add(new PropertyFilter() {
+
+                public boolean apply(Object source, String name, Object value) {
+                    return !propertyFliters.contains(name);
+                }
+
+            });
+            serializer.write(obj);
+            return out.toString();
+        } finally {
+            out.close();
+        }
+    }
+}

common/src/main/java/com/alibaba/otter/canal/common/utils/NamedThreadFactory.java

@@ -0,0 +1,56 @@
+package com.alibaba.otter.canal.common.utils;
+
+import java.lang.Thread.UncaughtExceptionHandler;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author zebin.xuzb 2012-9-20 下午3:47:47
+ * @version 1.0.0
+ */
+public class NamedThreadFactory implements ThreadFactory {
+
+    private static final Logger           logger                   = LoggerFactory.getLogger(NamedThreadFactory.class);
+    final private static String           DEFAULT_NAME             = "canal-worker";
+    final private String                  name;
+    final private boolean                 daemon;
+    final private ThreadGroup             group;
+    final private AtomicInteger           threadNumber             = new AtomicInteger(0);
+    final static UncaughtExceptionHandler uncaughtExceptionHandler = new UncaughtExceptionHandler() {
+
+                                                                       public void uncaughtException(Thread t,
+                                                                                                     Throwable e) {
+                                                                           logger.error("from " + t.getName(), e);
+                                                                       }
+                                                                   };
+
+    public NamedThreadFactory(){
+        this(DEFAULT_NAME, true);
+    }
+
+    public NamedThreadFactory(String name){
+        this(name, true);
+    }
+
+    public NamedThreadFactory(String name, boolean daemon){
+        this.name = name;
+        this.daemon = daemon;
+        SecurityManager s = System.getSecurityManager();
+        group = (s != null) ? s.getThreadGroup() : Thread.currentThread().getThreadGroup();
+    }
+
+    public Thread newThread(Runnable r) {
+        Thread t = new Thread(group, r, name + "-" + threadNumber.getAndIncrement(), 0);
+        t.setDaemon(daemon);
+        if (t.getPriority() != Thread.NORM_PRIORITY) {
+            t.setPriority(Thread.NORM_PRIORITY);
+        }
+
+        t.setUncaughtExceptionHandler(uncaughtExceptionHandler);
+        return t;
+    }
+
+}

common/src/main/java/com/alibaba/otter/canal/common/utils/UriUtils.java

@@ -0,0 +1,79 @@
+package com.alibaba.otter.canal.common.utils;
+
+import java.io.UnsupportedEncodingException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URLDecoder;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Scanner;
+
+import org.apache.commons.lang.StringUtils;
+
+/**
+ * @author zebin.xuzb 2012-11-15 下午3:53:09
+ * @since 1.0.0
+ */
+public final class UriUtils {
+
+    private final static String SPLIT            = "&";
+    private final static String EQUAL            = "=";
+    private final static String DEFAULT_ENCODING = "ISO_8859_1";
+
+    private UriUtils(){
+    }
+
+    public static Map<String, String> parseQuery(final String uriString) {
+        URI uri = null;
+        try {
+            uri = new URI(uriString);
+        } catch (URISyntaxException e) {
+            throw new IllegalArgumentException(e);
+        }
+        return parseQuery(uri);
+    }
+
+    public static Map<String, String> parseQuery(final String uriString, final String encoding) {
+        URI uri = null;
+        try {
+            uri = new URI(uriString);
+        } catch (URISyntaxException e) {
+            throw new IllegalArgumentException(e);
+        }
+        return parseQuery(uri, encoding);
+    }
+
+    public static Map<String, String> parseQuery(final URI uri) {
+        return parseQuery(uri, DEFAULT_ENCODING);
+    }
+
+    public static Map<String, String> parseQuery(final URI uri, final String encoding) {
+        if (uri == null || StringUtils.isBlank(uri.getQuery())) {
+            return Collections.EMPTY_MAP;
+        }
+        String query = uri.getRawQuery();
+        HashMap<String, String> params = new HashMap<String, String>();
+        Scanner scan = new Scanner(query);
+        scan.useDelimiter(SPLIT);
+        while (scan.hasNext()) {
+            String token = scan.next().trim();
+            String[] pair = token.split(EQUAL);
+            String key = decode(pair[0], encoding);
+            String value = null;
+            if (pair.length == 2) {
+                value = decode(pair[1], encoding);
+            }
+            params.put(key, value);
+        }
+        return params;
+    }
+
+    private static String decode(final String content, final String encoding) {
+        try {
+            return URLDecoder.decode(content, encoding != null ? encoding : DEFAULT_ENCODING);
+        } catch (UnsupportedEncodingException e) {
+            throw new IllegalArgumentException(e);
+        }
+    }
+}

common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ByteSerializer.java

@@ -0,0 +1,32 @@
+package com.alibaba.otter.canal.common.zookeeper;
+
+import java.io.UnsupportedEncodingException;
+
+import org.I0Itec.zkclient.exception.ZkMarshallingError;
+import org.I0Itec.zkclient.serialize.ZkSerializer;
+
+/**
+ * 基于string的序列化方式
+ * 
+ * @author jianghang 2012-7-11 下午02:57:09
+ * @version 1.0.0
+ */
+public class ByteSerializer implements ZkSerializer {
+
+    public Object deserialize(final byte[] bytes) throws ZkMarshallingError {
+        return bytes;
+    }
+
+    public byte[] serialize(final Object data) throws ZkMarshallingError {
+        try {
+            if (data instanceof byte[]) {
+                return (byte[]) data;
+            } else {
+                return ((String) data).getBytes("utf-8");
+            }
+        } catch (final UnsupportedEncodingException e) {
+            throw new ZkMarshallingError(e);
+        }
+    }
+
+}

common/src/main/java/com/alibaba/otter/canal/common/zookeeper/StringSerializer.java

@@ -0,0 +1,32 @@
+package com.alibaba.otter.canal.common.zookeeper;
+
+import java.io.UnsupportedEncodingException;
+
+import org.I0Itec.zkclient.exception.ZkMarshallingError;
+import org.I0Itec.zkclient.serialize.ZkSerializer;
+
+/**
+ * 基于string的序列化方式
+ * 
+ * @author jianghang 2012-7-11 下午02:57:09
+ * @version 1.0.0
+ */
+public class StringSerializer implements ZkSerializer {
+
+    public Object deserialize(final byte[] bytes) throws ZkMarshallingError {
+        try {
+            return new String(bytes, "utf-8");
+        } catch (final UnsupportedEncodingException e) {
+            throw new ZkMarshallingError(e);
+        }
+    }
+
+    public byte[] serialize(final Object data) throws ZkMarshallingError {
+        try {
+            return ((String) data).getBytes("utf-8");
+        } catch (final UnsupportedEncodingException e) {
+            throw new ZkMarshallingError(e);
+        }
+    }
+
+}

common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ZkClientx.java

@@ -0,0 +1,146 @@
+package com.alibaba.otter.canal.common.zookeeper;
+
+import java.util.Map;
+
+import org.I0Itec.zkclient.IZkConnection;
+import org.I0Itec.zkclient.ZkClient;
+import org.I0Itec.zkclient.exception.ZkException;
+import org.I0Itec.zkclient.exception.ZkInterruptedException;
+import org.I0Itec.zkclient.exception.ZkNoNodeException;
+import org.I0Itec.zkclient.exception.ZkNodeExistsException;
+import org.I0Itec.zkclient.serialize.ZkSerializer;
+import org.apache.zookeeper.CreateMode;
+
+import com.google.common.base.Function;
+import com.google.common.collect.MapMaker;
+
+/**
+ * 使用自定义的ZooKeeperx for zk connection
+ * 
+ * @author jianghang 2012-7-10 下午02:31:15
+ * @version 1.0.0
+ */
+public class ZkClientx extends ZkClient {
+
+    // 对于zkclient进行一次缓存，避免一个jvm内部使用多个zk connection
+    private static Map<String, ZkClientx> clients = new MapMaker().makeComputingMap(new Function<String, ZkClientx>() {
+
+                                                      public ZkClientx apply(String servers) {
+                                                          return new ZkClientx(servers);
+                                                      }
+                                                  });
+
+    public static ZkClientx getZkClient(String servers) {
+        return clients.get(servers);
+    }
+
+    public ZkClientx(String serverstring){
+        this(serverstring, Integer.MAX_VALUE);
+    }
+
+    public ZkClientx(String zkServers, int connectionTimeout){
+        this(new ZooKeeperx(zkServers), connectionTimeout);
+    }
+
+    public ZkClientx(String zkServers, int sessionTimeout, int connectionTimeout){
+        this(new ZooKeeperx(zkServers, sessionTimeout), connectionTimeout);
+    }
+
+    public ZkClientx(String zkServers, int sessionTimeout, int connectionTimeout, ZkSerializer zkSerializer){
+        this(new ZooKeeperx(zkServers, sessionTimeout), connectionTimeout, zkSerializer);
+    }
+
+    private ZkClientx(IZkConnection connection, int connectionTimeout){
+        this(connection, connectionTimeout, new ByteSerializer());
+    }
+
+    private ZkClientx(IZkConnection zkConnection, int connectionTimeout, ZkSerializer zkSerializer){
+        super(zkConnection, connectionTimeout, zkSerializer);
+    }
+
+    /**
+     * Create a persistent Sequential node.
+     * 
+     * @param path
+     * @param createParents if true all parent dirs are created as well and no {@link ZkNodeExistsException} is thrown
+     * in case the path already exists
+     * @throws ZkInterruptedException if operation was interrupted, or a required reconnection got interrupted
+     * @throws IllegalArgumentException if called from anything except the ZooKeeper event thread
+     * @throws ZkException if any ZooKeeper exception occurred
+     * @throws RuntimeException if any other exception occurs
+     */
+    public String createPersistentSequential(String path, boolean createParents) throws ZkInterruptedException,
+                                                                                IllegalArgumentException, ZkException,
+                                                                                RuntimeException {
+        try {
+            return create(path, null, CreateMode.PERSISTENT_SEQUENTIAL);
+        } catch (ZkNoNodeException e) {
+            if (!createParents) {
+                throw e;
+            }
+            String parentDir = path.substring(0, path.lastIndexOf('/'));
+            createPersistent(parentDir, createParents);
+            return createPersistentSequential(path, createParents);
+        }
+    }
+
+    /**
+     * Create a persistent Sequential node.
+     * 
+     * @param path
+     * @param data
+     * @param createParents if true all parent dirs are created as well and no {@link ZkNodeExistsException} is thrown
+     * in case the path already exists
+     * @throws ZkInterruptedException if operation was interrupted, or a required reconnection got interrupted
+     * @throws IllegalArgumentException if called from anything except the ZooKeeper event thread
+     * @throws ZkException if any ZooKeeper exception occurred
+     * @throws RuntimeException if any other exception occurs
+     */
+    public String createPersistentSequential(String path, Object data, boolean createParents)
+                                                                                             throws ZkInterruptedException,
+                                                                                             IllegalArgumentException,
+                                                                                             ZkException,
+                                                                                             RuntimeException {
+        try {
+            return create(path, data, CreateMode.PERSISTENT_SEQUENTIAL);
+        } catch (ZkNoNodeException e) {
+            if (!createParents) {
+                throw e;
+            }
+            String parentDir = path.substring(0, path.lastIndexOf('/'));
+            createPersistent(parentDir, createParents);
+            return createPersistentSequential(path, data, createParents);
+        }
+    }
+
+    /**
+     * Create a persistent Sequential node.
+     * 
+     * @param path
+     * @param data
+     * @param createParents if true all parent dirs are created as well and no {@link ZkNodeExistsException} is thrown
+     * in case the path already exists
+     * @throws ZkInterruptedException if operation was interrupted, or a required reconnection got interrupted
+     * @throws IllegalArgumentException if called from anything except the ZooKeeper event thread
+     * @throws ZkException if any ZooKeeper exception occurred
+     * @throws RuntimeException if any other exception occurs
+     */
+    public void createPersistent(String path, Object data, boolean createParents) throws ZkInterruptedException,
+                                                                                 IllegalArgumentException, ZkException,
+                                                                                 RuntimeException {
+        try {
+            create(path, data, CreateMode.PERSISTENT);
+        } catch (ZkNodeExistsException e) {
+            if (!createParents) {
+                throw e;
+            }
+        } catch (ZkNoNodeException e) {
+            if (!createParents) {
+                throw e;
+            }
+            String parentDir = path.substring(0, path.lastIndexOf('/'));
+            createPersistent(parentDir, createParents);
+            createPersistent(path, data, createParents);
+        }
+    }
+}

common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ZooKeeperx.java

@@ -0,0 +1,181 @@
+package com.alibaba.otter.canal.common.zookeeper;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.net.InetSocketAddress;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.I0Itec.zkclient.IZkConnection;
+import org.I0Itec.zkclient.exception.ZkException;
+import org.apache.commons.lang.StringUtils;
+import org.apache.zookeeper.ClientCnxn;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.ZooDefs.Ids;
+import org.apache.zookeeper.ZooKeeper.States;
+import org.apache.zookeeper.client.ConnectStringParser;
+import org.apache.zookeeper.client.HostProvider;
+import org.apache.zookeeper.client.StaticHostProvider;
+import org.apache.zookeeper.data.Stat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.util.ReflectionUtils;
+
+/**
+ * 封装了ZooKeeper，使其支持节点的优先顺序，比如美国机房的节点会优先加载美国对应的zk集群列表，都失败后才会选择加载杭州的zk集群列表 *
+ * 
+ * @author jianghang 2012-7-10 下午02:31:42
+ * @version 1.0.0
+ */
+public class ZooKeeperx implements IZkConnection {
+
+    private static final String SERVER_COMMA            = ";";
+    private static final Logger logger                  = LoggerFactory.getLogger(ZooKeeperx.class);
+    private static final Field  clientCnxnField         = ReflectionUtils.findField(ZooKeeper.class, "cnxn");
+    private static final Field  hostProviderField       = ReflectionUtils.findField(ClientCnxn.class, "hostProvider");
+    private static final Field  serverAddressesField    = ReflectionUtils.findField(StaticHostProvider.class,
+                                                                                    "serverAddresses");
+    private static final int    DEFAULT_SESSION_TIMEOUT = 90000;
+
+    private ZooKeeper           _zk                     = null;
+    private Lock                _zookeeperLock          = new ReentrantLock();
+
+    private final List<String>  _servers;
+    private final int           _sessionTimeOut;
+
+    public ZooKeeperx(String zkServers){
+        this(zkServers, DEFAULT_SESSION_TIMEOUT);
+    }
+
+    public ZooKeeperx(String zkServers, int sessionTimeOut){
+        _servers = Arrays.asList(StringUtils.split(zkServers, SERVER_COMMA));
+        _sessionTimeOut = sessionTimeOut;
+    }
+
+    @Override
+    public void connect(Watcher watcher) {
+        _zookeeperLock.lock();
+        try {
+            if (_zk != null) {
+                throw new IllegalStateException("zk client has already been started");
+            }
+
+            try {
+                logger.debug("Creating new ZookKeeper instance to connect to " + _servers + ".");
+                _zk = new ZooKeeper(_servers.get(0), _sessionTimeOut, watcher);
+                configMutliCluster(_zk);
+            } catch (IOException e) {
+                throw new ZkException("Unable to connect to " + _servers, e);
+            }
+        } finally {
+            _zookeeperLock.unlock();
+        }
+    }
+
+    public void close() throws InterruptedException {
+        _zookeeperLock.lock();
+        try {
+            if (_zk != null) {
+                logger.debug("Closing ZooKeeper connected to " + _servers);
+                _zk.close();
+                _zk = null;
+            }
+        } finally {
+            _zookeeperLock.unlock();
+        }
+    }
+
+    public String create(String path, byte[] data, CreateMode mode) throws KeeperException, InterruptedException {
+        return _zk.create(path, data, Ids.OPEN_ACL_UNSAFE, mode);
+    }
+
+    public void delete(String path) throws InterruptedException, KeeperException {
+        _zk.delete(path, -1);
+    }
+
+    public boolean exists(String path, boolean watch) throws KeeperException, InterruptedException {
+        return _zk.exists(path, watch) != null;
+    }
+
+    public List<String> getChildren(final String path, final boolean watch) throws KeeperException,
+                                                                           InterruptedException {
+        return _zk.getChildren(path, watch);
+    }
+
+    public byte[] readData(String path, Stat stat, boolean watch) throws KeeperException, InterruptedException {
+        return _zk.getData(path, watch, stat);
+    }
+
+    public void writeData(String path, byte[] data) throws KeeperException, InterruptedException {
+        writeData(path, data, -1);
+    }
+
+    public void writeData(String path, byte[] data, int version) throws KeeperException, InterruptedException {
+        _zk.setData(path, data, version);
+    }
+
+    public States getZookeeperState() {
+        return _zk != null ? _zk.getState() : null;
+    }
+
+    public ZooKeeper getZookeeper() {
+        return _zk;
+    }
+
+    public long getCreateTime(String path) throws KeeperException, InterruptedException {
+        Stat stat = _zk.exists(path, false);
+        if (stat != null) {
+            return stat.getCtime();
+        }
+        return -1;
+    }
+
+    public String getServers() {
+        return StringUtils.join(_servers, SERVER_COMMA);
+    }
+
+    // ===============================
+
+    public void configMutliCluster(ZooKeeper zk) {
+        if (_servers.size() == 1) {
+            return;
+        }
+        String cluster1 = _servers.get(0);
+        try {
+            if (_servers.size() > 1) {
+                // 强制的声明accessible
+                ReflectionUtils.makeAccessible(clientCnxnField);
+                ReflectionUtils.makeAccessible(hostProviderField);
+                ReflectionUtils.makeAccessible(serverAddressesField);
+
+                // 添加第二组集群列表
+                for (int i = 1; i < _servers.size(); i++) {
+                    String cluster = _servers.get(i);
+                    // 强制获取zk中的地址信息
+                    ClientCnxn cnxn = (ClientCnxn) ReflectionUtils.getField(clientCnxnField, zk);
+                    HostProvider hostProvider = (HostProvider) ReflectionUtils.getField(hostProviderField, cnxn);
+                    List<InetSocketAddress> serverAddrs = (List<InetSocketAddress>) ReflectionUtils.getField(
+                                                                                                             serverAddressesField,
+                                                                                                             hostProvider);
+                    // 添加第二组集群列表
+                    serverAddrs.addAll(new ConnectStringParser(cluster).getServerAddresses());
+                }
+            }
+        } catch (Exception e) {
+            try {
+                if (zk != null) {
+                    zk.close();
+                }
+            } catch (InterruptedException ie) {
+                // ignore interrupt
+            }
+            throw new ZkException("zookeeper_create_error, serveraddrs=" + cluster1, e);
+        }
+
+    }
+}

common/src/main/java/com/alibaba/otter/canal/common/zookeeper/ZookeeperPathUtils.java

@@ -0,0 +1,181 @@
+package com.alibaba.otter.canal.common.zookeeper;
+
+import java.text.MessageFormat;
+
+import org.apache.commons.lang.StringUtils;
+
+/**
+ * 存储结构：
+ * 
+ * <pre>
+ * /otter
+ *    canal
+ *      cluster
+ *      destinations
+ *        dest1
+ *          running (EPHEMERAL) 
+ *          cluster
+ *          client1
+ *            running (EPHEMERAL)
+ *            cluster
+ *            filter
+ *            cursor
+ *            mark
+ *              1
+ *              2
+ *              3
+ * </pre>
+ * 
+ * @author zebin.xuzb @ 2012-6-21
+ * @version 1.0.0
+ */
+public class ZookeeperPathUtils {
+
+    public static final String ZOOKEEPER_SEPARATOR                          = "/";
+
+    public static final String OTTER_ROOT_NODE                              = ZOOKEEPER_SEPARATOR + "otter";
+
+    public static final String CANAL_ROOT_NODE                              = OTTER_ROOT_NODE + ZOOKEEPER_SEPARATOR
+                                                                              + "canal";
+
+    public static final String DESTINATION_ROOT_NODE                        = CANAL_ROOT_NODE + ZOOKEEPER_SEPARATOR
+                                                                              + "destinations";
+
+    public static final String FILTER_NODE                                  = "filter";
+
+    public static final String BATCH_MARK_NODE                              = "mark";
+
+    public static final String PARSE_NODE                                   = "parse";
+
+    public static final String CURSOR_NODE                                  = "cursor";
+
+    public static final String RUNNING_NODE                                 = "running";
+
+    public static final String CLUSTER_NODE                                 = "cluster";
+
+    public static final String DESTINATION_NODE                             = DESTINATION_ROOT_NODE
+                                                                              + ZOOKEEPER_SEPARATOR + "{0}";
+
+    public static final String DESTINATION_PARSE_NODE                       = DESTINATION_NODE + ZOOKEEPER_SEPARATOR
+                                                                              + PARSE_NODE;
+
+    public static final String DESTINATION_CLIENTID_NODE                    = DESTINATION_NODE + ZOOKEEPER_SEPARATOR
+                                                                              + "{1}";
+
+    public static final String DESTINATION_CURSOR_NODE                      = DESTINATION_CLIENTID_NODE
+                                                                              + ZOOKEEPER_SEPARATOR + CURSOR_NODE;
+
+    public static final String DESTINATION_CLIENTID_FILTER_NODE             = DESTINATION_CLIENTID_NODE
+                                                                              + ZOOKEEPER_SEPARATOR + FILTER_NODE;
+
+    public static final String DESTINATION_CLIENTID_BATCH_MARK_NODE         = DESTINATION_CLIENTID_NODE
+                                                                              + ZOOKEEPER_SEPARATOR + BATCH_MARK_NODE;
+
+    public static final String DESTINATION_CLIENTID_BATCH_MARK_WITH_ID_PATH = DESTINATION_CLIENTID_BATCH_MARK_NODE
+                                                                              + ZOOKEEPER_SEPARATOR + "{2}";
+
+    /**
+     * 服务端当前正在提供服务的running节点
+     */
+    public static final String DESTINATION_RUNNING_NODE                     = DESTINATION_NODE + ZOOKEEPER_SEPARATOR
+                                                                              + RUNNING_NODE;
+
+    /**
+     * 客户端当前正在工作的running节点
+     */
+    public static final String DESTINATION_CLIENTID_RUNNING_NODE            = DESTINATION_CLIENTID_NODE
+                                                                              + ZOOKEEPER_SEPARATOR + RUNNING_NODE;
+
+    /**
+     * 整个canal server的集群列表
+     */
+    public static final String CANAL_CLUSTER_ROOT_NODE                      = CANAL_ROOT_NODE + ZOOKEEPER_SEPARATOR
+                                                                              + CLUSTER_NODE;
+
+    public static final String CANAL_CLUSTER_NODE                           = CANAL_CLUSTER_ROOT_NODE
+                                                                              + ZOOKEEPER_SEPARATOR + "{0}";
+
+    /**
+     * 针对某个destination的工作的集群列表
+     */
+    public static final String DESTINATION_CLUSTER_ROOT                     = DESTINATION_NODE + ZOOKEEPER_SEPARATOR
+                                                                              + CLUSTER_NODE;
+    public static final String DESTINATION_CLUSTER_NODE                     = DESTINATION_CLUSTER_ROOT
+                                                                              + ZOOKEEPER_SEPARATOR + "{1}";
+
+    public static String getDestinationPath(String destinationName) {
+        return MessageFormat.format(DESTINATION_NODE, destinationName);
+    }
+
+    public static String getClientIdNodePath(String destinationName, short clientId) {
+        return MessageFormat.format(DESTINATION_CLIENTID_NODE, destinationName, String.valueOf(clientId));
+    }
+
+    public static String getFilterPath(String destinationName, short clientId) {
+        return MessageFormat.format(DESTINATION_CLIENTID_FILTER_NODE, destinationName, String.valueOf(clientId));
+    }
+
+    public static String getBatchMarkPath(String destinationName, short clientId) {
+        return MessageFormat.format(DESTINATION_CLIENTID_BATCH_MARK_NODE, destinationName, String.valueOf(clientId));
+    }
+
+    public static String getBatchMarkWithIdPath(String destinationName, short clientId, Long batchId) {
+        return MessageFormat.format(DESTINATION_CLIENTID_BATCH_MARK_WITH_ID_PATH, destinationName,
+                                    String.valueOf(clientId), getBatchMarkNode(batchId));
+    }
+
+    public static String getCursorPath(String destination, short clientId) {
+        return MessageFormat.format(DESTINATION_CURSOR_NODE, destination, String.valueOf(clientId));
+    }
+
+    public static String getCanalClusterNode(String node) {
+        return MessageFormat.format(CANAL_CLUSTER_NODE, node);
+    }
+
+    /**
+     * 服务端当前正在提供服务的running节点
+     */
+    public static String getDestinationServerRunning(String destination) {
+        return MessageFormat.format(DESTINATION_RUNNING_NODE, destination);
+    }
+
+    /**
+     * 客户端当前正在工作的running节点
+     */
+    public static String getDestinationClientRunning(String destination, short clientId) {
+        return MessageFormat.format(DESTINATION_CLIENTID_RUNNING_NODE, destination, String.valueOf(clientId));
+    }
+
+    public static String getDestinationClusterNode(String destination, String node) {
+        return MessageFormat.format(DESTINATION_CLUSTER_NODE, destination, node);
+    }
+
+    public static String getDestinationClusterRoot(String destination) {
+        return MessageFormat.format(DESTINATION_CLUSTER_ROOT, destination);
+    }
+
+    public static String getParsePath(String destination) {
+        return MessageFormat.format(DESTINATION_PARSE_NODE, destination);
+    }
+
+    /**
+     * 将batchNode转换为Long
+     */
+    public static short getClientId(String clientNode) {
+        return Short.valueOf(clientNode);
+    }
+
+    /**
+     * 将batchNode转换为Long
+     */
+    public static long getBatchMarkId(String batchMarkNode) {
+        return Long.valueOf(batchMarkNode);
+    }
+
+    /**
+     * 将batchId转化为zookeeper中的node名称
+     */
+    public static String getBatchMarkNode(Long batchId) {
+        return StringUtils.leftPad(String.valueOf(batchId.intValue()), 10, '0');
+    }
+}

common/src/main/java/com/alibaba/otter/canal/common/zookeeper/running/ServerRunningData.java

@@ -0,0 +1,59 @@
+package com.alibaba.otter.canal.common.zookeeper.running;
+
+import java.io.Serializable;
+
+import org.apache.commons.lang.builder.ToStringBuilder;
+
+import com.alibaba.otter.canal.common.utils.CanalToStringStyle;
+
+/**
+ * 服务端running状态信息
+ * 
+ * @author jianghang 2012-11-22 下午03:11:30
+ * @version 1.0.0
+ */
+public class ServerRunningData implements Serializable {
+
+    private static final long serialVersionUID = 92260481691855281L;
+
+    private Long              cid;
+    private String            address;
+    private boolean           active           = true;
+
+    public ServerRunningData(){
+    }
+
+    public ServerRunningData(Long cid, String address){
+        this.cid = cid;
+        this.address = address;
+    }
+
+    public Long getCid() {
+        return cid;
+    }
+
+    public void setCid(Long cid) {
+        this.cid = cid;
+    }
+
+    public String getAddress() {
+        return address;
+    }
+
+    public void setAddress(String address) {
+        this.address = address;
+    }
+
+    public boolean isActive() {
+        return active;
+    }
+
+    public void setActive(boolean active) {
+        this.active = active;
+    }
+
+    public String toString() {
+        return ToStringBuilder.reflectionToString(this, CanalToStringStyle.DEFAULT_STYLE);
+    }
+
+}

common/src/main/java/com/alibaba/otter/canal/common/zookeeper/running/ServerRunningListener.java

@@ -0,0 +1,31 @@
+package com.alibaba.otter.canal.common.zookeeper.running;
+
+/**
+ * 触发一下mainstem发生切换
+ * 
+ * @author jianghang 2012-9-11 下午02:26:03
+ * @version 1.0.0
+ */
+public interface ServerRunningListener {
+
+    /**
+     * 启动时回调做点事情
+     */
+    public void processStart();
+
+    /**
+     * 关闭时回调做点事情
+     */
+    public void processStop();
+
+    /**
+     * 触发现在轮到自己做为active，需要载入上一个active的上下文数据
+     */
+    public void processActiveEnter();
+
+    /**
+     * 触发一下当前active模式失败
+     */
+    public void processActiveExit();
+
+}

common/src/main/java/com/alibaba/otter/canal/common/zookeeper/running/ServerRunningMonitor.java

@@ -0,0 +1,273 @@
+package com.alibaba.otter.canal.common.zookeeper.running;
+
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.I0Itec.zkclient.IZkDataListener;
+import org.I0Itec.zkclient.exception.ZkException;
+import org.I0Itec.zkclient.exception.ZkInterruptedException;
+import org.I0Itec.zkclient.exception.ZkNoNodeException;
+import org.I0Itec.zkclient.exception.ZkNodeExistsException;
+import org.apache.zookeeper.CreateMode;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
+
+import com.alibaba.otter.canal.common.AbstractCanalLifeCycle;
+import com.alibaba.otter.canal.common.utils.BooleanMutex;
+import com.alibaba.otter.canal.common.utils.JsonUtils;
+import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
+import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
+
+/**
+ * 针对server的running节点控制
+ * 
+ * @author jianghang 2012-11-22 下午02:59:42
+ * @version 1.0.0
+ */
+public class ServerRunningMonitor extends AbstractCanalLifeCycle {
+
+    private static final Logger        logger       = LoggerFactory.getLogger(ServerRunningMonitor.class);
+    private ZkClientx                  zkClient;
+    private String                     destination;
+    private IZkDataListener            dataListener;
+    private BooleanMutex               mutex        = new BooleanMutex(false);
+    private volatile boolean           release      = false;
+    // 当前服务节点状态信息
+    private ServerRunningData          serverData;
+    // 当前实际运行的节点状态信息
+    private volatile ServerRunningData activeData;
+    private ScheduledExecutorService   delayExector = Executors.newScheduledThreadPool(1);
+    private int                        delayTime    = 5;
+    private ServerRunningListener      listener;
+
+    public ServerRunningMonitor(ServerRunningData serverData){
+        this();
+        this.serverData = serverData;
+    }
+
+    public ServerRunningMonitor(){
+        // 创建父节点
+        dataListener = new IZkDataListener() {
+
+            public void handleDataChange(String dataPath, Object data) throws Exception {
+                MDC.put("destination", destination);
+                ServerRunningData runningData = JsonUtils.unmarshalFromByte((byte[]) data, ServerRunningData.class);
+                if (!isMine(runningData.getAddress())) {
+                    mutex.set(false);
+                }
+
+                if (!runningData.isActive() && isMine(runningData.getAddress())) { // 说明出现了主动释放的操作，并且本机之前是active
+                    release = true;
+                    releaseRunning();// 彻底释放mainstem
+                }
+
+                activeData = (ServerRunningData) runningData;
+            }
+
+            public void handleDataDeleted(String dataPath) throws Exception {
+                MDC.put("destination", destination);
+                mutex.set(false);
+                if (!release && activeData != null && isMine(activeData.getAddress())) {
+                    // 如果上一次active的状态就是本机，则即时触发一下active抢占
+                    initRunning();
+                } else {
+                    // 否则就是等待delayTime，避免因网络瞬端或者zk异常，导致出现频繁的切换操作
+                    delayExector.schedule(new Runnable() {
+
+                        public void run() {
+                            initRunning();
+                        }
+                    }, delayTime, TimeUnit.SECONDS);
+                }
+            }
+
+        };
+
+    }
+
+    public void start() {
+        super.start();
+        processStart();
+        if (zkClient != null) {
+            // 如果需要尽可能释放instance资源，不需要监听running节点，不然即使stop了这台机器，另一台机器立马会start
+            String path = ZookeeperPathUtils.getDestinationServerRunning(destination);
+            zkClient.subscribeDataChanges(path, dataListener);
+
+            initRunning();
+        } else {
+            processActiveEnter();// 没有zk，直接启动
+        }
+    }
+
+    public void release() {
+        if (zkClient != null) {
+            releaseRunning(); // 尝试一下release
+        } else {
+            processActiveExit(); // 没有zk，直接启动
+        }
+    }
+
+    public void stop() {
+        super.stop();
+
+        if (zkClient != null) {
+            String path = ZookeeperPathUtils.getDestinationServerRunning(destination);
+            zkClient.unsubscribeDataChanges(path, dataListener);
+
+            releaseRunning(); // 尝试一下release
+        } else {
+            processActiveExit(); // 没有zk，直接启动
+        }
+        processStop();
+    }
+
+    private void initRunning() {
+        if (!isStart()) {
+            return;
+        }
+
+        String path = ZookeeperPathUtils.getDestinationServerRunning(destination);
+        // 序列化
+        byte[] bytes = JsonUtils.marshalToByte(serverData);
+        try {
+            mutex.set(false);
+            zkClient.create(path, bytes, CreateMode.EPHEMERAL);
+            activeData = serverData;
+            processActiveEnter();// 触发一下事件
+            mutex.set(true);
+        } catch (ZkNodeExistsException e) {
+            bytes = zkClient.readData(path, true);
+            if (bytes == null) {// 如果不存在节点，立即尝试一次
+                initRunning();
+            } else {
+                activeData = JsonUtils.unmarshalFromByte(bytes, ServerRunningData.class);
+            }
+        } catch (ZkNoNodeException e) {
+            zkClient.createPersistent(ZookeeperPathUtils.getDestinationPath(destination), true); // 尝试创建父节点
+            initRunning();
+        }
+    }
+
+    /**
+     * 阻塞等待自己成为active，如果自己成为active，立马返回
+     * 
+     * @throws InterruptedException
+     */
+    public void waitForActive() throws InterruptedException {
+        initRunning();
+        mutex.get();
+    }
+
+    /**
+     * 检查当前的状态
+     */
+    public boolean check() {
+        String path = ZookeeperPathUtils.getDestinationServerRunning(destination);
+        try {
+            byte[] bytes = zkClient.readData(path);
+            ServerRunningData eventData = JsonUtils.unmarshalFromByte(bytes, ServerRunningData.class);
+            activeData = eventData;// 更新下为最新值
+            // 检查下nid是否为自己
+            boolean result = isMine(activeData.getAddress());
+            if (!result) {
+                logger.warn("canal is running in node[{}] , but not in node[{}]",
+                    activeData.getCid(),
+                    serverData.getCid());
+            }
+            return result;
+        } catch (ZkNoNodeException e) {
+            logger.warn("canal is not run any in node");
+            return false;
+        } catch (ZkInterruptedException e) {
+            logger.warn("canal check is interrupt");
+            Thread.interrupted();// 清除interrupt标记
+            return check();
+        } catch (ZkException e) {
+            logger.warn("canal check is failed");
+            return false;
+        }
+    }
+
+    private boolean releaseRunning() {
+        if (check()) {
+            String path = ZookeeperPathUtils.getDestinationServerRunning(destination);
+            zkClient.delete(path);
+            mutex.set(false);
+            processActiveExit();
+            return true;
+        }
+
+        return false;
+    }
+
+    // ====================== helper method ======================
+
+    private boolean isMine(String address) {
+        return address.equals(serverData.getAddress());
+    }
+
+    private void processStart() {
+        if (listener != null) {
+            try {
+                listener.processStart();
+            } catch (Exception e) {
+                logger.error("processStart failed", e);
+            }
+        }
+    }
+
+    private void processStop() {
+        if (listener != null) {
+            try {
+                listener.processStop();
+            } catch (Exception e) {
+                logger.error("processStop failed", e);
+            }
+        }
+    }
+
+    private void processActiveEnter() {
+        if (listener != null) {
+            try {
+                listener.processActiveEnter();
+            } catch (Exception e) {
+                logger.error("processActiveEnter failed", e);
+            }
+        }
+    }
+
+    private void processActiveExit() {
+        if (listener != null) {
+            try {
+                listener.processActiveExit();
+            } catch (Exception e) {
+                logger.error("processActiveExit failed", e);
+            }
+        }
+    }
+
+    public void setListener(ServerRunningListener listener) {
+        this.listener = listener;
+    }
+
+    // ===================== setter / getter =======================
+
+    public void setDelayTime(int delayTime) {
+        this.delayTime = delayTime;
+    }
+
+    public void setServerData(ServerRunningData serverData) {
+        this.serverData = serverData;
+    }
+
+    public void setDestination(String destination) {
+        this.destination = destination;
+    }
+
+    public void setZkClient(ZkClientx zkClient) {
+        this.zkClient = zkClient;
+    }
+
+}

common/src/main/java/com/alibaba/otter/canal/common/zookeeper/running/ServerRunningMonitors.java

@@ -0,0 +1,36 @@
+package com.alibaba.otter.canal.common.zookeeper.running;
+
+import java.util.Map;
+
+/**
+ * {@linkplain ServerRunningMonitor}管理容器，使用static进行数据全局共享
+ * 
+ * @author jianghang 2012-12-3 下午09:32:06
+ * @version 1.0.0
+ */
+public class ServerRunningMonitors {
+
+    private static ServerRunningData serverData;
+    private static Map               runningMonitors; // <String, ServerRunningMonitor>
+
+    public static ServerRunningData getServerData() {
+        return serverData;
+    }
+
+    public static Map<String, ServerRunningMonitor> getRunningMonitors() {
+        return runningMonitors;
+    }
+
+    public static ServerRunningMonitor getRunningMonitor(String destination) {
+        return (ServerRunningMonitor) runningMonitors.get(destination);
+    }
+
+    public static void setServerData(ServerRunningData serverData) {
+        ServerRunningMonitors.serverData = serverData;
+    }
+
+    public static void setRunningMonitors(Map runningMonitors) {
+        ServerRunningMonitors.runningMonitors = runningMonitors;
+    }
+
+}

common/src/test/java/com/alibaba/otter/canal/common/AbstractZkTest.java

@@ -0,0 +1,18 @@
+package com.alibaba.otter.canal.common;
+
+import org.junit.Assert;
+
+public class AbstractZkTest {
+
+    protected String destination = "ljhtest1";
+    protected String cluster1    = "127.0.0.1:2188";
+    protected String cluster2    = "127.0.0.1:2188,127.0.0.1:2188";
+
+    public void sleep(long time) {
+        try {
+            Thread.sleep(time);
+        } catch (InterruptedException e) {
+            Assert.fail(e.getMessage());
+        }
+    }
+}

common/src/test/java/com/alibaba/otter/canal/common/ServerRunningTest.java

@@ -0,0 +1,143 @@
+package com.alibaba.otter.canal.common;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import org.apache.commons.lang.math.RandomUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
+import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
+import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningData;
+import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningListener;
+import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningMonitor;
+
+public class ServerRunningTest extends AbstractZkTest {
+
+    private ZkClientx zkclientx = new ZkClientx(cluster1 + ";" + cluster2);
+
+    @Before
+    public void setUp() {
+        String path = ZookeeperPathUtils.getDestinationPath(destination);
+        zkclientx.deleteRecursive(path);
+
+        zkclientx.createPersistent(ZookeeperPathUtils.getDestinationPath(destination), true);
+    }
+
+    @After
+    public void tearDown() {
+        String path = ZookeeperPathUtils.getDestinationPath(destination);
+        zkclientx.deleteRecursive(path);
+    }
+
+    @Test
+    public void testOneServer() {
+        final CountDownLatch countLatch = new CountDownLatch(2);
+        ServerRunningMonitor runningMonitor = buildServerRunning(countLatch, 1L, "127.0.0.1", 2088);
+        runningMonitor.start();
+        sleep(2000L);
+        runningMonitor.stop();
+        sleep(2000L);
+
+        if (countLatch.getCount() != 0) {
+            Assert.fail();
+        }
+    }
+
+    @Test
+    public void testMultiServer() {
+        final CountDownLatch countLatch = new CountDownLatch(30);
+        final ServerRunningMonitor runningMonitor1 = buildServerRunning(countLatch, 1L, "127.0.0.1", 2088);
+        final ServerRunningMonitor runningMonitor2 = buildServerRunning(countLatch, 2L, "127.0.0.1", 2089);
+        final ServerRunningMonitor runningMonitor3 = buildServerRunning(countLatch, 3L, "127.0.0.1", 2090);
+        final ExecutorService executor = Executors.newFixedThreadPool(3);
+        executor.submit(new Runnable() {
+
+            public void run() {
+                for (int i = 0; i < 10; i++) {
+                    if (!runningMonitor1.isStart()) {
+                        runningMonitor1.start();
+                    }
+                    sleep(2000L + RandomUtils.nextInt(500));
+                    if (runningMonitor1.check()) {
+                        runningMonitor1.stop();
+                    }
+                    sleep(2000L + RandomUtils.nextInt(500));
+                }
+            }
+
+        });
+
+        executor.submit(new Runnable() {
+
+            public void run() {
+                for (int i = 0; i < 10; i++) {
+                    if (!runningMonitor2.isStart()) {
+                        runningMonitor2.start();
+                    }
+                    sleep(2000L + RandomUtils.nextInt(500));
+                    if (runningMonitor2.check()) {
+                        runningMonitor2.stop();
+                    }
+                    sleep(2000L + RandomUtils.nextInt(500));
+                }
+            }
+
+        });
+
+        executor.submit(new Runnable() {
+
+            public void run() {
+                for (int i = 0; i < 10; i++) {
+                    if (!runningMonitor3.isStart()) {
+                        runningMonitor3.start();
+                    }
+                    sleep(2000L + RandomUtils.nextInt(500));
+                    if (runningMonitor3.check()) {
+                        runningMonitor3.stop();
+                    }
+                    sleep(2000L + RandomUtils.nextInt(500));
+                }
+            }
+
+        });
+
+        sleep(30000L);
+    }
+
+    private ServerRunningMonitor buildServerRunning(final CountDownLatch countLatch, final Long cid, final String ip,
+                                                    final int port) {
+        ServerRunningData serverData = new ServerRunningData(cid, ip + ":" + port);
+        ServerRunningMonitor runningMonitor = new ServerRunningMonitor(serverData);
+        runningMonitor.setDestination(destination);
+        runningMonitor.setListener(new ServerRunningListener() {
+
+            public void processActiveEnter() {
+                System.out.println(String.format("cid:%s ip:%s:%s has start", cid, ip, port));
+                countLatch.countDown();
+            }
+
+            public void processActiveExit() {
+                System.out.println(String.format("cid:%s ip:%s:%s has stop", cid, ip, port));
+                countLatch.countDown();
+            }
+
+            public void processStart() {
+                System.out.println(String.format("cid:%s ip:%s:%s processStart", cid, ip, port));
+            }
+
+            public void processStop() {
+                System.out.println(String.format("cid:%s ip:%s:%s processStop", cid, ip, port));
+            }
+
+        });
+
+        runningMonitor.setZkClient(zkclientx);
+        runningMonitor.setDelayTime(1);
+        return runningMonitor;
+    }
+}

common/src/test/java/logback.xml

@@ -0,0 +1,14 @@
+<configuration scan="true" scanPeriod=" 5 seconds">
+
+	<jmxConfigurator />
+	<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+		<encoder>
+			<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n
+			</pattern>
+		</encoder>
+	</appender>
+	
+	<root level="WARN">
+		<appender-ref ref="STDOUT"/>
+	</root>
+</configuration>

dbsync/pom.xml

@@ -0,0 +1,43 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<parent>
+		<groupId>com.alibaba.otter</groupId>
+		<artifactId>canal</artifactId>
+		<version>1.0.19-SNAPSHOT</version>
+		<relativePath>../pom.xml</relativePath>
+	</parent>
+	<groupId>com.alibaba.otter</groupId>
+	<artifactId>canal.parse.dbsync</artifactId>
+	<packaging>jar</packaging>
+	<name>canal dbsync module for otter ${project.version}</name>
+	<dependencies>
+		<!-- log -->
+		<dependency>
+			<groupId>ch.qos.logback</groupId>
+			<artifactId>logback-core</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>ch.qos.logback</groupId>
+			<artifactId>logback-classic</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.slf4j</groupId>
+			<artifactId>jcl-over-slf4j</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.slf4j</groupId>
+			<artifactId>slf4j-api</artifactId>
+		</dependency>
+		<!-- test dependency -->
+		<dependency>
+			<groupId>junit</groupId>
+			<artifactId>junit</artifactId>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
+			<groupId>mysql</groupId>
+			<artifactId>mysql-connector-java</artifactId>
+			<scope>test</scope>
+		</dependency>
+	</dependencies>
+</project>

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/CharsetConversion.java

@@ -0,0 +1,367 @@
+package com.taobao.tddl.dbsync.binlog;
+
+import java.nio.charset.Charset;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * An utility class implements MySQL/Java charsets conversion.  you can see com.mysql.jdbc.CharsetMapping.
+ * 
+ * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
+ */
+public final class CharsetConversion
+{
+    static final Log logger = LogFactory.getLog(CharsetConversion.class);
+
+    static final class Entry
+    {
+        protected final int    charsetId;
+        protected final String mysqlCharset;
+        protected final String mysqlCollation;
+        protected final String javaCharset;
+
+        Entry(final int id, String mysqlCharset, // NL
+                String mysqlCollation, String javaCharset)
+        {
+            this.charsetId = id;
+            this.mysqlCharset = mysqlCharset;
+            this.mysqlCollation = mysqlCollation;
+            this.javaCharset = javaCharset;
+        }
+    }
+
+    // Character set data used in lookups. The array will be sparse.
+    static final Entry[] entries = new Entry[0xff];
+
+    static Entry getEntry(final int id)
+    {
+        if (id >= 0 && id < entries.length)
+        {
+            return entries[id];
+        }
+        else
+        {
+            throw new IllegalArgumentException("Invalid charset id: " + id);
+        }
+    }
+
+    // Loads character set information.
+    static void putEntry(final int charsetId, String mysqlCharset,
+            String mysqlCollation, String javaCharset)
+    {
+        entries[charsetId] = new Entry(charsetId, mysqlCharset, // NL
+                mysqlCollation, javaCharset);
+    }
+
+    // Loads character set information.
+    @Deprecated
+    static void putEntry(final int charsetId, String mysqlCharset,
+            String mysqlCollation)
+    {
+        entries[charsetId] = new Entry(charsetId, mysqlCharset, // NL
+                mysqlCollation, /* Unknown java charset */null);
+    }
+
+    // Load character set data statically.
+    static
+    {
+        putEntry(1, "big5", "big5_chinese_ci", "Big5");
+        putEntry(2, "latin2", "latin2_czech_cs", "ISO8859_2");
+        putEntry(3, "dec8", "dec8_swedish_ci", "ISO8859_1");
+        putEntry(4, "cp850", "cp850_general_ci", "Cp850");
+        putEntry(5, "latin1", "latin1_german1_ci", "ISO8859_1");
+        putEntry(6, "hp8", "hp8_english_ci", "ISO8859_1");
+        putEntry(7, "koi8r", "koi8r_general_ci", "KOI8_R");
+        putEntry(8, "latin1", "latin1_swedish_ci", "ISO8859_1");
+        putEntry(9, "latin2", "latin2_general_ci", "ISO8859_2");
+        putEntry(10, "swe7", "swe7_swedish_ci", "ISO8859_1");
+        putEntry(11, "ascii", "ascii_general_ci", "US-ASCII");
+        putEntry(12, "ujis", "ujis_japanese_ci", "EUC_JP");
+        putEntry(13, "sjis", "sjis_japanese_ci", "SJIS");
+        putEntry(14, "cp1251", "cp1251_bulgarian_ci", "Cp1251");
+        putEntry(15, "latin1", "latin1_danish_ci", "ISO8859_1");
+        putEntry(16, "hebrew", "hebrew_general_ci", "ISO8859_8");
+        putEntry(17, "filename", "filename", "ISO8859_1");
+        putEntry(18, "tis620", "tis620_thai_ci", "TIS620");
+        putEntry(19, "euckr", "euckr_korean_ci", "EUC_KR");
+        putEntry(20, "latin7", "latin7_estonian_cs", "ISO8859_7");
+        putEntry(21, "latin2", "latin2_hungarian_ci", "ISO8859_2");
+        putEntry(22, "koi8u", "koi8u_general_ci", "KOI8_U");
+        putEntry(23, "cp1251", "cp1251_ukrainian_ci", "Cp1251");
+        putEntry(24, "gb2312", "gb2312_chinese_ci", "EUC_CN");
+        putEntry(25, "greek", "greek_general_ci", "ISO8859_7");
+        putEntry(26, "cp1250", "cp1250_general_ci", "Cp1250");
+        putEntry(27, "latin2", "latin2_croatian_ci", "ISO8859_2");
+        putEntry(28, "gbk", "gbk_chinese_ci", "GBK");
+        putEntry(29, "cp1257", "cp1257_lithuanian_ci", "Cp1257");
+        putEntry(30, "latin5", "latin5_turkish_ci", "ISO8859_5");
+        putEntry(31, "latin1", "latin1_german2_ci", "ISO8859_1");
+        putEntry(32, "armscii8", "armscii8_general_ci", "ISO8859_1");
+        putEntry(33, "utf8", "utf8_general_ci", "UTF-8");
+        putEntry(34, "cp1250", "cp1250_czech_cs", "Cp1250");
+        putEntry(35, "ucs2", "ucs2_general_ci", "UnicodeBig");
+        putEntry(36, "cp866", "cp866_general_ci", "Cp866");
+        putEntry(37, "keybcs2", "keybcs2_general_ci", "Cp895");
+        putEntry(38, "macce", "macce_general_ci", "MacCentralEurope");
+        putEntry(39, "macroman", "macroman_general_ci", "MacRoman");
+        putEntry(40, "cp852", "cp852_general_ci", "Cp852");
+        putEntry(41, "latin7", "latin7_general_ci", "ISO8859_7");
+        putEntry(42, "latin7", "latin7_general_cs", "ISO8859_7");
+        putEntry(43, "macce", "macce_bin", "MacCentralEurope");
+        putEntry(44, "cp1250", "cp1250_croatian_ci", "Cp1250");
+        putEntry(45, "utf8mb4", "utf8mb4_general_ci", "MacCentralEurope");
+        putEntry(46, "utf8mb4", "utf8mb4_bin", "MacCentralEurope");
+        putEntry(47, "latin1", "latin1_bin", "ISO8859_1");
+        putEntry(48, "latin1", "latin1_general_ci", "ISO8859_1");
+        putEntry(49, "latin1", "latin1_general_cs", "ISO8859_1");
+        putEntry(50, "cp1251", "cp1251_bin", "Cp1251");
+        putEntry(51, "cp1251", "cp1251_general_ci", "Cp1251");
+        putEntry(52, "cp1251", "cp1251_general_cs", "Cp1251");
+        putEntry(53, "macroman", "macroman_bin", "MacRoman");
+        putEntry(54, "utf16", "utf16_general_ci", "UTF-16");
+        putEntry(55, "utf16", "utf16_bin", "UTF-16");
+        putEntry(57, "cp1256", "cp1256_general_ci", "Cp1256");
+        putEntry(58, "cp1257", "cp1257_bin", "Cp1257");
+        putEntry(59, "cp1257", "cp1257_general_ci", "Cp1257");
+        putEntry(60, "utf32", "utf32_general_ci", "UTF-32");
+        putEntry(61, "utf32", "utf32_bin", "UTF-32");
+        putEntry(63, "binary", "binary", "US-ASCII");
+        putEntry(64, "armscii8", "armscii8_bin", "ISO8859_2");
+        putEntry(65, "ascii", "ascii_bin", "US-ASCII");
+        putEntry(66, "cp1250", "cp1250_bin", "Cp1250");
+        putEntry(67, "cp1256", "cp1256_bin", "Cp1256");
+        putEntry(68, "cp866", "cp866_bin", "Cp866");
+        putEntry(69, "dec8", "dec8_bin", "US-ASCII");
+        putEntry(70, "greek", "greek_bin", "ISO8859_7");
+        putEntry(71, "hebrew", "hebrew_bin", "ISO8859_8");
+        putEntry(72, "hp8", "hp8_bin" , "US-ASCII");
+        putEntry(73, "keybcs2", "keybcs2_bin", "Cp895");
+        putEntry(74, "koi8r", "koi8r_bin", "KOI8_R");
+        putEntry(75, "koi8u", "koi8u_bin", "KOI8_U");
+        putEntry(77, "latin2", "latin2_bin", "ISO8859_2");
+        putEntry(78, "latin5", "latin5_bin", "ISO8859_5");
+        putEntry(79, "latin7", "latin7_bin", "ISO8859_7");
+        putEntry(80, "cp850", "cp850_bin", "Cp850");
+        putEntry(81, "cp852", "cp852_bin", "Cp852");
+        putEntry(82, "swe7", "swe7_bin", "ISO8859_1");
+        putEntry(83, "utf8", "utf8_bin", "UTF-8");
+        putEntry(84, "big5", "big5_bin", "Big5");
+        putEntry(85, "euckr", "euckr_bin", "EUC_KR");
+        putEntry(86, "gb2312", "gb2312_bin", "EUC_CN");
+        putEntry(87, "gbk", "gbk_bin", "GBK");
+        putEntry(88, "sjis", "sjis_bin", "SJIS");
+        putEntry(89, "tis620", "tis620_bin", "TIS620");
+        putEntry(90, "ucs2", "ucs2_bin", "UnicodeBig");
+        putEntry(91, "ujis", "ujis_bin", "EUC_JP");
+        putEntry(92, "geostd8", "geostd8_general_ci", "US-ASCII");
+        putEntry(93, "geostd8", "geostd8_bin", "US-ASCII");
+        putEntry(94, "latin1", "latin1_spanish_ci", "ISO8859_1");
+        putEntry(95, "cp932", "cp932_japanese_ci", "Shift_JIS");
+        putEntry(96, "cp932", "cp932_bin", "Shift_JIS");
+        putEntry(97, "eucjpms", "eucjpms_japanese_ci", "EUC_JP");
+        putEntry(98, "eucjpms", "eucjpms_bin", "EUC_JP");
+        putEntry(99, "cp1250", "cp1250_polish_ci", "Cp1250");
+
+        putEntry(101, "utf16", "utf16_unicode_ci", "UTF-16");
+        putEntry(102, "utf16", "utf16_icelandic_ci", "UTF-16");
+        putEntry(103, "utf16", "utf16_latvian_ci", "UTF-16");
+        putEntry(104, "utf16", "utf16_romanian_ci", "UTF-16");
+        putEntry(105, "utf16", "utf16_slovenian_ci", "UTF-16");
+        putEntry(106, "utf16", "utf16_polish_ci", "UTF-16");
+        putEntry(107, "utf16", "utf16_estonian_ci", "UTF-16");
+        putEntry(108, "utf16", "utf16_spanish_ci", "UTF-16");
+        putEntry(109, "utf16", "utf16_swedish_ci", "UTF-16");
+        putEntry(110, "utf16", "utf16_turkish_ci", "UTF-16");
+        putEntry(111, "utf16", "utf16_czech_ci", "UTF-16");
+        putEntry(112, "utf16", "utf16_danish_ci", "UTF-16");
+        putEntry(113, "utf16", "utf16_lithuanian_ci", "UTF-16");
+        putEntry(114, "utf16", "utf16_slovak_ci", "UTF-16");
+        putEntry(115, "utf16", "utf16_spanish2_ci", "UTF-16");
+        putEntry(116, "utf16", "utf16_roman_ci", "UTF-16");
+        putEntry(117, "utf16", "utf16_persian_ci", "UTF-16");
+        putEntry(118, "utf16", "utf16_esperanto_ci", "UTF-16");
+        putEntry(119, "utf16", "utf16_hungarian_ci", "UTF-16");
+        putEntry(120, "utf16", "utf16_sinhala_ci", "UTF-16");
+
+        putEntry(128, "ucs2", "ucs2_unicode_ci", "UnicodeBig");
+        putEntry(129, "ucs2", "ucs2_icelandic_ci", "UnicodeBig");
+        putEntry(130, "ucs2", "ucs2_latvian_ci", "UnicodeBig");
+        putEntry(131, "ucs2", "ucs2_romanian_ci", "UnicodeBig");
+        putEntry(132, "ucs2", "ucs2_slovenian_ci", "UnicodeBig");
+        putEntry(133, "ucs2", "ucs2_polish_ci", "UnicodeBig");
+        putEntry(134, "ucs2", "ucs2_estonian_ci", "UnicodeBig");
+        putEntry(135, "ucs2", "ucs2_spanish_ci", "UnicodeBig");
+        putEntry(136, "ucs2", "ucs2_swedish_ci", "UnicodeBig");
+        putEntry(137, "ucs2", "ucs2_turkish_ci", "UnicodeBig");
+        putEntry(138, "ucs2", "ucs2_czech_ci", "UnicodeBig");
+        putEntry(139, "ucs2", "ucs2_danish_ci", "UnicodeBig");
+        putEntry(140, "ucs2", "ucs2_lithuanian_ci", "UnicodeBig");
+        putEntry(141, "ucs2", "ucs2_slovak_ci", "UnicodeBig");
+        putEntry(142, "ucs2", "ucs2_spanish2_ci", "UnicodeBig");
+        putEntry(143, "ucs2", "ucs2_roman_ci", "UnicodeBig");
+        putEntry(144, "ucs2", "ucs2_persian_ci", "UnicodeBig");
+        putEntry(145, "ucs2", "ucs2_esperanto_ci", "UnicodeBig");
+        putEntry(146, "ucs2", "ucs2_hungarian_ci", "UnicodeBig");
+        putEntry(147, "ucs2", "ucs2_sinhala_ci", "UnicodeBig");
+
+        putEntry(160, "utf32", "utf32_unicode_ci", "UTF-32");
+        putEntry(161, "utf32", "utf32_icelandic_ci", "UTF-32");
+        putEntry(162, "utf32", "utf32_latvian_ci", "UTF-32");
+        putEntry(163, "utf32", "utf32_romanian_ci", "UTF-32");
+        putEntry(164, "utf32", "utf32_slovenian_ci", "UTF-32");
+        putEntry(165, "utf32", "utf32_polish_ci", "UTF-32");
+        putEntry(166, "utf32", "utf32_estonian_ci", "UTF-32");
+        putEntry(167, "utf32", "utf32_spanish_ci", "UTF-32");
+        putEntry(168, "utf32", "utf32_swedish_ci", "UTF-32");
+        putEntry(169, "utf32", "utf32_turkish_ci", "UTF-32");
+        putEntry(170, "utf32", "utf32_czech_ci", "UTF-32");
+        putEntry(171, "utf32", "utf32_danish_ci", "UTF-32");
+        putEntry(172, "utf32", "utf32_lithuanian_ci", "UTF-32");
+        putEntry(173, "utf32", "utf32_slovak_ci", "UTF-32");
+        putEntry(174, "utf32", "utf32_spanish2_ci", "UTF-32");
+        putEntry(175, "utf32", "utf32_roman_ci", "UTF-32");
+        putEntry(176, "utf32", "utf32_persian_ci", "UTF-32");
+        putEntry(177, "utf32", "utf32_esperanto_ci", "UTF-32");
+        putEntry(178, "utf32", "utf32_hungarian_ci", "UTF-32");
+        putEntry(179, "utf32", "utf32_sinhala_ci", "UTF-32");
+
+        putEntry(192, "utf8", "utf8_unicode_ci", "UTF-8");
+        putEntry(193, "utf8", "utf8_icelandic_ci", "UTF-8");
+        putEntry(194, "utf8", "utf8_latvian_ci", "UTF-8");
+        putEntry(195, "utf8", "utf8_romanian_ci", "UTF-8");
+        putEntry(196, "utf8", "utf8_slovenian_ci", "UTF-8");
+        putEntry(197, "utf8", "utf8_polish_ci", "UTF-8");
+        putEntry(198, "utf8", "utf8_estonian_ci", "UTF-8");
+        putEntry(199, "utf8", "utf8_spanish_ci", "UTF-8");
+        putEntry(200, "utf8", "utf8_swedish_ci", "UTF-8");
+        putEntry(201, "utf8", "utf8_turkish_ci", "UTF-8");
+        putEntry(202, "utf8", "utf8_czech_ci", "UTF-8");
+        putEntry(203, "utf8", "utf8_danish_ci", "UTF-8");
+        putEntry(204, "utf8", "utf8_lithuanian_ci", "UTF-8");
+        putEntry(205, "utf8", "utf8_slovak_ci", "UTF-8");
+        putEntry(206, "utf8", "utf8_spanish2_ci", "UTF-8");
+        putEntry(207, "utf8", "utf8_roman_ci", "UTF-8");
+        putEntry(208, "utf8", "utf8_persian_ci", "UTF-8");
+        putEntry(209, "utf8", "utf8_esperanto_ci", "UTF-8");
+        putEntry(210, "utf8", "utf8_hungarian_ci", "UTF-8");
+        putEntry(211, "utf8", "utf8_sinhala_ci", "UTF-8");
+
+        putEntry(224, "utf8mb4", "utf8mb4_unicode_ci", "UTF-8");
+        putEntry(225, "utf8mb4", "utf8mb4_icelandic_ci", "UTF-8");
+        putEntry(226, "utf8mb4", "utf8mb4_latvian_ci", "UTF-8");
+        putEntry(227, "utf8mb4", "utf8mb4_romanian_ci", "UTF-8");
+        putEntry(228, "utf8mb4", "utf8mb4_slovenian_ci", "UTF-8");
+        putEntry(229, "utf8mb4", "utf8mb4_polish_ci", "UTF-8");
+        putEntry(230, "utf8mb4", "utf8mb4_estonian_ci", "UTF-8");
+        putEntry(231, "utf8mb4", "utf8mb4_spanish_ci", "UTF-8");
+        putEntry(232, "utf8mb4", "utf8mb4_swedish_ci", "UTF-8");
+        putEntry(233, "utf8mb4", "utf8mb4_turkish_ci", "UTF-8");
+        putEntry(234, "utf8mb4", "utf8mb4_czech_ci", "UTF-8");
+        putEntry(235, "utf8mb4", "utf8mb4_danish_ci", "UTF-8");
+        putEntry(236, "utf8mb4", "utf8mb4_lithuanian_ci", "UTF-8");
+        putEntry(237, "utf8mb4", "utf8mb4_slovak_ci", "UTF-8");
+        putEntry(238, "utf8mb4", "utf8mb4_spanish2_ci", "UTF-8");
+        putEntry(239, "utf8mb4", "utf8mb4_roman_ci", "UTF-8");
+        putEntry(240, "utf8mb4", "utf8mb4_persian_ci", "UTF-8");
+        putEntry(241, "utf8mb4", "utf8mb4_esperanto_ci", "UTF-8");
+        putEntry(242, "utf8mb4", "utf8mb4_hungarian_ci", "UTF-8");
+        putEntry(243, "utf8mb4", "utf8mb4_sinhala_ci", "UTF-8");
+
+        putEntry(254, "utf8", "utf8_general_cs", "UTF-8");
+    }
+
+    /**
+     * Return defined charset name for mysql.
+     */
+    public static String getCharset(final int id)
+    {
+        Entry entry = getEntry(id);
+
+        if (entry != null)
+        {
+            return entry.mysqlCharset;
+        }
+        else
+        {
+            logger.warn("Unexpect mysql charset: " + id);
+            return null;
+        }
+    }
+
+    /**
+     * Return defined collaction name for mysql.
+     */
+    public static String getCollation(final int id)
+    {
+        Entry entry = getEntry(id);
+
+        if (entry != null)
+        {
+            return entry.mysqlCollation;
+        }
+        else
+        {
+            logger.warn("Unexpect mysql charset: " + id);
+            return null;
+        }
+    }
+
+    /**
+     * Return converted charset name for java.
+     */
+    public static String getJavaCharset(final int id)
+    {
+        Entry entry = getEntry(id);
+
+        if (entry != null)
+        {
+            if (entry.javaCharset != null)
+            {
+                return entry.javaCharset;
+            }
+            else
+            {
+                logger.warn("Unknown java charset for: id = " + id
+                        + ", name = " + entry.mysqlCharset + ", coll = "
+                        + entry.mysqlCollation);
+                return null;
+            }
+        }
+        else
+        {
+            logger.warn("Unexpect mysql charset: " + id);
+            return null;
+        }
+    }
+
+    public static void main(String[] args)
+    {
+        for (int i = 0; i < entries.length; i++)
+        {
+            Entry entry = entries[i];
+
+            System.out.print(i);
+            System.out.print(',');
+            System.out.print(' ');
+            if (entry != null)
+            {
+                System.out.print(entry.mysqlCharset);
+                System.out.print(',');
+                System.out.print(' ');
+                System.out.print(entry.javaCharset);
+                if (entry.javaCharset != null)
+                {
+                    System.out.print(',');
+                    System.out.print(' ');
+                    System.out.print(Charset.forName(entry.javaCharset).name());
+                }
+            }
+            else
+            {
+                System.out.print("null");
+            }
+            System.out.println();
+        }
+    }
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/DirectLogFetcher.java

@@ -0,0 +1,487 @@
+package com.taobao.tddl.dbsync.binlog;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InterruptedIOException;
+import java.io.OutputStream;
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.net.SocketTimeoutException;
+import java.sql.Connection;
+import java.sql.SQLException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * TODO: Document It!!
+ * 
+ * <pre>
+ * DirectLogFetcher fetcher = new DirectLogFetcher();
+ * fetcher.open(conn, file, 0, 13);
+ * 
+ * while (fetcher.fetch())
+ * {
+ *     LogEvent event;
+ *     do
+ *     {
+ *         event = decoder.decode(fetcher, context);
+ * 
+ *         // process log event.
+ *     }
+ *     while (event != null);
+ * }
+ * // connection closed.
+ * </pre>
+ * 
+ * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
+ * @version 1.0
+ */
+public final class DirectLogFetcher extends LogFetcher
+{
+    protected static final Log logger                = LogFactory.getLog(DirectLogFetcher.class);
+
+    /** Command to dump binlog */
+    public static final byte   COM_BINLOG_DUMP       = 18;
+
+    /** Packet header sizes */
+    public static final int    NET_HEADER_SIZE       = 4;
+    public static final int    SQLSTATE_LENGTH       = 5;
+
+    /** Packet offsets */
+    public static final int    PACKET_LEN_OFFSET     = 0;
+    public static final int    PACKET_SEQ_OFFSET     = 3;
+
+    /** Maximum packet length */
+    public static final int    MAX_PACKET_LENGTH     = (256 * 256 * 256 - 1);
+
+    /** BINLOG_DUMP options */
+    public static final int    BINLOG_DUMP_NON_BLOCK = 1;
+    public static final int    BINLOG_SEND_ANNOTATE_ROWS_EVENT = 2;
+
+    private Connection         conn;
+    private OutputStream       mysqlOutput;
+    private InputStream        mysqlInput;
+
+    public DirectLogFetcher()
+    {
+        super(DEFAULT_INITIAL_CAPACITY, DEFAULT_GROWTH_FACTOR);
+    }
+
+    public DirectLogFetcher(final int initialCapacity)
+    {
+        super(initialCapacity, DEFAULT_GROWTH_FACTOR);
+    }
+
+    public DirectLogFetcher(final int initialCapacity, final float growthFactor)
+    {
+        super(initialCapacity, growthFactor);
+    }
+
+    private static final Object unwrapConnection(Object conn, Class<?> connClazz)
+            throws IOException
+    {
+        while (!connClazz.isInstance(conn))
+        {
+            try
+            {
+                Class<?> connProxy = Class.forName("org.springframework.jdbc.datasource.ConnectionProxy");
+                if (connProxy.isInstance(conn))
+                {
+                    conn = invokeMethod(conn, connProxy, "getTargetConnection");
+                    continue;
+                }
+            }
+            catch (ClassNotFoundException e)
+            {
+                // org.springframework.jdbc.datasource.ConnectionProxy not found.
+            }
+
+            try
+            {
+                Class<?> connProxy = Class.forName("org.apache.commons.dbcp.DelegatingConnection");
+                if (connProxy.isInstance(conn))
+                {
+                    conn = getDeclaredField(conn, connProxy, "_conn");
+                    continue;
+                }
+            }
+            catch (ClassNotFoundException e)
+            {
+                // org.apache.commons.dbcp.DelegatingConnection not found.
+            }
+
+            try
+            {
+                if (conn instanceof java.sql.Wrapper)
+                {
+                    Class<?> connIface = Class.forName("com.mysql.jdbc.Connection");
+                    conn = ((java.sql.Wrapper) conn).unwrap(connIface);
+                    continue;
+                }
+            }
+            catch (ClassNotFoundException e)
+            {
+                // com.mysql.jdbc.Connection not found.
+            }
+            catch (SQLException e)
+            {
+                logger.warn("Unwrap " + conn.getClass().getName() + " to "
+                        + connClazz.getName() + " failed: " + e.getMessage(), e);
+            }
+
+            return null;
+        }
+        return conn;
+    }
+
+    private static final Object invokeMethod(Object obj, Class<?> objClazz,
+            String name)
+    {
+        try
+        {
+            Method method = objClazz.getMethod(name, (Class<?>[]) null);
+            return method.invoke(obj, (Object[]) null);
+        }
+        catch (NoSuchMethodException e)
+        {
+            throw new IllegalArgumentException("No such method: \'" + name
+                    + "\' @ " + objClazz.getName(), e);
+        }
+        catch (IllegalAccessException e)
+        {
+            throw new IllegalArgumentException("Cannot invoke method: \'"
+                    + name + "\' @ " + objClazz.getName(), e);
+        }
+        catch (InvocationTargetException e)
+        {
+            throw new IllegalArgumentException("Invoke method failed: \'"
+                    + name + "\' @ " + objClazz.getName(),
+                    e.getTargetException());
+        }
+    }
+
+    private static final Object getDeclaredField(Object obj, Class<?> objClazz,
+            String name)
+    {
+        try
+        {
+            Field field = objClazz.getDeclaredField(name);
+            field.setAccessible(true);
+            return field.get(obj);
+        }
+        catch (NoSuchFieldException e)
+        {
+            throw new IllegalArgumentException("No such field: \'" + name
+                    + "\' @ " + objClazz.getName(), e);
+        }
+        catch (IllegalAccessException e)
+        {
+            throw new IllegalArgumentException("Cannot get field: \'" + name
+                    + "\' @ " + objClazz.getName(), e);
+        }
+    }
+
+    /**
+     * Connect MySQL master to fetch binlog.
+     */
+    public void open(Connection conn, String fileName, final int serverId)
+            throws IOException
+    {
+        open(conn, fileName, BIN_LOG_HEADER_SIZE, serverId, false);
+    }
+
+    /**
+     * Connect MySQL master to fetch binlog.
+     */
+    public void open(Connection conn, String fileName, final int serverId,
+            boolean nonBlocking) throws IOException
+    {
+        open(conn, fileName, BIN_LOG_HEADER_SIZE, serverId, nonBlocking);
+    }
+
+    /**
+     * Connect MySQL master to fetch binlog.
+     */
+    public void open(Connection conn, String fileName, final long filePosition,
+            final int serverId) throws IOException
+    {
+        open(conn, fileName, filePosition, serverId, false);
+    }
+
+    /**
+     * Connect MySQL master to fetch binlog.
+     */
+    public void open(Connection conn, String fileName, long filePosition,
+            final int serverId, boolean nonBlocking) throws IOException
+    {
+        try
+        {
+            this.conn = conn;
+            Class<?> connClazz = Class.forName("com.mysql.jdbc.ConnectionImpl");
+            Object unwrapConn = unwrapConnection(conn, connClazz);
+            if (unwrapConn == null)
+            {
+                throw new IOException("Unable to unwrap "
+                        + conn.getClass().getName()
+                        + " to com.mysql.jdbc.ConnectionImpl");
+            }
+
+            // Get underlying IO streams for network communications.
+            Object connIo = getDeclaredField(unwrapConn, connClazz, "io");
+            if (connIo == null)
+            {
+                throw new IOException("Get null field:"
+                        + conn.getClass().getName() + "#io");
+            }
+            mysqlOutput = (OutputStream) getDeclaredField(connIo,
+                    connIo.getClass(), "mysqlOutput");
+            mysqlInput = (InputStream) getDeclaredField(connIo,
+                    connIo.getClass(), "mysqlInput");
+
+            if (filePosition == 0)
+                filePosition = BIN_LOG_HEADER_SIZE;
+            sendBinlogDump(fileName, filePosition, serverId, nonBlocking);
+            position = 0;
+        }
+        catch (IOException e)
+        {
+            close(); /* Do cleanup */
+            logger.error("Error on COM_BINLOG_DUMP: file = " + fileName
+                    + ", position = " + filePosition);
+            throw e;
+        }
+        catch (ClassNotFoundException e)
+        {
+            close(); /* Do cleanup */
+            throw new IOException(
+                    "Unable to load com.mysql.jdbc.ConnectionImpl", e);
+        }
+    }
+
+    /**
+     * Put a byte in the buffer.
+     * 
+     * @param b the byte to put in the buffer
+     */
+    protected final void putByte(byte b)
+    {
+        ensureCapacity(position + 1);
+
+        buffer[position++] = b;
+    }
+
+    /**
+     * Put 16-bit integer in the buffer.
+     * 
+     * @param i16 the integer to put in the buffer
+     */
+    protected final void putInt16(int i16)
+    {
+        ensureCapacity(position + 2);
+
+        byte[] buf = buffer;
+        buf[position++] = (byte) (i16 & 0xff);
+        buf[position++] = (byte) (i16 >>> 8);
+    }
+
+    /**
+     * Put 32-bit integer in the buffer.
+     * 
+     * @param i32 the integer to put in the buffer
+     */
+    protected final void putInt32(long i32)
+    {
+        ensureCapacity(position + 4);
+
+        byte[] buf = buffer;
+        buf[position++] = (byte) (i32 & 0xff);
+        buf[position++] = (byte) (i32 >>> 8);
+        buf[position++] = (byte) (i32 >>> 16);
+        buf[position++] = (byte) (i32 >>> 24);
+    }
+
+    /**
+     * Put a string in the buffer.
+     * 
+     * @param s the value to put in the buffer
+     */
+    protected final void putString(String s)
+    {
+        ensureCapacity(position + (s.length() * 2) + 1);
+
+        System.arraycopy(s.getBytes(), 0, buffer, position, s.length());
+        position += s.length();
+        buffer[position++] = 0;
+    }
+
+    protected final void sendBinlogDump(String fileName,
+            final long filePosition, final int serverId, boolean nonBlocking)
+            throws IOException
+    {
+        position = NET_HEADER_SIZE;
+
+        putByte(COM_BINLOG_DUMP);
+        putInt32(filePosition);
+        int binlog_flags = nonBlocking ? BINLOG_DUMP_NON_BLOCK : 0;
+        binlog_flags |= BINLOG_SEND_ANNOTATE_ROWS_EVENT;
+        putInt16(binlog_flags); // binlog_flags
+        putInt32(serverId); // slave's server-id 
+        putString(fileName);
+
+        final byte[] buf = buffer;
+        final int len = position - NET_HEADER_SIZE;
+        buf[0] = (byte) (len & 0xff);
+        buf[1] = (byte) (len >>> 8);
+        buf[2] = (byte) (len >>> 16);
+
+        mysqlOutput.write(buffer, 0, position);
+        mysqlOutput.flush();
+    }
+
+    /**
+     * {@inheritDoc}
+     * 
+     * @see com.taobao.tddl.dbsync.binlog.LogFetcher#fetch()
+     */
+    public boolean fetch() throws IOException
+    {
+        try
+        {
+            // Fetching packet header from input.
+            if (!fetch0(0, NET_HEADER_SIZE))
+            {
+                logger.warn("Reached end of input stream while fetching header");
+                return false;
+            }
+
+            // Fetching the first packet(may a multi-packet).
+            int netlen = getUint24(PACKET_LEN_OFFSET);
+            int netnum = getUint8(PACKET_SEQ_OFFSET);
+            if (!fetch0(NET_HEADER_SIZE, netlen))
+            {
+                logger.warn("Reached end of input stream: packet #" + netnum
+                        + ", len = " + netlen);
+                return false;
+            }
+
+            // Detecting error code.
+            final int mark = getUint8(NET_HEADER_SIZE);
+            if (mark != 0)
+            {
+                if (mark == 255) // error from master
+                {
+                    // Indicates an error, for example trying to fetch from wrong
+                    // binlog position.
+                    position = NET_HEADER_SIZE + 1;
+                    final int errno = getInt16();
+                    String sqlstate = forward(1).getFixString(SQLSTATE_LENGTH);
+                    String errmsg = getFixString(limit - position);
+                    throw new IOException("Received error packet:"
+                            + " errno = " + errno + ", sqlstate = " + sqlstate
+                            + " errmsg = " + errmsg);
+                }
+                else if (mark == 254)
+                {
+                    // Indicates end of stream. It's not clear when this would
+                    // be sent.
+                    logger.warn("Received EOF packet from server, apparent"
+                            + " master disconnected.");
+                    return false;
+                }
+                else
+                {
+                    // Should not happen.
+                    throw new IOException("Unexpected response " + mark
+                            + " while fetching binlog: packet #" + netnum
+                            + ", len = " + netlen);
+                }
+            }
+
+            // The first packet is a multi-packet, concatenate the packets.
+            while (netlen == MAX_PACKET_LENGTH)
+            {
+                if (!fetch0(0, NET_HEADER_SIZE))
+                {
+                    logger.warn("Reached end of input stream while fetching header");
+                    return false;
+                }
+
+                netlen = getUint24(PACKET_LEN_OFFSET);
+                netnum = getUint8(PACKET_SEQ_OFFSET);
+                if (!fetch0(limit, netlen))
+                {
+                    logger.warn("Reached end of input stream: packet #"
+                            + netnum + ", len = " + netlen);
+                    return false;
+                }
+            }
+
+            // Preparing buffer variables to decoding.
+            origin = NET_HEADER_SIZE + 1;
+            position = origin;
+            limit -= origin;
+            return true;
+        }
+        catch (SocketTimeoutException e)
+        {
+            close(); /* Do cleanup */
+            logger.error("Socket timeout expired, closing connection", e);
+            throw e;
+        }
+        catch (InterruptedIOException e)
+        {
+            close(); /* Do cleanup */
+            logger.warn("I/O interrupted while reading from client socket", e);
+            throw e;
+        }
+        catch (IOException e)
+        {
+            close(); /* Do cleanup */
+            logger.error("I/O error while reading from client socket", e);
+            throw e;
+        }
+    }
+
+    private final boolean fetch0(final int off, final int len)
+            throws IOException
+    {
+        ensureCapacity(off + len);
+
+        for (int count, n = 0; n < len; n += count)
+        {
+            if (0 > (count = mysqlInput.read(buffer, off + n, len - n)))
+            {
+                // Reached end of input stream
+                return false;
+            }
+        }
+
+        if (limit < off + len)
+            limit = off + len;
+        return true;
+    }
+
+    /**
+     * {@inheritDoc}
+     * 
+     * @see com.taobao.tddl.dbsync.binlog.LogFetcher#close()
+     */
+    public void close() throws IOException
+    {
+        try
+        {
+            if (conn != null)
+                conn.close();
+
+            conn = null;
+            mysqlInput = null;
+            mysqlOutput = null;
+        }
+        catch (SQLException e)
+        {
+            logger.warn("Unable to close connection", e);
+        }
+    }
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/FileLogFetcher.java

@@ -0,0 +1,160 @@
+package com.taobao.tddl.dbsync.binlog;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Arrays;
+
+import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent;
+
+/**
+ * TODO: Document It!!
+ * 
+ * <pre>
+ * FileLogFetcher fetcher = new FileLogFetcher();
+ * fetcher.open(file, 0);
+ * 
+ * while (fetcher.fetch()) {
+ *     LogEvent event;
+ *     do {
+ *         event = decoder.decode(fetcher, context);
+ * 
+ *         // process log event.
+ *     } while (event != null);
+ * }
+ * // file ending reached.
+ * </pre>
+ * 
+ * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
+ * @version 1.0
+ */
+public final class FileLogFetcher extends LogFetcher {
+
+    public static final byte[] BINLOG_MAGIC = { -2, 0x62, 0x69, 0x6e };
+
+    private FileInputStream    fin;
+
+    public FileLogFetcher(){
+        super(DEFAULT_INITIAL_CAPACITY, DEFAULT_GROWTH_FACTOR);
+    }
+
+    public FileLogFetcher(final int initialCapacity){
+        super(initialCapacity, DEFAULT_GROWTH_FACTOR);
+    }
+
+    public FileLogFetcher(final int initialCapacity, final float growthFactor){
+        super(initialCapacity, growthFactor);
+    }
+
+    /**
+     * Open binlog file in local disk to fetch.
+     */
+    public void open(File file) throws FileNotFoundException, IOException {
+        open(file, 0L);
+    }
+
+    /**
+     * Open binlog file in local disk to fetch.
+     */
+    public void open(String filePath) throws FileNotFoundException, IOException {
+        open(new File(filePath), 0L);
+    }
+
+    /**
+     * Open binlog file in local disk to fetch.
+     */
+    public void open(String filePath, final long filePosition) throws FileNotFoundException, IOException {
+        open(new File(filePath), filePosition);
+    }
+
+    /**
+     * Open binlog file in local disk to fetch.
+     */
+    public void open(File file, final long filePosition) throws FileNotFoundException, IOException {
+        fin = new FileInputStream(file);
+
+        ensureCapacity(BIN_LOG_HEADER_SIZE);
+        if (BIN_LOG_HEADER_SIZE != fin.read(buffer, 0, BIN_LOG_HEADER_SIZE)) throw new IOException("No binlog file header");
+
+        if (buffer[0] != BINLOG_MAGIC[0] || buffer[1] != BINLOG_MAGIC[1] || buffer[2] != BINLOG_MAGIC[2]
+            || buffer[3] != BINLOG_MAGIC[3]) {
+            throw new IOException("Error binlog file header: "
+                                  + Arrays.toString(Arrays.copyOf(buffer, BIN_LOG_HEADER_SIZE)));
+        }
+
+        limit = 0;
+        origin = 0;
+        position = 0;
+
+        if (filePosition > BIN_LOG_HEADER_SIZE) {
+            final int maxFormatDescriptionEventLen = FormatDescriptionLogEvent.LOG_EVENT_MINIMAL_HEADER_LEN
+                                                     + FormatDescriptionLogEvent.ST_COMMON_HEADER_LEN_OFFSET
+                                                     + LogEvent.ENUM_END_EVENT + LogEvent.BINLOG_CHECKSUM_ALG_DESC_LEN
+                                                     + LogEvent.CHECKSUM_CRC32_SIGNATURE_LEN;
+
+            ensureCapacity(maxFormatDescriptionEventLen);
+            limit = fin.read(buffer, 0, maxFormatDescriptionEventLen);
+            limit = (int) getUint32(LogEvent.EVENT_LEN_OFFSET);
+            fin.getChannel().position(filePosition);
+        }
+    }
+
+    /**
+     * {@inheritDoc}
+     * 
+     * @see com.taobao.tddl.dbsync.binlog.LogFetcher#fetch()
+     */
+    public boolean fetch() throws IOException {
+        if (limit == 0) {
+            final int len = fin.read(buffer, 0, buffer.length);
+            if (len >= 0) {
+                limit += len;
+                position = 0;
+                origin = 0;
+
+                /* More binlog to fetch */
+                return true;
+            }
+        } else if (origin == 0) {
+            if (limit > buffer.length / 2) {
+                ensureCapacity(buffer.length + limit);
+            }
+            final int len = fin.read(buffer, limit, buffer.length - limit);
+            if (len >= 0) {
+                limit += len;
+
+                /* More binlog to fetch */
+                return true;
+            }
+        } else if (limit > 0) {
+            System.arraycopy(buffer, origin, buffer, 0, limit);
+            final int len = fin.read(buffer, limit, buffer.length - limit);
+            if (len >= 0) {
+                limit += len;
+                position -= origin;
+                origin = 0;
+
+                /* More binlog to fetch */
+                return true;
+            }
+        } else {
+            /* Should not happen. */
+            throw new IllegalArgumentException("Unexcepted limit: " + limit);
+        }
+
+        /* Reach binlog file end */
+        return false;
+    }
+
+    /**
+     * {@inheritDoc}
+     * 
+     * @see com.taobao.tddl.dbsync.binlog.LogFetcher#close()
+     */
+    public void close() throws IOException {
+        if (fin != null) fin.close();
+
+        fin = null;
+    }
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogBuffer.java

@@ -0,0 +1,1995 @@
+package com.taobao.tddl.dbsync.binlog;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.UnsupportedEncodingException;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.Arrays;
+import java.util.BitSet;
+
+/**
+ * TODO: Document Me!!
+ * 
+ * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
+ * @version 1.0
+ */
+public class LogBuffer
+{
+    protected byte[] buffer;
+
+    protected int    origin, limit;
+    protected int    position;
+
+    protected LogBuffer()
+    {
+    }
+
+    public LogBuffer(byte[] buffer, final int origin, final int limit)
+    {
+        if (origin + limit > buffer.length)
+            throw new IllegalArgumentException("capacity excceed: "
+                    + (origin + limit));
+
+        this.buffer = buffer;
+        this.origin = origin;
+        this.position = origin;
+        this.limit = limit;
+    }
+
+    /**
+     * Return n bytes in this buffer.
+     */
+    public final LogBuffer duplicate(final int pos, final int len)
+    {
+        if (pos + len > limit)
+            throw new IllegalArgumentException("limit excceed: " + (pos + len));
+
+        // XXX: Do momery copy avoid buffer modified.
+        final int off = origin + pos;
+        byte[] buf = Arrays.copyOfRange(buffer, off, off + len);
+        return new LogBuffer(buf, 0, len);
+    }
+
+    /**
+     * Return next n bytes in this buffer.
+     */
+    public final LogBuffer duplicate(final int len)
+    {
+        if (position + len > origin + limit)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (position + len - origin));
+
+        // XXX: Do momery copy avoid buffer modified.
+        final int end = position + len;
+        byte[] buf = Arrays.copyOfRange(buffer, position, end);
+        LogBuffer dupBuffer = new LogBuffer(buf, 0, len);
+        position = end;
+        return dupBuffer;
+    }
+
+    /**
+     * Return next n bytes in this buffer.
+     */
+    public final LogBuffer duplicate()
+    {
+        // XXX: Do momery copy avoid buffer modified.
+        byte[] buf = Arrays.copyOfRange(buffer, origin, origin + limit);
+        return new LogBuffer(buf, 0, limit);
+    }
+
+    /**
+     * Returns this buffer's capacity. </p>
+     * 
+     * @return The capacity of this buffer
+     */
+    public final int capacity()
+    {
+        return buffer.length;
+    }
+
+    /**
+     * Returns this buffer's position. </p>
+     * 
+     * @return The position of this buffer
+     */
+    public final int position()
+    {
+        return position - origin;
+    }
+
+    /**
+     * Sets this buffer's position. If the mark is defined and larger than the
+     * new position then it is discarded. </p>
+     * 
+     * @param newPosition The new position value; must be non-negative and no
+     *            larger than the current limit
+     * 
+     * @return This buffer
+     * 
+     * @throws IllegalArgumentException If the preconditions on
+     *             <tt>newPosition</tt> do not hold
+     */
+    public final LogBuffer position(final int newPosition)
+    {
+        if (newPosition > limit || newPosition < 0)
+            throw new IllegalArgumentException("limit excceed: " + newPosition);
+
+        this.position = origin + newPosition;
+        return this;
+    }
+
+    /**
+     * Forwards this buffer's position.
+     * 
+     * @param len The forward distance
+     * 
+     * @return This buffer
+     */
+    public final LogBuffer forward(final int len)
+    {
+        if (position + len > origin + limit)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (position + len - origin));
+
+        this.position += len;
+        return this;
+    }
+
+    /**
+     * Consume this buffer, moving origin and position.
+     * 
+     * @param len The consume distance
+     * 
+     * @return This buffer
+     */
+    public final LogBuffer consume(final int len)
+    {
+        if (limit > len)
+        {
+            limit -= len;
+            origin += len;
+            position = origin;
+            return this;
+        }
+        else if (limit == len)
+        {
+            limit = 0;
+            origin = 0;
+            position = 0;
+            return this;
+        }
+        else
+        {
+            /* Should not happen. */
+            throw new IllegalArgumentException("limit excceed: " + len);
+        }
+    }
+
+    /**
+     * Rewinds this buffer. The position is set to zero.
+     * 
+     * @return This buffer
+     */
+    public final LogBuffer rewind()
+    {
+        position = origin;
+        return this;
+    }
+
+    /**
+     * Returns this buffer's limit. </p>
+     * 
+     * @return The limit of this buffer
+     */
+    public final int limit()
+    {
+        return limit;
+    }
+
+    /**
+     * Sets this buffer's limit. If the position is larger than the new limit
+     * then it is set to the new limit. If the mark is defined and larger than
+     * the new limit then it is discarded. </p>
+     * 
+     * @param newLimit The new limit value; must be non-negative and no larger
+     *            than this buffer's capacity
+     * 
+     * @return This buffer
+     * 
+     * @throws IllegalArgumentException If the preconditions on
+     *             <tt>newLimit</tt> do not hold
+     */
+    public final LogBuffer limit(int newLimit)
+    {
+        if (origin + newLimit > buffer.length || newLimit < 0)
+            throw new IllegalArgumentException("capacity excceed: "
+                    + (origin + newLimit));
+
+        limit = newLimit;
+        return this;
+    }
+
+    /**
+     * Returns the number of elements between the current position and the
+     * limit. </p>
+     * 
+     * @return The number of elements remaining in this buffer
+     */
+    public final int remaining()
+    {
+        return limit + origin - position;
+    }
+
+    /**
+     * Tells whether there are any elements between the current position and the
+     * limit. </p>
+     * 
+     * @return <tt>true</tt> if, and only if, there is at least one element
+     *         remaining in this buffer
+     */
+    public final boolean hasRemaining()
+    {
+        return position < limit + origin;
+    }
+
+    /**
+     * Return 8-bit signed int from buffer.
+     */
+    public final int getInt8(final int pos)
+    {
+        if (pos >= limit || pos < 0)
+            throw new IllegalArgumentException("limit excceed: " + pos);
+
+        return buffer[origin + pos];
+    }
+
+    /**
+     * Return next 8-bit signed int from buffer.
+     */
+    public final int getInt8()
+    {
+        if (position >= origin + limit)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (position - origin));
+
+        return buffer[position++];
+    }
+
+    /**
+     * Return 8-bit unsigned int from buffer.
+     */
+    public final int getUint8(final int pos)
+    {
+        if (pos >= limit || pos < 0)
+            throw new IllegalArgumentException("limit excceed: " + pos);
+
+        return 0xff & buffer[origin + pos];
+    }
+
+    /**
+     * Return next 8-bit unsigned int from buffer.
+     */
+    public final int getUint8()
+    {
+        if (position >= origin + limit)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (position - origin));
+
+        return 0xff & buffer[position++];
+    }
+
+    /**
+     * Return 16-bit signed int from buffer. (little-endian)
+     * 
+     * @see mysql-5.1.60/include/my_global.h - sint2korr
+     */
+    public final int getInt16(final int pos)
+    {
+        final int position = origin + pos;
+
+        if (pos + 1 >= limit || pos < 0)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (pos < 0 ? pos : (pos + 1)));
+
+        byte[] buf = buffer;
+        return (0xff & buf[position]) | ((buf[position + 1]) << 8);
+    }
+
+    /**
+     * Return next 16-bit signed int from buffer. (little-endian)
+     * 
+     * @see mysql-5.1.60/include/my_global.h - sint2korr
+     */
+    public final int getInt16()
+    {
+        if (position + 1 >= origin + limit)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (position - origin + 1));
+
+        byte[] buf = buffer;
+        return (0xff & buf[position++]) | ((buf[position++]) << 8);
+    }
+    
+    /**
+     * Return 16-bit unsigned int from buffer. (little-endian)
+     * 
+     * @see mysql-5.1.60/include/my_global.h - uint2korr
+     */
+    public final int getUint16(final int pos)
+    {
+        final int position = origin + pos;
+
+        if (pos + 1 >= limit || pos < 0)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (pos < 0 ? pos : (pos + 1)));
+
+        byte[] buf = buffer;
+        return (0xff & buf[position]) | ((0xff & buf[position + 1]) << 8);
+    }
+
+    /**
+     * Return next 16-bit unsigned int from buffer. (little-endian)
+     * 
+     * @see mysql-5.1.60/include/my_global.h - uint2korr
+     */
+    public final int getUint16()
+    {
+        if (position + 1 >= origin + limit)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (position - origin + 1));
+
+        byte[] buf = buffer;
+        return (0xff & buf[position++]) | ((0xff & buf[position++]) << 8);
+    }
+    
+    /**
+     * Return 16-bit signed int from buffer. (big-endian)
+     * 
+     * @see mysql-5.6.10/include/myisampack.h - mi_sint2korr
+     */
+    public final int getBeInt16(final int pos)
+    {
+        final int position = origin + pos;
+
+        if (pos + 1 >= limit || pos < 0)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (pos < 0 ? pos : (pos + 1)));
+
+        byte[] buf = buffer;
+        return (0xff & buf[position + 1]) | ((buf[position]) << 8);
+    }
+
+    /**
+     * Return next 16-bit signed int from buffer. (big-endian)
+     * 
+     * @see mysql-5.1.60/include/my_global.h - mi_sint2korr
+     */
+    public final int getBeInt16()
+    {
+        if (position + 1 >= origin + limit)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (position - origin + 1));
+
+        byte[] buf = buffer;
+        return (buf[position++] << 8) | (0xff & buf[position++]);
+    }
+
+    /**
+     * Return 16-bit unsigned int from buffer. (big-endian)
+     * 
+     * @see mysql-5.6.10/include/myisampack.h - mi_usint2korr
+     */
+    public final int getBeUint16(final int pos)
+    {
+        final int position = origin + pos;
+
+        if (pos + 1 >= limit || pos < 0)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (pos < 0 ? pos : (pos + 1)));
+
+        byte[] buf = buffer;
+        return (0xff & buf[position + 1]) | ((0xff & buf[position]) << 8);
+    }
+
+    /**
+     * Return next 16-bit unsigned int from buffer. (big-endian)
+     * 
+     * @see mysql-5.6.10/include/myisampack.h - mi_usint2korr
+     */
+    public final int getBeUint16()
+    {
+        if (position + 1 >= origin + limit)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (position - origin + 1));
+
+        byte[] buf = buffer;
+        return ((0xff & buf[position++]) << 8) | (0xff & buf[position++]);
+    }
+    
+    /**
+     * Return 24-bit signed int from buffer. (little-endian)
+     * 
+     * @see mysql-5.1.60/include/my_global.h - sint3korr
+     */
+    public final int getInt24(final int pos)
+    {
+        final int position = origin + pos;
+
+        if (pos + 2 >= limit || pos < 0)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (pos < 0 ? pos : (pos + 2)));
+
+        byte[] buf = buffer;
+        return (0xff & buf[position]) | ((0xff & buf[position + 1]) << 8)
+                | ((buf[position + 2]) << 16);
+    }
+
+    /**
+     * Return next 24-bit signed int from buffer. (little-endian)
+     * 
+     * @see mysql-5.1.60/include/my_global.h - sint3korr
+     */
+    public final int getInt24()
+    {
+        if (position + 2 >= origin + limit)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (position - origin + 2));
+
+        byte[] buf = buffer;
+        return (0xff & buf[position++]) | ((0xff & buf[position++]) << 8)
+                | ((buf[position++]) << 16);
+    }
+    
+    /**
+     * Return 24-bit signed int from buffer. (big-endian)
+     * 
+     * @see mysql-5.6.10/include/myisampack.h - mi_usint3korr
+     */
+    public final int getBeInt24(final int pos)
+    {
+        final int position = origin + pos;
+
+        if (pos + 2 >= limit || pos < 0)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (pos < 0 ? pos : (pos + 2)));
+
+        byte[] buf = buffer;
+        return (0xff & buf[position + 2]) | ((0xff & buf[position + 1]) << 8)
+                | ((buf[position]) << 16);
+    }
+
+    /**
+     * Return next 24-bit signed int from buffer. (big-endian)
+     * 
+     * @see mysql-5.6.10/include/myisampack.h - mi_usint3korr
+     */
+    public final int getBeInt24()
+    {
+        if (position + 2 >= origin + limit)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (position - origin + 2));
+
+        byte[] buf = buffer;
+        return ((buf[position++]) << 16) | ((0xff & buf[position++]) << 8)
+                | (0xff & buf[position++]);
+    }
+    
+
+    /**
+     * Return 24-bit unsigned int from buffer. (little-endian)
+     * 
+     * @see mysql-5.1.60/include/my_global.h - uint3korr
+     */
+    public final int getUint24(final int pos)
+    {
+        final int position = origin + pos;
+
+        if (pos + 2 >= limit || pos < 0)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (pos < 0 ? pos : (pos + 2)));
+
+        byte[] buf = buffer;
+        return (0xff & buf[position]) | ((0xff & buf[position + 1]) << 8)
+                | ((0xff & buf[position + 2]) << 16);
+    }
+
+    /**
+     * Return next 24-bit unsigned int from buffer. (little-endian)
+     * 
+     * @see mysql-5.1.60/include/my_global.h - uint3korr
+     */
+    public final int getUint24()
+    {
+        if (position + 2 >= origin + limit)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (position - origin + 2));
+
+        byte[] buf = buffer;
+        return (0xff & buf[position++]) | ((0xff & buf[position++]) << 8)
+                | ((0xff & buf[position++]) << 16);
+    }
+    
+    /**
+     * Return 24-bit unsigned int from buffer. (big-endian)
+     * 
+     * @see mysql-5.6.10/include/myisampack.h - mi_usint3korr
+     */
+    public final int getBeUint24(final int pos)
+    {
+        final int position = origin + pos;
+
+        if (pos + 2 >= limit || pos < 0)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (pos < 0 ? pos : (pos + 2)));
+
+        byte[] buf = buffer;
+        return (0xff & buf[position + 2]) | ((0xff & buf[position + 1]) << 8)
+                | ((0xff & buf[position]) << 16);
+    }
+
+    /**
+     * Return next 24-bit unsigned int from buffer. (big-endian)
+     * 
+     * @see mysql-5.6.10/include/myisampack.h - mi_usint3korr
+     */
+    public final int getBeUint24()
+    {
+        if (position + 2 >= origin + limit)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (position - origin + 2));
+
+        byte[] buf = buffer;
+        return ((0xff & buf[position++]) << 16) | ((0xff & buf[position++]) << 8)
+                | (0xff & buf[position++]);
+    }
+
+    /**
+     * Return 32-bit signed int from buffer. (little-endian)
+     * 
+     * @see mysql-5.1.60/include/my_global.h - sint4korr
+     */
+    public final int getInt32(final int pos)
+    {
+        final int position = origin + pos;
+
+        if (pos + 3 >= limit || pos < 0)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (pos < 0 ? pos : (pos + 3)));
+
+        byte[] buf = buffer;
+        return (0xff & buf[position]) | ((0xff & buf[position + 1]) << 8)
+                | ((0xff & buf[position + 2]) << 16)
+                | ((buf[position + 3]) << 24);
+    }
+    
+    /**
+     * Return 32-bit signed int from buffer. (big-endian)
+     * 
+     * @see mysql-5.6.10/include/myisampack.h - mi_sint4korr
+     */
+    public final int getBeInt32(final int pos)
+    {
+        final int position = origin + pos;
+
+        if (pos + 3 >= limit || pos < 0)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (pos < 0 ? pos : (pos + 3)));
+
+        byte[] buf = buffer;
+        return (0xff & buf[position + 3]) | ((0xff & buf[position + 2]) << 8)
+                | ((0xff & buf[position + 1]) << 16)
+                | ((buf[position]) << 24);
+    }
+
+    /**
+     * Return next 32-bit signed int from buffer. (little-endian)
+     * 
+     * @see mysql-5.1.60/include/my_global.h - sint4korr
+     */
+    public final int getInt32()
+    {
+        if (position + 3 >= origin + limit)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (position - origin + 3));
+
+        byte[] buf = buffer;
+        return (0xff & buf[position++]) | ((0xff & buf[position++]) << 8)
+                | ((0xff & buf[position++]) << 16) | ((buf[position++]) << 24);
+    }
+
+    /**
+     * Return next 32-bit signed int from buffer. (big-endian)
+     * 
+     * @see mysql-5.6.10/include/myisampack.h - mi_sint4korr
+     */
+    public final int getBeInt32()
+    {
+        if (position + 3 >= origin + limit)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (position - origin + 3));
+
+        byte[] buf = buffer;
+        return ((buf[position++]) << 24) | ((0xff & buf[position++]) << 16) | ((0xff & buf[position++]) << 8)
+               | (0xff & buf[position++]);
+    }
+    
+    /**
+     * Return 32-bit unsigned int from buffer. (little-endian)
+     * 
+     * @see mysql-5.1.60/include/my_global.h - uint4korr
+     */
+    public final long getUint32(final int pos)
+    {
+        final int position = origin + pos;
+
+        if (pos + 3 >= limit || pos < 0)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (pos < 0 ? pos : (pos + 3)));
+
+        byte[] buf = buffer;
+        return ((long) (0xff & buf[position]))
+                | ((long) (0xff & buf[position + 1]) << 8)
+                | ((long) (0xff & buf[position + 2]) << 16)
+                | ((long) (0xff & buf[position + 3]) << 24);
+    }
+
+    /**
+     * Return 32-bit unsigned int from buffer. (big-endian)
+     * 
+     * @see mysql-5.6.10/include/myisampack.h - mi_usint4korr
+     */
+    public final long getBeUint32(final int pos)
+    {
+        final int position = origin + pos;
+
+        if (pos + 3 >= limit || pos < 0)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (pos < 0 ? pos : (pos + 3)));
+
+        byte[] buf = buffer;
+        return ((long) (0xff & buf[position + 3]))
+                | ((long) (0xff & buf[position + 2]) << 8)
+                | ((long) (0xff & buf[position + 1]) << 16)
+                | ((long) (0xff & buf[position]) << 24);
+    }
+
+    
+    /**
+     * Return next 32-bit unsigned int from buffer. (little-endian)
+     * 
+     * @see mysql-5.1.60/include/my_global.h - uint4korr
+     */
+    public final long getUint32()
+    {
+        if (position + 3 >= origin + limit)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (position - origin + 3));
+
+        byte[] buf = buffer;
+        return ((long) (0xff & buf[position++]))
+                | ((long) (0xff & buf[position++]) << 8)
+                | ((long) (0xff & buf[position++]) << 16)
+                | ((long) (0xff & buf[position++]) << 24);
+    }
+    
+    /**
+     * Return next 32-bit unsigned int from buffer. (big-endian)
+     * 
+     * @see mysql-5.6.10/include/myisampack.h - mi_uint4korr
+     */
+    public final long getBeUint32()
+    {
+        if (position + 3 >= origin + limit)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (position - origin + 3));
+
+        byte[] buf = buffer;
+        return ((long) (0xff & buf[position++]) << 24)
+                | ((long) (0xff & buf[position++]) << 16)
+                | ((long) (0xff & buf[position++]) << 8)
+                | ((long) (0xff & buf[position++]));
+    }
+    
+    /**
+     * Return 40-bit unsigned int from buffer. (little-endian)
+     */
+    public final long getUlong40(final int pos)
+    {
+        final int position = origin + pos;
+
+        if (pos + 4 >= limit || pos < 0)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (pos < 0 ? pos : (pos + 4)));
+
+        byte[] buf = buffer;
+        return ((long) (0xff & buf[position]))
+                | ((long) (0xff & buf[position + 1]) << 8)
+                | ((long) (0xff & buf[position + 2]) << 16)
+                | ((long) (0xff & buf[position + 3]) << 24)
+                | ((long) (0xff & buf[position + 4]) << 32);
+    }
+
+    /**
+     * Return next 40-bit unsigned int from buffer. (little-endian)
+     */
+    public final long getUlong40()
+    {
+        if (position + 4 >= origin + limit)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (position - origin + 4));
+
+        byte[] buf = buffer;
+        return ((long) (0xff & buf[position++]))
+                | ((long) (0xff & buf[position++]) << 8)
+                | ((long) (0xff & buf[position++]) << 16)
+                | ((long) (0xff & buf[position++]) << 24)
+                | ((long) (0xff & buf[position++]) << 32);
+    }
+
+    
+    /**
+     * Return 40-bit unsigned int from buffer. (big-endian)
+     * 
+     * @see mysql-5.6.10/include/myisampack.h - mi_uint5korr
+     */
+    public final long getBeUlong40(final int pos)
+    {
+        final int position = origin + pos;
+
+        if (pos + 4 >= limit || pos < 0)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (pos < 0 ? pos : (pos + 4)));
+
+        byte[] buf = buffer;
+        return ((long) (0xff & buf[position + 4]))
+                | ((long) (0xff & buf[position + 3]) << 8)
+                | ((long) (0xff & buf[position + 2]) << 16)
+                | ((long) (0xff & buf[position + 1]) << 24)
+                | ((long) (0xff & buf[position]) << 32);
+    }
+
+    /**
+     * Return next 40-bit unsigned int from buffer. (big-endian)
+     * 
+     * @see mysql-5.6.10/include/myisampack.h - mi_uint5korr
+     */
+    public final long getBeUlong40()
+    {
+        if (position + 4 >= origin + limit)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (position - origin + 4));
+
+        byte[] buf = buffer;
+        return ((long) (0xff & buf[position++]) << 32)
+                | ((long) (0xff & buf[position++]) << 24)
+                | ((long) (0xff & buf[position++]) << 16)
+                | ((long) (0xff & buf[position++]) << 8)
+                | ((long) (0xff & buf[position++]));
+    }
+
+    /**
+     * Return 48-bit signed long from buffer. (little-endian)
+     * 
+     * @see mysql-5.1.60/include/my_global.h - sint6korr
+     */
+    public final long getLong48(final int pos)
+    {
+        final int position = origin + pos;
+
+        if (pos + 5 >= limit || pos < 0)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (pos < 0 ? pos : (pos + 5)));
+
+        byte[] buf = buffer;
+        return ((long) (0xff & buf[position]))
+                | ((long) (0xff & buf[position + 1]) << 8)
+                | ((long) (0xff & buf[position + 2]) << 16)
+                | ((long) (0xff & buf[position + 3]) << 24)
+                | ((long) (0xff & buf[position + 4]) << 32)
+                | ((long) (buf[position + 5]) << 40);
+    }
+    
+    /**
+     * Return 48-bit signed long from buffer. (big-endian)
+     * 
+     * @see mysql-5.6.10/include/myisampack.h - mi_sint6korr
+     */
+    public final long getBeLong48(final int pos)
+    {
+        final int position = origin + pos;
+
+        if (pos + 5 >= limit || pos < 0)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (pos < 0 ? pos : (pos + 5)));
+
+        byte[] buf = buffer;
+        return ((long) (0xff & buf[position + 5]))
+                | ((long) (0xff & buf[position + 4]) << 8)
+                | ((long) (0xff & buf[position + 3]) << 16)
+                | ((long) (0xff & buf[position + 2]) << 24)
+                | ((long) (0xff & buf[position + 1]) << 32)
+                | ((long) (buf[position]) << 40);
+    }
+
+    /**
+     * Return next 48-bit signed long from buffer. (little-endian)
+     * 
+     * @see mysql-5.1.60/include/my_global.h - sint6korr
+     */
+    public final long getLong48()
+    {
+        if (position + 5 >= origin + limit)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (position - origin + 5));
+
+        byte[] buf = buffer;
+        return ((long) (0xff & buf[position++]))
+                | ((long) (0xff & buf[position++]) << 8)
+                | ((long) (0xff & buf[position++]) << 16)
+                | ((long) (0xff & buf[position++]) << 24)
+                | ((long) (0xff & buf[position++]) << 32)
+                | ((long) (buf[position++]) << 40);
+    }
+    
+    /**
+     * Return next 48-bit signed long from buffer. (Big-endian)
+     * 
+     * @see mysql-5.6.10/include/myisampack.h - mi_sint6korr
+     */
+    public final long getBeLong48()
+    {
+        if (position + 5 >= origin + limit)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (position - origin + 5));
+
+        byte[] buf = buffer;
+        return ((long) (buf[position++]) << 40)
+                | ((long) (0xff & buf[position++]) << 32)
+                | ((long) (0xff & buf[position++]) << 24)
+                | ((long) (0xff & buf[position++]) << 16)
+                | ((long) (0xff & buf[position++]) << 8)
+                | ((long) (0xff & buf[position++]));
+    }
+
+    /**
+     * Return 48-bit unsigned long from buffer. (little-endian)
+     * 
+     * @see mysql-5.1.60/include/my_global.h - uint6korr
+     */
+    public final long getUlong48(final int pos)
+    {
+        final int position = origin + pos;
+
+        if (pos + 5 >= limit || pos < 0)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (pos < 0 ? pos : (pos + 5)));
+
+        byte[] buf = buffer;
+        return ((long) (0xff & buf[position]))
+                | ((long) (0xff & buf[position + 1]) << 8)
+                | ((long) (0xff & buf[position + 2]) << 16)
+                | ((long) (0xff & buf[position + 3]) << 24)
+                | ((long) (0xff & buf[position + 4]) << 32)
+                | ((long) (0xff & buf[position + 5]) << 40);
+    }
+    
+    /**
+     * Return 48-bit unsigned long from buffer. (big-endian)
+     * 
+     * @see mysql-5.6.10/include/myisampack.h - mi_uint6korr
+     */
+    public final long getBeUlong48(final int pos)
+    {
+        final int position = origin + pos;
+
+        if (pos + 5 >= limit || pos < 0)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (pos < 0 ? pos : (pos + 5)));
+
+        byte[] buf = buffer;
+        return ((long) (0xff & buf[position + 5]))
+                | ((long) (0xff & buf[position + 4]) << 8)
+                | ((long) (0xff & buf[position + 3]) << 16)
+                | ((long) (0xff & buf[position + 2]) << 24)
+                | ((long) (0xff & buf[position + 1]) << 32)
+                | ((long) (0xff & buf[position]) << 40);
+    }
+
+    /**
+     * Return next 48-bit unsigned long from buffer. (little-endian)
+     * 
+     * @see mysql-5.1.60/include/my_global.h - uint6korr
+     */
+    public final long getUlong48()
+    {
+        if (position + 5 >= origin + limit)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (position - origin + 5));
+
+        byte[] buf = buffer;
+        return ((long) (0xff & buf[position++]))
+                | ((long) (0xff & buf[position++]) << 8)
+                | ((long) (0xff & buf[position++]) << 16)
+                | ((long) (0xff & buf[position++]) << 24)
+                | ((long) (0xff & buf[position++]) << 32)
+                | ((long) (0xff & buf[position++]) << 40);
+    }
+    
+    /**
+     * Return next 48-bit unsigned long from buffer. (big-endian)
+     * 
+     * @see mysql-5.6.10/include/myisampack.h - mi_uint6korr
+     */
+    public final long getBeUlong48()
+    {
+        if (position + 5 >= origin + limit)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (position - origin + 5));
+
+        byte[] buf = buffer;
+        return ((long) (0xff & buf[position++]) << 40)
+                | ((long) (0xff & buf[position++]) << 32)
+                | ((long) (0xff & buf[position++]) << 24)
+                | ((long) (0xff & buf[position++]) << 16)
+                | ((long) (0xff & buf[position++]) << 8)
+                | ((long) (0xff & buf[position++]));
+    }
+    
+    /**
+     * Return 56-bit unsigned int from buffer. (little-endian)
+     * 
+     */
+    public final long getUlong56(final int pos)
+    {
+        final int position = origin + pos;
+
+        if (pos + 6 >= limit || pos < 0)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (pos < 0 ? pos : (pos + 6)));
+
+        byte[] buf = buffer;
+        return ((long) (0xff & buf[position]))
+                | ((long) (0xff & buf[position + 1]) << 8)
+                | ((long) (0xff & buf[position + 2]) << 16)
+                | ((long) (0xff & buf[position + 3]) << 24)
+                | ((long) (0xff & buf[position + 4]) << 32)
+                | ((long) (0xff & buf[position + 5]) << 40)
+                | ((long) (0xff & buf[position + 6]) << 48);
+    }
+
+    /**
+     * Return next 56-bit unsigned int from buffer. (little-endian)
+     * 
+     */
+    public final long getUlong56()
+    {
+        if (position + 6 >= origin + limit)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (position - origin + 6));
+
+        byte[] buf = buffer;
+        return ((long) (0xff & buf[position++]) )
+                | ((long) (0xff & buf[position++]) << 8)
+                | ((long) (0xff & buf[position++]) << 16)
+                | ((long) (0xff & buf[position++]) << 24)
+                | ((long) (0xff & buf[position++]) << 32)
+                | ((long) (0xff & buf[position++]) << 40)
+                | ((long) (0xff & buf[position++]) << 48);
+    }
+    
+    /**
+     * Return 56-bit unsigned int from buffer. (big-endian)
+     * 
+     */
+    public final long getBeUlong56(final int pos)
+    {
+        final int position = origin + pos;
+
+        if (pos + 6 >= limit || pos < 0)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (pos < 0 ? pos : (pos + 6)));
+
+        byte[] buf = buffer;
+        return ((long) (0xff & buf[position + 6]))
+                | ((long) (0xff & buf[position + 5]) << 8)
+                | ((long) (0xff & buf[position + 4]) << 16)
+                | ((long) (0xff & buf[position + 3]) << 24)
+                | ((long) (0xff & buf[position + 2]) << 32)
+                | ((long) (0xff & buf[position + 1]) << 40)
+                | ((long) (0xff & buf[position]) << 48);
+    }
+
+    /**
+     * Return next 56-bit unsigned int from buffer. (big-endian)
+     * 
+     */
+    public final long getBeUlong56()
+    {
+        if (position + 6 >= origin + limit)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (position - origin + 6));
+
+        byte[] buf = buffer;
+        return ((long) (0xff & buf[position++]) << 48)
+                | ((long) (0xff & buf[position++]) << 40)
+                | ((long) (0xff & buf[position++]) << 32)
+                | ((long) (0xff & buf[position++]) << 24)
+                | ((long) (0xff & buf[position++]) << 16)
+                | ((long) (0xff & buf[position++]) << 8)
+                | ((long) (0xff & buf[position++]));
+    }
+
+    /**
+     * Return 64-bit signed long from buffer. (little-endian)
+     * 
+     * @see mysql-5.1.60/include/my_global.h - sint8korr
+     */
+    public final long getLong64(final int pos)
+    {
+        final int position = origin + pos;
+
+        if (pos + 7 >= limit || pos < 0)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (pos < 0 ? pos : (pos + 7)));
+
+        byte[] buf = buffer;
+        return ((long) (0xff & buf[position]))
+                | ((long) (0xff & buf[position + 1]) << 8)
+                | ((long) (0xff & buf[position + 2]) << 16)
+                | ((long) (0xff & buf[position + 3]) << 24)
+                | ((long) (0xff & buf[position + 4]) << 32)
+                | ((long) (0xff & buf[position + 5]) << 40)
+                | ((long) (0xff & buf[position + 6]) << 48)
+                | ((long) (buf[position + 7]) << 56);
+    }
+    
+    /**
+     * Return 64-bit signed long from buffer. (big-endian)
+     * 
+     * @see mysql-5.6.10/include/myisampack.h - mi_sint8korr
+     */
+    public final long getBeLong64(final int pos)
+    {
+        final int position = origin + pos;
+
+        if (pos + 7 >= limit || pos < 0)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (pos < 0 ? pos : (pos + 7)));
+
+        byte[] buf = buffer;
+        return ((long) (0xff & buf[position + 7]))
+                | ((long) (0xff & buf[position + 6]) << 8)
+                | ((long) (0xff & buf[position + 5]) << 16)
+                | ((long) (0xff & buf[position + 4]) << 24)
+                | ((long) (0xff & buf[position + 3]) << 32)
+                | ((long) (0xff & buf[position + 2]) << 40)
+                | ((long) (0xff & buf[position + 1]) << 48)
+                | ((long) (buf[position]) << 56);
+    }
+
+    /**
+     * Return next 64-bit signed long from buffer. (little-endian)
+     * 
+     * @see mysql-5.1.60/include/my_global.h - sint8korr
+     */
+    public final long getLong64()
+    {
+        if (position + 7 >= origin + limit)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (position - origin + 7));
+
+        byte[] buf = buffer;
+        return ((long) (0xff & buf[position++]))
+                | ((long) (0xff & buf[position++]) << 8)
+                | ((long) (0xff & buf[position++]) << 16)
+                | ((long) (0xff & buf[position++]) << 24)
+                | ((long) (0xff & buf[position++]) << 32)
+                | ((long) (0xff & buf[position++]) << 40)
+                | ((long) (0xff & buf[position++]) << 48)
+                | ((long) (buf[position++]) << 56);
+    }
+    
+    /**
+     * Return next 64-bit signed long from buffer. (big-endian)
+     * 
+     * @see mysql-5.6.10/include/myisampack.h - mi_sint8korr
+     */
+    public final long getBeLong64()
+    {
+        if (position + 7 >= origin + limit)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (position - origin + 7));
+
+        byte[] buf = buffer;
+        return ((long) (buf[position++]) << 56)
+                | ((long) (0xff & buf[position++]) << 48)
+                | ((long) (0xff & buf[position++]) << 40)
+                | ((long) (0xff & buf[position++]) << 32)
+                | ((long) (0xff & buf[position++]) << 24)
+                | ((long) (0xff & buf[position++]) << 16)
+                | ((long) (0xff & buf[position++]) << 8)
+                | ((long) (0xff & buf[position++]));
+    }
+
+    /* The max ulonglong - 0x ff ff ff ff ff ff ff ff */
+    public static final BigInteger BIGINT_MAX_VALUE = new BigInteger(
+                                                            "18446744073709551615");
+
+    /**
+     * Return 64-bit unsigned long from buffer. (little-endian)
+     * 
+     * @see mysql-5.1.60/include/my_global.h - uint8korr
+     */
+    public final BigInteger getUlong64(final int pos)
+    {
+        final long long64 = getLong64(pos);
+
+        return (long64 >= 0) ? BigInteger.valueOf(long64)
+                : BIGINT_MAX_VALUE.add(BigInteger.valueOf(1 + long64));
+    }
+    
+    /**
+     * Return 64-bit unsigned long from buffer. (big-endian)
+     * 
+     * @see mysql-5.6.10/include/myisampack.h - mi_uint8korr
+     */
+    public final BigInteger getBeUlong64(final int pos)
+    {
+        final long long64 = getBeLong64(pos);
+
+        return (long64 >= 0) ? BigInteger.valueOf(long64)
+                : BIGINT_MAX_VALUE.add(BigInteger.valueOf(1 + long64));
+    }
+
+    /**
+     * Return next 64-bit unsigned long from buffer. (little-endian)
+     * 
+     * @see mysql-5.1.60/include/my_global.h - uint8korr
+     */
+    public final BigInteger getUlong64()
+    {
+        final long long64 = getLong64();
+
+        return (long64 >= 0) ? BigInteger.valueOf(long64)
+                : BIGINT_MAX_VALUE.add(BigInteger.valueOf(1 + long64));
+    }
+    
+    /**
+     * Return next 64-bit unsigned long from buffer. (big-endian)
+     * 
+     * @see mysql-5.6.10/include/myisampack.h - mi_uint8korr
+     */
+    public final BigInteger getBeUlong64()
+    {
+        final long long64 = getBeLong64();
+
+        return (long64 >= 0) ? BigInteger.valueOf(long64)
+                : BIGINT_MAX_VALUE.add(BigInteger.valueOf(1 + long64));
+    }
+
+    /**
+     * Return 32-bit float from buffer. (little-endian)
+     * 
+     * @see mysql-5.1.60/include/my_global.h - float4get
+     */
+    public final float getFloat32(final int pos)
+    {
+        return Float.intBitsToFloat(getInt32(pos));
+    }
+
+    /**
+     * Return next 32-bit float from buffer. (little-endian)
+     * 
+     * @see mysql-5.1.60/include/my_global.h - float4get
+     */
+    public final float getFloat32()
+    {
+        return Float.intBitsToFloat(getInt32());
+    }
+
+    /**
+     * Return 64-bit double from buffer. (little-endian)
+     * 
+     * @see mysql-5.1.60/include/my_global.h - float8get
+     */
+    public final double getDouble64(final int pos)
+    {
+        return Double.longBitsToDouble(getLong64(pos));
+    }
+
+    /**
+     * Return next 64-bit double from buffer. (little-endian)
+     * 
+     * @see mysql-5.1.60/include/my_global.h - float8get
+     */
+    public final double getDouble64()
+    {
+        return Double.longBitsToDouble(getLong64());
+    }
+
+    public static final long NULL_LENGTH = ((long) ~0);
+
+    /**
+     * Return packed number from buffer. (little-endian)
+     * 
+     * A Packed Integer has the capacity of storing up to 8-byte integers, while
+     * small integers still can use 1, 3, or 4 bytes. The value of the first
+     * byte determines how to read the number, according to the following table.
+     * 
+     * <ul>
+     * <li>0-250 The first byte is the number (in the range 0-250). No
+     * additional bytes are used.</li>
+     * <li>252 Two more bytes are used. The number is in the range 251-0xffff.</li>
+     * <li>253 Three more bytes are used. The number is in the range
+     * 0xffff-0xffffff.</li>
+     * <li>254 Eight more bytes are used. The number is in the range
+     * 0xffffff-0xffffffffffffffff.</li>
+     * </ul>
+     * 
+     * That representation allows a first byte value of 251 to represent the SQL
+     * NULL value.
+     */
+    public final long getPackedLong(final int pos)
+    {
+        final int lead = getUint8(pos);
+        if (lead < 251)
+            return lead;
+
+        switch (lead)
+        {
+        case 251:
+            return NULL_LENGTH;
+        case 252:
+            return getUint16(pos + 1);
+        case 253:
+            return getUint24(pos + 1);
+        default: /* Must be 254 when here */
+            return getUint32(pos + 1);
+        }
+    }
+
+    /**
+     * Return next packed number from buffer. (little-endian)
+     * 
+     * @see LogBuffer#getPackedLong(int)
+     */
+    public final long getPackedLong()
+    {
+        final int lead = getUint8();
+        if (lead < 251)
+            return lead;
+
+        switch (lead)
+        {
+        case 251:
+            return NULL_LENGTH;
+        case 252:
+            return getUint16();
+        case 253:
+            return getUint24();
+        default: /* Must be 254 when here */
+            final long value = getUint32();
+            position += 4; /* ignore other */
+            return value;
+        }
+    }
+
+    /* default ANSI charset */
+    public static final String ISO_8859_1 = "ISO-8859-1";
+
+    /**
+     * Return fix length string from buffer.
+     */
+    public final String getFixString(final int pos, final int len)
+    {
+        return getFixString(pos, len, ISO_8859_1);
+    }
+
+    /**
+     * Return next fix length string from buffer.
+     */
+    public final String getFixString(final int len)
+    {
+        return getFixString(len, ISO_8859_1);
+    }
+
+    /**
+     * Return fix length string from buffer.
+     */
+    public final String getFixString(final int pos, final int len,
+            String charsetName)
+    {
+        if (pos + len > limit || pos < 0)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (pos < 0 ? pos : (pos + len)));
+
+        final int from = origin + pos;
+        final int end = from + len;
+        byte[] buf = buffer;
+        int found = from;
+        for (; (found < end) && buf[found] != '\0'; found++)
+            /* empty loop */;
+
+        try
+        {
+            return new String(buf, from, found - from, charsetName);
+        }
+        catch (UnsupportedEncodingException e)
+        {
+            throw new IllegalArgumentException("Unsupported encoding: "
+                    + charsetName, e);
+        }
+    }
+
+    /**
+     * Return next fix length string from buffer.
+     */
+    public final String getFixString(final int len, String charsetName)
+    {
+        if (position + len > origin + limit)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (position + len - origin));
+
+        final int from = position;
+        final int end = from + len;
+        byte[] buf = buffer;
+        int found = from;
+        for (; (found < end) && buf[found] != '\0'; found++)
+            /* empty loop */;
+
+        try
+        {
+            String string = new String(buf, from, found - from, charsetName);
+            position += len;
+            return string;
+        }
+        catch (UnsupportedEncodingException e)
+        {
+            throw new IllegalArgumentException("Unsupported encoding: "
+                    + charsetName, e);
+        }
+    }
+
+    /**
+     * Return fix-length string from buffer without null-terminate checking.
+     * 
+     * Fix bug #17 {@link https://github.com/AlibabaTech/canal/issues/17 }
+     */
+    public final String getFullString(final int pos, final int len,
+            String charsetName)
+    {
+        if (pos + len > limit || pos < 0)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (pos < 0 ? pos : (pos + len)));
+
+        try
+        {
+            return new String(buffer, origin + pos, len, charsetName);
+        }
+        catch (UnsupportedEncodingException e)
+        {
+            throw new IllegalArgumentException("Unsupported encoding: "
+                    + charsetName, e);
+        }
+    }
+
+    /**
+     * Return next fix-length string from buffer without null-terminate
+     * checking.
+     * 
+     * Fix bug #17 {@link https://github.com/AlibabaTech/canal/issues/17 }
+     */
+    public final String getFullString(final int len, String charsetName)
+    {
+        if (position + len > origin + limit)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (position + len - origin));
+
+        try
+        {
+            String string = new String(buffer, position, len, charsetName);
+            position += len;
+            return string;
+        }
+        catch (UnsupportedEncodingException e)
+        {
+            throw new IllegalArgumentException("Unsupported encoding: "
+                    + charsetName, e);
+        }
+    }
+
+    /**
+     * Return dynamic length string from buffer.
+     */
+    public final String getString(final int pos)
+    {
+        return getString(pos, ISO_8859_1);
+    }
+
+    /**
+     * Return next dynamic length string from buffer.
+     */
+    public final String getString()
+    {
+        return getString(ISO_8859_1);
+    }
+
+    /**
+     * Return dynamic length string from buffer.
+     */
+    public final String getString(final int pos, String charsetName)
+    {
+        if (pos >= limit || pos < 0)
+            throw new IllegalArgumentException("limit excceed: " + pos);
+
+        byte[] buf = buffer;
+        final int len = (0xff & buf[origin + pos]);
+        if (pos + len + 1 > limit)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (pos + len + 1));
+
+        try
+        {
+            return new String(buf, origin + pos + 1, len, charsetName);
+        }
+        catch (UnsupportedEncodingException e)
+        {
+            throw new IllegalArgumentException("Unsupported encoding: "
+                    + charsetName, e);
+        }
+    }
+
+    /**
+     * Return next dynamic length string from buffer.
+     */
+    public final String getString(String charsetName)
+    {
+        if (position >= origin + limit)
+            throw new IllegalArgumentException("limit excceed: " + position);
+
+        byte[] buf = buffer;
+        final int len = (0xff & buf[position]);
+        if (position + len + 1 > origin + limit)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (position + len + 1 - origin));
+
+        try
+        {
+            String string = new String(buf, position + 1, len, charsetName);
+            position += len + 1;
+            return string;
+        }
+        catch (UnsupportedEncodingException e)
+        {
+            throw new IllegalArgumentException("Unsupported encoding: "
+                    + charsetName, e);
+        }
+    }
+
+    /**
+     * Return 16-bit signed int from buffer. (big-endian)
+     * 
+     * @see mysql-5.1.60/include/myisampack.h - mi_sint2korr
+     */
+    private static final int getInt16BE(byte[] buffer, final int pos)
+    {
+        return ((buffer[pos]) << 8) | (0xff & buffer[pos + 1]);
+    }
+
+    /**
+     * Return 24-bit signed int from buffer. (big-endian)
+     * 
+     * @see mysql-5.1.60/include/myisampack.h - mi_sint3korr
+     */
+    private static final int getInt24BE(byte[] buffer, final int pos)
+    {
+        return (buffer[pos] << 16) | ((0xff & buffer[pos + 1]) << 8)
+                | (0xff & buffer[pos + 2]);
+    }
+
+    /**
+     * Return 32-bit signed int from buffer. (big-endian)
+     * 
+     * @see mysql-5.1.60/include/myisampack.h - mi_sint4korr
+     */
+    private static final int getInt32BE(byte[] buffer, final int pos)
+    {
+        return (buffer[pos] << 24) | ((0xff & buffer[pos + 1]) << 16)
+                | ((0xff & buffer[pos + 2]) << 8) | (0xff & buffer[pos + 3]);
+    }
+
+    /* decimal representation */
+    public static final int DIG_PER_DEC1  = 9;
+    public static final int DIG_BASE      = 1000000000;
+    public static final int DIG_MAX       = DIG_BASE - 1;
+    public static final int dig2bytes[]   = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 4 };
+    public static final int powers10[]    = { 1, 10, 100, 1000, 10000, 100000,
+            1000000, 10000000, 100000000, 1000000000 };
+
+    public static final int DIG_PER_INT32 = 9;
+    public static final int SIZE_OF_INT32 = 4;
+
+    /**
+     * Return big decimal from buffer.
+     * 
+     * @see mysql-5.1.60/strings/decimal.c - bin2decimal()
+     */
+    public final BigDecimal getDecimal(final int pos, final int precision,
+            final int scale)
+    {
+        final int intg = precision - scale;
+        final int frac = scale;
+        final int intg0 = intg / DIG_PER_INT32;
+        final int frac0 = frac / DIG_PER_INT32;
+        final int intg0x = intg - intg0 * DIG_PER_INT32;
+        final int frac0x = frac - frac0 * DIG_PER_INT32;
+
+        final int binSize = intg0 * SIZE_OF_INT32 + dig2bytes[intg0x] + frac0
+                * SIZE_OF_INT32 + dig2bytes[frac0x];
+        if (pos + binSize > limit || pos < 0)
+        {
+            throw new IllegalArgumentException("limit excceed: "
+                    + (pos < 0 ? pos : (pos + binSize)));
+        }
+        return getDecimal0(origin + pos, intg, frac, // NL
+                intg0, frac0, intg0x, frac0x);
+    }
+
+    /**
+     * Return next big decimal from buffer.
+     * 
+     * @see mysql-5.1.60/strings/decimal.c - bin2decimal()
+     */
+    public final BigDecimal getDecimal(final int precision, final int scale)
+    {
+        final int intg = precision - scale;
+        final int frac = scale;
+        final int intg0 = intg / DIG_PER_INT32;
+        final int frac0 = frac / DIG_PER_INT32;
+        final int intg0x = intg - intg0 * DIG_PER_INT32;
+        final int frac0x = frac - frac0 * DIG_PER_INT32;
+
+        final int binSize = intg0 * SIZE_OF_INT32 + dig2bytes[intg0x] + frac0
+                * SIZE_OF_INT32 + dig2bytes[frac0x];
+        if (position + binSize > origin + limit)
+        {
+            throw new IllegalArgumentException("limit excceed: "
+                    + (position + binSize - origin));
+        }
+
+        BigDecimal decimal = getDecimal0(position, intg, frac, // NL
+                intg0, frac0, intg0x, frac0x);
+        position += binSize;
+        return decimal;
+    }
+
+    /**
+     * Return big decimal from buffer.
+     * 
+     * <pre>
+     * Decimal representation in binlog seems to be as follows:
+     * 
+     * 1st bit - sign such that set == +, unset == -
+     * every 4 bytes represent 9 digits in big-endian order, so that
+     * if you print the values of these quads as big-endian integers one after
+     * another, you get the whole number string representation in decimal. What
+     * remains is to put a sign and a decimal dot.
+     * 
+     * 80 00 00 05 1b 38 b0 60 00 means:
+     * 
+     *   0x80 - positive 
+     *   0x00000005 - 5
+     *   0x1b38b060 - 456700000
+     *   0x00       - 0
+     * 
+     * 54567000000 / 10^{10} = 5.4567
+     * </pre>
+     * 
+     * @see mysql-5.1.60/strings/decimal.c - bin2decimal()
+     * @see mysql-5.1.60/strings/decimal.c - decimal2string()
+     */
+    private final BigDecimal getDecimal0(final int begin, final int intg,
+            final int frac, final int intg0, final int frac0, final int intg0x,
+            final int frac0x)
+    {
+        final int mask = ((buffer[begin] & 0x80) == 0x80) ? 0 : -1;
+        int from = begin;
+
+        /* max string length */
+        final int len = ((mask != 0) ? 1 : 0) + ((intg != 0) ? intg : 1) // NL
+                + ((frac != 0) ? 1 : 0) + frac;
+        char[] buf = new char[len];
+        int pos = 0;
+
+        if (mask != 0) /* decimal sign */
+            buf[pos++] = ('-');
+
+        final byte[] d_copy = buffer;
+        d_copy[begin] ^= 0x80; /* clear sign */
+        int mark = pos;
+
+        if (intg0x != 0)
+        {
+            final int i = dig2bytes[intg0x];
+            int x = 0;
+            switch (i)
+            {
+            case 1:
+                x = d_copy[from] /* one byte */;
+                break;
+            case 2:
+                x = getInt16BE(d_copy, from);
+                break;
+            case 3:
+                x = getInt24BE(d_copy, from);
+                break;
+            case 4:
+                x = getInt32BE(d_copy, from);
+                break;
+            }
+            from += i;
+            x ^= mask;
+            if (x < 0 || x >= powers10[intg0x + 1])
+            {
+                throw new IllegalArgumentException("bad format, x exceed: " + x
+                        + ", " + powers10[intg0x + 1]);
+            }
+            if (x != 0 /* !digit || x != 0 */)
+            {
+                for (int j = intg0x; j > 0; j--)
+                {
+                    final int divisor = powers10[j - 1];
+                    final int y = x / divisor;
+                    if (mark < pos || y != 0)
+                    {
+                        buf[pos++] = ((char) ('0' + y));
+                    }
+                    x -= y * divisor;
+                }
+            }
+        }
+
+        for (final int stop = from + intg0 * SIZE_OF_INT32; from < stop; from += SIZE_OF_INT32)
+        {
+            int x = getInt32BE(d_copy, from);
+            x ^= mask;
+            if (x < 0 || x > DIG_MAX)
+            {
+                throw new IllegalArgumentException("bad format, x exceed: " + x
+                        + ", " + DIG_MAX);
+            }
+            if (x != 0)
+            {
+                if (mark < pos)
+                {
+                    for (int i = DIG_PER_DEC1; i > 0; i--)
+                    {
+                        final int divisor = powers10[i - 1];
+                        final int y = x / divisor;
+                        buf[pos++] = ((char) ('0' + y));
+                        x -= y * divisor;
+                    }
+                }
+                else
+                {
+                    for (int i = DIG_PER_DEC1; i > 0; i--)
+                    {
+                        final int divisor = powers10[i - 1];
+                        final int y = x / divisor;
+                        if (mark < pos || y != 0)
+                        {
+                            buf[pos++] = ((char) ('0' + y));
+                        }
+                        x -= y * divisor;
+                    }
+                }
+            }
+            else if (mark < pos)
+            {
+                for (int i = DIG_PER_DEC1; i > 0; i--)
+                    buf[pos++] = ('0');
+            }
+        }
+
+        if (mark == pos)
+            /* fix 0.0 problem, only '.' may cause BigDecimal parsing exception. */
+            buf[pos++] = ('0');
+
+        if (frac > 0)
+        {
+            buf[pos++] = ('.');
+            mark = pos;
+
+            for (final int stop = from + frac0 * SIZE_OF_INT32; from < stop; from += SIZE_OF_INT32)
+            {
+                int x = getInt32BE(d_copy, from);
+                x ^= mask;
+                if (x < 0 || x > DIG_MAX)
+                {
+                    throw new IllegalArgumentException("bad format, x exceed: "
+                            + x + ", " + DIG_MAX);
+                }
+                if (x != 0)
+                {
+                    for (int i = DIG_PER_DEC1; i > 0; i--)
+                    {
+                        final int divisor = powers10[i - 1];
+                        final int y = x / divisor;
+                        buf[pos++] = ((char) ('0' + y));
+                        x -= y * divisor;
+                    }
+                }
+                else
+                {
+                    for (int i = DIG_PER_DEC1; i > 0; i--)
+                        buf[pos++] = ('0');
+                }
+            }
+
+            if (frac0x != 0)
+            {
+                final int i = dig2bytes[frac0x];
+                int x = 0;
+                switch (i)
+                {
+                case 1:
+                    x = d_copy[from] /* one byte */;
+                    break;
+                case 2:
+                    x = getInt16BE(d_copy, from);
+                    break;
+                case 3:
+                    x = getInt24BE(d_copy, from);
+                    break;
+                case 4:
+                    x = getInt32BE(d_copy, from);
+                    break;
+                }
+                x ^= mask;
+                if (x != 0)
+                {
+                    final int dig = DIG_PER_DEC1 - frac0x;
+                    x *= powers10[dig];
+                    if (x < 0 || x > DIG_MAX)
+                    {
+                        throw new IllegalArgumentException(
+                                "bad format, x exceed: " + x + ", " + DIG_MAX);
+                    }
+                    for (int j = DIG_PER_DEC1; j > dig; j--)
+                    {
+                        final int divisor = powers10[j - 1];
+                        final int y = x / divisor;
+                        buf[pos++] = ((char) ('0' + y));
+                        x -= y * divisor;
+                    }
+                }
+            }
+
+            if (mark == pos)
+                /* make number more friendly */
+                buf[pos++] = ('0');
+        }
+
+        d_copy[begin] ^= 0x80; /* restore sign */
+        String decimal = String.valueOf(buf, 0, pos);
+        return new BigDecimal(decimal);
+    }
+
+    /**
+     * Fill MY_BITMAP structure from buffer.
+     * 
+     * @param len The length of MY_BITMAP in bits.
+     */
+    public final void fillBitmap(BitSet bitmap, final int pos, final int len)
+    {
+        if (pos + ((len + 7) / 8) > limit || pos < 0)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (pos + (len + 7) / 8));
+
+        fillBitmap0(bitmap, origin + pos, len);
+    }
+
+    /**
+     * Fill next MY_BITMAP structure from buffer.
+     * 
+     * @param len The length of MY_BITMAP in bits.
+     */
+    public final void fillBitmap(BitSet bitmap, final int len)
+    {
+        if (position + ((len + 7) / 8) > origin + limit)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (position + ((len + 7) / 8) - origin));
+
+        position = fillBitmap0(bitmap, position, len);
+    }
+
+    /**
+     * Fill MY_BITMAP structure from buffer.
+     * 
+     * @param len The length of MY_BITMAP in bits.
+     */
+    private final int fillBitmap0(BitSet bitmap, int pos, final int len)
+    {
+        final byte[] buf = buffer;
+
+        for (int bit = 0; bit < len; bit += 8)
+        {
+            int flag = ((int) buf[pos++]) & 0xff;
+            if (flag == 0)
+                continue;
+            if ((flag & 0x01) != 0)
+                bitmap.set(bit);
+            if ((flag & 0x02) != 0)
+                bitmap.set(bit + 1);
+            if ((flag & 0x04) != 0)
+                bitmap.set(bit + 2);
+            if ((flag & 0x08) != 0)
+                bitmap.set(bit + 3);
+            if ((flag & 0x10) != 0)
+                bitmap.set(bit + 4);
+            if ((flag & 0x20) != 0)
+                bitmap.set(bit + 5);
+            if ((flag & 0x40) != 0)
+                bitmap.set(bit + 6);
+            if ((flag & 0x80) != 0)
+                bitmap.set(bit + 7);
+        }
+        return pos;
+    }
+
+    /**
+     * Return MY_BITMAP structure from buffer.
+     * 
+     * @param len The length of MY_BITMAP in bits.
+     */
+    public final BitSet getBitmap(final int pos, final int len)
+    {
+        BitSet bitmap = new BitSet(len);
+        fillBitmap(bitmap, pos, len);
+        return bitmap;
+    }
+
+    /**
+     * Return next MY_BITMAP structure from buffer.
+     * 
+     * @param len The length of MY_BITMAP in bits.
+     */
+    public final BitSet getBitmap(final int len)
+    {
+        BitSet bitmap = new BitSet(len);
+        fillBitmap(bitmap, len);
+        return bitmap;
+    }
+
+    /**
+     * Fill n bytes into output stream.
+     */
+    public final void fillOutput(OutputStream out, final int pos, final int len)
+            throws IOException
+    {
+        if (pos + len > limit || pos < 0)
+            throw new IllegalArgumentException("limit excceed: " + (pos + len));
+
+        out.write(buffer, origin + pos, len);
+    }
+
+    /**
+     * Fill next n bytes into output stream.
+     */
+    public final void fillOutput(OutputStream out, final int len)
+            throws IOException
+    {
+        if (position + len > origin + limit)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (position + len - origin));
+
+        out.write(buffer, position, len);
+        position += len;
+    }
+
+    /**
+     * Fill n bytes in this buffer.
+     */
+    public final void fillBytes(final int pos, byte[] dest, final int destPos,
+            final int len)
+    {
+        if (pos + len > limit || pos < 0)
+            throw new IllegalArgumentException("limit excceed: " + (pos + len));
+
+        System.arraycopy(buffer, origin + pos, dest, destPos, len);
+    }
+
+    /**
+     * Fill next n bytes in this buffer.
+     */
+    public final void fillBytes(byte[] dest, final int destPos, final int len)
+    {
+        if (position + len > origin + limit)
+            throw new IllegalArgumentException("limit excceed: "
+                    + (position + len - origin));
+
+        System.arraycopy(buffer, position, dest, destPos, len);
+        position += len;
+    }
+
+    /**
+     * Return n-byte data from buffer.
+     */
+    public final byte[] getData(final int pos, final int len)
+    {
+        byte[] buf = new byte[len];
+        fillBytes(pos, buf, 0, len);
+        return buf;
+    }
+
+    /**
+     * Return next n-byte data from buffer.
+     */
+    public final byte[] getData(final int len)
+    {
+        byte[] buf = new byte[len];
+        fillBytes(buf, 0, len);
+        return buf;
+    }
+
+    /**
+     * Return all remaining data from buffer.
+     */
+    public final byte[] getData()
+    {
+        return getData(0, limit);
+    }
+
+    /**
+     * Return full hexdump from position.
+     */
+    public final String hexdump(final int pos)
+    {
+        if ((limit - pos) > 0)
+        {
+            final int begin = origin + pos;
+            final int end = origin + limit;
+
+            byte[] buf = buffer;
+            StringBuilder dump = new StringBuilder();
+            dump.append(Integer.toHexString(buf[begin] >> 4));
+            dump.append(Integer.toHexString(buf[begin] & 0xf));
+            for (int i = begin + 1; i < end; i++)
+            {
+                dump.append("_");
+                dump.append(Integer.toHexString(buf[i] >> 4));
+                dump.append(Integer.toHexString(buf[i] & 0xf));
+            }
+
+            return dump.toString();
+        }
+        return "";
+    }
+
+    /**
+     * Return hexdump from position, for len bytes.
+     */
+    public final String hexdump(final int pos, final int len)
+    {
+        if ((limit - pos) > 0)
+        {
+            final int begin = origin + pos;
+            final int end = Math.min(begin + len, origin + limit);
+
+            byte[] buf = buffer;
+            StringBuilder dump = new StringBuilder();
+            dump.append(Integer.toHexString(buf[begin] >> 4));
+            dump.append(Integer.toHexString(buf[begin] & 0xf));
+            for (int i = begin + 1; i < end; i++)
+            {
+                dump.append("_");
+                dump.append(Integer.toHexString(buf[i] >> 4));
+                dump.append(Integer.toHexString(buf[i] & 0xf));
+            }
+
+            return dump.toString();
+        }
+        return "";
+    }
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogContext.java

@@ -0,0 +1,77 @@
+package com.taobao.tddl.dbsync.binlog;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent;
+
+/**
+ * TODO: Document Me!!
+ * 
+ * NOTE: Log context will NOT write multi-threaded.
+ * 
+ * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
+ * @version 1.0
+ */
+public final class LogContext
+{
+    private final Map<Long, TableMapLogEvent> mapOfTable = new HashMap<Long, TableMapLogEvent>();
+
+    private FormatDescriptionLogEvent         formatDescription;
+
+    private LogPosition                       logPosition;
+
+    public LogContext()
+    {
+        this.formatDescription = FormatDescriptionLogEvent.FORMAT_DESCRIPTION_EVENT_5_x;
+    }
+
+    public LogContext(FormatDescriptionLogEvent descriptionEvent)
+    {
+        this.formatDescription = descriptionEvent;
+    }
+
+    public final LogPosition getLogPosition()
+    {
+        return logPosition;
+    }
+
+    public final void setLogPosition(LogPosition logPosition)
+    {
+        this.logPosition = logPosition;
+    }
+
+    public final FormatDescriptionLogEvent getFormatDescription()
+    {
+        return formatDescription;
+    }
+
+    public final void setFormatDescription(
+            FormatDescriptionLogEvent formatDescription)
+    {
+        this.formatDescription = formatDescription;
+    }
+
+    public final void putTable(TableMapLogEvent mapEvent)
+    {
+        mapOfTable.put(Long.valueOf(mapEvent.getTableId()), mapEvent);
+    }
+
+    public final TableMapLogEvent getTable(final long tableId)
+    {
+        return mapOfTable.get(Long.valueOf(tableId));
+    }
+
+    public final void clearAllTables()
+    {
+        mapOfTable.clear();
+    }
+
+    public void reset()
+    {
+        formatDescription = FormatDescriptionLogEvent.FORMAT_DESCRIPTION_EVENT_5_x;
+
+        mapOfTable.clear();
+    }
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogDecoder.java

@@ -0,0 +1,494 @@
+package com.taobao.tddl.dbsync.binlog;
+
+import java.io.IOException;
+import java.util.BitSet;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import com.taobao.tddl.dbsync.binlog.event.AppendBlockLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.BeginLoadQueryLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.CreateFileLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.DeleteFileLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.ExecuteLoadLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.ExecuteLoadQueryLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.GtidLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.HeartbeatLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.IgnorableLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.IncidentLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.IntvarLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.LoadLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.LogHeader;
+import com.taobao.tddl.dbsync.binlog.event.PreviousGtidsLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.QueryLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.RandLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.RotateLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.RowsLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.RowsQueryLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.StartLogEventV3;
+import com.taobao.tddl.dbsync.binlog.event.StopLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.UnknownLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.UpdateRowsLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.UserVarLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.XidLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.mariadb.AnnotateRowsEvent;
+import com.taobao.tddl.dbsync.binlog.event.mariadb.BinlogCheckPointLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.mariadb.MariaGtidListLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.mariadb.MariaGtidLogEvent;
+
+/**
+ * Implements a binary-log decoder.
+ * 
+ * <pre>
+ * LogDecoder decoder = new LogDecoder();
+ * decoder.handle(...);
+ * 
+ * LogEvent event;
+ * do
+ * {
+ *     event = decoder.decode(buffer, context);
+ * 
+ *     // process log event.
+ * }
+ * while (event != null);
+ * // no more events in buffer.
+ * </pre>
+ * 
+ * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
+ * @version 1.0
+ */
+public final class LogDecoder
+{
+    protected static final Log logger    = LogFactory.getLog(LogDecoder.class);
+
+    protected final BitSet     handleSet = new BitSet(LogEvent.ENUM_END_EVENT);
+
+    public LogDecoder()
+    {
+    }
+
+    public LogDecoder(final int fromIndex, final int toIndex)
+    {
+        handleSet.set(fromIndex, toIndex);
+    }
+
+    public final void handle(final int fromIndex, final int toIndex)
+    {
+        handleSet.set(fromIndex, toIndex);
+    }
+
+    public final void handle(final int flagIndex)
+    {
+        handleSet.set(flagIndex);
+    }
+
+    /**
+     * Decoding an event from binary-log buffer.
+     * 
+     * @return <code>UknownLogEvent</code> if event type is unknown or skipped,
+     *         <code>null</code> if buffer is not including a full event.
+     */
+    public LogEvent decode(LogBuffer buffer, LogContext context)
+            throws IOException
+    {
+        final int limit = buffer.limit();
+
+        if (limit >= FormatDescriptionLogEvent.LOG_EVENT_HEADER_LEN)
+        {
+            LogHeader header = new LogHeader(buffer,
+                    context.getFormatDescription());
+
+            final int len = header.getEventLen();
+            if (limit >= len)
+            {
+                LogEvent event;
+
+                /* Checking binary-log's header */
+                if (handleSet.get(header.getType()))
+                {
+                    buffer.limit(len);
+                    try
+                    {
+                        /* Decoding binary-log to event */
+                        event = decode(buffer, header, context);
+                    }
+                    catch (IOException e)
+                    {
+                        if (logger.isWarnEnabled())
+                            logger.warn("Decoding "
+                                    + LogEvent.getTypeName(header.getType())
+                                    + " failed from: "
+                                    + context.getLogPosition(), e);
+                        throw e;
+                    }
+                    finally
+                    {
+                        buffer.limit(limit); /* Restore limit */
+                    }
+                }
+                else
+                {
+                    /* Ignore unsupported binary-log. */
+                    event = new UnknownLogEvent(header);
+                }
+
+                /* consume this binary-log. */
+                buffer.consume(len);
+                return event;
+            }
+        }
+
+        /* Rewind buffer's position to 0. */
+        buffer.rewind();
+        return null;
+    }
+
+    /**
+     * Deserialize an event from buffer.
+     * 
+     * @return <code>UknownLogEvent</code> if event type is unknown or skipped.
+     */
+    public static LogEvent decode(LogBuffer buffer, LogHeader header,
+            LogContext context) throws IOException
+    {
+        FormatDescriptionLogEvent descriptionEvent = context.getFormatDescription();
+        LogPosition logPosition = context.getLogPosition();
+        
+        int checksumAlg = LogEvent.BINLOG_CHECKSUM_ALG_UNDEF;
+        if (header.getType() != LogEvent.FORMAT_DESCRIPTION_EVENT) {
+            checksumAlg = descriptionEvent.header.getChecksumAlg();
+        }else {
+            // 如果是format事件自己，也需要处理checksum
+            checksumAlg = header.getChecksumAlg();
+        }
+        
+        if (checksumAlg != LogEvent.BINLOG_CHECKSUM_ALG_OFF && checksumAlg != LogEvent.BINLOG_CHECKSUM_ALG_UNDEF) {
+            // remove checksum bytes
+            buffer.limit(header.getEventLen() - LogEvent.BINLOG_CHECKSUM_LEN);
+        }
+        
+        switch (header.getType())
+        {
+        case LogEvent.QUERY_EVENT:
+            {
+                QueryLogEvent event = new QueryLogEvent(header, buffer,
+                        descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+        case LogEvent.XID_EVENT:
+            {
+                XidLogEvent event = new XidLogEvent(header, buffer,
+                        descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+        case LogEvent.TABLE_MAP_EVENT:
+            {
+                TableMapLogEvent mapEvent = new TableMapLogEvent(header,
+                        buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                context.putTable(mapEvent);
+                return mapEvent;
+            }
+        case LogEvent.WRITE_ROWS_EVENT_V1:
+            {
+                RowsLogEvent event = new WriteRowsLogEvent(header, buffer,
+                        descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                event.fillTable(context);
+                return event;
+            }
+        case LogEvent.UPDATE_ROWS_EVENT_V1:
+            {
+                RowsLogEvent event = new UpdateRowsLogEvent(header, buffer,
+                        descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                event.fillTable(context);
+                return event;
+            }
+        case LogEvent.DELETE_ROWS_EVENT_V1:
+            {
+                RowsLogEvent event = new DeleteRowsLogEvent(header, buffer,
+                        descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                event.fillTable(context);
+                return event;
+            }
+        case LogEvent.ROTATE_EVENT:
+            {
+                RotateLogEvent event = new RotateLogEvent(header, buffer,
+                        descriptionEvent);
+                /* updating position in context */
+                logPosition = new LogPosition(event.getFilename(),
+                        event.getPosition());
+                context.setLogPosition(logPosition);
+                return event;
+            }
+        case LogEvent.LOAD_EVENT:
+        case LogEvent.NEW_LOAD_EVENT:
+            {
+                LoadLogEvent event = new LoadLogEvent(header, buffer,
+                        descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+        case LogEvent.SLAVE_EVENT: /* can never happen (unused event) */
+            {
+                if (logger.isWarnEnabled())
+                    logger.warn("Skipping unsupported SLAVE_EVENT from: "
+                            + context.getLogPosition());
+                break;
+            }
+        case LogEvent.CREATE_FILE_EVENT:
+            {
+                CreateFileLogEvent event = new CreateFileLogEvent(header,
+                        buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+        case LogEvent.APPEND_BLOCK_EVENT:
+            {
+                AppendBlockLogEvent event = new AppendBlockLogEvent(header,
+                        buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+        case LogEvent.DELETE_FILE_EVENT:
+            {
+                DeleteFileLogEvent event = new DeleteFileLogEvent(header,
+                        buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+        case LogEvent.EXEC_LOAD_EVENT:
+            {
+                ExecuteLoadLogEvent event = new ExecuteLoadLogEvent(header,
+                        buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+        case LogEvent.START_EVENT_V3:
+            {
+                /* This is sent only by MySQL <=4.x */
+                StartLogEventV3 event = new StartLogEventV3(header, buffer,
+                        descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+        case LogEvent.STOP_EVENT:
+            {
+                StopLogEvent event = new StopLogEvent(header, buffer,
+                        descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+        case LogEvent.INTVAR_EVENT:
+            {
+                IntvarLogEvent event = new IntvarLogEvent(header, buffer,
+                        descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+        case LogEvent.RAND_EVENT:
+            {
+                RandLogEvent event = new RandLogEvent(header, buffer,
+                        descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+        case LogEvent.USER_VAR_EVENT:
+            {
+                UserVarLogEvent event = new UserVarLogEvent(header, buffer,
+                        descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+        case LogEvent.FORMAT_DESCRIPTION_EVENT:
+            {
+                descriptionEvent = new FormatDescriptionLogEvent(header,
+                        buffer, descriptionEvent);
+                context.setFormatDescription(descriptionEvent);
+                return descriptionEvent;
+            }
+        case LogEvent.PRE_GA_WRITE_ROWS_EVENT:
+            {
+                if (logger.isWarnEnabled())
+                    logger.warn("Skipping unsupported PRE_GA_WRITE_ROWS_EVENT from: "
+                            + context.getLogPosition());
+                // ev = new Write_rows_log_event_old(buf, event_len,
+                // description_event);
+                break;
+            }
+        case LogEvent.PRE_GA_UPDATE_ROWS_EVENT:
+            {
+                if (logger.isWarnEnabled())
+                    logger.warn("Skipping unsupported PRE_GA_UPDATE_ROWS_EVENT from: "
+                            + context.getLogPosition());
+                // ev = new Update_rows_log_event_old(buf, event_len,
+                // description_event);
+                break;
+            }
+        case LogEvent.PRE_GA_DELETE_ROWS_EVENT:
+            {
+                if (logger.isWarnEnabled())
+                    logger.warn("Skipping unsupported PRE_GA_DELETE_ROWS_EVENT from: "
+                            + context.getLogPosition());
+                // ev = new Delete_rows_log_event_old(buf, event_len,
+                // description_event);
+                break;
+            }
+        case LogEvent.BEGIN_LOAD_QUERY_EVENT:
+            {
+                BeginLoadQueryLogEvent event = new BeginLoadQueryLogEvent(
+                        header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+        case LogEvent.EXECUTE_LOAD_QUERY_EVENT:
+            {
+                ExecuteLoadQueryLogEvent event = new ExecuteLoadQueryLogEvent(
+                        header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+        case LogEvent.INCIDENT_EVENT:
+            {
+                IncidentLogEvent event = new IncidentLogEvent(header, buffer,
+                        descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+        case LogEvent.HEARTBEAT_LOG_EVENT:
+        {
+            HeartbeatLogEvent event = new HeartbeatLogEvent(header, buffer,
+                    descriptionEvent);
+            /* updating position in context */
+            logPosition.position = header.getLogPos();
+            return event;
+        }
+        case LogEvent.IGNORABLE_LOG_EVENT:
+            {
+                IgnorableLogEvent event = new IgnorableLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+        case LogEvent.ROWS_QUERY_LOG_EVENT:
+            {
+                RowsQueryLogEvent event = new RowsQueryLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+        case LogEvent.WRITE_ROWS_EVENT: {
+                RowsLogEvent event = new WriteRowsLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                event.fillTable(context);
+                return event;
+            }
+        case LogEvent.UPDATE_ROWS_EVENT: {
+                RowsLogEvent event = new UpdateRowsLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                event.fillTable(context);
+                return event;
+            }
+        case LogEvent.DELETE_ROWS_EVENT: {
+                RowsLogEvent event = new DeleteRowsLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                event.fillTable(context);
+                return event;
+            }
+        case LogEvent.GTID_LOG_EVENT:
+        case LogEvent.ANONYMOUS_GTID_LOG_EVENT:
+            {
+                GtidLogEvent event = new GtidLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+        case LogEvent.PREVIOUS_GTIDS_LOG_EVENT:
+            {
+                PreviousGtidsLogEvent event = new PreviousGtidsLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }
+        case LogEvent.ANNOTATE_ROWS_EVENT:
+            {
+            AnnotateRowsEvent event = new AnnotateRowsEvent(header, buffer, descriptionEvent);
+            /* updating position in context */
+            logPosition.position = header.getLogPos();
+            return event;
+            }
+        case LogEvent.BINLOG_CHECKPOINT_EVENT:
+            {
+            BinlogCheckPointLogEvent event = new BinlogCheckPointLogEvent(header, buffer, descriptionEvent);
+            /* updating position in context */
+            logPosition.position = header.getLogPos();
+            return event;
+            }
+        case LogEvent.GTID_EVENT:
+            {
+            MariaGtidLogEvent event = new MariaGtidLogEvent(header, buffer, descriptionEvent);
+            /* updating position in context */
+            logPosition.position = header.getLogPos();
+            return event;
+            }
+        case LogEvent.GTID_LIST_EVENT:
+            {
+            MariaGtidListLogEvent event = new MariaGtidListLogEvent(header, buffer, descriptionEvent);
+            /* updating position in context */
+            logPosition.position = header.getLogPos();
+            return event;
+            }
+        default:
+            /*
+                Create an object of Ignorable_log_event for unrecognized sub-class.
+                So that SLAVE SQL THREAD will only update the position and continue.
+             */
+            if((buffer.getUint16(LogEvent.FLAGS_OFFSET) & LogEvent.LOG_EVENT_IGNORABLE_F) > 0){
+                IgnorableLogEvent event = new IgnorableLogEvent(header, buffer, descriptionEvent);
+                /* updating position in context */
+                logPosition.position = header.getLogPos();
+                return event;
+            }else {
+                if (logger.isWarnEnabled())
+                    logger.warn("Skipping unrecognized binlog event "
+                            + LogEvent.getTypeName(header.getType()) + " from: "
+                            + context.getLogPosition());
+            }
+        }
+
+        /* updating position in context */
+        logPosition.position = header.getLogPos();
+        /* Unknown or unsupported log event */
+        return new UnknownLogEvent(header);
+    }
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogEvent.java

@@ -0,0 +1,437 @@
+package com.taobao.tddl.dbsync.binlog;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import com.taobao.tddl.dbsync.binlog.event.LogHeader;
+
+/**
+ * Binary log event definitions. This includes generic code common to all types
+ * of log events, as well as specific code for each type of log event.
+ * 
+ * - All numbers, whether they are 16-, 24-, 32-, or 64-bit numbers, are stored
+ * in little endian, i.e., the least significant byte first, unless otherwise
+ * specified.
+ * 
+ * representation of unsigned integers, called Packed Integer. A Packed Integer
+ * has the capacity of storing up to 8-byte integers, while small integers still
+ * can use 1, 3, or 4 bytes. The value of the first byte determines how to read
+ * the number, according to the following table:
+ * 
+ * <table>
+ * <caption>Format of Packed Integer</caption>
+ * 
+ * <tr>
+ * <th>First byte</th>
+ * <th>Format</th>
+ * </tr>
+ * 
+ * <tr>
+ * <td>0-250</td>
+ * <td>The first byte is the number (in the range 0-250), and no more bytes are
+ * used.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>252</td>
+ * <td>Two more bytes are used. The number is in the range 251-0xffff.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>253</td>
+ * <td>Three more bytes are used. The number is in the range 0xffff-0xffffff.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>254</td>
+ * <td>Eight more bytes are used. The number is in the range
+ * 0xffffff-0xffffffffffffffff.</td>
+ * </tr>
+ * 
+ * </table>
+ * 
+ * - Strings are stored in various formats. The format of each string is
+ * documented separately.
+ * 
+ * @see mysql-5.1.60/sql/log_event.h
+ * 
+ * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
+ * @version 1.0
+ */
+public abstract class LogEvent
+{
+    /*
+     * 3 is MySQL 4.x; 4 is MySQL 5.0.0.
+     * Compared to version 3, version 4 has:
+     * - a different Start_log_event, which includes info about the binary log
+     * (sizes of headers); this info is included for better compatibility if the
+     * master's MySQL version is different from the slave's.
+     * - all events have a unique ID (the triplet (server_id, timestamp at server
+     * start, other) to be sure an event is not executed more than once in a
+     * multimaster setup, example:
+     *              M1
+     *            /   \
+     *           v     v
+     *           M2    M3
+     *           \     /
+     *            v   v
+     *              S
+     * if a query is run on M1, it will arrive twice on S, so we need that S
+     * remembers the last unique ID it has processed, to compare and know if the
+     * event should be skipped or not. Example of ID: we already have the server id
+     * (4 bytes), plus:
+     * timestamp_when_the_master_started (4 bytes), a counter (a sequence number
+     * which increments every time we write an event to the binlog) (3 bytes).
+     * Q: how do we handle when the counter is overflowed and restarts from 0 ?
+     * 
+     * - Query and Load (Create or Execute) events may have a more precise
+     *   timestamp (with microseconds), number of matched/affected/warnings rows
+     * and fields of session variables: SQL_MODE,
+     * FOREIGN_KEY_CHECKS, UNIQUE_CHECKS, SQL_AUTO_IS_NULL, the collations and
+     * charsets, the PASSWORD() version (old/new/...).
+     */
+    public static final int    BINLOG_VERSION           = 4;
+
+    /* Default 5.0 server version */
+    public static final String SERVER_VERSION           = "5.0";
+
+    /**
+     * Event header offsets; these point to places inside the fixed header.
+     */
+    public static final int    EVENT_TYPE_OFFSET        = 4;
+    public static final int    SERVER_ID_OFFSET         = 5;
+    public static final int    EVENT_LEN_OFFSET         = 9;
+    public static final int    LOG_POS_OFFSET           = 13;
+    public static final int    FLAGS_OFFSET             = 17;
+
+    /* event-specific post-header sizes */
+    // where 3.23, 4.x and 5.0 agree
+    public static final int    QUERY_HEADER_MINIMAL_LEN = (4 + 4 + 1 + 2);
+    // where 5.0 differs: 2 for len of N-bytes vars.
+    public static final int    QUERY_HEADER_LEN         = (QUERY_HEADER_MINIMAL_LEN + 2);
+
+    /* Enumeration type for the different types of log events. */
+    public static final int    UNKNOWN_EVENT            = 0;
+    public static final int    START_EVENT_V3           = 1;
+    public static final int    QUERY_EVENT              = 2;
+    public static final int    STOP_EVENT               = 3;
+    public static final int    ROTATE_EVENT             = 4;
+    public static final int    INTVAR_EVENT             = 5;
+    public static final int    LOAD_EVENT               = 6;
+    public static final int    SLAVE_EVENT              = 7;
+    public static final int    CREATE_FILE_EVENT        = 8;
+    public static final int    APPEND_BLOCK_EVENT       = 9;
+    public static final int    EXEC_LOAD_EVENT          = 10;
+    public static final int    DELETE_FILE_EVENT        = 11;
+
+    /**
+     * NEW_LOAD_EVENT is like LOAD_EVENT except that it has a longer sql_ex,
+     * allowing multibyte TERMINATED BY etc; both types share the same class
+     * (Load_log_event)
+     */
+    public static final int    NEW_LOAD_EVENT           = 12;
+    public static final int    RAND_EVENT               = 13;
+    public static final int    USER_VAR_EVENT           = 14;
+    public static final int    FORMAT_DESCRIPTION_EVENT = 15;
+    public static final int    XID_EVENT                = 16;
+    public static final int    BEGIN_LOAD_QUERY_EVENT   = 17;
+    public static final int    EXECUTE_LOAD_QUERY_EVENT = 18;
+
+    public static final int    TABLE_MAP_EVENT          = 19;
+
+    /**
+     * These event numbers were used for 5.1.0 to 5.1.15 and are therefore
+     * obsolete.
+     */
+    public static final int    PRE_GA_WRITE_ROWS_EVENT  = 20;
+    public static final int    PRE_GA_UPDATE_ROWS_EVENT = 21;
+    public static final int    PRE_GA_DELETE_ROWS_EVENT = 22;
+
+    /**
+     * These event numbers are used from 5.1.16 and forward
+     */
+    public static final int    WRITE_ROWS_EVENT_V1      = 23;
+    public static final int    UPDATE_ROWS_EVENT_V1     = 24;
+    public static final int    DELETE_ROWS_EVENT_V1     = 25;
+
+    /**
+     * Something out of the ordinary happened on the master
+     */
+    public static final int    INCIDENT_EVENT           = 26;
+
+    /**
+     * Heartbeat event to be send by master at its idle time to ensure master's online status to slave
+     */
+    public static final int    HEARTBEAT_LOG_EVENT      = 27;
+
+    /**
+     * In some situations, it is necessary to send over ignorable data to the slave: data that a slave can handle in
+     * case there is code for handling it, but which can be ignored if it is not recognized.
+     */
+    public static final int    IGNORABLE_LOG_EVENT      = 28;
+    public static final int    ROWS_QUERY_LOG_EVENT     = 29;
+
+    /** Version 2 of the Row events */
+    public static final int    WRITE_ROWS_EVENT         = 30;
+    public static final int    UPDATE_ROWS_EVENT        = 31;
+    public static final int    DELETE_ROWS_EVENT        = 32;
+
+    public static final int    GTID_LOG_EVENT           = 33;
+    public static final int    ANONYMOUS_GTID_LOG_EVENT = 34;
+
+    public static final int    PREVIOUS_GTIDS_LOG_EVENT = 35;
+    
+    // mariaDb 5.5.34
+    /* New MySQL/Sun events are to be added right above this comment */
+    public static final int    MYSQL_EVENTS_END         = 36;
+
+    public static final int    MARIA_EVENTS_BEGIN       = 160;
+    /* New Maria event numbers start from here */
+    public static final int    ANNOTATE_ROWS_EVENT      = 160;
+    /*
+    Binlog checkpoint event. Used for XA crash recovery on the master, not used
+    in replication.
+    A binlog checkpoint event specifies a binlog file such that XA crash
+    recovery can start from that file - and it is guaranteed to find all XIDs
+    that are prepared in storage engines but not yet committed.
+     */
+    public static final int     BINLOG_CHECKPOINT_EVENT = 161;
+    /*
+    Gtid event. For global transaction ID, used to start a new event group,
+    instead of the old BEGIN query event, and also to mark stand-alone
+    events.
+     */
+    public static final int    GTID_EVENT               = 162;
+    /*
+    Gtid list event. Logged at the start of every binlog, to record the
+    current replication state. This consists of the last GTID seen for
+    each replication domain.
+     */
+    public static final int    GTID_LIST_EVENT          = 163;
+
+    /** end marker */
+    public static final int    ENUM_END_EVENT           = 164;
+
+    /**
+    1 byte length, 1 byte format
+    Length is total length in bytes, including 2 byte header
+    Length values 0 and 1 are currently invalid and reserved.
+    */
+    public static final int    EXTRA_ROW_INFO_LEN_OFFSET    = 0;
+    public static final int    EXTRA_ROW_INFO_FORMAT_OFFSET = 1;
+    public static final int    EXTRA_ROW_INFO_HDR_BYTES     = 2;
+    public static final int    EXTRA_ROW_INFO_MAX_PAYLOAD   = (255 - EXTRA_ROW_INFO_HDR_BYTES);
+    
+    // Events are without checksum though its generator
+    public static final int    BINLOG_CHECKSUM_ALG_OFF      = 0;
+    // is checksum-capable New Master (NM).
+    // CRC32 of zlib algorithm.
+    public static final int    BINLOG_CHECKSUM_ALG_CRC32    = 1;
+    // the cut line: valid alg range is [1, 0x7f].
+    public static final int    BINLOG_CHECKSUM_ALG_ENUM_END = 2;
+    // special value to tag undetermined yet checksum
+    public static final int    BINLOG_CHECKSUM_ALG_UNDEF    = 255;
+    // or events from checksum-unaware servers
+
+    public static final int    CHECKSUM_CRC32_SIGNATURE_LEN = 4;
+    public static final int    BINLOG_CHECKSUM_ALG_DESC_LEN = 1;
+    /**
+     * defined statically while there is just one alg implemented
+     */
+    public static final int    BINLOG_CHECKSUM_LEN          = CHECKSUM_CRC32_SIGNATURE_LEN;
+    
+    /* MySQL or old MariaDB slave with no announced capability. */
+    public static final int    MARIA_SLAVE_CAPABILITY_UNKNOWN           = 0;
+
+    /* MariaDB >= 5.3, which understands ANNOTATE_ROWS_EVENT. */
+    public static final int    MARIA_SLAVE_CAPABILITY_ANNOTATE          = 1;
+    /*
+     * MariaDB >= 5.5. This version has the capability to tolerate events
+     * omitted from the binlog stream without breaking replication (MySQL slaves
+     * fail because they mis-compute the offsets into the master's binlog).
+     */
+    public static final int    MARIA_SLAVE_CAPABILITY_TOLERATE_HOLES    = 2;
+    /* MariaDB >= 10.0, which knows about binlog_checkpoint_log_event. */
+    public static final int    MARIA_SLAVE_CAPABILITY_BINLOG_CHECKPOINT = 3;
+    /* MariaDB >= 10.0.1, which knows about global transaction id events. */
+    public static final int    MARIA_SLAVE_CAPABILITY_GTID              = 4;
+
+    /* Our capability. */
+    public static final int    MARIA_SLAVE_CAPABILITY_MINE              = MARIA_SLAVE_CAPABILITY_GTID;
+    
+    /**
+        For an event, 'e', carrying a type code, that a slave,
+        's', does not recognize, 's' will check 'e' for
+        LOG_EVENT_IGNORABLE_F, and if the flag is set, then 'e'
+        is ignored. Otherwise, 's' acknowledges that it has
+        found an unknown event in the relay log.
+     */
+    public static final int   LOG_EVENT_IGNORABLE_F    = 0x80;
+    
+    /** enum_field_types */
+    public static final int    MYSQL_TYPE_DECIMAL       = 0;
+    public static final int    MYSQL_TYPE_TINY          = 1;
+    public static final int    MYSQL_TYPE_SHORT         = 2;
+    public static final int    MYSQL_TYPE_LONG          = 3;
+    public static final int    MYSQL_TYPE_FLOAT         = 4;
+    public static final int    MYSQL_TYPE_DOUBLE        = 5;
+    public static final int    MYSQL_TYPE_NULL          = 6;
+    public static final int    MYSQL_TYPE_TIMESTAMP     = 7;
+    public static final int    MYSQL_TYPE_LONGLONG      = 8;
+    public static final int    MYSQL_TYPE_INT24         = 9;
+    public static final int    MYSQL_TYPE_DATE          = 10;
+    public static final int    MYSQL_TYPE_TIME          = 11;
+    public static final int    MYSQL_TYPE_DATETIME      = 12;
+    public static final int    MYSQL_TYPE_YEAR          = 13;
+    public static final int    MYSQL_TYPE_NEWDATE       = 14;
+    public static final int    MYSQL_TYPE_VARCHAR       = 15;
+    public static final int    MYSQL_TYPE_BIT           = 16;
+    public static final int    MYSQL_TYPE_TIMESTAMP2    = 17;
+    public static final int    MYSQL_TYPE_DATETIME2     = 18;
+    public static final int    MYSQL_TYPE_TIME2         = 19;
+    public static final int    MYSQL_TYPE_NEWDECIMAL    = 246;
+    public static final int    MYSQL_TYPE_ENUM          = 247;
+    public static final int    MYSQL_TYPE_SET           = 248;
+    public static final int    MYSQL_TYPE_TINY_BLOB     = 249;
+    public static final int    MYSQL_TYPE_MEDIUM_BLOB   = 250;
+    public static final int    MYSQL_TYPE_LONG_BLOB     = 251;
+    public static final int    MYSQL_TYPE_BLOB          = 252;
+    public static final int    MYSQL_TYPE_VAR_STRING    = 253;
+    public static final int    MYSQL_TYPE_STRING        = 254;
+    public static final int    MYSQL_TYPE_GEOMETRY      = 255;
+
+    public static String getTypeName(final int type)
+    {
+        switch (type)
+        {
+        case START_EVENT_V3:
+            return "Start_v3";
+        case STOP_EVENT:
+            return "Stop";
+        case QUERY_EVENT:
+            return "Query";
+        case ROTATE_EVENT:
+            return "Rotate";
+        case INTVAR_EVENT:
+            return "Intvar";
+        case LOAD_EVENT:
+            return "Load";
+        case NEW_LOAD_EVENT:
+            return "New_load";
+        case SLAVE_EVENT:
+            return "Slave";
+        case CREATE_FILE_EVENT:
+            return "Create_file";
+        case APPEND_BLOCK_EVENT:
+            return "Append_block";
+        case DELETE_FILE_EVENT:
+            return "Delete_file";
+        case EXEC_LOAD_EVENT:
+            return "Exec_load";
+        case RAND_EVENT:
+            return "RAND";
+        case XID_EVENT:
+            return "Xid";
+        case USER_VAR_EVENT:
+            return "User var";
+        case FORMAT_DESCRIPTION_EVENT:
+            return "Format_desc";
+        case TABLE_MAP_EVENT:
+            return "Table_map";
+        case PRE_GA_WRITE_ROWS_EVENT:
+            return "Write_rows_event_old";
+        case PRE_GA_UPDATE_ROWS_EVENT:
+            return "Update_rows_event_old";
+        case PRE_GA_DELETE_ROWS_EVENT:
+            return "Delete_rows_event_old";
+        case WRITE_ROWS_EVENT_V1:
+            return "Write_rows_v1";
+        case UPDATE_ROWS_EVENT_V1:
+            return "Update_rows_v1";
+        case DELETE_ROWS_EVENT_V1:
+            return "Delete_rows_v1";
+        case BEGIN_LOAD_QUERY_EVENT:
+            return "Begin_load_query";
+        case EXECUTE_LOAD_QUERY_EVENT:
+            return "Execute_load_query";
+        case INCIDENT_EVENT:
+            return "Incident";
+        case HEARTBEAT_LOG_EVENT:
+            return "Heartbeat";
+        case IGNORABLE_LOG_EVENT:
+            return "Ignorable";
+        case ROWS_QUERY_LOG_EVENT:
+            return "Rows_query";
+        case WRITE_ROWS_EVENT:
+            return "Write_rows";
+        case UPDATE_ROWS_EVENT:
+            return "Update_rows";
+        case DELETE_ROWS_EVENT:
+            return "Delete_rows";
+        case GTID_LOG_EVENT:
+            return "Gtid";
+        case ANONYMOUS_GTID_LOG_EVENT:
+            return "Anonymous_Gtid";
+        case PREVIOUS_GTIDS_LOG_EVENT:
+            return "Previous_gtids";
+        default:
+            return "Unknown"; /* impossible */
+        }
+    }
+
+    protected static final Log logger = LogFactory.getLog(LogEvent.class);
+
+    protected final LogHeader  header;
+
+    protected LogEvent(LogHeader header)
+    {
+        this.header = header;
+    }
+
+    /**
+     * Return event header.
+     */
+    public final LogHeader getHeader()
+    {
+        return header;
+    }
+
+    /**
+     * The total size of this event, in bytes. In other words, this is the sum
+     * of the sizes of Common-Header, Post-Header, and Body.
+     */
+    public final int getEventLen()
+    {
+        return header.getEventLen();
+    }
+
+    /**
+     * Server ID of the server that created the event.
+     */
+    public final long getServerId()
+    {
+        return header.getServerId();
+    }
+
+    /**
+     * The position of the next event in the master binary log, in bytes from
+     * the beginning of the file. In a binlog that is not a relay log, this is
+     * just the position of the next event, in bytes from the beginning of the
+     * file. In a relay log, this is the position of the next event in the
+     * master's binlog.
+     */
+    public final long getLogPos()
+    {
+        return header.getLogPos();
+    }
+
+    /**
+     * The time when the query started, in seconds since 1970.
+     */
+    public final long getWhen()
+    {
+        return header.getWhen();
+    }
+    
+    
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogFetcher.java

@@ -0,0 +1,93 @@
+package com.taobao.tddl.dbsync.binlog;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Arrays;
+
+/**
+ * Declaration a binary-log fetcher. It extends from <code>LogBuffer</code>.
+ * 
+ * <pre>
+ * LogFetcher fetcher = new SomeLogFetcher();
+ * ...
+ * 
+ * while (fetcher.fetch())
+ * {
+ *     LogEvent event;
+ *     do
+ *     {
+ *         event = decoder.decode(fetcher, context);
+ * 
+ *         // process log event.
+ *     }
+ *     while (event != null);
+ * }
+ * // no more binlog.
+ * fetcher.close();
+ * </pre>
+ * 
+ * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
+ * @version 1.0
+ */
+public abstract class LogFetcher extends LogBuffer implements Closeable
+{
+    /** Default initial capacity. */
+    public static final int   DEFAULT_INITIAL_CAPACITY = 8192;
+
+    /** Default growth factor. */
+    public static final float DEFAULT_GROWTH_FACTOR    = 2.0f;
+
+    /** Binlog file header size */
+    public static final int   BIN_LOG_HEADER_SIZE      = 4;
+
+    protected final float     factor;
+
+    public LogFetcher()
+    {
+        this(DEFAULT_INITIAL_CAPACITY, DEFAULT_GROWTH_FACTOR);
+    }
+
+    public LogFetcher(final int initialCapacity)
+    {
+        this(initialCapacity, DEFAULT_GROWTH_FACTOR);
+    }
+
+    public LogFetcher(final int initialCapacity, final float growthFactor)
+    {
+        this.buffer = new byte[initialCapacity];
+        this.factor = growthFactor;
+    }
+
+    /**
+     * Increases the capacity of this <tt>LogFetcher</tt> instance, if
+     * necessary, to ensure that it can hold at least the number of elements
+     * specified by the minimum capacity argument.
+     * 
+     * @param minCapacity the desired minimum capacity
+     */
+    protected final void ensureCapacity(final int minCapacity)
+    {
+        final int oldCapacity = buffer.length;
+
+        if (minCapacity > oldCapacity)
+        {
+            int newCapacity = (int) (oldCapacity * factor);
+            if (newCapacity < minCapacity)
+                newCapacity = minCapacity;
+
+            buffer = Arrays.copyOf(buffer, newCapacity);
+        }
+    }
+
+    /**
+     * Fetches the next frame of binary-log, and fill it in buffer.
+     */
+    public abstract boolean fetch() throws IOException;
+
+    /**
+     * {@inheritDoc}
+     * 
+     * @see java.io.Closeable#close()
+     */
+    public abstract void close() throws IOException;
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/LogPosition.java

@@ -0,0 +1,127 @@
+package com.taobao.tddl.dbsync.binlog;
+
+/**
+ * Implements binlog position.
+ * 
+ * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
+ * @version 1.0
+ */
+public class LogPosition implements Cloneable, Comparable<LogPosition>
+{
+    /* binlog file's name */
+    protected String fileName;
+
+    /* position in file */
+    protected long   position;
+
+    /**
+     * Binlog position init.
+     * 
+     * @param fileName file name for binlog files: mysql-bin.000001
+     */
+    public LogPosition(String fileName)
+    {
+        this.fileName = fileName;
+        this.position = 0L;
+    }
+
+    /**
+     * Binlog position init.
+     * 
+     * @param fileName file name for binlog files: mysql-bin.000001
+     */
+    public LogPosition(String fileName, final long position)
+    {
+        this.fileName = fileName;
+        this.position = position;
+    }
+
+    /**
+     * Binlog position copy init.
+     */
+    public LogPosition(LogPosition source)
+    {
+        this.fileName = source.fileName;
+        this.position = source.position;
+    }
+
+    public final String getFileName()
+    {
+        return fileName;
+    }
+
+    public final long getPosition()
+    {
+        return position;
+    }
+
+    /* Clone binlog position without CloneNotSupportedException */
+    public LogPosition clone()
+    {
+        try
+        {
+            return (LogPosition) super.clone();
+        }
+        catch (CloneNotSupportedException e)
+        {
+            // Never happend
+            return null;
+        }
+    }
+
+    /**
+     * Compares with the specified fileName and position.
+     */
+    public final int compareTo(String fileName, final long position)
+    {
+        final int val = this.fileName.compareTo(fileName);
+
+        if (val == 0)
+        {
+            return (int) (this.position - position);
+        }
+        return val;
+    }
+
+    /**
+     * {@inheritDoc}
+     * 
+     * @see java.lang.Comparable#compareTo(java.lang.Object)
+     */
+    public int compareTo(LogPosition o)
+    {
+        final int val = fileName.compareTo(o.fileName);
+
+        if (val == 0)
+        {
+            return (int) (position - o.position);
+        }
+        return val;
+    }
+
+    /**
+     * {@inheritDoc}
+     * 
+     * @see java.lang.Object#equals(java.lang.Object)
+     */
+    public boolean equals(Object obj)
+    {
+        if (obj instanceof LogPosition)
+        {
+            LogPosition pos = ((LogPosition) obj);
+            return fileName.equals(pos.fileName)
+                    && (this.position == pos.position);
+        }
+        return false;
+    }
+
+    /**
+     * {@inheritDoc}
+     * 
+     * @see java.lang.Object#toString()
+     */
+    public String toString()
+    {
+        return fileName + ':' + position;
+    }
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/AppendBlockLogEvent.java

@@ -0,0 +1,53 @@
+package com.taobao.tddl.dbsync.binlog.event;
+
+import com.taobao.tddl.dbsync.binlog.LogBuffer;
+import com.taobao.tddl.dbsync.binlog.LogEvent;
+
+/**
+ * Append_block_log_event.
+ * 
+ * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
+ * @version 1.0
+ */
+public class AppendBlockLogEvent extends LogEvent
+{
+    private final LogBuffer blockBuf;
+    private final int       blockLen;
+
+    private final long      fileId;
+
+    /* AB = "Append Block" */
+    public static final int AB_FILE_ID_OFFSET = 0;
+
+    public AppendBlockLogEvent(LogHeader header, LogBuffer buffer,
+            FormatDescriptionLogEvent descriptionEvent)
+    {
+        super(header);
+
+        final int commonHeaderLen = descriptionEvent.commonHeaderLen;
+        final int postHeaderLen = descriptionEvent.postHeaderLen[header.type - 1];
+        final int totalHeaderLen = commonHeaderLen + postHeaderLen;
+
+        buffer.position(commonHeaderLen + AB_FILE_ID_OFFSET);
+        fileId = buffer.getUint32();
+
+        buffer.position(postHeaderLen);
+        blockLen = buffer.limit() - totalHeaderLen;
+        blockBuf = buffer.duplicate(blockLen);
+    }
+
+    public final long getFileId()
+    {
+        return fileId;
+    }
+
+    public final LogBuffer getBuffer()
+    {
+        return blockBuf;
+    }
+
+    public final byte[] getData()
+    {
+        return blockBuf.getData();
+    }
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/BeginLoadQueryLogEvent.java

@@ -0,0 +1,20 @@
+package com.taobao.tddl.dbsync.binlog.event;
+
+import com.taobao.tddl.dbsync.binlog.LogBuffer;
+
+/**
+ * Event for the first block of file to be loaded, its only difference from
+ * Append_block event is that this event creates or truncates existing file
+ * before writing data.
+ * 
+ * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
+ * @version 1.0
+ */
+public final class BeginLoadQueryLogEvent extends AppendBlockLogEvent
+{
+    public BeginLoadQueryLogEvent(LogHeader header, LogBuffer buffer,
+            FormatDescriptionLogEvent descriptionEvent)
+    {
+        super(header, buffer, descriptionEvent);
+    }
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/CreateFileLogEvent.java

@@ -0,0 +1,73 @@
+package com.taobao.tddl.dbsync.binlog.event;
+
+import com.taobao.tddl.dbsync.binlog.LogBuffer;
+
+/**
+ * Create_file_log_event.
+ * 
+ * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
+ * @version 1.0
+ */
+public final class CreateFileLogEvent extends LoadLogEvent
+{
+    protected LogBuffer     blockBuf;
+    protected int           blockLen;
+    protected long          fileId;
+
+    protected boolean       initedFromOld;
+
+    /* CF = "Create File" */
+    public static final int CF_FILE_ID_OFFSET = 0;
+    public static final int CF_DATA_OFFSET    = FormatDescriptionLogEvent.CREATE_FILE_HEADER_LEN;
+
+    public CreateFileLogEvent(LogHeader header, LogBuffer buffer,
+            FormatDescriptionLogEvent descriptionEvent)
+    {
+        super(header, buffer, descriptionEvent);
+
+        final int headerLen = descriptionEvent.commonHeaderLen;
+        final int loadHeaderLen = descriptionEvent.postHeaderLen[LOAD_EVENT - 1];
+        final int createFileHeaderLen = descriptionEvent.postHeaderLen[CREATE_FILE_EVENT - 1];
+
+        copyLogEvent(buffer,
+                ((header.type == LOAD_EVENT) ? (loadHeaderLen + headerLen)
+                        : (headerLen + loadHeaderLen + createFileHeaderLen)),
+                descriptionEvent);
+
+        if (descriptionEvent.binlogVersion != 1)
+        {
+            fileId = buffer.getUint32(headerLen + loadHeaderLen
+                    + CF_FILE_ID_OFFSET);
+            /*
+              Note that it's ok to use get_data_size() below, because it is computed
+              with values we have already read from this event (because we called
+              copy_log_event()); we are not using slave's format info to decode
+              master's format, we are really using master's format info.
+              Anyway, both formats should be identical (except the common_header_len)
+              as these Load events are not changed between 4.0 and 5.0 (as logging of
+              LOAD DATA INFILE does not use Load_log_event in 5.0).
+            */
+            blockLen = buffer.limit() - buffer.position();
+            blockBuf = buffer.duplicate(blockLen);
+        }
+        else
+        {
+            initedFromOld = true;
+        }
+    }
+
+    public final long getFileId()
+    {
+        return fileId;
+    }
+
+    public final LogBuffer getBuffer()
+    {
+        return blockBuf;
+    }
+
+    public final byte[] getData()
+    {
+        return blockBuf.getData();
+    }
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/DeleteFileLogEvent.java

@@ -0,0 +1,33 @@
+package com.taobao.tddl.dbsync.binlog.event;
+
+import com.taobao.tddl.dbsync.binlog.LogBuffer;
+import com.taobao.tddl.dbsync.binlog.LogEvent;
+
+/**
+ * Delete_file_log_event.
+ * 
+ * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
+ * @version 1.0
+ */
+public final class DeleteFileLogEvent extends LogEvent
+{
+    private final long      fileId;
+
+    /* DF = "Delete File" */
+    public static final int DF_FILE_ID_OFFSET = 0;
+
+    public DeleteFileLogEvent(LogHeader header, LogBuffer buffer,
+            FormatDescriptionLogEvent descriptionEvent)
+    {
+        super(header);
+
+        final int commonHeaderLen = descriptionEvent.commonHeaderLen;
+        buffer.position(commonHeaderLen + DF_FILE_ID_OFFSET);
+        fileId = buffer.getUint32(); //  DF_FILE_ID_OFFSET
+    }
+
+    public final long getFileId()
+    {
+        return fileId;
+    }
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/DeleteRowsLogEvent.java

@@ -0,0 +1,19 @@
+package com.taobao.tddl.dbsync.binlog.event;
+
+import com.taobao.tddl.dbsync.binlog.LogBuffer;
+
+/**
+ * Log row deletions. The event contain several delete rows for a table. Note
+ * that each event contains only rows for one table.
+ * 
+ * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
+ * @version 1.0
+ */
+public final class DeleteRowsLogEvent extends RowsLogEvent
+{
+    public DeleteRowsLogEvent(LogHeader header, LogBuffer buffer,
+            FormatDescriptionLogEvent descriptionEvent)
+    {
+        super(header, buffer, descriptionEvent);
+    }
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/ExecuteLoadLogEvent.java

@@ -0,0 +1,33 @@
+package com.taobao.tddl.dbsync.binlog.event;
+
+import com.taobao.tddl.dbsync.binlog.LogBuffer;
+import com.taobao.tddl.dbsync.binlog.LogEvent;
+
+/**
+ * Execute_load_log_event.
+ * 
+ * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
+ * @version 1.0
+ */
+public final class ExecuteLoadLogEvent extends LogEvent
+{
+    private final long      fileId;
+
+    /* EL = "Execute Load" */
+    public static final int EL_FILE_ID_OFFSET = 0;
+
+    public ExecuteLoadLogEvent(LogHeader header, LogBuffer buffer,
+            FormatDescriptionLogEvent descriptionEvent)
+    {
+        super(header);
+
+        final int commonHeaderLen = descriptionEvent.commonHeaderLen;
+        buffer.position(commonHeaderLen + EL_FILE_ID_OFFSET);
+        fileId = buffer.getUint32(); //  EL_FILE_ID_OFFSET
+    }
+
+    public final long getFileId()
+    {
+        return fileId;
+    }
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/ExecuteLoadQueryLogEvent.java

@@ -0,0 +1,101 @@
+package com.taobao.tddl.dbsync.binlog.event;
+
+import java.io.IOException;
+
+import com.taobao.tddl.dbsync.binlog.LogBuffer;
+
+/**
+ * Event responsible for LOAD DATA execution, it similar to Query_log_event but
+ * before executing the query it substitutes original filename in LOAD DATA
+ * query with name of temporary file.
+ * 
+ * <ul>
+ * <li>4 bytes. The ID of the file to load.</li>
+ * <li>4 bytes. The start position within the statement for filename
+ * substitution.</li>
+ * <li>4 bytes. The end position within the statement for filename substitution.
+ * </li>
+ * <li>1 byte. How to handle duplicates: LOAD_DUP_ERROR = 0, LOAD_DUP_IGNORE =
+ * 1, LOAD_DUP_REPLACE = 2</li>
+ * </ul>
+ * 
+ * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
+ * @version 1.0
+ */
+public final class ExecuteLoadQueryLogEvent extends QueryLogEvent
+{
+    /** file_id of temporary file */
+    private long            fileId;
+
+    /** pointer to the part of the query that should be substituted */
+    private int             fnPosStart;
+
+    /** pointer to the end of this part of query */
+    private int             fnPosEnd;
+
+    /*
+     * Elements of this enum describe how LOAD DATA handles duplicates.
+     */
+    public static final int LOAD_DUP_ERROR          = 0;
+    public static final int LOAD_DUP_IGNORE         = LOAD_DUP_ERROR + 1;
+    public static final int LOAD_DUP_REPLACE        = LOAD_DUP_IGNORE + 1;
+
+    /**
+     * We have to store type of duplicate handling explicitly, because for LOAD
+     * DATA it also depends on LOCAL option. And this part of query will be
+     * rewritten during replication so this information may be lost...
+     */
+    private int             dupHandling;
+
+    /* ELQ = "Execute Load Query" */
+    public static final int ELQ_FILE_ID_OFFSET      = QUERY_HEADER_LEN;
+    public static final int ELQ_FN_POS_START_OFFSET = ELQ_FILE_ID_OFFSET + 4;
+    public static final int ELQ_FN_POS_END_OFFSET   = ELQ_FILE_ID_OFFSET + 8;
+    public static final int ELQ_DUP_HANDLING_OFFSET = ELQ_FILE_ID_OFFSET + 12;
+
+    public ExecuteLoadQueryLogEvent(LogHeader header, LogBuffer buffer,
+            FormatDescriptionLogEvent descriptionEvent) throws IOException
+    {
+        super(header, buffer, descriptionEvent);
+
+        buffer.position(descriptionEvent.commonHeaderLen + ELQ_FILE_ID_OFFSET);
+
+        fileId = buffer.getUint32(); // ELQ_FILE_ID_OFFSET
+        fnPosStart = (int) buffer.getUint32(); // ELQ_FN_POS_START_OFFSET
+        fnPosEnd = (int) buffer.getUint32(); // ELQ_FN_POS_END_OFFSET
+        dupHandling = buffer.getInt8(); // ELQ_DUP_HANDLING_OFFSET
+
+        final int len = query.length();
+        if (fnPosStart > len || fnPosEnd > len
+                || dupHandling > LOAD_DUP_REPLACE)
+        {
+            throw new IOException(String.format(
+                    "Invalid ExecuteLoadQueryLogEvent: fn_pos_start=%d, "
+                            + "fn_pos_end=%d, dup_handling=%d", fnPosStart,
+                    fnPosEnd, dupHandling));
+        }
+    }
+
+    public final int getFilenamePosStart()
+    {
+        return fnPosStart;
+    }
+
+    public final int getFilenamePosEnd()
+    {
+        return fnPosEnd;
+    }
+
+    public final String getFilename()
+    {
+        if (query != null)
+            return query.substring(fnPosStart, fnPosEnd).trim();
+
+        return null;
+    }
+
+    public final long getFileId()
+    {
+        return fileId;
+    }
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/FormatDescriptionLogEvent.java

@@ -0,0 +1,305 @@
+package com.taobao.tddl.dbsync.binlog.event;
+
+import java.io.IOException;
+
+import com.taobao.tddl.dbsync.binlog.LogBuffer;
+
+/**
+ * For binlog version 4. This event is saved by threads which read it, as they need it for future use (to decode the
+ * ordinary events).
+ * 
+ * @see mysql-5.1.60/sql/log_event.cc - Format_description_log_event
+ * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
+ * @version 1.0
+ */
+public final class FormatDescriptionLogEvent extends StartLogEventV3 {
+
+    /**
+     * The number of types we handle in Format_description_log_event (UNKNOWN_EVENT is not to be handled, it does not
+     * exist in binlogs, it does not have a format).
+     */
+    public static final int    LOG_EVENT_TYPES                     = (ENUM_END_EVENT - 1);
+
+    public static final int    ST_COMMON_HEADER_LEN_OFFSET         = (ST_SERVER_VER_OFFSET + ST_SERVER_VER_LEN + 4);
+
+    public static final int    OLD_HEADER_LEN                      = 13;
+    public static final int    LOG_EVENT_HEADER_LEN                = 19;
+    public static final int    LOG_EVENT_MINIMAL_HEADER_LEN        = 19;
+
+    /* event-specific post-header sizes */
+    public static final int    STOP_HEADER_LEN                     = 0;
+    public static final int    LOAD_HEADER_LEN                     = (4 + 4 + 4 + 1 + 1 + 4);
+    public static final int    SLAVE_HEADER_LEN                    = 0;
+    public static final int    START_V3_HEADER_LEN                 = (2 + ST_SERVER_VER_LEN + 4);
+    public static final int    ROTATE_HEADER_LEN                   = 8;                                                       // this
+    // is
+    // FROZEN
+    // (the
+    // Rotate
+    // post-header
+    // is
+    // frozen)
+    public static final int    INTVAR_HEADER_LEN                   = 0;
+    public static final int    CREATE_FILE_HEADER_LEN              = 4;
+    public static final int    APPEND_BLOCK_HEADER_LEN             = 4;
+    public static final int    EXEC_LOAD_HEADER_LEN                = 4;
+    public static final int    DELETE_FILE_HEADER_LEN              = 4;
+    public static final int    NEW_LOAD_HEADER_LEN                 = LOAD_HEADER_LEN;
+    public static final int    RAND_HEADER_LEN                     = 0;
+    public static final int    USER_VAR_HEADER_LEN                 = 0;
+    public static final int    FORMAT_DESCRIPTION_HEADER_LEN       = (START_V3_HEADER_LEN + 1 + LOG_EVENT_TYPES);
+    public static final int    XID_HEADER_LEN                      = 0;
+    public static final int    BEGIN_LOAD_QUERY_HEADER_LEN         = APPEND_BLOCK_HEADER_LEN;
+    public static final int    ROWS_HEADER_LEN_V1                  = 8;
+    public static final int    TABLE_MAP_HEADER_LEN                = 8;
+    public static final int    EXECUTE_LOAD_QUERY_EXTRA_HEADER_LEN = (4 + 4 + 4 + 1);
+    public static final int    EXECUTE_LOAD_QUERY_HEADER_LEN       = (QUERY_HEADER_LEN + EXECUTE_LOAD_QUERY_EXTRA_HEADER_LEN);
+    public static final int    INCIDENT_HEADER_LEN                 = 2;
+    public static final int    HEARTBEAT_HEADER_LEN                = 0;
+    public static final int    IGNORABLE_HEADER_LEN                = 0;
+    public static final int    ROWS_HEADER_LEN_V2                  = 10;
+    public static final int    ANNOTATE_ROWS_HEADER_LEN            = 0;
+    public static final int    BINLOG_CHECKPOINT_HEADER_LEN        = 4;
+    public static final int    GTID_HEADER_LEN                     = 19;
+    public static final int    GTID_LIST_HEADER_LEN                = 4;
+
+    public static final int    POST_HEADER_LENGTH                  = 11;
+
+    public static final int   BINLOG_CHECKSUM_ALG_DESC_LEN        = 1;
+    public static final int[] checksumVersionSplit                = { 5, 6, 1 };
+    public static final long  checksumVersionProduct              = (checksumVersionSplit[0] * 256 + checksumVersionSplit[1])
+                                                                    * 256 + checksumVersionSplit[2];
+    /**
+     * The size of the fixed header which _all_ events have (for binlogs written by this version, this is equal to
+     * LOG_EVENT_HEADER_LEN), except FORMAT_DESCRIPTION_EVENT and ROTATE_EVENT (those have a header of size
+     * LOG_EVENT_MINIMAL_HEADER_LEN).
+     */
+    protected final int        commonHeaderLen;
+    protected int              numberOfEventTypes;
+
+    /** The list of post-headers' lengthes */
+    protected final short[]    postHeaderLen;
+    protected int[]            serverVersionSplit                  = new int[3];
+
+    public FormatDescriptionLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent)
+                                                                                                                    throws IOException{
+        /* Start_log_event_v3 */
+        super(header, buffer, descriptionEvent);
+
+        buffer.position(LOG_EVENT_MINIMAL_HEADER_LEN + ST_COMMON_HEADER_LEN_OFFSET);
+        commonHeaderLen = buffer.getUint8();
+        if (commonHeaderLen < OLD_HEADER_LEN) /* sanity check */
+        {
+            throw new IOException("Format Description event header length is too short");
+        }
+
+        numberOfEventTypes = buffer.limit() - (LOG_EVENT_MINIMAL_HEADER_LEN + ST_COMMON_HEADER_LEN_OFFSET + 1);
+
+        // buffer.position(LOG_EVENT_MINIMAL_HEADER_LEN
+        // + ST_COMMON_HEADER_LEN_OFFSET + 1);
+        postHeaderLen = new short[numberOfEventTypes];
+        for (int i = 0; i < numberOfEventTypes; i++) {
+            postHeaderLen[i] = (short) buffer.getUint8();
+        }
+
+        calcServerVersionSplit();
+        long calc = getVersionProduct();
+        if (calc >= checksumVersionProduct) {
+            /* the last bytes are the checksum alg desc and value (or value's room) */
+            numberOfEventTypes -= BINLOG_CHECKSUM_ALG_DESC_LEN;
+        }
+        
+        if (logger.isInfoEnabled()) logger.info("common_header_len= " + commonHeaderLen + ", number_of_event_types= "
+                + numberOfEventTypes);
+    }
+
+    /** MySQL 5.0 format descriptions. */
+    public static final FormatDescriptionLogEvent FORMAT_DESCRIPTION_EVENT_5_x   = new FormatDescriptionLogEvent(4);
+
+    /** MySQL 4.0.x (x>=2) format descriptions. */
+    public static final FormatDescriptionLogEvent FORMAT_DESCRIPTION_EVENT_4_0_x = new FormatDescriptionLogEvent(3);
+
+    /** MySQL 3.23 format descriptions. */
+    public static final FormatDescriptionLogEvent FORMAT_DESCRIPTION_EVENT_3_23  = new FormatDescriptionLogEvent(1);
+
+    public static FormatDescriptionLogEvent getFormatDescription(final int binlogVersion) throws IOException {
+        /* identify binlog format */
+        switch (binlogVersion) {
+            case 4: /* MySQL 5.0 */
+                return FORMAT_DESCRIPTION_EVENT_5_x;
+            case 3:
+                return FORMAT_DESCRIPTION_EVENT_4_0_x;
+            case 1:
+                return FORMAT_DESCRIPTION_EVENT_3_23;
+            default:
+                throw new IOException("Unknown binlog version: " + binlogVersion);
+        }
+    }
+
+    public FormatDescriptionLogEvent(final int binlogVersion){
+        this.binlogVersion = binlogVersion;
+
+        postHeaderLen = new short[ENUM_END_EVENT];
+        /* identify binlog format */
+        switch (binlogVersion) {
+            case 4: /* MySQL 5.0 */
+                serverVersion = SERVER_VERSION;
+                commonHeaderLen = LOG_EVENT_HEADER_LEN;
+                numberOfEventTypes = LOG_EVENT_TYPES;
+
+                /* Note: all event types must explicitly fill in their lengths here. */
+                postHeaderLen[START_EVENT_V3 - 1] = START_V3_HEADER_LEN;
+                postHeaderLen[QUERY_EVENT - 1] = QUERY_HEADER_LEN;
+                postHeaderLen[STOP_EVENT - 1] = STOP_HEADER_LEN;
+                postHeaderLen[ROTATE_EVENT - 1] = ROTATE_HEADER_LEN;
+                postHeaderLen[INTVAR_EVENT - 1] = INTVAR_HEADER_LEN;
+                postHeaderLen[LOAD_EVENT - 1] = LOAD_HEADER_LEN;
+                postHeaderLen[SLAVE_EVENT - 1] = SLAVE_HEADER_LEN;
+                postHeaderLen[CREATE_FILE_EVENT - 1] = CREATE_FILE_HEADER_LEN;
+                postHeaderLen[APPEND_BLOCK_EVENT - 1] = APPEND_BLOCK_HEADER_LEN;
+                postHeaderLen[EXEC_LOAD_EVENT - 1] = EXEC_LOAD_HEADER_LEN;
+                postHeaderLen[DELETE_FILE_EVENT - 1] = DELETE_FILE_HEADER_LEN;
+                postHeaderLen[NEW_LOAD_EVENT - 1] = NEW_LOAD_HEADER_LEN;
+                postHeaderLen[RAND_EVENT - 1] = RAND_HEADER_LEN;
+                postHeaderLen[USER_VAR_EVENT - 1] = USER_VAR_HEADER_LEN;
+                postHeaderLen[FORMAT_DESCRIPTION_EVENT - 1] = FORMAT_DESCRIPTION_HEADER_LEN;
+                postHeaderLen[XID_EVENT - 1] = XID_HEADER_LEN;
+                postHeaderLen[BEGIN_LOAD_QUERY_EVENT - 1] = BEGIN_LOAD_QUERY_HEADER_LEN;
+                postHeaderLen[EXECUTE_LOAD_QUERY_EVENT - 1] = EXECUTE_LOAD_QUERY_HEADER_LEN;
+                postHeaderLen[TABLE_MAP_EVENT - 1] = TABLE_MAP_HEADER_LEN;
+                postHeaderLen[WRITE_ROWS_EVENT_V1 - 1] = ROWS_HEADER_LEN_V1;
+                postHeaderLen[UPDATE_ROWS_EVENT_V1 - 1] = ROWS_HEADER_LEN_V1;
+                postHeaderLen[DELETE_ROWS_EVENT_V1 - 1] = ROWS_HEADER_LEN_V1;
+                /*
+                 * We here have the possibility to simulate a master of before we changed the table map id to be stored
+                 * in 6 bytes: when it was stored in 4 bytes (=> post_header_len was 6). This is used to test backward
+                 * compatibility. This code can be removed after a few months (today is Dec 21st 2005), when we know
+                 * that the 4-byte masters are not deployed anymore (check with Tomas Ulin first!), and the accompanying
+                 * test (rpl_row_4_bytes) too.
+                 */
+                postHeaderLen[HEARTBEAT_LOG_EVENT - 1] = 0;
+                postHeaderLen[IGNORABLE_LOG_EVENT - 1] = IGNORABLE_HEADER_LEN;
+                postHeaderLen[ROWS_QUERY_LOG_EVENT - 1] = IGNORABLE_HEADER_LEN;
+                postHeaderLen[WRITE_ROWS_EVENT - 1] = ROWS_HEADER_LEN_V2;
+                postHeaderLen[UPDATE_ROWS_EVENT - 1] = ROWS_HEADER_LEN_V2;
+                postHeaderLen[DELETE_ROWS_EVENT - 1] = ROWS_HEADER_LEN_V2;
+                postHeaderLen[GTID_LOG_EVENT - 1] = POST_HEADER_LENGTH;
+                postHeaderLen[ANONYMOUS_GTID_LOG_EVENT - 1] = POST_HEADER_LENGTH;
+                postHeaderLen[PREVIOUS_GTIDS_LOG_EVENT - 1] = IGNORABLE_HEADER_LEN;
+                // mariadb 10
+                postHeaderLen[ANNOTATE_ROWS_EVENT - 1] = ANNOTATE_ROWS_HEADER_LEN;
+                postHeaderLen[BINLOG_CHECKPOINT_EVENT - 1] = BINLOG_CHECKPOINT_HEADER_LEN;
+                postHeaderLen[GTID_EVENT - 1] = GTID_HEADER_LEN;
+                postHeaderLen[GTID_LIST_EVENT - 1] = GTID_LIST_HEADER_LEN;
+                break;
+
+            case 3: /* 4.0.x x>=2 */
+                /*
+                 * We build an artificial (i.e. not sent by the master) event, which describes what those old master
+                 * versions send.
+                 */
+                serverVersion = "4.0";
+                commonHeaderLen = LOG_EVENT_MINIMAL_HEADER_LEN;
+
+                /*
+                 * The first new event in binlog version 4 is Format_desc. So any event type after that does not exist
+                 * in older versions. We use the events known by version 3, even if version 1 had only a subset of them
+                 * (this is not a problem: it uses a few bytes for nothing but unifies code; it does not make the slave
+                 * detect less corruptions).
+                 */
+                numberOfEventTypes = FORMAT_DESCRIPTION_EVENT - 1;
+
+                postHeaderLen[START_EVENT_V3 - 1] = START_V3_HEADER_LEN;
+                postHeaderLen[QUERY_EVENT - 1] = QUERY_HEADER_MINIMAL_LEN;
+                postHeaderLen[ROTATE_EVENT - 1] = ROTATE_HEADER_LEN;
+                postHeaderLen[LOAD_EVENT - 1] = LOAD_HEADER_LEN;
+                postHeaderLen[CREATE_FILE_EVENT - 1] = CREATE_FILE_HEADER_LEN;
+                postHeaderLen[APPEND_BLOCK_EVENT - 1] = APPEND_BLOCK_HEADER_LEN;
+                postHeaderLen[EXEC_LOAD_EVENT - 1] = EXEC_LOAD_HEADER_LEN;
+                postHeaderLen[DELETE_FILE_EVENT - 1] = DELETE_FILE_HEADER_LEN;
+                postHeaderLen[NEW_LOAD_EVENT - 1] = postHeaderLen[LOAD_EVENT - 1];
+                break;
+
+            case 1: /* 3.23 */
+                /*
+                 * We build an artificial (i.e. not sent by the master) event, which describes what those old master
+                 * versions send.
+                 */
+                serverVersion = "3.23";
+                commonHeaderLen = OLD_HEADER_LEN;
+
+                /*
+                 * The first new event in binlog version 4 is Format_desc. So any event type after that does not exist
+                 * in older versions. We use the events known by version 3, even if version 1 had only a subset of them
+                 * (this is not a problem: it uses a few bytes for nothing but unifies code; it does not make the slave
+                 * detect less corruptions).
+                 */
+                numberOfEventTypes = FORMAT_DESCRIPTION_EVENT - 1;
+
+                postHeaderLen[START_EVENT_V3 - 1] = START_V3_HEADER_LEN;
+                postHeaderLen[QUERY_EVENT - 1] = QUERY_HEADER_MINIMAL_LEN;
+                postHeaderLen[LOAD_EVENT - 1] = LOAD_HEADER_LEN;
+                postHeaderLen[CREATE_FILE_EVENT - 1] = CREATE_FILE_HEADER_LEN;
+                postHeaderLen[APPEND_BLOCK_EVENT - 1] = APPEND_BLOCK_HEADER_LEN;
+                postHeaderLen[EXEC_LOAD_EVENT - 1] = EXEC_LOAD_HEADER_LEN;
+                postHeaderLen[DELETE_FILE_EVENT - 1] = DELETE_FILE_HEADER_LEN;
+                postHeaderLen[NEW_LOAD_EVENT - 1] = postHeaderLen[LOAD_EVENT - 1];
+                break;
+
+            default:
+                numberOfEventTypes = 0;
+                commonHeaderLen = 0;
+        }
+    }
+    
+    public void calcServerVersionSplit() {
+        doServerVersionSplit(serverVersion, serverVersionSplit);
+    }
+
+    public long getVersionProduct() {
+        return versionProduct(serverVersionSplit);
+    }
+
+    public boolean isVersionBeforeChecksum() {
+        return getVersionProduct() < checksumVersionProduct;
+    }
+
+    public static void doServerVersionSplit(String serverVersion, int[] versionSplit) {
+        String[] split = serverVersion.split("\\.");
+        if (split.length < 3) {
+            versionSplit[0] = 0;
+            versionSplit[1] = 0;
+            versionSplit[2] = 0;
+        } else {
+            int j = 0;
+            for (int i = 0; i <= 2; i++) {
+                String str = split[i];
+                for (j = 0; j < str.length(); j++) {
+                    if (Character.isDigit(str.charAt(j)) == false) {
+                        break;
+                    }
+                }
+                if (j > 0) {
+                    versionSplit[i] = Integer.valueOf(str.substring(0, j), 10);
+                } else {
+                    versionSplit[0] = 0;
+                    versionSplit[1] = 0;
+                    versionSplit[2] = 0;
+                }
+            }
+        }
+    }
+
+    public static long versionProduct(int[] versionSplit) {
+        return ((versionSplit[0] * 256 + versionSplit[1]) * 256 + versionSplit[2]);
+    }
+
+    public final int getCommonHeaderLen() {
+        return commonHeaderLen;
+    }
+
+    public final short[] getPostHeaderLen() {
+        return postHeaderLen;
+    }
+
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/GtidLogEvent.java

@@ -0,0 +1,45 @@
+package com.taobao.tddl.dbsync.binlog.event;
+
+import com.taobao.tddl.dbsync.binlog.LogBuffer;
+import com.taobao.tddl.dbsync.binlog.LogEvent;
+
+/**
+ * @author jianghang 2013-4-8 上午12:36:29
+ * @version 1.0.3
+ * @since mysql 5.6 / mariadb10
+ */
+public class GtidLogEvent extends LogEvent {
+
+    // / Length of the commit_flag in event encoding
+    public static final int ENCODED_FLAG_LENGTH = 1;
+    // / Length of SID in event encoding
+    public static final int ENCODED_SID_LENGTH  = 16;
+
+    private boolean         commitFlag;
+
+    public GtidLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
+        super(header);
+
+        final int commonHeaderLen = descriptionEvent.commonHeaderLen;
+        // final int postHeaderLen = descriptionEvent.postHeaderLen[header.type
+        // - 1];
+
+        buffer.position(commonHeaderLen);
+        commitFlag = (buffer.getUint8() != 0); // ENCODED_FLAG_LENGTH
+
+        // ignore gtid info read
+        // sid.copy_from((uchar *)ptr_buffer);
+        // ptr_buffer+= ENCODED_SID_LENGTH;
+        //
+        // // SIDNO is only generated if needed, in get_sidno().
+        // spec.gtid.sidno= -1;
+        //
+        // spec.gtid.gno= uint8korr(ptr_buffer);
+        // ptr_buffer+= ENCODED_GNO_LENGTH;
+    }
+
+    public boolean isCommitFlag() {
+        return commitFlag;
+    }
+
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/HeartbeatLogEvent.java

@@ -0,0 +1,49 @@
+package com.taobao.tddl.dbsync.binlog.event;
+
+import com.taobao.tddl.dbsync.binlog.LogBuffer;
+import com.taobao.tddl.dbsync.binlog.LogEvent;
+
+/**
+ * <pre>
+ * Replication event to ensure to slave that master is alive.
+ *   The event is originated by master's dump thread and sent straight to
+ *   slave without being logged. Slave itself does not store it in relay log
+ *   but rather uses a data for immediate checks and throws away the event.
+ * 
+ *   Two members of the class log_ident and Log_event::log_pos comprise 
+ *   @see the event_coordinates instance. The coordinates that a heartbeat
+ *   instance carries correspond to the last event master has sent from
+ *   its binlog.
+ * </pre>
+ * 
+ * @author jianghang 2013-4-8 上午12:36:29
+ * @version 1.0.3
+ * @since mysql 5.6
+ */
+public class HeartbeatLogEvent extends LogEvent {
+
+    public static final int FN_REFLEN = 512; /* Max length of full path-name */
+    private int             identLen;
+    private String          logIdent;
+
+    public HeartbeatLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
+        super(header);
+
+        final int commonHeaderLen = descriptionEvent.commonHeaderLen;
+        identLen = buffer.limit() - commonHeaderLen;
+        if (identLen > FN_REFLEN - 1) {
+            identLen = FN_REFLEN - 1;
+        }
+
+        logIdent = buffer.getFullString(commonHeaderLen, identLen, LogBuffer.ISO_8859_1);
+    }
+
+    public int getIdentLen() {
+        return identLen;
+    }
+
+    public String getLogIdent() {
+        return logIdent;
+    }
+
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/IgnorableLogEvent.java

@@ -0,0 +1,35 @@
+package com.taobao.tddl.dbsync.binlog.event;
+
+import com.taobao.tddl.dbsync.binlog.LogBuffer;
+import com.taobao.tddl.dbsync.binlog.LogEvent;
+
+/**
+ * <pre>
+ *   Base class for ignorable log events. Events deriving from
+  this class can be safely ignored by slaves that cannot
+  recognize them. Newer slaves, will be able to read and
+  handle them. This has been designed to be an open-ended
+  architecture, so adding new derived events shall not harm
+  the old slaves that support ignorable log event mechanism
+  (they will just ignore unrecognized ignorable events).
+
+  @note The only thing that makes an event ignorable is that it has
+  the LOG_EVENT_IGNORABLE_F flag set.  It is not strictly necessary
+  that ignorable event types derive from Ignorable_log_event; they may
+  just as well derive from Log_event and pass LOG_EVENT_IGNORABLE_F as
+  argument to the Log_event constructor.
+  </pre>
+ * 
+ * @author jianghang 2013-4-8 上午12:36:29
+ * @version 1.0.3
+ * @since mysql 5.6
+ */
+public class IgnorableLogEvent extends LogEvent {
+
+    public IgnorableLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
+        super(header);
+
+        // do nothing , just ignore log event
+    }
+
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/IncidentLogEvent.java

@@ -0,0 +1,88 @@
+package com.taobao.tddl.dbsync.binlog.event;
+
+import com.taobao.tddl.dbsync.binlog.LogBuffer;
+import com.taobao.tddl.dbsync.binlog.LogEvent;
+
+/**
+ * Class representing an incident, an occurance out of the ordinary, that
+ * happened on the master.
+ * 
+ * The event is used to inform the slave that something out of the ordinary
+ * happened on the master that might cause the database to be in an inconsistent
+ * state.
+ * 
+ * <table id="IncidentFormat">
+ * <caption>Incident event format</caption>
+ * <tr>
+ * <th>Symbol</th>
+ * <th>Format</th>
+ * <th>Description</th>
+ * </tr>
+ * <tr>
+ * <td>INCIDENT</td>
+ * <td align="right">2</td>
+ * <td>Incident number as an unsigned integer</td>
+ * </tr>
+ * <tr>
+ * <td>MSGLEN</td>
+ * <td align="right">1</td>
+ * <td>Message length as an unsigned integer</td>
+ * </tr>
+ * <tr>
+ * <td>MESSAGE</td>
+ * <td align="right">MSGLEN</td>
+ * <td>The message, if present. Not null terminated.</td>
+ * </tr>
+ * </table>
+ * 
+ * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
+ * @version 1.0
+ */
+public final class IncidentLogEvent extends LogEvent
+{
+    public static final int INCIDENT_NONE        = 0;
+
+    /** There are possibly lost events in the replication stream */
+    public static final int INCIDENT_LOST_EVENTS = 1;
+
+    /** Shall be last event of the enumeration */
+    public static final int INCIDENT_COUNT       = 2;
+
+    private final int       incident;
+    private final String    message;
+
+    public IncidentLogEvent(LogHeader header, LogBuffer buffer,
+            FormatDescriptionLogEvent descriptionEvent)
+    {
+        super(header);
+
+        final int commonHeaderLen = descriptionEvent.commonHeaderLen;
+        final int postHeaderLen = descriptionEvent.postHeaderLen[header.type - 1];
+
+        buffer.position(commonHeaderLen);
+        final int incidentNumber = buffer.getUint16();
+        if (incidentNumber >= INCIDENT_COUNT || incidentNumber <= INCIDENT_NONE)
+        {
+            // If the incident is not recognized, this binlog event is
+            // invalid.  If we set incident_number to INCIDENT_NONE, the
+            // invalidity will be detected by is_valid().
+            incident = INCIDENT_NONE;
+            message = null;
+            return;
+        }
+        incident = incidentNumber;
+
+        buffer.position(commonHeaderLen + postHeaderLen);
+        message = buffer.getString();
+    }
+
+    public final int getIncident()
+    {
+        return incident;
+    }
+
+    public final String getMessage()
+    {
+        return message;
+    }
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/IntvarLogEvent.java

@@ -0,0 +1,98 @@
+package com.taobao.tddl.dbsync.binlog.event;
+
+import com.taobao.tddl.dbsync.binlog.LogBuffer;
+import com.taobao.tddl.dbsync.binlog.LogEvent;
+
+/**
+ * An Intvar_log_event will be created just before a Query_log_event, if the
+ * query uses one of the variables LAST_INSERT_ID or INSERT_ID. Each
+ * Intvar_log_event holds the value of one of these variables.
+ * 
+ * Binary Format
+ * 
+ * The Post-Header for this event type is empty. The Body has two components:
+ * 
+ * <table>
+ * <caption>Body for Intvar_log_event</caption>
+ * 
+ * <tr>
+ * <th>Name</th>
+ * <th>Format</th>
+ * <th>Description</th>
+ * </tr>
+ * 
+ * <tr>
+ * <td>type</td>
+ * <td>1 byte enumeration</td>
+ * <td>One byte identifying the type of variable stored. Currently, two
+ * identifiers are supported: LAST_INSERT_ID_EVENT==1 and INSERT_ID_EVENT==2.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>value</td>
+ * <td>8 byte unsigned integer</td>
+ * <td>The value of the variable.</td>
+ * </tr>
+ * 
+ * </table>
+ * 
+ * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
+ * @version 1.0
+ */
+public final class IntvarLogEvent extends LogEvent
+{
+    /**
+     * Fixed data part: Empty
+     * 
+     * <p>
+     * Variable data part:
+     * 
+     * <ul>
+     * <li>1 byte. A value indicating the variable type: LAST_INSERT_ID_EVENT =
+     * 1 or INSERT_ID_EVENT = 2.</li>
+     * <li>8 bytes. An unsigned integer indicating the value to be used for the
+     * LAST_INSERT_ID() invocation or AUTO_INCREMENT column.</li>
+     * </ul>
+     * 
+     * Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
+     */
+    private final long      value;
+    private final int       type;
+
+    /* Intvar event data */
+    public static final int I_TYPE_OFFSET        = 0;
+    public static final int I_VAL_OFFSET         = 1;
+
+    // enum Int_event_type
+    public static final int INVALID_INT_EVENT    = 0;
+    public static final int LAST_INSERT_ID_EVENT = 1;
+    public static final int INSERT_ID_EVENT      = 2;
+
+    public IntvarLogEvent(LogHeader header, LogBuffer buffer,
+            FormatDescriptionLogEvent descriptionEvent)
+    {
+        super(header);
+
+        /* The Post-Header is empty. The Varible Data part begins immediately. */
+        buffer.position(descriptionEvent.commonHeaderLen
+                + descriptionEvent.postHeaderLen[INTVAR_EVENT - 1]
+                + I_TYPE_OFFSET);
+        type = buffer.getInt8(); // I_TYPE_OFFSET
+        value = buffer.getLong64(); // !uint8korr(buf + I_VAL_OFFSET);
+    }
+
+    public final int getType()
+    {
+        return type;
+    }
+
+    public final long getValue()
+    {
+        return value;
+    }
+
+    public final String getQuery()
+    {
+        return "SET INSERT_ID = " + value;
+    }
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/LoadLogEvent.java

@@ -0,0 +1,391 @@
+package com.taobao.tddl.dbsync.binlog.event;
+
+import com.taobao.tddl.dbsync.binlog.LogBuffer;
+import com.taobao.tddl.dbsync.binlog.LogEvent;
+
+/**
+ * This log event corresponds to a "LOAD DATA INFILE" SQL query on the following
+ * form:
+ * 
+ * <pre>
+ *    (1)    USE db;
+ *    (2)    LOAD DATA [CONCURRENT] [LOCAL] INFILE 'file_name'
+ *    (3)    [REPLACE | IGNORE]
+ *    (4)    INTO TABLE 'table_name'
+ *    (5)    [FIELDS
+ *    (6)      [TERMINATED BY 'field_term']
+ *    (7)      [[OPTIONALLY] ENCLOSED BY 'enclosed']
+ *    (8)      [ESCAPED BY 'escaped']
+ *    (9)    ]
+ *   (10)    [LINES
+ *   (11)      [TERMINATED BY 'line_term']
+ *   (12)      [LINES STARTING BY 'line_start']
+ *   (13)    ]
+ *   (14)    [IGNORE skip_lines LINES]
+ *   (15)    (field_1, field_2, ..., field_n)
+ * </pre>
+ * 
+ * Binary Format: The Post-Header consists of the following six components.
+ * 
+ * <table>
+ * <caption>Post-Header for Load_log_event</caption>
+ * 
+ * <tr>
+ * <th>Name</th>
+ * <th>Format</th>
+ * <th>Description</th>
+ * </tr>
+ * 
+ * <tr>
+ * <td>slave_proxy_id</td>
+ * <td>4 byte unsigned integer</td>
+ * <td>An integer identifying the client thread that issued the query. The id is
+ * unique per server. (Note, however, that two threads on different servers may
+ * have the same slave_proxy_id.) This is used when a client thread creates a
+ * temporary table local to the client. The slave_proxy_id is used to
+ * distinguish temporary tables that belong to different clients.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>exec_time</td>
+ * <td>4 byte unsigned integer</td>
+ * <td>The time from when the query started to when it was logged in the binlog,
+ * in seconds.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>skip_lines</td>
+ * <td>4 byte unsigned integer</td>
+ * <td>The number on line (14) above, if present, or 0 if line (14) is left out.
+ * </td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>table_name_len</td>
+ * <td>1 byte unsigned integer</td>
+ * <td>The length of 'table_name' on line (4) above.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>db_len</td>
+ * <td>1 byte unsigned integer</td>
+ * <td>The length of 'db' on line (1) above.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>num_fields</td>
+ * <td>4 byte unsigned integer</td>
+ * <td>The number n of fields on line (15) above.</td>
+ * </tr>
+ * </table>
+ * 
+ * The Body contains the following components.
+ * 
+ * <table>
+ * <caption>Body of Load_log_event</caption>
+ * 
+ * <tr>
+ * <th>Name</th>
+ * <th>Format</th>
+ * <th>Description</th>
+ * </tr>
+ * 
+ * <tr>
+ * <td>sql_ex</td>
+ * <td>variable length</td>
+ * 
+ * <td>Describes the part of the query on lines (3) and (5)&ndash;(13) above.
+ * More precisely, it stores the five strings (on lines) field_term (6),
+ * enclosed (7), escaped (8), line_term (11), and line_start (12); as well as a
+ * bitfield indicating the presence of the keywords REPLACE (3), IGNORE (3), and
+ * OPTIONALLY (7).
+ * 
+ * The data is stored in one of two formats, called "old" and "new". The type
+ * field of Common-Header determines which of these two formats is used: type
+ * LOAD_EVENT means that the old format is used, and type NEW_LOAD_EVENT means
+ * that the new format is used. When MySQL writes a Load_log_event, it uses the
+ * new format if at least one of the five strings is two or more bytes long.
+ * Otherwise (i.e., if all strings are 0 or 1 bytes long), the old format is
+ * used.
+ * 
+ * The new and old format differ in the way the five strings are stored.
+ * 
+ * <ul>
+ * <li>In the new format, the strings are stored in the order field_term,
+ * enclosed, escaped, line_term, line_start. Each string consists of a length (1
+ * byte), followed by a sequence of characters (0-255 bytes). Finally, a boolean
+ * combination of the following flags is stored in 1 byte: REPLACE_FLAG==0x4,
+ * IGNORE_FLAG==0x8, and OPT_ENCLOSED_FLAG==0x2. If a flag is set, it indicates
+ * the presence of the corresponding keyword in the SQL query.
+ * 
+ * <li>In the old format, we know that each string has length 0 or 1. Therefore,
+ * only the first byte of each string is stored. The order of the strings is the
+ * same as in the new format. These five bytes are followed by the same 1 byte
+ * bitfield as in the new format. Finally, a 1 byte bitfield called empty_flags
+ * is stored. The low 5 bits of empty_flags indicate which of the five strings
+ * have length 0. For each of the following flags that is set, the corresponding
+ * string has length 0; for the flags that are not set, the string has length 1:
+ * FIELD_TERM_EMPTY==0x1, ENCLOSED_EMPTY==0x2, LINE_TERM_EMPTY==0x4,
+ * LINE_START_EMPTY==0x8, ESCAPED_EMPTY==0x10.
+ * </ul>
+ * 
+ * Thus, the size of the new format is 6 bytes + the sum of the sizes of the
+ * five strings. The size of the old format is always 7 bytes.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>field_lens</td>
+ * <td>num_fields 1 byte unsigned integers</td>
+ * <td>An array of num_fields integers representing the length of each field in
+ * the query. (num_fields is from the Post-Header).</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>fields</td>
+ * <td>num_fields null-terminated strings</td>
+ * <td>An array of num_fields null-terminated strings, each representing a field
+ * in the query. (The trailing zero is redundant, since the length are stored in
+ * the num_fields array.) The total length of all strings equals to the sum of
+ * all field_lens, plus num_fields bytes for all the trailing zeros.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>table_name</td>
+ * <td>null-terminated string of length table_len+1 bytes</td>
+ * <td>The 'table_name' from the query, as a null-terminated string. (The
+ * trailing zero is actually redundant since the table_len is known from
+ * Post-Header.)</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>db</td>
+ * <td>null-terminated string of length db_len+1 bytes</td>
+ * <td>The 'db' from the query, as a null-terminated string. (The trailing zero
+ * is actually redundant since the db_len is known from Post-Header.)</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>file_name</td>
+ * <td>variable length string without trailing zero, extending to the end of the
+ * event (determined by the length field of the Common-Header)</td>
+ * <td>The 'file_name' from the query.</td>
+ * </tr>
+ * 
+ * </table>
+ * 
+ * This event type is understood by current versions, but only generated by
+ * MySQL 3.23 and earlier.
+ * 
+ * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
+ * @version 1.0
+ */
+public class LoadLogEvent extends LogEvent
+{
+    private String          table;
+    private String          db;
+    private String          fname;
+    private int             skipLines;
+    private int             numFields;
+    private String[]        fields;
+
+    /* sql_ex_info */
+    private String          fieldTerm;
+    private String          lineTerm;
+    private String          lineStart;
+    private String          enclosed;
+    private String          escaped;
+    private int             optFlags;
+    private int             emptyFlags;
+
+    private long            execTime;
+
+    /* Load event post-header */
+    public static final int L_THREAD_ID_OFFSET  = 0;
+    public static final int L_EXEC_TIME_OFFSET  = 4;
+    public static final int L_SKIP_LINES_OFFSET = 8;
+    public static final int L_TBL_LEN_OFFSET    = 12;
+    public static final int L_DB_LEN_OFFSET     = 13;
+    public static final int L_NUM_FIELDS_OFFSET = 14;
+    public static final int L_SQL_EX_OFFSET     = 18;
+    public static final int L_DATA_OFFSET       = FormatDescriptionLogEvent.LOAD_HEADER_LEN;
+
+    /*
+      These are flags and structs to handle all the LOAD DATA INFILE options (LINES
+      TERMINATED etc).
+      DUMPFILE_FLAG is probably useless (DUMPFILE is a clause of SELECT, not of LOAD
+      DATA).
+    */
+    public static final int DUMPFILE_FLAG       = 0x1;
+    public static final int OPT_ENCLOSED_FLAG   = 0x2;
+    public static final int REPLACE_FLAG        = 0x4;
+    public static final int IGNORE_FLAG         = 0x8;
+
+    public static final int FIELD_TERM_EMPTY    = 0x1;
+    public static final int ENCLOSED_EMPTY      = 0x2;
+    public static final int LINE_TERM_EMPTY     = 0x4;
+    public static final int LINE_START_EMPTY    = 0x8;
+    public static final int ESCAPED_EMPTY       = 0x10;
+
+    public LoadLogEvent(LogHeader header, LogBuffer buffer,
+            FormatDescriptionLogEvent descriptionEvent)
+    {
+        super(header);
+
+        final int loadHeaderLen = FormatDescriptionLogEvent.LOAD_HEADER_LEN;
+        /*
+         * I (Guilhem) manually tested replication of LOAD DATA INFILE for
+         * 3.23->5.0, 4.0->5.0 and 5.0->5.0 and it works.
+         */
+        copyLogEvent(buffer, ((header.type == LOAD_EVENT) ? loadHeaderLen
+                + descriptionEvent.commonHeaderLen : loadHeaderLen
+                + FormatDescriptionLogEvent.LOG_EVENT_HEADER_LEN),
+                descriptionEvent);
+    }
+
+    /**
+     * @see mysql-5.1.60/sql/log_event.cc - Load_log_event::copy_log_event
+     */
+    protected final void copyLogEvent(LogBuffer buffer, final int bodyOffset,
+            FormatDescriptionLogEvent descriptionEvent)
+    {
+        /* this is the beginning of the post-header */
+        buffer.position(descriptionEvent.commonHeaderLen + L_EXEC_TIME_OFFSET);
+
+        execTime = buffer.getUint32(); // L_EXEC_TIME_OFFSET
+        skipLines = (int) buffer.getUint32(); // L_SKIP_LINES_OFFSET
+        final int tableNameLen = buffer.getUint8(); // L_TBL_LEN_OFFSET
+        final int dbLen = buffer.getUint8(); // L_DB_LEN_OFFSET
+        numFields = (int) buffer.getUint32(); // L_NUM_FIELDS_OFFSET
+
+        buffer.position(bodyOffset);
+        /*
+         * Sql_ex.init() on success returns the pointer to the first byte after
+         * the sql_ex structure, which is the start of field lengths array.
+         */
+        if (header.type != LOAD_EVENT /* use_new_format */)
+        {
+            /*
+             * The code below assumes that buf will not disappear from under our
+             * feet during the lifetime of the event. This assumption holds true
+             * in the slave thread if the log is in new format, but is not the
+             * case when we have old format because we will be reusing net
+             * buffer to read the actual file before we write out the
+             * Create_file event.
+             */
+            fieldTerm = buffer.getString();
+            enclosed = buffer.getString();
+            lineTerm = buffer.getString();
+            lineStart = buffer.getString();
+            escaped = buffer.getString();
+            optFlags = buffer.getInt8();
+            emptyFlags = 0;
+        }
+        else
+        {
+            fieldTerm = buffer.getFixString(1);
+            enclosed = buffer.getFixString(1);
+            lineTerm = buffer.getFixString(1);
+            lineStart = buffer.getFixString(1);
+            escaped = buffer.getFixString(1);
+            optFlags = buffer.getUint8();
+            emptyFlags = buffer.getUint8();
+
+            if ((emptyFlags & FIELD_TERM_EMPTY) != 0)
+                fieldTerm = null;
+            if ((emptyFlags & ENCLOSED_EMPTY) != 0)
+                enclosed = null;
+            if ((emptyFlags & LINE_TERM_EMPTY) != 0)
+                lineTerm = null;
+            if ((emptyFlags & LINE_START_EMPTY) != 0)
+                lineStart = null;
+            if ((emptyFlags & ESCAPED_EMPTY) != 0)
+                escaped = null;
+        }
+
+        final int fieldLenPos = buffer.position();
+        buffer.forward(numFields);
+        fields = new String[numFields];
+        for (int i = 0; i < numFields; i++)
+        {
+            final int fieldLen = buffer.getUint8(fieldLenPos + i);
+            fields[i] = buffer.getFixString(fieldLen + 1);
+        }
+
+        table = buffer.getFixString(tableNameLen + 1);
+        db = buffer.getFixString(dbLen + 1);
+
+        // null termination is accomplished by the caller 
+        final int from = buffer.position();
+        final int end = from + buffer.limit();
+        int found = from;
+        for (; (found < end) && buffer.getInt8(found) != '\0'; found++)
+            /* empty loop */;
+        fname = buffer.getString(found);
+        buffer.forward(1); // The + 1 is for \0 terminating fname 
+    }
+
+    public final String getTable()
+    {
+        return table;
+    }
+
+    public final String getDb()
+    {
+        return db;
+    }
+
+    public final String getFname()
+    {
+        return fname;
+    }
+
+    public final int getSkipLines()
+    {
+        return skipLines;
+    }
+
+    public final String[] getFields()
+    {
+        return fields;
+    }
+
+    public final String getFieldTerm()
+    {
+        return fieldTerm;
+    }
+
+    public final String getLineTerm()
+    {
+        return lineTerm;
+    }
+
+    public final String getLineStart()
+    {
+        return lineStart;
+    }
+
+    public final String getEnclosed()
+    {
+        return enclosed;
+    }
+
+    public final String getEscaped()
+    {
+        return escaped;
+    }
+
+    public final int getOptFlags()
+    {
+        return optFlags;
+    }
+
+    public final int getEmptyFlags()
+    {
+        return emptyFlags;
+    }
+
+    public final long getExecTime()
+    {
+        return execTime;
+    }
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/LogHeader.java

@@ -0,0 +1,310 @@
+package com.taobao.tddl.dbsync.binlog.event;
+
+import com.taobao.tddl.dbsync.binlog.LogBuffer;
+import com.taobao.tddl.dbsync.binlog.LogEvent;
+
+/**
+ * The Common-Header, documented in the table @ref Table_common_header "below",
+ * always has the same form and length within one version of MySQL. Each event
+ * type specifies a format and length of the Post-Header. The length of the
+ * Common-Header is the same for all events of the same type. The Body may be of
+ * different format and length even for different events of the same type. The
+ * binary formats of Post-Header and Body are documented separately in each
+ * subclass. The binary format of Common-Header is as follows.
+ * 
+ * <table>
+ * <caption>Common-Header</caption>
+ * 
+ * <tr>
+ * <th>Name</th>
+ * <th>Format</th>
+ * <th>Description</th>
+ * </tr>
+ * 
+ * <tr>
+ * <td>timestamp</td>
+ * <td>4 byte unsigned integer</td>
+ * <td>The time when the query started, in seconds since 1970.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>type</td>
+ * <td>1 byte enumeration</td>
+ * <td>See enum #Log_event_type.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>server_id</td>
+ * <td>4 byte unsigned integer</td>
+ * <td>Server ID of the server that created the event.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>total_size</td>
+ * <td>4 byte unsigned integer</td>
+ * <td>The total size of this event, in bytes. In other words, this is the sum
+ * of the sizes of Common-Header, Post-Header, and Body.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>master_position</td>
+ * <td>4 byte unsigned integer</td>
+ * <td>The position of the next event in the master binary log, in bytes from
+ * the beginning of the file. In a binlog that is not a relay log, this is just
+ * the position of the next event, in bytes from the beginning of the file. In a
+ * relay log, this is the position of the next event in the master's binlog.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>flags</td>
+ * <td>2 byte bitfield</td>
+ * <td>See Log_event::flags.</td>
+ * </tr>
+ * </table>
+ * 
+ * Summing up the numbers above, we see that the total size of the common header
+ * is 19 bytes.
+ * 
+ * @see mysql-5.1.60/sql/log_event.cc
+ * 
+ * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
+ * @version 1.0
+ */
+public final class LogHeader
+{
+    protected final int type;
+
+    /**
+     * The offset in the log where this event originally appeared (it is
+     * preserved in relay logs, making SHOW SLAVE STATUS able to print
+     * coordinates of the event in the master's binlog). Note: when a
+     * transaction is written by the master to its binlog (wrapped in
+     * BEGIN/COMMIT) the log_pos of all the queries it contains is the one of
+     * the BEGIN (this way, when one does SHOW SLAVE STATUS it sees the offset
+     * of the BEGIN, which is logical as rollback may occur), except the COMMIT
+     * query which has its real offset.
+     */
+    protected long      logPos;
+
+    /**
+     * Timestamp on the master(for debugging and replication of
+     * NOW()/TIMESTAMP). It is important for queries and LOAD DATA INFILE. This
+     * is set at the event's creation time, except for Query and Load (et al.)
+     * events where this is set at the query's execution time, which guarantees
+     * good replication (otherwise, we could have a query and its event with
+     * different timestamps).
+     */
+    protected long      when;
+
+    /** Number of bytes written by write() function */
+    protected int       eventLen;
+
+    /**
+     * The master's server id (is preserved in the relay log; used to prevent
+     * from infinite loops in circular replication).
+     */
+    protected long      serverId;
+
+    /**
+     * Some 16 flags. See the definitions above for LOG_EVENT_TIME_F,
+     * LOG_EVENT_FORCED_ROTATE_F, LOG_EVENT_THREAD_SPECIFIC_F, and
+     * LOG_EVENT_SUPPRESS_USE_F for notes.
+     */
+    protected int       flags;
+    
+    /** 
+     * The value is set by caller of FD constructor and
+     * Log_event::write_header() for the rest.
+     * In the FD case it's propagated into the last byte 
+     * of post_header_len[] at FD::write().
+     * On the slave side the value is assigned from post_header_len[last] 
+     * of the last seen FD event.
+     */
+    protected int       checksumAlg;
+    /**
+       Placeholder for event checksum while writing to binlog.
+    */
+    protected long      crc;      // ha_checksum
+
+    /* for Start_event_v3 */
+    public LogHeader(final int type)
+    {
+        this.type = type;
+    }
+
+    public LogHeader(LogBuffer buffer,
+            FormatDescriptionLogEvent descriptionEvent)
+    {
+        when = buffer.getUint32();
+        type = buffer.getUint8(); // LogEvent.EVENT_TYPE_OFFSET;
+        serverId = buffer.getUint32(); // LogEvent.SERVER_ID_OFFSET;
+        eventLen = (int) buffer.getUint32(); // LogEvent.EVENT_LEN_OFFSET;
+        
+        if (descriptionEvent.binlogVersion == 1)
+        {
+            logPos = 0;
+            flags = 0;
+            return;
+        }
+
+        /* 4.0 or newer */
+        logPos = buffer.getUint32(); // LogEvent.LOG_POS_OFFSET
+        /*
+          If the log is 4.0 (so here it can only be a 4.0 relay log read by
+          the SQL thread or a 4.0 master binlog read by the I/O thread),
+          log_pos is the beginning of the event: we transform it into the end
+          of the event, which is more useful.
+          But how do you know that the log is 4.0: you know it if
+          description_event is version 3 *and* you are not reading a
+          Format_desc (remember that mysqlbinlog starts by assuming that 5.0
+          logs are in 4.0 format, until it finds a Format_desc).
+        */
+        if (descriptionEvent.binlogVersion == 3
+                && type < LogEvent.FORMAT_DESCRIPTION_EVENT && logPos != 0)
+        {
+            /*
+              If log_pos=0, don't change it. log_pos==0 is a marker to mean
+              "don't change rli->group_master_log_pos" (see
+              inc_group_relay_log_pos()). As it is unreal log_pos, adding the
+              event len's is nonsense. For example, a fake Rotate event should
+              not have its log_pos (which is 0) changed or it will modify
+              Exec_master_log_pos in SHOW SLAVE STATUS, displaying a nonsense
+              value of (a non-zero offset which does not exist in the master's
+              binlog, so which will cause problems if the user uses this value
+              in CHANGE MASTER).
+            */
+            logPos += eventLen; /* purecov: inspected */
+        }
+
+        flags = buffer.getUint16(); // LogEvent.FLAGS_OFFSET
+        if ((type == LogEvent.FORMAT_DESCRIPTION_EVENT)
+                || (type == LogEvent.ROTATE_EVENT))
+        {
+            /*
+              These events always have a header which stops here (i.e. their
+              header is FROZEN).
+            */
+            /*
+              Initialization to zero of all other Log_event members as they're
+              not specified. Currently there are no such members; in the future
+              there will be an event UID (but Format_description and Rotate
+              don't need this UID, as they are not propagated through
+              --log-slave-updates (remember the UID is used to not play a query
+              twice when you have two masters which are slaves of a 3rd master).
+              Then we are done.
+            */
+            
+            if (type == LogEvent.FORMAT_DESCRIPTION_EVENT) {
+                int commonHeaderLen = buffer.getUint8(FormatDescriptionLogEvent.LOG_EVENT_MINIMAL_HEADER_LEN
+                                                      + FormatDescriptionLogEvent.ST_COMMON_HEADER_LEN_OFFSET);
+                buffer.position(commonHeaderLen + FormatDescriptionLogEvent.ST_SERVER_VER_OFFSET);
+                String serverVersion = buffer.getFixString(FormatDescriptionLogEvent.ST_SERVER_VER_LEN); // ST_SERVER_VER_OFFSET
+                int versionSplit[] = new int[] { 0, 0, 0 };
+                FormatDescriptionLogEvent.doServerVersionSplit(serverVersion, versionSplit);
+                checksumAlg = LogEvent.BINLOG_CHECKSUM_ALG_UNDEF;
+                if (FormatDescriptionLogEvent.versionProduct(versionSplit) >= FormatDescriptionLogEvent.checksumVersionProduct) {
+                    buffer.position(eventLen - LogEvent.BINLOG_CHECKSUM_LEN - LogEvent.BINLOG_CHECKSUM_ALG_DESC_LEN);
+                    checksumAlg = buffer.getUint8();
+                }
+
+                processCheckSum(buffer);
+            }
+            return;
+        }
+        
+        /*
+        CRC verification by SQL and Show-Binlog-Events master side.
+        The caller has to provide @description_event->checksum_alg to
+        be the last seen FD's (A) descriptor.
+        If event is FD the descriptor is in it.
+        Notice, FD of the binlog can be only in one instance and therefore
+        Show-Binlog-Events executing master side thread needs just to know
+        the only FD's (A) value -  whereas RL can contain more.
+        In the RL case, the alg is kept in FD_e (@description_event) which is reset 
+        to the newer read-out event after its execution with possibly new alg descriptor.
+        Therefore in a typical sequence of RL:
+        {FD_s^0, FD_m, E_m^1} E_m^1 
+        will be verified with (A) of FD_m.
+
+        See legends definition on MYSQL_BIN_LOG::relay_log_checksum_alg docs
+        lines (log.h).
+
+        Notice, a pre-checksum FD version forces alg := BINLOG_CHECKSUM_ALG_UNDEF.
+        */
+        checksumAlg = descriptionEvent.getHeader().checksumAlg; // fetch checksum alg
+        processCheckSum(buffer);
+        /* otherwise, go on with reading the header from buf (nothing now) */
+    }
+
+    /**
+     * The different types of log events.
+     */
+    public final int getType()
+    {
+        return type;
+    }
+
+    /**
+     * The position of the next event in the master binary log, in bytes from
+     * the beginning of the file. In a binlog that is not a relay log, this is
+     * just the position of the next event, in bytes from the beginning of the
+     * file. In a relay log, this is the position of the next event in the
+     * master's binlog.
+     */
+    public final long getLogPos()
+    {
+        return logPos;
+    }
+
+    /**
+     * The total size of this event, in bytes. In other words, this is the sum
+     * of the sizes of Common-Header, Post-Header, and Body.
+     */
+    public final int getEventLen()
+    {
+        return eventLen;
+    }
+
+    /**
+     * The time when the query started, in seconds since 1970.
+     */
+    public final long getWhen()
+    {
+        return when;
+    }
+
+    /**
+     * Server ID of the server that created the event.
+     */
+    public final long getServerId()
+    {
+        return serverId;
+    }
+
+    /**
+     * Some 16 flags. See the definitions above for LOG_EVENT_TIME_F,
+     * LOG_EVENT_FORCED_ROTATE_F, LOG_EVENT_THREAD_SPECIFIC_F, and
+     * LOG_EVENT_SUPPRESS_USE_F for notes.
+     */
+    public final int getFlags()
+    {
+        return flags;
+    }
+
+    
+    public long getCrc() {
+        return crc;
+    }
+    
+    
+    public int getChecksumAlg() {
+        return checksumAlg;
+    }
+
+    private void processCheckSum(LogBuffer buffer) {
+        if (checksumAlg != LogEvent.BINLOG_CHECKSUM_ALG_OFF &&
+                checksumAlg != LogEvent.BINLOG_CHECKSUM_ALG_UNDEF){
+            crc = buffer.getUint32(eventLen -  LogEvent.BINLOG_CHECKSUM_LEN);
+        }
+    }
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/PreviousGtidsLogEvent.java

@@ -0,0 +1,19 @@
+package com.taobao.tddl.dbsync.binlog.event;
+
+import com.taobao.tddl.dbsync.binlog.LogBuffer;
+import com.taobao.tddl.dbsync.binlog.LogEvent;
+
+/**
+ * 
+ * @author jianghang 2013-4-8 上午12:36:29
+ * @version 1.0.3
+ * @since mysql 5.6
+ */
+public class PreviousGtidsLogEvent  extends LogEvent{
+
+    public PreviousGtidsLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
+        super(header);
+        // do nothing , just for mysql gtid search function
+    }
+}
+

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/QueryLogEvent.java

@@ -0,0 +1,920 @@
+package com.taobao.tddl.dbsync.binlog.event;
+
+import java.io.IOException;
+import java.nio.charset.Charset;
+
+import com.taobao.tddl.dbsync.binlog.CharsetConversion;
+import com.taobao.tddl.dbsync.binlog.LogBuffer;
+import com.taobao.tddl.dbsync.binlog.LogEvent;
+
+/**
+ * A Query_log_event is created for each query that modifies the database,
+ * unless the query is logged row-based.
+ * 
+ * The Post-Header has five components:
+ * 
+ * <table>
+ * <caption>Post-Header for Query_log_event</caption>
+ * 
+ * <tr>
+ * <th>Name</th>
+ * <th>Format</th>
+ * <th>Description</th>
+ * </tr>
+ * 
+ * <tr>
+ * <td>slave_proxy_id</td>
+ * <td>4 byte unsigned integer</td>
+ * <td>An integer identifying the client thread that issued the query. The id is
+ * unique per server. (Note, however, that two threads on different servers may
+ * have the same slave_proxy_id.) This is used when a client thread creates a
+ * temporary table local to the client. The slave_proxy_id is used to
+ * distinguish temporary tables that belong to different clients.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>exec_time</td>
+ * <td>4 byte unsigned integer</td>
+ * <td>The time from when the query started to when it was logged in the binlog,
+ * in seconds.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>db_len</td>
+ * <td>1 byte integer</td>
+ * <td>The length of the name of the currently selected database.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>error_code</td>
+ * <td>2 byte unsigned integer</td>
+ * <td>Error code generated by the master. If the master fails, the slave will
+ * fail with the same error code, except for the error codes ER_DB_CREATE_EXISTS
+ * == 1007 and ER_DB_DROP_EXISTS == 1008.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>status_vars_len</td>
+ * <td>2 byte unsigned integer</td>
+ * <td>The length of the status_vars block of the Body, in bytes. See
+ * query_log_event_status_vars "below".</td>
+ * </tr>
+ * </table>
+ * 
+ * The Body has the following components:
+ * 
+ * <table>
+ * <caption>Body for Query_log_event</caption>
+ * 
+ * <tr>
+ * <th>Name</th>
+ * <th>Format</th>
+ * <th>Description</th>
+ * </tr>
+ * 
+ * <tr>
+ * <td>query_log_event_status_vars status_vars</td>
+ * <td>status_vars_len bytes</td>
+ * <td>Zero or more status variables. Each status variable consists of one byte
+ * identifying the variable stored, followed by the value of the variable. The
+ * possible variables are listed separately in the table
+ * Table_query_log_event_status_vars "below". MySQL always writes events in the
+ * order defined below; however, it is capable of reading them in any order.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>db</td>
+ * <td>db_len+1</td>
+ * <td>The currently selected database, as a null-terminated string.
+ * 
+ * (The trailing zero is redundant since the length is already known; it is
+ * db_len from Post-Header.)</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>query</td>
+ * <td>variable length string without trailing zero, extending to the end of the
+ * event (determined by the length field of the Common-Header)</td>
+ * <td>The SQL query.</td>
+ * </tr>
+ * </table>
+ * 
+ * The following table lists the status variables that may appear in the
+ * status_vars field. Table_query_log_event_status_vars
+ * 
+ * <table>
+ * <caption>Status variables for Query_log_event</caption>
+ * 
+ * <tr>
+ * <th>Status variable</th>
+ * <th>1 byte identifier</th>
+ * <th>Format</th>
+ * <th>Description</th>
+ * </tr>
+ * 
+ * <tr>
+ * <td>flags2</td>
+ * <td>Q_FLAGS2_CODE == 0</td>
+ * <td>4 byte bitfield</td>
+ * <td>The flags in thd->options, binary AND-ed with OPTIONS_WRITTEN_TO_BIN_LOG.
+ * The thd->options bitfield contains options for "SELECT". OPTIONS_WRITTEN
+ * identifies those options that need to be written to the binlog (not all do).
+ * Specifically, OPTIONS_WRITTEN_TO_BIN_LOG equals (OPTION_AUTO_IS_NULL |
+ * OPTION_NO_FOREIGN_KEY_CHECKS | OPTION_RELAXED_UNIQUE_CHECKS |
+ * OPTION_NOT_AUTOCOMMIT), or 0x0c084000 in hex.
+ * 
+ * These flags correspond to the SQL variables SQL_AUTO_IS_NULL,
+ * FOREIGN_KEY_CHECKS, UNIQUE_CHECKS, and AUTOCOMMIT, documented in the
+ * "SET Syntax" section of the MySQL Manual.
+ * 
+ * This field is always written to the binlog in version >= 5.0, and never
+ * written in version < 5.0.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>sql_mode</td>
+ * <td>Q_SQL_MODE_CODE == 1</td>
+ * <td>8 byte bitfield</td>
+ * <td>The sql_mode variable. See the section "SQL Modes" in the MySQL manual,
+ * and see mysql_priv.h for a list of the possible flags. Currently
+ * (2007-10-04), the following flags are available:
+ * 
+ * <pre>
+ *     MODE_REAL_AS_FLOAT==0x1
+ *     MODE_PIPES_AS_CONCAT==0x2
+ *     MODE_ANSI_QUOTES==0x4
+ *     MODE_IGNORE_SPACE==0x8
+ *     MODE_NOT_USED==0x10
+ *     MODE_ONLY_FULL_GROUP_BY==0x20
+ *     MODE_NO_UNSIGNED_SUBTRACTION==0x40
+ *     MODE_NO_DIR_IN_CREATE==0x80
+ *     MODE_POSTGRESQL==0x100
+ *     MODE_ORACLE==0x200
+ *     MODE_MSSQL==0x400
+ *     MODE_DB2==0x800
+ *     MODE_MAXDB==0x1000
+ *     MODE_NO_KEY_OPTIONS==0x2000
+ *     MODE_NO_TABLE_OPTIONS==0x4000
+ *     MODE_NO_FIELD_OPTIONS==0x8000
+ *     MODE_MYSQL323==0x10000
+ *     MODE_MYSQL323==0x20000
+ *     MODE_MYSQL40==0x40000
+ *     MODE_ANSI==0x80000
+ *     MODE_NO_AUTO_VALUE_ON_ZERO==0x100000
+ *     MODE_NO_BACKSLASH_ESCAPES==0x200000
+ *     MODE_STRICT_TRANS_TABLES==0x400000
+ *     MODE_STRICT_ALL_TABLES==0x800000
+ *     MODE_NO_ZERO_IN_DATE==0x1000000
+ *     MODE_NO_ZERO_DATE==0x2000000
+ *     MODE_INVALID_DATES==0x4000000
+ *     MODE_ERROR_FOR_DIVISION_BY_ZERO==0x8000000
+ *     MODE_TRADITIONAL==0x10000000
+ *     MODE_NO_AUTO_CREATE_USER==0x20000000
+ *     MODE_HIGH_NOT_PRECEDENCE==0x40000000
+ *     MODE_PAD_CHAR_TO_FULL_LENGTH==0x80000000
+ * </pre>
+ * 
+ * All these flags are replicated from the server. However, all flags except
+ * MODE_NO_DIR_IN_CREATE are honored by the slave; the slave always preserves
+ * its old value of MODE_NO_DIR_IN_CREATE. For a rationale, see comment in
+ * Query_log_event::do_apply_event in log_event.cc.
+ * 
+ * This field is always written to the binlog.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>catalog</td>
+ * <td>Q_CATALOG_NZ_CODE == 6</td>
+ * <td>Variable-length string: the length in bytes (1 byte) followed by the
+ * characters (at most 255 bytes)</td>
+ * <td>Stores the client's current catalog. Every database belongs to a catalog,
+ * the same way that every table belongs to a database. Currently, there is only
+ * one catalog, "std".
+ * 
+ * This field is written if the length of the catalog is > 0; otherwise it is
+ * not written.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>auto_increment</td>
+ * <td>Q_AUTO_INCREMENT == 3</td>
+ * <td>two 2 byte unsigned integers, totally 2+2=4 bytes</td>
+ * 
+ * <td>The two variables auto_increment_increment and auto_increment_offset, in
+ * that order. For more information, see "System variables" in the MySQL manual.
+ * 
+ * This field is written if auto_increment > 1. Otherwise, it is not written.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>charset</td>
+ * <td>Q_CHARSET_CODE == 4</td>
+ * <td>three 2 byte unsigned integers, totally 2+2+2=6 bytes</td>
+ * <td>The three variables character_set_client, collation_connection, and
+ * collation_server, in that order. character_set_client is a code identifying
+ * the character set and collation used by the client to encode the query.
+ * collation_connection identifies the character set and collation that the
+ * master converts the query to when it receives it; this is useful when
+ * comparing literal strings. collation_server is the default character set and
+ * collation used when a new database is created.
+ * 
+ * See also "Connection Character Sets and Collations" in the MySQL 5.1 manual.
+ * 
+ * All three variables are codes identifying a (character set, collation) pair.
+ * To see which codes map to which pairs, run the query "SELECT id,
+ * character_set_name, collation_name FROM COLLATIONS".
+ * 
+ * Cf. Q_CHARSET_DATABASE_CODE below.
+ * 
+ * This field is always written.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>time_zone</td>
+ * <td>Q_TIME_ZONE_CODE == 5</td>
+ * <td>Variable-length string: the length in bytes (1 byte) followed by the
+ * characters (at most 255 bytes).
+ * <td>The time_zone of the master.
+ * 
+ * See also "System Variables" and "MySQL Server Time Zone Support" in the MySQL
+ * manual.
+ * 
+ * This field is written if the length of the time zone string is > 0;
+ * otherwise, it is not written.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>lc_time_names_number</td>
+ * <td>Q_LC_TIME_NAMES_CODE == 7</td>
+ * <td>2 byte integer</td>
+ * <td>A code identifying a table of month and day names. The mapping from codes
+ * to languages is defined in sql_locale.cc. This field is written if it is not
+ * 0, i.e., if the locale is not en_US.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>charset_database_number</td>
+ * <td>Q_CHARSET_DATABASE_CODE == 8</td>
+ * <td>2 byte integer</td>
+ * 
+ * <td>The value of the collation_database system variable (in the source code
+ * stored in thd->variables.collation_database), which holds the code for a
+ * (character set, collation) pair as described above (see Q_CHARSET_CODE).
+ * 
+ * collation_database was used in old versions (???WHEN). Its value was loaded
+ * when issuing a "use db" query and could be changed by issuing a
+ * "SET collation_database=xxx" query. It used to affect the "LOAD DATA INFILE"
+ * and "CREATE TABLE" commands.
+ * 
+ * In newer versions, "CREATE TABLE" has been changed to take the character set
+ * from the database of the created table, rather than the character set of the
+ * current database. This makes a difference when creating a table in another
+ * database than the current one. "LOAD DATA INFILE" has not yet changed to do
+ * this, but there are plans to eventually do it, and to make collation_database
+ * read-only.
+ * 
+ * This field is written if it is not 0.</td>
+ * </tr>
+ * <tr>
+ * <td>table_map_for_update</td>
+ * <td>Q_TABLE_MAP_FOR_UPDATE_CODE == 9</td>
+ * <td>8 byte integer</td>
+ * 
+ * <td>The value of the table map that is to be updated by the multi-table
+ * update query statement. Every bit of this variable represents a table, and is
+ * set to 1 if the corresponding table is to be updated by this statement.
+ * 
+ * The value of this variable is set when executing a multi-table update
+ * statement and used by slave to apply filter rules without opening all the
+ * tables on slave. This is required because some tables may not exist on slave
+ * because of the filter rules.</td>
+ * </tr>
+ * </table>
+ * 
+ * Query_log_event_notes_on_previous_versions Notes on Previous Versions
+ * 
+ * Status vars were introduced in version 5.0. To read earlier versions
+ * correctly, check the length of the Post-Header.
+ * 
+ * The status variable Q_CATALOG_CODE == 2 existed in MySQL 5.0.x, where
+ * 0<=x<=3. It was identical to Q_CATALOG_CODE, except that the string had a
+ * trailing '\0'. The '\0' was removed in 5.0.4 since it was redundant (the
+ * string length is stored before the string). The Q_CATALOG_CODE will never be
+ * written by a new master, but can still be understood by a new slave.
+ * 
+ * See Q_CHARSET_DATABASE_CODE in the table above.
+ * 
+ * When adding new status vars, please don't forget to update the
+ * MAX_SIZE_LOG_EVENT_STATUS, and update function code_name
+ * 
+ * @see mysql-5.1.6/sql/logevent.cc - Query_log_event
+ * 
+ * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
+ * @version 1.0
+ */
+public class QueryLogEvent extends LogEvent
+{
+    /**
+    The maximum number of updated databases that a status of
+    Query-log-event can carry.  It can redefined within a range
+    [1.. OVER_MAX_DBS_IN_EVENT_MTS].
+     */
+    public static final int MAX_DBS_IN_EVENT_MTS      = 16;
+    
+    /**
+    When the actual number of databases exceeds MAX_DBS_IN_EVENT_MTS
+    the value of OVER_MAX_DBS_IN_EVENT_MTS is is put into the
+    mts_accessed_dbs status.
+     */
+    public static final int OVER_MAX_DBS_IN_EVENT_MTS = 254;
+    
+    
+    public static final int SYSTEM_CHARSET_MBMAXLEN   = 3;
+    public static final int NAME_CHAR_LEN             = 64;
+    /* Field/table name length */
+    public static final int NAME_LEN                  = (NAME_CHAR_LEN * SYSTEM_CHARSET_MBMAXLEN);
+
+    /**
+     * Max number of possible extra bytes in a replication event compared to a
+     * packet (i.e. a query) sent from client to master; First, an auxiliary
+     * log_event status vars estimation:
+     */
+    public static final int MAX_SIZE_LOG_EVENT_STATUS = (1 + 4 /* type, flags2 */
+            + 1 + 8 /* type, sql_mode */
+            + 1 + 1 + 255 /* type, length, catalog */
+            + 1 + 4 /* type, auto_increment */
+            + 1 + 6 /* type, charset */
+            + 1 + 1 + 255 /* type, length, time_zone */
+            + 1 + 2 /* type, lc_time_names_number */
+            + 1 + 2 /* type, charset_database_number */
+            + 1 + 8 /* type, table_map_for_update */
+            + 1 + 4 /* type, master_data_written */
+            /* type, db_1, db_2, ... */ 
+            /* type, microseconds */
+            /* MariaDb type, sec_part of NOW() */ 
+            + 1 + (MAX_DBS_IN_EVENT_MTS * (1 + NAME_LEN)) + 3
+            + 1 + 16 + 1 + 60/* type, user_len, user, host_len, host */);
+    /**
+     * Fixed data part:
+     * 
+     * <ul>
+     * <li>4 bytes. The ID of the thread that issued this statement. Needed for
+     * temporary tables. This is also useful for a DBA for knowing who did what
+     * on the master.</li>
+     * <li>4 bytes. The time in seconds that the statement took to execute. Only
+     * useful for inspection by the DBA.</li>
+     * <li>1 byte. The length of the name of the database which was the default
+     * database when the statement was executed. This name appears later, in the
+     * variable data part. It is necessary for statements such as INSERT INTO t
+     * VALUES(1) that don't specify the database and rely on the default
+     * database previously selected by USE.</li>
+     * <li>2 bytes. The error code resulting from execution of the statement on
+     * the master. Error codes are defined in include/mysqld_error.h. 0 means no
+     * error. How come statements with a non-zero error code can exist in the
+     * binary log? This is mainly due to the use of non-transactional tables
+     * within transactions. For example, if an INSERT ... SELECT fails after
+     * inserting 1000 rows into a MyISAM table (for example, with a
+     * duplicate-key violation), we have to write this statement to the binary
+     * log, because it truly modified the MyISAM table. For transactional
+     * tables, there should be no event with a non-zero error code (though it
+     * can happen, for example if the connection was interrupted (Control-C)).
+     * The slave checks the error code: After executing the statement itself, it
+     * compares the error code it got with the error code in the event, and if
+     * they are different it stops replicating (unless --slave-skip-errors was
+     * used to ignore the error).</li>
+     * <li>2 bytes (not present in v1, v3). The length of the status variable
+     * block.</li>
+     * </ul>
+     * 
+     * Variable part:
+     * <ul>
+     * <li>Zero or more status variables (not present in v1, v3). Each status
+     * variable consists of one byte code identifying the variable stored,
+     * followed by the value of the variable. The format of the value is
+     * variable-specific, as described later.</li>
+     * <li>The default database name (null-terminated).</li>
+     * <li>The SQL statement. The slave knows the size of the other fields in
+     * the variable part (the sizes are given in the fixed data part), so by
+     * subtraction it can know the size of the statement.</li>
+     * </ul>
+     * 
+     * Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
+     */
+    private String          user;
+    private String          host;
+
+    /* using byte for query string */
+    protected String        query;
+    protected String        catalog;
+    protected final String  dbname;
+
+    /** The number of seconds the query took to run on the master. */
+    //  The time in seconds that the statement took to execute. Only useful for inspection by the DBA
+    private final long      execTime;
+    private final int       errorCode;
+    private final long      sessionId;                                                                                      /* thread_id */
+
+    /**
+     * 'flags2' is a second set of flags (on top of those in Log_event), for
+     * session variables. These are thd->options which is & against a mask
+     * (OPTIONS_WRITTEN_TO_BIN_LOG).
+     */
+    private long            flags2;
+
+    /** In connections sql_mode is 32 bits now but will be 64 bits soon */
+    private long            sql_mode;
+
+    private long            autoIncrementIncrement    = -1;
+    private long            autoIncrementOffset       = -1;
+
+    private int             clientCharset             = -1;
+    private int             clientCollation           = -1;
+    private int             serverCollation           = -1;
+    private String          charsetName;
+
+    private String          timezone;
+
+    public QueryLogEvent(LogHeader header, LogBuffer buffer,
+            FormatDescriptionLogEvent descriptionEvent) throws IOException
+    {
+        super(header);
+
+        final int commonHeaderLen = descriptionEvent.commonHeaderLen;
+        final int postHeaderLen = descriptionEvent.postHeaderLen[header.type - 1];
+        /*
+         * We test if the event's length is sensible, and if so we compute
+         * data_len. We cannot rely on QUERY_HEADER_LEN here as it would not be
+         * format-tolerant. We use QUERY_HEADER_MINIMAL_LEN which is the same
+         * for 3.23, 4.0 & 5.0.
+         */
+        if (buffer.limit() < (commonHeaderLen + postHeaderLen))
+        {
+            throw new IOException("Query event length is too short.");
+        }
+        int dataLen = buffer.limit() - (commonHeaderLen + postHeaderLen);
+        buffer.position(commonHeaderLen + Q_THREAD_ID_OFFSET);
+
+        sessionId = buffer.getUint32(); // Q_THREAD_ID_OFFSET
+        execTime = buffer.getUint32(); // Q_EXEC_TIME_OFFSET
+
+        // TODO: add a check of all *_len vars
+        final int dbLen = buffer.getUint8(); // Q_DB_LEN_OFFSET
+        errorCode = buffer.getUint16(); // Q_ERR_CODE_OFFSET
+
+        /*
+         * 5.0 format starts here. Depending on the format, we may or not
+         * have affected/warnings etc The remaining post-header to be parsed
+         * has length:
+         */
+        int statusVarsLen = 0;
+        if (postHeaderLen > QUERY_HEADER_MINIMAL_LEN)
+        {
+            statusVarsLen = buffer.getUint16(); // Q_STATUS_VARS_LEN_OFFSET
+            /*
+              Check if status variable length is corrupt and will lead to very
+              wrong data. We could be even more strict and require data_len to
+              be even bigger, but this will suffice to catch most corruption
+              errors that can lead to a crash.
+            */
+            if (statusVarsLen > Math.min(dataLen, MAX_SIZE_LOG_EVENT_STATUS))
+            {
+                throw new IOException("status_vars_len (" + statusVarsLen
+                        + ") > data_len (" + dataLen + ")");
+            }
+            dataLen -= statusVarsLen;
+        }
+        /*
+         * We have parsed everything we know in the post header for QUERY_EVENT,
+         * the rest of post header is either comes from older version MySQL or
+         * dedicated to derived events (e.g. Execute_load_query...)
+         */
+
+        /* variable-part: the status vars; only in MySQL 5.0  */
+        final int start = commonHeaderLen + postHeaderLen;
+        final int limit = buffer.limit(); /* for restore */
+        final int end = start + statusVarsLen;
+        buffer.position(start).limit(end);
+        unpackVariables(buffer, end);
+        buffer.position(end);
+        buffer.limit(limit);
+
+        /* A 2nd variable part; this is common to all versions */
+        final int queryLen = dataLen - dbLen - 1;
+        dbname = buffer.getFixString(dbLen + 1);
+        if (clientCharset >= 0)
+        {
+            charsetName = CharsetConversion.getJavaCharset(clientCharset);
+
+            if ((charsetName != null) && (Charset.isSupported(charsetName)))
+            {
+                query = buffer.getFixString(queryLen, charsetName);
+            }
+            else
+            {
+                logger.warn("unsupported character set in query log: "
+                        + "\n    ID = " + clientCharset + ", Charset = "
+                        + CharsetConversion.getCharset(clientCharset)
+                        + ", Collation = "
+                        + CharsetConversion.getCollation(clientCharset));
+
+                query = buffer.getFixString(queryLen);
+            }
+        }
+        else
+        {
+            query = buffer.getFixString(queryLen);
+        }
+    }
+
+    /* query event post-header */
+    public static final int Q_THREAD_ID_OFFSET          = 0;
+    public static final int Q_EXEC_TIME_OFFSET          = 4;
+    public static final int Q_DB_LEN_OFFSET             = 8;
+    public static final int Q_ERR_CODE_OFFSET           = 9;
+    public static final int Q_STATUS_VARS_LEN_OFFSET    = 11;
+    public static final int Q_DATA_OFFSET               = QUERY_HEADER_LEN;
+
+    /* these are codes, not offsets; not more than 256 values (1 byte). */
+    public static final int Q_FLAGS2_CODE               = 0;
+    public static final int Q_SQL_MODE_CODE             = 1;
+
+    /**
+     * Q_CATALOG_CODE is catalog with end zero stored; it is used only by MySQL
+     * 5.0.x where 0<=x<=3. We have to keep it to be able to replicate these old
+     * masters.
+     */
+    public static final int Q_CATALOG_CODE              = 2;
+    public static final int Q_AUTO_INCREMENT            = 3;
+    public static final int Q_CHARSET_CODE              = 4;
+    public static final int Q_TIME_ZONE_CODE            = 5;
+
+    /**
+     * Q_CATALOG_NZ_CODE is catalog withOUT end zero stored; it is used by MySQL
+     * 5.0.x where x>=4. Saves one byte in every Query_log_event in binlog,
+     * compared to Q_CATALOG_CODE. The reason we didn't simply re-use
+     * Q_CATALOG_CODE is that then a 5.0.3 slave of this 5.0.x (x>=4) master
+     * would crash (segfault etc) because it would expect a 0 when there is
+     * none.
+     */
+    public static final int Q_CATALOG_NZ_CODE           = 6;
+
+    public static final int Q_LC_TIME_NAMES_CODE        = 7;
+
+    public static final int Q_CHARSET_DATABASE_CODE     = 8;
+
+    public static final int Q_TABLE_MAP_FOR_UPDATE_CODE = 9;
+
+    public static final int Q_MASTER_DATA_WRITTEN_CODE  = 10;
+
+    public static final int Q_INVOKER                   = 11;
+    
+    /**
+        Q_UPDATED_DB_NAMES status variable collects of the updated databases
+        total number and their names to be propagated to the slave in order
+        to facilitate the parallel applying of the Query events.
+     */
+    public static final int Q_UPDATED_DB_NAMES          = 12;
+
+    public static final int Q_MICROSECONDS              = 13;
+    
+    /** 
+     *  FROM MariaDB 5.5.34 
+     */
+    public static final int Q_HRNOW                     = 128;
+
+    private final void unpackVariables(LogBuffer buffer, final int end)
+            throws IOException
+    {
+        int code = -1;
+        try
+        {
+            while (buffer.position() < end)
+            {
+                switch (code = buffer.getUint8())
+                {
+                case Q_FLAGS2_CODE:
+                    flags2 = buffer.getUint32();
+                    break;
+                case Q_SQL_MODE_CODE:
+                    sql_mode = buffer.getLong64(); // QQ: Fix when sql_mode is ulonglong
+                    break;
+                case Q_CATALOG_NZ_CODE:
+                    catalog = buffer.getString();
+                    break;
+                case Q_AUTO_INCREMENT:
+                    autoIncrementIncrement = buffer.getUint16();
+                    autoIncrementOffset = buffer.getUint16();
+                    break;
+                case Q_CHARSET_CODE:
+                    // Charset: 6 byte character set flag.
+                    //   1-2 = character set client
+                    //   3-4 = collation client
+                    //   5-6 = collation server
+                    clientCharset = buffer.getUint16();
+                    clientCollation = buffer.getUint16();
+                    serverCollation = buffer.getUint16();
+                    break;
+                case Q_TIME_ZONE_CODE:
+                    timezone = buffer.getString();
+                    break;
+                case Q_CATALOG_CODE: /* for 5.0.x where 0<=x<=3 masters */
+                    final int len = buffer.getUint8();
+                    catalog = buffer.getFixString(len + 1);
+                    break;
+                case Q_LC_TIME_NAMES_CODE:
+                    // lc_time_names_number = buffer.getUint16();
+                    buffer.forward(2);
+                    break;
+                case Q_CHARSET_DATABASE_CODE:
+                    // charset_database_number = buffer.getUint16();
+                    buffer.forward(2);
+                    break;
+                case Q_TABLE_MAP_FOR_UPDATE_CODE:
+                    // table_map_for_update = buffer.getUlong64();
+                    buffer.forward(8);
+                    break;
+                case Q_MASTER_DATA_WRITTEN_CODE:
+                    // data_written = master_data_written = buffer.getUint32();
+                    buffer.forward(4);
+                    break;
+                case Q_INVOKER:
+                    user = buffer.getString();
+                    host = buffer.getString();
+                    break;
+                case Q_MICROSECONDS:
+                    // when.tv_usec= uint3korr(pos);
+                    buffer.forward(3);
+                    break;
+                case Q_UPDATED_DB_NAMES:
+                    int mtsAccessedDbs = buffer.getUint8(); 
+                    /**
+                        Notice, the following check is positive also in case of
+                        the master's MAX_DBS_IN_EVENT_MTS > the slave's one and the event 
+                        contains e.g the master's MAX_DBS_IN_EVENT_MTS db:s.
+                     */
+                    if (mtsAccessedDbs > MAX_DBS_IN_EVENT_MTS) {
+                        mtsAccessedDbs = OVER_MAX_DBS_IN_EVENT_MTS;
+                        break;
+                    }
+                    String mtsAccessedDbNames[] = new String[mtsAccessedDbs];
+                    for (int i = 0; i < mtsAccessedDbs && buffer.position() < end; i++) {
+                        int length = end - buffer.position();
+                        mtsAccessedDbNames[i] = buffer.getFixString(length < NAME_LEN ? length : NAME_LEN);
+                    }
+                    break;
+                case Q_HRNOW: 
+                    // int when_sec_part = buffer.getUint24();
+                    buffer.forward(3);
+                    break;
+                default:
+                    /* That's why you must write status vars in growing order of code */
+                    if (logger.isDebugEnabled())
+                        logger.debug("Query_log_event has unknown status vars (first has code: "
+                                + code + "), skipping the rest of them");
+                    break; // Break loop
+                }
+            }
+        }
+        catch (RuntimeException e)
+        {
+            throw new IOException("Read " + findCodeName(code) + " error: "
+                    + e.getMessage(), e);
+        }
+    }
+
+    private static final String findCodeName(final int code)
+    {
+        switch (code)
+        {
+        case Q_FLAGS2_CODE:
+            return "Q_FLAGS2_CODE";
+        case Q_SQL_MODE_CODE:
+            return "Q_SQL_MODE_CODE";
+        case Q_CATALOG_CODE:
+            return "Q_CATALOG_CODE";
+        case Q_AUTO_INCREMENT:
+            return "Q_AUTO_INCREMENT";
+        case Q_CHARSET_CODE:
+            return "Q_CHARSET_CODE";
+        case Q_TIME_ZONE_CODE:
+            return "Q_TIME_ZONE_CODE";
+        case Q_CATALOG_NZ_CODE:
+            return "Q_CATALOG_NZ_CODE";
+        case Q_LC_TIME_NAMES_CODE:
+            return "Q_LC_TIME_NAMES_CODE";
+        case Q_CHARSET_DATABASE_CODE:
+            return "Q_CHARSET_DATABASE_CODE";
+        case Q_TABLE_MAP_FOR_UPDATE_CODE:
+            return "Q_TABLE_MAP_FOR_UPDATE_CODE";
+        case Q_MASTER_DATA_WRITTEN_CODE:
+            return "Q_MASTER_DATA_WRITTEN_CODE";
+        case Q_UPDATED_DB_NAMES: 
+            return "Q_UPDATED_DB_NAMES";
+        case Q_MICROSECONDS:
+            return "Q_MICROSECONDS";
+        }
+        return "CODE#" + code;
+    }
+
+    public final String getUser()
+    {
+        return user;
+    }
+
+    public final String getHost()
+    {
+        return host;
+    }
+
+    public final String getQuery()
+    {
+        return query;
+    }
+
+    public final String getCatalog()
+    {
+        return catalog;
+    }
+
+    public final String getDbName()
+    {
+        return dbname;
+    }
+
+    /**
+     * The number of seconds the query took to run on the master.
+     */
+    public final long getExecTime()
+    {
+        return execTime;
+    }
+
+    public final int getErrorCode()
+    {
+        return errorCode;
+    }
+
+    public final long getSessionId()
+    {
+        return sessionId;
+    }
+
+    public final long getAutoIncrementIncrement()
+    {
+        return autoIncrementIncrement;
+    }
+
+    public final long getAutoIncrementOffset()
+    {
+        return autoIncrementOffset;
+    }
+
+    public final String getCharsetName()
+    {
+        return charsetName;
+    }
+
+    public final String getTimezone()
+    {
+        return timezone;
+    }
+
+    /**
+     * Returns the charsetID value.
+     * 
+     * @return Returns the charsetID.
+     */
+    public final int getClientCharset()
+    {
+        return clientCharset;
+    }
+
+    /**
+     * Returns the clientCollationId value.
+     * 
+     * @return Returns the clientCollationId.
+     */
+    public final int getClientCollation()
+    {
+        return clientCollation;
+    }
+
+    /**
+     * Returns the serverCollationId value.
+     * 
+     * @return Returns the serverCollationId.
+     */
+    public final int getServerCollation()
+    {
+        return serverCollation;
+    }
+
+    /**
+     * Returns the sql_mode value.
+     * 
+     * <p>
+     * The sql_mode variable. See the section "SQL Modes" in the MySQL manual,
+     * and see mysql_priv.h for a list of the possible flags. Currently
+     * (2007-10-04), the following flags are available:
+     * 
+     * <ul>
+     * <li>MODE_REAL_AS_FLOAT==0x1</li>
+     * <li>MODE_PIPES_AS_CONCAT==0x2</li>
+     * <li>MODE_ANSI_QUOTES==0x4</li>
+     * <li>MODE_IGNORE_SPACE==0x8</li>
+     * <li>MODE_NOT_USED==0x10</li>
+     * <li>MODE_ONLY_FULL_GROUP_BY==0x20</li>
+     * <li>MODE_NO_UNSIGNED_SUBTRACTION==0x40</li>
+     * <li>MODE_NO_DIR_IN_CREATE==0x80</li>
+     * <li>MODE_POSTGRESQL==0x100</li>
+     * <li>MODE_ORACLE==0x200</li>
+     * <li>MODE_MSSQL==0x400</li>
+     * <li>MODE_DB2==0x800</li>
+     * <li>MODE_MAXDB==0x1000</li>
+     * <li>MODE_NO_KEY_OPTIONS==0x2000</li>
+     * <li>MODE_NO_TABLE_OPTIONS==0x4000</li>
+     * <li>MODE_NO_FIELD_OPTIONS==0x8000</li>
+     * <li>MODE_MYSQL323==0x10000</li>
+     * <li>MODE_MYSQL40==0x20000</li>
+     * <li>MODE_ANSI==0x40000</li>
+     * <li>MODE_NO_AUTO_VALUE_ON_ZERO==0x80000</li>
+     * <li>MODE_NO_BACKSLASH_ESCAPES==0x100000</li>
+     * <li>MODE_STRICT_TRANS_TABLES==0x200000</li>
+     * <li>MODE_STRICT_ALL_TABLES==0x400000</li>
+     * <li>MODE_NO_ZERO_IN_DATE==0x800000</li>
+     * <li>MODE_NO_ZERO_DATE==0x1000000</li>
+     * <li>MODE_INVALID_DATES==0x2000000</li>
+     * <li>MODE_ERROR_FOR_DIVISION_BY_ZERO==0x4000000</li>
+     * <li>MODE_TRADITIONAL==0x8000000</li>
+     * <li>MODE_NO_AUTO_CREATE_USER==0x10000000</li>
+     * <li>MODE_HIGH_NOT_PRECEDENCE==0x20000000</li>
+     * <li>MODE_NO_ENGINE_SUBSTITUTION=0x40000000</li>
+     * <li>MODE_PAD_CHAR_TO_FULL_LENGTH==0x80000000</li>
+     * </ul>
+     * 
+     * All these flags are replicated from the server. However, all flags except
+     * MODE_NO_DIR_IN_CREATE are honored by the slave; the slave always
+     * preserves its old value of MODE_NO_DIR_IN_CREATE. This field is always
+     * written to the binlog.
+     */
+    public final long getSqlMode()
+    {
+        return sql_mode;
+    }
+
+    /* FLAGS2 values that can be represented inside the binlog */
+    public static final int OPTION_AUTO_IS_NULL          = 1 << 14;
+    public static final int OPTION_NOT_AUTOCOMMIT        = 1 << 19;
+    public static final int OPTION_NO_FOREIGN_KEY_CHECKS = 1 << 26;
+    public static final int OPTION_RELAXED_UNIQUE_CHECKS = 1 << 27;
+
+    /**
+     * The flags in thd->options, binary AND-ed with OPTIONS_WRITTEN_TO_BIN_LOG.
+     * The thd->options bitfield contains options for "SELECT". OPTIONS_WRITTEN
+     * identifies those options that need to be written to the binlog (not all
+     * do). Specifically, OPTIONS_WRITTEN_TO_BIN_LOG equals (OPTION_AUTO_IS_NULL
+     * | OPTION_NO_FOREIGN_KEY_CHECKS | OPTION_RELAXED_UNIQUE_CHECKS |
+     * OPTION_NOT_AUTOCOMMIT), or 0x0c084000 in hex. These flags correspond to
+     * the SQL variables SQL_AUTO_IS_NULL, FOREIGN_KEY_CHECKS, UNIQUE_CHECKS,
+     * and AUTOCOMMIT, documented in the "SET Syntax" section of the MySQL
+     * Manual. This field is always written to the binlog in version >= 5.0, and
+     * never written in version < 5.0.
+     */
+    public final long getFlags2()
+    {
+        return flags2;
+    }
+
+    /**
+     * Returns the OPTION_AUTO_IS_NULL flag.
+     */
+    public final boolean isAutoIsNull()
+    {
+        return ((flags2 & OPTION_AUTO_IS_NULL) == OPTION_AUTO_IS_NULL);
+    }
+
+    /**
+     * Returns the OPTION_NO_FOREIGN_KEY_CHECKS flag.
+     */
+    public final boolean isForeignKeyChecks()
+    {
+        return ((flags2 & OPTION_NO_FOREIGN_KEY_CHECKS) != OPTION_NO_FOREIGN_KEY_CHECKS);
+    }
+
+    /**
+     * Returns the OPTION_NOT_AUTOCOMMIT flag.
+     */
+    public final boolean isAutocommit()
+    {
+        return ((flags2 & OPTION_NOT_AUTOCOMMIT) != OPTION_NOT_AUTOCOMMIT);
+    }
+
+    /**
+     * Returns the OPTION_NO_FOREIGN_KEY_CHECKS flag.
+     */
+    public final boolean isUniqueChecks()
+    {
+        return ((flags2 & OPTION_RELAXED_UNIQUE_CHECKS) != OPTION_RELAXED_UNIQUE_CHECKS);
+    }
+
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RandLogEvent.java

@@ -0,0 +1,83 @@
+package com.taobao.tddl.dbsync.binlog.event;
+
+import com.taobao.tddl.dbsync.binlog.LogBuffer;
+import com.taobao.tddl.dbsync.binlog.LogEvent;
+
+/**
+ * Logs random seed used by the next RAND(), and by PASSWORD() in 4.1.0. 4.1.1
+ * does not need it (it's repeatable again) so this event needn't be written in
+ * 4.1.1 for PASSWORD() (but the fact that it is written is just a waste, it
+ * does not cause bugs).
+ * 
+ * The state of the random number generation consists of 128 bits, which are
+ * stored internally as two 64-bit numbers.
+ * 
+ * Binary Format
+ * 
+ * The Post-Header for this event type is empty. The Body has two components:
+ * 
+ * <table>
+ * <caption>Body for Rand_log_event</caption>
+ * 
+ * <tr>
+ * <th>Name</th>
+ * <th>Format</th>
+ * <th>Description</th>
+ * </tr>
+ * 
+ * <tr>
+ * <td>seed1</td>
+ * <td>8 byte unsigned integer</td>
+ * <td>64 bit random seed1.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>seed2</td>
+ * <td>8 byte unsigned integer</td>
+ * <td>64 bit random seed2.</td>
+ * </tr>
+ * </table>
+ * 
+ * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
+ * @version 1.0
+ */
+public final class RandLogEvent extends LogEvent
+{
+    /**
+     * Fixed data part: Empty
+     * 
+     * <p>
+     * Variable data part:
+     * 
+     * <ul>
+     * <li>8 bytes. The value for the first seed.</li>
+     * <li>8 bytes. The value for the second seed.</li>
+     * </ul>
+     * 
+     * Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
+     */
+    private final long      seed1;
+    private final long      seed2;
+
+    /* Rand event data */
+    public static final int RAND_SEED1_OFFSET = 0;
+    public static final int RAND_SEED2_OFFSET = 8;
+
+    public RandLogEvent(LogHeader header, LogBuffer buffer,
+            FormatDescriptionLogEvent descriptionEvent)
+    {
+        super(header);
+
+        /* The Post-Header is empty. The Variable Data part begins immediately. */
+        buffer.position(descriptionEvent.commonHeaderLen
+                + descriptionEvent.postHeaderLen[RAND_EVENT - 1]
+                + RAND_SEED1_OFFSET);
+        seed1 = buffer.getLong64(); // !uint8korr(buf+RAND_SEED1_OFFSET);
+        seed2 = buffer.getLong64(); // !uint8korr(buf+RAND_SEED2_OFFSET);
+    }
+
+    public final String getQuery()
+    {
+        return "SET SESSION rand_seed1 = " + seed1 + " , rand_seed2 = " + seed2;
+    }
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RotateLogEvent.java

@@ -0,0 +1,145 @@
+package com.taobao.tddl.dbsync.binlog.event;
+
+import com.taobao.tddl.dbsync.binlog.LogBuffer;
+import com.taobao.tddl.dbsync.binlog.LogEvent;
+
+/**
+ * This will be deprecated when we move to using sequence ids.
+ * 
+ * Binary Format
+ * 
+ * The Post-Header has one component:
+ * 
+ * <table>
+ * <caption>Post-Header for Rotate_log_event</caption>
+ * 
+ * <tr>
+ * <th>Name</th>
+ * <th>Format</th>
+ * <th>Description</th>
+ * </tr>
+ * 
+ * <tr>
+ * <td>position</td>
+ * <td>8 byte integer</td>
+ * <td>The position within the binlog to rotate to.</td>
+ * </tr>
+ * 
+ * </table>
+ * 
+ * The Body has one component:
+ * 
+ * <table>
+ * <caption>Body for Rotate_log_event</caption>
+ * 
+ * <tr>
+ * <th>Name</th>
+ * <th>Format</th>
+ * <th>Description</th>
+ * </tr>
+ * 
+ * <tr>
+ * <td>new_log</td>
+ * <td>variable length string without trailing zero, extending to the end of the
+ * event (determined by the length field of the Common-Header)</td>
+ * <td>Name of the binlog to rotate to.</td>
+ * </tr>
+ * 
+ * </table>
+ * 
+ * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
+ * @version 1.0
+ */
+public final class RotateLogEvent extends LogEvent
+{
+    /**
+     * Fixed data part:
+     * 
+     * <ul>
+     * <li>8 bytes. The position of the first event in the next log file. Always
+     * contains the number 4 (meaning the next event starts at position 4 in the
+     * next binary log). This field is not present in v1; presumably the value
+     * is assumed to be 4.</li>
+     * </ul>
+     * <p>
+     * 
+     * Variable data part:
+     * 
+     * <ul>
+     * <li>The name of the next binary log. The filename is not null-terminated.
+     * Its length is the event size minus the size of the fixed parts.</li>
+     * </ul>
+     * 
+     * Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
+     */
+    private final String          filename;
+    private final long            position;
+
+    /* Rotate event post-header */
+    public static final int       R_POS_OFFSET   = 0;
+    public static final int       R_IDENT_OFFSET = 8;
+
+    /* Max length of full path-name */
+    public static final int       FN_REFLEN      = 512;
+
+    // Rotate header with all empty fields.
+    public static final LogHeader ROTATE_HEADER  = new LogHeader(ROTATE_EVENT);
+
+    /**
+     * Creates a new <code>Rotate_log_event</code> object read normally from
+     * log.
+     * 
+     * @throws MySQLExtractException
+     */
+    public RotateLogEvent(LogHeader header, LogBuffer buffer,
+            FormatDescriptionLogEvent descriptionEvent)
+    {
+        super(header);
+
+        final int headerSize = descriptionEvent.commonHeaderLen;
+        final int postHeaderLen = descriptionEvent.postHeaderLen[ROTATE_EVENT - 1];
+
+        buffer.position(headerSize + R_POS_OFFSET);
+        position = (postHeaderLen != 0) ? buffer.getLong64() : 4; // !uint8korr(buf + R_POS_OFFSET)
+
+        final int filenameOffset = headerSize + postHeaderLen;
+        int filenameLen = buffer.limit() - filenameOffset;
+        if (filenameLen > FN_REFLEN - 1)
+            filenameLen = FN_REFLEN - 1;
+        buffer.position(filenameOffset);
+        filename = buffer.getFixString(filenameLen);
+    }
+
+    /**
+     * Creates a new <code>Rotate_log_event</code> without log information. This
+     * is used to generate missing log rotation events.
+     */
+    public RotateLogEvent(String filename)
+    {
+        super(ROTATE_HEADER);
+
+        this.filename = filename;
+        this.position = 4;
+    }
+
+    /**
+     * Creates a new <code>Rotate_log_event</code> without log information.
+     */
+    public RotateLogEvent(String filename, final long position)
+    {
+        super(ROTATE_HEADER);
+
+        this.filename = filename;
+        this.position = position;
+    }
+
+    public final String getFilename()
+    {
+        return filename;
+    }
+
+    public final long getPosition()
+    {
+        return position;
+    }
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogBuffer.java

@@ -0,0 +1,976 @@
+package com.taobao.tddl.dbsync.binlog.event;
+
+import java.io.Serializable;
+import java.sql.Timestamp;
+import java.sql.Types;
+import java.util.BitSet;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import com.taobao.tddl.dbsync.binlog.LogBuffer;
+import com.taobao.tddl.dbsync.binlog.LogEvent;
+
+/**
+ * Extracting JDBC type & value information from packed rows-buffer.
+ * 
+ * @see mysql-5.1.60/sql/log_event.cc - Rows_log_event::print_verbose_one_row
+ * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
+ * @version 1.0
+ */
+public final class RowsLogBuffer {
+
+    protected static final Log logger            = LogFactory.getLog(RowsLogBuffer.class);
+
+    public static final long   DATETIMEF_INT_OFS = 0x8000000000L;
+    public static final long   TIMEF_INT_OFS     = 0x800000L;
+    private final LogBuffer    buffer;
+    private final int          columnLen;
+    private final String       charsetName;
+    // private Calendar cal;
+
+    private final BitSet       nullBits;
+    private int                nullBitIndex;
+
+    private boolean            fNull;
+    private int                javaType;
+    private int                length;
+    private Serializable       value;
+
+    public RowsLogBuffer(LogBuffer buffer, final int columnLen, String charsetName){
+        this.buffer = buffer;
+        this.columnLen = columnLen;
+        this.charsetName = charsetName;
+        this.nullBits = new BitSet(columnLen);
+    }
+
+    /**
+     * Extracting next row from packed buffer.
+     * 
+     * @see mysql-5.1.60/sql/log_event.cc -
+     * Rows_log_event::print_verbose_one_row
+     */
+    public final boolean nextOneRow(BitSet columns) {
+        final boolean hasOneRow = buffer.hasRemaining();
+
+        if (hasOneRow) {
+            int column = 0;
+
+            for (int i = 0; i < columnLen; i++)
+                if (columns.get(i)) column++;
+
+            nullBitIndex = 0;
+            nullBits.clear();
+            buffer.fillBitmap(nullBits, column);
+        }
+        return hasOneRow;
+    }
+
+    /**
+     * Extracting next field value from packed buffer.
+     * 
+     * @see mysql-5.1.60/sql/log_event.cc -
+     * Rows_log_event::print_verbose_one_row
+     */
+    public final Serializable nextValue(final int type, final int meta) {
+        return nextValue(type, meta, false);
+    }
+
+    /**
+     * Extracting next field value from packed buffer.
+     * 
+     * @see mysql-5.1.60/sql/log_event.cc -
+     * Rows_log_event::print_verbose_one_row
+     */
+    public final Serializable nextValue(final int type, final int meta, boolean isBinary) {
+        fNull = nullBits.get(nullBitIndex++);
+
+        if (fNull) {
+            value = null;
+            javaType = mysqlToJavaType(type, meta, isBinary);
+            length = 0;
+            return null;
+        } else {
+            // Extracting field value from packed buffer.
+            return fetchValue(type, meta, isBinary);
+        }
+    }
+
+    /**
+     * Maps the given MySQL type to the correct JDBC type.
+     */
+    static int mysqlToJavaType(int type, final int meta, boolean isBinary) {
+        int javaType;
+
+        if (type == LogEvent.MYSQL_TYPE_STRING) {
+            if (meta >= 256) {
+                int byte0 = meta >> 8;
+                if ((byte0 & 0x30) != 0x30) {
+                    /* a long CHAR() field: see #37426 */
+                    type = byte0 | 0x30;
+                } else {
+                    switch (byte0) {
+                        case LogEvent.MYSQL_TYPE_SET:
+                        case LogEvent.MYSQL_TYPE_ENUM:
+                        case LogEvent.MYSQL_TYPE_STRING:
+                            type = byte0;
+                    }
+                }
+            }
+        }
+
+        switch (type) {
+            case LogEvent.MYSQL_TYPE_LONG:
+                javaType = Types.INTEGER;
+                break;
+
+            case LogEvent.MYSQL_TYPE_TINY:
+                javaType = Types.TINYINT;
+                break;
+
+            case LogEvent.MYSQL_TYPE_SHORT:
+                javaType = Types.SMALLINT;
+                break;
+
+            case LogEvent.MYSQL_TYPE_INT24:
+                javaType = Types.INTEGER;
+                break;
+
+            case LogEvent.MYSQL_TYPE_LONGLONG:
+                javaType = Types.BIGINT;
+                break;
+
+            case LogEvent.MYSQL_TYPE_DECIMAL:
+                javaType = Types.DECIMAL;
+                break;
+
+            case LogEvent.MYSQL_TYPE_NEWDECIMAL:
+                javaType = Types.DECIMAL;
+                break;
+
+            case LogEvent.MYSQL_TYPE_FLOAT:
+                javaType = Types.REAL; // Types.FLOAT;
+                break;
+
+            case LogEvent.MYSQL_TYPE_DOUBLE:
+                javaType = Types.DOUBLE;
+                break;
+
+            case LogEvent.MYSQL_TYPE_BIT:
+                javaType = Types.BIT;
+                break;
+
+            case LogEvent.MYSQL_TYPE_TIMESTAMP:
+            case LogEvent.MYSQL_TYPE_DATETIME:
+                javaType = Types.TIMESTAMP;
+                break;
+
+            case LogEvent.MYSQL_TYPE_TIME:
+                javaType = Types.TIME;
+                break;
+
+            case LogEvent.MYSQL_TYPE_NEWDATE:
+            case LogEvent.MYSQL_TYPE_DATE:
+                javaType = Types.DATE;
+                break;
+
+            case LogEvent.MYSQL_TYPE_YEAR:
+                javaType = Types.VARCHAR;
+                break;
+
+            case LogEvent.MYSQL_TYPE_ENUM:
+                javaType = Types.INTEGER;
+                break;
+
+            case LogEvent.MYSQL_TYPE_SET:
+                javaType = Types.BINARY;
+                break;
+
+            case LogEvent.MYSQL_TYPE_TINY_BLOB:
+            case LogEvent.MYSQL_TYPE_MEDIUM_BLOB:
+            case LogEvent.MYSQL_TYPE_LONG_BLOB:
+            case LogEvent.MYSQL_TYPE_BLOB:
+                if (meta == 1) {
+                    javaType = Types.VARBINARY;
+                } else {
+                    javaType = Types.LONGVARBINARY;
+                }
+                break;
+
+            case LogEvent.MYSQL_TYPE_VARCHAR:
+            case LogEvent.MYSQL_TYPE_VAR_STRING:
+                if (isBinary) {
+                    // varbinary在binlog中为var_string类型
+                    javaType = Types.VARBINARY;
+                } else {
+                    javaType = Types.VARCHAR;
+                }
+                break;
+
+            case LogEvent.MYSQL_TYPE_STRING:
+                if (isBinary) {
+                    // binary在binlog中为string类型
+                    javaType = Types.BINARY;
+                } else {
+                    javaType = Types.CHAR;
+                }
+                break;
+
+            case LogEvent.MYSQL_TYPE_GEOMETRY:
+                javaType = Types.BINARY;
+                break;
+
+            // case LogEvent.MYSQL_TYPE_BINARY:
+            // javaType = Types.BINARY;
+            // break;
+            //
+            // case LogEvent.MYSQL_TYPE_VARBINARY:
+            // javaType = Types.VARBINARY;
+            // break;
+
+            default:
+                javaType = Types.OTHER;
+        }
+
+        return javaType;
+    }
+
+    /**
+     * Extracting next field value from packed buffer.
+     * 
+     * @see mysql-5.1.60/sql/log_event.cc - log_event_print_value
+     */
+    final Serializable fetchValue(int type, final int meta, boolean isBinary) {
+        int len = 0;
+
+        if (type == LogEvent.MYSQL_TYPE_STRING) {
+            if (meta >= 256) {
+                int byte0 = meta >> 8;
+                int byte1 = meta & 0xff;
+                if ((byte0 & 0x30) != 0x30) {
+                    /* a long CHAR() field: see #37426 */
+                    len = byte1 | (((byte0 & 0x30) ^ 0x30) << 4);
+                    type = byte0 | 0x30;
+                } else {
+                    switch (byte0) {
+                        case LogEvent.MYSQL_TYPE_SET:
+                        case LogEvent.MYSQL_TYPE_ENUM:
+                        case LogEvent.MYSQL_TYPE_STRING:
+                            type = byte0;
+                            len = byte1;
+                            break;
+                        default:
+                            throw new IllegalArgumentException(String.format("!! Don't know how to handle column type=%d meta=%d (%04X)",
+                                type,
+                                meta,
+                                meta));
+                    }
+                }
+            } else {
+                len = meta;
+            }
+        }
+
+        switch (type) {
+            case LogEvent.MYSQL_TYPE_LONG: {
+                // XXX: How to check signed / unsigned?
+                // value = unsigned ? Long.valueOf(buffer.getUint32()) :
+                // Integer.valueOf(buffer.getInt32());
+                value = Integer.valueOf(buffer.getInt32());
+                javaType = Types.INTEGER;
+                length = 4;
+                break;
+            }
+            case LogEvent.MYSQL_TYPE_TINY: {
+                // XXX: How to check signed / unsigned?
+                // value = Integer.valueOf(unsigned ? buffer.getUint8() :
+                // buffer.getInt8());
+                value = Integer.valueOf(buffer.getInt8());
+                javaType = Types.TINYINT; // java.sql.Types.INTEGER;
+                length = 1;
+                break;
+            }
+            case LogEvent.MYSQL_TYPE_SHORT: {
+                // XXX: How to check signed / unsigned?
+                // value = Integer.valueOf(unsigned ? buffer.getUint16() :
+                // buffer.getInt16());
+                value = Integer.valueOf((short) buffer.getInt16());
+                javaType = Types.SMALLINT; // java.sql.Types.INTEGER;
+                length = 2;
+                break;
+            }
+            case LogEvent.MYSQL_TYPE_INT24: {
+                // XXX: How to check signed / unsigned?
+                // value = Integer.valueOf(unsigned ? buffer.getUint24() :
+                // buffer.getInt24());
+                value = Integer.valueOf(buffer.getInt24());
+                javaType = Types.INTEGER;
+                length = 3;
+                break;
+            }
+            case LogEvent.MYSQL_TYPE_LONGLONG: {
+                // XXX: How to check signed / unsigned?
+                // value = unsigned ? buffer.getUlong64()) :
+                // Long.valueOf(buffer.getLong64());
+                value = Long.valueOf(buffer.getLong64());
+                javaType = Types.BIGINT; // Types.INTEGER;
+                length = 8;
+                break;
+            }
+            case LogEvent.MYSQL_TYPE_DECIMAL: {
+                /*
+                 * log_event.h : This enumeration value is only used internally
+                 * and cannot exist in a binlog.
+                 */
+                logger.warn("MYSQL_TYPE_DECIMAL : This enumeration value is "
+                            + "only used internally and cannot exist in a binlog!");
+                javaType = Types.DECIMAL;
+                value = null; /* unknown format */
+                length = 0;
+                break;
+            }
+            case LogEvent.MYSQL_TYPE_NEWDECIMAL: {
+                final int precision = meta >> 8;
+                final int decimals = meta & 0xff;
+                value = buffer.getDecimal(precision, decimals);
+                javaType = Types.DECIMAL;
+                length = precision;
+                break;
+            }
+            case LogEvent.MYSQL_TYPE_FLOAT: {
+                value = Float.valueOf(buffer.getFloat32());
+                javaType = Types.REAL; // Types.FLOAT;
+                length = 4;
+                break;
+            }
+            case LogEvent.MYSQL_TYPE_DOUBLE: {
+                value = Double.valueOf(buffer.getDouble64());
+                javaType = Types.DOUBLE;
+                length = 8;
+                break;
+            }
+            case LogEvent.MYSQL_TYPE_BIT: {
+                /* Meta-data: bit_len, bytes_in_rec, 2 bytes */
+                final int nbits = ((meta >> 8) * 8) + (meta & 0xff);
+                len = (nbits + 7) / 8;
+                if (nbits > 1) {
+                    // byte[] bits = new byte[len];
+                    // buffer.fillBytes(bits, 0, len);
+                    // 转化为unsign long
+                    switch (len) {
+                        case 1:
+                            value = buffer.getInt8();
+                            break;
+                        case 2:
+                            value = buffer.getBeUint16();
+                            break;
+                        case 3:
+                            value = buffer.getBeUint24();
+                            break;
+                        case 4:
+                            value = buffer.getBeUint32();
+                            break;
+                        case 5:
+                            value = buffer.getBeUlong40();
+                            break;
+                        case 6:
+                            value = buffer.getBeUlong48();
+                            break;
+                        case 7:
+                            value = buffer.getBeUlong56();
+                            break;
+                        case 8:
+                            value = buffer.getBeUlong64();
+                            break;
+                        default:
+                            throw new IllegalArgumentException("!! Unknown Bit len = " + len);
+                    }
+                } else {
+                    final int bit = buffer.getInt8();
+                    // value = (bit != 0) ? Boolean.TRUE : Boolean.FALSE;
+                    value = bit;
+                }
+                javaType = Types.BIT;
+                length = nbits;
+                break;
+            }
+            case LogEvent.MYSQL_TYPE_TIMESTAMP: {
+                // MYSQL DataTypes: TIMESTAMP
+                // range is '1970-01-01 00:00:01' UTC to '2038-01-19 03:14:07'
+                // UTC
+                // A TIMESTAMP cannot represent the value '1970-01-01 00:00:00'
+                // because that is equivalent to 0 seconds from the epoch and
+                // the value 0 is reserved for representing '0000-00-00
+                // 00:00:00', the “zero” TIMESTAMP value.
+                final long i32 = buffer.getUint32();
+                if (i32 == 0) {
+                    value = "0000-00-00 00:00:00";
+                } else {
+                    String v = new Timestamp(i32 * 1000).toString();
+                    value = v.substring(0, v.length() - 2);
+                }
+                javaType = Types.TIMESTAMP;
+                length = 4;
+                break;
+            }
+            case LogEvent.MYSQL_TYPE_TIMESTAMP2: {
+                final long tv_sec = buffer.getBeUint32(); // big-endian
+                int tv_usec = 0;
+                switch (meta) {
+                    case 0:
+                        tv_usec = 0;
+                        break;
+                    case 1:
+                    case 2:
+                        tv_usec = buffer.getInt8() * 10000;
+                        break;
+                    case 3:
+                    case 4:
+                        tv_usec = buffer.getBeInt16() * 100;
+                        break;
+                    case 5:
+                    case 6:
+                        tv_usec = buffer.getBeInt24();
+                        break;
+                    default:
+                        tv_usec = 0;
+                        break;
+                }
+
+                if (tv_sec == 0) {
+                    value = "0000-00-00 00:00:00";
+                } else {
+                    Timestamp time = new Timestamp(tv_sec * 1000);
+                    time.setNanos(tv_usec * 1000);
+                    String v = time.toString();
+                    value = v.substring(0, v.length() - 2);
+                }
+                javaType = Types.TIMESTAMP;
+                length = 4 + (meta + 1) / 2;
+                break;
+            }
+            case LogEvent.MYSQL_TYPE_DATETIME: {
+                // MYSQL DataTypes: DATETIME
+                // range is '0000-01-01 00:00:00' to '9999-12-31 23:59:59'
+                final long i64 = buffer.getLong64(); /* YYYYMMDDhhmmss */
+                if (i64 == 0) {
+                    value = "0000-00-00 00:00:00";
+                } else {
+                    final int d = (int) (i64 / 1000000);
+                    final int t = (int) (i64 % 1000000);
+                    // if (cal == null) cal = Calendar.getInstance();
+                    // cal.clear();
+                    /* month is 0-based, 0 for january. */
+                    // cal.set(d / 10000, (d % 10000) / 100 - 1, d % 100, t /
+                    // 10000, (t % 10000) / 100, t % 100);
+                    // value = new Timestamp(cal.getTimeInMillis());
+                    value = String.format("%04d-%02d-%02d %02d:%02d:%02d",
+                        d / 10000,
+                        (d % 10000) / 100,
+                        d % 100,
+                        t / 10000,
+                        (t % 10000) / 100,
+                        t % 100);
+                }
+                javaType = Types.TIMESTAMP;
+                length = 8;
+                break;
+            }
+            case LogEvent.MYSQL_TYPE_DATETIME2: {
+                /*
+                 * DATETIME and DATE low-level memory and disk representation
+                 * routines 1 bit sign (used when on disk) 17 bits year*13+month
+                 * (year 0-9999, month 0-12) 5 bits day (0-31) 5 bits hour
+                 * (0-23) 6 bits minute (0-59) 6 bits second (0-59) 24 bits
+                 * microseconds (0-999999) Total: 64 bits = 8 bytes
+                 * SYYYYYYY.YYYYYYYY
+                 * .YYdddddh.hhhhmmmm.mmssssss.ffffffff.ffffffff.ffffffff
+                 */
+                long intpart = buffer.getBeUlong40() - DATETIMEF_INT_OFS; // big-endian
+                @SuppressWarnings("unused")
+                int frac = 0;
+                switch (meta) {
+                    case 0:
+                        frac = 0;
+                        break;
+                    case 1:
+                    case 2:
+                        frac = buffer.getInt8() * 10000;
+                        break;
+                    case 3:
+                    case 4:
+                        frac = buffer.getBeInt16() * 100;
+                        break;
+                    case 5:
+                    case 6:
+                        frac = buffer.getBeInt24();
+                        break;
+                    default:
+                        frac = 0;
+                        break;
+                }
+
+                if (intpart == 0) {
+                    value = "0000-00-00 00:00:00";
+                } else {
+                    // 构造TimeStamp只处理到秒
+                    long ymd = intpart >> 17;
+                    long ym = ymd >> 5;
+                    long hms = intpart % (1 << 17);
+
+                    // if (cal == null) cal = Calendar.getInstance();
+                    // cal.clear();
+                    // cal.set((int) (ym / 13), (int) (ym % 13) - 1, (int) (ymd
+                    // % (1 << 5)), (int) (hms >> 12),
+                    // (int) ((hms >> 6) % (1 << 6)), (int) (hms % (1 << 6)));
+                    // value = new Timestamp(cal.getTimeInMillis());
+                    value = String.format("%04d-%02d-%02d %02d:%02d:%02d",
+                        (int) (ym / 13),
+                        (int) (ym % 13),
+                        (int) (ymd % (1 << 5)),
+                        (int) (hms >> 12),
+                        (int) ((hms >> 6) % (1 << 6)),
+                        (int) (hms % (1 << 6)));
+                }
+                javaType = Types.TIMESTAMP;
+                length = 5 + (meta + 1) / 2;
+                break;
+            }
+            case LogEvent.MYSQL_TYPE_TIME: {
+                // MYSQL DataTypes: TIME
+                // The range is '-838:59:59' to '838:59:59'
+                // final int i32 = buffer.getUint24();
+                final int i32 = buffer.getInt24();
+                final int u32 = Math.abs(i32);
+                if (i32 == 0) {
+                    value = "00:00:00";
+                } else {
+                    // if (cal == null) cal = Calendar.getInstance();
+                    // cal.clear();
+                    // cal.set(70, 0, 1, i32 / 10000, (i32 % 10000) / 100, i32 %
+                    // 100);
+                    // value = new Time(cal.getTimeInMillis());
+                    value = String.format("%s%02d:%02d:%02d",
+                        (i32 >= 0) ? "" : "-",
+                        u32 / 10000,
+                        (u32 % 10000) / 100,
+                        u32 % 100);
+                }
+                javaType = Types.TIME;
+                length = 3;
+                break;
+            }
+            case LogEvent.MYSQL_TYPE_TIME2: {
+                /*
+                 * TIME low-level memory and disk representation routines
+                 * In-memory format: 1 bit sign (Used for sign, when on disk) 1
+                 * bit unused (Reserved for wider hour range, e.g. for
+                 * intervals) 10 bit hour (0-836) 6 bit minute (0-59) 6 bit
+                 * second (0-59) 24 bits microseconds (0-999999) Total: 48 bits
+                 * = 6 bytes
+                 * Suhhhhhh.hhhhmmmm.mmssssss.ffffffff.ffffffff.ffffffff
+                 */
+                long intpart = 0;
+                int frac = 0;
+                long ltime = 0;
+                switch (meta) {
+                    case 0:
+                        intpart = buffer.getBeUint24() - TIMEF_INT_OFS; // big-endian
+                        ltime = intpart << 24;
+                        break;
+                    case 1:
+                    case 2:
+                        intpart = buffer.getBeUint24() - TIMEF_INT_OFS;
+                        frac = buffer.getUint8();
+                        if (intpart < 0 && frac > 0) {
+                            /*
+                             * Negative values are stored with reverse
+                             * fractional part order, for binary sort
+                             * compatibility. Disk value intpart frac Time value
+                             * Memory value 800000.00 0 0 00:00:00.00
+                             * 0000000000.000000 7FFFFF.FF -1 255 -00:00:00.01
+                             * FFFFFFFFFF.FFD8F0 7FFFFF.9D -1 99 -00:00:00.99
+                             * FFFFFFFFFF.F0E4D0 7FFFFF.00 -1 0 -00:00:01.00
+                             * FFFFFFFFFF.000000 7FFFFE.FF -1 255 -00:00:01.01
+                             * FFFFFFFFFE.FFD8F0 7FFFFE.F6 -2 246 -00:00:01.10
+                             * FFFFFFFFFE.FE7960 Formula to convert fractional
+                             * part from disk format (now stored in "frac"
+                             * variable) to absolute value: "0x100 - frac". To
+                             * reconstruct in-memory value, we shift to the next
+                             * integer value and then substruct fractional part.
+                             */
+                            intpart++; /* Shift to the next integer value */
+                            frac -= 0x100; /* -(0x100 - frac) */
+                            // fraclong = frac * 10000;
+                        }
+                        ltime = intpart << 24 + frac * 10000;
+                        break;
+                    case 3:
+                    case 4:
+                        intpart = buffer.getBeUint24() - TIMEF_INT_OFS;
+                        frac = buffer.getBeUint16();
+                        if (intpart < 0 && frac > 0) {
+                            /*
+                             * Fix reverse fractional part order:
+                             * "0x10000 - frac". See comments for FSP=1 and
+                             * FSP=2 above.
+                             */
+                            intpart++; /* Shift to the next integer value */
+                            frac -= 0x10000; /* -(0x10000-frac) */
+                            // fraclong = frac * 100;
+                        }
+                        ltime = intpart << 24 + frac * 100;
+                        break;
+                    case 5:
+                    case 6:
+                        intpart = buffer.getBeUlong48() - TIMEF_INT_OFS;
+                        ltime = intpart;
+                        break;
+                    default:
+                        intpart = buffer.getBeUint24() - TIMEF_INT_OFS;
+                        ltime = intpart << 24;
+                        break;
+                }
+
+                if (intpart == 0) {
+                    value = "00:00:00";
+                } else {
+                    // 目前只记录秒，不处理us frac
+                    // if (cal == null) cal = Calendar.getInstance();
+                    // cal.clear();
+                    // cal.set(70, 0, 1, (int) ((intpart >> 12) % (1 << 10)),
+                    // (int) ((intpart >> 6) % (1 << 6)),
+                    // (int) (intpart % (1 << 6)));
+                    // value = new Time(cal.getTimeInMillis());
+                    long ultime = Math.abs(ltime);
+                    intpart = ultime >> 24;
+                    value = String.format("%s%02d:%02d:%02d",
+                        ltime >= 0 ? "" : "-",
+                        (int) ((intpart >> 12) % (1 << 10)),
+                        (int) ((intpart >> 6) % (1 << 6)),
+                        (int) (intpart % (1 << 6)));
+                }
+
+                javaType = Types.TIME;
+                length = 3 + (meta + 1) / 2;
+                break;
+            }
+            case LogEvent.MYSQL_TYPE_NEWDATE: {
+                /*
+                 * log_event.h : This enumeration value is only used internally
+                 * and cannot exist in a binlog.
+                 */
+                logger.warn("MYSQL_TYPE_NEWDATE : This enumeration value is "
+                            + "only used internally and cannot exist in a binlog!");
+                javaType = Types.DATE;
+                value = null; /* unknown format */
+                length = 0;
+                break;
+            }
+            case LogEvent.MYSQL_TYPE_DATE: {
+                // MYSQL DataTypes:
+                // range: 0000-00-00 ~ 9999-12-31
+                final int i32 = buffer.getUint24();
+                if (i32 == 0) {
+                    value = "0000-00-00";
+                } else {
+                    // if (cal == null) cal = Calendar.getInstance();
+                    // cal.clear();
+                    /* month is 0-based, 0 for january. */
+                    // cal.set((i32 / (16 * 32)), (i32 / 32 % 16) - 1, (i32 %
+                    // 32));
+                    // value = new java.sql.Date(cal.getTimeInMillis());
+                    value = String.format("%04d-%02d-%02d", i32 / (16 * 32), i32 / 32 % 16, i32 % 32);
+                }
+                javaType = Types.DATE;
+                length = 3;
+                break;
+            }
+            case LogEvent.MYSQL_TYPE_YEAR: {
+                // MYSQL DataTypes: YEAR[(2|4)]
+                // In four-digit format, values display as 1901 to 2155, and
+                // 0000.
+                // In two-digit format, values display as 70 to 69, representing
+                // years from 1970 to 2069.
+
+                final int i32 = buffer.getUint8();
+                // If connection property 'YearIsDateType' has
+                // set, value is java.sql.Date.
+                /*
+                 * if (cal == null) cal = Calendar.getInstance(); cal.clear();
+                 * cal.set(Calendar.YEAR, i32 + 1900); value = new
+                 * java.sql.Date(cal.getTimeInMillis());
+                 */
+                // The else, value is java.lang.Short.
+                if (i32 == 0) {
+                    value = "0000";
+                } else {
+                    value = String.valueOf((short) (i32 + 1900));
+                }
+                // It might seem more correct to create a java.sql.Types.DATE
+                // value
+                // for this date, but it is much simpler to pass the value as an
+                // integer. The MySQL JDBC specification states that one can
+                // pass a java int between 1901 and 2055. Creating a DATE value
+                // causes truncation errors with certain SQL_MODES
+                // (e.g."STRICT_TRANS_TABLES").
+                javaType = Types.VARCHAR; // Types.INTEGER;
+                length = 1;
+                break;
+            }
+            case LogEvent.MYSQL_TYPE_ENUM: {
+                final int int32;
+                /*
+                 * log_event.h : This enumeration value is only used internally
+                 * and cannot exist in a binlog.
+                 */
+                switch (len) {
+                    case 1:
+                        int32 = buffer.getUint8();
+                        break;
+                    case 2:
+                        int32 = buffer.getUint16();
+                        break;
+                    default:
+                        throw new IllegalArgumentException("!! Unknown ENUM packlen = " + len);
+                }
+                // logger.warn("MYSQL_TYPE_ENUM : This enumeration value is "
+                // + "only used internally and cannot exist in a binlog!");
+                value = Integer.valueOf(int32);
+                javaType = Types.INTEGER;
+                length = len;
+                break;
+            }
+            case LogEvent.MYSQL_TYPE_SET: {
+                final int nbits = (meta & 0xFF) * 8;
+                len = (nbits + 7) / 8;
+                if (nbits > 1) {
+                    // byte[] bits = new byte[len];
+                    // buffer.fillBytes(bits, 0, len);
+                    // 转化为unsign long
+                    switch (len) {
+                        case 1:
+                            value = buffer.getInt8();
+                            break;
+                        case 2:
+                            value = buffer.getUint16();
+                            break;
+                        case 3:
+                            value = buffer.getUint24();
+                            break;
+                        case 4:
+                            value = buffer.getUint32();
+                            break;
+                        case 5:
+                            value = buffer.getUlong40();
+                            break;
+                        case 6:
+                            value = buffer.getUlong48();
+                            break;
+                        case 7:
+                            value = buffer.getUlong56();
+                            break;
+                        case 8:
+                            value = buffer.getUlong64();
+                            break;
+                        default:
+                            throw new IllegalArgumentException("!! Unknown Set len = " + len);
+                    }
+                } else {
+                    final int bit = buffer.getInt8();
+                    // value = (bit != 0) ? Boolean.TRUE : Boolean.FALSE;
+                    value = bit;
+                }
+
+                javaType = Types.BIT;
+                length = len;
+                break;
+            }
+            case LogEvent.MYSQL_TYPE_TINY_BLOB: {
+                /*
+                 * log_event.h : This enumeration value is only used internally
+                 * and cannot exist in a binlog.
+                 */
+                logger.warn("MYSQL_TYPE_TINY_BLOB : This enumeration value is "
+                            + "only used internally and cannot exist in a binlog!");
+            }
+            case LogEvent.MYSQL_TYPE_MEDIUM_BLOB: {
+                /*
+                 * log_event.h : This enumeration value is only used internally
+                 * and cannot exist in a binlog.
+                 */
+                logger.warn("MYSQL_TYPE_MEDIUM_BLOB : This enumeration value is "
+                            + "only used internally and cannot exist in a binlog!");
+            }
+            case LogEvent.MYSQL_TYPE_LONG_BLOB: {
+                /*
+                 * log_event.h : This enumeration value is only used internally
+                 * and cannot exist in a binlog.
+                 */
+                logger.warn("MYSQL_TYPE_LONG_BLOB : This enumeration value is "
+                            + "only used internally and cannot exist in a binlog!");
+            }
+            case LogEvent.MYSQL_TYPE_BLOB: {
+                /*
+                 * BLOB or TEXT datatype
+                 */
+                switch (meta) {
+                    case 1: {
+                        /* TINYBLOB/TINYTEXT */
+                        final int len8 = buffer.getUint8();
+                        byte[] binary = new byte[len8];
+                        buffer.fillBytes(binary, 0, len8);
+                        value = binary;
+                        javaType = Types.VARBINARY;
+                        length = len8;
+                        break;
+                    }
+                    case 2: {
+                        /* BLOB/TEXT */
+                        final int len16 = buffer.getUint16();
+                        byte[] binary = new byte[len16];
+                        buffer.fillBytes(binary, 0, len16);
+                        value = binary;
+                        javaType = Types.LONGVARBINARY;
+                        length = len16;
+                        break;
+                    }
+                    case 3: {
+                        /* MEDIUMBLOB/MEDIUMTEXT */
+                        final int len24 = buffer.getUint24();
+                        byte[] binary = new byte[len24];
+                        buffer.fillBytes(binary, 0, len24);
+                        value = binary;
+                        javaType = Types.LONGVARBINARY;
+                        length = len24;
+                        break;
+                    }
+                    case 4: {
+                        /* LONGBLOB/LONGTEXT */
+                        final int len32 = (int) buffer.getUint32();
+                        byte[] binary = new byte[len32];
+                        buffer.fillBytes(binary, 0, len32);
+                        value = binary;
+                        javaType = Types.LONGVARBINARY;
+                        length = len32;
+                        break;
+                    }
+                    default:
+                        throw new IllegalArgumentException("!! Unknown BLOB packlen = " + meta);
+                }
+                break;
+            }
+            case LogEvent.MYSQL_TYPE_VARCHAR:
+            case LogEvent.MYSQL_TYPE_VAR_STRING: {
+                /*
+                 * Except for the data length calculation, MYSQL_TYPE_VARCHAR,
+                 * MYSQL_TYPE_VAR_STRING and MYSQL_TYPE_STRING are handled the
+                 * same way.
+                 */
+                len = meta;
+                if (len < 256) {
+                    len = buffer.getUint8();
+                } else {
+                    len = buffer.getUint16();
+                }
+
+                if (isBinary) {
+                    // fixed issue #66 ,binary类型在binlog中为var_string
+                    /* fill binary */
+                    byte[] binary = new byte[len];
+                    buffer.fillBytes(binary, 0, len);
+
+                    javaType = Types.VARBINARY;
+                    value = binary;
+                } else {
+                    value = buffer.getFullString(len, charsetName);
+                    javaType = Types.VARCHAR;
+                }
+
+                length = len;
+                break;
+            }
+            case LogEvent.MYSQL_TYPE_STRING: {
+                if (len < 256) {
+                    len = buffer.getUint8();
+                } else {
+                    len = buffer.getUint16();
+                }
+
+                if (isBinary) {
+                    /* fill binary */
+                    byte[] binary = new byte[len];
+                    buffer.fillBytes(binary, 0, len);
+
+                    javaType = Types.BINARY;
+                    value = binary;
+                } else {
+                    value = buffer.getFullString(len, charsetName);
+                    javaType = Types.CHAR; // Types.VARCHAR;
+                }
+                length = len;
+                break;
+            }
+            case LogEvent.MYSQL_TYPE_GEOMETRY: {
+                /*
+                 * MYSQL_TYPE_GEOMETRY: copy from BLOB or TEXT
+                 */
+                switch (meta) {
+                    case 1:
+                        len = buffer.getUint8();
+                        break;
+                    case 2:
+                        len = buffer.getUint16();
+                        break;
+                    case 3:
+                        len = buffer.getUint24();
+                        break;
+                    case 4:
+                        len = (int) buffer.getUint32();
+                        break;
+                    default:
+                        throw new IllegalArgumentException("!! Unknown MYSQL_TYPE_GEOMETRY packlen = " + meta);
+                }
+                /* fill binary */
+                byte[] binary = new byte[len];
+                buffer.fillBytes(binary, 0, len);
+
+                /* Warning unsupport cloumn type */
+                logger.warn(String.format("!! Unsupport column type MYSQL_TYPE_GEOMETRY: meta=%d (%04X), len = %d",
+                    meta,
+                    meta,
+                    len));
+                javaType = Types.BINARY;
+                value = binary;
+                length = len;
+                break;
+            }
+            default:
+                logger.error(String.format("!! Don't know how to handle column type=%d meta=%d (%04X)",
+                    type,
+                    meta,
+                    meta));
+                javaType = Types.OTHER;
+                value = null;
+                length = 0;
+        }
+
+        return value;
+    }
+
+    public final boolean isNull() {
+        return fNull;
+    }
+
+    public final int getJavaType() {
+        return javaType;
+    }
+
+    public final Serializable getValue() {
+        return value;
+    }
+
+    public final int getLength() {
+        return length;
+    }
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogEvent.java

@@ -0,0 +1,221 @@
+package com.taobao.tddl.dbsync.binlog.event;
+
+import java.util.BitSet;
+
+import com.taobao.tddl.dbsync.binlog.LogBuffer;
+import com.taobao.tddl.dbsync.binlog.LogContext;
+import com.taobao.tddl.dbsync.binlog.LogEvent;
+
+/**
+ * Common base class for all row-containing log events.
+ * 
+ * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
+ * @version 1.0
+ */
+public abstract class RowsLogEvent extends LogEvent
+{
+    /**
+     * Fixed data part:
+     * 
+     * <ul>
+     * <li>6 bytes. The table ID.</li>
+     * <li>2 bytes. Reserved for future use.</li>
+     * </ul>
+     * 
+     * <p>
+     * Variable data part:
+     * 
+     * <ul>
+     * <li>Packed integer. The number of columns in the table.</li>
+     * 
+     * <li>Variable-sized. Bit-field indicating whether each column is used, one
+     * bit per column. For this field, the amount of storage required for N
+     * columns is INT((N+7)/8) bytes.</li>
+     * 
+     * <li>Variable-sized (for UPDATE_ROWS_LOG_EVENT only). Bit-field indicating
+     * whether each column is used in the UPDATE_ROWS_LOG_EVENT after-image; one
+     * bit per column. For this field, the amount of storage required for N
+     * columns is INT((N+7)/8) bytes.</li>
+     * 
+     * <li>Variable-sized. A sequence of zero or more rows. The end is
+     * determined by the size of the event. Each row has the following format:
+     * 
+     * <ul>
+     * <li>Variable-sized. Bit-field indicating whether each field in the row is
+     * NULL. Only columns that are "used" according to the second field in the
+     * variable data part are listed here. If the second field in the variable
+     * data part has N one-bits, the amount of storage required for this field
+     * is INT((N+7)/8) bytes.</li>
+     * 
+     * <li>Variable-sized. The row-image, containing values of all table fields.
+     * This only lists table fields that are used (according to the second field
+     * of the variable data part) and non-NULL (according to the previous
+     * field). In other words, the number of values listed here is equal to the
+     * number of zero bits in the previous field (not counting padding bits in
+     * the last byte). The format of each value is described in the
+     * log_event_print_value() function in log_event.cc.</li>
+     * 
+     * <li>(for UPDATE_ROWS_EVENT only) the previous two fields are repeated,
+     * representing a second table row.</li>
+     * </ul>
+     * </ul>
+     * 
+     * Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
+     */
+    private final long       tableId;                           /* Table ID */
+    private TableMapLogEvent table;                             /* The table the rows belong to */
+
+    /** Bitmap denoting columns available */
+    protected final int      columnLen;
+    protected final BitSet   columns;
+
+    /**
+     * Bitmap for columns available in the after image, if present. These fields
+     * are only available for Update_rows events. Observe that the width of both
+     * the before image COLS vector and the after image COLS vector is the same:
+     * the number of columns of the table on the master.
+     */
+    protected final BitSet   changeColumns;
+
+    /** XXX: Don't handle buffer in another thread. */
+    private final LogBuffer  rowsBuf;                           /* The rows in packed format */
+
+    /**
+     * enum enum_flag
+     * 
+     * These definitions allow you to combine the flags into an appropriate flag
+     * set using the normal bitwise operators. The implicit conversion from an
+     * enum-constant to an integer is accepted by the compiler, which is then
+     * used to set the real set of flags.
+     */
+    private final int        flags;
+
+    /** Last event of a statement */
+    public static final int  STMT_END_F              = 1;
+
+    /** Value of the OPTION_NO_FOREIGN_KEY_CHECKS flag in thd->options */
+    public static final int  NO_FOREIGN_KEY_CHECKS_F = (1 << 1);
+
+    /** Value of the OPTION_RELAXED_UNIQUE_CHECKS flag in thd->options */
+    public static final int  RELAXED_UNIQUE_CHECKS_F = (1 << 2);
+
+    /**
+     * Indicates that rows in this event are complete, that is contain values
+     * for all columns of the table.
+     */
+    public static final int  COMPLETE_ROWS_F         = (1 << 3);
+
+    /* RW = "RoWs" */
+    public static final int  RW_MAPID_OFFSET         = 0;
+    public static final int  RW_FLAGS_OFFSET         = 6;
+    public static final int  RW_VHLEN_OFFSET         = 8;
+    public static final int  RW_V_TAG_LEN            = 1;
+    public static final int  RW_V_EXTRAINFO_TAG      = 0;
+
+
+    public RowsLogEvent(LogHeader header, LogBuffer buffer,
+            FormatDescriptionLogEvent descriptionEvent)
+    {
+        super(header);
+
+        final int commonHeaderLen = descriptionEvent.commonHeaderLen;
+        final int postHeaderLen = descriptionEvent.postHeaderLen[header.type - 1];
+        int headerLen = 0;
+        buffer.position(commonHeaderLen + RW_MAPID_OFFSET);
+        if (postHeaderLen == 6)
+        {
+            /* Master is of an intermediate source tree before 5.1.4. Id is 4 bytes */
+            tableId = buffer.getUint32();
+        }
+        else
+        {
+            tableId = buffer.getUlong48(); // RW_FLAGS_OFFSET
+        }
+        flags = buffer.getUint16();
+        
+        if (postHeaderLen == FormatDescriptionLogEvent.ROWS_HEADER_LEN_V2)
+        {
+            headerLen = buffer.getUint16();
+            headerLen -= 2;
+            int start = buffer.position();
+            int end = start + headerLen;
+            for(int i = start ;i < end; ){
+                switch (buffer.getUint8(i++)) {
+                    case RW_V_EXTRAINFO_TAG:
+                        // int infoLen = buffer.getUint8();
+                        buffer.position(i + EXTRA_ROW_INFO_LEN_OFFSET);
+                        int checkLen = buffer.getUint8(); // EXTRA_ROW_INFO_LEN_OFFSET
+                        int val= checkLen - EXTRA_ROW_INFO_HDR_BYTES;
+                        assert(buffer.getUint8() == val); //EXTRA_ROW_INFO_FORMAT_OFFSET
+                        for (int j= 0; j < val; j++) {
+                          assert(buffer.getUint8() == val); // EXTRA_ROW_INFO_HDR_BYTES + i
+                        }
+                        break;
+                    default:
+                        i = end;
+                        break;
+                }
+            }
+        }
+        
+        buffer.position(commonHeaderLen + postHeaderLen + headerLen);
+        columnLen = (int) buffer.getPackedLong();
+        columns = buffer.getBitmap(columnLen);
+
+        if (header.type == UPDATE_ROWS_EVENT_V1 || header.type == UPDATE_ROWS_EVENT)
+        {
+            changeColumns = buffer.getBitmap(columnLen);
+        }
+        else
+        {
+            changeColumns = columns;
+        }
+
+        // XXX: Don't handle buffer in another thread.
+        int dataSize = buffer.limit() - buffer.position();
+        rowsBuf = buffer.duplicate(dataSize);
+    }
+
+    public final void fillTable(LogContext context)
+    {
+        table = context.getTable(tableId);
+
+        // end of statement check:
+        if ((flags & RowsLogEvent.STMT_END_F) != 0)
+        {
+            // Now is safe to clear ignored map (clear_tables will also
+            // delete original table map events stored in the map).
+            context.clearAllTables();
+        }
+    }
+
+    public final long getTableId()
+    {
+        return tableId;
+    }
+
+    public final TableMapLogEvent getTable()
+    {
+        return table;
+    }
+
+    public final BitSet getColumns()
+    {
+        return columns;
+    }
+
+    public final BitSet getChangeColumns()
+    {
+        return changeColumns;
+    }
+
+    public final RowsLogBuffer getRowsBuf(String charsetName)
+    {
+        return new RowsLogBuffer(rowsBuf.duplicate(), columnLen, charsetName);
+    }
+
+    public final int getFlags(final int flags)
+    {
+        return this.flags & flags;
+    }
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsQueryLogEvent.java

@@ -0,0 +1,33 @@
+package com.taobao.tddl.dbsync.binlog.event;
+
+import com.taobao.tddl.dbsync.binlog.LogBuffer;
+
+/**
+ * @author jianghang 2013-4-8 上午12:36:29
+ * @version 1.0.3
+ * @since mysql 5.6
+ */
+public class RowsQueryLogEvent extends IgnorableLogEvent {
+
+    private String rowsQuery;
+
+    public RowsQueryLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
+        super(header, buffer, descriptionEvent);
+
+        final int commonHeaderLen = descriptionEvent.commonHeaderLen;
+        final int postHeaderLen = descriptionEvent.postHeaderLen[header.type - 1];
+
+        /*
+         * m_rows_query length is stored using only one byte, but that length is
+         * ignored and the complete query is read.
+         */
+        int offset = commonHeaderLen + postHeaderLen + 1;
+        int len = buffer.limit() - offset;
+        rowsQuery = buffer.getFullString(offset, len, LogBuffer.ISO_8859_1);
+    }
+
+    public String getRowsQuery() {
+        return rowsQuery;
+    }
+
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/StartLogEventV3.java

@@ -0,0 +1,61 @@
+package com.taobao.tddl.dbsync.binlog.event;
+
+import com.taobao.tddl.dbsync.binlog.LogBuffer;
+import com.taobao.tddl.dbsync.binlog.LogEvent;
+
+/**
+ * Start_log_event_v3 is the Start_log_event of binlog format 3 (MySQL 3.23 and
+ * 4.x).
+ * 
+ * Format_description_log_event derives from Start_log_event_v3; it is the
+ * Start_log_event of binlog format 4 (MySQL 5.0), that is, the event that
+ * describes the other events' Common-Header/Post-Header lengths. This event is
+ * sent by MySQL 5.0 whenever it starts sending a new binlog if the requested
+ * position is >4 (otherwise if ==4 the event will be sent naturally).
+ * 
+ * @see mysql-5.1.60/sql/log_event.cc - Start_log_event_v3
+ * 
+ * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
+ * @version 1.0
+ */
+public class StartLogEventV3 extends LogEvent
+{
+    /**
+     * We could have used SERVER_VERSION_LENGTH, but this introduces an obscure
+     * dependency - if somebody decided to change SERVER_VERSION_LENGTH this
+     * would break the replication protocol
+     */
+    public static final int ST_SERVER_VER_LEN    = 50;
+
+    /* start event post-header (for v3 and v4) */
+    public static final int ST_BINLOG_VER_OFFSET = 0;
+    public static final int ST_SERVER_VER_OFFSET = 2;
+
+    protected int           binlogVersion;
+    protected String        serverVersion;
+
+    public StartLogEventV3(LogHeader header, LogBuffer buffer,
+            FormatDescriptionLogEvent descriptionEvent)
+    {
+        super(header);
+
+        buffer.position(descriptionEvent.commonHeaderLen);
+        binlogVersion = buffer.getUint16(); // ST_BINLOG_VER_OFFSET
+        serverVersion = buffer.getFixString(ST_SERVER_VER_LEN); // ST_SERVER_VER_OFFSET
+    }
+
+    public StartLogEventV3()
+    {
+        super(new LogHeader(START_EVENT_V3));
+    }
+
+    public final String getServerVersion()
+    {
+        return serverVersion;
+    }
+
+    public final int getBinlogVersion()
+    {
+        return binlogVersion;
+    }
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/StopLogEvent.java

@@ -0,0 +1,22 @@
+package com.taobao.tddl.dbsync.binlog.event;
+
+import com.taobao.tddl.dbsync.binlog.LogBuffer;
+import com.taobao.tddl.dbsync.binlog.LogEvent;
+
+/**
+ * Stop_log_event.
+ * 
+ * The Post-Header and Body for this event type are empty; it only has the
+ * Common-Header.
+ * 
+ * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
+ * @version 1.0
+ */
+public final class StopLogEvent extends LogEvent
+{
+    public StopLogEvent(LogHeader header, LogBuffer buffer,
+            FormatDescriptionLogEvent description_event)
+    {
+        super(header);
+    }
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/TableMapLogEvent.java

@@ -0,0 +1,543 @@
+package com.taobao.tddl.dbsync.binlog.event;
+
+import java.util.BitSet;
+
+import com.taobao.tddl.dbsync.binlog.LogBuffer;
+import com.taobao.tddl.dbsync.binlog.LogEvent;
+
+/**
+ * In row-based mode, every row operation event is preceded by a
+ * Table_map_log_event which maps a table definition to a number. The table
+ * definition consists of database name, table name, and column definitions.
+ * 
+ * The Post-Header has the following components:
+ * 
+ * <table>
+ * <caption>Post-Header for Table_map_log_event</caption>
+ * 
+ * <tr>
+ * <th>Name</th>
+ * <th>Format</th>
+ * <th>Description</th>
+ * </tr>
+ * 
+ * <tr>
+ * <td>table_id</td>
+ * <td>6 bytes unsigned integer</td>
+ * <td>The number that identifies the table.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>flags</td>
+ * <td>2 byte bitfield</td>
+ * <td>Reserved for future use; currently always 0.</td>
+ * </tr>
+ * 
+ * </table>
+ * 
+ * The Body has the following components:
+ * 
+ * <table>
+ * <caption>Body for Table_map_log_event</caption>
+ * 
+ * <tr>
+ * <th>Name</th>
+ * <th>Format</th>
+ * <th>Description</th>
+ * </tr>
+ * 
+ * <tr>
+ * <td>database_name</td>
+ * <td>one byte string length, followed by null-terminated string</td>
+ * <td>The name of the database in which the table resides. The name is
+ * represented as a one byte unsigned integer representing the number of bytes
+ * in the name, followed by length bytes containing the database name, followed
+ * by a terminating 0 byte. (Note the redundancy in the representation of the
+ * length.)</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>table_name</td>
+ * <td>one byte string length, followed by null-terminated string</td>
+ * <td>The name of the table, encoded the same way as the database name above.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>column_count</td>
+ * <td>packed_integer "Packed Integer"</td>
+ * <td>The number of columns in the table, represented as a packed
+ * variable-length integer.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>column_type</td>
+ * <td>List of column_count 1 byte enumeration values</td>
+ * <td>The type of each column in the table, listed from left to right. Each
+ * byte is mapped to a column type according to the enumeration type
+ * enum_field_types defined in mysql_com.h. The mapping of types to numbers is
+ * listed in the table Table_table_map_log_event_column_types "below" (along
+ * with description of the associated metadata field).</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>metadata_length</td>
+ * <td>packed_integer "Packed Integer"</td>
+ * <td>The length of the following metadata block</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>metadata</td>
+ * <td>list of metadata for each column</td>
+ * <td>For each column from left to right, a chunk of data who's length and
+ * semantics depends on the type of the column. The length and semantics for the
+ * metadata for each column are listed in the table
+ * Table_table_map_log_event_column_types "below".</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>null_bits</td>
+ * <td>column_count bits, rounded up to nearest byte</td>
+ * <td>For each column, a bit indicating whether data in the column can be NULL
+ * or not. The number of bytes needed for this is int((column_count+7)/8). The
+ * flag for the first column from the left is in the least-significant bit of
+ * the first byte, the second is in the second least significant bit of the
+ * first byte, the ninth is in the least significant bit of the second byte, and
+ * so on.</td>
+ * </tr>
+ * 
+ * </table>
+ * 
+ * The table below lists all column types, along with the numerical identifier
+ * for it and the size and interpretation of meta-data used to describe the
+ * type.
+ * 
+ * <table>
+ * <caption>Table_map_log_event column types: numerical identifier and
+ * metadata</caption>
+ * <tr>
+ * <th>Name</th>
+ * <th>Identifier</th>
+ * <th>Size of metadata in bytes</th>
+ * <th>Description of metadata</th>
+ * </tr>
+ * 
+ * <tr>
+ * <td>MYSQL_TYPE_DECIMAL</td>
+ * <td>0</td>
+ * <td>0</td>
+ * <td>No column metadata.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>MYSQL_TYPE_TINY</td>
+ * <td>1</td>
+ * <td>0</td>
+ * <td>No column metadata.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>MYSQL_TYPE_SHORT</td>
+ * <td>2</td>
+ * <td>0</td>
+ * <td>No column metadata.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>MYSQL_TYPE_LONG</td>
+ * <td>3</td>
+ * <td>0</td>
+ * <td>No column metadata.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>MYSQL_TYPE_FLOAT</td>
+ * <td>4</td>
+ * <td>1 byte</td>
+ * <td>1 byte unsigned integer, representing the "pack_length", which is equal
+ * to sizeof(float) on the server from which the event originates.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>MYSQL_TYPE_DOUBLE</td>
+ * <td>5</td>
+ * <td>1 byte</td>
+ * <td>1 byte unsigned integer, representing the "pack_length", which is equal
+ * to sizeof(double) on the server from which the event originates.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>MYSQL_TYPE_NULL</td>
+ * <td>6</td>
+ * <td>0</td>
+ * <td>No column metadata.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>MYSQL_TYPE_TIMESTAMP</td>
+ * <td>7</td>
+ * <td>0</td>
+ * <td>No column metadata.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>MYSQL_TYPE_LONGLONG</td>
+ * <td>8</td>
+ * <td>0</td>
+ * <td>No column metadata.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>MYSQL_TYPE_INT24</td>
+ * <td>9</td>
+ * <td>0</td>
+ * <td>No column metadata.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>MYSQL_TYPE_DATE</td>
+ * <td>10</td>
+ * <td>0</td>
+ * <td>No column metadata.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>MYSQL_TYPE_TIME</td>
+ * <td>11</td>
+ * <td>0</td>
+ * <td>No column metadata.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>MYSQL_TYPE_DATETIME</td>
+ * <td>12</td>
+ * <td>0</td>
+ * <td>No column metadata.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>MYSQL_TYPE_YEAR</td>
+ * <td>13</td>
+ * <td>0</td>
+ * <td>No column metadata.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td><i>MYSQL_TYPE_NEWDATE</i></td>
+ * <td><i>14</i></td>
+ * <td>&ndash;</td>
+ * <td><i>This enumeration value is only used internally and cannot exist in a
+ * binlog.</i></td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>MYSQL_TYPE_VARCHAR</td>
+ * <td>15</td>
+ * <td>2 bytes</td>
+ * <td>2 byte unsigned integer representing the maximum length of the string.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>MYSQL_TYPE_BIT</td>
+ * <td>16</td>
+ * <td>2 bytes</td>
+ * <td>A 1 byte unsigned int representing the length in bits of the bitfield (0
+ * to 64), followed by a 1 byte unsigned int representing the number of bytes
+ * occupied by the bitfield. The number of bytes is either int((length+7)/8) or
+ * int(length/8).</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>MYSQL_TYPE_NEWDECIMAL</td>
+ * <td>246</td>
+ * <td>2 bytes</td>
+ * <td>A 1 byte unsigned int representing the precision, followed by a 1 byte
+ * unsigned int representing the number of decimals.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td><i>MYSQL_TYPE_ENUM</i></td>
+ * <td><i>247</i></td>
+ * <td>&ndash;</td>
+ * <td><i>This enumeration value is only used internally and cannot exist in a
+ * binlog.</i></td>
+ * </tr>
+ * 
+ * <tr>
+ * <td><i>MYSQL_TYPE_SET</i></td>
+ * <td><i>248</i></td>
+ * <td>&ndash;</td>
+ * <td><i>This enumeration value is only used internally and cannot exist in a
+ * binlog.</i></td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>MYSQL_TYPE_TINY_BLOB</td>
+ * <td>249</td>
+ * <td>&ndash;</td>
+ * <td><i>This enumeration value is only used internally and cannot exist in a
+ * binlog.</i></td>
+ * </tr>
+ * 
+ * <tr>
+ * <td><i>MYSQL_TYPE_MEDIUM_BLOB</i></td>
+ * <td><i>250</i></td>
+ * <td>&ndash;</td>
+ * <td><i>This enumeration value is only used internally and cannot exist in a
+ * binlog.</i></td>
+ * </tr>
+ * 
+ * <tr>
+ * <td><i>MYSQL_TYPE_LONG_BLOB</i></td>
+ * <td><i>251</i></td>
+ * <td>&ndash;</td>
+ * <td><i>This enumeration value is only used internally and cannot exist in a
+ * binlog.</i></td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>MYSQL_TYPE_BLOB</td>
+ * <td>252</td>
+ * <td>1 byte</td>
+ * <td>The pack length, i.e., the number of bytes needed to represent the length
+ * of the blob: 1, 2, 3, or 4.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>MYSQL_TYPE_VAR_STRING</td>
+ * <td>253</td>
+ * <td>2 bytes</td>
+ * <td>This is used to store both strings and enumeration values. The first byte
+ * is a enumeration value storing the <i>real type</i>, which may be either
+ * MYSQL_TYPE_VAR_STRING or MYSQL_TYPE_ENUM. The second byte is a 1 byte
+ * unsigned integer representing the field size, i.e., the number of bytes
+ * needed to store the length of the string.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>MYSQL_TYPE_STRING</td>
+ * <td>254</td>
+ * <td>2 bytes</td>
+ * <td>The first byte is always MYSQL_TYPE_VAR_STRING (i.e., 253). The second
+ * byte is the field size, i.e., the number of bytes in the representation of
+ * size of the string: 3 or 4.</td>
+ * </tr>
+ * 
+ * <tr>
+ * <td>MYSQL_TYPE_GEOMETRY</td>
+ * <td>255</td>
+ * <td>1 byte</td>
+ * <td>The pack length, i.e., the number of bytes needed to represent the length
+ * of the geometry: 1, 2, 3, or 4.</td>
+ * </tr>
+ * 
+ * </table>
+ * 
+ * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
+ * @version 1.0
+ */
+public final class TableMapLogEvent extends LogEvent
+{
+    /**
+     * Fixed data part:
+     * <ul>
+     * <li>6 bytes. The table ID.</li>
+     * <li>2 bytes. Reserved for future use.</li>
+     * </ul>
+     * 
+     * <p>
+     * Variable data part:
+     * <ul>
+     * <li>1 byte. The length of the database name.</li>
+     * <li>Variable-sized. The database name (null-terminated).</li>
+     * <li>1 byte. The length of the table name.</li>
+     * <li>Variable-sized. The table name (null-terminated).</li>
+     * <li>Packed integer. The number of columns in the table.</li>
+     * <li>Variable-sized. An array of column types, one byte per column.</li>
+     * <li>Packed integer. The length of the metadata block.</li>
+     * <li>Variable-sized. The metadata block; see log_event.h for contents and
+     * format.</li>
+     * <li>Variable-sized. Bit-field indicating whether each column can be NULL,
+     * one bit per column. For this field, the amount of storage required for N
+     * columns is INT((N+7)/8) bytes.</li>
+     * </ul>
+     * 
+     * Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
+     */
+    protected final String dbname;
+    protected final String tblname;
+
+    /**
+     * Holding mysql column information.
+     */
+    public static final class ColumnInfo
+    {
+        public int type;
+        public int meta;
+    }
+
+    protected final int          columnCnt;
+    protected final ColumnInfo[] columnInfo;         // buffer for field metadata
+
+    protected final long         tableId;
+    protected BitSet             nullBits;
+
+    /** TM = "Table Map" */
+    public static final int      TM_MAPID_OFFSET = 0;
+    public static final int      TM_FLAGS_OFFSET = 6;
+
+    /**
+     * Constructor used by slave to read the event from the binary log.
+     */
+    public TableMapLogEvent(LogHeader header, LogBuffer buffer,
+            FormatDescriptionLogEvent descriptionEvent)
+    {
+        super(header);
+
+        final int commonHeaderLen = descriptionEvent.commonHeaderLen;
+        final int postHeaderLen = descriptionEvent.postHeaderLen[header.type - 1];
+        /* Read the post-header */
+        buffer.position(commonHeaderLen + TM_MAPID_OFFSET);
+        if (postHeaderLen == 6)
+        {
+            /* Master is of an intermediate source tree before 5.1.4. Id is 4 bytes */
+            tableId = buffer.getUint32();
+        }
+        else
+        {
+            // DBUG_ASSERT(post_header_len == TABLE_MAP_HEADER_LEN);
+            tableId = buffer.getUlong48();
+        }
+        // flags = buffer.getUint16();
+
+        /* Read the variable part of the event */
+        buffer.position(commonHeaderLen + postHeaderLen);
+        dbname = buffer.getString();
+        buffer.forward(1); /* termination null */
+        tblname = buffer.getString();
+        buffer.forward(1); /* termination null */
+
+        // Read column information from buffer
+        columnCnt = (int) buffer.getPackedLong();
+        columnInfo = new ColumnInfo[columnCnt];
+        for (int i = 0; i < columnCnt; i++)
+        {
+            ColumnInfo info = new ColumnInfo();
+            info.type = buffer.getUint8();
+            columnInfo[i] = info;
+        }
+
+        if (buffer.position() < buffer.limit())
+        {
+            final int fieldSize = (int) buffer.getPackedLong();
+            decodeFields(buffer, fieldSize);
+            nullBits = buffer.getBitmap(columnCnt);
+        }
+    }
+
+    /**
+     * Decode field metadata by column types.
+     * 
+     * @see mysql-5.1.60/sql/rpl_utility.h
+     */
+    private final void decodeFields(LogBuffer buffer, final int len)
+    {
+        final int limit = buffer.limit();
+
+        buffer.limit(len + buffer.position());
+        for (int i = 0; i < columnCnt; i++)
+        {
+            ColumnInfo info = columnInfo[i];
+
+            switch (info.type)
+            {
+            case MYSQL_TYPE_TINY_BLOB:
+            case MYSQL_TYPE_BLOB:
+            case MYSQL_TYPE_MEDIUM_BLOB:
+            case MYSQL_TYPE_LONG_BLOB:
+            case MYSQL_TYPE_DOUBLE:
+            case MYSQL_TYPE_FLOAT:
+            case MYSQL_TYPE_GEOMETRY:
+                /*
+                  These types store a single byte.
+                */
+                info.meta = buffer.getUint8();
+                break;
+            case MYSQL_TYPE_SET:
+            case MYSQL_TYPE_ENUM:
+                /*
+                 * log_event.h : MYSQL_TYPE_SET & MYSQL_TYPE_ENUM : This
+                 * enumeration value is only used internally and cannot
+                 * exist in a binlog.
+                 */
+                logger.warn("This enumeration value is only used internally "
+                        + "and cannot exist in a binlog: type=" + info.type);
+                break;
+            case MYSQL_TYPE_STRING:
+                {
+                    /*
+                     * log_event.h : The first byte is always
+                     * MYSQL_TYPE_VAR_STRING (i.e., 253). The second byte is the
+                     * field size, i.e., the number of bytes in the
+                     * representation of size of the string: 3 or 4.
+                     */
+                    int x = (buffer.getUint8() << 8); // real_type
+                    x += buffer.getUint8(); // pack or field length
+                    info.meta = x;
+                    break;
+                }
+            case MYSQL_TYPE_BIT:
+                info.meta = buffer.getUint16();
+                break;
+            case MYSQL_TYPE_VARCHAR:
+                /*
+                 * These types store two bytes.
+                 */
+                info.meta = buffer.getUint16();
+                break;
+            case MYSQL_TYPE_NEWDECIMAL:
+                {
+                    int x = buffer.getUint8() << 8; // precision
+                    x += buffer.getUint8(); // decimals
+                    info.meta = x;
+                    break;
+                }
+            case MYSQL_TYPE_TIME2:
+            case MYSQL_TYPE_DATETIME2:
+            case MYSQL_TYPE_TIMESTAMP2:
+                {
+                    info.meta = buffer.getUint8();
+                    break;
+                }
+            default:
+                info.meta = 0;
+                break;
+            }
+        }
+        buffer.limit(limit);
+    }
+
+    public final String getDbName()
+    {
+        return dbname;
+    }
+
+    public final String getTableName()
+    {
+        return tblname;
+    }
+
+    public final int getColumnCnt()
+    {
+        return columnCnt;
+    }
+
+    public final ColumnInfo[] getColumnInfo()
+    {
+        return columnInfo;
+    }
+
+    public final long getTableId()
+    {
+        return tableId;
+    }
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UnknownLogEvent.java

@@ -0,0 +1,17 @@
+package com.taobao.tddl.dbsync.binlog.event;
+
+import com.taobao.tddl.dbsync.binlog.LogEvent;
+
+/**
+ * Unknown_log_event
+ * 
+ * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
+ * @version 1.0
+ */
+public final class UnknownLogEvent extends LogEvent
+{
+    public UnknownLogEvent(LogHeader header)
+    {
+        super(header);
+    }
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UpdateRowsLogEvent.java

@@ -0,0 +1,22 @@
+package com.taobao.tddl.dbsync.binlog.event;
+
+import com.taobao.tddl.dbsync.binlog.LogBuffer;
+
+/**
+ * Log row updates with a before image. The event contain several update rows
+ * for a table. Note that each event contains only rows for one table.
+ * 
+ * Also note that the row data consists of pairs of row data: one row for the
+ * old data and one row for the new data.
+ * 
+ * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
+ * @version 1.0
+ */
+public final class UpdateRowsLogEvent extends RowsLogEvent
+{
+    public UpdateRowsLogEvent(LogHeader header, LogBuffer buffer,
+            FormatDescriptionLogEvent descriptionEvent)
+    {
+        super(header, buffer, descriptionEvent);
+    }
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/UserVarLogEvent.java

@@ -0,0 +1,146 @@
+package com.taobao.tddl.dbsync.binlog.event;
+
+import java.io.IOException;
+import java.io.Serializable;
+
+import com.taobao.tddl.dbsync.binlog.CharsetConversion;
+import com.taobao.tddl.dbsync.binlog.LogBuffer;
+import com.taobao.tddl.dbsync.binlog.LogEvent;
+
+/**
+ * User_var_log_event.
+ * 
+ * Every time a query uses the value of a user variable, a User_var_log_event is
+ * written before the Query_log_event, to set the user variable.
+ * 
+ * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
+ * @version 1.0
+ */
+public final class UserVarLogEvent extends LogEvent
+{
+    /**
+     * Fixed data part: Empty
+     * 
+     * <p>
+     * Variable data part:
+     * 
+     * <ul>
+     * <li>4 bytes. the size of the user variable name.</li>
+     * <li>The user variable name.</li>
+     * <li>1 byte. Non-zero if the variable value is the SQL NULL value, 0
+     * otherwise. If this byte is 0, the following parts exist in the event.</li>
+     * <li>1 byte. The user variable type. The value corresponds to elements of
+     * enum Item_result defined in include/mysql_com.h.</li>
+     * <li>4 bytes. The number of the character set for the user variable
+     * (needed for a string variable). The character set number is really a
+     * collation number that indicates a character set/collation pair.</li>
+     * <li>4 bytes. The size of the user variable value (corresponds to member
+     * val_len of class Item_string).</li>
+     * <li>Variable-sized. For a string variable, this is the string. For a
+     * float or integer variable, this is its value in 8 bytes.</li>
+     * </ul>
+     * 
+     * Source : http://forge.mysql.com/wiki/MySQL_Internals_Binary_Log
+     */
+    private final String       name;
+    private final Serializable value;
+    private final int          type;
+    private final int          charsetNumber;
+    private final boolean      isNull;
+
+    /**
+     * The following is for user defined functions
+     * 
+     * @see mysql-5.1.60//include/mysql_com.h
+     */
+    public static final int    STRING_RESULT          = 0;
+    public static final int    REAL_RESULT            = 1;
+    public static final int    INT_RESULT             = 2;
+    public static final int    ROW_RESULT             = 3;
+    public static final int    DECIMAL_RESULT         = 4;
+
+    /* User_var event data */
+    public static final int    UV_VAL_LEN_SIZE        = 4;
+    public static final int    UV_VAL_IS_NULL         = 1;
+    public static final int    UV_VAL_TYPE_SIZE       = 1;
+    public static final int    UV_NAME_LEN_SIZE       = 4;
+    public static final int    UV_CHARSET_NUMBER_SIZE = 4;
+
+    public UserVarLogEvent(LogHeader header, LogBuffer buffer,
+            FormatDescriptionLogEvent descriptionEvent) throws IOException
+    {
+        super(header);
+
+        /* The Post-Header is empty. The Variable Data part begins immediately. */
+        buffer.position(descriptionEvent.commonHeaderLen
+                + descriptionEvent.postHeaderLen[USER_VAR_EVENT - 1]);
+        final int nameLen = (int) buffer.getUint32();
+        name = buffer.getFixString(nameLen); // UV_NAME_LEN_SIZE
+        isNull = (0 != buffer.getInt8());
+
+        if (isNull)
+        {
+            type = STRING_RESULT;
+            charsetNumber = 63; /* binary */
+            value = null;
+        }
+        else
+        {
+            type = buffer.getInt8(); // UV_VAL_IS_NULL
+            charsetNumber = (int) buffer.getUint32(); // buf + UV_VAL_TYPE_SIZE
+            final int valueLen = (int) buffer.getUint32(); // buf +  UV_CHARSET_NUMBER_SIZE
+            final int limit = buffer.limit(); /* for restore */
+            buffer.limit(buffer.position() + valueLen);
+
+            /* @see User_var_log_event::print */
+            switch (type)
+            {
+            case REAL_RESULT:
+                value = Double.valueOf(buffer.getDouble64()); // float8get
+                break;
+            case INT_RESULT:
+                if (valueLen == 8)
+                    value = Long.valueOf(buffer.getLong64()); // !uint8korr
+                else if (valueLen == 4)
+                    value = Long.valueOf(buffer.getUint32());
+                else
+                    throw new IOException("Error INT_RESULT length: "
+                            + valueLen);
+                break;
+            case DECIMAL_RESULT:
+                final int precision = buffer.getInt8();
+                final int scale = buffer.getInt8();
+                value = buffer.getDecimal(precision, scale); // bin2decimal
+                break;
+            case STRING_RESULT:
+                String charsetName = CharsetConversion.getJavaCharset(charsetNumber);
+                value = buffer.getFixString(valueLen, charsetName);
+                break;
+            case ROW_RESULT:
+                // this seems to be banned in MySQL altogether
+                throw new IOException("ROW_RESULT is unsupported");
+            default:
+                value = null;
+                break;
+            }
+            buffer.limit(limit);
+        }
+    }
+
+    public final String getQuery()
+    {
+        if (value == null)
+        {
+            return "SET @" + name + " := NULL";
+        }
+        else if (type == STRING_RESULT)
+        {
+            // TODO: do escaping !?
+            return "SET @" + name + " := \'" + value + '\'';
+        }
+        else
+        {
+            return "SET @" + name + " := " + String.valueOf(value);
+        }
+    }
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/WriteRowsLogEvent.java

@@ -0,0 +1,19 @@
+package com.taobao.tddl.dbsync.binlog.event;
+
+import com.taobao.tddl.dbsync.binlog.LogBuffer;
+
+/**
+ * Log row insertions and updates. The event contain several insert/update rows
+ * for a table. Note that each event contains only rows for one table.
+ * 
+ * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
+ * @version 1.0
+ */
+public final class WriteRowsLogEvent extends RowsLogEvent
+{
+    public WriteRowsLogEvent(LogHeader header, LogBuffer buffer,
+            FormatDescriptionLogEvent descriptionEvent)
+    {
+        super(header, buffer, descriptionEvent);
+    }
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/XidLogEvent.java

@@ -0,0 +1,32 @@
+package com.taobao.tddl.dbsync.binlog.event;
+
+import com.taobao.tddl.dbsync.binlog.LogBuffer;
+import com.taobao.tddl.dbsync.binlog.LogEvent;
+
+/**
+ * Logs xid of the transaction-to-be-committed in the 2pc protocol. Has no
+ * meaning in replication, slaves ignore it.
+ * 
+ * @author <a href="mailto:changyuan.lh@taobao.com">Changyuan.lh</a>
+ * @version 1.0
+ */
+public final class XidLogEvent extends LogEvent
+{
+    private final long xid;
+
+    public XidLogEvent(LogHeader header, LogBuffer buffer,
+            FormatDescriptionLogEvent descriptionEvent)
+    {
+        super(header);
+
+        /* The Post-Header is empty. The Variable Data part begins immediately. */
+        buffer.position(descriptionEvent.commonHeaderLen
+                + descriptionEvent.postHeaderLen[XID_EVENT - 1]);
+        xid = buffer.getLong64(); // !uint8korr
+    }
+
+    public final long getXid()
+    {
+        return xid;
+    }
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/AnnotateRowsEvent.java

@@ -0,0 +1,33 @@
+package com.taobao.tddl.dbsync.binlog.event.mariadb;
+
+import com.taobao.tddl.dbsync.binlog.LogBuffer;
+import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.IgnorableLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.LogHeader;
+
+/**
+ * mariadb的ANNOTATE_ROWS_EVENT类型
+ * 
+ * @author jianghang 2014-1-20 下午2:20:35
+ * @since 1.0.17
+ */
+public class AnnotateRowsEvent extends IgnorableLogEvent {
+
+    private String rowsQuery;
+
+    public AnnotateRowsEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
+        super(header, buffer, descriptionEvent);
+
+        final int commonHeaderLen = descriptionEvent.getCommonHeaderLen();
+        final int postHeaderLen = descriptionEvent.getPostHeaderLen()[header.getType() - 1];
+
+        int offset = commonHeaderLen + postHeaderLen;
+        int len = buffer.limit() - offset;
+        rowsQuery = buffer.getFullString(offset, len, LogBuffer.ISO_8859_1);
+    }
+
+    public String getRowsQuery() {
+        return rowsQuery;
+    }
+
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/BinlogCheckPointLogEvent.java

@@ -0,0 +1,21 @@
+package com.taobao.tddl.dbsync.binlog.event.mariadb;
+
+import com.taobao.tddl.dbsync.binlog.LogBuffer;
+import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.IgnorableLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.LogHeader;
+
+/**
+ * mariadb10的BINLOG_CHECKPOINT_EVENT类型
+ * 
+ * @author jianghang 2014-1-20 下午2:22:04
+ * @since 1.0.17
+ */
+public class BinlogCheckPointLogEvent extends IgnorableLogEvent {
+
+    public BinlogCheckPointLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
+        super(header, buffer, descriptionEvent);
+        // do nothing , just mariadb binlog checkpoint
+    }
+
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/MariaGtidListLogEvent.java

@@ -0,0 +1,21 @@
+package com.taobao.tddl.dbsync.binlog.event.mariadb;
+
+import com.taobao.tddl.dbsync.binlog.LogBuffer;
+import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.IgnorableLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.LogHeader;
+
+/**
+ * mariadb的GTID_LIST_EVENT类型
+ * 
+ * @author jianghang 2014-1-20 下午4:51:50
+ * @since 1.0.17
+ */
+public class MariaGtidListLogEvent extends IgnorableLogEvent {
+
+    public MariaGtidListLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
+        super(header, buffer, descriptionEvent);
+        // do nothing , just ignore log event
+    }
+
+}

dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/mariadb/MariaGtidLogEvent.java

@@ -0,0 +1,21 @@
+package com.taobao.tddl.dbsync.binlog.event.mariadb;
+
+import com.taobao.tddl.dbsync.binlog.LogBuffer;
+import com.taobao.tddl.dbsync.binlog.event.FormatDescriptionLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.IgnorableLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.LogHeader;
+
+/**
+ * mariadb的GTID_EVENT类型
+ * 
+ * @author jianghang 2014-1-20 下午4:49:10
+ * @since 1.0.17
+ */
+public class MariaGtidLogEvent extends IgnorableLogEvent {
+
+    public MariaGtidLogEvent(LogHeader header, LogBuffer buffer, FormatDescriptionLogEvent descriptionEvent){
+        super(header, buffer, descriptionEvent);
+        // do nothing , just ignore log event
+    }
+
+}

dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/BaseLogFetcherTest.java

@@ -0,0 +1,109 @@
+package com.taobao.tddl.dbsync.binlog;
+
+import java.io.Serializable;
+import java.io.UnsupportedEncodingException;
+import java.nio.charset.Charset;
+import java.util.BitSet;
+
+import com.taobao.tddl.dbsync.binlog.event.QueryLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.RowsLogBuffer;
+import com.taobao.tddl.dbsync.binlog.event.RowsLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.RowsQueryLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.XidLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.TableMapLogEvent.ColumnInfo;
+import com.taobao.tddl.dbsync.binlog.event.mariadb.AnnotateRowsEvent;
+
+public class BaseLogFetcherTest {
+
+    protected String  binlogFileName = "mysql-bin.000001";
+    protected Charset charset        = Charset.forName("utf-8");
+
+    protected void parseQueryEvent(QueryLogEvent event) {
+        System.out.println(String.format("================> binlog[%s:%s] , name[%s]", binlogFileName,
+                                         event.getHeader().getLogPos() - event.getHeader().getEventLen(),
+                                         event.getCatalog()));
+        System.out.println("sql : " + event.getQuery());
+    }
+
+    protected void parseRowsQueryEvent(RowsQueryLogEvent event) throws Exception {
+        System.out.println(String.format("================> binlog[%s:%s]", binlogFileName,
+                                         event.getHeader().getLogPos() - event.getHeader().getEventLen()));
+        System.out.println("sql : " + new String(event.getRowsQuery().getBytes("ISO-8859-1"), charset.name()));
+    }
+    
+    protected void parseAnnotateRowsEvent(AnnotateRowsEvent event) throws Exception {
+        System.out.println(String.format("================> binlog[%s:%s]", binlogFileName,
+                                         event.getHeader().getLogPos() - event.getHeader().getEventLen()));
+        System.out.println("sql : " + new String(event.getRowsQuery().getBytes("ISO-8859-1"), charset.name()));
+    }
+    
+    protected void parseXidEvent(XidLogEvent event) throws Exception {
+        System.out.println(String.format("================> binlog[%s:%s]", binlogFileName,
+                                         event.getHeader().getLogPos() - event.getHeader().getEventLen()));
+        System.out.println("xid : " + event.getXid());
+    }
+
+
+    protected void parseRowsEvent(RowsLogEvent event) {
+        try {
+            System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s]", binlogFileName,
+                                             event.getHeader().getLogPos() - event.getHeader().getEventLen(),
+                                             event.getTable().getDbName(), event.getTable().getTableName()));
+            RowsLogBuffer buffer = event.getRowsBuf(charset.name());
+            BitSet columns = event.getColumns();
+            BitSet changeColumns = event.getChangeColumns();
+            while (buffer.nextOneRow(columns)) {
+                // 处理row记录
+                int type = event.getHeader().getType();
+                if (LogEvent.WRITE_ROWS_EVENT_V1 == type || LogEvent.WRITE_ROWS_EVENT == type) {
+                    // insert的记录放在before字段中
+                    parseOneRow(event, buffer, columns, true);
+                } else if (LogEvent.DELETE_ROWS_EVENT_V1 == type || LogEvent.DELETE_ROWS_EVENT == type) {
+                    // delete的记录放在before字段中
+                    parseOneRow(event, buffer, columns, false);
+                } else {
+                    // update需要处理before/after
+                    System.out.println("-------> before");
+                    parseOneRow(event, buffer, columns, false);
+                    if (!buffer.nextOneRow(changeColumns)) {
+                        break;
+                    }
+                    System.out.println("-------> after");
+                    parseOneRow(event, buffer, changeColumns, true);
+                }
+
+            }
+        } catch (Exception e) {
+            throw new RuntimeException("parse row data failed.", e);
+        }
+    }
+
+    protected void parseOneRow(RowsLogEvent event, RowsLogBuffer buffer, BitSet cols, boolean isAfter)
+                                                                                                      throws UnsupportedEncodingException {
+        TableMapLogEvent map = event.getTable();
+        if (map == null) {
+            throw new RuntimeException("not found TableMap with tid=" + event.getTableId());
+        }
+
+        final int columnCnt = map.getColumnCnt();
+        final ColumnInfo[] columnInfo = map.getColumnInfo();
+
+        for (int i = 0; i < columnCnt; i++) {
+            ColumnInfo info = columnInfo[i];
+            buffer.nextValue(info.type, info.meta);
+
+            if (buffer.isNull()) {
+                //
+            } else {
+                final Serializable value = buffer.getValue();
+                if (value instanceof byte[]) {
+                    System.out.println(new String((byte[]) value));
+                } else {
+                    System.out.println(value);
+                }
+            }
+        }
+
+    }
+}

dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/DirectLogFetcherTest.java

@@ -0,0 +1,90 @@
+package com.taobao.tddl.dbsync.binlog;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.Statement;
+
+import junit.framework.Assert;
+
+import org.junit.Test;
+
+import com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.QueryLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.RotateLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.RowsQueryLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.UpdateRowsLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.XidLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.mariadb.AnnotateRowsEvent;
+
+public class DirectLogFetcherTest extends BaseLogFetcherTest {
+
+    @Test
+    public void testSimple() {
+        DirectLogFetcher fecther = new DirectLogFetcher();
+        try {
+            Class.forName("com.mysql.jdbc.Driver");
+            Connection connection = DriverManager.getConnection("jdbc:mysql://127.0.0.1:3306", "xxxxx", "xxxxx");
+            Statement statement = connection.createStatement();
+            statement.execute("SET @master_binlog_checksum='@@global.binlog_checksum'");
+            statement.execute("SET @mariadb_slave_capability='" + LogEvent.MARIA_SLAVE_CAPABILITY_MINE + "'");
+
+            fecther.open(connection, "mysql-bin.000003", 4L, 2);
+
+            LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
+            LogContext context = new LogContext();
+            while (fecther.fetch()) {
+                LogEvent event = null;
+                event = decoder.decode(fecther, context);
+
+                if (event == null) {
+                    throw new RuntimeException("parse failed");
+                }
+
+                int eventType = event.getHeader().getType();
+                switch (eventType) {
+                    case LogEvent.ROTATE_EVENT:
+                        binlogFileName = ((RotateLogEvent) event).getFilename();
+                        break;
+                    case LogEvent.WRITE_ROWS_EVENT_V1:
+                    case LogEvent.WRITE_ROWS_EVENT:
+                        parseRowsEvent((WriteRowsLogEvent) event);
+                        break;
+                    case LogEvent.UPDATE_ROWS_EVENT_V1:
+                    case LogEvent.UPDATE_ROWS_EVENT:
+                        parseRowsEvent((UpdateRowsLogEvent) event);
+                        break;
+                    case LogEvent.DELETE_ROWS_EVENT_V1:
+                    case LogEvent.DELETE_ROWS_EVENT:
+                        parseRowsEvent((DeleteRowsLogEvent) event);
+                        break;
+                    case LogEvent.QUERY_EVENT:
+                        parseQueryEvent((QueryLogEvent) event);
+                        break;
+                    case LogEvent.ROWS_QUERY_LOG_EVENT:
+                        parseRowsQueryEvent((RowsQueryLogEvent) event);
+                        break;
+                    case LogEvent.ANNOTATE_ROWS_EVENT:
+                        parseAnnotateRowsEvent((AnnotateRowsEvent) event);
+                        break;
+                    case LogEvent.XID_EVENT:
+                        parseXidEvent((XidLogEvent) event);
+                        break;
+                    default:
+                        break;
+                }
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+            Assert.fail(e.getMessage());
+        } finally {
+            try {
+                fecther.close();
+            } catch (IOException e) {
+                Assert.fail(e.getMessage());
+            }
+        }
+
+    }
+}

dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/FileLogFetcherTest.java

@@ -0,0 +1,93 @@
+package com.taobao.tddl.dbsync.binlog;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+
+import junit.framework.Assert;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import com.taobao.tddl.dbsync.binlog.event.DeleteRowsLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.QueryLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.RotateLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.RowsQueryLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.UpdateRowsLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.WriteRowsLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.XidLogEvent;
+import com.taobao.tddl.dbsync.binlog.event.mariadb.AnnotateRowsEvent;
+
+public class FileLogFetcherTest extends BaseLogFetcherTest {
+
+    private String directory;
+
+    @Before
+    public void setUp() {
+        URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt");
+        File dummyFile = new File(url.getFile());
+        directory = new File(dummyFile.getParent() + "/binlog").getPath();
+        // directory = "/home/jianghang/tmp/binlog";
+    }
+
+    @Test
+    public void testSimple() {
+        FileLogFetcher fetcher = new FileLogFetcher(1024 * 16);
+        try {
+            LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
+            LogContext context = new LogContext();
+
+            File current = new File(directory, "mysql-bin.000006");
+            fetcher.open(current);
+            context.setLogPosition(new LogPosition(current.getName()));
+
+            while (fetcher.fetch()) {
+                LogEvent event = null;
+                event = decoder.decode(fetcher, context);
+                if (event != null) {
+                    int eventType = event.getHeader().getType();
+                    switch (eventType) {
+                        case LogEvent.ROTATE_EVENT:
+                             binlogFileName = ((RotateLogEvent)
+                             event).getFilename();
+                            break;
+                        case LogEvent.WRITE_ROWS_EVENT_V1:
+                        case LogEvent.WRITE_ROWS_EVENT:
+                             parseRowsEvent((WriteRowsLogEvent) event);
+                            break;
+                        case LogEvent.UPDATE_ROWS_EVENT_V1:
+                        case LogEvent.UPDATE_ROWS_EVENT:
+                             parseRowsEvent((UpdateRowsLogEvent) event);
+                            break;
+                        case LogEvent.DELETE_ROWS_EVENT_V1:
+                        case LogEvent.DELETE_ROWS_EVENT:
+                             parseRowsEvent((DeleteRowsLogEvent) event);
+                            break;
+                        case LogEvent.QUERY_EVENT:
+                             parseQueryEvent((QueryLogEvent) event);
+                            break;
+                        case LogEvent.ROWS_QUERY_LOG_EVENT:
+                            parseRowsQueryEvent((RowsQueryLogEvent) event);
+                            break;
+                        case LogEvent.ANNOTATE_ROWS_EVENT:
+                            parseAnnotateRowsEvent((AnnotateRowsEvent) event);
+                            break;
+                        case LogEvent.XID_EVENT:
+                            parseXidEvent((XidLogEvent) event);
+                            break;
+                        default:
+                            break;
+                    }
+                }
+            }
+        } catch (Exception e) {
+            Assert.fail(e.getMessage());
+        } finally {
+            try {
+                fetcher.close();
+            } catch (IOException e) {
+                Assert.fail(e.getMessage());
+            }
+        }
+    }
+}

dbsync/src/test/java/com/taobao/tddl/dbsync/binlog/LogBufferTest.java

@@ -0,0 +1,356 @@
+package com.taobao.tddl.dbsync.binlog;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+
+import junit.framework.TestCase;
+
+import com.taobao.tddl.dbsync.binlog.LogBuffer;
+
+public class LogBufferTest extends TestCase
+{
+    public static final int LOOP = 10000;
+
+    public void testSigned()
+    {
+        byte[] array = { 0, 0, 0, (byte) 0xff };
+
+        LogBuffer buffer = new LogBuffer(array, 0, array.length);
+
+        System.out.println(buffer.getInt32(0));
+        System.out.println(buffer.getUint32(0));
+
+        System.out.println(buffer.getInt24(1));
+        System.out.println(buffer.getUint24(1));
+    }
+
+    public void testBigInteger()
+    {
+        byte[] array = { (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff,
+                (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff };
+
+        LogBuffer buffer = new LogBuffer(array, 0, array.length);
+
+        long tt1 = 0;
+        long l1 = 0;
+        for (int i = 0; i < LOOP; i++)
+        {
+            final long t1 = System.nanoTime();
+            l1 = buffer.getLong64(0);
+            tt1 += System.nanoTime() - t1;
+        }
+        System.out.print(tt1 / LOOP);
+        System.out.print("ns >> ");
+        System.out.println(l1);
+
+        long tt2 = 0;
+        BigInteger l2 = null;
+        for (int i = 0; i < LOOP; i++)
+        {
+            final long t2 = System.nanoTime();
+            l2 = buffer.getUlong64(0);
+            tt2 += System.nanoTime() - t2;
+        }
+        System.out.print(tt2 / LOOP);
+        System.out.print("ns >> ");
+        System.out.println(l2);
+    }
+
+    /* Reads big-endian integer from no more than 4 bytes */
+    private static int convertNBytesToInt(byte[] buffer, int offset, int length)
+    {
+        int ret = 0;
+        for (int i = offset; i < (offset + length); i++)
+        {
+            ret = (ret << 8) | (0xff & buffer[i]);
+        }
+        return ret;
+    }
+
+    private static int convert4BytesToInt(byte[] buffer, int offset)
+    {
+        int value;
+        value = (0xff & buffer[offset + 3]);
+        value += (0xff & buffer[offset + 2]) << 8;
+        value += (0xff & buffer[offset + 1]) << 16;
+        value += (0xff & buffer[offset]) << 24;
+        return value;
+    }
+
+    public static short convert1ByteToShort(byte[] buffer, int offset)
+    {
+        short value;
+        value = (short) buffer[offset + 0];
+        return value;
+    }
+
+    public static short convert2bytesToShort(byte[] buffer, int offset)
+    {
+        short value;
+        value = (short) (buffer[offset + 0] << 8);
+        value += (short) (buffer[offset + 1] & 0xff);
+        return value;
+    }
+
+    public static final BigDecimal extractDecimal(byte[] buffer, int precision,
+            int scale)
+    {
+        //
+        // Decimal representation in binlog seems to be as follows:
+        // 1 byte - 'precision'
+        // 1 byte - 'scale'
+        // remaining n bytes - integer such that value = n / (10^scale)
+        // Integer is represented as follows:
+        // 1st bit - sign such that set == +, unset == -
+        // every 4 bytes represent 9 digits in big-endian order, so that if
+        // you print the values of these quads as big-endian integers one after
+        // another, you get the whole number string representation in decimal.
+        // What remains is to put a sign and a decimal dot.
+        // 13 0a 80 00 00 05 1b 38 b0 60 00 means:
+        // 0x13 - precision = 19
+        // 0x0a - scale = 10
+        // 0x80 - positive
+        // 0x00000005 0x1b38b060 0x00
+        // 5 456700000 0
+        // 54567000000 / 10^{10} = 5.4567
+        // 
+        // int_size below shows how long is integer part
+        //
+        // offset = offset + 2; // offset of the number part
+        //
+        int intg = precision - scale;
+        int intg0 = intg / LogBuffer.DIG_PER_INT32;
+        int frac0 = scale / LogBuffer.DIG_PER_INT32;
+        int intg0x = intg - intg0 * LogBuffer.DIG_PER_INT32;
+        int frac0x = scale - frac0 * LogBuffer.DIG_PER_INT32;
+
+        int offset = 0;
+
+        int sign = (buffer[offset] & 0x80) == 0x80 ? 1 : -1;
+
+        // how many bytes are used to represent given amount of digits?
+        int integerSize = intg0 * LogBuffer.SIZE_OF_INT32
+                + LogBuffer.dig2bytes[intg0x];
+        int decimalSize = frac0 * LogBuffer.SIZE_OF_INT32
+                + LogBuffer.dig2bytes[frac0x];
+
+        int bin_size = integerSize + decimalSize; // total bytes
+        byte[] d_copy = new byte[bin_size];
+
+        if (bin_size > buffer.length)
+        {
+            throw new ArrayIndexOutOfBoundsException("Calculated bin_size: "
+                    + bin_size + ", available bytes: " + buffer.length);
+        }
+
+        // Invert first bit
+        d_copy[0] = buffer[0];
+        d_copy[0] ^= 0x80;
+        if (sign == -1)
+        {
+            // Invert every byte
+            d_copy[0] ^= 0xFF;
+        }
+
+        for (int i = 1; i < bin_size; i++)
+        {
+            d_copy[i] = buffer[i];
+            if (sign == -1)
+            {
+                // Invert every byte
+                d_copy[i] ^= 0xFF;
+            }
+        }
+
+        // Integer part
+        offset = LogBuffer.dig2bytes[intg0x];
+
+        BigDecimal intPart = new BigDecimal(0);
+
+        if (offset > 0)
+            intPart = BigDecimal.valueOf(convertNBytesToInt(d_copy, 0, offset));
+
+        while (offset < integerSize)
+        {
+            intPart = intPart.movePointRight(LogBuffer.DIG_PER_DEC1).add(
+                    BigDecimal.valueOf(convert4BytesToInt(d_copy, offset)));
+            offset += 4;
+        }
+
+        // Decimal part
+        BigDecimal fracPart = new BigDecimal(0);
+        int shift = 0;
+        for (int i = 0; i < frac0; i++)
+        {
+            shift += LogBuffer.DIG_PER_DEC1;
+            fracPart = fracPart.add(BigDecimal.valueOf(
+                    convert4BytesToInt(d_copy, offset)).movePointLeft(shift));
+            offset += 4;
+        }
+
+        if (LogBuffer.dig2bytes[frac0x] > 0)
+        {
+            fracPart = fracPart.add(BigDecimal.valueOf(
+                    convertNBytesToInt(d_copy, offset,
+                            LogBuffer.dig2bytes[frac0x])).movePointLeft(
+                    shift + frac0x));
+        }
+
+        return BigDecimal.valueOf(sign).multiply(intPart.add(fracPart));
+    }
+
+    public static final byte[] array1 = { (byte) 0x80, 0x00, 0x00, 0x05, 0x1b,
+            0x38, (byte) 0xb0, 0x60, 0x00 };
+
+    public static final byte[] array2 = { (byte) 0x7f, (byte) 0xff,
+            (byte) 0xff, (byte) 0xfb, (byte) 0xe4, (byte) 0xc7, (byte) 0x4f,
+            (byte) 0xa0, (byte) 0xff };
+
+    public static final byte[] array3 = { -128, 0, 6, 20, 113, 56, 6, 26, -123 };
+
+    public static final byte[] array4 = { -128, 7, 0, 0, 0, 1, 0, 0, 3 };
+
+    public static final byte[] array5 = { -128, 0, 0, 0, 0, 1, 1, -122, -96,
+            -108                     };
+
+    public void testBigDecimal() throws InterruptedException
+    {
+        do
+        {
+            System.out.println("old extract decimal: ");
+
+            long tt1 = 0;
+            BigDecimal bd1 = null;
+            for (int i = 0; i < LOOP; i++)
+            {
+                final long t1 = System.nanoTime();
+                bd1 = extractDecimal(array2, 19, 10);
+                tt1 += System.nanoTime() - t1;
+            }
+            System.out.print(tt1 / LOOP);
+            System.out.print("ns >> ");
+            System.out.println(bd1);
+
+            long tt2 = 0;
+            BigDecimal bd2 = null;
+            for (int i = 0; i < LOOP; i++)
+            {
+                final long t2 = System.nanoTime();
+                bd2 = extractDecimal(array1, 19, 10);
+                tt2 += System.nanoTime() - t2;
+            }
+            System.out.print(tt2 / LOOP);
+            System.out.print("ns >> ");
+            System.out.println(bd2);
+
+            long tt3 = 0;
+            BigDecimal bd3 = null;
+            for (int i = 0; i < LOOP; i++)
+            {
+                final long t3 = System.nanoTime();
+                bd3 = extractDecimal(array3, 18, 6);
+                tt3 += System.nanoTime() - t3;
+            }
+            System.out.print(tt3 / LOOP);
+            System.out.print("ns >> ");
+            System.out.println(bd3);
+
+            long tt4 = 0;
+            BigDecimal bd4 = null;
+            for (int i = 0; i < LOOP; i++)
+            {
+                final long t4 = System.nanoTime();
+                bd4 = extractDecimal(array4, 18, 6);
+                tt4 += System.nanoTime() - t4;
+            }
+            System.out.print(tt4 / LOOP);
+            System.out.print("ns >> ");
+            System.out.println(bd4);
+
+            long tt5 = 0;
+            BigDecimal bd5 = null;
+            for (int i = 0; i < LOOP; i++)
+            {
+                final long t5 = System.nanoTime();
+                bd5 = extractDecimal(array5, 18, 6);
+                tt5 += System.nanoTime() - t5;
+            }
+            System.out.print(tt5 / LOOP);
+            System.out.print("ns >> ");
+            System.out.println(bd5);
+        }
+        while (false);
+
+        do
+        {
+            System.out.println("new extract decimal: ");
+
+            LogBuffer buffer1 = new LogBuffer(array2, 0, array2.length);
+            LogBuffer buffer2 = new LogBuffer(array1, 0, array1.length);
+            LogBuffer buffer3 = new LogBuffer(array3, 0, array3.length);
+            LogBuffer buffer4 = new LogBuffer(array4, 0, array4.length);
+            LogBuffer buffer5 = new LogBuffer(array5, 0, array5.length);
+
+            long tt1 = 0;
+            BigDecimal bd1 = null;
+            for (int i = 0; i < LOOP; i++)
+            {
+                final long t1 = System.nanoTime();
+                bd1 = buffer1.getDecimal(0, 19, 10);
+                tt1 += System.nanoTime() - t1;
+            }
+            System.out.print(tt1 / LOOP);
+            System.out.print("ns >> ");
+            System.out.println(bd1);
+
+            long tt2 = 0;
+            BigDecimal bd2 = null;
+            for (int i = 0; i < LOOP; i++)
+            {
+                final long t2 = System.nanoTime();
+                bd2 = buffer2.getDecimal(0, 19, 10);
+                tt2 += System.nanoTime() - t2;
+            }
+            System.out.print(tt2 / LOOP);
+            System.out.print("ns >> ");
+            System.out.println(bd2);
+
+            long tt3 = 0;
+            BigDecimal bd3 = null;
+            for (int i = 0; i < LOOP; i++)
+            {
+                final long t3 = System.nanoTime();
+                bd3 = buffer3.getDecimal(0, 18, 6);
+                tt3 += System.nanoTime() - t3;
+            }
+            System.out.print(tt3 / LOOP);
+            System.out.print("ns >> ");
+            System.out.println(bd3);
+
+            long tt4 = 0;
+            BigDecimal bd4 = null;
+            for (int i = 0; i < LOOP; i++)
+            {
+                final long t4 = System.nanoTime();
+                bd4 = buffer4.getDecimal(0, 18, 6);
+                tt4 += System.nanoTime() - t4;
+            }
+            System.out.print(tt4 / LOOP);
+            System.out.print("ns >> ");
+            System.out.println(bd4);
+
+            long tt5 = 0;
+            BigDecimal bd5 = null;
+            for (int i = 0; i < LOOP; i++)
+            {
+                final long t5 = System.nanoTime();
+                bd5 = buffer5.getDecimal(0, 18, 6);
+                tt5 += System.nanoTime() - t5;
+            }
+            System.out.print(tt5 / LOOP);
+            System.out.print("ns >> ");
+            System.out.println(bd5);
+        }
+        while (false);
+    }
+}

dbsync/src/test/resources/dummy.txt

@@ -0,0 +1 @@
+本文件仅仅为定位绝对路径使用

deployer/pom.xml

@@ -0,0 +1,118 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<parent>
+		<groupId>com.alibaba.otter</groupId>
+		<artifactId>canal</artifactId>
+		<version>1.0.19-SNAPSHOT</version>
+		<relativePath>../pom.xml</relativePath>
+	</parent>
+	<groupId>com.alibaba.otter</groupId>
+	<artifactId>canal.deployer</artifactId>
+	<packaging>jar</packaging>
+	<name>canal deployer module for otter ${project.version}</name>
+	<dependencies>
+		<dependency>
+			<groupId>com.alibaba.otter</groupId>
+			<artifactId>canal.server</artifactId>
+			<version>${project.version}</version>
+		</dependency>
+	</dependencies>
+	
+	<build>
+		<plugins>
+			<!-- deploy模块的packaging通常是jar，如果项目中没有java 源代码或资源文件，加上这一段配置使项目能通过构建 -->
+			<plugin>
+				<artifactId>maven-jar-plugin</artifactId>
+				<configuration>
+					<archive>
+						<addMavenDescriptor>true</addMavenDescriptor>
+					</archive>
+					<excludes>
+						<exclude>**/logback.xml</exclude>
+						<exclude>**/canal.properties</exclude>
+						<exclude>**/spring/**</exclude>
+						<exclude>**/example/**</exclude>
+					</excludes>
+				</configuration>
+			</plugin>
+
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-assembly-plugin</artifactId>
+				<!-- 这是最新版本，推荐使用这个版本 -->
+				<version>2.2.1</version>
+				<executions>
+					<execution>
+						<id>assemble</id>
+						<goals>
+							<goal>single</goal>
+						</goals>
+						<phase>package</phase>
+					</execution>
+				</executions>
+				<configuration>
+					<appendAssemblyId>false</appendAssemblyId>
+					<attach>false</attach>
+				</configuration>
+			</plugin>
+		</plugins>
+	</build>
+
+	<profiles>
+		<profile>
+			<id>dev</id>
+			<activation>
+				<activeByDefault>true</activeByDefault>
+				<property>
+					<name>env</name>
+					<value>!release</value>
+				</property>
+			</activation>
+
+			<build>
+				<plugins>
+					<plugin>
+						<artifactId>maven-assembly-plugin</artifactId>
+						<configuration>
+							<!-- maven assembly插件需要一个描述文件 来告诉插件包的结构以及打包所需的文件来自哪里 -->
+							<descriptors>
+								<descriptor>${basedir}/src/main/assembly/dev.xml</descriptor>
+							</descriptors>
+							<finalName>canal</finalName>
+							<outputDirectory>${project.build.directory}</outputDirectory>
+						</configuration>
+					</plugin>
+				</plugins>
+			</build>
+
+		</profile>
+
+		<profile>
+			<id>release</id>
+			<activation>
+				<property>
+					<name>env</name>
+					<value>release</value>
+				</property>
+			</activation>
+
+			<build>
+				<plugins>
+					<plugin>
+						<artifactId>maven-assembly-plugin</artifactId>
+						<configuration>
+							<!-- 发布模式使用的maven assembly插件描述文件 -->
+							<descriptors>
+								<descriptor>${basedir}/src/main/assembly/release.xml</descriptor>
+							</descriptors>
+							<!-- 如果一个应用的包含多个deploy模块，如果使用同样的包名， 如果把它们复制的一个目录中可能会失败，所以包名加了 artifactId以示区分 -->
+							<finalName>${project.artifactId}-${project.version}</finalName>
+							<!-- scm 要求 release 模式打出的包放到顶级目录下的target子目录中 -->
+							<outputDirectory>${project.parent.build.directory}</outputDirectory>
+						</configuration>
+					</plugin>
+				</plugins>
+			</build>
+		</profile>
+	</profiles>
+</project>

deployer/src/main/assembly/dev.xml

@@ -0,0 +1,54 @@
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+	<id>dist</id>
+	<formats>
+		<format>dir</format>
+	</formats>
+	<includeBaseDirectory>false</includeBaseDirectory>
+	<fileSets>
+		<fileSet>
+			<directory>.</directory>
+			<outputDirectory>/</outputDirectory>
+			<includes>
+				<include>README*</include>
+			</includes>
+		</fileSet>
+		<fileSet>
+			<directory>./src/main/bin</directory>
+			<outputDirectory>bin</outputDirectory>
+			<includes>
+				<include>**/*</include>
+			</includes>
+			<fileMode>0755</fileMode>
+		</fileSet>
+		<fileSet>
+			<directory>./src/main/conf</directory>
+			<outputDirectory>/conf</outputDirectory>
+			<includes>
+				<include>**/*</include>
+			</includes>
+		</fileSet>
+		<fileSet>
+			<directory>./src/main/resources</directory>
+			<outputDirectory>/conf</outputDirectory>
+			<includes>
+				<include>**/*</include>
+			</includes>
+		</fileSet>
+		<fileSet>
+			<directory>target</directory>
+			<outputDirectory>logs</outputDirectory>
+			<excludes>
+				<exclude>**/*</exclude>
+			</excludes>
+		</fileSet>
+	</fileSets>
+	<dependencySets>
+		<dependencySet>
+			<outputDirectory>lib</outputDirectory>
+			<excludes>
+				<exclude>junit:junit</exclude>
+			</excludes>
+		</dependencySet>
+	</dependencySets>
+</assembly>

deployer/src/main/assembly/release.xml

@@ -0,0 +1,54 @@
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+	<id>dist</id>
+	<formats>
+		<format>tar.gz</format>
+	</formats>
+	<includeBaseDirectory>false</includeBaseDirectory>
+	<fileSets>
+		<fileSet>
+			<directory>.</directory>
+			<outputDirectory>/</outputDirectory>
+			<includes>
+				<include>README*</include>
+			</includes>
+		</fileSet>
+		<fileSet>
+			<directory>./src/main/bin</directory>
+			<outputDirectory>bin</outputDirectory>
+			<includes>
+				<include>**/*</include>
+			</includes>
+			<fileMode>0755</fileMode>
+		</fileSet>
+		<fileSet>
+			<directory>./src/main/conf</directory>
+			<outputDirectory>/conf</outputDirectory>
+			<includes>
+				<include>**/*</include>
+			</includes>
+		</fileSet>
+		<fileSet>
+			<directory>./src/main/resources</directory>
+			<outputDirectory>/conf</outputDirectory>
+			<includes>
+				<include>**/*</include>
+			</includes>
+		</fileSet>
+		<fileSet>
+			<directory>target</directory>
+			<outputDirectory>logs</outputDirectory>
+			<excludes>
+				<exclude>**/*</exclude>
+			</excludes>
+		</fileSet>
+	</fileSets>
+	<dependencySets>
+		<dependencySet>
+			<outputDirectory>lib</outputDirectory>
+			<excludes>
+				<exclude>junit:junit</exclude>
+			</excludes>
+		</dependencySet>
+	</dependencySets>
+</assembly>

deployer/src/main/bin/startup.bat

@@ -0,0 +1,25 @@
+@echo off
+@if not "%ECHO%" == ""  echo %ECHO%
+@if "%OS%" == "Windows_NT"  setlocal
+
+set ENV_PATH=.\
+if "%OS%" == "Windows_NT" set ENV_PATH=%~dp0%
+
+set conf_dir=%ENV_PATH%\..\conf
+set canal_conf=%conf_dir%\canal.properties
+set logback_configurationFile=%conf_dir%\logback.xml
+
+set CLASSPATH=%conf_dir%
+set CLASSPATH=%conf_dir%\..\lib\*;%CLASSPATH%
+
+set JAVA_MEM_OPTS= -Xms128m -Xmx512m -XX:PermSize=128m
+set JAVA_OPTS_EXT= -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dapplication.codeset=UTF-8 -Dfile.encoding=UTF-8
+set JAVA_DEBUG_OPT= -server -Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=9099,server=y,suspend=n
+set CANAL_OPTS= -DappName=otter-canal -Dlogback.configurationFile="%logback_configurationFile%" -Dcanal.conf="%canal_conf%"
+
+set JAVA_OPTS= %JAVA_MEM_OPTS% %JAVA_OPTS_EXT% %JAVA_DEBUG_OPT% %CANAL_OPTS%
+
+set CMD_STR= java %JAVA_OPTS% -classpath "%CLASSPATH%" java %JAVA_OPTS% -classpath "%CLASSPATH%" com.alibaba.otter.canal.deployer.CanalLauncher
+echo start cmd : %CMD_STR%
+
+java %JAVA_OPTS% -classpath "%CLASSPATH%" com.alibaba.otter.canal.deployer.CanalLauncher

deployer/src/main/bin/startup.sh

@@ -0,0 +1,104 @@
+#!/bin/bash 
+
+current_path=`pwd`
+case "`uname`" in
+    Linux)
+		bin_abs_path=$(readlink -f $(dirname $0))
+		;;
+	*)
+		bin_abs_path=`cd $(dirname $0); pwd`
+		;;
+esac
+base=${bin_abs_path}/..
+canal_conf=$base/conf/canal.properties
+logback_configurationFile=$base/conf/logback.xml
+export LANG=en_US.UTF-8
+export BASE=$base
+
+if [ -f $base/bin/canal.pid ] ; then
+	echo "found canal.pid , Please run stop.sh first ,then startup.sh" 2>&2
+    exit 1
+fi
+
+if [ ! -d $base/logs/canal ] ; then 
+	mkdir -p $base/logs/canal
+fi
+
+## set java path
+if [ -z "$JAVA" ] ; then
+  JAVA=$(which java)
+fi
+
+ALIBABA_JAVA="/usr/alibaba/java/bin/java"
+TAOBAO_JAVA="/opt/taobao/java/bin/java"
+if [ -z "$JAVA" ]; then
+  if [ -f $ALIBABA_JAVA ] ; then
+  	JAVA=$ALIBABA_JAVA
+  elif [ -f $TAOBAO_JAVA ] ; then
+  	JAVA=$TAOBAO_JAVA
+  else
+  	echo "Cannot find a Java JDK. Please set either set JAVA or put java (>=1.5) in your PATH." 2>&2
+    exit 1
+  fi
+fi
+
+case "$#" 
+in
+0 ) 
+	;;
+1 )	
+	var=$*
+	if [ -f $var ] ; then 
+		canal_conf=$var
+	else
+		echo "THE PARAMETER IS NOT CORRECT.PLEASE CHECK AGAIN."
+        exit
+	fi;;
+2 )	
+	var=$1
+	if [ -f $var ] ; then
+		canal_conf=$var
+	else 
+		if [ "$1" = "debug" ]; then
+			DEBUG_PORT=$2
+			DEBUG_SUSPEND="n"
+			JAVA_DEBUG_OPT="-Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=$DEBUG_PORT,server=y,suspend=$DEBUG_SUSPEND"
+		fi
+     fi;;
+* )
+	echo "THE PARAMETERS MUST BE TWO OR LESS.PLEASE CHECK AGAIN."
+	exit;;
+esac
+
+str=`file -L $JAVA | grep 64-bit`
+if [ -n "$str" ]; then
+	JAVA_OPTS="-server -Xms2048m -Xmx3072m -Xmn1024m -XX:SurvivorRatio=2 -XX:PermSize=96m -XX:MaxPermSize=256m -Xss256k -XX:-UseAdaptiveSizePolicy -XX:MaxTenuringThreshold=15 -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+UseCMSCompactAtFullCollection -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:+HeapDumpOnOutOfMemoryError"
+else
+	JAVA_OPTS="-server -Xms1024m -Xmx1024m -XX:NewSize=256m -XX:MaxNewSize=256m -XX:MaxPermSize=128m "
+fi
+
+JAVA_OPTS=" $JAVA_OPTS -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dfile.encoding=UTF-8"
+CANAL_OPTS="-DappName=otter-canal -Dlogback.configurationFile=$logback_configurationFile -Dcanal.conf=$canal_conf"
+
+if [ -e $canal_conf -a -e $logback_configurationFile ]
+then 
+	
+	for i in $base/lib/*;
+		do CLASSPATH=$i:"$CLASSPATH";
+	done
+ 	CLASSPATH="$base/conf:$CLASSPATH";
+ 	
+ 	echo "cd to $bin_abs_path for workaround relative path"
+  	cd $bin_abs_path
+ 	
+	echo LOG CONFIGURATION : $logback_configurationFile
+	echo canal conf : $canal_conf 
+	echo CLASSPATH :$CLASSPATH
+	$JAVA $JAVA_OPTS $JAVA_DEBUG_OPT $CANAL_OPTS -classpath .:$CLASSPATH com.alibaba.otter.canal.deployer.CanalLauncher 1>>$base/logs/canal/canal.log 2>&1 &
+	echo $! > $base/bin/canal.pid 
+	
+	echo "cd to $current_path for continue"
+  	cd $current_path
+else 
+	echo "canal conf("$canal_conf") OR log configration file($logback_configurationFile) is not exist,please create then first!"
+fi

deployer/src/main/bin/stop.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+cygwin=false;
+case "`uname`" in
+    CYGWIN*)
+        cygwin=true
+        ;;
+esac
+
+get_pid() {	
+	STR=$1
+	PID=$2
+    if $cygwin; then
+        JAVA_CMD="$JAVA_HOME\bin\java"
+        JAVA_CMD=`cygpath --path --unix $JAVA_CMD`
+        JAVA_PID=`ps |grep $JAVA_CMD |awk '{print $1}'`
+    else
+        if [ ! -z "$PID" ]; then
+        	JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep "$PID"|grep -v grep|awk '{print $2}'`
+	    else 
+	        JAVA_PID=`ps -C java -f --width 1000|grep "$STR"|grep -v grep|awk '{print $2}'`
+        fi
+    fi
+    echo $JAVA_PID;
+}
+
+base=`dirname $0`/..
+pidfile=$base/bin/canal.pid
+if [ ! -f "$pidfile" ];then
+	echo "canal is not running. exists"
+	exit
+fi
+
+pid=`cat $pidfile`
+if [ "$pid" == "" ] ; then
+	pid=`get_pid "appName=otter-canal"`
+fi
+
+echo -e "`hostname`: stopping canal $pid ... "
+kill $pid
+
+LOOPS=0
+while (true); 
+do 
+	gpid=`get_pid "appName=otter-canal" "$pid"`
+    if [ "$gpid" == "" ] ; then
+    	echo "Oook! cost:$LOOPS"
+    	`rm $pidfile`
+    	break;
+    fi
+    let LOOPS=LOOPS+1
+    sleep 1
+done

deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalConstants.java

@@ -0,0 +1,50 @@
+package com.alibaba.otter.canal.deployer;
+
+import java.text.MessageFormat;
+
+/**
+ * 启动常用变量
+ * 
+ * @author jianghang 2012-11-8 下午03:15:55
+ * @version 1.0.0
+ */
+public class CanalConstants {
+
+    public static final String MDC_DESTINATION                   = "destination";
+    public static final String ROOT                              = "canal";
+    public static final String CANAL_ID                          = ROOT + "." + "id";
+    public static final String CANAL_IP                          = ROOT + "." + "ip";
+    public static final String CANAL_PORT                        = ROOT + "." + "port";
+    public static final String CANAL_ZKSERVERS                   = ROOT + "." + "zkServers";
+
+    public static final String CANAL_DESTINATIONS                = ROOT + "." + "destinations";
+    public static final String CANAL_AUTO_SCAN                   = ROOT + "." + "auto.scan";
+    public static final String CANAL_AUTO_SCAN_INTERVAL          = ROOT + "." + "auto.scan.interval";
+    public static final String CANAL_CONF_DIR                    = ROOT + "." + "conf.dir";
+
+    public static final String CANAL_DESTINATION_SPLIT           = ",";
+    public static final String GLOBAL_NAME                       = "global";
+
+    public static final String INSTANCE_MODE_TEMPLATE            = ROOT + "." + "instance.{0}.mode";
+    public static final String INSTANCE_LAZY_TEMPLATE            = ROOT + "." + "instance.{0}.lazy";
+    public static final String INSTANCE_MANAGER_ADDRESS_TEMPLATE = ROOT + "." + "instance.{0}.manager.address";
+    public static final String INSTANCE_SPRING_XML_TEMPLATE      = ROOT + "." + "instance.{0}.spring.xml";
+
+    public static final String CANAL_DESTINATION_PROPERTY        = ROOT + ".instance.destination";
+
+    public static String getInstanceModeKey(String destination) {
+        return MessageFormat.format(INSTANCE_MODE_TEMPLATE, destination);
+    }
+
+    public static String getInstanceManagerAddressKey(String destination) {
+        return MessageFormat.format(INSTANCE_MANAGER_ADDRESS_TEMPLATE, destination);
+    }
+
+    public static String getInstancSpringXmlKey(String destination) {
+        return MessageFormat.format(INSTANCE_SPRING_XML_TEMPLATE, destination);
+    }
+
+    public static String getInstancLazyKey(String destination) {
+        return MessageFormat.format(INSTANCE_LAZY_TEMPLATE, destination);
+    }
+}

deployer/src/main/java/com/alibaba/otter/canal/deployer/CanalController.java

@@ -0,0 +1,452 @@
+package com.alibaba.otter.canal.deployer;
+
+import java.util.Map;
+import java.util.Properties;
+
+import org.I0Itec.zkclient.IZkStateListener;
+import org.I0Itec.zkclient.exception.ZkNoNodeException;
+import org.apache.commons.lang.BooleanUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.zookeeper.Watcher.Event.KeeperState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
+import org.springframework.beans.factory.BeanFactory;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.support.ClassPathXmlApplicationContext;
+
+import com.alibaba.otter.canal.common.utils.AddressUtils;
+import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
+import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
+import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningData;
+import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningListener;
+import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningMonitor;
+import com.alibaba.otter.canal.common.zookeeper.running.ServerRunningMonitors;
+import com.alibaba.otter.canal.deployer.InstanceConfig.InstanceMode;
+import com.alibaba.otter.canal.deployer.monitor.InstanceAction;
+import com.alibaba.otter.canal.deployer.monitor.InstanceConfigMonitor;
+import com.alibaba.otter.canal.deployer.monitor.ManagerInstanceConfigMonitor;
+import com.alibaba.otter.canal.deployer.monitor.SpringInstanceConfigMonitor;
+import com.alibaba.otter.canal.instance.core.CanalInstance;
+import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator;
+import com.alibaba.otter.canal.instance.manager.CanalConfigClient;
+import com.alibaba.otter.canal.instance.manager.ManagerCanalInstanceGenerator;
+import com.alibaba.otter.canal.instance.spring.SpringCanalInstanceGenerator;
+import com.alibaba.otter.canal.server.embeded.CanalServerWithEmbeded;
+import com.alibaba.otter.canal.server.exception.CanalServerException;
+import com.alibaba.otter.canal.server.netty.CanalServerWithNetty;
+import com.google.common.base.Function;
+import com.google.common.collect.MapMaker;
+
+/**
+ * canal调度控制器
+ * 
+ * @author jianghang 2012-11-8 下午12:03:11
+ * @version 1.0.0
+ */
+public class CanalController {
+
+    private static final Logger                      logger   = LoggerFactory.getLogger(CanalController.class);
+    private Long                                     cid;
+    private String                                   ip;
+    private int                                      port;
+    // 默认使用spring的方式载入
+    private Map<String, InstanceConfig>              instanceConfigs;
+    private InstanceConfig                           globalInstanceConfig;
+    private Map<String, CanalConfigClient>           managerClients;
+    // 监听instance config的变化
+    private boolean                                  autoScan = true;
+    private InstanceAction                           defaultAction;
+    private Map<InstanceMode, InstanceConfigMonitor> instanceConfigMonitors;
+    private CanalServerWithEmbeded                   embededCanalServer;
+    private CanalServerWithNetty                     canalServer;
+
+    private CanalInstanceGenerator                   instanceGenerator;
+    private ZkClientx                                zkclientx;
+
+    public CanalController(){
+        this(System.getProperties());
+    }
+
+    public CanalController(final Properties properties){
+        managerClients = new MapMaker().makeComputingMap(new Function<String, CanalConfigClient>() {
+
+            public CanalConfigClient apply(String managerAddress) {
+                return getManagerClient(managerAddress);
+            }
+        });
+
+        // 初始化全局参数设置
+        globalInstanceConfig = initGlobalConfig(properties);
+        instanceConfigs = new MapMaker().makeMap();
+        // 初始化instance config
+        initInstanceConfig(properties);
+
+        // 准备canal server
+        cid = Long.valueOf(getProperty(properties, CanalConstants.CANAL_ID));
+        ip = getProperty(properties, CanalConstants.CANAL_IP);
+        port = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_PORT));
+        embededCanalServer = new CanalServerWithEmbeded();
+        embededCanalServer.setCanalInstanceGenerator(instanceGenerator);// 设置自定义的instanceGenerator
+        canalServer = new CanalServerWithNetty(embededCanalServer);
+        canalServer.setIp(ip);
+        canalServer.setPort(port);
+
+        // 处理下ip为空，默认使用hostIp暴露到zk中
+        if (StringUtils.isEmpty(ip)) {
+            ip = AddressUtils.getHostIp();
+        }
+        final String zkServers = getProperty(properties, CanalConstants.CANAL_ZKSERVERS);
+        if (StringUtils.isNotEmpty(zkServers)) {
+            zkclientx = ZkClientx.getZkClient(zkServers);
+            // 初始化系统目录
+            zkclientx.createPersistent(ZookeeperPathUtils.DESTINATION_ROOT_NODE, true);
+            zkclientx.createPersistent(ZookeeperPathUtils.CANAL_CLUSTER_ROOT_NODE, true);
+        }
+
+        final ServerRunningData serverData = new ServerRunningData(cid, ip + ":" + port);
+        ServerRunningMonitors.setServerData(serverData);
+        ServerRunningMonitors.setRunningMonitors(new MapMaker().makeComputingMap(new Function<String, ServerRunningMonitor>() {
+
+            public ServerRunningMonitor apply(final String destination) {
+                ServerRunningMonitor runningMonitor = new ServerRunningMonitor(serverData);
+                runningMonitor.setDestination(destination);
+                runningMonitor.setListener(new ServerRunningListener() {
+
+                    public void processActiveEnter() {
+                        try {
+                            MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
+                            embededCanalServer.start(destination);
+                        } finally {
+                            MDC.remove(CanalConstants.MDC_DESTINATION);
+                        }
+                    }
+
+                    public void processActiveExit() {
+                        try {
+                            MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
+                            embededCanalServer.stop(destination);
+                        } finally {
+                            MDC.remove(CanalConstants.MDC_DESTINATION);
+                        }
+                    }
+
+                    public void processStart() {
+                        try {
+                            if (zkclientx != null) {
+                                final String path = ZookeeperPathUtils.getDestinationClusterNode(destination, ip + ":"
+                                                                                                              + port);
+                                initCid(path);
+                                zkclientx.subscribeStateChanges(new IZkStateListener() {
+
+                                    public void handleStateChanged(KeeperState state) throws Exception {
+
+                                    }
+
+                                    public void handleNewSession() throws Exception {
+                                        initCid(path);
+                                    }
+                                });
+                            }
+                        } finally {
+                            MDC.remove(CanalConstants.MDC_DESTINATION);
+                        }
+                    }
+
+                    public void processStop() {
+                        try {
+                            MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
+                            if (zkclientx != null) {
+                                final String path = ZookeeperPathUtils.getDestinationClusterNode(destination, ip + ":"
+                                                                                                              + port);
+                                releaseCid(path);
+                            }
+                        } finally {
+                            MDC.remove(CanalConstants.MDC_DESTINATION);
+                        }
+                    }
+
+                });
+                if (zkclientx != null) {
+                    runningMonitor.setZkClient(zkclientx);
+                }
+                return runningMonitor;
+            }
+        }));
+
+        // 初始化monitor机制
+        autoScan = BooleanUtils.toBoolean(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN));
+        if (autoScan) {
+            defaultAction = new InstanceAction() {
+
+                public void start(String destination) {
+                    InstanceConfig config = instanceConfigs.get(destination);
+                    if (config == null) {
+                        config = new InstanceConfig(globalInstanceConfig);
+                        instanceConfigs.put(destination, config);
+                    }
+
+                    if (!config.getLazy() && !embededCanalServer.isStart(destination)) {
+                        // HA机制启动
+                        ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(destination);
+                        if (!runningMonitor.isStart()) {
+                            runningMonitor.start();
+                        }
+                    }
+                }
+
+                public void stop(String destination) {
+                    // 此处的stop，代表强制退出，非HA机制，所以需要退出HA的monitor和配置信息
+                    InstanceConfig config = instanceConfigs.remove(destination);
+                    if (config != null) {
+                        embededCanalServer.stop(destination);
+                        ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(destination);
+                        if (runningMonitor.isStart()) {
+                            runningMonitor.stop();
+                        }
+                    }
+                }
+
+                public void reload(String destination) {
+                    // 目前任何配置变化，直接重启，简单处理
+                    stop(destination);
+                    start(destination);
+                }
+            };
+
+            instanceConfigMonitors = new MapMaker().makeComputingMap(new Function<InstanceMode, InstanceConfigMonitor>() {
+
+                public InstanceConfigMonitor apply(InstanceMode mode) {
+                    int scanInterval = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN_INTERVAL));
+
+                    if (mode.isSpring()) {
+                        SpringInstanceConfigMonitor monitor = new SpringInstanceConfigMonitor();
+                        monitor.setScanIntervalInSecond(scanInterval);
+                        monitor.setDefaultAction(defaultAction);
+                        // 设置conf目录，默认是user.dir + conf目录组成
+                        String rootDir = getProperty(properties, CanalConstants.CANAL_CONF_DIR);
+                        if (StringUtils.isEmpty(rootDir)) {
+                            rootDir = "../conf";
+                        }
+                        monitor.setRootConf(rootDir);
+                        return monitor;
+                    } else if (mode.isManager()) {
+                        return new ManagerInstanceConfigMonitor();
+                    } else {
+                        throw new UnsupportedOperationException("unknow mode :" + mode + " for monitor");
+                    }
+                }
+            });
+        }
+    }
+
+    private InstanceConfig initGlobalConfig(Properties properties) {
+        InstanceConfig globalConfig = new InstanceConfig();
+        String modeStr = getProperty(properties, CanalConstants.getInstanceModeKey(CanalConstants.GLOBAL_NAME));
+        if (StringUtils.isNotEmpty(modeStr)) {
+            globalConfig.setMode(InstanceMode.valueOf(StringUtils.upperCase(modeStr)));
+        }
+
+        String lazyStr = getProperty(properties, CanalConstants.getInstancLazyKey(CanalConstants.GLOBAL_NAME));
+        if (StringUtils.isNotEmpty(lazyStr)) {
+            globalConfig.setLazy(Boolean.valueOf(lazyStr));
+        }
+
+        String managerAddress = getProperty(properties,
+            CanalConstants.getInstanceManagerAddressKey(CanalConstants.GLOBAL_NAME));
+        if (StringUtils.isNotEmpty(managerAddress)) {
+            globalConfig.setManagerAddress(managerAddress);
+        }
+
+        String springXml = getProperty(properties, CanalConstants.getInstancSpringXmlKey(CanalConstants.GLOBAL_NAME));
+        if (StringUtils.isNotEmpty(springXml)) {
+            globalConfig.setSpringXml(springXml);
+        }
+
+        instanceGenerator = new CanalInstanceGenerator() {
+
+            public CanalInstance generate(String destination) {
+                InstanceConfig config = instanceConfigs.get(destination);
+                if (config == null) {
+                    throw new CanalServerException("can't find destination:{}");
+                }
+
+                if (config.getMode().isManager()) {
+                    ManagerCanalInstanceGenerator instanceGenerator = new ManagerCanalInstanceGenerator();
+                    instanceGenerator.setCanalConfigClient(managerClients.get(config.getManagerAddress()));
+                    return instanceGenerator.generate(destination);
+                } else if (config.getMode().isSpring()) {
+                    SpringCanalInstanceGenerator instanceGenerator = new SpringCanalInstanceGenerator();
+                    synchronized (this) {
+                        try {
+                            // 设置当前正在加载的通道，加载spring查找文件时会用到该变量
+                            System.setProperty(CanalConstants.CANAL_DESTINATION_PROPERTY, destination);
+                            instanceGenerator.setBeanFactory(getBeanFactory(config.getSpringXml()));
+                            return instanceGenerator.generate(destination);
+                        } finally {
+                            System.setProperty(CanalConstants.CANAL_DESTINATION_PROPERTY, "");
+                        }
+                    }
+                } else {
+                    throw new UnsupportedOperationException("unknow mode :" + config.getMode());
+                }
+
+            }
+
+        };
+
+        return globalConfig;
+    }
+
+    private CanalConfigClient getManagerClient(String managerAddress) {
+        return new CanalConfigClient();
+    }
+
+    private BeanFactory getBeanFactory(String springXml) {
+        ApplicationContext applicationContext = new ClassPathXmlApplicationContext(springXml);
+        return applicationContext;
+    }
+
+    private void initInstanceConfig(Properties properties) {
+        String destinationStr = getProperty(properties, CanalConstants.CANAL_DESTINATIONS);
+        String[] destinations = StringUtils.split(destinationStr, CanalConstants.CANAL_DESTINATION_SPLIT);
+
+        for (String destination : destinations) {
+            InstanceConfig config = parseInstanceConfig(properties, destination);
+            InstanceConfig oldConfig = instanceConfigs.put(destination, config);
+
+            if (oldConfig != null) {
+                logger.warn("destination:{} old config:{} has replace by new config:{}", new Object[] { destination,
+                        oldConfig, config });
+            }
+        }
+    }
+
+    private InstanceConfig parseInstanceConfig(Properties properties, String destination) {
+        InstanceConfig config = new InstanceConfig(globalInstanceConfig);
+        String modeStr = getProperty(properties, CanalConstants.getInstanceModeKey(destination));
+        if (!StringUtils.isEmpty(modeStr)) {
+            config.setMode(InstanceMode.valueOf(StringUtils.upperCase(modeStr)));
+        }
+
+        String lazyStr = getProperty(properties, CanalConstants.getInstancLazyKey(destination));
+        if (!StringUtils.isEmpty(lazyStr)) {
+            config.setLazy(Boolean.valueOf(lazyStr));
+        }
+
+        if (config.getMode().isManager()) {
+            String managerAddress = getProperty(properties, CanalConstants.getInstanceManagerAddressKey(destination));
+            if (StringUtils.isNotEmpty(managerAddress)) {
+                config.setManagerAddress(managerAddress);
+            }
+        } else if (config.getMode().isSpring()) {
+            String springXml = getProperty(properties, CanalConstants.getInstancSpringXmlKey(destination));
+            if (StringUtils.isNotEmpty(springXml)) {
+                config.setSpringXml(springXml);
+            }
+        }
+
+        return config;
+    }
+
+    private String getProperty(Properties properties, String key) {
+        return StringUtils.trim(properties.getProperty(StringUtils.trim(key)));
+    }
+
+    public void start() throws Throwable {
+        logger.info("## start the canal server[{}:{}]", ip, port);
+        // 创建整个canal的工作节点
+        final String path = ZookeeperPathUtils.getCanalClusterNode(ip + ":" + port);
+        initCid(path);
+        if (zkclientx != null) {
+            this.zkclientx.subscribeStateChanges(new IZkStateListener() {
+
+                public void handleStateChanged(KeeperState state) throws Exception {
+
+                }
+
+                public void handleNewSession() throws Exception {
+                    initCid(path);
+                }
+            });
+        }
+        // 优先启动embeded服务
+        embededCanalServer.start();
+        // 尝试启动一下非lazy状态的通道
+        for (Map.Entry<String, InstanceConfig> entry : instanceConfigs.entrySet()) {
+            final String destination = entry.getKey();
+            InstanceConfig config = entry.getValue();
+            // 创建destination的工作节点
+            if (!config.getLazy() && !embededCanalServer.isStart(destination)) {
+                // HA机制启动
+                ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(destination);
+                if (!runningMonitor.isStart()) {
+                    runningMonitor.start();
+                }
+            }
+
+            if (autoScan) {
+                instanceConfigMonitors.get(config.getMode()).regeister(destination, defaultAction);
+            }
+        }
+
+        if (autoScan) {
+            instanceConfigMonitors.get(globalInstanceConfig.getMode()).start();
+            for (InstanceConfigMonitor monitor : instanceConfigMonitors.values()) {
+                if (!monitor.isStart()) {
+                    monitor.start();
+                }
+            }
+        }
+
+        // 启动网络接口
+        canalServer.start();
+    }
+
+    public void stop() throws Throwable {
+        canalServer.stop();
+
+        if (autoScan) {
+            for (InstanceConfigMonitor monitor : instanceConfigMonitors.values()) {
+                if (monitor.isStart()) {
+                    monitor.stop();
+                }
+            }
+        }
+
+        for (ServerRunningMonitor runningMonitor : ServerRunningMonitors.getRunningMonitors().values()) {
+            if (runningMonitor.isStart()) {
+                runningMonitor.stop();
+            }
+        }
+
+        // 释放canal的工作节点
+        releaseCid(ZookeeperPathUtils.getCanalClusterNode(ip + ":" + port));
+        logger.info("## stop the canal server[{}:{}]", ip, port);
+    }
+
+    private void initCid(String path) {
+        // logger.info("## init the canalId = {}", cid);
+        // 初始化系统目录
+        if (zkclientx != null) {
+            try {
+                zkclientx.createEphemeral(path);
+            } catch (ZkNoNodeException e) {
+                // 如果父目录不存在，则创建
+                String parentDir = path.substring(0, path.lastIndexOf('/'));
+                zkclientx.createPersistent(parentDir, true);
+                zkclientx.createEphemeral(path);
+            }
+
+        }
+    }
+
+    private void releaseCid(String path) {
+        // logger.info("## release the canalId = {}", cid);
+        // 初始化系统目录
+        if (zkclientx != null) {
+            zkclientx.delete(path);
+        }
+    }
+
+}