|
|
@@ -17,6 +17,7 @@ import org.springframework.context.ApplicationContext;
|
|
|
import org.springframework.context.support.ClassPathXmlApplicationContext;
|
|
|
|
|
|
import com.alibaba.otter.canal.common.CanalException;
|
|
|
+import com.alibaba.otter.canal.common.MQProperties;
|
|
|
import com.alibaba.otter.canal.common.utils.AddressUtils;
|
|
|
import com.alibaba.otter.canal.common.zookeeper.ZkClientx;
|
|
|
import com.alibaba.otter.canal.common.zookeeper.ZookeeperPathUtils;
|
|
|
@@ -35,6 +36,7 @@ import com.alibaba.otter.canal.instance.manager.CanalConfigClient;
|
|
|
import com.alibaba.otter.canal.instance.manager.ManagerCanalInstanceGenerator;
|
|
|
import com.alibaba.otter.canal.instance.spring.SpringCanalInstanceGenerator;
|
|
|
import com.alibaba.otter.canal.parse.CanalEventParser;
|
|
|
+import com.alibaba.otter.canal.server.CanalMQStarter;
|
|
|
import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded;
|
|
|
import com.alibaba.otter.canal.server.exception.CanalServerException;
|
|
|
import com.alibaba.otter.canal.server.netty.CanalServerWithNetty;
|
|
|
@@ -68,6 +70,8 @@ public class CanalController {
|
|
|
private CanalInstanceGenerator instanceGenerator;
|
|
|
private ZkClientx zkclientx;
|
|
|
|
|
|
+ private CanalMQStarter canalMQStarter;
|
|
|
+
|
|
|
public CanalController(){
|
|
|
this(System.getProperties());
|
|
|
}
|
|
|
@@ -105,7 +109,7 @@ public class CanalController {
|
|
|
logger.info("No valid metrics server port found, use default 11112.");
|
|
|
embededCanalServer.setMetricsPort(11112);
|
|
|
}
|
|
|
-
|
|
|
+
|
|
|
String canalWithoutNetty = getProperty(properties, CanalConstants.CANAL_WITHOUT_NETTY);
|
|
|
if (canalWithoutNetty == null || "false".equals(canalWithoutNetty)) {
|
|
|
canalServer = CanalServerWithNetty.instance();
|
|
|
@@ -127,80 +131,81 @@ public class CanalController {
|
|
|
|
|
|
final ServerRunningData serverData = new ServerRunningData(cid, ip + ":" + port);
|
|
|
ServerRunningMonitors.setServerData(serverData);
|
|
|
- ServerRunningMonitors.setRunningMonitors(MigrateMap.makeComputingMap(new Function<String, ServerRunningMonitor>() {
|
|
|
-
|
|
|
- public ServerRunningMonitor apply(final String destination) {
|
|
|
- ServerRunningMonitor runningMonitor = new ServerRunningMonitor(serverData);
|
|
|
- runningMonitor.setDestination(destination);
|
|
|
- runningMonitor.setListener(new ServerRunningListener() {
|
|
|
-
|
|
|
- public void processActiveEnter() {
|
|
|
- try {
|
|
|
- MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
|
|
|
- embededCanalServer.start(destination);
|
|
|
- } finally {
|
|
|
- MDC.remove(CanalConstants.MDC_DESTINATION);
|
|
|
+ ServerRunningMonitors
|
|
|
+ .setRunningMonitors(MigrateMap.makeComputingMap(new Function<String, ServerRunningMonitor>() {
|
|
|
+
|
|
|
+ public ServerRunningMonitor apply(final String destination) {
|
|
|
+ ServerRunningMonitor runningMonitor = new ServerRunningMonitor(serverData);
|
|
|
+ runningMonitor.setDestination(destination);
|
|
|
+ runningMonitor.setListener(new ServerRunningListener() {
|
|
|
+
|
|
|
+ public void processActiveEnter() {
|
|
|
+ try {
|
|
|
+ MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
|
|
|
+ embededCanalServer.start(destination);
|
|
|
+ } finally {
|
|
|
+ MDC.remove(CanalConstants.MDC_DESTINATION);
|
|
|
+ }
|
|
|
}
|
|
|
- }
|
|
|
|
|
|
- public void processActiveExit() {
|
|
|
- try {
|
|
|
- MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
|
|
|
- embededCanalServer.stop(destination);
|
|
|
- } finally {
|
|
|
- MDC.remove(CanalConstants.MDC_DESTINATION);
|
|
|
+ public void processActiveExit() {
|
|
|
+ try {
|
|
|
+ MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
|
|
|
+ embededCanalServer.stop(destination);
|
|
|
+ } finally {
|
|
|
+ MDC.remove(CanalConstants.MDC_DESTINATION);
|
|
|
+ }
|
|
|
}
|
|
|
- }
|
|
|
-
|
|
|
- public void processStart() {
|
|
|
- try {
|
|
|
- if (zkclientx != null) {
|
|
|
- final String path = ZookeeperPathUtils.getDestinationClusterNode(destination, ip + ":"
|
|
|
- + port);
|
|
|
- initCid(path);
|
|
|
- zkclientx.subscribeStateChanges(new IZkStateListener() {
|
|
|
-
|
|
|
- public void handleStateChanged(KeeperState state) throws Exception {
|
|
|
-
|
|
|
- }
|
|
|
|
|
|
- public void handleNewSession() throws Exception {
|
|
|
- initCid(path);
|
|
|
- }
|
|
|
-
|
|
|
- @Override
|
|
|
- public void handleSessionEstablishmentError(Throwable error) throws Exception {
|
|
|
- logger.error("failed to connect to zookeeper", error);
|
|
|
- }
|
|
|
- });
|
|
|
+ public void processStart() {
|
|
|
+ try {
|
|
|
+ if (zkclientx != null) {
|
|
|
+ final String path = ZookeeperPathUtils.getDestinationClusterNode(destination,
|
|
|
+ ip + ":" + port);
|
|
|
+ initCid(path);
|
|
|
+ zkclientx.subscribeStateChanges(new IZkStateListener() {
|
|
|
+
|
|
|
+ public void handleStateChanged(KeeperState state) throws Exception {
|
|
|
+
|
|
|
+ }
|
|
|
+
|
|
|
+ public void handleNewSession() throws Exception {
|
|
|
+ initCid(path);
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public void handleSessionEstablishmentError(Throwable error) throws Exception {
|
|
|
+ logger.error("failed to connect to zookeeper", error);
|
|
|
+ }
|
|
|
+ });
|
|
|
+ }
|
|
|
+ } finally {
|
|
|
+ MDC.remove(CanalConstants.MDC_DESTINATION);
|
|
|
}
|
|
|
- } finally {
|
|
|
- MDC.remove(CanalConstants.MDC_DESTINATION);
|
|
|
}
|
|
|
- }
|
|
|
|
|
|
- public void processStop() {
|
|
|
- try {
|
|
|
- MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
|
|
|
- if (zkclientx != null) {
|
|
|
- final String path = ZookeeperPathUtils.getDestinationClusterNode(destination, ip + ":"
|
|
|
- + port);
|
|
|
- releaseCid(path);
|
|
|
+ public void processStop() {
|
|
|
+ try {
|
|
|
+ MDC.put(CanalConstants.MDC_DESTINATION, String.valueOf(destination));
|
|
|
+ if (zkclientx != null) {
|
|
|
+ final String path = ZookeeperPathUtils.getDestinationClusterNode(destination,
|
|
|
+ ip + ":" + port);
|
|
|
+ releaseCid(path);
|
|
|
+ }
|
|
|
+ } finally {
|
|
|
+ MDC.remove(CanalConstants.MDC_DESTINATION);
|
|
|
}
|
|
|
- } finally {
|
|
|
- MDC.remove(CanalConstants.MDC_DESTINATION);
|
|
|
}
|
|
|
- }
|
|
|
|
|
|
- });
|
|
|
- if (zkclientx != null) {
|
|
|
- runningMonitor.setZkClient(zkclientx);
|
|
|
+ });
|
|
|
+ if (zkclientx != null) {
|
|
|
+ runningMonitor.setZkClient(zkclientx);
|
|
|
+ }
|
|
|
+ // 触发创建一下cid节点
|
|
|
+ runningMonitor.init();
|
|
|
+ return runningMonitor;
|
|
|
}
|
|
|
- // 触发创建一下cid节点
|
|
|
- runningMonitor.init();
|
|
|
- return runningMonitor;
|
|
|
- }
|
|
|
- }));
|
|
|
+ }));
|
|
|
|
|
|
// 初始化monitor机制
|
|
|
autoScan = BooleanUtils.toBoolean(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN));
|
|
|
@@ -220,6 +225,9 @@ public class CanalController {
|
|
|
ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(destination);
|
|
|
if (!config.getLazy() && !runningMonitor.isStart()) {
|
|
|
runningMonitor.start();
|
|
|
+ if (canalMQStarter != null) {
|
|
|
+ canalMQStarter.start(getMQProperties(properties));
|
|
|
+ }
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
@@ -228,6 +236,9 @@ public class CanalController {
|
|
|
// 此处的stop，代表强制退出，非HA机制，所以需要退出HA的monitor和配置信息
|
|
|
InstanceConfig config = instanceConfigs.remove(destination);
|
|
|
if (config != null) {
|
|
|
+ if (canalMQStarter != null) {
|
|
|
+ canalMQStarter.stop();
|
|
|
+ }
|
|
|
embededCanalServer.stop(destination);
|
|
|
ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(destination);
|
|
|
if (runningMonitor.isStart()) {
|
|
|
@@ -246,7 +257,8 @@ public class CanalController {
|
|
|
instanceConfigMonitors = MigrateMap.makeComputingMap(new Function<InstanceMode, InstanceConfigMonitor>() {
|
|
|
|
|
|
public InstanceConfigMonitor apply(InstanceMode mode) {
|
|
|
- int scanInterval = Integer.valueOf(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN_INTERVAL));
|
|
|
+ int scanInterval = Integer
|
|
|
+ .valueOf(getProperty(properties, CanalConstants.CANAL_AUTO_SCAN_INTERVAL));
|
|
|
|
|
|
if (mode.isSpring()) {
|
|
|
SpringInstanceConfigMonitor monitor = new SpringInstanceConfigMonitor();
|
|
|
@@ -354,8 +366,8 @@ public class CanalController {
|
|
|
InstanceConfig oldConfig = instanceConfigs.put(destination, config);
|
|
|
|
|
|
if (oldConfig != null) {
|
|
|
- logger.warn("destination:{} old config:{} has replace by new config:{}", new Object[] { destination,
|
|
|
- oldConfig, config });
|
|
|
+ logger
|
|
|
+ .warn("destination:{} old config:{} has replace by new config:{}", destination, oldConfig, config);
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
@@ -514,4 +526,31 @@ public class CanalController {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+ public CanalMQStarter getCanalMQStarter() {
|
|
|
+ return canalMQStarter;
|
|
|
+ }
|
|
|
+
|
|
|
+ public void setCanalMQStarter(CanalMQStarter canalMQStarter) {
|
|
|
+ this.canalMQStarter = canalMQStarter;
|
|
|
+ }
|
|
|
+
|
|
|
+ public static MQProperties getMQProperties(Properties properties) {
|
|
|
+ MQProperties mqProperties = new MQProperties();
|
|
|
+ mqProperties.setServers(CanalController.getProperty(properties, CanalConstants.CANAL_MQ_SERVERS));
|
|
|
+ mqProperties
|
|
|
+ .setRetries(Integer.valueOf(CanalController.getProperty(properties, CanalConstants.CANAL_MQ_RETRIES)));
|
|
|
+ mqProperties
|
|
|
+ .setBatchSize(Integer.valueOf(CanalController.getProperty(properties, CanalConstants.CANAL_MQ_BATCHSIZE)));
|
|
|
+ mqProperties
|
|
|
+ .setLingerMs(Integer.valueOf(CanalController.getProperty(properties, CanalConstants.CANAL_MQ_LINGERMS)));
|
|
|
+ mqProperties.setBufferMemory(
|
|
|
+ Long.valueOf(CanalController.getProperty(properties, CanalConstants.CANAL_MQ_BUFFERMEMORY)));
|
|
|
+ mqProperties.setCanalBatchSize(
|
|
|
+ Integer.valueOf(CanalController.getProperty(properties, CanalConstants.CANAL_MQ_CANALBATCHSIZE)));
|
|
|
+ mqProperties.setCanalGetTimeout(
|
|
|
+ Long.valueOf(CanalController.getProperty(properties, CanalConstants.CANAL_MQ_CANALGETTIMEOUT)));
|
|
|
+ mqProperties.setFlatMessage(
|
|
|
+ Boolean.valueOf(CanalController.getProperty(properties, CanalConstants.CANAL_MQ_FLATMESSAGE)));
|
|
|
+ return mqProperties;
|
|
|
+ }
|
|
|
}
|
mcy