|
|
@@ -10,6 +10,8 @@ import java.util.concurrent.ScheduledExecutorService;
|
|
|
import java.util.concurrent.ScheduledFuture;
|
|
|
import java.util.concurrent.ThreadFactory;
|
|
|
import java.util.concurrent.TimeUnit;
|
|
|
+import java.util.concurrent.locks.ReadWriteLock;
|
|
|
+import java.util.concurrent.locks.ReentrantReadWriteLock;
|
|
|
import java.util.regex.Pattern;
|
|
|
|
|
|
import org.apache.commons.beanutils.BeanUtils;
|
|
|
@@ -45,30 +47,34 @@ import com.alibaba.otter.canal.protocol.position.EntryPosition;
|
|
|
*/
|
|
|
public class DatabaseTableMeta implements TableMetaTSDB {
|
|
|
|
|
|
- public static final EntryPosition INIT_POSITION = new EntryPosition("0", 0L, -2L, -1L);
|
|
|
- private static Logger logger = LoggerFactory.getLogger(DatabaseTableMeta.class);
|
|
|
- private static Pattern pattern = Pattern.compile("Duplicate entry '.*' for key '*'");
|
|
|
- private static Pattern h2Pattern = Pattern.compile("Unique index or primary key violation");
|
|
|
- private static ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {
|
|
|
- @Override
|
|
|
- public Thread newThread(Runnable r) {
|
|
|
- Thread thread = new Thread(r, "[scheduler-table-meta-snapshot]");
|
|
|
- thread.setDaemon(true);
|
|
|
- return thread;
|
|
|
- }
|
|
|
- });
|
|
|
- private String destination;
|
|
|
- private MemoryTableMeta memoryTableMeta;
|
|
|
- private MysqlConnection connection; // 查询meta信息的链接
|
|
|
- private CanalEventFilter filter;
|
|
|
- private CanalEventFilter blackFilter;
|
|
|
- private EntryPosition lastPosition;
|
|
|
- private MetaHistoryDAO metaHistoryDAO;
|
|
|
- private MetaSnapshotDAO metaSnapshotDAO;
|
|
|
- private int snapshotInterval = 24;
|
|
|
- private int snapshotExpire = 360;
|
|
|
- private ScheduledFuture<?> scheduleSnapshotFuture;
|
|
|
-
|
|
|
+ public static final EntryPosition INIT_POSITION = new EntryPosition("0", 0L, -2L, -1L);
|
|
|
+ private static Logger logger = LoggerFactory.getLogger(DatabaseTableMeta.class);
|
|
|
+ private static Pattern pattern = Pattern.compile("Duplicate entry '.*' for key '*'");
|
|
|
+ private static Pattern h2Pattern = Pattern.compile("Unique index or primary key violation");
|
|
|
+ private static ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public Thread newThread(Runnable r) {
|
|
|
+ Thread thread = new Thread(r,
|
|
|
+ "[scheduler-table-meta-snapshot]");
|
|
|
+ thread.setDaemon(true);
|
|
|
+ return thread;
|
|
|
+ }
|
|
|
+ });
|
|
|
+ private ReadWriteLock lock = new ReentrantReadWriteLock();
|
|
|
+ private String destination;
|
|
|
+ private MemoryTableMeta memoryTableMeta;
|
|
|
+ private MysqlConnection connection; // 查询meta信息的链接
|
|
|
+ private CanalEventFilter filter;
|
|
|
+ private CanalEventFilter blackFilter;
|
|
|
+ private EntryPosition lastPosition;
|
|
|
+ private boolean hasNewDdl;
|
|
|
+ private MetaHistoryDAO metaHistoryDAO;
|
|
|
+ private MetaSnapshotDAO metaSnapshotDAO;
|
|
|
+ private int snapshotInterval = 24;
|
|
|
+ private int snapshotExpire = 360;
|
|
|
+ private ScheduledFuture<?> scheduleSnapshotFuture;
|
|
|
+
|
|
|
public DatabaseTableMeta(){
|
|
|
|
|
|
}
|
|
|
@@ -105,13 +111,13 @@ public class DatabaseTableMeta implements TableMetaTSDB {
|
|
|
}
|
|
|
return true;
|
|
|
}
|
|
|
-
|
|
|
+
|
|
|
@Override
|
|
|
public void destory() {
|
|
|
if (memoryTableMeta != null) {
|
|
|
memoryTableMeta.destory();
|
|
|
}
|
|
|
-
|
|
|
+
|
|
|
if (connection != null) {
|
|
|
try {
|
|
|
connection.disconnect();
|
|
|
@@ -120,7 +126,7 @@ public class DatabaseTableMeta implements TableMetaTSDB {
|
|
|
.getAddress(), e);
|
|
|
}
|
|
|
}
|
|
|
-
|
|
|
+
|
|
|
if (scheduleSnapshotFuture != null) {
|
|
|
scheduleSnapshotFuture.cancel(false);
|
|
|
}
|
|
|
@@ -128,22 +134,29 @@ public class DatabaseTableMeta implements TableMetaTSDB {
|
|
|
|
|
|
@Override
|
|
|
public TableMeta find(String schema, String table) {
|
|
|
- synchronized (memoryTableMeta) {
|
|
|
+ lock.readLock().lock();
|
|
|
+ try {
|
|
|
return memoryTableMeta.find(schema, table);
|
|
|
+ } finally {
|
|
|
+ lock.readLock().unlock();
|
|
|
}
|
|
|
}
|
|
|
|
|
|
@Override
|
|
|
public boolean apply(EntryPosition position, String schema, String ddl, String extra) {
|
|
|
// 首先记录到内存结构
|
|
|
- synchronized (memoryTableMeta) {
|
|
|
+ lock.writeLock().lock();
|
|
|
+ try {
|
|
|
if (memoryTableMeta.apply(position, schema, ddl, extra)) {
|
|
|
this.lastPosition = position;
|
|
|
+ this.hasNewDdl = true;
|
|
|
// 同步每次变更给远程做历史记录
|
|
|
return applyHistoryToDB(position, schema, ddl, extra);
|
|
|
} else {
|
|
|
throw new RuntimeException("apply to memory is failed");
|
|
|
}
|
|
|
+ } finally {
|
|
|
+ lock.writeLock().unlock();
|
|
|
}
|
|
|
}
|
|
|
|
|
|
@@ -267,17 +280,22 @@ public class DatabaseTableMeta implements TableMetaTSDB {
|
|
|
*/
|
|
|
private boolean applySnapshotToDB(EntryPosition position, boolean init) {
|
|
|
// 获取一份快照
|
|
|
- MemoryTableMeta tmpMemoryTableMeta = new MemoryTableMeta();
|
|
|
Map<String, String> schemaDdls = null;
|
|
|
- synchronized (memoryTableMeta) {
|
|
|
- if (!init && position == null) {
|
|
|
+ lock.readLock().lock();
|
|
|
+ try {
|
|
|
+ if (!init && !hasNewDdl) {
|
|
|
// 如果是持续构建,则识别一下是否有DDL变更过,如果没有就忽略了
|
|
|
return false;
|
|
|
}
|
|
|
+ this.hasNewDdl = false;
|
|
|
schemaDdls = memoryTableMeta.snapshot();
|
|
|
- for (Map.Entry<String, String> entry : schemaDdls.entrySet()) {
|
|
|
- tmpMemoryTableMeta.apply(position, entry.getKey(), entry.getValue(), null);
|
|
|
- }
|
|
|
+ } finally {
|
|
|
+ lock.readLock().unlock();
|
|
|
+ }
|
|
|
+
|
|
|
+ MemoryTableMeta tmpMemoryTableMeta = new MemoryTableMeta();
|
|
|
+ for (Map.Entry<String, String> entry : schemaDdls.entrySet()) {
|
|
|
+ tmpMemoryTableMeta.apply(position, entry.getKey(), entry.getValue(), null);
|
|
|
}
|
|
|
|
|
|
// 基于临时内存对象进行对比
|
|
|
@@ -472,7 +490,26 @@ public class DatabaseTableMeta implements TableMetaTSDB {
|
|
|
return false;
|
|
|
}
|
|
|
|
|
|
- if (!StringUtils.equalsIgnoreCase(sourceField.getColumnType(), targetField.getColumnType())) {
|
|
|
+ // if (!StringUtils.equalsIgnoreCase(sourceField.getColumnType(),
|
|
|
+ // targetField.getColumnType())) {
|
|
|
+ // return false;
|
|
|
+ // }
|
|
|
+
|
|
|
+ // https://github.com/alibaba/canal/issues/1100
|
|
|
+ // 支持一下 int vs int(10)
|
|
|
+ if ((sourceField.isUnsigned() && !targetField.isUnsigned())
|
|
|
+ || (!sourceField.isUnsigned() && targetField.isUnsigned())) {
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+
|
|
|
+ String sign = sourceField.isUnsigned() ? "unsigned" : "signed";
|
|
|
+ String sourceColumnType = StringUtils.removeEndIgnoreCase(sourceField.getColumnType(), sign).trim();
|
|
|
+ String targetColumnType = StringUtils.removeEndIgnoreCase(targetField.getColumnType(), sign).trim();
|
|
|
+
|
|
|
+ boolean columnTypeCompare = false;
|
|
|
+ columnTypeCompare |= StringUtils.containsIgnoreCase(sourceColumnType, targetColumnType);
|
|
|
+ columnTypeCompare |= StringUtils.containsIgnoreCase(targetColumnType, sourceColumnType);
|
|
|
+ if (!columnTypeCompare) {
|
|
|
return false;
|
|
|
}
|
|
|
|
machey