use es6x/es7x instead of elaticsearch project

client-adapter/elasticsearch/pom.xml

@@ -1,105 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <parent>
-        <artifactId>canal.client-adapter</artifactId>
-        <groupId>com.alibaba.otter</groupId>
-        <version>1.1.5-SNAPSHOT</version>
-    </parent>
-    <modelVersion>4.0.0</modelVersion>
-    <groupId>com.alibaba.otter</groupId>
-    <artifactId>client-adapter.elasticsearch</artifactId>
-    <packaging>jar</packaging>
-    <name>canal client adapter elasticsearch module for otter ${project.version}</name>
-
-    <dependencies>
-        <dependency>
-            <groupId>com.alibaba.otter</groupId>
-            <artifactId>client-adapter.common</artifactId>
-            <version>${project.version}</version>
-            <scope>provided</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.alibaba.fastsql</groupId>
-            <artifactId>fastsql</artifactId>
-            <version>2.0.0_preview_855</version>
-        </dependency>
-        <dependency>
-            <groupId>org.elasticsearch</groupId>
-            <artifactId>elasticsearch</artifactId>
-            <version>6.4.3</version>
-        </dependency>
-        <dependency>
-            <groupId>org.elasticsearch.client</groupId>
-            <artifactId>transport</artifactId>
-            <version>6.4.3</version>
-        </dependency>
-        <dependency>
-            <groupId>org.elasticsearch.client</groupId>
-            <artifactId>elasticsearch-rest-client</artifactId>
-            <version>6.4.3</version>
-        </dependency>
-        <dependency>
-            <groupId>org.elasticsearch.client</groupId>
-            <artifactId>elasticsearch-rest-high-level-client</artifactId>
-            <version>6.4.3</version>
-        </dependency>
-
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <version>4.12</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>mysql</groupId>
-            <artifactId>mysql-connector-java</artifactId>
-            <version>5.1.40</version>
-            <scope>test</scope>
-        </dependency>
-    </dependencies>
-
-    <build>
-        <plugins>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-assembly-plugin</artifactId>
-                <version>2.4</version>
-                <configuration>
-                    <descriptorRefs>
-                        <descriptorRef>jar-with-dependencies</descriptorRef>
-                    </descriptorRefs>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>make-assembly</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>single</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
-            <plugin>
-                <artifactId>maven-antrun-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>run</goal>
-                        </goals>
-                        <configuration>
-                            <tasks>
-                                <copy todir="${project.basedir}/../launcher/target/classes/es" overwrite="true">
-                                    <fileset dir="${project.basedir}/target/classes/es" erroronmissingdir="true">
-                                        <include name="*.yml" />
-                                    </fileset>
-                                </copy>
-                            </tasks>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-        </plugins>
-    </build>
-
-</project>

client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/ESAdapter.java

@@ -1,256 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.es;
-
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import javax.sql.DataSource;
-
-import org.apache.commons.lang.StringUtils;
-import org.elasticsearch.action.search.SearchResponse;
-
-import com.alibaba.druid.pool.DruidDataSource;
-import com.alibaba.otter.canal.client.adapter.OuterAdapter;
-import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
-import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping;
-import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfigLoader;
-import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem;
-import com.alibaba.otter.canal.client.adapter.es.config.SqlParser;
-import com.alibaba.otter.canal.client.adapter.es.monitor.ESConfigMonitor;
-import com.alibaba.otter.canal.client.adapter.es.service.ESEtlService;
-import com.alibaba.otter.canal.client.adapter.es.service.ESSyncService;
-import com.alibaba.otter.canal.client.adapter.es.support.ESConnection;
-import com.alibaba.otter.canal.client.adapter.es.support.ESTemplate;
-import com.alibaba.otter.canal.client.adapter.support.*;
-
-/**
- * ES外部适配器
- *
- * @author rewerma 2018-10-20
- * @version 1.0.0
- */
-@SPI("es")
-public class ESAdapter implements OuterAdapter {
-
-    private Map<String, ESSyncConfig>              esSyncConfig        = new ConcurrentHashMap<>(); // 文件名对应配置
-    private Map<String, Map<String, ESSyncConfig>> dbTableEsSyncConfig = new ConcurrentHashMap<>(); // schema-table对应配置
-
-    private ESConnection                           esConnection;
-
-    private ESSyncService                          esSyncService;
-
-    private ESConfigMonitor                        esConfigMonitor;
-
-    private Properties                             envProperties;
-
-    public ESSyncService getEsSyncService() {
-        return esSyncService;
-    }
-
-    public Map<String, ESSyncConfig> getEsSyncConfig() {
-        return esSyncConfig;
-    }
-
-    public Map<String, Map<String, ESSyncConfig>> getDbTableEsSyncConfig() {
-        return dbTableEsSyncConfig;
-    }
-
-    public ESConnection getEsConnection() {
-        return esConnection;
-    }
-
-    public void setEsConnection(ESConnection esConnection) {
-        this.esConnection = esConnection;
-    }
-
-    @Override
-    public void init(OuterAdapterConfig configuration, Properties envProperties) {
-        try {
-            this.envProperties = envProperties;
-            Map<String, ESSyncConfig> esSyncConfigTmp = ESSyncConfigLoader.load(envProperties);
-            // 过滤不匹配的key的配置
-            esSyncConfigTmp.forEach((key, config) -> {
-                if ((config.getOuterAdapterKey() == null && configuration.getKey() == null)
-                    || (config.getOuterAdapterKey() != null
-                        && config.getOuterAdapterKey().equalsIgnoreCase(configuration.getKey()))) {
-                    esSyncConfig.put(key, config);
-                }
-            });
-
-            for (Map.Entry<String, ESSyncConfig> entry : esSyncConfig.entrySet()) {
-                String configName = entry.getKey();
-                ESSyncConfig config = entry.getValue();
-                addSyncConfigToCache(configName, config);
-            }
-
-            Map<String, String> properties = configuration.getProperties();
-
-            String[] hostArray = configuration.getHosts().split(",");
-            String mode = properties.get("mode");
-            if ("rest".equalsIgnoreCase(mode) || "http".equalsIgnoreCase(mode)) {
-                esConnection = new ESConnection(hostArray, properties, ESConnection.ESClientMode.REST);
-            } else {
-                esConnection = new ESConnection(hostArray, properties, ESConnection.ESClientMode.TRANSPORT);
-            }
-
-            ESTemplate esTemplate = new ESTemplate(esConnection);
-            esSyncService = new ESSyncService(esTemplate);
-
-            esConfigMonitor = new ESConfigMonitor();
-            esConfigMonitor.init(this, envProperties);
-        } catch (Throwable e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    public void addSyncConfigToCache(String configName, ESSyncConfig config) {
-        Properties envProperties = this.envProperties;
-        SchemaItem schemaItem = SqlParser.parse(config.getEsMapping().getSql());
-        config.getEsMapping().setSchemaItem(schemaItem);
-
-        DruidDataSource dataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
-        if (dataSource == null || dataSource.getUrl() == null) {
-            throw new RuntimeException("No data source found: " + config.getDataSourceKey());
-        }
-        Pattern pattern = Pattern.compile(".*:(.*)://.*/(.*)\\?.*$");
-        Matcher matcher = pattern.matcher(dataSource.getUrl());
-        if (!matcher.find()) {
-            throw new RuntimeException("Not found the schema of jdbc-url: " + config.getDataSourceKey());
-        }
-        String schema = matcher.group(2);
-
-        schemaItem.getAliasTableItems().values().forEach(tableItem -> {
-            Map<String, ESSyncConfig> esSyncConfigMap;
-            if (envProperties != null
-                && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
-                esSyncConfigMap = dbTableEsSyncConfig
-                    .computeIfAbsent(StringUtils.trimToEmpty(config.getDestination()) + "-"
-                                     + StringUtils.trimToEmpty(config.getGroupId()) + "_" + schema + "-"
-                                     + tableItem.getTableName(),
-                        k -> new ConcurrentHashMap<>());
-            } else {
-                esSyncConfigMap = dbTableEsSyncConfig
-                    .computeIfAbsent(StringUtils.trimToEmpty(config.getDestination()) + "_" + schema + "-"
-                                     + tableItem.getTableName(),
-                        k -> new ConcurrentHashMap<>());
-            }
-
-            esSyncConfigMap.put(configName, config);
-        });
-    }
-
-
-    @Override
-    public void sync(List<Dml> dmls) {
-        if (dmls == null || dmls.isEmpty()) {
-            return;
-        }
-        for (Dml dml : dmls) {
-            if (!dml.getIsDdl()) {
-                sync(dml);
-            }
-        }
-        esSyncService.commit(); // 批次统一提交
-
-    }
-
-    private void sync(Dml dml) {
-        String database = dml.getDatabase();
-        String table = dml.getTable();
-        Map<String, ESSyncConfig> configMap;
-        if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
-            configMap = dbTableEsSyncConfig
-                .get(StringUtils.trimToEmpty(dml.getDestination()) + "-" + StringUtils.trimToEmpty(dml.getGroupId())
-                     + "_" + database + "-" + table);
-        } else {
-            configMap = dbTableEsSyncConfig
-                .get(StringUtils.trimToEmpty(dml.getDestination()) + "_" + database + "-" + table);
-        }
-
-        if (configMap != null && !configMap.values().isEmpty()) {
-            esSyncService.sync(configMap.values(), dml);
-        }
-    }
-
-    @Override
-    public EtlResult etl(String task, List<String> params) {
-        EtlResult etlResult = new EtlResult();
-        ESSyncConfig config = esSyncConfig.get(task);
-        if (config != null) {
-            DataSource dataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
-            ESEtlService esEtlService = new ESEtlService(esConnection, config);
-            if (dataSource != null) {
-                return esEtlService.importData(params);
-            } else {
-                etlResult.setSucceeded(false);
-                etlResult.setErrorMessage("DataSource not found");
-                return etlResult;
-            }
-        } else {
-            StringBuilder resultMsg = new StringBuilder();
-            boolean resSuccess = true;
-            for (ESSyncConfig configTmp : esSyncConfig.values()) {
-                // 取所有的destination为task的配置
-                if (configTmp.getDestination().equals(task)) {
-                    ESEtlService esEtlService = new ESEtlService(esConnection, configTmp);
-                    EtlResult etlRes = esEtlService.importData(params);
-                    if (!etlRes.getSucceeded()) {
-                        resSuccess = false;
-                        resultMsg.append(etlRes.getErrorMessage()).append("\n");
-                    } else {
-                        resultMsg.append(etlRes.getResultMessage()).append("\n");
-                    }
-                }
-            }
-            if (resultMsg.length() > 0) {
-                etlResult.setSucceeded(resSuccess);
-                if (resSuccess) {
-                    etlResult.setResultMessage(resultMsg.toString());
-                } else {
-                    etlResult.setErrorMessage(resultMsg.toString());
-                }
-                return etlResult;
-            }
-        }
-        etlResult.setSucceeded(false);
-        etlResult.setErrorMessage("Task not found");
-        return etlResult;
-    }
-
-    @Override
-    public Map<String, Object> count(String task) {
-        ESSyncConfig config = esSyncConfig.get(task);
-        ESMapping mapping = config.getEsMapping();
-        SearchResponse response = this.esConnection.new ESSearchRequest(mapping.get_index(), mapping.get_type()).size(0)
-            .getResponse();
-
-        long rowCount = response.getHits().getTotalHits();
-        Map<String, Object> res = new LinkedHashMap<>();
-        res.put("esIndex", mapping.get_index());
-        res.put("count", rowCount);
-        return res;
-    }
-
-    @Override
-    public void destroy() {
-        if (esConfigMonitor != null) {
-            esConfigMonitor.destroy();
-        }
-        if (esConnection != null) {
-            esConnection.close();
-        }
-    }
-
-    @Override
-    public String getDestination(String task) {
-        ESSyncConfig config = esSyncConfig.get(task);
-        if (config != null) {
-            return config.getDestination();
-        }
-        return null;
-    }
-}

client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfig.java

@@ -1,241 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.es.config;
-
-import com.alibaba.otter.canal.client.adapter.support.AdapterConfig;
-
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * ES 映射配置
- *
- * @author rewerma 2018-11-01
- * @version 1.0.0
- */
-public class ESSyncConfig implements AdapterConfig {
-
-    private String    dataSourceKey;   // 数据源key
-
-    private String    outerAdapterKey; // adapter key
-
-    private String    groupId;         // group id
-
-    private String    destination;     // canal destination
-
-    private ESMapping esMapping;
-
-    public void validate() {
-        if (esMapping._index == null) {
-            throw new NullPointerException("esMapping._index");
-        }
-        if (esMapping._type == null) {
-            throw new NullPointerException("esMapping._type");
-        }
-        if (esMapping._id == null && esMapping.getPk() == null) {
-            throw new NullPointerException("esMapping._id or esMapping.pk");
-        }
-        if (esMapping.sql == null) {
-            throw new NullPointerException("esMapping.sql");
-        }
-    }
-
-    public String getDataSourceKey() {
-        return dataSourceKey;
-    }
-
-    public void setDataSourceKey(String dataSourceKey) {
-        this.dataSourceKey = dataSourceKey;
-    }
-
-    public String getOuterAdapterKey() {
-        return outerAdapterKey;
-    }
-
-    public void setOuterAdapterKey(String outerAdapterKey) {
-        this.outerAdapterKey = outerAdapterKey;
-    }
-
-    public String getGroupId() {
-        return groupId;
-    }
-
-    public void setGroupId(String groupId) {
-        this.groupId = groupId;
-    }
-
-    public String getDestination() {
-        return destination;
-    }
-
-    public void setDestination(String destination) {
-        this.destination = destination;
-    }
-
-    public ESMapping getEsMapping() {
-        return esMapping;
-    }
-
-    public void setEsMapping(ESMapping esMapping) {
-        this.esMapping = esMapping;
-    }
-
-    public ESMapping getMapping(){
-        return esMapping;
-    }
-
-    public static class ESMapping implements AdapterMapping{
-
-        private String                       _index;
-        private String                       _type;
-        private String                       _id;
-        private boolean                      upsert          = false;
-        private String                       pk;
-        private Map<String, RelationMapping> relations       = new LinkedHashMap<>();
-        private String                       sql;
-        // 对象字段, 例: objFields:
-        // - _labels: array:;
-        private Map<String, String>          objFields       = new LinkedHashMap<>();
-        private List<String>                 skips           = new ArrayList<>();
-        private int                          commitBatch     = 1000;
-        private String                       etlCondition;
-        private boolean                      syncByTimestamp = false;                // 是否按时间戳定时同步
-        private Long                         syncInterval;                           // 同步时间间隔
-
-        private SchemaItem                   schemaItem;                             // sql解析结果模型
-
-        public String get_index() {
-            return _index;
-        }
-
-        public void set_index(String _index) {
-            this._index = _index;
-        }
-
-        public String get_type() {
-            return _type;
-        }
-
-        public void set_type(String _type) {
-            this._type = _type;
-        }
-
-        public String get_id() {
-            return _id;
-        }
-
-        public void set_id(String _id) {
-            this._id = _id;
-        }
-
-        public boolean isUpsert() {
-            return upsert;
-        }
-
-        public void setUpsert(boolean upsert) {
-            this.upsert = upsert;
-        }
-
-        public String getPk() {
-            return pk;
-        }
-
-        public void setPk(String pk) {
-            this.pk = pk;
-        }
-
-        public Map<String, String> getObjFields() {
-            return objFields;
-        }
-
-        public void setObjFields(Map<String, String> objFields) {
-            this.objFields = objFields;
-        }
-
-        public List<String> getSkips() {
-            return skips;
-        }
-
-        public void setSkips(List<String> skips) {
-            this.skips = skips;
-        }
-
-        public Map<String, RelationMapping> getRelations() {
-            return relations;
-        }
-
-        public void setRelations(Map<String, RelationMapping> relations) {
-            this.relations = relations;
-        }
-
-        public String getSql() {
-            return sql;
-        }
-
-        public void setSql(String sql) {
-            this.sql = sql;
-        }
-
-        public int getCommitBatch() {
-            return commitBatch;
-        }
-
-        public void setCommitBatch(int commitBatch) {
-            this.commitBatch = commitBatch;
-        }
-
-        public String getEtlCondition() {
-            return etlCondition;
-        }
-
-        public void setEtlCondition(String etlCondition) {
-            this.etlCondition = etlCondition;
-        }
-
-        public Long getSyncInterval() {
-            return syncInterval;
-        }
-
-        public void setSyncInterval(Long syncInterval) {
-            this.syncInterval = syncInterval;
-        }
-
-        public boolean isSyncByTimestamp() {
-            return syncByTimestamp;
-        }
-
-        public void setSyncByTimestamp(boolean syncByTimestamp) {
-            this.syncByTimestamp = syncByTimestamp;
-        }
-
-        public SchemaItem getSchemaItem() {
-            return schemaItem;
-        }
-
-        public void setSchemaItem(SchemaItem schemaItem) {
-            this.schemaItem = schemaItem;
-        }
-    }
-
-    public static class RelationMapping {
-
-        private String name;
-        private String parent;
-
-        public String getName() {
-            return name;
-        }
-
-        public void setName(String name) {
-            this.name = name;
-        }
-
-        public String getParent() {
-            return parent;
-        }
-
-        public void setParent(String parent) {
-            this.parent = parent;
-        }
-    }
-}

client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/ESSyncConfigLoader.java

@@ -1,45 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.es.config;
-
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.Properties;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
-import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
-
-/**
- * ES 配置装载器
- *
- * @author rewerma 2018-11-01
- * @version 1.0.0
- */
-public class ESSyncConfigLoader {
-
-    private static Logger logger = LoggerFactory.getLogger(ESSyncConfigLoader.class);
-
-    public static synchronized Map<String, ESSyncConfig> load(Properties envProperties) {
-        logger.info("## Start loading es mapping config ... ");
-
-        Map<String, ESSyncConfig> esSyncConfig = new LinkedHashMap<>();
-
-        Map<String, String> configContentMap = MappingConfigsLoader.loadConfigs("es");
-        configContentMap.forEach((fileName, content) -> {
-            ESSyncConfig config = YmlConfigBinder.bindYmlToObj(null, content, ESSyncConfig.class, null, envProperties);
-            if (config == null) {
-                return;
-            }
-            try {
-                config.validate();
-            } catch (Exception e) {
-                throw new RuntimeException("ERROR Config: " + fileName + " " + e.getMessage(), e);
-            }
-            esSyncConfig.put(fileName, config);
-        });
-
-        logger.info("## ES mapping config loaded");
-        return esSyncConfig;
-    }
-}

client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SchemaItem.java

@@ -1,440 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.es.config;
-
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-
-import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping;
-
-/**
- * ES 映射配置视图
- *
- * @author rewerma 2018-11-01
- * @version 1.0.0
- */
-public class SchemaItem {
-
-    private Map<String, TableItem>                aliasTableItems = new LinkedHashMap<>(); // 别名对应表名
-    private Map<String, FieldItem>                selectFields    = new LinkedHashMap<>(); // 查询字段
-    private String                                sql;
-
-    private volatile Map<String, List<TableItem>> tableItemAliases;
-    private volatile Map<String, List<FieldItem>> columnFields;
-    private volatile Boolean                      allFieldsSimple;
-
-    public void init() {
-        this.getTableItemAliases();
-        this.getColumnFields();
-        this.isAllFieldsSimple();
-        aliasTableItems.values().forEach(tableItem -> {
-            tableItem.getRelationTableFields();
-            tableItem.getRelationSelectFieldItems();
-        });
-    }
-
-    public Map<String, TableItem> getAliasTableItems() {
-        return aliasTableItems;
-    }
-
-    public void setAliasTableItems(Map<String, TableItem> aliasTableItems) {
-        this.aliasTableItems = aliasTableItems;
-    }
-
-    public String getSql() {
-        return sql;
-    }
-
-    public void setSql(String sql) {
-        this.sql = sql;
-    }
-
-    public Map<String, FieldItem> getSelectFields() {
-        return selectFields;
-    }
-
-    public void setSelectFields(Map<String, FieldItem> selectFields) {
-        this.selectFields = selectFields;
-    }
-
-    public String toSql() {
-        // todo
-        return null;
-    }
-
-    public Map<String, List<TableItem>> getTableItemAliases() {
-        if (tableItemAliases == null) {
-            synchronized (SchemaItem.class) {
-                if (tableItemAliases == null) {
-                    tableItemAliases = new LinkedHashMap<>();
-                    aliasTableItems.forEach((alias, tableItem) -> {
-                        List<TableItem> aliases = tableItemAliases
-                            .computeIfAbsent(tableItem.getTableName().toLowerCase(), k -> new ArrayList<>());
-                        aliases.add(tableItem);
-                    });
-                }
-            }
-        }
-        return tableItemAliases;
-    }
-
-    public Map<String, List<FieldItem>> getColumnFields() {
-        if (columnFields == null) {
-            synchronized (SchemaItem.class) {
-                if (columnFields == null) {
-                    columnFields = new LinkedHashMap<>();
-                    getSelectFields()
-                        .forEach((fieldName, fieldItem) -> fieldItem.getColumnItems().forEach(columnItem -> {
-                            // TableItem tableItem = getAliasTableItems().get(columnItem.getOwner());
-                            // if (!tableItem.isSubQuery()) {
-                            //当数据列并非原始列时，columnName是空的，例如concat('px',id)
-                            if(columnItem.getColumnName() != null) {
-                                List<FieldItem> fieldItems = columnFields.computeIfAbsent(
-                                    columnItem.getOwner() + "." + columnItem.getColumnName(),
-                                    k -> new ArrayList<>());
-                                fieldItems.add(fieldItem);
-                            }
-                            // } else {
-                            // tableItem.getSubQueryFields().forEach(subQueryField -> {
-                            // List<FieldItem> fieldItems = columnFields.computeIfAbsent(
-                            // columnItem.getOwner() + "." + subQueryField.getColumn().getColumnName(),
-                            // k -> new ArrayList<>());
-                            // fieldItems.add(fieldItem);
-                            // });
-                            // }
-                        }));
-                }
-            }
-        }
-        return columnFields;
-    }
-
-    public boolean isAllFieldsSimple() {
-        if (allFieldsSimple == null) {
-            synchronized (SchemaItem.class) {
-                if (allFieldsSimple == null) {
-                    allFieldsSimple = true;
-
-                    for (FieldItem fieldItem : getSelectFields().values()) {
-                        if (fieldItem.isMethod() || fieldItem.isBinaryOp()) {
-                            allFieldsSimple = false;
-                            break;
-                        }
-                    }
-                }
-            }
-        }
-
-        return allFieldsSimple;
-    }
-
-    public TableItem getMainTable() {
-        if (!aliasTableItems.isEmpty()) {
-            return aliasTableItems.values().iterator().next();
-        } else {
-            return null;
-        }
-    }
-
-    public FieldItem getIdFieldItem(ESMapping mapping) {
-        if (mapping.get_id() != null) {
-            return getSelectFields().get(mapping.get_id());
-        } else {
-            return getSelectFields().get(mapping.getPk());
-        }
-    }
-
-    public static class TableItem {
-
-        private SchemaItem                               schemaItem;
-
-        private String                                   schema;
-        private String                                   tableName;
-        private String                                   alias;
-        private String                                   subQuerySql;
-        private List<FieldItem>                          subQueryFields = new ArrayList<>();
-        private List<RelationFieldsPair>                 relationFields = new ArrayList<>();
-
-        private boolean                                  main;
-        private boolean                                  subQuery;
-
-        private volatile Map<FieldItem, List<FieldItem>> relationTableFields;               // 当前表关联条件字段对应主表查询字段
-        private volatile List<FieldItem>                 relationSelectFieldItems;          // 子表所在主表的查询字段
-
-        public TableItem(SchemaItem schemaItem){
-            this.schemaItem = schemaItem;
-        }
-
-        public SchemaItem getSchemaItem() {
-            return schemaItem;
-        }
-
-        public void setSchemaItem(SchemaItem schemaItem) {
-            this.schemaItem = schemaItem;
-        }
-
-        public String getSchema() {
-            return schema;
-        }
-
-        public void setSchema(String schema) {
-            this.schema = schema;
-        }
-
-        public String getTableName() {
-            return tableName;
-        }
-
-        public void setTableName(String tableName) {
-            this.tableName = tableName;
-        }
-
-        public String getAlias() {
-            return alias;
-        }
-
-        public void setAlias(String alias) {
-            this.alias = alias;
-        }
-
-        public String getSubQuerySql() {
-            return subQuerySql;
-        }
-
-        public void setSubQuerySql(String subQuerySql) {
-            this.subQuerySql = subQuerySql;
-        }
-
-        public boolean isMain() {
-            return main;
-        }
-
-        public void setMain(boolean main) {
-            this.main = main;
-        }
-
-        public boolean isSubQuery() {
-            return subQuery;
-        }
-
-        public void setSubQuery(boolean subQuery) {
-            this.subQuery = subQuery;
-        }
-
-        public List<FieldItem> getSubQueryFields() {
-            return subQueryFields;
-        }
-
-        public void setSubQueryFields(List<FieldItem> subQueryFields) {
-            this.subQueryFields = subQueryFields;
-        }
-
-        public List<RelationFieldsPair> getRelationFields() {
-            return relationFields;
-        }
-
-        public void setRelationFields(List<RelationFieldsPair> relationFields) {
-            this.relationFields = relationFields;
-        }
-
-        public Map<FieldItem, List<FieldItem>> getRelationTableFields() {
-            if (relationTableFields == null) {
-                synchronized (SchemaItem.class) {
-                    if (relationTableFields == null) {
-                        relationTableFields = new LinkedHashMap<>();
-
-                        getRelationFields().forEach(relationFieldsPair -> {
-                            FieldItem leftFieldItem = relationFieldsPair.getLeftFieldItem();
-                            FieldItem rightFieldItem = relationFieldsPair.getRightFieldItem();
-                            FieldItem currentTableRelField = null;
-                            if (getAlias().equals(leftFieldItem.getOwner())) {
-                                currentTableRelField = leftFieldItem;
-                            } else if (getAlias().equals(rightFieldItem.getOwner())) {
-                                currentTableRelField = rightFieldItem;
-                            }
-
-                            if (currentTableRelField != null) {
-                                List<FieldItem> selectFieldItem = getSchemaItem().getColumnFields()
-                                    .get(leftFieldItem.getOwner() + "." + leftFieldItem.getColumn().getColumnName());
-                                if (selectFieldItem != null && !selectFieldItem.isEmpty()) {
-                                    relationTableFields.put(currentTableRelField, selectFieldItem);
-                                } else {
-                                    selectFieldItem = getSchemaItem().getColumnFields()
-                                        .get(rightFieldItem.getOwner() + "."
-                                             + rightFieldItem.getColumn().getColumnName());
-                                    if (selectFieldItem != null && !selectFieldItem.isEmpty()) {
-                                        relationTableFields.put(currentTableRelField, selectFieldItem);
-                                    } else {
-                                        throw new UnsupportedOperationException(
-                                            "Relation condition column must in select columns.");
-                                    }
-                                }
-                            }
-                        });
-                    }
-                }
-            }
-            return relationTableFields;
-        }
-
-        public List<FieldItem> getRelationSelectFieldItems() {
-            if (relationSelectFieldItems == null) {
-                synchronized (SchemaItem.class) {
-                    if (relationSelectFieldItems == null) {
-                        List<FieldItem> relationSelectFieldItemsTmp = new ArrayList<>();
-                        for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
-                            if (fieldItem.getOwners().contains(getAlias())) {
-                                relationSelectFieldItemsTmp.add(fieldItem);
-                            }
-                        }
-                        relationSelectFieldItems = relationSelectFieldItemsTmp;
-                    }
-                }
-            }
-            return relationSelectFieldItems;
-        }
-    }
-
-    public static class RelationFieldsPair {
-
-        private FieldItem leftFieldItem;
-        private FieldItem rightFieldItem;
-
-        public RelationFieldsPair(FieldItem leftFieldItem, FieldItem rightFieldItem){
-            this.leftFieldItem = leftFieldItem;
-            this.rightFieldItem = rightFieldItem;
-        }
-
-        public FieldItem getLeftFieldItem() {
-            return leftFieldItem;
-        }
-
-        public void setLeftFieldItem(FieldItem leftFieldItem) {
-            this.leftFieldItem = leftFieldItem;
-        }
-
-        public FieldItem getRightFieldItem() {
-            return rightFieldItem;
-        }
-
-        public void setRightFieldItem(FieldItem rightFieldItem) {
-            this.rightFieldItem = rightFieldItem;
-        }
-    }
-
-    public static class FieldItem {
-
-        private String           fieldName;
-        private String           expr;
-        private List<ColumnItem> columnItems = new ArrayList<>();
-        private List<String>     owners      = new ArrayList<>();
-
-        private boolean          method;
-        private boolean          binaryOp;
-
-        public String getFieldName() {
-            return fieldName;
-        }
-
-        public void setFieldName(String fieldName) {
-            this.fieldName = fieldName;
-        }
-
-        public String getExpr() {
-            return expr;
-        }
-
-        public void setExpr(String expr) {
-            this.expr = expr;
-        }
-
-        public List<ColumnItem> getColumnItems() {
-            return columnItems;
-        }
-
-        public void setColumnItems(List<ColumnItem> columnItems) {
-            this.columnItems = columnItems;
-        }
-
-        public boolean isMethod() {
-            return method;
-        }
-
-        public void setMethod(boolean method) {
-            this.method = method;
-        }
-
-        public boolean isBinaryOp() {
-            return binaryOp;
-        }
-
-        public void setBinaryOp(boolean binaryOp) {
-            this.binaryOp = binaryOp;
-        }
-
-        public List<String> getOwners() {
-            return owners;
-        }
-
-        public void setOwners(List<String> owners) {
-            this.owners = owners;
-        }
-
-        public void addColumn(ColumnItem columnItem) {
-            columnItems.add(columnItem);
-        }
-
-        public ColumnItem getColumn() {
-            if (!columnItems.isEmpty()) {
-                return columnItems.get(0);
-            } else {
-                return null;
-            }
-        }
-
-        public String getOwner() {
-            if (!owners.isEmpty()) {
-                return owners.get(0);
-            } else {
-                return null;
-            }
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-
-            FieldItem fieldItem = (FieldItem) o;
-
-            return fieldName != null ? fieldName.equals(fieldItem.fieldName) : fieldItem.fieldName == null;
-        }
-
-        @Override
-        public int hashCode() {
-            return fieldName != null ? fieldName.hashCode() : 0;
-        }
-    }
-
-    public static class ColumnItem {
-
-        private String owner;
-        private String columnName;
-
-        public String getOwner() {
-            return owner;
-        }
-
-        public void setOwner(String owner) {
-            this.owner = owner;
-        }
-
-        public String getColumnName() {
-            return columnName;
-        }
-
-        public void setColumnName(String columnName) {
-            this.columnName = columnName;
-        }
-    }
-}

client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/config/SqlParser.java

@@ -1,276 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.es.config;
-
-import static com.alibaba.fastsql.sql.ast.expr.SQLBinaryOperator.BooleanAnd;
-import static com.alibaba.fastsql.sql.ast.expr.SQLBinaryOperator.Equality;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.stream.Collectors;
-
-import com.alibaba.fastsql.sql.SQLUtils;
-import com.alibaba.fastsql.sql.ast.SQLExpr;
-import com.alibaba.fastsql.sql.ast.expr.*;
-import com.alibaba.fastsql.sql.ast.statement.*;
-import com.alibaba.fastsql.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock;
-import com.alibaba.fastsql.sql.dialect.mysql.parser.MySqlStatementParser;
-import com.alibaba.fastsql.sql.parser.ParserException;
-import com.alibaba.fastsql.sql.parser.SQLStatementParser;
-import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.ColumnItem;
-import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.FieldItem;
-import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.RelationFieldsPair;
-import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.TableItem;
-
-/**
- * ES同步指定sql格式解析
- *
- * @author rewerma 2018-10-26 下午03:45:49
- * @version 1.0.0
- */
-public class SqlParser {
-
-    /**
-     * 解析sql
-     *
-     * @param sql sql
-     * @return 视图对象
-     */
-    public static SchemaItem parse(String sql) {
-        try {
-            SQLStatementParser parser = new MySqlStatementParser(sql);
-            SQLSelectStatement statement = (SQLSelectStatement) parser.parseStatement();
-            MySqlSelectQueryBlock sqlSelectQueryBlock = (MySqlSelectQueryBlock) statement.getSelect().getQuery();
-
-            SchemaItem schemaItem = new SchemaItem();
-            schemaItem.setSql(SQLUtils.toMySqlString(sqlSelectQueryBlock));
-            SQLTableSource sqlTableSource = sqlSelectQueryBlock.getFrom();
-            List<TableItem> tableItems = new ArrayList<>();
-            SqlParser.visitSelectTable(schemaItem, sqlTableSource, tableItems, null);
-            tableItems.forEach(tableItem -> schemaItem.getAliasTableItems().put(tableItem.getAlias(), tableItem));
-
-            List<FieldItem> fieldItems = collectSelectQueryFields(sqlSelectQueryBlock);
-            fieldItems.forEach(fieldItem -> schemaItem.getSelectFields().put(fieldItem.getFieldName(), fieldItem));
-
-            schemaItem.init();
-
-            if (schemaItem.getAliasTableItems().isEmpty() || schemaItem.getSelectFields().isEmpty()) {
-                throw new ParserException("Parse sql error");
-            }
-            return schemaItem;
-        } catch (Exception e) {
-            throw new ParserException();
-        }
-    }
-
-    /**
-     * 归集字段
-     *
-     * @param sqlSelectQueryBlock sqlSelectQueryBlock
-     * @return 字段属性列表
-     */
-    private static List<FieldItem> collectSelectQueryFields(MySqlSelectQueryBlock sqlSelectQueryBlock) {
-        return sqlSelectQueryBlock.getSelectList().stream().map(selectItem -> {
-            FieldItem fieldItem = new FieldItem();
-            fieldItem.setFieldName(selectItem.getAlias());
-            fieldItem.setExpr(selectItem.toString());
-            visitColumn(selectItem.getExpr(), fieldItem);
-            return fieldItem;
-        }).collect(Collectors.toList());
-    }
-
-    /**
-     * 解析字段
-     *
-     * @param expr sql expr
-     * @param fieldItem 字段属性
-     */
-    private static void visitColumn(SQLExpr expr, FieldItem fieldItem) {
-        if (expr instanceof SQLIdentifierExpr) {
-            // 无owner
-            SQLIdentifierExpr identifierExpr = (SQLIdentifierExpr) expr;
-            if (fieldItem.getFieldName() == null) {
-                fieldItem.setFieldName(identifierExpr.getName());
-                fieldItem.setExpr(identifierExpr.toString());
-            }
-            ColumnItem columnItem = new ColumnItem();
-            columnItem.setColumnName(identifierExpr.getName());
-            fieldItem.getOwners().add(null);
-            fieldItem.addColumn(columnItem);
-        } else if (expr instanceof SQLPropertyExpr) {
-            // 有owner
-            SQLPropertyExpr sqlPropertyExpr = (SQLPropertyExpr) expr;
-            if (fieldItem.getFieldName() == null) {
-                fieldItem.setFieldName(sqlPropertyExpr.getName());
-                fieldItem.setExpr(sqlPropertyExpr.toString());
-            }
-            fieldItem.getOwners().add(sqlPropertyExpr.getOwnernName());
-            ColumnItem columnItem = new ColumnItem();
-            columnItem.setColumnName(sqlPropertyExpr.getName());
-            columnItem.setOwner(sqlPropertyExpr.getOwnernName());
-            fieldItem.addColumn(columnItem);
-        } else if (expr instanceof SQLMethodInvokeExpr) {
-            SQLMethodInvokeExpr methodInvokeExpr = (SQLMethodInvokeExpr) expr;
-            fieldItem.setMethod(true);
-            for (SQLExpr sqlExpr : methodInvokeExpr.getArguments()) {
-                visitColumn(sqlExpr, fieldItem);
-            }
-        } else if (expr instanceof SQLBinaryOpExpr) {
-            SQLBinaryOpExpr sqlBinaryOpExpr = (SQLBinaryOpExpr) expr;
-            fieldItem.setBinaryOp(true);
-            visitColumn(sqlBinaryOpExpr.getLeft(), fieldItem);
-            visitColumn(sqlBinaryOpExpr.getRight(), fieldItem);
-        } else if (expr instanceof SQLCaseExpr) {
-            SQLCaseExpr sqlCaseExpr = (SQLCaseExpr) expr;
-            fieldItem.setMethod(true);
-            sqlCaseExpr.getItems().forEach(item -> visitColumn(item.getConditionExpr(), fieldItem));
-        } else if (expr instanceof SQLCharExpr) {
-            SQLCharExpr sqlCharExpr = (SQLCharExpr) expr;
-            String owner = null;
-            String columnName = null;
-            if (sqlCharExpr.getParent() instanceof SQLCaseExpr.Item) {
-                owner = ((SQLPropertyExpr) ((SQLCaseExpr.Item) sqlCharExpr.getParent()).getValueExpr()).getOwnernName();
-                columnName = ((SQLPropertyExpr) ((SQLCaseExpr.Item) sqlCharExpr.getParent()).getValueExpr()).getName();
-            }
-            if (fieldItem.getFieldName() == null) {
-                fieldItem.setFieldName(columnName);
-                fieldItem.setExpr(sqlCharExpr.toString());
-            }
-            ColumnItem columnItem = new ColumnItem();
-            columnItem.setColumnName(columnName);
-            columnItem.setOwner(owner);
-            fieldItem.getOwners().add(owner);
-            fieldItem.addColumn(columnItem);
-        }
-    }
-
-    /**
-     * 解析表
-     *
-     * @param schemaItem 视图对象
-     * @param sqlTableSource sqlTableSource
-     * @param tableItems 表对象列表
-     * @param tableItemTmp 表对象(临时)
-     */
-    private static void visitSelectTable(SchemaItem schemaItem, SQLTableSource sqlTableSource,
-                                         List<TableItem> tableItems, TableItem tableItemTmp) {
-        if (sqlTableSource instanceof SQLExprTableSource) {
-            SQLExprTableSource sqlExprTableSource = (SQLExprTableSource) sqlTableSource;
-            TableItem tableItem;
-            if (tableItemTmp != null) {
-                tableItem = tableItemTmp;
-            } else {
-                tableItem = new TableItem(schemaItem);
-            }
-            tableItem.setSchema(sqlExprTableSource.getSchema());
-            tableItem.setTableName(sqlExprTableSource.getTableName());
-            if (tableItem.getAlias() == null) {
-                tableItem.setAlias(sqlExprTableSource.getAlias());
-            }
-            if (tableItems.isEmpty()) {
-                // 第一张表为主表
-                tableItem.setMain(true);
-            }
-            tableItems.add(tableItem);
-        } else if (sqlTableSource instanceof SQLJoinTableSource) {
-            SQLJoinTableSource sqlJoinTableSource = (SQLJoinTableSource) sqlTableSource;
-            SQLTableSource leftTableSource = sqlJoinTableSource.getLeft();
-            visitSelectTable(schemaItem, leftTableSource, tableItems, null);
-            SQLTableSource rightTableSource = sqlJoinTableSource.getRight();
-            TableItem rightTableItem = new TableItem(schemaItem);
-            // 解析on条件字段
-            visitOnCondition(sqlJoinTableSource.getCondition(), rightTableItem);
-            visitSelectTable(schemaItem, rightTableSource, tableItems, rightTableItem);
-
-        } else if (sqlTableSource instanceof SQLSubqueryTableSource) {
-            SQLSubqueryTableSource subQueryTableSource = (SQLSubqueryTableSource) sqlTableSource;
-            MySqlSelectQueryBlock sqlSelectQuery = (MySqlSelectQueryBlock) subQueryTableSource.getSelect().getQuery();
-            TableItem tableItem;
-            if (tableItemTmp != null) {
-                tableItem = tableItemTmp;
-            } else {
-                tableItem = new TableItem(schemaItem);
-            }
-            tableItem.setAlias(subQueryTableSource.getAlias());
-            tableItem.setSubQuerySql(SQLUtils.toMySqlString(sqlSelectQuery));
-            tableItem.setSubQuery(true);
-            tableItem.setSubQueryFields(collectSelectQueryFields(sqlSelectQuery));
-            visitSelectTable(schemaItem, sqlSelectQuery.getFrom(), tableItems, tableItem);
-        }
-    }
-
-    /**
-     * 解析on条件
-     *
-     * @param expr sql expr
-     * @param tableItem 表对象
-     */
-    private static void visitOnCondition(SQLExpr expr, TableItem tableItem) {
-        if (!(expr instanceof SQLBinaryOpExpr)) {
-            throw new UnsupportedOperationException();
-        }
-        SQLBinaryOpExpr sqlBinaryOpExpr = (SQLBinaryOpExpr) expr;
-        if (sqlBinaryOpExpr.getOperator() == BooleanAnd) {
-            visitOnCondition(sqlBinaryOpExpr.getLeft(), tableItem);
-            visitOnCondition(sqlBinaryOpExpr.getRight(), tableItem);
-        } else if (sqlBinaryOpExpr.getOperator() == Equality) {
-            FieldItem leftFieldItem = new FieldItem();
-            visitColumn(sqlBinaryOpExpr.getLeft(), leftFieldItem);
-            if (leftFieldItem.getColumnItems().size() != 1 || leftFieldItem.isMethod() || leftFieldItem.isBinaryOp()) {
-                throw new UnsupportedOperationException("Unsupported for complex of on-condition");
-            }
-            FieldItem rightFieldItem = new FieldItem();
-            visitColumn(sqlBinaryOpExpr.getRight(), rightFieldItem);
-            if (rightFieldItem.getColumnItems().size() != 1 || rightFieldItem.isMethod()
-                || rightFieldItem.isBinaryOp()) {
-                throw new UnsupportedOperationException("Unsupported for complex of on-condition");
-            }
-            tableItem.getRelationFields().add(new RelationFieldsPair(leftFieldItem, rightFieldItem));
-        } else {
-            throw new UnsupportedOperationException("Unsupported for complex of on-condition");
-        }
-    }
-
-    public static MySqlSelectQueryBlock parseSQLSelectQueryBlock(String sql) {
-        if (sql == null || "".equals(sql)) {
-            return null;
-        }
-        SQLStatementParser parser = new MySqlStatementParser(sql);
-        SQLSelectStatement statement = (SQLSelectStatement) parser.parseStatement();
-        return (MySqlSelectQueryBlock) statement.getSelect().getQuery();
-    }
-
-    public static String parse4SQLSelectItem(MySqlSelectQueryBlock sqlSelectQueryBlock) {
-        List<SQLSelectItem> selectItems = sqlSelectQueryBlock.getSelectList();
-        StringBuilder subSql = new StringBuilder();
-        int i = 0;
-        for (SQLSelectItem sqlSelectItem : selectItems) {
-            if (i != 0) {
-                subSql.append(",");
-            } else {
-                i++;
-            }
-            subSql.append(SQLUtils.toMySqlString(sqlSelectItem));
-        }
-        return subSql.toString();
-    }
-
-    public static String parse4FromTableSource(MySqlSelectQueryBlock sqlSelectQueryBlock) {
-        SQLTableSource sqlTableSource = sqlSelectQueryBlock.getFrom();
-        return SQLUtils.toMySqlString(sqlTableSource);
-    }
-
-    public static String parse4WhereItem(MySqlSelectQueryBlock sqlSelectQueryBlock) {
-        SQLExpr sqlExpr = sqlSelectQueryBlock.getWhere();
-        if (sqlExpr != null) {
-            return SQLUtils.toMySqlString(sqlExpr);
-        }
-        return null;
-    }
-
-    public static String parse4GroupBy(MySqlSelectQueryBlock sqlSelectQueryBlock) {
-        SQLSelectGroupByClause expr = sqlSelectQueryBlock.getGroupBy();
-        if (expr != null) {
-            return SQLUtils.toMySqlString(expr);
-        }
-        return null;
-    }
-}

client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/monitor/ESConfigMonitor.java

@@ -1,147 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.es.monitor;
-
-import java.io.File;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Properties;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.commons.io.filefilter.FileFilterUtils;
-import org.apache.commons.io.monitor.FileAlterationListenerAdaptor;
-import org.apache.commons.io.monitor.FileAlterationMonitor;
-import org.apache.commons.io.monitor.FileAlterationObserver;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.alibaba.druid.pool.DruidDataSource;
-import com.alibaba.otter.canal.client.adapter.config.YmlConfigBinder;
-import com.alibaba.otter.canal.client.adapter.es.ESAdapter;
-import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
-import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem;
-import com.alibaba.otter.canal.client.adapter.es.config.SqlParser;
-import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
-import com.alibaba.otter.canal.client.adapter.support.MappingConfigsLoader;
-import com.alibaba.otter.canal.client.adapter.support.Util;
-
-public class ESConfigMonitor {
-
-    private static final Logger   logger      = LoggerFactory.getLogger(ESConfigMonitor.class);
-
-    private static final String   adapterName = "es";
-
-    private ESAdapter             esAdapter;
-
-    private Properties            envProperties;
-
-    private FileAlterationMonitor fileMonitor;
-
-    public void init(ESAdapter esAdapter, Properties envProperties) {
-        this.esAdapter = esAdapter;
-        this.envProperties = envProperties;
-        File confDir = Util.getConfDirPath(adapterName);
-        try {
-            FileAlterationObserver observer = new FileAlterationObserver(confDir,
-                FileFilterUtils.and(FileFilterUtils.fileFileFilter(), FileFilterUtils.suffixFileFilter("yml")));
-            FileListener listener = new FileListener();
-            observer.addListener(listener);
-            fileMonitor = new FileAlterationMonitor(3000, observer);
-            fileMonitor.start();
-
-        } catch (Exception e) {
-            logger.error(e.getMessage(), e);
-        }
-    }
-
-    public void destroy() {
-        try {
-            fileMonitor.stop();
-        } catch (Exception e) {
-            logger.error(e.getMessage(), e);
-        }
-    }
-
-    private class FileListener extends FileAlterationListenerAdaptor {
-
-        @Override
-        public void onFileCreate(File file) {
-            super.onFileCreate(file);
-            try {
-                // 加载新增的配置文件
-                String configContent = MappingConfigsLoader.loadConfig(adapterName + File.separator + file.getName());
-                ESSyncConfig config = YmlConfigBinder
-                    .bindYmlToObj(null, configContent, ESSyncConfig.class, null, envProperties);
-                if (config != null) {
-                    config.validate();
-                    addConfigToCache(file, config);
-                    logger.info("Add a new es mapping config: {} to canal adapter", file.getName());
-                }
-            } catch (Exception e) {
-                logger.error(e.getMessage(), e);
-            }
-        }
-
-        @Override
-        public void onFileChange(File file) {
-            super.onFileChange(file);
-
-            try {
-                if (esAdapter.getEsSyncConfig().containsKey(file.getName())) {
-                    // 加载配置文件
-                    String configContent = MappingConfigsLoader
-                        .loadConfig(adapterName + File.separator + file.getName());
-                    if (configContent == null) {
-                        onFileDelete(file);
-                        return;
-                    }
-                    ESSyncConfig config = YmlConfigBinder
-                        .bindYmlToObj(null, configContent, ESSyncConfig.class, null, envProperties);
-                    if (config == null) {
-                        return;
-                    }
-                    config.validate();
-                    if (esAdapter.getEsSyncConfig().containsKey(file.getName())) {
-                        deleteConfigFromCache(file);
-                    }
-                    addConfigToCache(file, config);
-
-                    logger.info("Change a es mapping config: {} of canal adapter", file.getName());
-                }else{
-                    onFileCreate(file);
-                }
-            } catch (Exception e) {
-                logger.error(e.getMessage(), e);
-            }
-        }
-
-        @Override
-        public void onFileDelete(File file) {
-            super.onFileDelete(file);
-
-            try {
-                if (esAdapter.getEsSyncConfig().containsKey(file.getName())) {
-                    deleteConfigFromCache(file);
-
-                    logger.info("Delete a es mapping config: {} of canal adapter", file.getName());
-                }
-            } catch (Exception e) {
-                logger.error(e.getMessage(), e);
-            }
-        }
-
-        private void addConfigToCache(File file, ESSyncConfig config) {
-            esAdapter.getEsSyncConfig().put(file.getName(), config);
-            esAdapter.addSyncConfigToCache(file.getName(),config);
-        }
-
-        private void deleteConfigFromCache(File file) {
-            esAdapter.getEsSyncConfig().remove(file.getName());
-            for (Map<String, ESSyncConfig> configMap : esAdapter.getDbTableEsSyncConfig().values()) {
-                if (configMap != null) {
-                    configMap.remove(file.getName());
-                }
-            }
-
-        }
-    }
-}

client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESEtlService.java

@@ -1,217 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.es.service;
-
-import java.sql.SQLException;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
-
-import javax.sql.DataSource;
-
-import org.apache.commons.lang.StringUtils;
-import org.elasticsearch.action.bulk.BulkItemResponse;
-import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.search.SearchHit;
-
-import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
-import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping;
-import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.FieldItem;
-import com.alibaba.otter.canal.client.adapter.es.support.ESConnection;
-import com.alibaba.otter.canal.client.adapter.es.support.ESConnection.ESBulkRequest;
-import com.alibaba.otter.canal.client.adapter.es.support.ESConnection.ESIndexRequest;
-import com.alibaba.otter.canal.client.adapter.es.support.ESConnection.ESSearchRequest;
-import com.alibaba.otter.canal.client.adapter.es.support.ESConnection.ESUpdateRequest;
-import com.alibaba.otter.canal.client.adapter.es.support.ESTemplate;
-import com.alibaba.otter.canal.client.adapter.support.AbstractEtlService;
-import com.alibaba.otter.canal.client.adapter.support.AdapterConfig;
-import com.alibaba.otter.canal.client.adapter.support.EtlResult;
-import com.alibaba.otter.canal.client.adapter.support.Util;
-
-/**
- * ES ETL Service
- *
- * @author rewerma 2018-11-01
- * @version 1.0.0
- */
-public class ESEtlService extends AbstractEtlService {
-
-    private ESConnection esConnection;
-    private ESTemplate   esTemplate;
-    private ESSyncConfig config;
-
-    public ESEtlService(ESConnection esConnection, ESSyncConfig config){
-        super("ES", config);
-        this.esConnection = esConnection;
-        this.esTemplate = new ESTemplate(esConnection);
-        this.config = config;
-    }
-
-    public EtlResult importData(List<String> params) {
-        ESMapping mapping = config.getEsMapping();
-        logger.info("start etl to import data to index: {}", mapping.get_index());
-        String sql = mapping.getSql();
-        return importData(sql, params);
-    }
-
-    private void processFailBulkResponse(BulkResponse bulkResponse) {
-        for (BulkItemResponse response : bulkResponse.getItems()) {
-            if (!response.isFailed()) {
-                continue;
-            }
-
-            if (response.getFailure().getStatus() == RestStatus.NOT_FOUND) {
-                logger.warn(response.getFailureMessage());
-            } else {
-                logger.error("全量导入数据有误 {}", response.getFailureMessage());
-                throw new RuntimeException("全量数据 etl 异常: " + response.getFailureMessage());
-            }
-        }
-    }
-
-    protected boolean executeSqlImport(DataSource ds, String sql, List<Object> values,
-                                       AdapterConfig.AdapterMapping adapterMapping, AtomicLong impCount,
-                                       List<String> errMsg) {
-        try {
-            ESMapping mapping = (ESMapping) adapterMapping;
-            Util.sqlRS(ds, sql, values, rs -> {
-                int count = 0;
-                try {
-                    ESBulkRequest esBulkRequest = this.esConnection.new ESBulkRequest();
-
-                    long batchBegin = System.currentTimeMillis();
-                    while (rs.next()) {
-                        Map<String, Object> esFieldData = new LinkedHashMap<>();
-                        Object idVal = null;
-                        for (FieldItem fieldItem : mapping.getSchemaItem().getSelectFields().values()) {
-
-                            String fieldName = fieldItem.getFieldName();
-                            if (mapping.getSkips().contains(fieldName)) {
-                                continue;
-                            }
-
-                            // 如果是主键字段则不插入
-                            if (fieldItem.getFieldName().equals(mapping.get_id())) {
-                                idVal = esTemplate.getValFromRS(mapping, rs, fieldName, fieldName);
-                            } else {
-                                Object val = esTemplate.getValFromRS(mapping, rs, fieldName, fieldName);
-                                esFieldData.put(Util.cleanColumn(fieldName), val);
-                            }
-
-                        }
-
-                        if (!mapping.getRelations().isEmpty()) {
-                            mapping.getRelations().forEach((relationField, relationMapping) -> {
-                                Map<String, Object> relations = new HashMap<>();
-                                relations.put("name", relationMapping.getName());
-                                if (StringUtils.isNotEmpty(relationMapping.getParent())) {
-                                    FieldItem parentFieldItem = mapping.getSchemaItem()
-                                        .getSelectFields()
-                                        .get(relationMapping.getParent());
-                                    Object parentVal;
-                                    try {
-                                        parentVal = esTemplate.getValFromRS(mapping,
-                                            rs,
-                                            parentFieldItem.getFieldName(),
-                                            parentFieldItem.getFieldName());
-                                    } catch (SQLException e) {
-                                        throw new RuntimeException(e);
-                                    }
-                                    if (parentVal != null) {
-                                        relations.put("parent", parentVal.toString());
-                                        esFieldData.put("$parent_routing", parentVal.toString());
-
-                                    }
-                                }
-                                esFieldData.put(Util.cleanColumn(relationField), relations);
-                            });
-                        }
-
-                        if (idVal != null) {
-                            String parentVal = (String) esFieldData.remove("$parent_routing");
-                            if (mapping.isUpsert()) {
-                                ESUpdateRequest esUpdateRequest = this.esConnection.new ESUpdateRequest(
-                                    mapping.get_index(),
-                                    mapping.get_type(),
-                                    idVal.toString()).setDoc(esFieldData).setDocAsUpsert(true);
-
-                                if (StringUtils.isNotEmpty(parentVal)) {
-                                    esUpdateRequest.setRouting(parentVal);
-                                }
-
-                                esBulkRequest.add(esUpdateRequest);
-                            } else {
-                                ESIndexRequest esIndexRequest = this.esConnection.new ESIndexRequest(mapping
-                                    .get_index(), mapping.get_type(), idVal.toString()).setSource(esFieldData);
-                                if (StringUtils.isNotEmpty(parentVal)) {
-                                    esIndexRequest.setRouting(parentVal);
-                                }
-                                esBulkRequest.add(esIndexRequest);
-                            }
-                        } else {
-                            idVal = esFieldData.get(mapping.getPk());
-                            ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index(),
-                                mapping.get_type()).setQuery(QueryBuilders.termQuery(mapping.getPk(), idVal))
-                                    .size(10000);
-                            SearchResponse response = esSearchRequest.getResponse();
-                            for (SearchHit hit : response.getHits()) {
-                                ESUpdateRequest esUpdateRequest = this.esConnection.new ESUpdateRequest(mapping
-                                    .get_index(), mapping.get_type(), hit.getId()).setDoc(esFieldData);
-                                esBulkRequest.add(esUpdateRequest);
-                            }
-                        }
-
-                        if (esBulkRequest.numberOfActions() % mapping.getCommitBatch() == 0
-                            && esBulkRequest.numberOfActions() > 0) {
-                            long esBatchBegin = System.currentTimeMillis();
-                            BulkResponse rp = esBulkRequest.bulk();
-                            if (rp.hasFailures()) {
-                                this.processFailBulkResponse(rp);
-                            }
-
-                            if (logger.isTraceEnabled()) {
-                                logger.trace("全量数据批量导入批次耗时: {}, es执行时间: {}, 批次大小: {}, index; {}",
-                                    (System.currentTimeMillis() - batchBegin),
-                                    (System.currentTimeMillis() - esBatchBegin),
-                                    esBulkRequest.numberOfActions(),
-                                    mapping.get_index());
-                            }
-                            batchBegin = System.currentTimeMillis();
-                            esBulkRequest.resetBulk();
-                        }
-                        count++;
-                        impCount.incrementAndGet();
-                    }
-
-                    if (esBulkRequest.numberOfActions() > 0) {
-                        long esBatchBegin = System.currentTimeMillis();
-                        BulkResponse rp = esBulkRequest.bulk();
-                        if (rp.hasFailures()) {
-                            this.processFailBulkResponse(rp);
-                        }
-                        if (logger.isTraceEnabled()) {
-                            logger.trace("全量数据批量导入最后批次耗时: {}, es执行时间: {}, 批次大小: {}, index; {}",
-                                (System.currentTimeMillis() - batchBegin),
-                                (System.currentTimeMillis() - esBatchBegin),
-                                esBulkRequest.numberOfActions(),
-                                mapping.get_index());
-                        }
-                    }
-                } catch (Exception e) {
-                    logger.error(e.getMessage(), e);
-                    errMsg.add(mapping.get_index() + " etl failed! ==>" + e.getMessage());
-                    throw new RuntimeException(e);
-                }
-                return count;
-            });
-
-            return true;
-        } catch (Exception e) {
-            logger.error(e.getMessage(), e);
-            return false;
-        }
-    }
-}

client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/service/ESSyncService.java

@@ -1,877 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.es.service;
-
-import java.util.*;
-
-import javax.sql.DataSource;
-
-import com.alibaba.fastsql.sql.dialect.mysql.ast.statement.MySqlSelectQueryBlock;
-import com.alibaba.otter.canal.client.adapter.es.config.SqlParser;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.alibaba.fastjson.JSON;
-import com.alibaba.fastjson.serializer.SerializerFeature;
-import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
-import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping;
-import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem;
-import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.ColumnItem;
-import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.FieldItem;
-import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.TableItem;
-import com.alibaba.otter.canal.client.adapter.es.support.ESSyncUtil;
-import com.alibaba.otter.canal.client.adapter.es.support.ESTemplate;
-import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
-import com.alibaba.otter.canal.client.adapter.support.Dml;
-import com.alibaba.otter.canal.client.adapter.support.Util;
-
-/**
- * ES 同步 Service
- *
- * @author rewerma 2018-11-01
- * @version 1.0.0
- */
-public class ESSyncService {
-
-    private static Logger logger = LoggerFactory.getLogger(ESSyncService.class);
-
-    private ESTemplate    esTemplate;
-
-    public ESSyncService(ESTemplate esTemplate){
-        this.esTemplate = esTemplate;
-    }
-
-    public void sync(Collection<ESSyncConfig> esSyncConfigs, Dml dml) {
-        long begin = System.currentTimeMillis();
-        if (esSyncConfigs != null) {
-            if (logger.isTraceEnabled()) {
-                logger.trace("Destination: {}, database:{}, table:{}, type:{}, affected index count: {}",
-                    dml.getDestination(),
-                    dml.getDatabase(),
-                    dml.getTable(),
-                    dml.getType(),
-                    esSyncConfigs.size());
-            }
-
-            for (ESSyncConfig config : esSyncConfigs) {
-                if (logger.isTraceEnabled()) {
-                    logger.trace("Prepared to sync index: {}, destination: {}",
-                        config.getEsMapping().get_index(),
-                        dml.getDestination());
-                }
-                this.sync(config, dml);
-                if (logger.isTraceEnabled()) {
-                    logger.trace("Sync completed: {}, destination: {}",
-                        config.getEsMapping().get_index(),
-                        dml.getDestination());
-                }
-            }
-            if (logger.isTraceEnabled()) {
-                logger.trace("Sync elapsed time: {} ms, affected indexes count：{}, destination: {}",
-                    (System.currentTimeMillis() - begin),
-                    esSyncConfigs.size(),
-                    dml.getDestination());
-            }
-            if (logger.isDebugEnabled()) {
-                StringBuilder configIndexes = new StringBuilder();
-                esSyncConfigs
-                    .forEach(esSyncConfig -> configIndexes.append(esSyncConfig.getEsMapping().get_index()).append(" "));
-                logger.debug("DML: {} \nAffected indexes: {}",
-                    JSON.toJSONString(dml, SerializerFeature.WriteMapNullValue),
-                    configIndexes.toString());
-            }
-        }
-    }
-
-    public void sync(ESSyncConfig config, Dml dml) {
-        try {
-            // 如果是按时间戳定时更新则返回
-            if (config.getEsMapping().isSyncByTimestamp()) {
-                return;
-            }
-
-            long begin = System.currentTimeMillis();
-
-            String type = dml.getType();
-            if (type != null && type.equalsIgnoreCase("INSERT")) {
-                insert(config, dml);
-            } else if (type != null && type.equalsIgnoreCase("UPDATE")) {
-                update(config, dml);
-            } else if (type != null && type.equalsIgnoreCase("DELETE")) {
-                delete(config, dml);
-            } else {
-                return;
-            }
-
-            if (logger.isTraceEnabled()) {
-                logger.trace("Sync elapsed time: {} ms,destination: {}, es index: {}",
-                    (System.currentTimeMillis() - begin),
-                    dml.getDestination(),
-                    config.getEsMapping().get_index());
-            }
-        } catch (Throwable e) {
-            logger.error("sync error, es index: {}, DML : {}", config.getEsMapping().get_index(), dml);
-            throw new RuntimeException(e);
-        }
-    }
-
-    /**
-     * 插入操作dml
-     *
-     * @param config es配置
-     * @param dml dml数据
-     */
-    private void insert(ESSyncConfig config, Dml dml) {
-        List<Map<String, Object>> dataList = dml.getData();
-        if (dataList == null || dataList.isEmpty()) {
-            return;
-        }
-        SchemaItem schemaItem = config.getEsMapping().getSchemaItem();
-        for (Map<String, Object> data : dataList) {
-            if (data == null || data.isEmpty()) {
-                continue;
-            }
-
-            if (schemaItem.getAliasTableItems().size() == 1 && schemaItem.isAllFieldsSimple()) {
-                // ------单表 & 所有字段都为简单字段------
-                singleTableSimpleFiledInsert(config, dml, data);
-            } else {
-                // ------是主表 查询sql来插入------
-                if (schemaItem.getMainTable().getTableName().equalsIgnoreCase(dml.getTable())) {
-                    mainTableInsert(config, dml, data);
-                }
-
-                // 从表的操作
-                for (TableItem tableItem : schemaItem.getAliasTableItems().values()) {
-                    if (tableItem.isMain()) {
-                        continue;
-                    }
-                    if (!tableItem.getTableName().equals(dml.getTable())) {
-                        continue;
-                    }
-                    // 关联条件出现在主表查询条件是否为简单字段
-                    boolean allFieldsSimple = true;
-                    for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) {
-                        if (fieldItem.isMethod() || fieldItem.isBinaryOp()) {
-                            allFieldsSimple = false;
-                            break;
-                        }
-                    }
-                    // 所有查询字段均为简单字段
-                    if (allFieldsSimple) {
-                        // 不是子查询
-                        if (!tableItem.isSubQuery()) {
-                            // ------关联表简单字段插入------
-                            Map<String, Object> esFieldData = new LinkedHashMap<>();
-                            for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) {
-                                Object value = esTemplate.getValFromData(config.getEsMapping(),
-                                    data,
-                                    fieldItem.getFieldName(),
-                                    fieldItem.getColumn().getColumnName());
-                                esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), value);
-                            }
-
-                            joinTableSimpleFieldOperation(config, dml, data, tableItem, esFieldData);
-                        } else {
-                            // ------关联子表简单字段插入------
-                            subTableSimpleFieldOperation(config, dml, data, null, tableItem);
-                        }
-                    } else {
-                        // ------关联子表复杂字段插入 执行全sql更新es------
-                        wholeSqlOperation(config, dml, data, null, tableItem);
-                    }
-                }
-            }
-        }
-    }
-
-    /**
-     * 更新操作dml
-     *
-     * @param config es配置
-     * @param dml dml数据
-     */
-    private void update(ESSyncConfig config, Dml dml) {
-        List<Map<String, Object>> dataList = dml.getData();
-        List<Map<String, Object>> oldList = dml.getOld();
-        if (dataList == null || dataList.isEmpty() || oldList == null || oldList.isEmpty()) {
-            return;
-        }
-        SchemaItem schemaItem = config.getEsMapping().getSchemaItem();
-        int i = 0;
-        for (Map<String, Object> data : dataList) {
-            Map<String, Object> old = oldList.get(i);
-            if (data == null || data.isEmpty() || old == null || old.isEmpty()) {
-                continue;
-            }
-
-            if (schemaItem.getAliasTableItems().size() == 1 && schemaItem.isAllFieldsSimple()) {
-                // ------单表 & 所有字段都为简单字段------
-                singleTableSimpleFiledUpdate(config, schemaItem.getMainTable().getAlias(), dml, data, old);
-            } else {
-                // ------主表 查询sql来更新------
-                if (schemaItem.getMainTable().getTableName().equalsIgnoreCase(dml.getTable())) {
-                    ESMapping mapping = config.getEsMapping();
-                    String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
-                    FieldItem idFieldItem = schemaItem.getSelectFields().get(idFieldName);
-
-                    boolean idFieldSimple = true;
-                    if (idFieldItem.isMethod() || idFieldItem.isBinaryOp()) {
-                        idFieldSimple = false;
-                    }
-
-                    boolean allUpdateFieldSimple = true;
-                    out: for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
-                        for (ColumnItem columnItem : fieldItem.getColumnItems()) {
-                            if (schemaItem.getMainTable().getAlias().equals(columnItem.getOwner())) { // 如果是主表的字段
-                                if (old.containsKey(columnItem.getColumnName())) {
-                                    if (fieldItem.isMethod() || fieldItem.isBinaryOp()) {
-                                        allUpdateFieldSimple = false;
-                                        break out;
-                                    }
-                                }
-                            }
-                        }
-                    }
-
-                    // 不支持主键更新!!
-
-                    // 判断是否有外键更新
-                    boolean fkChanged = false;
-                    for (TableItem tableItem : schemaItem.getAliasTableItems().values()) {
-                        if (tableItem.isMain()) {
-                            continue;
-                        }
-                        boolean changed = false;
-                        for (List<FieldItem> fieldItems : tableItem.getRelationTableFields().values()) {
-                            for (FieldItem fieldItem : fieldItems) {
-                                if (old.containsKey(fieldItem.getColumn().getColumnName())) {
-                                    fkChanged = true;
-                                    changed = true;
-                                    break;
-                                }
-                            }
-                        }
-                        // 如果外键有修改,则更新所对应该表的所有查询条件数据
-                        if (changed) {
-                            for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) {
-                                fieldItem.getColumnItems()
-                                    .forEach(columnItem -> old.put(columnItem.getColumnName(), null));
-                            }
-                        }
-                    }
-
-                    // 判断主键和所更新的字段是否全为简单字段
-                    if (idFieldSimple && allUpdateFieldSimple && !fkChanged) {
-                        singleTableSimpleFiledUpdate(config, schemaItem.getMainTable().getAlias(), dml, data, old);
-                    } else {
-                        mainTableUpdate(config, dml, data, old);
-                    }
-                }
-
-                // 从表的操作
-                for (TableItem tableItem : schemaItem.getAliasTableItems().values()) {
-                    if (tableItem.isMain()) {
-                        continue;
-                    }
-                    if (!tableItem.getTableName().equals(dml.getTable())) {
-                        continue;
-                    }
-
-                    // 关联条件出现在主表查询条件是否为简单字段
-                    boolean allFieldsSimple = true;
-                    for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) {
-                        if (fieldItem.isMethod() || fieldItem.isBinaryOp()) {
-                            allFieldsSimple = false;
-                            break;
-                        }
-                    }
-
-                    // 所有查询字段均为简单字段
-                    if (allFieldsSimple) {
-                        // 不是子查询
-                        if (!tableItem.isSubQuery()) {
-                            // ------关联表简单字段更新------
-                            Map<String, Object> esFieldData = new LinkedHashMap<>();
-                            for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) {
-                                if (old.containsKey(fieldItem.getColumn().getColumnName())) {
-                                    Object value = esTemplate.getValFromData(config.getEsMapping(),
-                                        data,
-                                        fieldItem.getFieldName(),
-                                        fieldItem.getColumn().getColumnName());
-                                    esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), value);
-                                }
-                            }
-                            joinTableSimpleFieldOperation(config, dml, data, tableItem, esFieldData);
-                        } else {
-                            // ------关联子表简单字段更新------
-                            subTableSimpleFieldOperation(config, dml, data, old, tableItem);
-                        }
-                    } else {
-                        // ------关联子表复杂字段更新 执行全sql更新es------
-                        wholeSqlOperation(config, dml, data, old, tableItem);
-                    }
-                }
-            }
-
-            i++;
-        }
-    }
-
-    /**
-     * 删除操作dml
-     *
-     * @param config es配置
-     * @param dml dml数据
-     */
-    private void delete(ESSyncConfig config, Dml dml) {
-        List<Map<String, Object>> dataList = dml.getData();
-        if (dataList == null || dataList.isEmpty()) {
-            return;
-        }
-        SchemaItem schemaItem = config.getEsMapping().getSchemaItem();
-
-        for (Map<String, Object> data : dataList) {
-            if (data == null || data.isEmpty()) {
-                continue;
-            }
-
-            ESMapping mapping = config.getEsMapping();
-
-            // ------是主表------
-            if (schemaItem.getMainTable().getTableName().equalsIgnoreCase(dml.getTable())) {
-                if (mapping.get_id() != null) {
-                    FieldItem idFieldItem = schemaItem.getIdFieldItem(mapping);
-                    // 主键为简单字段
-                    if (!idFieldItem.isMethod() && !idFieldItem.isBinaryOp()) {
-                        Object idVal = esTemplate.getValFromData(mapping,
-                            data,
-                            idFieldItem.getFieldName(),
-                            idFieldItem.getColumn().getColumnName());
-
-                        if (logger.isTraceEnabled()) {
-                            logger.trace("Main table delete es index, destination:{}, table: {}, index: {}, id: {}",
-                                config.getDestination(),
-                                dml.getTable(),
-                                mapping.get_index(),
-                                idVal);
-                        }
-                        esTemplate.delete(mapping, idVal, null);
-                    } else {
-                        // ------主键带函数, 查询sql获取主键删除------
-                        // FIXME 删除时反查sql为空记录, 无法获获取 id field 值
-                        mainTableDelete(config, dml, data);
-                    }
-                } else {
-                    FieldItem pkFieldItem = schemaItem.getIdFieldItem(mapping);
-                    if (!pkFieldItem.isMethod() && !pkFieldItem.isBinaryOp()) {
-                        Map<String, Object> esFieldData = new LinkedHashMap<>();
-                        Object pkVal = esTemplate.getESDataFromDmlData(mapping, data, esFieldData);
-
-                        if (logger.isTraceEnabled()) {
-                            logger.trace("Main table delete es index, destination:{}, table: {}, index: {}, pk: {}",
-                                config.getDestination(),
-                                dml.getTable(),
-                                mapping.get_index(),
-                                pkVal);
-                        }
-                        esFieldData.remove(pkFieldItem.getFieldName());
-                        esFieldData.keySet().forEach(key -> esFieldData.put(key, null));
-                        esTemplate.delete(mapping, pkVal, esFieldData);
-                    } else {
-                        // ------主键带函数, 查询sql获取主键删除------
-                        mainTableDelete(config, dml, data);
-                    }
-                }
-
-            }
-
-            // 从表的操作
-            for (TableItem tableItem : schemaItem.getAliasTableItems().values()) {
-                if (tableItem.isMain()) {
-                    continue;
-                }
-                if (!tableItem.getTableName().equals(dml.getTable())) {
-                    continue;
-                }
-
-                // 关联条件出现在主表查询条件是否为简单字段
-                boolean allFieldsSimple = true;
-                for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) {
-                    if (fieldItem.isMethod() || fieldItem.isBinaryOp()) {
-                        allFieldsSimple = false;
-                        break;
-                    }
-                }
-
-                // 所有查询字段均为简单字段
-                if (allFieldsSimple) {
-                    // 不是子查询
-                    if (!tableItem.isSubQuery()) {
-                        // ------关联表简单字段更新为null------
-                        Map<String, Object> esFieldData = new LinkedHashMap<>();
-                        for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) {
-                            esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), null);
-                        }
-                        joinTableSimpleFieldOperation(config, dml, data, tableItem, esFieldData);
-                    } else {
-                        // ------关联子表简单字段更新------
-                        subTableSimpleFieldOperation(config, dml, data, null, tableItem);
-                    }
-                } else {
-                    // ------关联子表复杂字段更新 执行全sql更新es------
-                    wholeSqlOperation(config, dml, data, null, tableItem);
-                }
-            }
-        }
-    }
-
-    /**
-     * 单表简单字段insert
-     *
-     * @param config es配置
-     * @param dml dml信息
-     * @param data 单行dml数据
-     */
-    private void singleTableSimpleFiledInsert(ESSyncConfig config, Dml dml, Map<String, Object> data) {
-        ESMapping mapping = config.getEsMapping();
-        Map<String, Object> esFieldData = new LinkedHashMap<>();
-        Object idVal = esTemplate.getESDataFromDmlData(mapping, data, esFieldData);
-
-        if (logger.isTraceEnabled()) {
-            logger.trace("Single table insert to es index, destination:{}, table: {}, index: {}, id: {}",
-                config.getDestination(),
-                dml.getTable(),
-                mapping.get_index(),
-                idVal);
-        }
-        esTemplate.insert(mapping, idVal, esFieldData);
-    }
-
-    /**
-     * 主表(单表)复杂字段insert
-     *
-     * @param config es配置
-     * @param dml dml信息
-     * @param data 单行dml数据
-     */
-    private void mainTableInsert(ESSyncConfig config, Dml dml, Map<String, Object> data) {
-        ESMapping mapping = config.getEsMapping();
-        String sql = mapping.getSql();
-        String condition = ESSyncUtil.pkConditionSql(mapping, data);
-        sql = ESSyncUtil.appendCondition(sql, condition);
-        DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
-        if (logger.isTraceEnabled()) {
-            logger.trace("Main table insert to es index by query sql, destination:{}, table: {}, index: {}, sql: {}",
-                config.getDestination(),
-                dml.getTable(),
-                mapping.get_index(),
-                sql.replace("\n", " "));
-        }
-        Util.sqlRS(ds, sql, rs -> {
-            try {
-                while (rs.next()) {
-                    Map<String, Object> esFieldData = new LinkedHashMap<>();
-                    Object idVal = esTemplate.getESDataFromRS(mapping, rs, esFieldData);
-
-                    if (logger.isTraceEnabled()) {
-                        logger.trace(
-                            "Main table insert to es index by query sql, destination:{}, table: {}, index: {}, id: {}",
-                            config.getDestination(),
-                            dml.getTable(),
-                            mapping.get_index(),
-                            idVal);
-                    }
-                    esTemplate.insert(mapping, idVal, esFieldData);
-                }
-            } catch (Exception e) {
-                throw new RuntimeException(e);
-            }
-            return 0;
-        });
-    }
-
-    private void mainTableDelete(ESSyncConfig config, Dml dml, Map<String, Object> data) {
-        ESMapping mapping = config.getEsMapping();
-        String sql = mapping.getSql();
-        String condition = ESSyncUtil.pkConditionSql(mapping, data);
-        sql = ESSyncUtil.appendCondition(sql, condition);
-        DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
-        if (logger.isTraceEnabled()) {
-            logger.trace("Main table delete es index by query sql, destination:{}, table: {}, index: {}, sql: {}",
-                config.getDestination(),
-                dml.getTable(),
-                mapping.get_index(),
-                sql.replace("\n", " "));
-        }
-        Util.sqlRS(ds, sql, rs -> {
-            try {
-                Map<String, Object> esFieldData = null;
-                if (mapping.getPk() != null) {
-                    esFieldData = new LinkedHashMap<>();
-                    esTemplate.getESDataFromDmlData(mapping, data, esFieldData);
-                    esFieldData.remove(mapping.getPk());
-                    for (String key : esFieldData.keySet()) {
-                        esFieldData.put(Util.cleanColumn(key), null);
-                    }
-                }
-                while (rs.next()) {
-                    Object idVal = esTemplate.getIdValFromRS(mapping, rs);
-
-                    if (logger.isTraceEnabled()) {
-                        logger.trace(
-                            "Main table delete to es index by query sql, destination:{}, table: {}, index: {}, id: {}",
-                            config.getDestination(),
-                            dml.getTable(),
-                            mapping.get_index(),
-                            idVal);
-                    }
-                    esTemplate.delete(mapping, idVal, esFieldData);
-                }
-            } catch (Exception e) {
-                throw new RuntimeException(e);
-            }
-            return 0;
-        });
-    }
-
-    /**
-     * 关联表主表简单字段operation
-     *
-     * @param config es配置
-     * @param dml dml信息
-     * @param data 单行dml数据
-     * @param tableItem 当前表配置
-     */
-    private void joinTableSimpleFieldOperation(ESSyncConfig config, Dml dml, Map<String, Object> data,
-                                               TableItem tableItem, Map<String, Object> esFieldData) {
-        ESMapping mapping = config.getEsMapping();
-
-        Map<String, Object> paramsTmp = new LinkedHashMap<>();
-        for (Map.Entry<FieldItem, List<FieldItem>> entry : tableItem.getRelationTableFields().entrySet()) {
-            for (FieldItem fieldItem : entry.getValue()) {
-                if (fieldItem.getColumnItems().size() == 1) {
-                    Object value = esTemplate.getValFromData(mapping,
-                        data,
-                        fieldItem.getFieldName(),
-                        entry.getKey().getColumn().getColumnName());
-
-                    String fieldName = fieldItem.getFieldName();
-                    // 判断是否是主键
-                    if (fieldName.equals(mapping.get_id())) {
-                        fieldName = "_id";
-                    }
-                    paramsTmp.put(fieldName, value);
-                }
-            }
-        }
-
-        if (logger.isDebugEnabled()) {
-            logger.trace("Join table update es index by foreign key, destination:{}, table: {}, index: {}",
-                config.getDestination(),
-                dml.getTable(),
-                mapping.get_index());
-        }
-        esTemplate.updateByQuery(config, paramsTmp, esFieldData);
-    }
-
-    /**
-     * 关联子查询, 主表简单字段operation
-     *
-     * @param config es配置
-     * @param dml dml信息
-     * @param data 单行dml数据
-     * @param old 单行old数据
-     * @param tableItem 当前表配置
-     */
-    private void subTableSimpleFieldOperation(ESSyncConfig config, Dml dml, Map<String, Object> data,
-                                              Map<String, Object> old, TableItem tableItem) {
-        ESMapping mapping = config.getEsMapping();
-
-        MySqlSelectQueryBlock queryBlock = SqlParser.parseSQLSelectQueryBlock(tableItem.getSubQuerySql());
-        StringBuilder sql = new StringBuilder();
-        sql.append("SELECT ")
-            .append(SqlParser.parse4SQLSelectItem(queryBlock))
-            .append(" FROM ")
-            .append(SqlParser.parse4FromTableSource(queryBlock));
-
-        String whereSql = SqlParser.parse4WhereItem(queryBlock);
-        if (whereSql != null) {
-            sql.append(" WHERE ").append(whereSql);
-        } else {
-            sql.append(" WHERE 1=1 ");
-        }
-
-        List<Object> values = new ArrayList<>();
-        for (FieldItem fkFieldItem : tableItem.getRelationTableFields().keySet()) {
-            String columnName = fkFieldItem.getColumn().getColumnName();
-            Object value = esTemplate.getValFromData(mapping, data, fkFieldItem.getFieldName(), columnName);
-            sql.append(" AND ").append(columnName).append("=? ");
-            values.add(value);
-        }
-
-        String groupSql = SqlParser.parse4GroupBy(queryBlock);
-        if (groupSql != null) {
-            sql.append(groupSql);
-        }
-
-        DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
-        if (logger.isTraceEnabled()) {
-            logger.trace("Join table update es index by query sql, destination:{}, table: {}, index: {}, sql: {}",
-                config.getDestination(),
-                dml.getTable(),
-                mapping.get_index(),
-                sql.toString().replace("\n", " "));
-        }
-        Util.sqlRS(ds, sql.toString(), values, rs -> {
-            try {
-                while (rs.next()) {
-                    Map<String, Object> esFieldData = new LinkedHashMap<>();
-
-                    for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) {
-                        if (old != null) {
-                            out: for (FieldItem fieldItem1 : tableItem.getSubQueryFields()) {
-                                for (ColumnItem columnItem0 : fieldItem.getColumnItems()) {
-                                    if (fieldItem1.getFieldName().equals(columnItem0.getColumnName()))
-                                        for (ColumnItem columnItem : fieldItem1.getColumnItems()) {
-                                            if (old.containsKey(columnItem.getColumnName())) {
-                                                Object val = esTemplate.getValFromRS(mapping,
-                                                    rs,
-                                                    fieldItem.getFieldName(),
-                                                    fieldItem.getColumn().getColumnName());
-                                                esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), val);
-                                                break out;
-                                            }
-                                        }
-                                }
-                            }
-                        } else {
-                            Object val = esTemplate.getValFromRS(mapping,
-                                rs,
-                                fieldItem.getFieldName(),
-                                fieldItem.getColumn().getColumnName());
-                            esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), val);
-                        }
-                    }
-
-                    Map<String, Object> paramsTmp = new LinkedHashMap<>();
-                    for (Map.Entry<FieldItem, List<FieldItem>> entry : tableItem.getRelationTableFields().entrySet()) {
-                        for (FieldItem fieldItem : entry.getValue()) {
-                            if (fieldItem.getColumnItems().size() == 1) {
-                                Object value = esTemplate.getValFromRS(mapping,
-                                    rs,
-                                    fieldItem.getFieldName(),
-                                    entry.getKey().getColumn().getColumnName());
-                                String fieldName = fieldItem.getFieldName();
-                                // 判断是否是主键
-                                if (fieldName.equals(mapping.get_id())) {
-                                    fieldName = "_id";
-                                }
-                                paramsTmp.put(fieldName, value);
-                            }
-                        }
-                    }
-
-                    if (logger.isDebugEnabled()) {
-                        logger.trace("Join table update es index by query sql, destination:{}, table: {}, index: {}",
-                            config.getDestination(),
-                            dml.getTable(),
-                            mapping.get_index());
-                    }
-                    esTemplate.updateByQuery(config, paramsTmp, esFieldData);
-                }
-            } catch (Exception e) {
-                throw new RuntimeException(e);
-            }
-            return 0;
-        });
-    }
-
-    /**
-     * 关联(子查询), 主表复杂字段operation, 全sql执行
-     *
-     * @param config es配置
-     * @param dml dml信息
-     * @param data 单行dml数据
-     * @param tableItem 当前表配置
-     */
-    private void wholeSqlOperation(ESSyncConfig config, Dml dml, Map<String, Object> data, Map<String, Object> old,
-                                   TableItem tableItem) {
-        ESMapping mapping = config.getEsMapping();
-        // 防止最后出现groupby 导致sql解析异常
-        String[] sqlSplit = mapping.getSql().split("GROUP\\ BY(?!(.*)ON)");
-        String sqlNoWhere = sqlSplit[0];
-
-        String sqlGroupBy = "";
-
-        if (sqlSplit.length > 1) {
-            sqlGroupBy = "GROUP BY " + sqlSplit[1];
-        }
-
-        StringBuilder sql = new StringBuilder(sqlNoWhere + " WHERE ");
-
-        for (FieldItem fkFieldItem : tableItem.getRelationTableFields().keySet()) {
-            String columnName = fkFieldItem.getColumn().getColumnName();
-            Object value = esTemplate.getValFromData(mapping, data, fkFieldItem.getFieldName(), columnName);
-            ESSyncUtil.appendCondition(sql, value, tableItem.getAlias(), columnName);
-        }
-        int len = sql.length();
-        sql.delete(len - 5, len);
-        sql.append(sqlGroupBy);
-
-        DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
-        if (logger.isTraceEnabled()) {
-            logger.trace("Join table update es index by query whole sql, destination:{}, table: {}, index: {}, sql: {}",
-                config.getDestination(),
-                dml.getTable(),
-                mapping.get_index(),
-                sql.toString().replace("\n", " "));
-        }
-        Util.sqlRS(ds, sql.toString(), rs -> {
-            try {
-                while (rs.next()) {
-                    Map<String, Object> esFieldData = new LinkedHashMap<>();
-                    for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) {
-                        if (old != null) {
-                            // 从表子查询
-                            out: for (FieldItem fieldItem1 : tableItem.getSubQueryFields()) {
-                                for (ColumnItem columnItem0 : fieldItem.getColumnItems()) {
-                                    if (fieldItem1.getFieldName().equals(columnItem0.getColumnName()))
-                                        for (ColumnItem columnItem : fieldItem1.getColumnItems()) {
-                                            if (old.containsKey(columnItem.getColumnName())) {
-                                                Object val = esTemplate.getValFromRS(mapping,
-                                                    rs,
-                                                    fieldItem.getFieldName(),
-                                                    fieldItem.getFieldName());
-                                                esFieldData.put(fieldItem.getFieldName(), val);
-                                                break out;
-                                            }
-                                        }
-                                }
-                            }
-                            // 从表非子查询
-                            for (FieldItem fieldItem1 : tableItem.getRelationSelectFieldItems()) {
-                                if (fieldItem1.equals(fieldItem)) {
-                                    for (ColumnItem columnItem : fieldItem1.getColumnItems()) {
-                                        if (old.containsKey(columnItem.getColumnName())) {
-                                            Object val = esTemplate.getValFromRS(mapping,
-                                                rs,
-                                                fieldItem.getFieldName(),
-                                                fieldItem.getFieldName());
-                                            esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), val);
-                                            break;
-                                        }
-                                    }
-                                }
-                            }
-                        } else {
-                            Object val = esTemplate
-                                .getValFromRS(mapping, rs, fieldItem.getFieldName(), fieldItem.getFieldName());
-                            esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), val);
-                        }
-                    }
-
-                    Map<String, Object> paramsTmp = new LinkedHashMap<>();
-                    for (Map.Entry<FieldItem, List<FieldItem>> entry : tableItem.getRelationTableFields().entrySet()) {
-                        for (FieldItem fieldItem : entry.getValue()) {
-                            Object value = esTemplate
-                                .getValFromRS(mapping, rs, fieldItem.getFieldName(), fieldItem.getFieldName());
-                            String fieldName = fieldItem.getFieldName();
-                            // 判断是否是主键
-                            if (fieldName.equals(mapping.get_id())) {
-                                fieldName = "_id";
-                            }
-                            paramsTmp.put(fieldName, value);
-                        }
-                    }
-
-                    if (logger.isDebugEnabled()) {
-                        logger.trace(
-                            "Join table update es index by query whole sql, destination:{}, table: {}, index: {}",
-                            config.getDestination(),
-                            dml.getTable(),
-                            mapping.get_index());
-                    }
-                    esTemplate.updateByQuery(config, paramsTmp, esFieldData);
-                }
-            } catch (Exception e) {
-                throw new RuntimeException(e);
-            }
-            return 0;
-        });
-    }
-
-    /**
-     * 单表简单字段update
-     *
-     * @param config es配置
-     * @param owner 所属表
-     * @param dml dml信息
-     * @param data 单行data数据
-     * @param old 单行old数据
-     */
-    private void singleTableSimpleFiledUpdate(ESSyncConfig config, String owner, Dml dml, Map<String, Object> data,
-                                              Map<String, Object> old) {
-        ESMapping mapping = config.getEsMapping();
-        Map<String, Object> esFieldData = new LinkedHashMap<>();
-
-        Object idVal = esTemplate.getESDataFromDmlData(mapping, owner, data, old, esFieldData);
-
-        if (logger.isTraceEnabled()) {
-            logger.trace("Main table update to es index, destination:{}, table: {}, index: {}, id: {}",
-                config.getDestination(),
-                dml.getTable(),
-                mapping.get_index(),
-                idVal);
-        }
-        esTemplate.update(mapping, idVal, esFieldData);
-    }
-
-    /**
-     * 主表(单表)复杂字段update
-     *
-     * @param config es配置
-     * @param dml dml信息
-     * @param data 单行dml数据
-     */
-    private void mainTableUpdate(ESSyncConfig config, Dml dml, Map<String, Object> data, Map<String, Object> old) {
-        ESMapping mapping = config.getEsMapping();
-        String sql = mapping.getSql();
-        String condition = ESSyncUtil.pkConditionSql(mapping, data);
-        sql = ESSyncUtil.appendCondition(sql, condition);
-        DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
-        if (logger.isTraceEnabled()) {
-            logger.trace("Main table update to es index by query sql, destination:{}, table: {}, index: {}, sql: {}",
-                config.getDestination(),
-                dml.getTable(),
-                mapping.get_index(),
-                sql.replace("\n", " "));
-        }
-        Util.sqlRS(ds, sql, rs -> {
-            try {
-                while (rs.next()) {
-                    Map<String, Object> esFieldData = new LinkedHashMap<>();
-                    Object idVal = esTemplate.getESDataFromRS(mapping, rs, old, esFieldData);
-
-                    if (logger.isTraceEnabled()) {
-                        logger.trace(
-                            "Main table update to es index by query sql, destination:{}, table: {}, index: {}, id: {}",
-                            config.getDestination(),
-                            dml.getTable(),
-                            mapping.get_index(),
-                            idVal);
-                    }
-                    esTemplate.update(mapping, idVal, esFieldData);
-                }
-            } catch (Exception e) {
-                throw new RuntimeException(e);
-            }
-            return 0;
-        });
-    }
-
-    /**
-     * 提交批次
-     */
-    public void commit() {
-        esTemplate.commit();
-    }
-}

client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESConnection.java

@@ -1,469 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.es.support;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.util.Map;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.http.HttpHost;
-import org.apache.http.auth.AuthScope;
-import org.apache.http.auth.UsernamePasswordCredentials;
-import org.apache.http.client.CredentialsProvider;
-import org.apache.http.impl.client.BasicCredentialsProvider;
-import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
-import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
-import org.elasticsearch.action.bulk.BulkRequest;
-import org.elasticsearch.action.bulk.BulkRequestBuilder;
-import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.delete.DeleteRequest;
-import org.elasticsearch.action.delete.DeleteRequestBuilder;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.index.IndexRequestBuilder;
-import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.search.SearchRequestBuilder;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.update.UpdateRequest;
-import org.elasticsearch.action.update.UpdateRequestBuilder;
-import org.elasticsearch.client.*;
-import org.elasticsearch.client.transport.TransportClient;
-import org.elasticsearch.cluster.metadata.MappingMetaData;
-import org.elasticsearch.common.collect.ImmutableOpenMap;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.TransportAddress;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.transport.client.PreBuiltTransportClient;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * ES 连接器, Transport Rest 两种方式
- *
- * @author rewerma 2019-08-01
- * @version 1.0.0
- */
-public class ESConnection {
-
-    private static final Logger logger = LoggerFactory.getLogger(ESConnection.class);
-
-    public enum ESClientMode {
-                              TRANSPORT, REST
-    }
-
-    private ESClientMode        mode;
-
-    private TransportClient     transportClient;
-
-    private RestHighLevelClient restHighLevelClient;
-
-    public ESConnection(String[] hosts, Map<String, String> properties, ESClientMode mode) throws UnknownHostException{
-        this.mode = mode;
-        if (mode == ESClientMode.TRANSPORT) {
-            Settings.Builder settingBuilder = Settings.builder();
-            // properties.forEach(settingBuilder::put);
-            settingBuilder.put("cluster.name", properties.get("properties"));
-            Settings settings = settingBuilder.build();
-            transportClient = new PreBuiltTransportClient(settings);
-            for (String host : hosts) {
-                int i = host.indexOf(":");
-                transportClient.addTransportAddress(new TransportAddress(InetAddress.getByName(host.substring(0, i)),
-                    Integer.parseInt(host.substring(i + 1))));
-            }
-        } else {
-            HttpHost[] httpHosts = new HttpHost[hosts.length];
-            for (int i = 0; i < hosts.length; i++) {
-                String host = hosts[i];
-                int j = host.indexOf(":");
-                HttpHost httpHost = new HttpHost(InetAddress.getByName(host.substring(0, j)),
-                    Integer.parseInt(host.substring(j + 1)));
-                httpHosts[i] = httpHost;
-            }
-            RestClientBuilder restClientBuilder = RestClient.builder(httpHosts);
-            String nameAndPwd = properties.get("security.auth");
-            if (StringUtils.isNotEmpty(nameAndPwd) && nameAndPwd.contains(":")) {
-                String[] nameAndPwdArr = nameAndPwd.split(":");
-                final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
-                credentialsProvider.setCredentials(AuthScope.ANY,
-                    new UsernamePasswordCredentials(nameAndPwdArr[0], nameAndPwdArr[1]));
-                restClientBuilder.setHttpClientConfigCallback(
-                    httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
-            }
-            restHighLevelClient = new RestHighLevelClient(restClientBuilder);
-        }
-    }
-
-    public void close() {
-        if (mode == ESClientMode.TRANSPORT) {
-            transportClient.close();
-        } else {
-            try {
-                restHighLevelClient.close();
-            } catch (IOException e) {
-                throw new RuntimeException(e);
-            }
-        }
-    }
-
-    public MappingMetaData getMapping(String index, String type) {
-        MappingMetaData mappingMetaData = null;
-        if (mode == ESClientMode.TRANSPORT) {
-            ImmutableOpenMap<String, MappingMetaData> mappings;
-            try {
-                mappings = transportClient.admin()
-                    .cluster()
-                    .prepareState()
-                    .execute()
-                    .actionGet()
-                    .getState()
-                    .getMetaData()
-                    .getIndices()
-                    .get(index)
-                    .getMappings();
-            } catch (NullPointerException e) {
-                throw new IllegalArgumentException("Not found the mapping info of index: " + index);
-            }
-            mappingMetaData = mappings.get(type);
-
-        } else {
-            ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings;
-            try {
-                GetMappingsRequest request = new GetMappingsRequest();
-                request.indices(index);
-                GetMappingsResponse response;
-                // try {
-                // response = restHighLevelClient
-                // .indices()
-                // .getMapping(request, RequestOptions.DEFAULT);
-                // // 6.4以下版本直接使用该接口会报错
-                // } catch (Exception e) {
-                // logger.warn("Low ElasticSearch version for getMapping");
-                response = RestHighLevelClientExt.getMapping(restHighLevelClient, request, RequestOptions.DEFAULT);
-                // }
-
-                mappings = response.mappings();
-            } catch (NullPointerException e) {
-                throw new IllegalArgumentException("Not found the mapping info of index: " + index);
-            } catch (IOException e) {
-                logger.error(e.getMessage(), e);
-                return null;
-            }
-            mappingMetaData = mappings.get(index).get(type);
-        }
-        return mappingMetaData;
-    }
-
-    public class ESIndexRequest {
-
-        private IndexRequestBuilder indexRequestBuilder;
-
-        private IndexRequest        indexRequest;
-
-        public ESIndexRequest(String index, String type, String id){
-            if (mode == ESClientMode.TRANSPORT) {
-                indexRequestBuilder = transportClient.prepareIndex(index, type, id);
-            } else {
-                indexRequest = new IndexRequest(index, type, id);
-            }
-        }
-
-        public ESIndexRequest setSource(Map<String, ?> source) {
-            if (mode == ESClientMode.TRANSPORT) {
-                indexRequestBuilder.setSource(source);
-            } else {
-                indexRequest.source(source);
-            }
-            return this;
-        }
-
-        public ESIndexRequest setRouting(String routing) {
-            if (mode == ESClientMode.TRANSPORT) {
-                indexRequestBuilder.setRouting(routing);
-            } else {
-                indexRequest.routing(routing);
-            }
-            return this;
-        }
-
-        public IndexRequestBuilder getIndexRequestBuilder() {
-            return indexRequestBuilder;
-        }
-
-        public void setIndexRequestBuilder(IndexRequestBuilder indexRequestBuilder) {
-            this.indexRequestBuilder = indexRequestBuilder;
-        }
-
-        public IndexRequest getIndexRequest() {
-            return indexRequest;
-        }
-
-        public void setIndexRequest(IndexRequest indexRequest) {
-            this.indexRequest = indexRequest;
-        }
-    }
-
-    public class ESUpdateRequest {
-
-        private UpdateRequestBuilder updateRequestBuilder;
-
-        private UpdateRequest        updateRequest;
-
-        public ESUpdateRequest(String index, String type, String id){
-            if (mode == ESClientMode.TRANSPORT) {
-                updateRequestBuilder = transportClient.prepareUpdate(index, type, id);
-            } else {
-                updateRequest = new UpdateRequest(index, type, id);
-            }
-        }
-
-        public ESUpdateRequest setDoc(Map source) {
-            if (mode == ESClientMode.TRANSPORT) {
-                updateRequestBuilder.setDoc(source);
-            } else {
-                updateRequest.doc(source);
-            }
-            return this;
-        }
-
-        public ESUpdateRequest setDocAsUpsert(boolean shouldUpsertDoc) {
-            if (mode == ESClientMode.TRANSPORT) {
-                updateRequestBuilder.setDocAsUpsert(shouldUpsertDoc);
-            } else {
-                updateRequest.docAsUpsert(shouldUpsertDoc);
-            }
-            return this;
-        }
-
-        public ESUpdateRequest setRouting(String routing) {
-            if (mode == ESClientMode.TRANSPORT) {
-                updateRequestBuilder.setRouting(routing);
-            } else {
-                updateRequest.routing(routing);
-            }
-            return this;
-        }
-
-        public UpdateRequestBuilder getUpdateRequestBuilder() {
-            return updateRequestBuilder;
-        }
-
-        public void setUpdateRequestBuilder(UpdateRequestBuilder updateRequestBuilder) {
-            this.updateRequestBuilder = updateRequestBuilder;
-        }
-
-        public UpdateRequest getUpdateRequest() {
-            return updateRequest;
-        }
-
-        public void setUpdateRequest(UpdateRequest updateRequest) {
-            this.updateRequest = updateRequest;
-        }
-    }
-
-    public class ESDeleteRequest {
-
-        private DeleteRequestBuilder deleteRequestBuilder;
-
-        private DeleteRequest        deleteRequest;
-
-        public ESDeleteRequest(String index, String type, String id){
-            if (mode == ESClientMode.TRANSPORT) {
-                deleteRequestBuilder = transportClient.prepareDelete(index, type, id);
-            } else {
-                deleteRequest = new DeleteRequest(index, type, id);
-            }
-        }
-
-        public DeleteRequestBuilder getDeleteRequestBuilder() {
-            return deleteRequestBuilder;
-        }
-
-        public void setDeleteRequestBuilder(DeleteRequestBuilder deleteRequestBuilder) {
-            this.deleteRequestBuilder = deleteRequestBuilder;
-        }
-
-        public DeleteRequest getDeleteRequest() {
-            return deleteRequest;
-        }
-
-        public void setDeleteRequest(DeleteRequest deleteRequest) {
-            this.deleteRequest = deleteRequest;
-        }
-    }
-
-    public class ESSearchRequest {
-
-        private SearchRequestBuilder searchRequestBuilder;
-
-        private SearchRequest        searchRequest;
-
-        private SearchSourceBuilder  sourceBuilder;
-
-        public ESSearchRequest(String index, String... types){
-            if (mode == ESClientMode.TRANSPORT) {
-                searchRequestBuilder = transportClient.prepareSearch(index).setTypes(types);
-            } else {
-                searchRequest = new SearchRequest(index).types(types);
-                sourceBuilder = new SearchSourceBuilder();
-            }
-        }
-
-        public ESSearchRequest setQuery(QueryBuilder queryBuilder) {
-            if (mode == ESClientMode.TRANSPORT) {
-                searchRequestBuilder.setQuery(queryBuilder);
-            } else {
-                sourceBuilder.query(queryBuilder);
-            }
-            return this;
-        }
-
-        public ESSearchRequest size(int size) {
-            if (mode == ESClientMode.TRANSPORT) {
-                searchRequestBuilder.setSize(size);
-            } else {
-                sourceBuilder.size(size);
-            }
-            return this;
-        }
-
-        public SearchResponse getResponse() {
-            if (mode == ESClientMode.TRANSPORT) {
-                return searchRequestBuilder.get();
-            } else {
-                searchRequest.source(sourceBuilder);
-                try {
-                    return restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
-                } catch (IOException e) {
-                    throw new RuntimeException(e);
-                }
-            }
-        }
-
-        public SearchRequestBuilder getSearchRequestBuilder() {
-            return searchRequestBuilder;
-        }
-
-        public void setSearchRequestBuilder(SearchRequestBuilder searchRequestBuilder) {
-            this.searchRequestBuilder = searchRequestBuilder;
-        }
-
-        public SearchRequest getSearchRequest() {
-            return searchRequest;
-        }
-
-        public void setSearchRequest(SearchRequest searchRequest) {
-            this.searchRequest = searchRequest;
-        }
-    }
-
-    public class ESBulkRequest {
-
-        private BulkRequestBuilder bulkRequestBuilder;
-
-        private BulkRequest        bulkRequest;
-
-        public ESBulkRequest(){
-            if (mode == ESClientMode.TRANSPORT) {
-                bulkRequestBuilder = transportClient.prepareBulk();
-            } else {
-                bulkRequest = new BulkRequest();
-            }
-        }
-
-        public void resetBulk() {
-            if (mode == ESClientMode.TRANSPORT) {
-                bulkRequestBuilder = transportClient.prepareBulk();
-            } else {
-                bulkRequest = new BulkRequest();
-            }
-        }
-
-        public ESBulkRequest add(ESIndexRequest esIndexRequest) {
-            if (mode == ESClientMode.TRANSPORT) {
-                bulkRequestBuilder.add(esIndexRequest.indexRequestBuilder);
-            } else {
-                bulkRequest.add(esIndexRequest.indexRequest);
-            }
-            return this;
-        }
-
-        public ESBulkRequest add(ESUpdateRequest esUpdateRequest) {
-            if (mode == ESClientMode.TRANSPORT) {
-                bulkRequestBuilder.add(esUpdateRequest.updateRequestBuilder);
-            } else {
-                bulkRequest.add(esUpdateRequest.updateRequest);
-            }
-            return this;
-        }
-
-        public ESBulkRequest add(ESDeleteRequest esDeleteRequest) {
-            if (mode == ESClientMode.TRANSPORT) {
-                bulkRequestBuilder.add(esDeleteRequest.deleteRequestBuilder);
-            } else {
-                bulkRequest.add(esDeleteRequest.deleteRequest);
-            }
-            return this;
-        }
-
-        public int numberOfActions() {
-            if (mode == ESClientMode.TRANSPORT) {
-                return bulkRequestBuilder.numberOfActions();
-            } else {
-                return bulkRequest.numberOfActions();
-            }
-        }
-
-        public BulkResponse bulk() {
-            if (mode == ESClientMode.TRANSPORT) {
-                return bulkRequestBuilder.execute().actionGet();
-            } else {
-                try {
-                    return restHighLevelClient.bulk(bulkRequest);
-                } catch (IOException e) {
-                    throw new RuntimeException(e);
-                }
-            }
-        }
-
-        public BulkRequestBuilder getBulkRequestBuilder() {
-            return bulkRequestBuilder;
-        }
-
-        public void setBulkRequestBuilder(BulkRequestBuilder bulkRequestBuilder) {
-            this.bulkRequestBuilder = bulkRequestBuilder;
-        }
-
-        public BulkRequest getBulkRequest() {
-            return bulkRequest;
-        }
-
-        public void setBulkRequest(BulkRequest bulkRequest) {
-            this.bulkRequest = bulkRequest;
-        }
-    }
-
-    // ------ get/set ------
-    public ESClientMode getMode() {
-        return mode;
-    }
-
-    public void setMode(ESClientMode mode) {
-        this.mode = mode;
-    }
-
-    public TransportClient getTransportClient() {
-        return transportClient;
-    }
-
-    public void setTransportClient(TransportClient transportClient) {
-        this.transportClient = transportClient;
-    }
-
-    public RestHighLevelClient getRestHighLevelClient() {
-        return restHighLevelClient;
-    }
-
-    public void setRestHighLevelClient(RestHighLevelClient restHighLevelClient) {
-        this.restHighLevelClient = restHighLevelClient;
-    }
-}

client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESSyncUtil.java

@@ -1,309 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.es.support;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.sql.Blob;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.LinkedHashSet;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.commons.codec.binary.Base64;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.alibaba.fastjson.JSON;
-import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping;
-import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem;
-import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.ColumnItem;
-import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.TableItem;
-import com.alibaba.otter.canal.client.adapter.support.Util;
-
-/**
- * ES 同步工具同类
- *
- * @author rewerma 2018-11-01
- * @version 1.0.0
- */
-public class ESSyncUtil {
-
-    private static Logger logger = LoggerFactory.getLogger(ESSyncUtil.class);
-
-    public static Object convertToEsObj(Object val, String fieldInfo) {
-        if (val == null) {
-            return null;
-        }
-        if (fieldInfo.startsWith("array:")) {
-            String separator = fieldInfo.substring("array:".length()).trim();
-            String[] values = val.toString().split(separator);
-            return Arrays.asList(values);
-        } else if (fieldInfo.startsWith("object")) {
-            if (val instanceof String){
-                return JSON.parse(val.toString());
-            }
-            return JSON.parse(new String((byte[])val));
-        }
-        return null;
-    }
-
-    /**
-     * 类型转换为Mapping中对应的类型
-     */
-    public static Object typeConvert(Object val, String esType) {
-        if (val == null) {
-            return null;
-        }
-        if (esType == null) {
-            return val;
-        }
-        Object res = null;
-        if ("integer".equals(esType)) {
-            if (val instanceof Number) {
-                res = ((Number) val).intValue();
-            } else {
-                res = Integer.parseInt(val.toString());
-            }
-        } else if ("long".equals(esType)) {
-            if (val instanceof Number) {
-                res = ((Number) val).longValue();
-            } else {
-                res = Long.parseLong(val.toString());
-            }
-        } else if ("short".equals(esType)) {
-            if (val instanceof Number) {
-                res = ((Number) val).shortValue();
-            } else {
-                res = Short.parseShort(val.toString());
-            }
-        } else if ("byte".equals(esType)) {
-            if (val instanceof Number) {
-                res = ((Number) val).byteValue();
-            } else {
-                res = Byte.parseByte(val.toString());
-            }
-        } else if ("double".equals(esType)) {
-            if (val instanceof Number) {
-                res = ((Number) val).doubleValue();
-            } else {
-                res = Double.parseDouble(val.toString());
-            }
-        } else if ("float".equals(esType) || "half_float".equals(esType) || "scaled_float".equals(esType)) {
-            if (val instanceof Number) {
-                res = ((Number) val).floatValue();
-            } else {
-                res = Float.parseFloat(val.toString());
-            }
-        } else if ("boolean".equals(esType)) {
-            if (val instanceof Boolean) {
-                res = val;
-            } else if (val instanceof Number) {
-                int v = ((Number) val).intValue();
-                res = v != 0;
-            } else {
-                res = Boolean.parseBoolean(val.toString());
-            }
-        } else if ("date".equals(esType)) {
-            if (val instanceof java.sql.Time) {
-                DateTime dateTime = new DateTime(((java.sql.Time) val).getTime());
-                if (dateTime.getMillisOfSecond() != 0) {
-                    res = dateTime.toString("HH:mm:ss.SSS");
-                } else {
-                    res = dateTime.toString("HH:mm:ss");
-                }
-            } else if (val instanceof java.sql.Timestamp) {
-                DateTime dateTime = new DateTime(((java.sql.Timestamp) val).getTime());
-                if (dateTime.getMillisOfSecond() != 0) {
-                    res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS" + Util.timeZone);
-                } else {
-                    res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss" + Util.timeZone);
-                }
-            } else if (val instanceof java.sql.Date || val instanceof Date) {
-                DateTime dateTime;
-                if (val instanceof java.sql.Date) {
-                    dateTime = new DateTime(((java.sql.Date) val).getTime());
-                } else {
-                    dateTime = new DateTime(((Date) val).getTime());
-                }
-                if (dateTime.getHourOfDay() == 0 && dateTime.getMinuteOfHour() == 0 && dateTime.getSecondOfMinute() == 0
-                    && dateTime.getMillisOfSecond() == 0) {
-                    res = dateTime.toString("yyyy-MM-dd");
-                } else {
-                    if (dateTime.getMillisOfSecond() != 0) {
-                        res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS" + Util.timeZone);
-                    } else {
-                        res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss" + Util.timeZone);
-                    }
-                }
-            } else if (val instanceof Long) {
-                DateTime dateTime = new DateTime(((Long) val).longValue());
-                if (dateTime.getHourOfDay() == 0 && dateTime.getMinuteOfHour() == 0 && dateTime.getSecondOfMinute() == 0
-                    && dateTime.getMillisOfSecond() == 0) {
-                    res = dateTime.toString("yyyy-MM-dd");
-                } else if (dateTime.getMillisOfSecond() != 0) {
-                    res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS" + Util.timeZone);
-                } else {
-                    res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss" + Util.timeZone);
-                }
-            } else if (val instanceof String) {
-                String v = ((String) val).trim();
-                if (v.length() > 18 && v.charAt(4) == '-' && v.charAt(7) == '-' && v.charAt(10) == ' '
-                    && v.charAt(13) == ':' && v.charAt(16) == ':') {
-                    String dt = v.substring(0, 10) + "T" + v.substring(11);
-                    Date date = Util.parseDate(dt);
-                    if (date != null) {
-                        DateTime dateTime = new DateTime(date);
-                        if (dateTime.getMillisOfSecond() != 0) {
-                            res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss.SSS" + Util.timeZone);
-                        } else {
-                            res = dateTime.toString("yyyy-MM-dd'T'HH:mm:ss" + Util.timeZone);
-                        }
-                    }
-                } else if (v.length() == 10 && v.charAt(4) == '-' && v.charAt(7) == '-') {
-                    Date date = Util.parseDate(v);
-                    if (date != null) {
-                        DateTime dateTime = new DateTime(date);
-                        res = dateTime.toString("yyyy-MM-dd");
-                    }
-                }
-            }
-        } else if ("binary".equals(esType)) {
-            if (val instanceof byte[]) {
-                Base64 base64 = new Base64();
-                res = base64.encodeAsString((byte[]) val);
-            } else if (val instanceof Blob) {
-                byte[] b = blobToBytes((Blob) val);
-                Base64 base64 = new Base64();
-                res = base64.encodeAsString(b);
-            } else if (val instanceof String) {
-                // 对应canal中的单字节编码
-                byte[] b = ((String) val).getBytes(StandardCharsets.ISO_8859_1);
-                Base64 base64 = new Base64();
-                res = base64.encodeAsString(b);
-            }
-        } else if ("geo_point".equals(esType)) {
-            if (!(val instanceof String)) {
-                logger.error("es type is geo_point, but source type is not String");
-                return val;
-            }
-
-            if (!((String) val).contains(",")) {
-                logger.error("es type is geo_point, source value not contains ',' separator");
-                return val;
-            }
-
-            String[] point = ((String) val).split(",");
-            Map<String, Double> location = new HashMap<>();
-            location.put("lat", Double.valueOf(point[0].trim()));
-            location.put("lon", Double.valueOf(point[1].trim()));
-            return location;
-        } else if ("array".equals(esType)) {
-            if ("".equals(val.toString().trim())) {
-                res = new ArrayList<>();
-            } else {
-                String value = val.toString();
-                String separator = ",";
-                if (!value.contains(",")) {
-                    if (value.contains(";")) {
-                        separator = ";";
-                    } else if (value.contains("|")) {
-                        separator = "\\|";
-                    } else if (value.contains("-")) {
-                        separator = "-";
-                    }
-                }
-                String[] values = value.split(separator);
-                return Arrays.asList(values);
-            }
-        } else if ("object".equals(esType)) {
-            if ("".equals(val.toString().trim())) {
-                res = new HashMap<>();
-            } else {
-                res = JSON.parseObject(val.toString(), Map.class);
-            }
-        } else {
-            // 其他类全以字符串处理
-            res = val.toString();
-        }
-
-        return res;
-    }
-
-    /**
-     * Blob转byte[]
-     */
-    private static byte[] blobToBytes(Blob blob) {
-        try (InputStream is = blob.getBinaryStream()) {
-            byte[] b = new byte[(int) blob.length()];
-            if (is.read(b) != -1) {
-                return b;
-            } else {
-                return new byte[0];
-            }
-        } catch (IOException | SQLException e) {
-            logger.error(e.getMessage());
-            return null;
-        }
-    }
-
-    /**
-     * 拼接主键条件
-     *
-     * @param mapping
-     * @param data
-     * @return
-     */
-    public static String pkConditionSql(ESMapping mapping, Map<String, Object> data) {
-        Set<ColumnItem> idColumns = new LinkedHashSet<>();
-        SchemaItem schemaItem = mapping.getSchemaItem();
-
-        TableItem mainTable = schemaItem.getMainTable();
-
-        for (ColumnItem idColumnItem : schemaItem.getIdFieldItem(mapping).getColumnItems()) {
-            if ((mainTable.getAlias() == null && idColumnItem.getOwner() == null)
-                || (mainTable.getAlias() != null && mainTable.getAlias().equals(idColumnItem.getOwner()))) {
-                idColumns.add(idColumnItem);
-            }
-        }
-
-        if (idColumns.isEmpty()) {
-            throw new RuntimeException("Not found primary key field in main table");
-        }
-
-        // 拼接condition
-        StringBuilder condition = new StringBuilder(" ");
-        for (ColumnItem idColumn : idColumns) {
-            Object idVal = data.get(idColumn.getColumnName());
-            if (mainTable.getAlias() != null) condition.append(mainTable.getAlias()).append(".");
-            condition.append(idColumn.getColumnName()).append("=");
-            if (idVal instanceof String) {
-                condition.append("'").append(idVal).append("' AND ");
-            } else {
-                condition.append(idVal).append(" AND ");
-            }
-        }
-
-        if (condition.toString().endsWith("AND ")) {
-            int len2 = condition.length();
-            condition.delete(len2 - 4, len2);
-        }
-        return condition.toString();
-    }
-
-    public static String appendCondition(String sql, String condition) {
-        return sql + " WHERE " + condition + " ";
-    }
-
-    public static void appendCondition(StringBuilder sql, Object value, String owner, String columnName) {
-        if (value instanceof String) {
-            sql.append(owner).append(".").append(columnName).append("='").append(value).append("'  AND ");
-        } else {
-            sql.append(owner).append(".").append(columnName).append("=").append(value).append("  AND ");
-        }
-    }
-}

client-adapter/elasticsearch/src/main/java/com/alibaba/otter/canal/client/adapter/es/support/ESTemplate.java

@@ -1,523 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.es.support;
-
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.*;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-
-import javax.sql.DataSource;
-
-import org.apache.commons.lang.StringUtils;
-import org.elasticsearch.action.bulk.BulkItemResponse;
-import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.cluster.metadata.MappingMetaData;
-import org.elasticsearch.index.query.BoolQueryBuilder;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.search.SearchHit;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
-import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig.ESMapping;
-import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem;
-import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.ColumnItem;
-import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.FieldItem;
-import com.alibaba.otter.canal.client.adapter.es.support.ESConnection.*;
-import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
-import com.alibaba.otter.canal.client.adapter.support.Util;
-
-/**
- * ES 操作模板
- *
- * @author rewerma 2018-11-01
- * @version 1.0.0
- */
-public class ESTemplate {
-
-    private static final Logger logger         = LoggerFactory.getLogger(ESTemplate.class);
-
-    private static final int    MAX_BATCH_SIZE = 1000;
-
-    private ESConnection        esConnection;
-
-    private ESBulkRequest       esBulkRequest;
-
-    public ESTemplate(ESConnection esConnection){
-        this.esConnection = esConnection;
-        this.esBulkRequest = this.esConnection.new ESBulkRequest();
-    }
-
-    public ESBulkRequest getBulk() {
-        return esBulkRequest;
-    }
-
-    public void resetBulkRequestBuilder() {
-        this.esBulkRequest.resetBulk();
-    }
-
-    /**
-     * 插入数据
-     *
-     * @param mapping 配置对象
-     * @param pkVal 主键值
-     * @param esFieldData 数据Map
-     */
-    public void insert(ESMapping mapping, Object pkVal, Map<String, Object> esFieldData) {
-        if (mapping.get_id() != null) {
-            String parentVal = (String) esFieldData.remove("$parent_routing");
-            if (mapping.isUpsert()) {
-                ESUpdateRequest updateRequest = esConnection.new ESUpdateRequest(mapping.get_index(),
-                    mapping.get_type(),
-                    pkVal.toString()).setDoc(esFieldData).setDocAsUpsert(true);
-                if (StringUtils.isNotEmpty(parentVal)) {
-                    updateRequest.setRouting(parentVal);
-                }
-                getBulk().add(updateRequest);
-            } else {
-                ESIndexRequest indexRequest = esConnection.new ESIndexRequest(mapping.get_index(),
-                    mapping.get_type(),
-                    pkVal.toString()).setSource(esFieldData);
-                if (StringUtils.isNotEmpty(parentVal)) {
-                    indexRequest.setRouting(parentVal);
-                }
-                getBulk().add(indexRequest);
-            }
-            commitBulk();
-        } else {
-            ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index(),
-                mapping.get_type()).setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)).size(10000);
-            SearchResponse response = esSearchRequest.getResponse();
-
-            for (SearchHit hit : response.getHits()) {
-                ESUpdateRequest esUpdateRequest = this.esConnection.new ESUpdateRequest(mapping.get_index(),
-                    mapping.get_type(),
-                    hit.getId()).setDoc(esFieldData);
-                getBulk().add(esUpdateRequest);
-                commitBulk();
-            }
-        }
-
-    }
-
-    /**
-     * 根据主键更新数据
-     *
-     * @param mapping 配置对象
-     * @param pkVal 主键值
-     * @param esFieldData 数据Map
-     */
-    public void update(ESMapping mapping, Object pkVal, Map<String, Object> esFieldData) {
-        Map<String, Object> esFieldDataTmp = new LinkedHashMap<>(esFieldData.size());
-        esFieldData.forEach((k, v) -> esFieldDataTmp.put(Util.cleanColumn(k), v));
-        append4Update(mapping, pkVal, esFieldDataTmp);
-        commitBulk();
-    }
-
-    /**
-     * update by query
-     *
-     * @param config 配置对象
-     * @param paramsTmp sql查询条件
-     * @param esFieldData 数据Map
-     */
-    public void updateByQuery(ESSyncConfig config, Map<String, Object> paramsTmp, Map<String, Object> esFieldData) {
-        if (paramsTmp.isEmpty()) {
-            return;
-        }
-        ESMapping mapping = config.getEsMapping();
-        BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
-        paramsTmp.forEach((fieldName, value) -> queryBuilder.must(QueryBuilders.termsQuery(fieldName, value)));
-
-        // 查询sql批量更新
-        DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
-        StringBuilder sql = new StringBuilder("SELECT * FROM (" + mapping.getSql() + ") _v WHERE ");
-        List<Object> values = new ArrayList<>();
-        paramsTmp.forEach((fieldName, value) -> {
-            sql.append("_v.").append(fieldName).append("=? AND ");
-            values.add(value);
-        });
-        // TODO 直接外部包裹sql会导致全表扫描性能低, 待优化拼接内部where条件
-        int len = sql.length();
-        sql.delete(len - 4, len);
-        Integer syncCount = (Integer) Util.sqlRS(ds, sql.toString(), values, rs -> {
-            int count = 0;
-            try {
-                while (rs.next()) {
-                    Object idVal = getIdValFromRS(mapping, rs);
-                    append4Update(mapping, idVal, esFieldData);
-                    commitBulk();
-                    count++;
-                }
-            } catch (Exception e) {
-                throw new RuntimeException(e);
-            }
-            return count;
-        });
-        if (logger.isTraceEnabled()) {
-            logger.trace("Update ES by query affected {} records", syncCount);
-        }
-    }
-
-    /**
-     * 通过主键删除数据
-     *
-     * @param mapping 配置对象
-     * @param pkVal 主键值
-     * @param esFieldData 数据Map
-     */
-    public void delete(ESMapping mapping, Object pkVal, Map<String, Object> esFieldData) {
-        if (mapping.get_id() != null) {
-            ESDeleteRequest esDeleteRequest = this.esConnection.new ESDeleteRequest(mapping.get_index(),
-                mapping.get_type(),
-                pkVal.toString());
-            getBulk().add(esDeleteRequest);
-            commitBulk();
-        } else {
-            ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index(),
-                mapping.get_type()).setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)).size(10000);
-            SearchResponse response = esSearchRequest.getResponse();
-            for (SearchHit hit : response.getHits()) {
-                ESUpdateRequest esUpdateRequest = this.esConnection.new ESUpdateRequest(mapping.get_index(),
-                    mapping.get_type(),
-                    hit.getId()).setDoc(esFieldData);
-                getBulk().add(esUpdateRequest);
-                commitBulk();
-            }
-        }
-
-    }
-
-    /**
-     * 提交批次
-     */
-    public void commit() {
-        if (getBulk().numberOfActions() > 0) {
-            BulkResponse response = getBulk().bulk();
-            if (response.hasFailures()) {
-                for (BulkItemResponse itemResponse : response.getItems()) {
-                    if (!itemResponse.isFailed()) {
-                        continue;
-                    }
-
-                    if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) {
-                        logger.error(itemResponse.getFailureMessage());
-                    } else {
-                        throw new RuntimeException("ES sync commit error" + itemResponse.getFailureMessage());
-                    }
-                }
-            }
-            resetBulkRequestBuilder();
-        }
-    }
-
-    /**
-     * 如果大于批量数则提交批次
-     */
-    private void commitBulk() {
-        if (getBulk().numberOfActions() >= MAX_BATCH_SIZE) {
-            commit();
-        }
-    }
-
-    private void append4Update(ESMapping mapping, Object pkVal, Map<String, Object> esFieldData) {
-        if (mapping.get_id() != null) {
-            String parentVal = (String) esFieldData.remove("$parent_routing");
-            if (mapping.isUpsert()) {
-                ESUpdateRequest esUpdateRequest = this.esConnection.new ESUpdateRequest(mapping.get_index(),
-                    mapping.get_type(),
-                    pkVal.toString()).setDoc(esFieldData).setDocAsUpsert(true);
-                if (StringUtils.isNotEmpty(parentVal)) {
-                    esUpdateRequest.setRouting(parentVal);
-                }
-                getBulk().add(esUpdateRequest);
-            } else {
-                ESUpdateRequest esUpdateRequest = this.esConnection.new ESUpdateRequest(mapping.get_index(),
-                    mapping.get_type(),
-                    pkVal.toString()).setDoc(esFieldData);
-                if (StringUtils.isNotEmpty(parentVal)) {
-                    esUpdateRequest.setRouting(parentVal);
-                }
-                getBulk().add(esUpdateRequest);
-            }
-        } else {
-            ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index(),
-                mapping.get_type()).setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)).size(10000);
-            SearchResponse response = esSearchRequest.getResponse();
-            for (SearchHit hit : response.getHits()) {
-                ESUpdateRequest esUpdateRequest = this.esConnection.new ESUpdateRequest(mapping.get_index(),
-                    mapping.get_type(),
-                    hit.getId()).setDoc(esFieldData);
-                getBulk().add(esUpdateRequest);
-            }
-        }
-    }
-
-    public Object getValFromRS(ESMapping mapping, ResultSet resultSet, String fieldName,
-                               String columnName) throws SQLException {
-        fieldName = Util.cleanColumn(fieldName);
-        columnName = Util.cleanColumn(columnName);
-        String esType = getEsType(mapping, fieldName);
-
-        Object value = resultSet.getObject(columnName);
-        if (value instanceof Boolean) {
-            if (!"boolean".equals(esType)) {
-                value = resultSet.getByte(columnName);
-            }
-        }
-
-        // 如果是对象类型
-        if (mapping.getObjFields().containsKey(fieldName)) {
-            return ESSyncUtil.convertToEsObj(value, mapping.getObjFields().get(fieldName));
-        } else {
-            return ESSyncUtil.typeConvert(value, esType);
-        }
-    }
-
-    public Object getESDataFromRS(ESMapping mapping, ResultSet resultSet,
-                                  Map<String, Object> esFieldData) throws SQLException {
-        SchemaItem schemaItem = mapping.getSchemaItem();
-        String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
-        Object resultIdVal = null;
-        for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
-            Object value = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName());
-
-            if (fieldItem.getFieldName().equals(idFieldName)) {
-                resultIdVal = value;
-            }
-
-            if (!fieldItem.getFieldName().equals(mapping.get_id())
-                && !mapping.getSkips().contains(fieldItem.getFieldName())) {
-                esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), value);
-            }
-        }
-
-        // 添加父子文档关联信息
-        putRelationDataFromRS(mapping, schemaItem, resultSet, esFieldData);
-
-        return resultIdVal;
-    }
-
-    public Object getIdValFromRS(ESMapping mapping, ResultSet resultSet) throws SQLException {
-        SchemaItem schemaItem = mapping.getSchemaItem();
-        String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
-        Object resultIdVal = null;
-        for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
-            Object value = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName());
-
-            if (fieldItem.getFieldName().equals(idFieldName)) {
-                resultIdVal = value;
-                break;
-            }
-        }
-        return resultIdVal;
-    }
-
-    public Object getESDataFromRS(ESMapping mapping, ResultSet resultSet, Map<String, Object> dmlOld,
-                                  Map<String, Object> esFieldData) throws SQLException {
-        SchemaItem schemaItem = mapping.getSchemaItem();
-        String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
-        Object resultIdVal = null;
-        for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
-            if (fieldItem.getFieldName().equals(idFieldName)) {
-                resultIdVal = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName());
-            }
-
-            for (ColumnItem columnItem : fieldItem.getColumnItems()) {
-                if (dmlOld.containsKey(columnItem.getColumnName())
-                    && !mapping.getSkips().contains(fieldItem.getFieldName())) {
-                    esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()),
-                        getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName()));
-                    break;
-                }
-            }
-        }
-
-        // 添加父子文档关联信息
-        putRelationDataFromRS(mapping, schemaItem, resultSet, esFieldData);
-
-        return resultIdVal;
-    }
-
-    public Object getValFromData(ESMapping mapping, Map<String, Object> dmlData, String fieldName, String columnName) {
-        String esType = getEsType(mapping, fieldName);
-        Object value = dmlData.get(columnName);
-        if (value instanceof Byte) {
-            if ("boolean".equals(esType)) {
-                value = ((Byte) value).intValue() != 0;
-            }
-        }
-
-        // 如果是对象类型
-        if (mapping.getObjFields().containsKey(fieldName)) {
-            return ESSyncUtil.convertToEsObj(value, mapping.getObjFields().get(fieldName));
-        } else {
-            return ESSyncUtil.typeConvert(value, esType);
-        }
-    }
-
-    /**
-     * 将dml的data转换为es的data
-     *
-     * @param mapping 配置mapping
-     * @param dmlData dml data
-     * @param esFieldData es data
-     * @return 返回 id 值
-     */
-    public Object getESDataFromDmlData(ESMapping mapping, Map<String, Object> dmlData,
-                                       Map<String, Object> esFieldData) {
-        SchemaItem schemaItem = mapping.getSchemaItem();
-        String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
-        Object resultIdVal = null;
-        for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
-            String columnName = fieldItem.getColumnItems().iterator().next().getColumnName();
-            Object value = getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName);
-
-            if (fieldItem.getFieldName().equals(idFieldName)) {
-                resultIdVal = value;
-            }
-
-            if (!fieldItem.getFieldName().equals(mapping.get_id())
-                && !mapping.getSkips().contains(fieldItem.getFieldName())) {
-                esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), value);
-            }
-        }
-
-        // 添加父子文档关联信息
-        putRelationData(mapping, schemaItem, dmlData, esFieldData);
-        return resultIdVal;
-    }
-
-    /**
-     * 将dml的data, old转换为es的data
-     *
-     * @param mapping 配置mapping
-     * @param owner 所属表
-     * @param dmlData dml data
-     * @param esFieldData es data
-     * @return 返回 id 值
-     */
-    public Object getESDataFromDmlData(ESMapping mapping, String owner, Map<String, Object> dmlData,
-                                       Map<String, Object> dmlOld, Map<String, Object> esFieldData) {
-        SchemaItem schemaItem = mapping.getSchemaItem();
-        String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
-        Object resultIdVal = null;
-        for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
-            ColumnItem columnItem = fieldItem.getColumnItems().iterator().next();
-            if (!columnItem.getOwner().equals(owner)) {
-                continue;
-            }
-            String columnName = columnItem.getColumnName();
-
-            if (fieldItem.getFieldName().equals(idFieldName)) {
-                resultIdVal = getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName);
-            }
-
-            if (dmlOld.containsKey(columnName) && !mapping.getSkips().contains(fieldItem.getFieldName())) {
-                esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()),
-                    getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName));
-            }
-        }
-
-        // 添加父子文档关联信息
-        putRelationData(mapping, schemaItem, dmlOld, esFieldData);
-        return resultIdVal;
-    }
-
-    private void putRelationDataFromRS(ESMapping mapping, SchemaItem schemaItem, ResultSet resultSet,
-                                       Map<String, Object> esFieldData) {
-        // 添加父子文档关联信息
-        if (!mapping.getRelations().isEmpty()) {
-            mapping.getRelations().forEach((relationField, relationMapping) -> {
-                Map<String, Object> relations = new HashMap<>();
-                relations.put("name", relationMapping.getName());
-                if (StringUtils.isNotEmpty(relationMapping.getParent())) {
-                    FieldItem parentFieldItem = schemaItem.getSelectFields().get(relationMapping.getParent());
-                    Object parentVal;
-                    try {
-                        parentVal = getValFromRS(mapping,
-                            resultSet,
-                            parentFieldItem.getFieldName(),
-                            parentFieldItem.getFieldName());
-                    } catch (SQLException e) {
-                        throw new RuntimeException(e);
-                    }
-                    if (parentVal != null) {
-                        relations.put("parent", parentVal.toString());
-                        esFieldData.put("$parent_routing", parentVal.toString());
-
-                    }
-                }
-                esFieldData.put(relationField, relations);
-            });
-        }
-    }
-
-    private void putRelationData(ESMapping mapping, SchemaItem schemaItem, Map<String, Object> dmlData,
-                                 Map<String, Object> esFieldData) {
-        // 添加父子文档关联信息
-        if (!mapping.getRelations().isEmpty()) {
-            mapping.getRelations().forEach((relationField, relationMapping) -> {
-                Map<String, Object> relations = new HashMap<>();
-                relations.put("name", relationMapping.getName());
-                if (StringUtils.isNotEmpty(relationMapping.getParent())) {
-                    FieldItem parentFieldItem = schemaItem.getSelectFields().get(relationMapping.getParent());
-                    String columnName = parentFieldItem.getColumnItems().iterator().next().getColumnName();
-                    Object parentVal = getValFromData(mapping, dmlData, parentFieldItem.getFieldName(), columnName);
-                    if (parentVal != null) {
-                        relations.put("parent", parentVal.toString());
-                        esFieldData.put("$parent_routing", parentVal.toString());
-
-                    }
-                }
-                esFieldData.put(relationField, relations);
-            });
-        }
-    }
-
-    /**
-     * es 字段类型本地缓存
-     */
-    private static ConcurrentMap<String, Map<String, String>> esFieldTypes = new ConcurrentHashMap<>();
-
-    /**
-     * 获取es mapping中的属性类型
-     *
-     * @param mapping mapping配置
-     * @param fieldName 属性名
-     * @return 类型
-     */
-    @SuppressWarnings("unchecked")
-    private String getEsType(ESMapping mapping, String fieldName) {
-        String key = mapping.get_index() + "-" + mapping.get_type();
-        Map<String, String> fieldType = esFieldTypes.get(key);
-        if (fieldType != null) {
-            return fieldType.get(fieldName);
-        } else {
-            MappingMetaData mappingMetaData = esConnection.getMapping(mapping.get_index(), mapping.get_type());
-
-            if (mappingMetaData == null) {
-                throw new IllegalArgumentException("Not found the mapping info of index: " + mapping.get_index());
-            }
-
-            fieldType = new LinkedHashMap<>();
-
-            Map<String, Object> sourceMap = mappingMetaData.getSourceAsMap();
-            Map<String, Object> esMapping = (Map<String, Object>) sourceMap.get("properties");
-            for (Map.Entry<String, Object> entry : esMapping.entrySet()) {
-                Map<String, Object> value = (Map<String, Object>) entry.getValue();
-                if (value.containsKey("properties")) {
-                    fieldType.put(entry.getKey(), "object");
-                } else {
-                    fieldType.put(entry.getKey(), (String) value.get("type"));
-                }
-            }
-            esFieldTypes.put(key, fieldType);
-
-            return fieldType.get(fieldName);
-        }
-    }
-
-}

client-adapter/elasticsearch/src/main/java/org/elasticsearch/client/RequestConvertersExt.java

@@ -1,30 +0,0 @@
-package org.elasticsearch.client;
-
-import org.apache.http.client.methods.HttpGet;
-import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
-import org.elasticsearch.common.Strings;
-
-import java.io.IOException;
-
-/**
- * RequestConverters扩展
- *
- * @author rewerma 2019-08-01
- * @version 1.0.0
- */
-public class RequestConvertersExt {
-
-    /**
-     * 修改 getMappings 去掉request参数
-     *
-     * @param getMappingsRequest
-     * @return
-     * @throws IOException
-     */
-    static Request getMappings(GetMappingsRequest getMappingsRequest) throws IOException {
-        String[] indices = getMappingsRequest.indices() == null ? Strings.EMPTY_ARRAY : getMappingsRequest.indices();
-        String[] types = getMappingsRequest.types() == null ? Strings.EMPTY_ARRAY : getMappingsRequest.types();
-
-        return new Request(HttpGet.METHOD_NAME, RequestConverters.endpoint(indices, "_mapping", types));
-    }
-}

client-adapter/elasticsearch/src/main/java/org/elasticsearch/client/RestHighLevelClientExt.java

@@ -1,29 +0,0 @@
-package org.elasticsearch.client;
-
-import static java.util.Collections.emptySet;
-
-import java.io.IOException;
-
-import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
-import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
-
-/**
- * RestHighLevelClient扩展
- *
- * @author rewerma 2019-08-01
- * @version 1.0.0
- */
-public class RestHighLevelClientExt {
-
-    public static GetMappingsResponse getMapping(RestHighLevelClient restHighLevelClient,
-                                                 GetMappingsRequest getMappingsRequest,
-                                                 RequestOptions options) throws IOException {
-        return restHighLevelClient.performRequestAndParseEntity(getMappingsRequest,
-            RequestConvertersExt::getMappings,
-            options,
-            GetMappingsResponse::fromXContent,
-            emptySet());
-
-    }
-
-}

client-adapter/elasticsearch/src/main/resources/META-INF/canal/com.alibaba.otter.canal.client.adapter.OuterAdapter

@@ -1 +0,0 @@
-es=com.alibaba.otter.canal.client.adapter.es.ESAdapter

client-adapter/elasticsearch/src/main/resources/es/biz_order.yml

@@ -1,21 +0,0 @@
-dataSourceKey: defaultDS
-destination: example
-groupId: g1
-esMapping:
-  _index: customer
-  _type: _doc
-  _id: _id
-  relations:
-    customer_order:
-      name: order
-      parent: customer_id
-  sql: "select concat('oid_', t.id) as _id,
-        t.customer_id,
-        t.id as order_id,
-        t.serial_code as order_serial,
-        t.c_time as order_time
-        from biz_order t"
-  skips:
-    - customer_id
-  etlCondition: "where t.c_time>={}"
-  commitBatch: 3000

client-adapter/elasticsearch/src/main/resources/es/customer.yml

@@ -1,47 +0,0 @@
-dataSourceKey: defaultDS
-destination: example
-groupId: g1
-esMapping:
-  _index: customer
-  _type: _doc
-  _id: id
-  relations:
-    customer_order:
-      name: customer
-  sql: "select t.id, t.name, t.email from customer t"
-  etlCondition: "where t.c_time>={}"
-  commitBatch: 3000
-
-
-#{
-#  "mappings":{
-#    "_doc":{
-#      "properties":{
-#        "id": {
-#          "type": "long"
-#        },
-#        "name": {
-#          "type": "text"
-#        },
-#        "email": {
-#          "type": "text"
-#        },
-#        "order_id": {
-#          "type": "long"
-#        },
-#        "order_serial": {
-#          "type": "text"
-#        },
-#        "order_time": {
-#          "type": "date"
-#        },
-#        "customer_order":{
-#          "type":"join",
-#          "relations":{
-#            "customer":"order"
-#          }
-#        }
-#      }
-#    }
-#  }
-#}

client-adapter/elasticsearch/src/main/resources/es/mytest_user.yml

@@ -1,16 +0,0 @@
-dataSourceKey: defaultDS
-destination: example
-groupId: g1
-esMapping:
-  _index: mytest_user
-  _type: _doc
-  _id: _id
-  upsert: true
-#  pk: id
-  sql: "select a.id as _id, a.name as _name, a.role_id as _role_id, b.role_name as _role_name,
-        a.c_time as _c_time from user a
-        left join role b on b.id=a.role_id"
-#  objFields:
-#    _labels: array:;
-  etlCondition: "where a.c_time>={}"
-  commitBatch: 3000

client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/ConfigLoadTest.java

@@ -1,40 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.es.test;
-
-import java.util.Map;
-
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
-import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfigLoader;
-import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
-@Ignore
-public class ConfigLoadTest {
-
-    @Before
-    public void before() {
-        // AdapterConfigs.put("es", "mytest_user.yml");
-        // 加载数据源连接池
-        DatasourceConfig.DATA_SOURCES.put("defaultDS", TestConstant.dataSource);
-    }
-
-    @Test
-    public void testLoad() {
-        Map<String, ESSyncConfig> configMap = ESSyncConfigLoader.load(null);
-        ESSyncConfig config = configMap.get("mytest_user.yml");
-        config.validate();
-        Assert.assertNotNull(config);
-        Assert.assertEquals("defaultDS", config.getDataSourceKey());
-        ESSyncConfig.ESMapping esMapping = config.getEsMapping();
-        Assert.assertEquals("mytest_user", esMapping.get_index());
-        Assert.assertEquals("_doc", esMapping.get_type());
-        Assert.assertEquals("id", esMapping.get_id());
-        Assert.assertNotNull(esMapping.getSql());
-
-        // Map<String, List<ESSyncConfig>> dbTableEsSyncConfig =
-        // ESSyncConfigLoader.getDbTableEsSyncConfig();
-        // Assert.assertFalse(dbTableEsSyncConfig.isEmpty());
-    }
-}

client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/ESTest.java

@@ -1,119 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.es.test;
-
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.util.LinkedHashMap;
-import java.util.Map;
-
-import org.elasticsearch.action.bulk.BulkItemResponse;
-import org.elasticsearch.action.bulk.BulkRequestBuilder;
-import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.client.transport.TransportClient;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.TransportAddress;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.rest.RestStatus;
-import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.transport.client.PreBuiltTransportClient;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-@Ignore
-public class ESTest {
-
-    private TransportClient transportClient;
-
-    @Before
-    public void init() throws UnknownHostException {
-        Settings.Builder settingBuilder = Settings.builder();
-        settingBuilder.put("cluster.name", TestConstant.clusterName);
-        Settings settings = settingBuilder.build();
-        transportClient = new PreBuiltTransportClient(settings);
-        String[] hostArray = TestConstant.esHosts.split(",");
-        for (String host : hostArray) {
-            int i = host.indexOf(":");
-            transportClient.addTransportAddress(new TransportAddress(InetAddress.getByName(host.substring(0, i)),
-                Integer.parseInt(host.substring(i + 1))));
-        }
-    }
-
-    @Test
-    public void test01() {
-        SearchResponse response = transportClient.prepareSearch("test")
-            .setTypes("osm")
-            .setQuery(QueryBuilders.termQuery("_id", "1"))
-            .setSize(10000)
-            .get();
-        for (SearchHit hit : response.getHits()) {
-            System.out.println(hit.getSourceAsMap().get("data").getClass());
-        }
-    }
-
-    @Test
-    public void test02() {
-        Map<String, Object> esFieldData = new LinkedHashMap<>();
-        esFieldData.put("userId", 2L);
-        esFieldData.put("eventId", 4L);
-        esFieldData.put("eventName", "网络异常");
-        esFieldData.put("description", "第四个事件信息");
-
-        Map<String, Object> relations = new LinkedHashMap<>();
-        esFieldData.put("user_event", relations);
-        relations.put("name", "event");
-        relations.put("parent", "2");
-
-        BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk();
-        bulkRequestBuilder
-            .add(transportClient.prepareIndex("test", "osm", "2_4").setRouting("2").setSource(esFieldData));
-        commit(bulkRequestBuilder);
-    }
-
-    @Test
-    public void test03() {
-        Map<String, Object> esFieldData = new LinkedHashMap<>();
-        esFieldData.put("userId", 2L);
-        esFieldData.put("eventName", "网络异常1");
-
-        Map<String, Object> relations = new LinkedHashMap<>();
-        esFieldData.put("user_event", relations);
-        relations.put("name", "event");
-        relations.put("parent", "2");
-
-        BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk();
-        bulkRequestBuilder.add(transportClient.prepareUpdate("test", "osm", "2_4").setRouting("2").setDoc(esFieldData));
-        commit(bulkRequestBuilder);
-    }
-
-    @Test
-    public void test04() {
-        BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk();
-        bulkRequestBuilder.add(transportClient.prepareDelete("test", "osm", "2_4"));
-        commit(bulkRequestBuilder);
-    }
-
-    private void commit(BulkRequestBuilder bulkRequestBuilder) {
-        if (bulkRequestBuilder.numberOfActions() > 0) {
-            BulkResponse response = bulkRequestBuilder.execute().actionGet();
-            if (response.hasFailures()) {
-                for (BulkItemResponse itemResponse : response.getItems()) {
-                    if (!itemResponse.isFailed()) {
-                        continue;
-                    }
-
-                    if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) {
-                        System.out.println(itemResponse.getFailureMessage());
-                    } else {
-                        System.out.println("ES bulk commit error" + itemResponse.getFailureMessage());
-                    }
-                }
-            }
-        }
-    }
-
-    @After
-    public void after() {
-        transportClient.close();
-    }
-}

client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/SqlParseTest.java

@@ -1,48 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.es.test;
-
-import java.util.List;
-import java.util.Map;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem;
-import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.FieldItem;
-import com.alibaba.otter.canal.client.adapter.es.config.SchemaItem.TableItem;
-import com.alibaba.otter.canal.client.adapter.es.config.SqlParser;
-
-public class SqlParseTest {
-
-    @Test
-    public void parseTest() {
-        String sql = "select a.id, CASE WHEN a.id <= 500 THEN '1' else '2' end as id2, "
-                     + "concat(a.name,'_test') as name, a.role_id, b.name as role_name, c.labels from user a "
-                     + "left join role b on a.role_id=b.id "
-                     + "left join (select user_id, group_concat(label,',') as labels from user_label "
-                     + "group by user_id) c on c.user_id=a.id";
-        SchemaItem schemaItem = SqlParser.parse(sql);
-
-        // 通过表名找 TableItem
-        List<TableItem> tableItems = schemaItem.getTableItemAliases().get("user_label".toLowerCase());
-        tableItems.forEach(tableItem -> Assert.assertEquals("c", tableItem.getAlias()));
-
-        TableItem tableItem = tableItems.get(0);
-        Assert.assertFalse(tableItem.isMain());
-        Assert.assertTrue(tableItem.isSubQuery());
-        // 通过字段名找 FieldItem
-        List<FieldItem> fieldItems = schemaItem.getColumnFields().get(tableItem.getAlias() + ".labels".toLowerCase());
-        fieldItems.forEach(
-            fieldItem -> Assert.assertEquals("c.labels", fieldItem.getOwner() + "." + fieldItem.getFieldName()));
-
-        // 获取当前表关联条件字段
-        Map<FieldItem, List<FieldItem>> relationTableFields = tableItem.getRelationTableFields();
-        relationTableFields.keySet()
-            .forEach(fieldItem -> Assert.assertEquals("user_id", fieldItem.getColumn().getColumnName()));
-
-        // 获取关联字段在select中的对应字段
-        // List<FieldItem> relationSelectFieldItem =
-        // tableItem.getRelationKeyFieldItems();
-        // relationSelectFieldItem.forEach(fieldItem -> Assert.assertEquals("c.labels",
-        // fieldItem.getOwner() + "." + fieldItem.getColumn().getColumnName()));
-    }
-}

client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/TestConstant.java

@@ -1,40 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.es.test;
-
-import java.sql.SQLException;
-
-import com.alibaba.druid.pool.DruidDataSource;
-
-public class TestConstant {
-
-    public final static String    jdbcUrl      = "jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true";
-    public final static String    jdbcUser     = "root";
-    public final static String    jdbcPassword = "121212";
-
-    public final static String    esHosts      = "127.0.0.1:9300";
-    public final static String    clusterName  = "elasticsearch";
-
-    public final static DruidDataSource dataSource;
-
-    static {
-        dataSource = new DruidDataSource();
-        dataSource.setDriverClassName("com.mysql.jdbc.Driver");
-        dataSource.setUrl(jdbcUrl);
-        dataSource.setUsername(jdbcUser);
-        dataSource.setPassword(jdbcPassword);
-        dataSource.setInitialSize(1);
-        dataSource.setMinIdle(1);
-        dataSource.setMaxActive(1);
-        dataSource.setMaxWait(60000);
-        dataSource.setTimeBetweenEvictionRunsMillis(60000);
-        dataSource.setMinEvictableIdleTimeMillis(300000);
-        dataSource.setPoolPreparedStatements(false);
-        dataSource.setMaxPoolPreparedStatementPerConnectionSize(20);
-        dataSource.setValidationQuery("select 1");
-        try {
-            dataSource.init();
-        } catch (SQLException e) {
-            e.printStackTrace();
-        }
-    }
-
-}

client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/Common.java

@@ -1,68 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.es.test.sync;
-
-import java.sql.Connection;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.HashMap;
-import java.util.Map;
-
-import javax.sql.DataSource;
-
-import com.alibaba.otter.canal.client.adapter.es.ESAdapter;
-import com.alibaba.otter.canal.client.adapter.es.test.TestConstant;
-import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
-import com.alibaba.otter.canal.client.adapter.support.OuterAdapterConfig;
-
-public class Common {
-
-    public static ESAdapter init() {
-        DatasourceConfig.DATA_SOURCES.put("defaultDS", TestConstant.dataSource);
-
-        OuterAdapterConfig outerAdapterConfig = new OuterAdapterConfig();
-        outerAdapterConfig.setName("es");
-        outerAdapterConfig.setHosts(TestConstant.esHosts);
-        Map<String, String> properties = new HashMap<>();
-        properties.put("cluster.name", TestConstant.clusterName);
-        outerAdapterConfig.setProperties(properties);
-
-        ESAdapter esAdapter = new ESAdapter();
-        esAdapter.init(outerAdapterConfig, null);
-        return esAdapter;
-    }
-
-    public static void sqlExe(DataSource dataSource, String sql) {
-        Connection conn = null;
-        Statement stmt = null;
-        try {
-            conn = dataSource.getConnection();
-            conn.setAutoCommit(false);
-            stmt = conn.createStatement();
-            stmt.execute(sql);
-            conn.commit();
-        } catch (Exception e) {
-            if (conn != null) {
-                try {
-                    conn.rollback();
-                } catch (SQLException e1) {
-                    // ignore
-                }
-            }
-            e.printStackTrace();
-        } finally {
-            if (stmt != null) {
-                try {
-                    stmt.close();
-                } catch (SQLException e) {
-                    // ignore
-                }
-            }
-            if (conn != null) {
-                try {
-                    conn.close();
-                } catch (SQLException e) {
-                    // ignore
-                }
-            }
-        }
-    }
-}

client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/LabelSyncJoinSub2Test.java

@@ -1,145 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.es.test.sync;
-
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-
-import javax.sql.DataSource;
-
-import org.elasticsearch.action.get.GetResponse;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import com.alibaba.otter.canal.client.adapter.es.ESAdapter;
-import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
-import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
-import com.alibaba.otter.canal.client.adapter.support.Dml;
-
-@Ignore
-public class LabelSyncJoinSub2Test {
-
-    private ESAdapter esAdapter;
-
-    @Before
-    public void init() {
-        // AdapterConfigs.put("es", "mytest_user_join_sub2.yml");
-        esAdapter = Common.init();
-    }
-
-    /**
-     * 带函数子查询从表插入
-     */
-    @Test
-    public void test01() {
-        DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
-        Common.sqlExe(ds, "delete from label where id=1 or id=2");
-        Common.sqlExe(ds, "insert into label (id,user_id,label) values (1,1,'a')");
-        Common.sqlExe(ds, "insert into label (id,user_id,label) values (2,1,'b')");
-
-        Dml dml = new Dml();
-        dml.setDestination("example");
-        dml.setTs(new Date().getTime());
-        dml.setType("INSERT");
-        dml.setDatabase("mytest");
-        dml.setTable("label");
-        List<Map<String, Object>> dataList = new ArrayList<>();
-        Map<String, Object> data = new LinkedHashMap<>();
-        dataList.add(data);
-        data.put("id", 2L);
-        data.put("user_id", 1L);
-        data.put("label", "b");
-        dml.setData(dataList);
-
-        String database = dml.getDatabase();
-        String table = dml.getTable();
-        Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
-
-        esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
-
-        GetResponse response = esAdapter.getEsConnection()
-            .getTransportClient()
-            .prepareGet("mytest_user", "_doc", "1")
-            .get();
-        Assert.assertEquals("b;a_", response.getSource().get("_labels"));
-    }
-
-    /**
-     * 带函数子查询从表更新
-     */
-    @Test
-    public void test02() {
-        DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
-        Common.sqlExe(ds, "update label set label='aa' where id=1");
-
-        Dml dml = new Dml();
-        dml.setDestination("example");
-        dml.setTs(new Date().getTime());
-        dml.setType("UPDATE");
-        dml.setDatabase("mytest");
-        dml.setTable("label");
-        List<Map<String, Object>> dataList = new ArrayList<>();
-        Map<String, Object> data = new LinkedHashMap<>();
-        dataList.add(data);
-        data.put("id", 1L);
-        data.put("user_id", 1L);
-        data.put("label", "aa");
-        dml.setData(dataList);
-
-        List<Map<String, Object>> oldList = new ArrayList<>();
-        Map<String, Object> old = new LinkedHashMap<>();
-        oldList.add(old);
-        old.put("label", "v");
-        dml.setOld(oldList);
-
-        String database = dml.getDatabase();
-        String table = dml.getTable();
-        Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
-
-        esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
-
-        GetResponse response = esAdapter.getEsConnection()
-            .getTransportClient()
-            .prepareGet("mytest_user", "_doc", "1")
-            .get();
-        Assert.assertEquals("b;aa_", response.getSource().get("_labels"));
-    }
-
-    /**
-     * 带函数子查询从表删除
-     */
-    @Test
-    public void test03() {
-        DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
-        Common.sqlExe(ds, "delete from label where id=1");
-
-        Dml dml = new Dml();
-        dml.setDestination("example");
-        dml.setTs(new Date().getTime());
-        dml.setType("DELETE");
-        dml.setDatabase("mytest");
-        dml.setTable("label");
-        List<Map<String, Object>> dataList = new ArrayList<>();
-        Map<String, Object> data = new LinkedHashMap<>();
-        dataList.add(data);
-        data.put("id", 1L);
-        data.put("user_id", 1L);
-        data.put("label", "a");
-        dml.setData(dataList);
-
-        String database = dml.getDatabase();
-        String table = dml.getTable();
-        Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
-
-        esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
-
-        GetResponse response = esAdapter.getEsConnection()
-            .getTransportClient()
-            .prepareGet("mytest_user", "_doc", "1")
-            .get();
-        Assert.assertEquals("b_", response.getSource().get("_labels"));
-    }
-}

client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/LabelSyncJoinSubTest.java

@@ -1,145 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.es.test.sync;
-
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-
-import javax.sql.DataSource;
-
-import org.elasticsearch.action.get.GetResponse;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import com.alibaba.otter.canal.client.adapter.es.ESAdapter;
-import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
-import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
-import com.alibaba.otter.canal.client.adapter.support.Dml;
-
-@Ignore
-public class LabelSyncJoinSubTest {
-
-    private ESAdapter esAdapter;
-
-    @Before
-    public void init() {
-        // AdapterConfigs.put("es", "mytest_user_join_sub.yml");
-        esAdapter = Common.init();
-    }
-
-    /**
-     * 子查询从表插入
-     */
-    @Test
-    public void test01() {
-        DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
-        Common.sqlExe(ds, "delete from label where id=1 or id=2");
-        Common.sqlExe(ds, "insert into label (id,user_id,label) values (1,1,'a')");
-        Common.sqlExe(ds, "insert into label (id,user_id,label) values (2,1,'b')");
-
-        Dml dml = new Dml();
-        dml.setDestination("example");
-        dml.setTs(new Date().getTime());
-        dml.setType("INSERT");
-        dml.setDatabase("mytest");
-        dml.setTable("label");
-        List<Map<String, Object>> dataList = new ArrayList<>();
-        Map<String, Object> data = new LinkedHashMap<>();
-        dataList.add(data);
-        data.put("id", 2L);
-        data.put("user_id", 1L);
-        data.put("label", "b");
-        dml.setData(dataList);
-
-        String database = dml.getDatabase();
-        String table = dml.getTable();
-        Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
-
-        esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
-
-        GetResponse response = esAdapter.getEsConnection()
-            .getTransportClient()
-            .prepareGet("mytest_user", "_doc", "1")
-            .get();
-        Assert.assertEquals("b;a", response.getSource().get("_labels"));
-    }
-
-    /**
-     * 子查询从表更新
-     */
-    @Test
-    public void test02() {
-        DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
-        Common.sqlExe(ds, "update label set label='aa' where id=1");
-
-        Dml dml = new Dml();
-        dml.setDestination("example");
-        dml.setTs(new Date().getTime());
-        dml.setType("UPDATE");
-        dml.setDatabase("mytest");
-        dml.setTable("label");
-        List<Map<String, Object>> dataList = new ArrayList<>();
-        Map<String, Object> data = new LinkedHashMap<>();
-        dataList.add(data);
-        data.put("id", 1L);
-        data.put("user_id", 1L);
-        data.put("label", "aa");
-        dml.setData(dataList);
-
-        List<Map<String, Object>> oldList = new ArrayList<>();
-        Map<String, Object> old = new LinkedHashMap<>();
-        oldList.add(old);
-        old.put("label", "a");
-        dml.setOld(oldList);
-
-        String database = dml.getDatabase();
-        String table = dml.getTable();
-        Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
-
-        esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
-
-        GetResponse response = esAdapter.getEsConnection()
-            .getTransportClient()
-            .prepareGet("mytest_user", "_doc", "1")
-            .get();
-        Assert.assertEquals("b;aa", response.getSource().get("_labels"));
-    }
-
-    /**
-     * 子查询从表删除
-     */
-    @Test
-    public void test03() {
-        DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
-        Common.sqlExe(ds, "delete from label where id=1");
-
-        Dml dml = new Dml();
-        dml.setDestination("example");
-        dml.setTs(new Date().getTime());
-        dml.setType("DELETE");
-        dml.setDatabase("mytest");
-        dml.setTable("label");
-        List<Map<String, Object>> dataList = new ArrayList<>();
-        Map<String, Object> data = new LinkedHashMap<>();
-        dataList.add(data);
-        data.put("id", 1L);
-        data.put("user_id", 1L);
-        data.put("label", "a");
-        dml.setData(dataList);
-
-        String database = dml.getDatabase();
-        String table = dml.getTable();
-        Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
-
-        esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
-
-        GetResponse response = esAdapter.getEsConnection()
-            .getTransportClient()
-            .prepareGet("mytest_user", "_doc", "1")
-            .get();
-        Assert.assertEquals("b", response.getSource().get("_labels"));
-    }
-}

client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOne2Test.java

@@ -1,107 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.es.test.sync;
-
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-
-import javax.sql.DataSource;
-
-import org.elasticsearch.action.get.GetResponse;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import com.alibaba.otter.canal.client.adapter.es.ESAdapter;
-import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
-import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
-import com.alibaba.otter.canal.client.adapter.support.Dml;
-
-@Ignore
-public class RoleSyncJoinOne2Test {
-
-    private ESAdapter esAdapter;
-
-    @Before
-    public void init() {
-        // AdapterConfigs.put("es", "mytest_user_join_one2.yml");
-        esAdapter = Common.init();
-    }
-
-    /**
-     * 带函数非子查询从表插入
-     */
-    @Test
-    public void test01() {
-        DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
-        Common.sqlExe(ds, "delete from role where id=1");
-        Common.sqlExe(ds, "insert into role (id,role_name) values (1,'admin')");
-
-        Dml dml = new Dml();
-        dml.setDestination("example");
-        dml.setTs(new Date().getTime());
-        dml.setType("INSERT");
-        dml.setDatabase("mytest");
-        dml.setTable("role");
-        List<Map<String, Object>> dataList = new ArrayList<>();
-        Map<String, Object> data = new LinkedHashMap<>();
-        dataList.add(data);
-        data.put("id", 1L);
-        data.put("role_name", "admin");
-        dml.setData(dataList);
-
-        String database = dml.getDatabase();
-        String table = dml.getTable();
-        Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
-
-        esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
-
-        GetResponse response = esAdapter.getEsConnection()
-            .getTransportClient()
-            .prepareGet("mytest_user", "_doc", "1")
-            .get();
-        Assert.assertEquals("admin_", response.getSource().get("_role_name"));
-    }
-
-    /**
-     * 带函数非子查询从表更新
-     */
-    @Test
-    public void test02() {
-        DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
-        Common.sqlExe(ds, "update role set role_name='admin3' where id=1");
-
-        Dml dml = new Dml();
-        dml.setDestination("example");
-        dml.setTs(new Date().getTime());
-        dml.setType("UPDATE");
-        dml.setDatabase("mytest");
-        dml.setTable("role");
-        List<Map<String, Object>> dataList = new ArrayList<>();
-        Map<String, Object> data = new LinkedHashMap<>();
-        dataList.add(data);
-        data.put("id", 1L);
-        data.put("role_name", "admin3");
-        dml.setData(dataList);
-
-        List<Map<String, Object>> oldList = new ArrayList<>();
-        Map<String, Object> old = new LinkedHashMap<>();
-        oldList.add(old);
-        old.put("role_name", "admin");
-        dml.setOld(oldList);
-
-        String database = dml.getDatabase();
-        String table = dml.getTable();
-        Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
-
-        esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
-
-        GetResponse response = esAdapter.getEsConnection()
-            .getTransportClient()
-            .prepareGet("mytest_user", "_doc", "1")
-            .get();
-        Assert.assertEquals("admin3_", response.getSource().get("_role_name"));
-    }
-}

client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/RoleSyncJoinOneTest.java

@@ -1,197 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.es.test.sync;
-
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-
-import javax.sql.DataSource;
-
-import org.elasticsearch.action.get.GetResponse;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import com.alibaba.otter.canal.client.adapter.es.ESAdapter;
-import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
-import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
-import com.alibaba.otter.canal.client.adapter.support.Dml;
-@Ignore
-public class RoleSyncJoinOneTest {
-
-    private ESAdapter esAdapter;
-
-    @Before
-    public void init() {
-        // AdapterConfigs.put("es", "mytest_user_join_one.yml");
-        esAdapter = Common.init();
-    }
-
-    /**
-     * 非子查询从表插入
-     */
-    @Test
-    public void test01() {
-        DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
-        Common.sqlExe(ds, "delete from role where id=1");
-        Common.sqlExe(ds, "insert into role (id,role_name) values (1,'admin')");
-
-        Dml dml = new Dml();
-        dml.setDestination("example");
-        dml.setTs(new Date().getTime());
-        dml.setType("INSERT");
-        dml.setDatabase("mytest");
-        dml.setTable("role");
-        List<Map<String, Object>> dataList = new ArrayList<>();
-        Map<String, Object> data = new LinkedHashMap<>();
-        dataList.add(data);
-        data.put("id", 1L);
-        data.put("role_name", "admin");
-
-        dml.setData(dataList);
-
-        String database = dml.getDatabase();
-        String table = dml.getTable();
-        Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
-
-        esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
-
-        GetResponse response = esAdapter.getEsConnection().getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
-        Assert.assertEquals("admin", response.getSource().get("_role_name"));
-    }
-
-    /**
-     * 非子查询从表更新
-     */
-    @Test
-    public void test02() {
-        DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
-        Common.sqlExe(ds, "update role set role_name='admin2' where id=1");
-
-        Dml dml = new Dml();
-        dml.setDestination("example");
-        dml.setTs(new Date().getTime());
-        dml.setType("UPDATE");
-        dml.setDatabase("mytest");
-        dml.setTable("role");
-        List<Map<String, Object>> dataList = new ArrayList<>();
-        Map<String, Object> data = new LinkedHashMap<>();
-        dataList.add(data);
-        data.put("id", 1L);
-        data.put("role_name", "admin2");
-        dml.setData(dataList);
-
-        List<Map<String, Object>> oldList = new ArrayList<>();
-        Map<String, Object> old = new LinkedHashMap<>();
-        oldList.add(old);
-        old.put("role_name", "admin");
-        dml.setOld(oldList);
-
-        String database = dml.getDatabase();
-        String table = dml.getTable();
-        Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
-
-        esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
-
-        GetResponse response = esAdapter.getEsConnection().getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
-        Assert.assertEquals("admin2", response.getSource().get("_role_name"));
-    }
-
-    /**
-     * 主表更新外键值
-     */
-    @Test
-    public void test03() {
-        DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
-        Common.sqlExe(ds, "delete from role where id=2");
-        Common.sqlExe(ds, "insert into role (id,role_name) values (2,'operator')");
-        Common.sqlExe(ds, "update user set role_id=2 where id=1");
-
-        Dml dml = new Dml();
-        dml.setDestination("example");
-        dml.setTs(new Date().getTime());
-        dml.setType("UPDATE");
-        dml.setDatabase("mytest");
-        dml.setTable("user");
-        List<Map<String, Object>> dataList = new ArrayList<>();
-        Map<String, Object> data = new LinkedHashMap<>();
-        dataList.add(data);
-        data.put("id", 1L);
-        data.put("role_id", 2L);
-        dml.setData(dataList);
-        List<Map<String, Object>> oldList = new ArrayList<>();
-        Map<String, Object> old = new LinkedHashMap<>();
-        oldList.add(old);
-        old.put("role_id", 1L);
-        dml.setOld(oldList);
-
-        String database = dml.getDatabase();
-        String table = dml.getTable();
-        Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
-
-        esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
-
-        GetResponse response = esAdapter.getEsConnection().getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
-        Assert.assertEquals("operator", response.getSource().get("_role_name"));
-
-        Common.sqlExe(ds, "update user set role_id=1 where id=1");
-
-        Dml dml2 = new Dml();
-        dml2.setDestination("example");
-        dml2.setTs(new Date().getTime());
-        dml2.setType("UPDATE");
-        dml2.setDatabase("mytest");
-        dml2.setTable("user");
-        List<Map<String, Object>> dataList2 = new ArrayList<>();
-        Map<String, Object> data2 = new LinkedHashMap<>();
-        dataList2.add(data2);
-        data2.put("id", 1L);
-        data2.put("role_id", 1L);
-        dml2.setData(dataList2);
-        List<Map<String, Object>> oldList2 = new ArrayList<>();
-        Map<String, Object> old2 = new LinkedHashMap<>();
-        oldList2.add(old2);
-        old2.put("role_id", 2L);
-        dml2.setOld(oldList2);
-
-        esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml2);
-
-        GetResponse response2 = esAdapter.getEsConnection().getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
-        Assert.assertEquals("admin2", response2.getSource().get("_role_name"));
-    }
-
-    /**
-     * 非子查询从表删除
-     */
-    @Test
-    public void test04() {
-        DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
-        Common.sqlExe(ds, "delete from role where id=1");
-
-        Dml dml = new Dml();
-        dml.setDestination("example");
-        dml.setTs(new Date().getTime());
-        dml.setType("DELETE");
-        dml.setDatabase("mytest");
-        dml.setTable("role");
-        List<Map<String, Object>> dataList = new ArrayList<>();
-        Map<String, Object> data = new LinkedHashMap<>();
-        dataList.add(data);
-        data.put("id", 1L);
-        data.put("role_name", "admin");
-
-        dml.setData(dataList);
-
-        String database = dml.getDatabase();
-        String table = dml.getTable();
-        Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
-
-        esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
-
-        GetResponse response = esAdapter.getEsConnection()
-            .getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
-        Assert.assertNull(response.getSource().get("_role_name"));
-    }
-}

client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncJoinOneTest.java

@@ -1,108 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.es.test.sync;
-
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-
-import javax.sql.DataSource;
-
-import org.elasticsearch.action.get.GetResponse;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import com.alibaba.otter.canal.client.adapter.es.ESAdapter;
-import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
-import com.alibaba.otter.canal.client.adapter.support.DatasourceConfig;
-import com.alibaba.otter.canal.client.adapter.support.Dml;
-
-@Ignore
-public class UserSyncJoinOneTest {
-
-    private ESAdapter esAdapter;
-
-    @Before
-    public void init() {
-        // AdapterConfigs.put("es", "mytest_user_join_one.yml");
-        esAdapter = Common.init();
-    }
-
-    /**
-     * 主表带函数插入
-     */
-    @Test
-    public void test01() {
-        DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
-        Common.sqlExe(ds, "delete from user where id=1");
-        Common.sqlExe(ds, "insert into user (id,name,role_id) values (1,'Eric',1)");
-
-        Dml dml = new Dml();
-        dml.setDestination("example");
-        dml.setTs(new Date().getTime());
-        dml.setType("INSERT");
-        dml.setDatabase("mytest");
-        dml.setTable("user");
-        List<Map<String, Object>> dataList = new ArrayList<>();
-        Map<String, Object> data = new LinkedHashMap<>();
-        dataList.add(data);
-        data.put("id", 1L);
-        data.put("name", "Eric");
-        data.put("role_id", 1L);
-        data.put("c_time", new Date());
-        dml.setData(dataList);
-
-        String database = dml.getDatabase();
-        String table = dml.getTable();
-        Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
-
-        esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
-
-        GetResponse response = esAdapter.getEsConnection()
-            .getTransportClient()
-            .prepareGet("mytest_user", "_doc", "1")
-            .get();
-        Assert.assertEquals("Eric_", response.getSource().get("_name"));
-    }
-
-    /**
-     * 主表带函数更新
-     */
-    @Test
-    public void test02() {
-        DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
-        Common.sqlExe(ds, "update user set name='Eric2' where id=1");
-
-        Dml dml = new Dml();
-        dml.setDestination("example");
-        dml.setTs(new Date().getTime());
-        dml.setType("UPDATE");
-        dml.setDatabase("mytest");
-        dml.setTable("user");
-        List<Map<String, Object>> dataList = new ArrayList<>();
-        Map<String, Object> data = new LinkedHashMap<>();
-        dataList.add(data);
-        data.put("id", 1L);
-        data.put("name", "Eric2");
-        dml.setData(dataList);
-        List<Map<String, Object>> oldList = new ArrayList<>();
-        Map<String, Object> old = new LinkedHashMap<>();
-        oldList.add(old);
-        old.put("name", "Eric");
-        dml.setOld(oldList);
-
-        String database = dml.getDatabase();
-        String table = dml.getTable();
-        Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
-
-        esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
-
-        GetResponse response = esAdapter.getEsConnection()
-            .getTransportClient()
-            .prepareGet("mytest_user", "_doc", "1")
-            .get();
-        Assert.assertEquals("Eric2_", response.getSource().get("_name"));
-    }
-}

client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/UserSyncSingleTest.java

@@ -1,137 +0,0 @@
-package com.alibaba.otter.canal.client.adapter.es.test.sync;
-
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.elasticsearch.action.get.GetResponse;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import com.alibaba.otter.canal.client.adapter.es.ESAdapter;
-import com.alibaba.otter.canal.client.adapter.es.config.ESSyncConfig;
-import com.alibaba.otter.canal.client.adapter.support.Dml;
-
-@Ignore
-public class UserSyncSingleTest {
-
-    private ESAdapter esAdapter;
-
-    @Before
-    public void init() {
-        // AdapterConfigs.put("es", "mytest_user_single.yml");
-        esAdapter = Common.init();
-    }
-
-    /**
-     * 单表插入
-     */
-    @Test
-    public void test01() {
-        Dml dml = new Dml();
-        dml.setDestination("example");
-        dml.setTs(new Date().getTime());
-        dml.setType("INSERT");
-        dml.setDatabase("mytest");
-        dml.setTable("user");
-        List<Map<String, Object>> dataList = new ArrayList<>();
-        Map<String, Object> data = new LinkedHashMap<>();
-        dataList.add(data);
-        data.put("id", 1L);
-        data.put("name", "Eric");
-        data.put("role_id", 1L);
-        data.put("c_time", new Date());
-        dml.setData(dataList);
-
-        String database = dml.getDatabase();
-        String table = dml.getTable();
-        Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
-
-        esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
-
-        GetResponse response = esAdapter.getEsConnection()
-            .getTransportClient()
-            .prepareGet("mytest_user", "_doc", "1")
-            .get();
-        Assert.assertEquals("Eric", response.getSource().get("_name"));
-    }
-
-    /**
-     * 单表更新
-     */
-    @Test
-    public void test02() {
-        Dml dml = new Dml();
-        dml.setDestination("example");
-        dml.setTs(new Date().getTime());
-        dml.setType("UPDATE");
-        dml.setDatabase("mytest");
-        dml.setTable("user");
-        List<Map<String, Object>> dataList = new ArrayList<>();
-        Map<String, Object> data = new LinkedHashMap<>();
-        dataList.add(data);
-        data.put("id", 1L);
-        data.put("name", "Eric2");
-        dml.setData(dataList);
-        List<Map<String, Object>> oldList = new ArrayList<>();
-        Map<String, Object> old = new LinkedHashMap<>();
-        oldList.add(old);
-        old.put("name", "Eric");
-        dml.setOld(oldList);
-
-        String database = dml.getDatabase();
-        String table = dml.getTable();
-        Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
-
-        esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
-
-        GetResponse response = esAdapter.getEsConnection()
-            .getTransportClient()
-            .prepareGet("mytest_user", "_doc", "1")
-            .get();
-        Assert.assertEquals("Eric2", response.getSource().get("_name"));
-    }
-
-    /**
-     * 单表删除
-     */
-    @Test
-    public void test03() {
-        Dml dml = new Dml();
-        dml.setDestination("example");
-        dml.setTs(new Date().getTime());
-        dml.setType("DELETE");
-        dml.setDatabase("mytest");
-        dml.setTable("user");
-        List<Map<String, Object>> dataList = new ArrayList<>();
-        Map<String, Object> data = new LinkedHashMap<>();
-        dataList.add(data);
-        data.put("id", 1L);
-        data.put("name", "Eric");
-        data.put("role_id", 1L);
-        data.put("c_time", new Date());
-        dml.setData(dataList);
-
-        String database = dml.getDatabase();
-        String table = dml.getTable();
-        Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
-
-        esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
-
-        GetResponse response = esAdapter.getEsConnection()
-            .getTransportClient()
-            .prepareGet("mytest_user", "_doc", "1")
-            .get();
-        Assert.assertNull(response.getSource());
-    }
-
-    // @After
-    // public void after() {
-    // esAdapter.destroy();
-    // DatasourceConfig.DATA_SOURCES.values().forEach(DruidDataSource::close);
-    // }
-}

client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/db_schema.sql

@@ -1,39 +0,0 @@
--- ----------------------------
--- Table structure for label
--- ----------------------------
-DROP TABLE IF EXISTS `label`;
-CREATE TABLE `label` (
-  `id` bigint(20) NOT NULL AUTO_INCREMENT,
-  `user_id` bigint(20) NOT NULL,
-  `label` varchar(30) NOT NULL,
-  PRIMARY KEY (`id`)
-) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb4;
-
--- ----------------------------
--- Table structure for role
--- ----------------------------
-DROP TABLE IF EXISTS `role`;
-CREATE TABLE `role` (
-  `id` bigint(20) NOT NULL AUTO_INCREMENT,
-  `role_name` varchar(30) NOT NULL,
-  PRIMARY KEY (`id`)
-) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb4;
-
--- ----------------------------
--- Table structure for user
--- ----------------------------
-DROP TABLE IF EXISTS `user`;
-CREATE TABLE `user` (
-  `id` bigint(20) NOT NULL AUTO_INCREMENT,
-  `name` varchar(30) NOT NULL,
-  `c_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
-  `role_id` bigint(20) DEFAULT NULL,
-  PRIMARY KEY (`id`)
-) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb4;
-
-insert into user (id,name,role_id) values (1,'Eric',1);
-insert into role (id,role_name) values (1,'admin');
-insert into role (id,role_name) values (2,'operator');
-insert into label (id,user_id,label) values (1,1,'a');
-insert into label (id,user_id,label) values (2,1,'b');
-commit;

client-adapter/elasticsearch/src/test/java/com/alibaba/otter/canal/client/adapter/es/test/sync/es_mapping.json

@@ -1,21 +0,0 @@
-{
-  "_doc": {
-    "properties": {
-      "_name": {
-        "type": "text"
-      },
-      "_role_id": {
-        "type": "long"
-      },
-      "_role_name": {
-        "type": "text"
-      },
-      "_labels": {
-        "type": "text"
-      },
-      "_c_time": {
-        "type": "date"
-      }
-    }
-  }
-}

client-adapter/elasticsearch/src/test/resources/es/mytest_user_single.yml_

@@ -1,8 +0,0 @@
-dataSourceKey: defaultDS
-destination: example
-esMapping:
-  _index: mytest_user
-  _type: _doc
-  _id: _id
-  sql: "select a.id as _id, a.name as _name, a.role_id as _role_id, a.c_time as _c_time from user a"
-  commitBatch: 3000

client-adapter/elasticsearch/src/test/resources/log4j2-test.xml

@@ -1,13 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<Configuration status="WARN">
-    <Appenders>
-        <Console name="Console" target="SYSTEM_OUT">
-            <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
-        </Console>
-    </Appenders>
-    <Loggers>
-        <Root level="ERROR">
-            <AppenderRef ref="Console"/>
-        </Root>
-    </Loggers>
-</Configuration>

client-adapter/elasticsearch/src/test/resources/logback-test.xml

@@ -1,13 +0,0 @@
-<configuration scan="true" scanPeriod=" 5 seconds">
-	<jmxConfigurator />
-	<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
-		<encoder>
-			<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{56} - %msg%n
-			</pattern>
-		</encoder>
-	</appender>
-
-	<root level="TRACE">
-		<appender-ref ref="STDOUT"/>
-	</root>
-</configuration>

client-adapter/pom.xml

@@ -20,7 +20,6 @@
         <module>common</module>
         <module>logger</module>
         <module>hbase</module>
-        <module>elasticsearch</module>
         <module>launcher</module>
         <module>rdb</module>
         <module>es6x</module>