template config load (#2123)

* template config load

* compile vue frontend for product

canal-admin/canal-admin-server/src/main/assembly/dev.xml

@@ -29,6 +29,8 @@
                 <include>logback.xml</include>
                 <include>public/**</include>
                 <include>canal_manager.sql</include>
+                <include>canal-template.properties</include>
+                <include>instance-template.properties</include>
             </includes>
         </fileSet>
         <fileSet>

canal-admin/canal-admin-server/src/main/assembly/release.xml

@@ -30,6 +30,8 @@
                 <include>logback.xml</include>
                 <include>public/**</include>
                 <include>canal_manager.sql</include>
+                <include>canal-template.properties</include>
+                <include>instance-template.properties</include>
             </includes>
         </fileSet>
         <fileSet>

canal-admin/canal-admin-server/src/main/java/com/alibaba/otter/canal/admin/common/TemplateConfigLoader.java

@@ -0,0 +1,50 @@
+package com.alibaba.otter.canal.admin.common;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TemplateConfigLoader {
+
+    private static final Logger logger              = LoggerFactory.getLogger(TemplateConfigLoader.class);
+
+    public static final String  CONF_DIR            = "conf";
+    public static final String  CANAL_CONFIG_TMP    = "canal-template.properties";
+    public static final String  INSTANCE_CONFIG_TMP = "instance-template.properties";
+
+    public static String loadCanalConfig() {
+        return loadFile(CANAL_CONFIG_TMP);
+    }
+
+    public static String loadInstanceConfig() {
+        return loadFile(INSTANCE_CONFIG_TMP);
+    }
+
+    private static String loadFile(String fileName) {
+        File configFile = new File(".." + File.separator + CONF_DIR + File.separator + fileName);
+        if (!configFile.exists()) {
+            URL url = TemplateConfigLoader.class.getClassLoader().getResource("");
+            if (url != null) {
+                configFile = new File(url.getPath() + fileName + File.separator);
+            }
+        }
+        if (!configFile.exists()) {
+            return null;
+        }
+        try (InputStream in = new FileInputStream(configFile)) {
+            byte[] bytes = new byte[in.available()];
+            in.read(bytes);
+            return new String(bytes, StandardCharsets.UTF_8);
+        } catch (IOException e) {
+            logger.error("Read " + fileName + " error", e);
+        }
+
+        return null;
+    }
+}

canal-admin/canal-admin-server/src/main/java/com/alibaba/otter/canal/admin/controller/CanalConfigController.java

@@ -1,5 +1,6 @@
 package com.alibaba.otter.canal.admin.controller;
 
+import com.alibaba.otter.canal.admin.common.TemplateConfigLoader;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.PathVariable;
@@ -50,4 +51,8 @@ public class CanalConfigController {
         return BaseModel.getInstance("success");
     }
 
+    @GetMapping(value = "/config/template")
+    public BaseModel<String> template(@PathVariable String env) {
+        return BaseModel.getInstance(TemplateConfigLoader.loadCanalConfig());
+    }
 }

canal-admin/canal-admin-server/src/main/java/com/alibaba/otter/canal/admin/controller/CanalInstanceController.java

@@ -3,6 +3,7 @@ package com.alibaba.otter.canal.admin.controller;
 import java.util.List;
 import java.util.Map;
 
+import com.alibaba.otter.canal.admin.common.TemplateConfigLoader;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.DeleteMapping;
 import org.springframework.web.bind.annotation.GetMapping;
@@ -142,7 +143,8 @@ public class CanalInstanceController {
      * @return 是否成功
      */
     @PutMapping(value = "/instance/status/{id}")
-    public BaseModel<Boolean> instanceStart(@PathVariable Long id, @RequestParam String option, @PathVariable String env) {
+    public BaseModel<Boolean> instanceStart(@PathVariable Long id, @RequestParam String option,
+                                            @PathVariable String env) {
         return BaseModel.getInstance(canalInstanceConfigService.instanceOperation(id, option));
     }
 
@@ -171,4 +173,9 @@ public class CanalInstanceController {
     public BaseModel<List<CanalInstanceConfig>> activeInstances(@PathVariable Long serverId, @PathVariable String env) {
         return BaseModel.getInstance(canalInstanceConfigService.findActiveInstanceByServerId(serverId));
     }
+
+    @GetMapping(value = "/instance/template")
+    public BaseModel<String> template(@PathVariable String env) {
+        return BaseModel.getInstance(TemplateConfigLoader.loadInstanceConfig());
+    }
 }

canal-admin/canal-admin-server/src/main/java/com/alibaba/otter/canal/admin/model/CanalConfig.java

@@ -39,6 +39,10 @@ public class CanalConfig extends Model {
     private String status;
     private Date   modifiedTime;
 
+    public void init() {
+        this.name = "canal.properties";
+    }
+
     public Long getId() {
         return id;
     }

canal-admin/canal-admin-server/src/main/java/com/alibaba/otter/canal/admin/service/impl/NodeServerServiceImpl.java

@@ -1,7 +1,6 @@
 package com.alibaba.otter.canal.admin.service.impl;
 
-import io.ebean.Query;
-
+import java.security.NoSuchAlgorithmException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.ExecutionException;
@@ -12,6 +11,7 @@ import java.util.concurrent.TimeoutException;
 import org.apache.commons.lang.StringUtils;
 import org.springframework.stereotype.Service;
 
+import com.alibaba.otter.canal.admin.common.TemplateConfigLoader;
 import com.alibaba.otter.canal.admin.common.Threads;
 import com.alibaba.otter.canal.admin.common.exception.ServiceException;
 import com.alibaba.otter.canal.admin.connector.AdminConnector;
@@ -21,6 +21,9 @@ import com.alibaba.otter.canal.admin.model.CanalInstanceConfig;
 import com.alibaba.otter.canal.admin.model.NodeServer;
 import com.alibaba.otter.canal.admin.model.Pager;
 import com.alibaba.otter.canal.admin.service.NodeServerService;
+import com.alibaba.otter.canal.protocol.SecurityUtil;
+
+import io.ebean.Query;
 
 /**
  * 节点信息业务层
@@ -42,6 +45,20 @@ public class NodeServerServiceImpl implements NodeServerService {
         }
 
         nodeServer.save();
+
+        if (nodeServer.getClusterId() == null) { // 单机模式
+            CanalConfig canalConfig = new CanalConfig();
+            canalConfig.setServerId(nodeServer.getId());
+            String configTmp = TemplateConfigLoader.loadCanalConfig();
+            canalConfig.setContent(configTmp);
+            try {
+                String contentMd5 = SecurityUtil.md5String(canalConfig.getContent());
+                canalConfig.setContentMd5(contentMd5);
+            } catch (NoSuchAlgorithmException e) {
+                e.printStackTrace();
+            }
+            canalConfig.save();
+        }
     }
 
     public NodeServer detail(Long id) {
@@ -159,9 +176,8 @@ public class NodeServerServiceImpl implements NodeServerService {
         if (nodeServer == null) {
             return "";
         }
-        return SimpleAdminConnectors.execute(nodeServer.getIp(),
-            nodeServer.getAdminPort(),
-            adminConnector -> adminConnector.canalLog(100));
+        return SimpleAdminConnectors
+            .execute(nodeServer.getIp(), nodeServer.getAdminPort(), adminConnector -> adminConnector.canalLog(100));
     }
 
     public boolean remoteOperation(Long id, String option) {
@@ -171,7 +187,8 @@ public class NodeServerServiceImpl implements NodeServerService {
         }
         Boolean result = null;
         if ("start".equals(option)) {
-            result = SimpleAdminConnectors.execute(nodeServer.getIp(), nodeServer.getAdminPort(), AdminConnector::start);
+            result = SimpleAdminConnectors
+                .execute(nodeServer.getIp(), nodeServer.getAdminPort(), AdminConnector::start);
         } else if ("stop".equals(option)) {
             result = SimpleAdminConnectors.execute(nodeServer.getIp(), nodeServer.getAdminPort(), AdminConnector::stop);
         } else {

canal-admin/canal-admin-server/src/main/resources/application.yml

@@ -7,8 +7,8 @@ spring:
 
 spring.datasource:
   url: jdbc:mysql://127.0.0.1:3306/canal_manager?useUnicode=true&characterEncoding=UTF-8&useSSL=false
-  username: canal
-  password: canal
+  username: root
+  password: 121212
   driver-class-name: com.mysql.jdbc.Driver
   hikari:
     maximum-pool-size: 10

deployer/src/main/resources/canal_manager.properties → canal-admin/canal-admin-server/src/main/resources/canal-template.properties

@@ -93,7 +93,7 @@ canal.aliyun.secretKey =
 #################################################
 ######### 		destinations		#############
 #################################################
-canal.destinations = 
+canal.destinations =
 # conf root dir
 canal.conf.dir = ../conf
 # auto scan instance dir add/remove and start/stop instance
@@ -136,4 +136,4 @@ canal.mq.accessChannel = local
 ##################################################
 canal.mq.kafka.kerberos.enable = false
 canal.mq.kafka.kerberos.krb5FilePath = "../conf/kerberos/krb5.conf"
-canal.mq.kafka.kerberos.jaasFilePath = "../conf/kerberos/jaas.conf"
+canal.mq.kafka.kerberos.jaasFilePath = "../conf/kerberos/jaas.conf"

canal-admin/canal-admin-server/src/main/resources/instance-template.properties

@@ -0,0 +1,57 @@
+#################################################
+## mysql serverId , v1.0.26+ will autoGen
+# canal.instance.mysql.slaveId=0
+
+# enable gtid use true/false
+canal.instance.gtidon=false
+
+# position info
+canal.instance.master.address=127.0.0.1:3306
+canal.instance.master.journal.name=
+canal.instance.master.position=
+canal.instance.master.timestamp=
+canal.instance.master.gtid=
+
+# rds oss binlog
+canal.instance.rds.accesskey=
+canal.instance.rds.secretkey=
+canal.instance.rds.instanceId=
+
+# table meta tsdb info
+canal.instance.tsdb.enable=true
+#canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/canal_tsdb
+#canal.instance.tsdb.dbUsername=canal
+#canal.instance.tsdb.dbPassword=canal
+
+#canal.instance.standby.address =
+#canal.instance.standby.journal.name =
+#canal.instance.standby.position =
+#canal.instance.standby.timestamp =
+#canal.instance.standby.gtid=
+
+# username/password
+canal.instance.dbUsername=canal
+canal.instance.dbPassword=canal
+canal.instance.connectionCharset = UTF-8
+# enable druid Decrypt database password
+canal.instance.enableDruid=false
+#canal.instance.pwdPublicKey=MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBALK4BUxdDltRRE5/zXpVEVPUgunvscYFtEip3pmLlhrWpacX7y7GCMo2/JM6LeHmiiNdH1FWgGCpUfircSwlWKUCAwEAAQ==
+
+# table regex
+canal.instance.filter.regex=.*\\..*
+# table black regex
+canal.instance.filter.black.regex=
+# table field filter(format: schema1.tableName1:field1/field2,schema2.tableName2:field1/field2)
+#canal.instance.filter.field=test1.t_product:id/subject/keywords,test2.t_company:id/name/contact/ch
+# table field black filter(format: schema1.tableName1:field1/field2,schema2.tableName2:field1/field2)
+#canal.instance.filter.black.field=test1.t_product:subject/product_image,test2.t_company:id/name/contact/ch
+
+# mq config
+canal.mq.topic=example
+# dynamic topic route by schema or table regex
+#canal.mq.dynamicTopic=mytest1.user,mytest2\\..*,.*\\..*
+canal.mq.partition=0
+# hash partition config
+#canal.mq.partitionsNum=3
+#canal.mq.partitionHash=test.table:id^name,.*\\..*
+#################################################

canal-admin/canal-admin-server/src/main/resources/public/static/css/chunk-2b9b6c5c.c89f8844.css

@@ -0,0 +1 @@
+.pagination-container[data-v-38ef71f0]{background:#fff;padding:32px 16px}.pagination-container.hidden[data-v-38ef71f0]{display:none}

canal-admin/canal-admin-server/src/main/resources/public/static/css/chunk-5afa45f5.a34bb177.css

@@ -0,0 +1 @@
+.line[data-v-2e6a6604]{text-align:center}

canal-admin/canal-admin-server/src/main/resources/public/static/css/chunk-5b373aad.b966d36b.css

@@ -0,0 +1 @@
+.line[data-v-e884438a]{text-align:center}

canal-admin/canal-admin-server/src/main/resources/public/static/css/chunk-da289616.c89f8844.css

@@ -0,0 +1 @@
+.pagination-container[data-v-38ef71f0]{background:#fff;padding:32px 16px}.pagination-container.hidden[data-v-38ef71f0]{display:none}

canal-admin/canal-admin-server/src/main/resources/public/static/js/chunk-37c49cbf.64d26540.js

@@ -0,0 +1 @@
+(window["webpackJsonp"]=window["webpackJsonp"]||[]).push([["chunk-37c49cbf"],{3671:function(t,e,n){"use strict";var a=n("afe4"),c=n.n(a);c.a},9406:function(t,e,n){"use strict";n.r(e);var a=function(){var t=this,e=t.$createElement;t._self._c;return t._m(0)},c=[function(){var t=this,e=t.$createElement,n=t._self._c||e;return n("div",{staticClass:"dashboard-container"},[n("div",{staticClass:"dashboard-text"},[t._v(" ")])])}],s=n("db72"),r=n("2f62"),u={name:"Dashboard",computed:Object(s["a"])({},Object(r["b"])(["name"])),mounted:function(){this.$router.push("/canalServer")}},i=u,o=(n("3671"),n("2877")),d=Object(o["a"])(i,a,c,!1,null,"42037c2b",null);e["default"]=d.exports},afe4:function(t,e,n){}}]);

canal-admin/canal-admin-server/src/main/resources/public/static/js/chunk-49959c8b.6d226f70.js

@@ -0,0 +1 @@
+(window["webpackJsonp"]=window["webpackJsonp"]||[]).push([["chunk-49959c8b"],{"26fc":function(t,s,a){t.exports=a.p+"static/img/404_cloud.0f4bc32b.png"},"8cdb":function(t,s,a){"use strict";a.r(s);var e=function(){var t=this,s=t.$createElement,a=t._self._c||s;return a("div",{staticClass:"wscn-http404-container"},[a("div",{staticClass:"wscn-http404"},[t._m(0),t._v(" "),a("div",{staticClass:"bullshit"},[a("div",{staticClass:"bullshit__oops"},[t._v("OOPS!")]),t._v(" "),t._m(1),t._v(" "),a("div",{staticClass:"bullshit__headline"},[t._v(t._s(t.message))]),t._v(" "),a("div",{staticClass:"bullshit__info"},[t._v("Please check that the URL you entered is correct, or click the button below to return to the homepage.")]),t._v(" "),a("a",{staticClass:"bullshit__return-home",attrs:{href:""}},[t._v("Back to home")])])])])},c=[function(){var t=this,s=t.$createElement,e=t._self._c||s;return e("div",{staticClass:"pic-404"},[e("img",{staticClass:"pic-404__parent",attrs:{src:a("a36b"),alt:"404"}}),t._v(" "),e("img",{staticClass:"pic-404__child left",attrs:{src:a("26fc"),alt:"404"}}),t._v(" "),e("img",{staticClass:"pic-404__child mid",attrs:{src:a("26fc"),alt:"404"}}),t._v(" "),e("img",{staticClass:"pic-404__child right",attrs:{src:a("26fc"),alt:"404"}})])},function(){var t=this,s=t.$createElement,a=t._self._c||s;return a("div",{staticClass:"bullshit__info"},[t._v("All rights reserved\n        "),a("a",{staticStyle:{color:"#20a0ff"},attrs:{href:"https://wallstreetcn.com",target:"_blank"}},[t._v("wallstreetcn")])])}],i={name:"Page404",computed:{message:function(){return"The webmaster said that you can not enter this page..."}}},l=i,n=(a("97ef"),a("2877")),r=Object(n["a"])(l,e,c,!1,null,"c095f994",null);s["default"]=r.exports},"97ef":function(t,s,a){"use strict";var e=a("b51e"),c=a.n(e);c.a},a36b:function(t,s,a){t.exports=a.p+"static/img/404.a57b6f31.png"},b51e:function(t,s,a){}}]);

canal-admin/canal-admin-ui/src/api/canalConfig.js

@@ -14,3 +14,10 @@ export function updateCanalConfig(data) {
     data
   })
 }
+
+export function getTemplateConfig() {
+  return request({
+    url: '/canal/config/template',
+    method: 'get'
+  })
+}

canal-admin/canal-admin-ui/src/api/canalInstance.js

@@ -72,3 +72,10 @@ export function getActiveInstances(serverId) {
     method: 'get'
   })
 }
+
+export function getTemplateInstance() {
+  return request({
+    url: '/canal/instance/template',
+    method: 'get'
+  })
+}

canal-admin/canal-admin-ui/src/components/Pagination/index.vue

@@ -9,7 +9,8 @@
       :total="total"
       v-bind="$attrs"
       @size-change="handleSizeChange"
-      @current-change="handleCurrentChange"/>
+      @current-change="handleCurrentChange"
+    />
   </div>
 </template>
 

canal-admin/canal-admin-ui/src/views/canalServer/CanalConfig.vue

@@ -6,6 +6,7 @@
           {{ form.name }}    
           <el-button type="primary" @click="onSubmit">保存</el-button>
           <el-button type="warning" @click="onCancel">重置</el-button>
+          <el-button type="success" @click="onLoadTemplate">载入模板</el-button>
           <el-button type="info" @click="onBack">返回</el-button>
         </el-form-item>
       </div>
@@ -15,7 +16,7 @@
 </template>
 
 <script>
-import { getCanalConfig, updateCanalConfig } from '@/api/canalConfig'
+import { getCanalConfig, updateCanalConfig, getTemplateConfig } from '@/api/canalConfig'
 
 export default {
   components: {
@@ -101,6 +102,11 @@ export default {
     },
     onBack() {
       history.go(-1)
+    },
+    onLoadTemplate() {
+      getTemplateConfig().then(res => {
+        this.form.content = res.data
+      })
     }
   }
 }

canal-admin/canal-admin-ui/src/views/canalServer/CanalInstanceAdd.vue

@@ -9,6 +9,7 @@
           </el-option-group>
         </el-select>
         <el-button class="filter-item" type="primary" @click="onSubmit">保存</el-button>
+        <el-button class="filter-item" type="success" @click="onLoadTemplate">载入模板</el-button>
         <el-button class="filter-item" type="info" @click="onBack">返回</el-button>
       </div>
       <editor v-model="form.content" lang="properties" theme="chrome" width="100%" :height="800" @init="editorInit" />
@@ -17,7 +18,7 @@
 </template>
 
 <script>
-import { addCanalInstance } from '@/api/canalInstance'
+import { addCanalInstance, getTemplateInstance } from '@/api/canalInstance'
 import { getClustersAndServers } from '@/api/canalCluster'
 
 export default {
@@ -99,6 +100,11 @@ export default {
     },
     onBack() {
       history.go(-1)
+    },
+    onLoadTemplate() {
+      getTemplateInstance().then(res => {
+        this.form.content = res.data
+      })
     }
   }
 }

canal-admin/canal-admin-ui/src/views/canalServer/NodeServer.vue

@@ -311,8 +311,8 @@ export default {
     },
     handleConfig(row) {
       if (row.canalCluster !== null) {
-         this.$message({ message: '集群模式Server不允许单独变更配置，请在集群配置变更', type: 'error' })
-         return
+        this.$message({ message: '集群模式Server不允许单独变更配置，请在集群配置变更', type: 'error' })
+        return
       }
       this.$router.push('/canalServer/nodeServer/config?serverId=' + row.id)
     },