Merge pull request #1045 from rewerma/master

修改打包配置

client-adapter/hbase/pom.xml

@@ -43,60 +43,28 @@
         </dependency>
     </dependencies>
 
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>2.4</version>
+                <configuration>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>make-assembly</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
 
-    <profiles>
-        <profile>
-            <id>release</id>
-            <activation>
-                <property>
-                    <name>env</name>
-                    <value>release</value>
-                </property>
-            </activation>
-
-            <build>
-                <plugins>
-                    <plugin>
-                        <groupId>org.apache.maven.plugins</groupId>
-                        <artifactId>maven-assembly-plugin</artifactId>
-                        <version>2.4</version>
-                        <configuration>
-                            <descriptorRefs>
-                                <descriptorRef>jar-with-dependencies</descriptorRef>
-                            </descriptorRefs>
-                        </configuration>
-                        <executions>
-                            <execution>
-                                <id>make-assembly</id>
-                                <phase>package</phase>
-                                <goals>
-                                    <goal>single</goal>
-                                </goals>
-                            </execution>
-                        </executions>
-                    </plugin>
-                    <plugin>
-                        <artifactId>maven-antrun-plugin</artifactId>
-                        <executions>
-                            <execution>
-                                <phase>package</phase>
-                                <goals>
-                                    <goal>run</goal>
-                                </goals>
-                                <configuration>
-                                    <tasks>
-                                        <copy todir="${project.basedir}/../launcher/target/config/hbase" overwrite="true" >
-                                            <fileset dir="${project.basedir}/src/main/resources/hbase" erroronmissingdir="true">
-                                                <include name="*.yml"/>
-                                            </fileset>
-                                        </copy>
-                                    </tasks>
-                                </configuration>
-                            </execution>
-                        </executions>
-                    </plugin>
-                </plugins>
-            </build>
-        </profile>
-    </profiles>
 </project>

client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/config/MappingConfig.java

@@ -152,6 +152,7 @@ public class MappingConfig {
         private boolean                 autoCreateTable    = false;                 // 同步时HBase中表不存在的情况下自动建表
         private String                  rowKey;                                     // 指定复合主键为rowKey
         private Map<String, String>     columns;                                    // 字段映射
+        private List<String>            excludeColumns;                             // 不映射的字段
         private ColumnItem              rowKeyColumn;                               // rowKey字段
         private String                  etlCondition;                               // etl条件sql
 
@@ -292,6 +293,14 @@ public class MappingConfig {
             }
         }
 
+        public List<String> getExcludeColumns() {
+            return excludeColumns;
+        }
+
+        public void setExcludeColumns(List<String> excludeColumns) {
+            this.excludeColumns = excludeColumns;
+        }
+
         public String getFamily() {
             return family;
         }

client-adapter/hbase/src/main/java/com/alibaba/otter/canal/client/adapter/hbase/service/HbaseSyncService.java

@@ -110,6 +110,9 @@ public class HbaseSyncService {
         Map<String, MappingConfig.ColumnItem> columnItems = hbaseOrm.getColumnItems();
         int i = 0;
         for (Map.Entry<String, Object> entry : data.entrySet()) {
+            if (hbaseOrm.getExcludeColumns() != null && hbaseOrm.getExcludeColumns().contains(entry.getKey())) {
+                continue;
+            }
             if (entry.getValue() != null) {
                 MappingConfig.ColumnItem columnItem = columnItems.get(entry.getKey());
 
@@ -196,6 +199,9 @@ public class HbaseSyncService {
             Map<String, MappingConfig.ColumnItem> columnItems = hbaseOrm.getColumnItems();
             HRow hRow = new HRow(rowKeyBytes);
             for (String updateColumn : old.get(index).keySet()) {
+                if (hbaseOrm.getExcludeColumns() != null && hbaseOrm.getExcludeColumns().contains(updateColumn)) {
+                    continue;
+                }
                 MappingConfig.ColumnItem columnItem = columnItems.get(updateColumn);
                 if (columnItem == null) {
                     String family = hbaseOrm.getFamily();

client-adapter/hbase/src/main/resources/hbase/mytest_person2.yml

@@ -17,6 +17,8 @@ hbaseOrm:
     type: $DECIMAL
     c_time: C_TIME$UNSIGNED_TIMESTAMP
     birthday: BIRTHDAY$DATE
+  excludeColumns:
+    - lat
 
 # -- NATIVE类型
 # $DEFAULT

client-adapter/launcher/pom.xml

@@ -139,6 +139,11 @@
                                         <include name="*.yml"/>
                                     </fileset>
                                 </copy>
+                                <copy todir="${project.basedir}/target/config/hbase" overwrite="true" >
+                                    <fileset dir="${project.basedir}/../hbase/src/main/resources/hbase" erroronmissingdir="true">
+                                        <include name="*.yml"/>
+                                    </fileset>
+                                </copy>
                             </tasks>
                         </configuration>
                     </execution>

client-adapter/launcher/src/main/resources/application.yml

@@ -23,12 +23,12 @@ canal.conf:
   - instance: example
     adapterGroups:
     - outAdapters:
-      - name: logger
-#      - name: hbase
-#        properties:
-#          hbase.zookeeper.quorum: ${hbase.zookeeper.quorum}
-#          hbase.zookeeper.property.clientPort: ${hbase.zookeeper.property.clientPort}
-#          zookeeper.znode.parent: ${hbase.zookeeper.znode.parent}
+#      - name: logger
+      - name: hbase
+        properties:
+          hbase.zookeeper.quorum: ${hbase.zookeeper.quorum}
+          hbase.zookeeper.property.clientPort: ${hbase.zookeeper.property.clientPort}
+          zookeeper.znode.parent: ${hbase.zookeeper.znode.parent}
 #  mqTopics:
 #  - mqMode: kafka
 #    topic: example
@@ -37,11 +37,11 @@ canal.conf:
 #      outAdapters:
 #      - name: logger
 
-#adapter.conf:
-#  datasourceConfigs:
-#    defaultDS:
-#      url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true
-#      username: root
-#      password: 121212
-#  adapterConfigs:
-#  - hbase/mytest_person2.yml
+adapter.conf:
+  datasourceConfigs:
+    defaultDS:
+      url: jdbc:mysql://127.0.0.1:3306/mytest?useUnicode=true
+      username: root
+      password: 121212
+  adapterConfigs:
+  - hbase/mytest_person2.yml

client-adapter/logger/pom.xml

@@ -21,39 +21,28 @@
         </dependency>
     </dependencies>
 
-    <profiles>
-        <profile>
-            <id>release</id>
-            <activation>
-                <property>
-                    <name>env</name>
-                    <value>release</value>
-                </property>
-            </activation>
 
-            <build>
-                <plugins>
-                    <plugin>
-                        <groupId>org.apache.maven.plugins</groupId>
-                        <artifactId>maven-assembly-plugin</artifactId>
-                        <version>2.4</version>
-                        <configuration>
-                            <descriptorRefs>
-                                <descriptorRef>jar-with-dependencies</descriptorRef>
-                            </descriptorRefs>
-                        </configuration>
-                        <executions>
-                            <execution>
-                                <id>make-assembly</id>
-                                <phase>package</phase>
-                                <goals>
-                                    <goal>single</goal>
-                                </goals>
-                            </execution>
-                        </executions>
-                    </plugin>
-                </plugins>
-            </build>
-        </profile>
-    </profiles>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>2.4</version>
+                <configuration>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>make-assembly</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
 </project>