Skip to content

Commit 874a256

Browse files
dingxiaobodingxiaobo
committed
DataX 2209, add plugin: datahubreader/datahubwriter/loghubreader/loghubwriter/starrocksreader, update plugin: odpsreader/oceanbasev10reader/oceanbasev10writer/elasticserachwriter/mysqlreader/mysqlwriter
1 parent ced5a45 commit 874a256

File tree

133 files changed

+5505
-2537
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

133 files changed

+5505
-2537
lines changed

‎README.md

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ DataX本身作为数据同步框架，将不同数据源的同步抽象为从源
2525

2626
# Quick Start
2727

28-
##### Download [DataX下载地址](https://datax-opensource.oss-cn-hangzhou.aliyuncs.com/20220530/datax.tar.gz)
28+
##### Download [DataX下载地址](https://datax-opensource.oss-cn-hangzhou.aliyuncs.com/202209/datax.tar.gz)
2929

3030

3131
##### 请点击：[Quick Start](https://github.com/alibaba/DataX/blob/master/userGuid.md)
@@ -95,6 +95,9 @@ DataX目前已经有了比较全面的插件体系，主流的RDBMS数据库、N
9595

9696
DataX 后续计划月度迭代更新，也欢迎感兴趣的同学提交 Pull requests，月度更新内容会介绍介绍如下。
9797

98+
- [datax_v202209]（https://github.com/alibaba/DataX/releases/tag/datax_v202209)
99+
- 涉及通道能力更新（MaxCompute、Datahub、SLS等）、安全漏洞更新、通用打包更新等
100+
98101
- [datax_v202205]（https://github.com/alibaba/DataX/releases/tag/datax_v202205)
99102
- 涉及通道能力更新（MaxCompute、Hologres、OSS、Tdengine等）、安全漏洞更新、通用打包更新等
100103

‎clickhousewriter/src/main/resources/plugin.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,5 @@
22
"name": "clickhousewriter",
33
"class": "com.alibaba.datax.plugin.writer.clickhousewriter.ClickhouseWriter",
44
"description": "useScene: prod. mechanism: Jdbc connection using the database, execute insert sql.",
5-
"developer": "jiye.tjy"
5+
"developer": "alibaba"
66
}

‎common/src/main/java/com/alibaba/datax/common/util/Configuration.java

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -411,6 +411,15 @@ public List<Object> getList(final String path) {
411411
return list;
412412
}
413413

414+
public <T> List<T> getListWithJson(final String path, Class<T> t) {
415+
Object object = this.get(path, List.class);
416+
if (null == object) {
417+
return null;
418+
}
419+
420+
return JSON.parseArray(JSON.toJSONString(object),t);
421+
}
422+
414423
/**
415424
* 根据用户提供的json path，寻址List对象，如果对象不存在，返回null
416425
*/

‎common/src/main/java/com/alibaba/datax/common/util/StrUtil.java

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
import org.apache.commons.lang3.StringUtils;
44
import org.apache.commons.lang3.Validate;
55

6+
import java.security.MessageDigest;
7+
import java.security.NoSuchAlgorithmException;
68
import java.text.DecimalFormat;
79
import java.util.HashMap;
810
import java.util.Map;
@@ -82,4 +84,20 @@ public static String compressMiddle(String s, int headLength, int tailLength) {
8284
return s.substring(0, headLength) + "..." + s.substring(s.length() - tailLength);
8385
}
8486

87+
public static String getMd5(String plainText) {
88+
try {
89+
StringBuilder builder = new StringBuilder();
90+
for (byte b : MessageDigest.getInstance("MD5").digest(plainText.getBytes())) {
91+
int i = b & 0xff;
92+
if (i < 0x10) {
93+
builder.append('0');
94+
}
95+
builder.append(Integer.toHexString(i));
96+
}
97+
return builder.toString();
98+
} catch (NoSuchAlgorithmException e) {
99+
throw new RuntimeException(e);
100+
}
101+
}
102+
85103
}

‎datahubreader/pom.xml

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
3+
<parent>
4+
<artifactId>datax-all</artifactId>
5+
<groupId>com.alibaba.datax</groupId>
6+
<version>0.0.1-SNAPSHOT</version>
7+
</parent>
8+
<modelVersion>4.0.0</modelVersion>
9+
10+
<artifactId>datahubreader</artifactId>
11+
12+
<version>0.0.1-SNAPSHOT</version>
13+
14+
<dependencies>
15+
<dependency>
16+
<groupId>com.alibaba.datax</groupId>
17+
<artifactId>datax-common</artifactId>
18+
<version>${datax-project-version}</version>
19+
<exclusions>
20+
<exclusion>
21+
<artifactId>slf4j-log4j12</artifactId>
22+
<groupId>org.slf4j</groupId>
23+
</exclusion>
24+
</exclusions>
25+
</dependency>
26+
<dependency>
27+
<groupId>org.slf4j</groupId>
28+
<artifactId>slf4j-api</artifactId>
29+
</dependency>
30+
<dependency>
31+
<groupId>ch.qos.logback</groupId>
32+
<artifactId>logback-classic</artifactId>
33+
</dependency>
34+
<dependency>
35+
<groupId>com.aliyun.datahub</groupId>
36+
<artifactId>aliyun-sdk-datahub</artifactId>
37+
<version>2.21.6-public</version>
38+
</dependency>
39+
<dependency>
40+
<groupId>junit</groupId>
41+
<artifactId>junit</artifactId>
42+
<version>4.12</version>
43+
<scope>test</scope>
44+
</dependency>
45+
</dependencies>
46+
47+
<build>
48+
<plugins>
49+
<!-- compiler plugin -->
50+
<plugin>
51+
<artifactId>maven-compiler-plugin</artifactId>
52+
<configuration>
53+
<source>${jdk-version}</source>
54+
<target>${jdk-version}</target>
55+
<encoding>${project-sourceEncoding}</encoding>
56+
</configuration>
57+
</plugin>
58+
<!-- assembly plugin -->
59+
<plugin>
60+
<artifactId>maven-assembly-plugin</artifactId>
61+
<configuration>
62+
<descriptors>
63+
<descriptor>src/main/assembly/package.xml</descriptor>
64+
</descriptors>
65+
<finalName>datax</finalName>
66+
</configuration>
67+
<executions>
68+
<execution>
69+
<id>dwzip</id>
70+
<phase>package</phase>
71+
<goals>
72+
<goal>single</goal>
73+
</goals>
74+
</execution>
75+
</executions>
76+
</plugin>
77+
</plugins>
78+
</build>
79+
</project>

‎datahubreader/src/main/assembly/package.xml

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
<assembly
2+
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
3+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4+
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
5+
<id></id>
6+
<formats>
7+
<format>dir</format>
8+
</formats>
9+
<includeBaseDirectory>false</includeBaseDirectory>
10+
<fileSets>
11+
<fileSet>
12+
<directory>src/main/resources</directory>
13+
<includes>
14+
<include>plugin.json</include>
15+
</includes>
16+
<outputDirectory>plugin/reader/datahubreader</outputDirectory>
17+
</fileSet>
18+
<fileSet>
19+
<directory>target/</directory>
20+
<includes>
21+
<include>datahubreader-0.0.1-SNAPSHOT.jar</include>
22+
</includes>
23+
<outputDirectory>plugin/reader/datahubreader</outputDirectory>
24+
</fileSet>
25+
</fileSets>
26+
27+
<dependencySets>
28+
<dependencySet>
29+
<useProjectArtifact>false</useProjectArtifact>
30+
<outputDirectory>plugin/reader/datahubreader/libs</outputDirectory>
31+
<scope>runtime</scope>
32+
</dependencySet>
33+
</dependencySets>
34+
</assembly>

‎datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/Constant.java

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
package com.alibaba.datax.plugin.reader.datahubreader;
2+
3+
public class Constant {
4+
5+
public static String DATETIME_FORMAT = "yyyyMMddHHmmss";
6+
public static String DATE_FORMAT = "yyyyMMdd";
7+
8+
}

‎datahubreader/src/main/java/com/alibaba/datax/plugin/reader/datahubreader/DatahubClientHelper.java

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
package com.alibaba.datax.plugin.reader.datahubreader;
2+
3+
import com.alibaba.datax.common.util.Configuration;
4+
import com.alibaba.fastjson.JSON;
5+
import com.alibaba.fastjson.TypeReference;
6+
import com.aliyun.datahub.client.DatahubClient;
7+
import com.aliyun.datahub.client.DatahubClientBuilder;
8+
import com.aliyun.datahub.client.auth.Account;
9+
import com.aliyun.datahub.client.auth.AliyunAccount;
10+
import com.aliyun.datahub.client.common.DatahubConfig;
11+
import com.aliyun.datahub.client.http.HttpConfig;
12+
import org.apache.commons.lang3.StringUtils;
13+
14+
public class DatahubClientHelper {
15+
public static DatahubClient getDatahubClient(Configuration jobConfig) {
16+
String accessId = jobConfig.getNecessaryValue(Key.CONFIG_KEY_ACCESS_ID,
17+
DatahubWriterErrorCode.MISSING_REQUIRED_VALUE);
18+
String accessKey = jobConfig.getNecessaryValue(Key.CONFIG_KEY_ACCESS_KEY,
19+
DatahubWriterErrorCode.MISSING_REQUIRED_VALUE);
20+
String endpoint = jobConfig.getNecessaryValue(Key.CONFIG_KEY_ENDPOINT,
21+
DatahubWriterErrorCode.MISSING_REQUIRED_VALUE);
22+
Account account = new AliyunAccount(accessId, accessKey);
23+
// 是否开启二进制传输，服务端2.12版本开始支持
24+
boolean enableBinary = jobConfig.getBool("enableBinary", false);
25+
DatahubConfig datahubConfig = new DatahubConfig(endpoint, account, enableBinary);
26+
// HttpConfig可不设置，不设置时采用默认值
27+
// 读写数据推荐打开网络传输 LZ4压缩
28+
HttpConfig httpConfig = null;
29+
String httpConfigStr = jobConfig.getString("httpConfig");
30+
if (StringUtils.isNotBlank(httpConfigStr)) {
31+
httpConfig = JSON.parseObject(httpConfigStr, new TypeReference<HttpConfig>() {
32+
});
33+
}
34+
35+
DatahubClientBuilder builder = DatahubClientBuilder.newBuilder().setDatahubConfig(datahubConfig);
36+
if (null != httpConfig) {
37+
builder.setHttpConfig(httpConfig);
38+
}
39+
DatahubClient datahubClient = builder.build();
40+
return datahubClient;
41+
}
42+
}

0 commit comments

Comments
 (0)