Skip to content

Commit 5372528

Browse files
kalencayawangqi
kalencaya
and
wangqi
authored
[Feature][scaleph-workspace-seatunnel] update scaleph-seatunnel image (#730)
* fix: kafka params error * feature: update elasticsearch docker compose * feature: update doris docker compose * feature: update doris docker compose * feature: update doris and kafka sink --------- Co-authored-by: wangqi <[email protected]>
1 parent 319905e commit 5372528

File tree

12 files changed

+240
-209
lines changed

12 files changed

+240
-209
lines changed

‎pom.xml

+12
Original file line numberDiff line numberDiff line change
@@ -795,6 +795,18 @@
795795
<groupId>org.apache.logging.log4j</groupId>
796796
<artifactId>log4j-slf4j2-impl</artifactId>
797797
</exclusion>
798+
<exclusion>
799+
<groupId>org.apache.logging.log4j</groupId>
800+
<artifactId>log4j-api</artifactId>
801+
</exclusion>
802+
<exclusion>
803+
<groupId>org.apache.logging.log4j</groupId>
804+
<artifactId>log4j-core</artifactId>
805+
</exclusion>
806+
<exclusion>
807+
<groupId>org.apache.logging.log4j</groupId>
808+
<artifactId>log4j-1.2-api</artifactId>
809+
</exclusion>
798810
</exclusions>
799811
</dependency>
800812
</dependencies>

‎scaleph-datasource/pom.xml

+2-2
Original file line numberDiff line numberDiff line change
@@ -37,14 +37,14 @@
3737
<artifactId>scaleph-resource</artifactId>
3838
</dependency>
3939

40-
<dependency>
40+
<!--<dependency>
4141
<groupId>com.datastrato.gravitino</groupId>
4242
<artifactId>client-java-runtime</artifactId>
4343
</dependency>
4444
<dependency>
4545
<groupId>com.datastrato.gravitino</groupId>
4646
<artifactId>client-java</artifactId>
47-
</dependency>
47+
</dependency>-->
4848
</dependencies>
4949

5050
</project>

‎scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/gravitino/GravitinoConfig.java

+51-51
Original file line numberDiff line numberDiff line change
@@ -1,51 +1,51 @@
1-
/*
2-
* Licensed to the Apache Software Foundation (ASF) under one
3-
* or more contributor license agreements. See the NOTICE file
4-
* distributed with this work for additional information
5-
* regarding copyright ownership. The ASF licenses this file
6-
* to you under the Apache License, Version 2.0 (the
7-
* "License"); you may not use this file except in compliance
8-
* with the License. You may obtain a copy of the License at
9-
*
10-
* http://www.apache.org/licenses/LICENSE-2.0
11-
*
12-
* Unless required by applicable law or agreed to in writing, software
13-
* distributed under the License is distributed on an "AS IS" BASIS,
14-
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15-
* See the License for the specific language governing permissions and
16-
* limitations under the License.
17-
*/
18-
19-
package cn.sliew.scaleph.ds.gravitino;
20-
21-
import cn.sliew.scaleph.common.util.NetUtils;
22-
import com.datastrato.gravitino.client.GravitinoAdminClient;
23-
import com.datastrato.gravitino.client.GravitinoClient;
24-
import org.springframework.beans.factory.annotation.Autowired;
25-
import org.springframework.boot.context.properties.EnableConfigurationProperties;
26-
import org.springframework.context.annotation.Bean;
27-
import org.springframework.context.annotation.Configuration;
28-
29-
@Configuration
30-
@EnableConfigurationProperties(GravitinoProperties.class)
31-
public class GravitinoConfig {
32-
33-
@Autowired
34-
private GravitinoProperties properties;
35-
36-
@Bean
37-
public GravitinoAdminClient gravitinoAdminClient() {
38-
return GravitinoAdminClient.builder(NetUtils.replaceLocalhost(properties.getUrl()))
39-
.withSimpleAuth()
40-
.build();
41-
}
42-
43-
/**
44-
* fixme 必须添加 metalakeName
45-
*/
46-
public GravitinoClient gravitinoClient() {
47-
return GravitinoClient.builder(NetUtils.replaceLocalhost(properties.getUrl()))
48-
.withSimpleAuth()
49-
.build();
50-
}
51-
}
1+
///*
2+
// * Licensed to the Apache Software Foundation (ASF) under one
3+
// * or more contributor license agreements. See the NOTICE file
4+
// * distributed with this work for additional information
5+
// * regarding copyright ownership. The ASF licenses this file
6+
// * to you under the Apache License, Version 2.0 (the
7+
// * "License"); you may not use this file except in compliance
8+
// * with the License. You may obtain a copy of the License at
9+
// *
10+
// * http://www.apache.org/licenses/LICENSE-2.0
11+
// *
12+
// * Unless required by applicable law or agreed to in writing, software
13+
// * distributed under the License is distributed on an "AS IS" BASIS,
14+
// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
// * See the License for the specific language governing permissions and
16+
// * limitations under the License.
17+
// */
18+
//
19+
//package cn.sliew.scaleph.ds.gravitino;
20+
//
21+
//import cn.sliew.scaleph.common.util.NetUtils;
22+
//import com.datastrato.gravitino.client.GravitinoAdminClient;
23+
//import com.datastrato.gravitino.client.GravitinoClient;
24+
//import org.springframework.beans.factory.annotation.Autowired;
25+
//import org.springframework.boot.context.properties.EnableConfigurationProperties;
26+
//import org.springframework.context.annotation.Bean;
27+
//import org.springframework.context.annotation.Configuration;
28+
//
29+
//@Configuration
30+
//@EnableConfigurationProperties(GravitinoProperties.class)
31+
//public class GravitinoConfig {
32+
//
33+
// @Autowired
34+
// private GravitinoProperties properties;
35+
//
36+
// @Bean
37+
// public GravitinoAdminClient gravitinoAdminClient() {
38+
// return GravitinoAdminClient.builder(NetUtils.replaceLocalhost(properties.getUrl()))
39+
// .withSimpleAuth()
40+
// .build();
41+
// }
42+
//
43+
// /**
44+
// * fixme 必须添加 metalakeName
45+
// */
46+
// public GravitinoClient gravitinoClient() {
47+
// return GravitinoClient.builder(NetUtils.replaceLocalhost(properties.getUrl()))
48+
// .withSimpleAuth()
49+
// .build();
50+
// }
51+
//}

‎scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/gravitino/GravitinoInitializer.java

+118-118
Original file line numberDiff line numberDiff line change
@@ -1,118 +1,118 @@
1-
/*
2-
* Licensed to the Apache Software Foundation (ASF) under one
3-
* or more contributor license agreements. See the NOTICE file
4-
* distributed with this work for additional information
5-
* regarding copyright ownership. The ASF licenses this file
6-
* to you under the Apache License, Version 2.0 (the
7-
* "License"); you may not use this file except in compliance
8-
* with the License. You may obtain a copy of the License at
9-
*
10-
* http://www.apache.org/licenses/LICENSE-2.0
11-
*
12-
* Unless required by applicable law or agreed to in writing, software
13-
* distributed under the License is distributed on an "AS IS" BASIS,
14-
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15-
* See the License for the specific language governing permissions and
16-
* limitations under the License.
17-
*/
18-
19-
package cn.sliew.scaleph.ds.gravitino;
20-
21-
import cn.sliew.milky.common.util.JacksonUtil;
22-
import cn.sliew.scaleph.common.dict.job.DataSourceType;
23-
import cn.sliew.scaleph.ds.modal.AbstractDataSource;
24-
import cn.sliew.scaleph.ds.modal.jdbc.JdbcDataSource;
25-
import cn.sliew.scaleph.ds.service.DsInfoService;
26-
import cn.sliew.scaleph.ds.service.dto.DsInfoDTO;
27-
import com.datastrato.gravitino.Catalog;
28-
import com.datastrato.gravitino.NameIdentifier;
29-
import com.datastrato.gravitino.client.GravitinoAdminClient;
30-
import com.datastrato.gravitino.client.GravitinoMetalake;
31-
import com.fasterxml.jackson.databind.node.ObjectNode;
32-
import org.springframework.beans.factory.InitializingBean;
33-
import org.springframework.beans.factory.annotation.Autowired;
34-
import org.springframework.stereotype.Component;
35-
36-
import java.util.Collections;
37-
import java.util.HashMap;
38-
import java.util.List;
39-
import java.util.Map;
40-
41-
@Component
42-
public class GravitinoInitializer implements InitializingBean {
43-
44-
@Autowired
45-
private GravitinoProperties properties;
46-
@Autowired
47-
private GravitinoAdminClient adminClient;
48-
@Autowired
49-
private DsInfoService dsInfoService;
50-
51-
@Override
52-
public void afterPropertiesSet() throws Exception {
53-
initialize();
54-
}
55-
56-
private void initialize() {
57-
NameIdentifier nameIdentifier = NameIdentifier.ofMetalake(properties.getMetalake());
58-
59-
// 初始化 metalake
60-
initMetalake(nameIdentifier);
61-
// 初始化 catalog
62-
initDataSource(properties.getMetalake(), DataSourceType.MYSQL);
63-
initDataSource(properties.getMetalake(), DataSourceType.POSTGRESQL);
64-
initDataSource(properties.getMetalake(), DataSourceType.HIVE);
65-
initDataSource(properties.getMetalake(), DataSourceType.ICEBERG);
66-
initDataSource(properties.getMetalake(), DataSourceType.DORIS);
67-
initDataSource(properties.getMetalake(), DataSourceType.KAFKA);
68-
initDataSource(properties.getMetalake(), DataSourceType.HDFS);
69-
}
70-
71-
private void initMetalake(NameIdentifier nameIdentifier) {
72-
if (adminClient.metalakeExists(nameIdentifier) == false) {
73-
adminClient.createMetalake(nameIdentifier, "scaleph created", Collections.emptyMap());
74-
}
75-
}
76-
77-
private void initDataSource(String metalakeName, DataSourceType type) {
78-
List<DsInfoDTO> dsInfoDTOS = dsInfoService.listByType(type);
79-
for (DsInfoDTO dsInfoDTO : dsInfoDTOS) {
80-
doInitDataSource(metalakeName, type, dsInfoDTO);
81-
}
82-
}
83-
84-
private void doInitDataSource(String metalakeName, DataSourceType type, DsInfoDTO dsInfoDTO) {
85-
GravitinoMetalake metalake = adminClient.loadMetalake(NameIdentifier.ofMetalake(metalakeName));
86-
NameIdentifier catalogName = NameIdentifier.ofCatalog(metalakeName, dsInfoDTO.getName());
87-
if (metalake.catalogExists(catalogName) == false) {
88-
switch (type) {
89-
case MYSQL:
90-
initMySQL(metalake, catalogName, dsInfoDTO);
91-
break;
92-
case POSTGRESQL:
93-
break;
94-
case HIVE:
95-
break;
96-
case ICEBERG:
97-
break;
98-
case DORIS:
99-
break;
100-
case KAFKA:
101-
break;
102-
case HDFS:
103-
break;
104-
default:
105-
}
106-
}
107-
}
108-
109-
private void initMySQL(GravitinoMetalake metalake, NameIdentifier catalogName, DsInfoDTO dsInfoDTO) {
110-
JdbcDataSource dataSource = (JdbcDataSource) AbstractDataSource.fromDsInfo((ObjectNode) JacksonUtil.toJsonNode(dsInfoDTO));
111-
Map<String, String> properties = new HashMap<>();
112-
properties.put("jdbc-driver", dataSource.getDriverClassName());
113-
properties.put("jdbc-url", dataSource.getUrl());
114-
properties.put("jdbc-user", dataSource.getUser());
115-
properties.put("jdbc-password", dataSource.getPassword());
116-
metalake.createCatalog(catalogName, Catalog.Type.RELATIONAL, "jdbc-mysql", dataSource.getRemark(), properties);
117-
}
118-
}
1+
///*
2+
// * Licensed to the Apache Software Foundation (ASF) under one
3+
// * or more contributor license agreements. See the NOTICE file
4+
// * distributed with this work for additional information
5+
// * regarding copyright ownership. The ASF licenses this file
6+
// * to you under the Apache License, Version 2.0 (the
7+
// * "License"); you may not use this file except in compliance
8+
// * with the License. You may obtain a copy of the License at
9+
// *
10+
// * http://www.apache.org/licenses/LICENSE-2.0
11+
// *
12+
// * Unless required by applicable law or agreed to in writing, software
13+
// * distributed under the License is distributed on an "AS IS" BASIS,
14+
// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
// * See the License for the specific language governing permissions and
16+
// * limitations under the License.
17+
// */
18+
//
19+
//package cn.sliew.scaleph.ds.gravitino;
20+
//
21+
//import cn.sliew.milky.common.util.JacksonUtil;
22+
//import cn.sliew.scaleph.common.dict.job.DataSourceType;
23+
//import cn.sliew.scaleph.ds.modal.AbstractDataSource;
24+
//import cn.sliew.scaleph.ds.modal.jdbc.JdbcDataSource;
25+
//import cn.sliew.scaleph.ds.service.DsInfoService;
26+
//import cn.sliew.scaleph.ds.service.dto.DsInfoDTO;
27+
//import com.datastrato.gravitino.Catalog;
28+
//import com.datastrato.gravitino.NameIdentifier;
29+
//import com.datastrato.gravitino.client.GravitinoAdminClient;
30+
//import com.datastrato.gravitino.client.GravitinoMetalake;
31+
//import com.fasterxml.jackson.databind.node.ObjectNode;
32+
//import org.springframework.beans.factory.InitializingBean;
33+
//import org.springframework.beans.factory.annotation.Autowired;
34+
//import org.springframework.stereotype.Component;
35+
//
36+
//import java.util.Collections;
37+
//import java.util.HashMap;
38+
//import java.util.List;
39+
//import java.util.Map;
40+
//
41+
//@Component
42+
//public class GravitinoInitializer implements InitializingBean {
43+
//
44+
// @Autowired
45+
// private GravitinoProperties properties;
46+
// @Autowired
47+
// private GravitinoAdminClient adminClient;
48+
// @Autowired
49+
// private DsInfoService dsInfoService;
50+
//
51+
// @Override
52+
// public void afterPropertiesSet() throws Exception {
53+
// initialize();
54+
// }
55+
//
56+
// private void initialize() {
57+
// NameIdentifier nameIdentifier = NameIdentifier.ofMetalake(properties.getMetalake());
58+
//
59+
// // 初始化 metalake
60+
// initMetalake(nameIdentifier);
61+
// // 初始化 catalog
62+
// initDataSource(properties.getMetalake(), DataSourceType.MYSQL);
63+
// initDataSource(properties.getMetalake(), DataSourceType.POSTGRESQL);
64+
// initDataSource(properties.getMetalake(), DataSourceType.HIVE);
65+
// initDataSource(properties.getMetalake(), DataSourceType.ICEBERG);
66+
// initDataSource(properties.getMetalake(), DataSourceType.DORIS);
67+
// initDataSource(properties.getMetalake(), DataSourceType.KAFKA);
68+
// initDataSource(properties.getMetalake(), DataSourceType.HDFS);
69+
// }
70+
//
71+
// private void initMetalake(NameIdentifier nameIdentifier) {
72+
// if (adminClient.metalakeExists(nameIdentifier) == false) {
73+
// adminClient.createMetalake(nameIdentifier, "scaleph created", Collections.emptyMap());
74+
// }
75+
// }
76+
//
77+
// private void initDataSource(String metalakeName, DataSourceType type) {
78+
// List<DsInfoDTO> dsInfoDTOS = dsInfoService.listByType(type);
79+
// for (DsInfoDTO dsInfoDTO : dsInfoDTOS) {
80+
// doInitDataSource(metalakeName, type, dsInfoDTO);
81+
// }
82+
// }
83+
//
84+
// private void doInitDataSource(String metalakeName, DataSourceType type, DsInfoDTO dsInfoDTO) {
85+
// GravitinoMetalake metalake = adminClient.loadMetalake(NameIdentifier.ofMetalake(metalakeName));
86+
// NameIdentifier catalogName = NameIdentifier.ofCatalog(metalakeName, dsInfoDTO.getName());
87+
// if (metalake.catalogExists(catalogName) == false) {
88+
// switch (type) {
89+
// case MYSQL:
90+
// initMySQL(metalake, catalogName, dsInfoDTO);
91+
// break;
92+
// case POSTGRESQL:
93+
// break;
94+
// case HIVE:
95+
// break;
96+
// case ICEBERG:
97+
// break;
98+
// case DORIS:
99+
// break;
100+
// case KAFKA:
101+
// break;
102+
// case HDFS:
103+
// break;
104+
// default:
105+
// }
106+
// }
107+
// }
108+
//
109+
// private void initMySQL(GravitinoMetalake metalake, NameIdentifier catalogName, DsInfoDTO dsInfoDTO) {
110+
// JdbcDataSource dataSource = (JdbcDataSource) AbstractDataSource.fromDsInfo((ObjectNode) JacksonUtil.toJsonNode(dsInfoDTO));
111+
// Map<String, String> properties = new HashMap<>();
112+
// properties.put("jdbc-driver", dataSource.getDriverClassName());
113+
// properties.put("jdbc-url", dataSource.getUrl());
114+
// properties.put("jdbc-user", dataSource.getUser());
115+
// properties.put("jdbc-password", dataSource.getPassword());
116+
// metalake.createCatalog(catalogName, Catalog.Type.RELATIONAL, "jdbc-mysql", dataSource.getRemark(), properties);
117+
// }
118+
//}

‎scaleph-datasource/src/main/java/cn/sliew/scaleph/ds/modal/olap/DorisDataSource.java

-1
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,6 @@ public class DorisDataSource extends AbstractDataSource {
4646
@Schema(description = "username")
4747
private String username;
4848

49-
@NotBlank
5049
@Schema(description = "password")
5150
private String password;
5251

‎scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/doris/sink/DorisSinkPlugin.java

+2
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,8 @@ public ObjectNode createConf() {
8585
}
8686
if (StringUtils.hasText(dataSource.getPassword())) {
8787
conf.putPOJO(PASSWORD.getName(), dataSource.getPassword());
88+
} else {
89+
conf.putPOJO(PASSWORD.getName(), "");
8890
}
8991
if (dataSource.getQueryPort() != null) {
9092
conf.putPOJO(QUERY_PORT.getName(), dataSource.getQueryPort());

0 commit comments

Comments
 (0)