Skip to content

Commit cf62469

Browse files
MrkuhuoMrkuhuo
committed
新增seatunnel_conf案例
新增seatunnel_conf案例
1 parent fe631dd commit cf62469

File tree

268 files changed

+25479
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

268 files changed

+25479
-0
lines changed

‎seatunnel_samples/connector-amazondynamodb-conf-template/amazondynamodbIT_source_to_sink.conf

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
######
18+
###### This config file is a demonstration of streaming processing in seatunnel config
19+
######
20+
21+
env {
22+
execution.parallelism = 1
23+
job.mode = "BATCH"
24+
}
25+
26+
source {
27+
# This is a example source plugin **only for test and demonstrate the feature source plugin**
28+
Amazondynamodb {
29+
url = "http://dynamodb-host:8000"
30+
region = "us-east-1"
31+
access_key_id = "dummy-key"
32+
secret_access_key = "dummy-secret"
33+
table = "source_table"
34+
schema = {
35+
fields {
36+
id = string
37+
c_map = "map<string, smallint>"
38+
c_array = "array<tinyint>"
39+
c_string = string
40+
c_boolean = boolean
41+
c_tinyint = tinyint
42+
c_smallint = smallint
43+
c_int = int
44+
c_bigint = bigint
45+
c_float = float
46+
c_double = double
47+
c_decimal = "decimal(2, 1)"
48+
c_bytes = bytes
49+
c_date = date
50+
c_timestamp = timestamp
51+
}
52+
}
53+
}
54+
}
55+
56+
transform {
57+
58+
# If you would like to get more information about how to configure seatunnel and see full list of transform plugins,
59+
# please go to https://seatunnel.apache.org/docs/transform-v2/sql
60+
}
61+
62+
sink {
63+
Amazondynamodb {
64+
url = "http://dynamodb-host:8000"
65+
region = "us-east-1"
66+
access_key_id = "dummy-key"
67+
secret_access_key = "dummy-secret"
68+
table = "sink_table"
69+
}
70+
# If you would like to get more information about how to configure seatunnel and see full list of sink plugins,
71+
# please go to https://seatunnel.apache.org/docs/category/sink-v2
72+
}

‎seatunnel_samples/connector-assert-conf-template/fakesource_to_assert.conf

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
18+
env {
19+
# You can set flink configuration here
20+
execution.parallelism = 1
21+
job.mode = BATCH
22+
#execution.checkpoint.interval = 10000
23+
#execution.checkpoint.data-uri = "hdfs://localhost:9000/checkpoint"
24+
}
25+
26+
source {
27+
# This is a example source plugin **only for test and demonstrate the feature source plugin**
28+
FakeSource {
29+
result_table_name = "fake"
30+
row.num = 100
31+
split.row = 25
32+
split.read-interval = 2000
33+
int.min = 32767
34+
schema = {
35+
fields {
36+
name = "string"
37+
age = "int"
38+
}
39+
}
40+
}
41+
# If you would like to get more information about how to configure seatunnel and see full list of source plugins,
42+
# please go to https://seatunnel.apache.org/docs/flink/configuration/source-plugins/Fake
43+
}
44+
45+
transform {
46+
Filter {
47+
source_table_name = "fake"
48+
result_table_name = "fake1"
49+
fields = ["name", "age"]
50+
}
51+
}
52+
53+
sink {
54+
Assert {
55+
source_table_name = "fake1"
56+
rules =
57+
{
58+
row_rules = [
59+
{
60+
rule_type = MAX_ROW
61+
rule_value = 100
62+
},
63+
{
64+
rule_type = MIN_ROW
65+
rule_value = 5
66+
}
67+
],
68+
field_rules = [{
69+
field_name = name
70+
field_type = string
71+
field_value = [
72+
{
73+
rule_type = NOT_NULL
74+
},
75+
{
76+
rule_type = MIN_LENGTH
77+
rule_value = 5
78+
},
79+
{
80+
rule_type = MAX_LENGTH
81+
rule_value = 65535
82+
}
83+
]
84+
}, {
85+
field_name = age
86+
field_type = int
87+
field_value = [
88+
{
89+
rule_type = NOT_NULL
90+
},
91+
{
92+
rule_type = MIN
93+
rule_value = 32767
94+
},
95+
{
96+
rule_type = MAX
97+
rule_value = 2147483647
98+
}
99+
]
100+
}
101+
]
102+
}
103+
104+
}
105+
# If you would like to get more information about how to configure seatunnel and see full list of sink plugins,
106+
# please go to https://seatunnel.apache.org/docs/connector-v2/sink/Assert
107+
}

‎seatunnel_samples/connector-cassandra-conf-template/application.conf

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
18+
datastax-java-driver {
19+
advanced.protocol.version = V5
20+
profiles {
21+
slow {
22+
basic.request.timeout = 10 seconds
23+
}
24+
}
25+
}

‎seatunnel_samples/connector-cassandra-conf-template/cassandra_to_cassandra.conf

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
18+
env {
19+
# You can set spark configuration here
20+
execution.parallelism = 1
21+
job.mode = "BATCH"
22+
}
23+
24+
source {
25+
# This is a example source plugin **only for test and demonstrate the feature source plugin**
26+
Cassandra {
27+
host = "cassandra:9042"
28+
username = ""
29+
password = ""
30+
datacenter = "datacenter1"
31+
keyspace = "test"
32+
cql = "select * from source_table"
33+
result_table_name = "source_table"
34+
}
35+
}
36+
37+
transform {
38+
# If you would like to get more information about how to configure seatunnel and see full list of transform plugins,
39+
# please go to https://seatunnel.apache.org/docs/transform-v2/sql
40+
}
41+
42+
sink {
43+
Cassandra {
44+
host = "cassandra:9042"
45+
username = ""
46+
password = ""
47+
datacenter = "datacenter1"
48+
keyspace = "test"
49+
async_write = "true"
50+
table = "sink_table"
51+
}
52+
}

‎seatunnel_samples/connector-cassandra-conf-template/init/cassandra_init.conf

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
18+
source_table = """
19+
create table if not exists source_table(
20+
id bigint,
21+
c_ascii ascii,
22+
c_bigint bigint,
23+
c_blob blob,
24+
c_boolean boolean,
25+
c_decimal decimal,
26+
c_double double,
27+
c_float float,
28+
c_int int,
29+
c_timestamp timestamp,
30+
c_uuid uuid,
31+
c_text text,
32+
c_varint varint,
33+
c_timeuuid timeuuid,
34+
c_inet inet,
35+
c_date date,
36+
c_smallint smallint,
37+
c_tinyint tinyint,
38+
c_list_float list<float>,
39+
c_list_int list<int>,
40+
c_set_double set<double>,
41+
c_set_bigint set<bigint>,
42+
c_map map<text,int>,
43+
PRIMARY KEY (id)
44+
);
45+
"""
46+
47+
sink_table = """
48+
create table if not exists sink_table(
49+
id bigint,
50+
c_ascii ascii,
51+
c_bigint bigint,
52+
c_blob blob,
53+
c_boolean boolean,
54+
c_decimal decimal,
55+
c_double double,
56+
c_float float,
57+
c_int int,
58+
c_timestamp timestamp,
59+
c_uuid uuid,
60+
c_text text,
61+
c_varint varint,
62+
c_timeuuid timeuuid,
63+
c_inet inet,
64+
c_date date,
65+
c_smallint smallint,
66+
c_tinyint tinyint,
67+
c_list_float list<float>,
68+
c_list_int list<int>,
69+
c_set_double set<double>,
70+
c_set_bigint set<bigint>,
71+
c_map map<text,int>,
72+
PRIMARY KEY (id)
73+
);
74+
"""
75+
76+
insert_cql = """
77+
insert into source_table
78+
(
79+
id,
80+
c_ascii,
81+
c_bigint,
82+
c_blob,
83+
c_boolean,
84+
c_decimal,
85+
c_double,
86+
c_float,
87+
c_int,
88+
c_timestamp,
89+
c_uuid,
90+
c_text,
91+
c_varint,
92+
c_timeuuid,
93+
c_inet,
94+
c_date,
95+
c_smallint,
96+
c_tinyint,
97+
c_list_float,
98+
c_list_int,
99+
c_set_double,
100+
c_set_bigint,
101+
c_map
102+
)
103+
values
104+
(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
105+
"""

‎seatunnel_samples/connector-cdc-mongodb-conf-template/ddl/inventory.js

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one or more
2+
// contributor license agreements. See the NOTICE file distributed with
3+
// -- this work for additional information regarding copyright ownership.
4+
// The ASF licenses this file to You under the Apache License, Version 2.0
5+
// (the "License"); you may not use this file except in compliance with
6+
// the License. You may obtain a copy of the License at
7+
//
8+
// http://www.apache.org/licenses/LICENSE-2.0
9+
//
10+
// Unless required by applicable law or agreed to in writing, software
11+
// distributed under the License is distributed on an "AS IS" BASIS,
12+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
// See the License for the specific language governing permissions and
14+
// limitations under the License.
15+
16+
db.getCollection('products').insertOne({"_id": ObjectId("100000000000000000000101"), "name": "scooter", "description": "Small 2-wheel scooter", "weight": "314"});
17+
db.getCollection('products').insertOne({"_id": ObjectId("100000000000000000000102"), "name": "car battery", "description": "12V car battery", "weight": "81"});
18+
db.getCollection('products').insertOne({"_id": ObjectId("100000000000000000000103"), "name": "12-pack drill bits", "description": "12-pack of drill bits with sizes ranging from #40 to #3", "weight": "8"});
19+
db.getCollection('products').insertOne({"_id": ObjectId("100000000000000000000104"), "name": "hammer", "description": "12oz carpenter''s hammer", "weight": "75"});
20+
db.getCollection('products').insertOne({"_id": ObjectId("100000000000000000000105"), "name": "hammer", "description": "12oz carpenter''s hammer", "weight": "875"});
21+
db.getCollection('products').insertOne({"_id": ObjectId("100000000000000000000106"), "name": "hammer", "description": "12oz carpenter''s hammer", "weight": "10"});
22+
db.getCollection('products').insertOne({"_id": ObjectId("100000000000000000000107"), "name": "rocks", "description": "box of assorted rocks", "weight": "53"});
23+
db.getCollection('products').insertOne({"_id": ObjectId("100000000000000000000108"), "name": "jacket", "description": "water resistent black wind breaker", "weight": "1"});
24+

0 commit comments

Comments
 (0)