Skip to content

Commit a583b1d

Browse files
YenchangChanYenchangChan
committed
refactor: ckmanctl command
1 parent 1148d15 commit a583b1d

File tree

18 files changed

+397
-212
lines changed

18 files changed

+397
-212
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ ckmanpasswd
1414
migrate
1515
znodefix
1616
znode_count
17+
cmd/ckmanctl/ckmanctl
1718
pkged.go
1819
static/dist/
1920
coverage.xml

‎Dockerfile

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,7 @@ FROM debian:stable-slim
33
RUN mkdir -p /etc/ckman && cd /etc/ckman && \
44
mkdir bin run logs conf package
55
ADD ./ckman /etc/ckman/bin/ckman
6-
ADD ./migrate /etc/ckman/bin/migrate
7-
ADD ./ckmanpasswd /etc/ckman/bin/ckmanpasswd
8-
ADD ./znodefix /etc/ckman/bin/znodefix
9-
ADD ./znode_count /etc/ckman/bin/znode_count
6+
ADD ./cmd/ckmanctl/ckmanctl /etc/ckman/bin/ckmanctl
107
ADD ./README.md /etc/ckman/package/README.md
118
ADD ./resources/ckman.hjson /etc/ckman/conf
129
ADD ./resources/migrate.hjson /etc/ckman/conf

‎Makefile

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -29,19 +29,13 @@ frontend:
2929
backend:
3030
@rm -rf ${PKGFULLDIR}
3131
go build ${LDFLAGS}
32-
go build ${LDFLAGS} -o ckmanpasswd cmd/password/password.go
33-
go build ${LDFLAGS} -o migrate cmd/migrate/migrate.go
34-
go build ${LDFLAGS} -o znodefix cmd/znodefix/znodefix.go
35-
go build ${LDFLAGS} -o znode_count cmd/znodecnt/znodecount.go
32+
go build ${LDFLAGS} -o cmd/ckmanctl/ckmanctl cmd/ckmanctl/ckmanctl.go
3633

3734
.PHONY: debug
3835
debug:
3936
@rm -rf ${PKGFULLDIR}
4037
go build ${GCFLAGS} ${LDFLAGS}
41-
go build ${GCFLAGS} ${LDFLAGS} -o ckmanpasswd cmd/password/password.go
42-
go build ${GCFLAGS} ${LDFLAGS} -o migrate cmd/migrate/migrate.go
43-
go build ${GCFLAGS} ${LDFLAGS} -o znodefix cmd/znodefix/znodefix.go
44-
go build ${GCFLAGS} ${LDFLAGS} -o znode_count cmd/znodecnt/znodecount.go
38+
go build ${LDFLAGS} -o cmd/ckmanctl/ckmanctl cmd/ckmanctl/ckmanctl.go
4539

4640
.PHONY: pre
4741
pre:
@@ -76,10 +70,7 @@ package:build
7670
@rm -rf ${PKGFULLDIR_TMP}
7771
@mkdir -p ${PKGFULLDIR_TMP}/bin ${PKGFULLDIR_TMP}/conf ${PKGFULLDIR_TMP}/run ${PKGFULLDIR_TMP}/logs ${PKGFULLDIR_TMP}/package ${PKGFULLDIR_TMP}/dbscript
7872
@mv ${SHDIR}/ckman ${PKGFULLDIR_TMP}/bin
79-
@mv ${SHDIR}/ckmanpasswd ${PKGFULLDIR_TMP}/bin
80-
@mv ${SHDIR}/migrate ${PKGFULLDIR_TMP}/bin
81-
@mv ${SHDIR}/znodefix ${PKGFULLDIR_TMP}/bin
82-
@mv ${SHDIR}/znode_count ${PKGFULLDIR_TMP}/bin
73+
@mv ${SHDIR}/cmd/ckmanctl/ckmanctl ${PKGFULLDIR_TMP}/binn
8374
@cp ${SHDIR}/resources/start ${PKGFULLDIR_TMP}/bin
8475
@cp ${SHDIR}/resources/stop ${PKGFULLDIR_TMP}/bin
8576
@cp ${SHDIR}/resources/yaml2json.${GOARCH} ${PKGFULLDIR_TMP}/bin/yaml2json

‎cmd/ckmanctl/ckmanctl.go

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
package main
2+
3+
/*
4+
ckmanctl migrate -f /etc/ckman/conf/migrate.hjson
5+
ckmanctl password
6+
ckmanctl get znodes --path
7+
ckmanctl delete znodes cluster suball
8+
ckmanctl delete znodes cluster replica_queue
9+
*/
10+
11+
import (
12+
"strings"
13+
14+
"github.com/alecthomas/kingpin/v2"
15+
16+
"github.com/housepower/ckman/cmd/metacache"
17+
"github.com/housepower/ckman/cmd/migrate"
18+
"github.com/housepower/ckman/cmd/password"
19+
"github.com/housepower/ckman/cmd/znodes"
20+
"github.com/housepower/ckman/log"
21+
)
22+
23+
var (
24+
migrateCmd = kingpin.Command("migrate", "migrate cluster config from old repersistence to new persistence")
25+
m_conf = migrateCmd.Flag("conf", "migrate config file path").Default("/etc/ckman/conf/migrate.hjson").Short('c').String()
26+
27+
//passwordCmd = kingpin.Command("password", "encrypt password")
28+
29+
// ckmanctl get znodes /clickhouse/tables/chenyc1 -r -s 20
30+
getCmd = kingpin.Command("get", "get options")
31+
g_znodes = getCmd.Command("znodes", "get znodes")
32+
gz_host = g_znodes.Flag("host", "host").Short('h').Default("127.0.0.1:2181").String()
33+
gz_recursive = g_znodes.Flag("recursive", "recursive").Short('r').Bool()
34+
gz_sort = g_znodes.Flag("sort", "sort number").Short('s').Int()
35+
gz_path = g_znodes.Arg("path", "path").Default("/clickhouse").String()
36+
37+
deleteCmd = kingpin.Command("delete", "delete options")
38+
d_znodes = deleteCmd.Command("znodes", "delete znodes")
39+
dz_suball = d_znodes.Command("suball", "delete all subnodes")
40+
dzs_cluster = dz_suball.Arg("cluster", "cluster").String()
41+
dzs_dryrun = dz_suball.Flag("dryrun", "dryrun").Short('d').Bool()
42+
dzs_path = dz_suball.Flag("path", "znode path").Short('p').String()
43+
dzs_conf = dz_suball.Flag("conf", "config file path").Short('c').Default("/etc/ckman/conf/ckman.hjson").String()
44+
45+
// ckmanctl delete znodes queue abc
46+
dz_queue = d_znodes.Command("queue", "delete replica queue")
47+
dzq_cluster = dz_queue.Arg("cluster", "cluster").String()
48+
dzq_dryrun = dz_queue.Flag("dryrun", "dryrun").Short('d').Bool()
49+
dzq_numtries = dz_queue.Flag("trynum", "num_tries").Short('n').Default("100").Int()
50+
dzq_conf = dz_queue.Flag("conf", "config file path").Short('c').Default("/etc/ckman/conf/ckman.hjson").String()
51+
52+
setCmd = kingpin.Command("set", "set options")
53+
s_metacacheCmd = setCmd.Command("metacache", "set metacache options")
54+
sm_cluster = s_metacacheCmd.Arg("cluster", "cluster").String()
55+
sm_conf = s_metacacheCmd.Flag("conf", "config file path").Short('c').Default("/etc/ckman/conf/ckman.hjson").String()
56+
sm_dryrun = s_metacacheCmd.Flag("dryrun", "dryrun").Short('d').Bool()
57+
)
58+
59+
func main() {
60+
log.InitLoggerDefault("debug", []string{"/var/log/ckmanctl.log"})
61+
command := kingpin.Parse()
62+
firstCmd := strings.Split(command, " ")[0]
63+
switch firstCmd {
64+
case "migrate":
65+
migrate.MigrateHandle(*m_conf)
66+
case "password":
67+
password.PasswordHandle()
68+
case "get":
69+
znodes.ZCntHandle(znodes.ZCntOpts{
70+
Path: *gz_path,
71+
Recursive: *gz_recursive,
72+
SortNumber: *gz_sort,
73+
Zkhosts: *gz_host,
74+
})
75+
case "delete":
76+
thirdCmd := strings.Split(command, " ")[2]
77+
if thirdCmd == "suball" {
78+
znodes.SuballHandle(znodes.ZSuballOpts{
79+
ClusterName: *dzs_cluster,
80+
ConfigFile: *dzs_conf,
81+
Dryrun: *dzs_dryrun,
82+
Node: *dzs_path,
83+
})
84+
} else if thirdCmd == "queue" {
85+
znodes.ReplicaQueueHandle(znodes.ZReplicaQueueOpts{
86+
ClusterName: *dzq_cluster,
87+
Dryrun: *dzq_dryrun,
88+
NumTries: *dzq_numtries,
89+
ConfigFile: *dzq_conf,
90+
})
91+
}
92+
case "set":
93+
secondCmd := strings.Split(command, " ")[1]
94+
if secondCmd == "metacache" {
95+
metacache.MetacacheHandle(metacache.MetacacheOpts{
96+
ClusterName: *sm_cluster,
97+
ConfigFile: *sm_conf,
98+
Dryrun: *sm_dryrun,
99+
})
100+
}
101+
}
102+
}

‎cmd/metacache/metacache.go

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
package metacache
2+
3+
import (
4+
"fmt"
5+
"os"
6+
7+
"github.com/housepower/ckman/common"
8+
"github.com/housepower/ckman/config"
9+
"github.com/housepower/ckman/log"
10+
"github.com/housepower/ckman/model"
11+
"github.com/housepower/ckman/repository"
12+
)
13+
14+
type MetacacheOpts struct {
15+
ClusterName string
16+
ConfigFile string
17+
Dryrun bool
18+
}
19+
20+
func MetacacheHandle(opts MetacacheOpts) {
21+
if err := config.ParseConfigFile(opts.ConfigFile, ""); err != nil {
22+
fmt.Printf("Parse config file %s fail: %v\n", opts.ConfigFile, err)
23+
os.Exit(-1)
24+
}
25+
26+
var conf config.CKManConfig
27+
_ = common.DeepCopyByGob(&conf, &config.GlobalConfig)
28+
err := common.Gsypt.Unmarshal(&config.GlobalConfig)
29+
if err != nil {
30+
fmt.Printf("gsypt config file %s fail: %v\n", opts.ConfigFile, err)
31+
os.Exit(-2)
32+
}
33+
err = repository.InitPersistent()
34+
if err != nil {
35+
fmt.Printf("init persistent failed:%v\n", err)
36+
os.Exit(-3)
37+
}
38+
39+
cluster, err := repository.Ps.GetClusterbyName(opts.ClusterName)
40+
if err != nil {
41+
fmt.Printf("get cluster %s failed:%v\n", opts.ClusterName, err)
42+
os.Exit(-4)
43+
}
44+
45+
if common.CompareClickHouseVersion(cluster.Version, "22.4.x") < 0 {
46+
fmt.Printf("cluster %s version %s not support metacache\n", opts.ClusterName, cluster.Version)
47+
os.Exit(-5)
48+
}
49+
50+
ckConns := make(map[string]*common.Conn, len(cluster.Hosts))
51+
for _, host := range cluster.Hosts {
52+
conn, err := common.ConnectClickHouse(host, model.ClickHouseDefaultDB, cluster.GetConnOption())
53+
if err == nil {
54+
ckConns[host] = conn
55+
}
56+
}
57+
58+
if len(ckConns) == 0 {
59+
fmt.Printf("connect to cluster %s failed\n", opts.ClusterName)
60+
os.Exit(-6)
61+
}
62+
63+
i := 0
64+
for host, conn := range ckConns {
65+
_, dbTbls, err := common.GetMergeTreeTables("MergeTree", "", conn)
66+
if err != nil {
67+
fmt.Printf("[%s]get tables of cluster %s failed:%v\n", host, opts.ClusterName, err)
68+
continue
69+
}
70+
for db, tbls := range dbTbls {
71+
for _, table := range tbls {
72+
query := fmt.Sprintf("ALTER TABLE `%s`.`%s` MODIFY SETTING use_metadata_cache = true", db, table)
73+
log.Logger.Debugf("[%s][%s]%s", opts.ClusterName, host, query)
74+
if opts.Dryrun {
75+
fmt.Printf("[%4d][%s][%s]%s\n", i, opts.ClusterName, host, query)
76+
i++
77+
} else {
78+
if err = conn.Exec(query); err != nil {
79+
fmt.Printf("[%s]%s failed:%v\n", host, query, err)
80+
}
81+
}
82+
}
83+
}
84+
}
85+
}

‎cmd/migrate/migrate.go

Lines changed: 8 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,17 @@
1-
package main
1+
package migrate
22

33
import (
4-
"flag"
5-
"fmt"
64
"io"
75
"os"
86

97
"github.com/hjson/hjson-go/v4"
10-
"github.com/housepower/ckman/common"
118
"github.com/housepower/ckman/log"
129
"github.com/housepower/ckman/repository"
1310
_ "github.com/housepower/ckman/repository/local"
1411
_ "github.com/housepower/ckman/repository/mysql"
1512
"github.com/pkg/errors"
1613
)
1714

18-
/*
19-
*
20-
auto migrate cluster config between diffrent persistent policy
21-
eg.
22-
23-
migrate --config=/etc/ckman/conf/migrate.hjson
24-
*/
25-
type CmdOptions struct {
26-
ShowVer bool
27-
ConfigFile string
28-
}
29-
3015
type PersistentConfig struct {
3116
Policy string
3217
Config map[string]interface{}
@@ -39,29 +24,13 @@ type MigrateConfig struct {
3924
}
4025

4126
var (
42-
cmdOps CmdOptions
43-
GitCommitHash string
44-
BuildTimeStamp string
45-
psrc repository.PersistentMgr
46-
pdst repository.PersistentMgr
27+
psrc repository.PersistentMgr
28+
pdst repository.PersistentMgr
4729
)
4830

49-
func initCmdOptions() {
50-
cmdOps = CmdOptions{
51-
ShowVer: false,
52-
ConfigFile: "/etc/ckman/conf/migrate.hjson",
53-
}
54-
common.EnvBoolVar(&cmdOps.ShowVer, "v")
55-
common.EnvStringVar(&cmdOps.ConfigFile, "config")
56-
57-
flag.BoolVar(&cmdOps.ShowVer, "v", cmdOps.ShowVer, "show build version and quit")
58-
flag.StringVar(&cmdOps.ConfigFile, "config", cmdOps.ConfigFile, "migrate config file")
59-
flag.Parse()
60-
}
61-
62-
func ParseConfig() (MigrateConfig, error) {
31+
func ParseConfig(conf string) (MigrateConfig, error) {
6332
var config MigrateConfig
64-
f, err := os.Open(cmdOps.ConfigFile)
33+
f, err := os.Open(conf)
6534
if err != nil {
6635
return MigrateConfig{}, errors.Wrap(err, "")
6736
}
@@ -163,18 +132,10 @@ func Migrate() error {
163132
return nil
164133
}
165134

166-
func main() {
167-
log.InitLoggerConsole()
168-
initCmdOptions()
169-
if cmdOps.ShowVer {
170-
fmt.Println("Build Timestamp:", BuildTimeStamp)
171-
fmt.Println("Git Commit Hash:", GitCommitHash)
172-
os.Exit(0)
173-
}
174-
175-
config, err := ParseConfig()
135+
func MigrateHandle(conf string) {
136+
config, err := ParseConfig(conf)
176137
if err != nil {
177-
log.Logger.Fatalf("parse config file %s failed: %v", cmdOps.ConfigFile, err)
138+
log.Logger.Fatalf("parse config file %s failed: %v", conf, err)
178139
}
179140
psrc, err = PersistentCheck(config, config.Source)
180141
if err != nil {

‎cmd/password/password.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package main
1+
package password
22

33
import (
44
"crypto/md5"
@@ -11,7 +11,7 @@ import (
1111
"golang.org/x/term"
1212
)
1313

14-
func main() {
14+
func PasswordHandle() {
1515
common.LoadUsers(path.Join(common.GetWorkDirectory(), "conf"))
1616
fmt.Println(`Password must be at least 8 characters long.
1717
Password must contain at least three character categories among the following:

0 commit comments

Comments
 (0)