Skip to content

Commit 78e1124

Browse files
ks-ci-botks-ci-bot
authored
Merge pull request #2111 from chilianyi/support-cri-docker
Support use docker for version >= v1.24.0
2 parents fb2a8ce + ace1ce7 commit 78e1124

File tree

12 files changed

+259
-23
lines changed

12 files changed

+259
-23
lines changed

‎cmd/kk/apis/kubekey/v1alpha2/default.go

+8-1
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
"strings"
2323

2424
"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util"
25+
"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubernetes"
2526
)
2627

2728
const (
@@ -42,6 +43,7 @@ const (
4243
DefaultEtcdVersion = "v3.5.6"
4344
DefaultEtcdPort = "2379"
4445
DefaultDockerVersion = "24.0.6"
46+
DefaultCriDockerdVersion = "0.3.9"
4547
DefaultContainerdVersion = "1.7.12"
4648
DefaultRuncVersion = "v1.1.11"
4749
DefaultCrictlVersion = "v1.29.0"
@@ -68,6 +70,7 @@ const (
6870
DefaultProxyMode = "ipvs"
6971
DefaultCrioEndpoint = "unix:///var/run/crio/crio.sock"
7072
DefaultContainerdEndpoint = "unix:///run/containerd/containerd.sock"
73+
DefaultCriDockerdEndpoint = "unix:///var/run/cri-dockerd.sock"
7174
DefaultIsulaEndpoint = "unix:///var/run/isulad.sock"
7275
Etcd = "etcd"
7376
Master = "master"
@@ -315,7 +318,11 @@ func SetDefaultClusterCfg(cfg *ClusterSpec) Kubernetes {
315318
if cfg.Kubernetes.ContainerRuntimeEndpoint == "" {
316319
switch cfg.Kubernetes.ContainerManager {
317320
case Docker:
318-
cfg.Kubernetes.ContainerRuntimeEndpoint = ""
321+
if kubernetes.IsAtLeastV124(cfg.Kubernetes.Version){
322+
cfg.Kubernetes.ContainerRuntimeEndpoint = DefaultCriDockerdEndpoint
323+
} else {
324+
cfg.Kubernetes.ContainerRuntimeEndpoint = ""
325+
}
319326
case Crio:
320327
cfg.Kubernetes.ContainerRuntimeEndpoint = DefaultCrioEndpoint
321328
case Containerd:

‎cmd/kk/pkg/binaries/kubernetes.go

+6-2
Original file line numberDiff line numberDiff line change
@@ -20,14 +20,14 @@ import (
2020
"fmt"
2121
"os/exec"
2222

23-
"github.com/pkg/errors"
24-
2523
kubekeyapiv1alpha2 "github.com/kubesphere/kubekey/v3/cmd/kk/apis/kubekey/v1alpha2"
2624
"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common"
2725
"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/cache"
2826
"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger"
2927
"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util"
3028
"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/files"
29+
"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubernetes"
30+
"github.com/pkg/errors"
3131
)
3232

3333
// K8sFilesDownloadHTTP defines the kubernetes' binaries that need to be downloaded in advance and downloads them.
@@ -40,6 +40,7 @@ func K8sFilesDownloadHTTP(kubeConf *common.KubeConf, path, version, arch string,
4040
kubecni := files.NewKubeBinary("kubecni", arch, kubekeyapiv1alpha2.DefaultCniVersion, path, kubeConf.Arg.DownloadCommand)
4141
helm := files.NewKubeBinary("helm", arch, kubekeyapiv1alpha2.DefaultHelmVersion, path, kubeConf.Arg.DownloadCommand)
4242
docker := files.NewKubeBinary("docker", arch, kubekeyapiv1alpha2.DefaultDockerVersion, path, kubeConf.Arg.DownloadCommand)
43+
criDockerd := files.NewKubeBinary("cri-dockerd", arch, kubekeyapiv1alpha2.DefaultCriDockerdVersion, path, kubeConf.Arg.DownloadCommand)
4344
crictl := files.NewKubeBinary("crictl", arch, kubekeyapiv1alpha2.DefaultCrictlVersion, path, kubeConf.Arg.DownloadCommand)
4445
containerd := files.NewKubeBinary("containerd", arch, kubekeyapiv1alpha2.DefaultContainerdVersion, path, kubeConf.Arg.DownloadCommand)
4546
runc := files.NewKubeBinary("runc", arch, kubekeyapiv1alpha2.DefaultRuncVersion, path, kubeConf.Arg.DownloadCommand)
@@ -49,6 +50,9 @@ func K8sFilesDownloadHTTP(kubeConf *common.KubeConf, path, version, arch string,
4950

5051
if kubeConf.Cluster.Kubernetes.ContainerManager == kubekeyapiv1alpha2.Docker {
5152
binaries = append(binaries, docker)
53+
if kubernetes.IsAtLeastV124(kubeConf.Cluster.Kubernetes.Version) && kubeConf.Cluster.Kubernetes.ContainerManager == common.Docker {
54+
binaries = append(binaries, criDockerd)
55+
}
5256
} else if kubeConf.Cluster.Kubernetes.ContainerManager == kubekeyapiv1alpha2.Containerd {
5357
binaries = append(binaries, containerd, runc)
5458
}

‎cmd/kk/pkg/bootstrap/confirm/tasks.go

+16-17
Original file line numberDiff line numberDiff line change
@@ -23,16 +23,16 @@ import (
2323
"regexp"
2424
"strings"
2525

26-
"github.com/mitchellh/mapstructure"
27-
"github.com/modood/table"
28-
"github.com/pkg/errors"
29-
versionutil "k8s.io/apimachinery/pkg/util/version"
30-
3126
"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common"
3227
"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/action"
3328
"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector"
3429
"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/logger"
3530
"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/util"
31+
"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubernetes"
32+
"github.com/mitchellh/mapstructure"
33+
"github.com/modood/table"
34+
"github.com/pkg/errors"
35+
versionutil "k8s.io/apimachinery/pkg/util/version"
3636
)
3737

3838
// PreCheckResults defines the items to be checked.
@@ -107,18 +107,17 @@ func (i *InstallationConfirm) Execute(runtime connector.Runtime) error {
107107
fmt.Println("https://github.com/kubesphere/kubekey#requirements-and-recommendations")
108108
fmt.Println("")
109109

110-
if k8sVersion, err := versionutil.ParseGeneric(i.KubeConf.Cluster.Kubernetes.Version); err == nil {
111-
if k8sVersion.AtLeast(versionutil.MustParseSemantic("v1.24.0")) && i.KubeConf.Cluster.Kubernetes.ContainerManager == common.Docker {
112-
fmt.Println("[Notice]")
113-
fmt.Println("Incorrect runtime. Please specify a container runtime other than Docker to install Kubernetes v1.24 or later.")
114-
fmt.Println("You can set \"spec.kubernetes.containerManager\" in the configuration file to \"containerd\" or add \"--container-manager containerd\" to the \"./kk create cluster\" command.")
115-
fmt.Println("For more information, see:")
116-
fmt.Println("https://github.com/kubesphere/kubekey/blob/master/docs/commands/kk-create-cluster.md")
117-
fmt.Println("https://kubernetes.io/docs/setup/production-environment/container-runtimes/#container-runtimes")
118-
fmt.Println("https://kubernetes.io/blog/2022/02/17/dockershim-faq/")
119-
fmt.Println("")
120-
stopFlag = true
121-
}
110+
if kubernetes.IsAtLeastV124(i.KubeConf.Cluster.Kubernetes.Version) && i.KubeConf.Cluster.Kubernetes.ContainerManager == common.Docker &&
111+
i.KubeConf.Cluster.Kubernetes.Type != common.Kubernetes {
112+
fmt.Println("[Notice]")
113+
fmt.Println("Incorrect runtime. Please specify a container runtime other than Docker to install Kubernetes v1.24 or later.")
114+
fmt.Println("You can set \"spec.kubernetes.containerManager\" in the configuration file to \"containerd\" or add \"--container-manager containerd\" to the \"./kk create cluster\" command.")
115+
fmt.Println("For more information, see:")
116+
fmt.Println("https://github.com/kubesphere/kubekey/blob/master/docs/commands/kk-create-cluster.md")
117+
fmt.Println("https://kubernetes.io/docs/setup/production-environment/container-runtimes/#container-runtimes")
118+
fmt.Println("https://kubernetes.io/blog/2022/02/17/dockershim-faq/")
119+
fmt.Println("")
120+
stopFlag = true
122121
}
123122

124123
if stopFlag {

‎cmd/kk/pkg/common/common.go

+1
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ const (
6262
Hybridnet = "hybridnet"
6363

6464
Docker = "docker"
65+
CriDockerd = "cri-dockerd"
6566
Crictl = "crictl"
6667
Containerd = "containerd"
6768
Crio = "crio"

‎cmd/kk/pkg/container/docker.go

+57-2
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,14 @@ import (
2121
"path/filepath"
2222
"strings"
2323

24-
"github.com/pkg/errors"
25-
2624
"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/common"
2725
"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/container/templates"
2826
"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/core/connector"
2927
"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/files"
3028
"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/registry"
3129
"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/utils"
30+
"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubernetes"
31+
"github.com/pkg/errors"
3232
)
3333

3434
type SyncDockerBinaries struct {
@@ -64,6 +64,39 @@ func (s *SyncDockerBinaries) Execute(runtime connector.Runtime) error {
6464
return nil
6565
}
6666

67+
type SyncCriDockerdBinaries struct {
68+
common.KubeAction
69+
}
70+
71+
func (s *SyncCriDockerdBinaries) Execute(runtime connector.Runtime) error {
72+
if err := utils.ResetTmpDir(runtime); err != nil {
73+
return err
74+
}
75+
76+
binariesMapObj, ok := s.PipelineCache.Get(common.KubeBinaries + "-" + runtime.RemoteHost().GetArch())
77+
if !ok {
78+
return errors.New("get KubeBinary by pipeline cache failed")
79+
}
80+
binariesMap := binariesMapObj.(map[string]*files.KubeBinary)
81+
82+
criDockerd, ok := binariesMap[common.CriDockerd]
83+
if !ok {
84+
return errors.New("get KubeBinary key cri-dockerd by pipeline cache failed")
85+
}
86+
87+
dst := filepath.Join(common.TmpDir, criDockerd.FileName)
88+
if err := runtime.GetRunner().Scp(criDockerd.Path(), dst); err != nil {
89+
return errors.Wrap(errors.WithStack(err), fmt.Sprintf("sync cri-dockerd binaries failed"))
90+
}
91+
92+
if _, err := runtime.GetRunner().SudoCmd(
93+
fmt.Sprintf("mkdir -p /usr/bin && tar -zxf %s && mv cri-dockerd/* /usr/bin && rm -rf cri-dockerd", dst),
94+
false); err != nil {
95+
return errors.Wrap(errors.WithStack(err), fmt.Sprintf("install container runtime cri-dockerd binaries failed"))
96+
}
97+
return nil
98+
}
99+
67100
type EnableContainerdForDocker struct {
68101
common.KubeAction
69102
}
@@ -90,6 +123,19 @@ func (e *EnableDocker) Execute(runtime connector.Runtime) error {
90123
return nil
91124
}
92125

126+
type EnableCriDockerd struct {
127+
common.KubeAction
128+
}
129+
130+
func (e *EnableCriDockerd) Execute(runtime connector.Runtime) error {
131+
if _, err := runtime.GetRunner().SudoCmd(
132+
"systemctl daemon-reload && systemctl enable cri-docker && systemctl start cri-docker",
133+
false); err != nil {
134+
return errors.Wrap(errors.WithStack(err), fmt.Sprintf("enable and start cri-docker failed"))
135+
}
136+
return nil
137+
}
138+
93139
type DockerLoginRegistry struct {
94140
common.KubeAction
95141
}
@@ -141,6 +187,15 @@ func (d *DisableDocker) Execute(runtime connector.Runtime) error {
141187
filepath.Join("/etc/systemd/system", templates.DockerService.Name()),
142188
filepath.Join("/etc/docker", templates.DockerConfig.Name()),
143189
}
190+
191+
if kubernetes.IsAtLeastV124(d.KubeConf.Cluster.Kubernetes.Version) && d.KubeConf.Cluster.Kubernetes.ContainerManager == common.Docker {
192+
if _, err := runtime.GetRunner().SudoCmd("systemctl disable cri-docker && systemctl stop cri-docker",
193+
false); err != nil {
194+
return errors.Wrap(errors.WithStack(err), fmt.Sprintf("disable and stop cri-docker failed"))
195+
}
196+
files = append(files, filepath.Join("/etc/systemd/system", templates.CriDockerService.Name()))
197+
}
198+
144199
if d.KubeConf.Cluster.Registry.DataRoot != "" {
145200
files = append(files, d.KubeConf.Cluster.Registry.DataRoot)
146201
} else {

‎cmd/kk/pkg/container/module.go

+56
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import (
3030
"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/images"
3131
"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/kubernetes"
3232
"github.com/kubesphere/kubekey/v3/cmd/kk/pkg/registry"
33+
versionk8s "github.com/kubesphere/kubekey/v3/cmd/kk/pkg/version/kubernetes"
3334
)
3435

3536
type InstallContainerModule struct {
@@ -73,6 +74,19 @@ func InstallDocker(m *InstallContainerModule) []task.Interface {
7374
Retry: 2,
7475
}
7576

77+
syncCriDockerdBinaries := &task.RemoteTask{
78+
Name: "SyncCriDockerdBinaries",
79+
Desc: "Sync cri-dockerd binaries",
80+
Hosts: m.Runtime.GetHostsByRole(common.K8s),
81+
Prepare: &prepare.PrepareCollection{
82+
&kubernetes.NodeInCluster{Not: true},
83+
&CriDockerdExist{Not: true},
84+
},
85+
Action: new(SyncCriDockerdBinaries),
86+
Parallel: true,
87+
Retry: 2,
88+
}
89+
7690
generateContainerdService := &task.RemoteTask{
7791
Name: "GenerateContainerdService",
7892
Desc: "Generate containerd service",
@@ -161,6 +175,48 @@ func InstallDocker(m *InstallContainerModule) []task.Interface {
161175
Parallel: true,
162176
}
163177

178+
generateCriDockerdService := &task.RemoteTask{
179+
Name: "GenerateCriDockerdService",
180+
Desc: "Generate cri-dockerd service",
181+
Hosts: m.Runtime.GetHostsByRole(common.K8s),
182+
Prepare: &prepare.PrepareCollection{
183+
&kubernetes.NodeInCluster{Not: true},
184+
&CriDockerdExist{Not: true},
185+
},
186+
Action: &action.Template{
187+
Template: templates.CriDockerService,
188+
Dst: filepath.Join("/etc/systemd/system", templates.CriDockerService.Name()),
189+
},
190+
Parallel: true,
191+
}
192+
193+
enableCriDockerd := &task.RemoteTask{
194+
Name: "EnableCriDockerd",
195+
Desc: "Enable cri-dockerd",
196+
Hosts: m.Runtime.GetHostsByRole(common.K8s),
197+
Prepare: &prepare.PrepareCollection{
198+
&kubernetes.NodeInCluster{Not: true},
199+
&CriDockerdExist{Not: true},
200+
},
201+
Action: new(EnableCriDockerd),
202+
Parallel: true,
203+
}
204+
205+
if versionk8s.IsAtLeastV124(m.KubeConf.Cluster.Kubernetes.Version) && m.KubeConf.Cluster.Kubernetes.ContainerManager == common.Docker {
206+
return []task.Interface{
207+
syncBinaries,
208+
syncCriDockerdBinaries,
209+
generateContainerdService,
210+
generateDockerService,
211+
generateDockerConfig,
212+
enableContainerdForDocker,
213+
enableDocker,
214+
dockerLoginRegistry,
215+
generateCriDockerdService,
216+
enableCriDockerd,
217+
}
218+
}
219+
164220
return []task.Interface{
165221
syncBinaries,
166222
generateContainerdService,

‎cmd/kk/pkg/container/prepares.go

+18
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,24 @@ func (d *DockerExist) PreCheck(runtime connector.Runtime) (bool, error) {
4141
return !d.Not, nil
4242
}
4343

44+
type CriDockerdExist struct {
45+
common.KubePrepare
46+
Not bool
47+
}
48+
49+
func (d *CriDockerdExist) PreCheck(runtime connector.Runtime) (bool, error) {
50+
output, err := runtime.GetRunner().SudoCmd("if [ -z $(command -v cri-dockerd) ] || [ ! -e /var/run/cri-dockerd.sock ]; "+
51+
"then echo 'not exist'; "+
52+
"fi", false)
53+
if err != nil {
54+
return false, err
55+
}
56+
if strings.Contains(output, "not exist") {
57+
return d.Not, nil
58+
}
59+
return !d.Not, nil
60+
}
61+
4462
type CrictlExist struct {
4563
common.KubePrepare
4664
Not bool

‎cmd/kk/pkg/container/templates/cri_dockerd_service.go

+63
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
/*
2+
Copyright 2021 The KubeSphere Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package templates
18+
19+
import (
20+
"text/template"
21+
22+
"github.com/lithammer/dedent"
23+
)
24+
25+
var CriDockerService = template.Must(template.New("cri-docker.service").Parse(
26+
dedent.Dedent(`[Unit]
27+
Description=CRI Interface for Docker Application Container Engine
28+
Documentation=https://docs.mirantis.com
29+
30+
[Service]
31+
Type=notify
32+
ExecStart=/usr/bin/cri-dockerd --pod-infra-container-image docker.io/kubesphere/pause:3.8
33+
ExecReload=/bin/kill -s HUP $MAINPID
34+
TimeoutSec=0
35+
RestartSec=2
36+
Restart=always
37+
38+
# Note that StartLimit* options were moved from "Service" to "Unit" in systemd 229.
39+
# Both the old, and new location are accepted by systemd 229 and up, so using the old location
40+
# to make them work for either version of systemd.
41+
StartLimitBurst=3
42+
43+
# Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230.
44+
# Both the old, and new name are accepted by systemd 230 and up, so using the old name to make
45+
# this option work for either version of systemd.
46+
StartLimitInterval=60s
47+
48+
# Having non-zero Limit*s causes performance problems due to accounting overhead
49+
# in the kernel. We recommend using cgroups to do container-local accounting.
50+
LimitNOFILE=infinity
51+
LimitNPROC=infinity
52+
LimitCORE=infinity
53+
54+
# Comment TasksMax if your systemd version does not support it.
55+
# Only systemd 226 and above support this option.
56+
TasksMax=infinity
57+
Delegate=yes
58+
KillMode=process
59+
60+
[Install]
61+
WantedBy=multi-user.target
62+
63+
`)))

0 commit comments

Comments
 (0)