Skip to content

feat: use registryMirror addon as Containerd mirror #1117

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: dkoshkin/feat-registry-mirror-addon
Choose a base branch
from
Open

feat: use registryMirror addon as Containerd mirror #1117

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1d0f700
feat: use registryMirror addon as Containerd mirror
dkoshkin May 2, 2025
1054c68
fixup! fix: remove unused function args
dkoshkin May 7, 2025
71bc9ca
fixup! after rebase
dkoshkin May 8, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 47 additions & 4 deletions pkg/handlers/generic/mutation/mirrors/inject.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
Expand All @@ -22,6 +23,7 @@ import (
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches"
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches/selectors"
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/variables"
registryutils "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/registry/utils"
handlersutils "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/utils"
)

Expand Down Expand Up @@ -62,8 +64,8 @@ func (h *globalMirrorPatchHandler) Mutate(
obj *unstructured.Unstructured,
vars map[string]apiextensionsv1.JSON,
holderRef runtimehooksv1.HolderReference,
clusterKey ctrlclient.ObjectKey,
_ mutation.ClusterGetter,
_ ctrlclient.ObjectKey,
clusterGetter mutation.ClusterGetter,
) error {
log := ctrl.LoggerFrom(ctx).WithValues(
"holderRef", holderRef,
Expand All @@ -82,14 +84,24 @@ func (h *globalMirrorPatchHandler) Mutate(
v1alpha1.ImageRegistriesVariableName,
)

_, registryMirrorErr := variables.Get[v1alpha1.RegistryAddon](
vars,
v1alpha1.ClusterConfigVariableName,
[]string{"addons", v1alpha1.RegistryAddonVariableName}...)

switch {
case variables.IsNotFoundError(imageRegistriesErr) && variables.IsNotFoundError(globalMirrorErr):
log.V(5).Info("Image Registry Credentials and Global Registry Mirror variable not defined")
case variables.IsNotFoundError(imageRegistriesErr) &&
variables.IsNotFoundError(globalMirrorErr) &&
variables.IsNotFoundError(registryMirrorErr):
log.V(5).
Info("Image Registry Credentials and Global Registry Mirror and Registry Mirror Addon variable not defined")
return nil
case imageRegistriesErr != nil && !variables.IsNotFoundError(imageRegistriesErr):
return imageRegistriesErr
case globalMirrorErr != nil && !variables.IsNotFoundError(globalMirrorErr):
return globalMirrorErr
case registryMirrorErr != nil && !variables.IsNotFoundError(registryMirrorErr):
return registryMirrorErr
}

var registriesWithOptionalCA []containerdConfig //nolint:prealloc // We don't know the size of the slice yet.
Expand Down Expand Up @@ -121,6 +133,22 @@ func (h *globalMirrorPatchHandler) Mutate(
registryWithOptionalCredentials,
)
}
if registryMirrorErr == nil {
cluster, err := clusterGetter(ctx)
if err != nil {
log.Error(
err,
"failed to get cluster from Global Mirror mutation handler",
)
return err
}

registryConfig, err := containerdConfigFromRegistryAddon(cluster)
if err != nil {
return err
}
registriesWithOptionalCA = append(registriesWithOptionalCA, registryConfig)
}

needConfiguration := needContainerdConfiguration(
registriesWithOptionalCA,
Expand Down Expand Up @@ -234,6 +262,21 @@ func containerdConfigFromImageRegistry(
return configWithOptionalCACert, nil
}

func containerdConfigFromRegistryAddon(cluster *clusterv1.Cluster) (containerdConfig, error) {
serviceIP, err := registryutils.ServiceIPForCluster(cluster)
if err != nil {
return containerdConfig{}, fmt.Errorf("error getting service IP for the registry mirror addon: %w", err)
}

config := containerdConfig{
// FIXME: Generate a self-signed CA.
URL: fmt.Sprintf("http://%s", serviceIP),
Mirror: true,
}

return config, nil
}

func generateFiles(
registriesWithOptionalCA []containerdConfig,
) ([]bootstrapv1.File, error) {
Expand Down
92 changes: 89 additions & 3 deletions pkg/handlers/generic/mutation/mirrors/inject_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,11 @@ import (
"github.com/stretchr/testify/assert"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apiserver/pkg/storage/names"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"

"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1"
Expand All @@ -32,6 +36,10 @@ func TestMirrorsPatch(t *testing.T) {
}

var _ = Describe("Generate Global mirror patches", func() {
clientScheme := runtime.NewScheme()
utilruntime.Must(clientgoscheme.AddToScheme(clientScheme))
utilruntime.Must(clusterv1.AddToScheme(clientScheme))

patchGenerator := func() mutation.GeneratePatches {
// Always initialize the testEnv variable in the closure.
// This will allow ginkgo to initialize testEnv variable during test execution time.
Expand All @@ -40,7 +48,7 @@ var _ = Describe("Generate Global mirror patches", func() {
// that are written by the tests.
// Test cases writes credentials secret that the mutator handler reads.
// Using direct client will enable reading it immediately.
client, err := testEnv.GetK8sClient()
client, err := testEnv.GetK8sClientWithScheme(clientScheme)
gomega.Expect(err).To(gomega.BeNil())
return mutation.NewMetaGeneratePatchesHandler("", client, NewPatch(client)).(mutation.GeneratePatches)
}
Expand Down Expand Up @@ -330,11 +338,69 @@ var _ = Describe("Generate Global mirror patches", func() {
},
},
},
{
Name: "files added in KubeadmControlPlaneTemplate for registry mirror addon",
Vars: []runtimehooksv1.Variable{
capitest.VariableWithValue(
v1alpha1.ClusterConfigVariableName,
v1alpha1.RegistryAddon{},
[]string{"addons", v1alpha1.RegistryAddonVariableName}...,
),
},
RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""),
ExpectedPatchMatchers: []capitest.JSONPatchMatcher{
{
Operation: "add",
Path: "/spec/template/spec/kubeadmConfigSpec/files",
ValueMatcher: gomega.HaveExactElements(
gomega.HaveKeyWithValue(
"path", "/etc/containerd/certs.d/_default/hosts.toml",
),
gomega.HaveKeyWithValue(
"path", "/etc/caren/containerd/patches/registry-config.toml",
),
),
},
},
},
{
Name: "files added in KubeadmConfigTemplate for registry mirror addon",
Vars: []runtimehooksv1.Variable{
capitest.VariableWithValue(
v1alpha1.ClusterConfigVariableName,
v1alpha1.RegistryAddon{},
[]string{"addons", v1alpha1.RegistryAddonVariableName}...,
),
capitest.VariableWithValue(
"builtin",
map[string]any{
"machineDeployment": map[string]any{
"class": names.SimpleNameGenerator.GenerateName("worker-"),
},
},
),
},
RequestItem: request.NewKubeadmConfigTemplateRequestItem(""),
ExpectedPatchMatchers: []capitest.JSONPatchMatcher{
{
Operation: "add",
Path: "/spec/template/spec/files",
ValueMatcher: gomega.HaveExactElements(
gomega.HaveKeyWithValue(
"path", "/etc/containerd/certs.d/_default/hosts.toml",
),
gomega.HaveKeyWithValue(
"path", "/etc/caren/containerd/patches/registry-config.toml",
),
),
},
},
},
}

// Create credentials secret before each test
BeforeEach(func(ctx SpecContext) {
client, err := helpers.TestEnv.GetK8sClient()
client, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme)
gomega.Expect(err).To(gomega.BeNil())
gomega.Expect(client.Create(
ctx,
Expand All @@ -344,11 +410,21 @@ var _ = Describe("Generate Global mirror patches", func() {
ctx,
newMirrorSecretWithoutCA(validMirrorNoCASecretName, request.Namespace),
)).To(gomega.BeNil())

gomega.Expect(client.Create(
ctx,
&clusterv1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: request.ClusterName,
Namespace: request.Namespace,
},
},
)).To(gomega.BeNil())
})

// Delete credentials secret after each test
AfterEach(func(ctx SpecContext) {
client, err := helpers.TestEnv.GetK8sClient()
client, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme)
gomega.Expect(err).To(gomega.BeNil())
gomega.Expect(client.Delete(
ctx,
Expand All @@ -358,6 +434,16 @@ var _ = Describe("Generate Global mirror patches", func() {
ctx,
newMirrorSecretWithoutCA(validMirrorNoCASecretName, request.Namespace),
)).To(gomega.BeNil())

gomega.Expect(client.Delete(
ctx,
&clusterv1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: request.ClusterName,
Namespace: request.Namespace,
},
},
)).To(gomega.BeNil())
})

// create test node for each case
Expand Down
Loading