Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
129 changes: 127 additions & 2 deletions cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,15 @@ limitations under the License.
package main

import (
"context"
"crypto/tls"
"flag"
"fmt"
"os"
"time"

"github.com/go-logr/logr"
ostls "github.com/openshift/controller-runtime-common/pkg/tls"
appconfig "github.com/securesign/operator/internal/config"
"github.com/securesign/operator/internal/constants"
"github.com/securesign/operator/internal/controller"
Expand All @@ -34,6 +38,8 @@ import (
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
apiErrors "k8s.io/apimachinery/pkg/api/errors"
apimeta "k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/klog/v2"
Expand Down Expand Up @@ -93,6 +99,11 @@ func init() {
}

func main() {
// Create a cancellable context from the signal handler. The TLS profile watcher
// uses cancel() to trigger a graceful restart when the cluster policy changes.
ctx, cancel := context.WithCancel(ctrl.SetupSignalHandler())
defer cancel()

var (
metricsAddr string
enableLeaderElection bool
Expand All @@ -119,6 +130,10 @@ func main() {
utils.BoolFlagOrEnv(&appconfig.Openshift, "openshift", "OPENSHIFT", false, "Enable to ensures the operator applies OpenShift specific configurations.")
utils.StringFlagOrEnv(&appconfig.OpenshiftAPIServerName, "openshift-apiserver-name", "OPENSHIFT_APISERVER_NAME", "openshift-apiserver", "The OpenShift API Server name.")
utils.DurationFlagOrEnv(&appconfig.APIServerTimeout, "apiserver-timeout", "APISERVER_TIMEOUT", 30*time.Second, "The initial timeout for contacting the API Server, defaults to 30 seconds.")
utils.BoolFlagOrEnv(&appconfig.DisableClusterTLSProfile, "disable-cluster-tls-profile", "DISABLE_CLUSTER_TLS_PROFILE", false,
"Disable reading the cluster-wide TLS security profile from configv1.APIServer. "+
"When set, the operator uses Intermediate TLS profile defaults (TLS 1.2 minimum). "+
"Use this as an escape hatch if the cluster profile causes compatibility issues.")
utils.StringFlagOrEnv(&appconfig.IngressHostTemplate, "ingress-host-template", "INGRESS_HOST_TEMPLATE", appconfig.IngressHostTemplate,
"Default hostname template for non-OpenShift Ingress resources when ExternalAccess.Host is not set. "+
"Uses Go fmt.Sprintf with %[1]s=service name, %[2]s=namespace. Ignored on OpenShift.")
Expand Down Expand Up @@ -162,6 +177,38 @@ func main() {
setupLog.Info("Platform explicitly configured via flag/env", "openshift", appconfig.Openshift)
}

// Resolve the cluster TLS security profile once at startup, before the webhook and metrics
// servers are configured. A dedicated bootstrap client is used because the manager has not
// started yet. On vanilla Kubernetes (no configv1.APIServer) or when the flag is set,
// this falls back to the Intermediate profile.
var bootstrapClient client.Client
if appconfig.Openshift && !appconfig.DisableClusterTLSProfile {
var bootErr error
bootstrapClient, bootErr = client.New(ctrl.GetConfigOrDie(), client.Options{Scheme: scheme})
if bootErr != nil {
setupLog.Error(bootErr, "unable to create bootstrap client for TLS profile resolution")
os.Exit(1)
}
}

// Bound the profile resolution by the same timeout used for OpenShift auto-detection and
// derive it from the cancellable startup context, so a SIGTERM or a network partition during
// a slow bootstrap API call aborts startup instead of hanging indefinitely.
resolveCtx, resolveCancel := context.WithTimeout(ctx, appconfig.APIServerTimeout)
tlsProfileSpec, tlsAdherence, resolveErr := resolveClusterTLSProfile(
resolveCtx, bootstrapClient, appconfig.Openshift, appconfig.DisableClusterTLSProfile, setupLog)
resolveCancel()
if resolveErr != nil {
setupLog.Error(resolveErr, "unable to resolve cluster TLS security profile")
os.Exit(1)
}

tlsConfigFn, unsupportedCiphers := ostls.NewTLSConfigFromProfile(tlsProfileSpec)
if len(unsupportedCiphers) > 0 {
setupLog.Info("cluster TLS profile contains ciphers unsupported by Go TLS",
"ciphers", unsupportedCiphers)
}

// if the enable-http2 flag is false (the default), http/2 should be disabled
// due to its vulnerabilities. More specifically, disabling http/2 will
// prevent from being vulnerable to the HTTP/2 Stream Cancelation and
Expand All @@ -177,7 +224,7 @@ func main() {
if !enableHTTP2 {
tlsOpts = append(tlsOpts, disableHTTP2)
}

tlsOpts = append(tlsOpts, tlsConfigFn)
webhookServer := webhook.NewServer(webhook.Options{
TLSOpts: tlsOpts,
})
Expand Down Expand Up @@ -221,6 +268,16 @@ func main() {
"metadata.name": "cluster",
}),
}
if !appconfig.DisableClusterTLSProfile {
// Restrict the APIServer cache to the single "cluster" object.
// The ClusterRole only grants access to this named resource, so a
// full cluster-wide list would be forbidden.
cacheOpts.ByObject[&configv1.APIServer{}] = cache.ByObject{
Field: fields.SelectorFromSet(fields.Set{
"metadata.name": "cluster",
}),
}
}
}

metricsOpts := metricsserver.Options{
Expand Down Expand Up @@ -264,6 +321,30 @@ func main() {
os.Exit(1)
}

// Watch the cluster TLS security profile for changes. When the profile or adherence
// policy changes, cancel() triggers a graceful shutdown so the operator restarts
// and picks up the new configuration.
if appconfig.Openshift && !appconfig.DisableClusterTLSProfile {
if err := (&ostls.SecurityProfileWatcher{
Client: mgr.GetClient(),
InitialTLSProfileSpec: tlsProfileSpec,
InitialTLSAdherencePolicy: tlsAdherence,
OnProfileChange: func(_ context.Context, old, new configv1.TLSProfileSpec) {
setupLog.Info("cluster TLS profile changed; restarting to apply new configuration",
"old", old, "new", new)
cancel()
},
OnAdherencePolicyChange: func(_ context.Context, old, new configv1.TLSAdherencePolicy) {
setupLog.Info("cluster TLS adherence policy changed; restarting to apply new configuration",
"old", old, "new", new)
cancel()
},
}).SetupWithManager(mgr); err != nil {
setupLog.Error(err, "unable to set up TLS security profile watcher")
os.Exit(1)
}
}

setupController("securesign", securesign.NewReconciler, mgr)
setupController("fulcio", fulcio.NewReconciler, mgr)
setupController("trillian", trillian.NewReconciler, mgr)
Expand Down Expand Up @@ -314,12 +395,56 @@ func main() {
}

setupLog.Info("starting manager")
if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
if err := mgr.Start(ctx); err != nil {
setupLog.Error(err, "problem running manager")
os.Exit(1)
}
}

// resolveClusterTLSProfile determines the cluster TLS security profile and adherence policy to
// apply to the operator's webhook and metrics servers.
//
// On vanilla Kubernetes or when disabled is set, it returns the Intermediate profile defaults
// (TLS 1.2 minimum) without contacting the API server. On OpenShift it fetches the cluster-wide
// profile from the config.openshift.io APIServer, falling back to Intermediate defaults when the
// resource or API is unavailable. A non-nil error is returned only for unexpected failures that
// should abort startup.
func resolveClusterTLSProfile(ctx context.Context, cli client.Client, openshift, disabled bool, log logr.Logger) (configv1.TLSProfileSpec, configv1.TLSAdherencePolicy, error) {
intermediateSpec := *configv1.TLSProfiles[configv1.TLSProfileIntermediateType]

if !openshift || disabled {
if !openshift {
log.Info("not running on OpenShift; using Intermediate TLS defaults")
} else {
log.Info("cluster TLS profile resolution disabled via flag; using Intermediate defaults")
}
return intermediateSpec, configv1.TLSAdherencePolicyNoOpinion, nil
}

tlsProfileSpec, err := ostls.FetchAPIServerTLSProfile(ctx, cli)
if err != nil {
if apiErrors.IsNotFound(err) || apimeta.IsNoMatchError(err) {
log.Info("config.openshift.io APIServer not available; using Intermediate TLS defaults")
tlsProfileSpec = intermediateSpec
} else {
return configv1.TLSProfileSpec{}, "", fmt.Errorf("unable to fetch cluster TLS security profile: %w", err)
}
}

tlsAdherence, err := ostls.FetchAPIServerTLSAdherencePolicy(ctx, cli)
if err != nil {
if apiErrors.IsNotFound(err) || apimeta.IsNoMatchError(err) {
log.Info("TLSAdherencePolicy API not available; defaulting to NoOpinion")
} else {
log.Error(err, "unable to fetch cluster TLS adherence policy; defaulting to NoOpinion")
}
tlsAdherence = configv1.TLSAdherencePolicyNoOpinion
}

log.Info("cluster TLS security profile resolved")
return tlsProfileSpec, tlsAdherence, nil
}

func setupController(name string, constructor controller.Constructor, manager ctrl.Manager) {
if err := constructor(
manager.GetClient(),
Expand Down
195 changes: 195 additions & 0 deletions cmd/main_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,195 @@
/*
Copyright 2023.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
"context"
"errors"
"testing"

"github.com/go-logr/logr"
"github.com/onsi/gomega"
configv1 "github.com/openshift/api/config/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
)

func tlsTestScheme(t *testing.T) *runtime.Scheme {
t.Helper()
s := runtime.NewScheme()
if err := configv1.AddToScheme(s); err != nil {
t.Fatalf("failed to add configv1 to scheme: %v", err)
}
return s
}

func apiServerWith(profile *configv1.TLSSecurityProfile, adherence configv1.TLSAdherencePolicy) *configv1.APIServer {
return &configv1.APIServer{
ObjectMeta: metav1.ObjectMeta{Name: "cluster"},
Spec: configv1.APIServerSpec{
TLSSecurityProfile: profile,
TLSAdherence: adherence,
},
}
}

// resolveClusterTLSProfile: non-OpenShift and disabled paths return Intermediate defaults
// without touching the client.
func TestResolveClusterTLSProfile_Defaults(t *testing.T) {
t.Parallel()
g := gomega.NewWithT(t)
intermediate := *configv1.TLSProfiles[configv1.TLSProfileIntermediateType]

tests := []struct {
name string
openshift bool
disabled bool
}{
{"vanilla kubernetes", false, false},
{"openshift with resolution disabled", true, true},
{"non-openshift with disable flag set", false, true},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// A nil client must never be dereferenced on these paths.
profile, adherence, err := resolveClusterTLSProfile(
context.Background(), nil, tt.openshift, tt.disabled, logr.Discard())

g.Expect(err).ToNot(gomega.HaveOccurred())
g.Expect(profile.MinTLSVersion).To(gomega.Equal(intermediate.MinTLSVersion))
g.Expect(adherence).To(gomega.Equal(configv1.TLSAdherencePolicyNoOpinion))
})
}
}

// resolveClusterTLSProfile: on OpenShift the configured cluster profile and adherence policy
// are returned.
func TestResolveClusterTLSProfile_FetchesConfiguredProfile(t *testing.T) {
t.Parallel()
g := gomega.NewWithT(t)
modern := *configv1.TLSProfiles[configv1.TLSProfileModernType]

cli := fake.NewClientBuilder().
WithScheme(tlsTestScheme(t)).
WithObjects(apiServerWith(
&configv1.TLSSecurityProfile{Type: configv1.TLSProfileModernType},
configv1.TLSAdherencePolicyStrictAllComponents,
)).
Build()

profile, adherence, err := resolveClusterTLSProfile(
context.Background(), cli, true, false, logr.Discard())

g.Expect(err).ToNot(gomega.HaveOccurred())
g.Expect(profile.MinTLSVersion).To(gomega.Equal(modern.MinTLSVersion))
g.Expect(adherence).To(gomega.Equal(configv1.TLSAdherencePolicyStrictAllComponents))
}

// resolveClusterTLSProfile: when the APIServer resource is absent the resolver falls back to
// Intermediate defaults and NoOpinion adherence without erroring.
func TestResolveClusterTLSProfile_APIServerNotFound(t *testing.T) {
t.Parallel()
g := gomega.NewWithT(t)
intermediate := *configv1.TLSProfiles[configv1.TLSProfileIntermediateType]

cli := fake.NewClientBuilder().WithScheme(tlsTestScheme(t)).Build()

profile, adherence, err := resolveClusterTLSProfile(
context.Background(), cli, true, false, logr.Discard())

g.Expect(err).ToNot(gomega.HaveOccurred())
g.Expect(profile.MinTLSVersion).To(gomega.Equal(intermediate.MinTLSVersion))
g.Expect(adherence).To(gomega.Equal(configv1.TLSAdherencePolicyNoOpinion))
}

// resolveClusterTLSProfile: an unexpected error fetching the profile aborts startup.
func TestResolveClusterTLSProfile_ProfileFetchError(t *testing.T) {
t.Parallel()
g := gomega.NewWithT(t)

boom := errors.New("connection refused")
cli := fake.NewClientBuilder().
WithScheme(tlsTestScheme(t)).
WithInterceptorFuncs(interceptor.Funcs{
Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error {
return boom
},
}).
Build()

_, _, err := resolveClusterTLSProfile(
context.Background(), cli, true, false, logr.Discard())

g.Expect(err).To(gomega.HaveOccurred())
g.Expect(err.Error()).To(gomega.ContainSubstring("unable to fetch cluster TLS security profile"))
}

// resolveClusterTLSProfile: an unexpected error fetching only the adherence policy is tolerated;
// the resolver keeps the profile and defaults adherence to NoOpinion.
func TestResolveClusterTLSProfile_AdherenceFetchError(t *testing.T) {
t.Parallel()
g := gomega.NewWithT(t)
modern := *configv1.TLSProfiles[configv1.TLSProfileModernType]

apiServer := apiServerWith(
&configv1.TLSSecurityProfile{Type: configv1.TLSProfileModernType},
configv1.TLSAdherencePolicyStrictAllComponents,
)

// Succeed on the first Get (profile) and fail on the second (adherence).
// NOTE: this assumes controller-runtime-common's FetchAPIServerTLSProfile and
// FetchAPIServerTLSAdherencePolicy each issue exactly one Get, in that order. That holds for the
// pinned version; if the dependency changes its internal call count/order, revisit this counter.
var calls int
cli := fake.NewClientBuilder().
WithScheme(tlsTestScheme(t)).
WithObjects(apiServer).
WithInterceptorFuncs(interceptor.Funcs{
Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
calls++
if calls >= 2 {
return errors.New("connection refused")
}
return c.Get(ctx, key, obj, opts...)
},
}).
Build()

profile, adherence, err := resolveClusterTLSProfile(
context.Background(), cli, true, false, logr.Discard())

g.Expect(err).ToNot(gomega.HaveOccurred())
g.Expect(profile.MinTLSVersion).To(gomega.Equal(modern.MinTLSVersion))
g.Expect(adherence).To(gomega.Equal(configv1.TLSAdherencePolicyNoOpinion))
}

// Guard: the NotFound classification the resolver relies on must hold for the fake client so the
// fallback path stays correct if dependencies change.
func TestResolveClusterTLSProfile_NotFoundClassification(t *testing.T) {
t.Parallel()
g := gomega.NewWithT(t)

cli := fake.NewClientBuilder().WithScheme(tlsTestScheme(t)).Build()
err := cli.Get(context.Background(), client.ObjectKey{Name: "cluster"}, &configv1.APIServer{})

g.Expect(apierrors.IsNotFound(err)).To(gomega.BeTrue())
}
Loading
Loading