Skip to content

Commit

Permalink
move the handlers, servers, etc needed for a kubeapiserver to the kub…
Browse files Browse the repository at this point in the history
…eapiserver
  • Loading branch information
deads2k committed Aug 14, 2018
1 parent e2fd943 commit 364a599
Show file tree
Hide file tree
Showing 17 changed files with 488 additions and 233 deletions.
1 change: 1 addition & 0 deletions hack/import-restrictions.json
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@
"ignoredSubTrees": [
"github.com/openshift/origin/pkg/oauthserver",

"github.com/openshift/origin/pkg/cmd/openshift-kube-apiserver/openshiftkubeapiserver",
"github.com/openshift/origin/pkg/cmd/server/origin",
"github.com/openshift/origin/pkg/cmd/server/apis/config/validation",
"github.com/openshift/origin/pkg/oc/cli/admin",
Expand Down
2 changes: 1 addition & 1 deletion pkg/client/genericinformers/interface.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ type GenericResourceInformer interface {
Start(stopCh <-chan struct{})
}

// genericInternalResourceInformerFunc will return an internal informer for any resource matching
// GenericInternalResourceInformerFunc will return an internal informer for any resource matching
// its group resource, instead of the external version. Only valid for use where the type is accessed
// via generic interfaces, such as the garbage collector with ObjectMeta.
type GenericInternalResourceInformerFunc func(resource schema.GroupVersionResource) (informers.GenericInformer, error)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package origin
package openshiftkubeapiserver

import (
"bytes"
Expand Down Expand Up @@ -51,8 +51,8 @@ func (f *userAgentFilter) matches(verb, userAgent string) bool {

// versionSkewFilter adds a filter that may deny requests from skewed
// oc clients, since we know that those clients will strip unknown fields which can lead to unexpected outcomes
func (c *MasterConfig) versionSkewFilter(handler http.Handler) http.Handler {
filterConfig := c.Options.PolicyConfig.UserAgentMatchingConfig
func versionSkewFilter(handler http.Handler, kubeAPIServerConfig *configapi.MasterConfig) http.Handler {
filterConfig := kubeAPIServerConfig.PolicyConfig.UserAgentMatchingConfig
if len(filterConfig.RequiredClients) == 0 && len(filterConfig.DeniedClients) == 0 {
return handler
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package origin
package openshiftkubeapiserver

import (
"io/ioutil"
Expand Down Expand Up @@ -117,12 +117,12 @@ func TestVersionSkewFilterDenyOld(t *testing.T) {
verbs := []string{"PATCH", "POST"}
doNothingHandler := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
})
config := MasterConfig{}
config.Options.PolicyConfig.UserAgentMatchingConfig.DeniedClients = []configapi.UserAgentDenyRule{
config := &configapi.MasterConfig{}
config.PolicyConfig.UserAgentMatchingConfig.DeniedClients = []configapi.UserAgentDenyRule{
{UserAgentMatchRule: configapi.UserAgentMatchRule{Regex: `\w+/v1\.1\.10 \(.+/.+\) kubernetes/\w{7}`, HTTPVerbs: verbs}, RejectionMessage: "rejected for reasons!"},
{UserAgentMatchRule: configapi.UserAgentMatchRule{Regex: `\w+/v(?:(?:1\.1\.1)|(?:1\.0\.1)) \(.+/.+\) openshift/\w{7}`, HTTPVerbs: verbs}, RejectionMessage: "rejected for reasons!"},
}
handler := config.versionSkewFilter(doNothingHandler)
handler := versionSkewFilter(doNothingHandler, config)
server := httptest.NewServer(testHandlerChain(handler))
defer server.Close()

Expand Down Expand Up @@ -164,13 +164,13 @@ func TestVersionSkewFilterDenySkewed(t *testing.T) {
verbs := []string{"PUT", "DELETE"}
doNothingHandler := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
})
config := MasterConfig{}
config.Options.PolicyConfig.UserAgentMatchingConfig.RequiredClients = []configapi.UserAgentMatchRule{
config := &configapi.MasterConfig{}
config.PolicyConfig.UserAgentMatchingConfig.RequiredClients = []configapi.UserAgentMatchRule{
{Regex: `\w+/` + kubeServerVersion + ` \(.+/.+\) kubernetes/\w{7}`, HTTPVerbs: verbs},
{Regex: `\w+/` + openshiftServerVersion + ` \(.+/.+\) openshift/\w{7}`, HTTPVerbs: verbs},
}
config.Options.PolicyConfig.UserAgentMatchingConfig.DefaultRejectionMessage = "rejected for reasons!"
handler := config.versionSkewFilter(doNothingHandler)
config.PolicyConfig.UserAgentMatchingConfig.DefaultRejectionMessage = "rejected for reasons!"
handler := versionSkewFilter(doNothingHandler, config)
server := httptest.NewServer(testHandlerChain(handler))
defer server.Close()

Expand Down Expand Up @@ -215,14 +215,14 @@ func TestVersionSkewFilterSkippedOnNonAPIRequest(t *testing.T) {
verbs := []string{"PUT", "DELETE"}
doNothingHandler := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
})
config := MasterConfig{}
config.Options.PolicyConfig.UserAgentMatchingConfig.RequiredClients = []configapi.UserAgentMatchRule{
config := &configapi.MasterConfig{}
config.PolicyConfig.UserAgentMatchingConfig.RequiredClients = []configapi.UserAgentMatchRule{
{Regex: `\w+/` + kubeServerVersion + ` \(.+/.+\) kubernetes/\w{7}`, HTTPVerbs: verbs},
{Regex: `\w+/` + openshiftServerVersion + ` \(.+/.+\) openshift/\w{7}`, HTTPVerbs: verbs},
}
config.Options.PolicyConfig.UserAgentMatchingConfig.DefaultRejectionMessage = "rejected for reasons!"
config.PolicyConfig.UserAgentMatchingConfig.DefaultRejectionMessage = "rejected for reasons!"

handler := config.versionSkewFilter(doNothingHandler)
handler := versionSkewFilter(doNothingHandler, config)
server := httptest.NewServer(testHandlerChain(handler))
defer server.Close()

Expand Down
Original file line number Diff line number Diff line change
@@ -1,13 +1,31 @@
package origin
package openshiftkubeapiserver

import (
"net/http"

genericmux "k8s.io/apiserver/pkg/server/mux"

configapi "github.com/openshift/origin/pkg/cmd/server/apis/config"
oauthutil "github.com/openshift/origin/pkg/oauth/util"
genericapiserver "k8s.io/apiserver/pkg/server"
genericmux "k8s.io/apiserver/pkg/server/mux"
"k8s.io/client-go/informers"
)

func NewOpenshiftNonAPIConfig(generiConfig *genericapiserver.Config, kubeInformers informers.SharedInformerFactory, kubeAPIServerConfig *configapi.MasterConfig) (*OpenshiftNonAPIConfig, error) {
var err error
ret := &OpenshiftNonAPIConfig{
GenericConfig: &genericapiserver.RecommendedConfig{
Config: *generiConfig,
SharedInformerFactory: kubeInformers,
},
}
ret.ExtraConfig.OAuthMetadata, _, err = oauthutil.PrepOauthMetadata(*kubeAPIServerConfig)
if err != nil {
return nil, err
}

return ret, nil
}

type NonAPIExtraConfig struct {
OAuthMetadata []byte
}
Expand Down
235 changes: 235 additions & 0 deletions pkg/cmd/openshift-kube-apiserver/openshiftkubeapiserver/patch.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,235 @@
package openshiftkubeapiserver

import (
configapi "github.com/openshift/origin/pkg/cmd/server/apis/config"
usercache "github.com/openshift/origin/pkg/user/cache"
"k8s.io/apiserver/pkg/admission"
genericapiserver "k8s.io/apiserver/pkg/server"
cacheddiscovery "k8s.io/client-go/discovery/cached"
clientgoinformers "k8s.io/client-go/informers"
kexternalinformers "k8s.io/client-go/informers"
"k8s.io/client-go/rest"
"k8s.io/client-go/restmapper"
internalinformers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion"
kinternalinformers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion"
"k8s.io/kubernetes/pkg/master"

oauthclient "github.com/openshift/client-go/oauth/clientset/versioned"
oauthinformer "github.com/openshift/client-go/oauth/informers/externalversions"
userclient "github.com/openshift/client-go/user/clientset/versioned"
userinformer "github.com/openshift/client-go/user/informers/externalversions"
originadmission "github.com/openshift/origin/pkg/cmd/server/origin/admission"
imageinformer "github.com/openshift/origin/pkg/image/generated/informers/internalversion"
imageclient "github.com/openshift/origin/pkg/image/generated/internalclientset"
quotainformer "github.com/openshift/origin/pkg/quota/generated/informers/internalversion"
quotaclient "github.com/openshift/origin/pkg/quota/generated/internalclientset"
securityinformer "github.com/openshift/origin/pkg/security/generated/informers/internalversion"
securityclient "github.com/openshift/origin/pkg/security/generated/internalclientset"

"fmt"
"time"

"github.com/openshift/origin/pkg/cmd/openshift-apiserver/openshiftapiserver"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/cache"
)

type KubeAPIServerServerPatchContext struct {
initialized bool

RESTMapper *restmapper.DeferredDiscoveryRESTMapper
postStartHooks map[string]genericapiserver.PostStartHookFunc
informerStartFuncs []func(stopCh <-chan struct{})
}

type KubeAPIServerConfigFunc func(config *master.Config, internalInformers internalinformers.SharedInformerFactory, kubeInformers clientgoinformers.SharedInformerFactory, pluginInitializers *[]admission.PluginInitializer, stopCh <-chan struct{}) (genericapiserver.DelegationTarget, error)

func NewOpenShiftKubeAPIServerConfigPatch(delegateAPIServer genericapiserver.DelegationTarget, kubeAPIServerConfig *configapi.MasterConfig) (KubeAPIServerConfigFunc, *KubeAPIServerServerPatchContext) {
patchContext := &KubeAPIServerServerPatchContext{
postStartHooks: map[string]genericapiserver.PostStartHookFunc{},
}
return func(config *master.Config, internalInformers internalinformers.SharedInformerFactory, kubeInformers clientgoinformers.SharedInformerFactory, pluginInitializers *[]admission.PluginInitializer, stopCh <-chan struct{}) (genericapiserver.DelegationTarget, error) {
kubeAPIServerInformers, err := NewInformers(internalInformers, kubeInformers, config.GenericConfig.LoopbackClientConfig)
if err != nil {
return nil, err
}

// AUTHENTICATOR
authenticator, postStartHooks, err := NewAuthenticator(
*kubeAPIServerConfig,
config.GenericConfig.LoopbackClientConfig,
kubeAPIServerInformers.OpenshiftOAuthInformers.Oauth().V1().OAuthClients().Lister(),
kubeAPIServerInformers.OpenshiftUserInformers.User().V1().Groups())
if err != nil {
return nil, err
}
config.GenericConfig.Authentication.Authenticator = authenticator
for key, fn := range postStartHooks {
patchContext.postStartHooks[key] = fn
}
// END AUTHENTICATOR

// AUTHORIZER
authorizer := NewAuthorizer(internalInformers, kubeInformers)
config.GenericConfig.Authorization.Authorizer = authorizer
// END AUTHORIZER

// ADMISSION
projectCache, err := openshiftapiserver.NewProjectCache(kubeAPIServerInformers.InternalKubernetesInformers.Core().InternalVersion().Namespaces(), config.GenericConfig.LoopbackClientConfig, kubeAPIServerConfig.ProjectConfig.DefaultNodeSelector)
if err != nil {
return nil, err
}
clusterQuotaMappingController := openshiftapiserver.NewClusterQuotaMappingController(kubeAPIServerInformers.InternalKubernetesInformers.Core().InternalVersion().Namespaces(), kubeAPIServerInformers.InternalOpenshiftQuotaInformers.Quota().InternalVersion().ClusterResourceQuotas())
kubeClient, err := kubernetes.NewForConfig(config.GenericConfig.LoopbackClientConfig)
if err != nil {
return nil, err
}
discoveryClient := cacheddiscovery.NewMemCacheClient(kubeClient.Discovery())
restMapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryClient)
admissionInitializer, err := originadmission.NewPluginInitializer(*kubeAPIServerConfig, config.GenericConfig.LoopbackClientConfig, kubeAPIServerInformers, config.GenericConfig.Authorization.Authorizer, projectCache, restMapper, clusterQuotaMappingController)
if err != nil {
return nil, err
}
*pluginInitializers = []admission.PluginInitializer{admissionInitializer}
// END ADMISSION

// HANDLER CHAIN (with oauth server and web console)
config.GenericConfig.BuildHandlerChainFunc, postStartHooks, err = BuildHandlerChain(config.GenericConfig, kubeInformers, kubeAPIServerConfig, stopCh)
if err != nil {
return nil, err
}
for key, fn := range postStartHooks {
patchContext.postStartHooks[key] = fn
}
// END HANDLER CHAIN

// CONSTRUCT DELEGATE
nonAPIServerConfig, err := NewOpenshiftNonAPIConfig(config.GenericConfig, kubeInformers, kubeAPIServerConfig)
if err != nil {
return nil, err
}
openshiftNonAPIServer, err := nonAPIServerConfig.Complete().New(delegateAPIServer)
if err != nil {
return nil, err
}
// END CONSTRUCT DELEGATE

patchContext.informerStartFuncs = append(patchContext.informerStartFuncs, kubeAPIServerInformers.Start)
patchContext.RESTMapper = restMapper
patchContext.initialized = true

return openshiftNonAPIServer.GenericAPIServer, nil
}, patchContext
}

func (c *KubeAPIServerServerPatchContext) PatchServer(server *master.Master) error {
if !c.initialized {
return fmt.Errorf("not initialized with config")
}

for name, fn := range c.postStartHooks {
server.GenericAPIServer.AddPostStartHookOrDie(name, fn)
}
server.GenericAPIServer.AddPostStartHookOrDie("openshift.io-startkubeinformers", func(context genericapiserver.PostStartHookContext) error {
for _, fn := range c.informerStartFuncs {
fn(context.StopCh)
}
return nil
})
server.GenericAPIServer.AddPostStartHookOrDie("openshift.io-restmapperupdater", func(context genericapiserver.PostStartHookContext) error {
c.RESTMapper.Reset()
go func() {
wait.Until(func() {
c.RESTMapper.Reset()
}, 10*time.Second, context.StopCh)
}()
return nil
})

return nil
}

// NewInformers is only exposed for the build's integration testing until it can be fixed more appropriately.
func NewInformers(internalInformers internalinformers.SharedInformerFactory, versionedInformers clientgoinformers.SharedInformerFactory, loopbackClientConfig *rest.Config) (*KubeAPIServerInformers, error) {
imageClient, err := imageclient.NewForConfig(loopbackClientConfig)
if err != nil {
return nil, err
}
oauthClient, err := oauthclient.NewForConfig(loopbackClientConfig)
if err != nil {
return nil, err
}
quotaClient, err := quotaclient.NewForConfig(loopbackClientConfig)
if err != nil {
return nil, err
}
securityClient, err := securityclient.NewForConfig(loopbackClientConfig)
if err != nil {
return nil, err
}
userClient, err := userclient.NewForConfig(loopbackClientConfig)
if err != nil {
return nil, err
}

// TODO find a single place to create and start informers. During the 1.7 rebase this will come more naturally in a config object,
// before then we should try to eliminate our direct to storage access. It's making us do weird things.
const defaultInformerResyncPeriod = 10 * time.Minute

ret := &KubeAPIServerInformers{
InternalKubernetesInformers: internalInformers,
KubernetesInformers: versionedInformers,
InternalOpenshiftImageInformers: imageinformer.NewSharedInformerFactory(imageClient, defaultInformerResyncPeriod),
OpenshiftOAuthInformers: oauthinformer.NewSharedInformerFactory(oauthClient, defaultInformerResyncPeriod),
InternalOpenshiftQuotaInformers: quotainformer.NewSharedInformerFactory(quotaClient, defaultInformerResyncPeriod),
InternalOpenshiftSecurityInformers: securityinformer.NewSharedInformerFactory(securityClient, defaultInformerResyncPeriod),
OpenshiftUserInformers: userinformer.NewSharedInformerFactory(userClient, defaultInformerResyncPeriod),
}
if err := ret.OpenshiftUserInformers.User().V1().Groups().Informer().AddIndexers(cache.Indexers{
usercache.ByUserIndexName: usercache.ByUserIndexKeys,
}); err != nil {
return nil, err
}

return ret, nil
}

type KubeAPIServerInformers struct {
InternalKubernetesInformers kinternalinformers.SharedInformerFactory
KubernetesInformers kexternalinformers.SharedInformerFactory
OpenshiftOAuthInformers oauthinformer.SharedInformerFactory
InternalOpenshiftImageInformers imageinformer.SharedInformerFactory
InternalOpenshiftQuotaInformers quotainformer.SharedInformerFactory
InternalOpenshiftSecurityInformers securityinformer.SharedInformerFactory
OpenshiftUserInformers userinformer.SharedInformerFactory
}

func (i *KubeAPIServerInformers) GetInternalKubernetesInformers() kinternalinformers.SharedInformerFactory {
return i.InternalKubernetesInformers
}
func (i *KubeAPIServerInformers) GetKubernetesInformers() kexternalinformers.SharedInformerFactory {
return i.KubernetesInformers
}
func (i *KubeAPIServerInformers) GetInternalOpenshiftImageInformers() imageinformer.SharedInformerFactory {
return i.InternalOpenshiftImageInformers
}
func (i *KubeAPIServerInformers) GetInternalOpenshiftQuotaInformers() quotainformer.SharedInformerFactory {
return i.InternalOpenshiftQuotaInformers
}
func (i *KubeAPIServerInformers) GetInternalOpenshiftSecurityInformers() securityinformer.SharedInformerFactory {
return i.InternalOpenshiftSecurityInformers
}
func (i *KubeAPIServerInformers) GetOpenshiftUserInformers() userinformer.SharedInformerFactory {
return i.OpenshiftUserInformers
}

func (i *KubeAPIServerInformers) Start(stopCh <-chan struct{}) {
i.InternalKubernetesInformers.Start(stopCh)
i.KubernetesInformers.Start(stopCh)
i.OpenshiftOAuthInformers.Start(stopCh)
i.InternalOpenshiftImageInformers.Start(stopCh)
i.InternalOpenshiftQuotaInformers.Start(stopCh)
i.InternalOpenshiftSecurityInformers.Start(stopCh)
i.OpenshiftUserInformers.Start(stopCh)
}
Loading

0 comments on commit 364a599

Please sign in to comment.