目录

OpenShift webconsole proxy实现原理


前言

今天线上出问题了,访问console直接503。

根据console的地址查,发现console的访问地址对应的服务是apiserver,令我很吃惊…看apiserver对应的报错日志:

I1229 19:15:18.181339   20697 logs.go:49] http: proxy error: x509: certificate has expired or is not yet valid: current time 2021-12-29T19:15:18+08:00 is after 2021-10-30T13:47:51Z

这….根本定位不出来啊! 只能去看apiserver的源代码。 (吐槽一下OpenShift的魔改!)

预备知识

  • 大概看过apiserver的代码
  • 了解go-restful

handler chain

api server的handler类似于java框架的filter机制(或者是Django的middleware),但是又有点不同,说多了反而不容易理解,比如我们有一个handlerabc,比如http配置的aa有自己的逻辑,a的如果满足了某种逻辑会调用b,而b又可能由于某种逻辑会调用c,这就是handler chain,感兴趣的看去看BuildHandlerChain里面的逻辑。

约定

  • OpenShift版本:3.11

Kubernetes APIServer 启动流程

前面的解析不赘述了,直接跳到启动流程代码pkg/cmd/server/origin/master.go:179

func (c *MasterConfig) Run(stopCh <-chan struct{}) error {
	var err error
	var apiExtensionsInformers apiextensionsinformers.SharedInformerFactory
	var delegateAPIServer apiserver.DelegationTarget
	var extraPostStartHooks map[string]apiserver.PostStartHookFunc

	c.kubeAPIServerConfig.GenericConfig.BuildHandlerChainFunc, extraPostStartHooks, err = openshiftkubeapiserver.BuildHandlerChain(
		c.kubeAPIServerConfig.GenericConfig, c.ClientGoKubeInformers,
		c.Options.ControllerConfig.ServiceServingCert.Signer.CertFile, c.Options.OAuthConfig, c.Options.PolicyConfig.UserAgentMatchingConfig)
	if err != nil {
		return err
	}
	# ....
}

接着看BuildHandlerChain函数,跳到pkg/cmd/openshift-kube-apiserver/openshiftkubeapiserver/patch_handlerchain.go:28

func BuildHandlerChain(genericConfig *genericapiserver.Config, kubeInformers informers.SharedInformerFactory, legacyServiceServingCertSignerCABundle string, oauthConfig *configapi.OAuthConfig, userAgentMatchingConfig configapi.UserAgentMatchingConfig) (func(apiHandler http.Handler, kc *genericapiserver.Config) http.Handler, map[string]genericapiserver.PostStartHookFunc, error) {
	extraPostStartHooks := map[string]genericapiserver.PostStartHookFunc{}

	webconsoleProxyHandler, err := newWebConsoleProxy(kubeInformers, legacyServiceServingCertSignerCABundle)
	if err != nil {
		return nil, nil, err
	}
	oauthServerHandler, newPostStartHooks, err := NewOAuthServerHandler(genericConfig, oauthConfig)
	if err != nil {
		return nil, nil, err
	}
	for name, fn := range newPostStartHooks {
		extraPostStartHooks[name] = fn
	}

	return func(apiHandler http.Handler, genericConfig *genericapiserver.Config) http.Handler {
			// Machinery that let's use discover the Web Console Public URL
			accessor := newWebConsolePublicURLAccessor(genericConfig.LoopbackClientConfig)
			// the webconsole is proxied through the API server.  This starts a small controller that keeps track of where to proxy.
			// TODO stop proxying the webconsole. Should happen in a future release.
			extraPostStartHooks["openshift.io-webconsolepublicurl"] = func(context genericapiserver.PostStartHookContext) error {
				go accessor.Run(context.StopCh)
				return nil
			}

			// these are after the kube handler
			handler := versionSkewFilter(apiHandler, userAgentMatchingConfig)

			// this is the normal kube handler chain
			handler = genericapiserver.DefaultBuildHandlerChain(handler, genericConfig)

			// these handlers are all before the normal kube chain
			handler = translateLegacyScopeImpersonation(handler)
			handler = configprocessing.WithCacheControl(handler, "no-store") // protected endpoints should not be cached

			// redirects from / to /console if you're using a browser
			handler = withAssetServerRedirect(handler, accessor)

			// these handlers are actually separate API servers which have their own handler chains.
			// our server embeds these
			handler = withConsoleRedirection(handler, webconsoleProxyHandler, accessor)
			handler = withOAuthRedirection(oauthConfig, handler, oauthServerHandler)

			return handler
		},
		extraPostStartHooks,
		nil
}

newWebConsoleProxy的逻辑:

func newWebConsoleProxy(kubeInformers informers.SharedInformerFactory, legacyServiceServingCertSignerCABundle string) (http.Handler, error) {
	caBundle, err := ioutil.ReadFile(legacyServiceServingCertSignerCABundle)
	if err != nil {
		return nil, err
	}
	proxyHandler, err := newServiceProxyHandler("webconsole", "openshift-web-console", aggregatorapiserver.NewClusterIPServiceResolver(kubeInformers.Core().V1().Services().Lister()), caBundle, "OpenShift web console")
	if err != nil {
		return nil, err
	}
	return proxyHandler, nil
}

newServiceProxyHandler的逻辑:

// newServiceProxyHandler is a simple proxy that doesn't handle upgrades, passes headers directly through, and doesn't assert any identity.
func newServiceProxyHandler(serviceName string, serviceNamespace string, serviceResolver ServiceResolver, caBundle []byte, applicationDisplayName string) (*serviceProxyHandler, error) {
	restConfig := &restclient.Config{
		TLSClientConfig: restclient.TLSClientConfig{
			ServerName: serviceName + "." + serviceNamespace + ".svc",
			CAData:     caBundle,
		},
	}
	proxyRoundTripper, err := restclient.TransportFor(restConfig)
	if err != nil {
		return nil, err
	}

	return &serviceProxyHandler{
		serviceName:            serviceName,
		serviceNamespace:       serviceNamespace,
		serviceResolver:        serviceResolver,
		applicationDisplayName: applicationDisplayName,
		proxyRoundTripper:      proxyRoundTripper,
		restConfig:             restConfig,
	}, nil
}

serviceProxyHandler 结构体的说明:

proxyHandler provides a http.Handler which will proxy traffic to locations specified by items implementing Redirector.

serviceProxyHandler实现了ServeHTTP,在请求来了之后就会把请求透传到后端。 accessor会定期从configmap openshift-web-console/webconsole-config 读取console的URL,withConsoleRedirection会用到这个URL来判断请求是不是访问console的,如果是就把流量交给console,不是则调用下面的handler来处理请求。