Kubernetes controller源码分析小记（基于release-1.21）

在学习讲解执行kubectl run ...命令时发生了什么的这篇文章时，想到一个问题，既然controller是状态驱动的，只有当资源发生了改变才会触发controller的处理，那如果在处理资源的过程中controller挂掉，因为apiserver是无状态的，不存在事件交付处理过期等机制，那这个资源就一直处于未处理的状态吗。例如新建了一个deployment，再deployment controller准备新增replicaset的时候挂掉，那这个deployment会一直没有replicaset吗。首先答案当然是否定的，controller会以某种机制，确保已知的资源是已经被正确处理了，那具体代码是怎么实现的，本文主要基于这篇博客进行一些源码阅读和分析，这篇博客已经讲得非常好了，但是基于的codebase比较老旧，本文从release-1.21分支出发去扒一下具体的实现过程，对大部分函数都进行了一定程度的简化。

首先看到deployment controller的核心代码，主要就是在几个informer中注册handler，这些handler主要是将相关的deployment enqueue到队列（deployment controller自己的队列，和informer的队列不是同一个）中，交给处理协程。

// pkg/controller/deployment/deployment_controller.go
func NewDeploymentController(
	dInformer appsinformers.DeploymentInformer,
	rsInformer appsinformers.ReplicaSetInformer,
	podInformer coreinformers.PodInformer,
	client clientset.Interface,
) (*DeploymentController, error) {
	dc := &DeploymentController{}
	dInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc:    dc.addDeployment,
		UpdateFunc: dc.updateDeployment,
		DeleteFunc: dc.deleteDeployment,
	})
	rsInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc:    dc.addReplicaSet,
		UpdateFunc: dc.updateReplicaSet,
		DeleteFunc: dc.deleteReplicaSet,
	})
	podInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
		DeleteFunc: dc.deletePod,
	})
}

既然deployment controller中只是根据到来的event进行处理，那么相关机制应该隐藏在了informer中。NewDeploymentController在app中被调用：

// cmd/kube-controller-manager/app/apps.go
func startDeploymentController(ctx ControllerContext) (http.Handler, bool, error) {
	dc, err := deployment.NewDeploymentController(
		ctx.InformerFactory.Apps().V1().Deployments(),
		ctx.InformerFactory.Apps().V1().ReplicaSets(),
		ctx.InformerFactory.Core().V1().Pods(),
		ctx.ClientBuilder.ClientOrDie("deployment-controller"),
	)
	if err != nil {
		return nil, true, fmt.Errorf("error creating Deployment controller: %v", err)
	}
	go dc.Run(int(ctx.ComponentConfig.DeploymentController.ConcurrentDeploymentSyncs), ctx.Stop)
	return nil, true, nil
}

而ctx变量则是在CreateControllerContext中被创建，其中的InformerFactory由NewSharedInformerFactory函数创建，并且在这里计算了默认resync的间隔时长：

// cmd/kube-controller-manager/app/controllermanager.go
func CreateControllerContext() ControllerContext {
    sharedInformers := informers.NewSharedInformerFactory(resyncPeriod)
    return ControllerContext{InformerFactory: sharedInformers}
}

追踪InformerFactory.Apps().V1().Deployments()的调用过程：

// staging/src/k8s.io/client-go/informers/factory.go
type SharedInformerFactory interface {
    internalinterfaces.SharedInformerFactory
    Apps() apps.Interface
}
type sharedInformerFactory struct {
    defaultResync time.Duration
    infromers     map[reflect.Type]cache.SharedIndexInformer
}
func NewSharedInformerFactory(defaultResync time.Duration) SharedInformerFactory {
    return NewSharedInformerFactoryWithOptions()
}
func NewSharedInformerFactoryWithOptions(defaultResync time.Duration) SharedInformerFactory {
    return &sharedInformerFactory{
        defaultResync: defaultResync,
        informers:     make(map[reflect.Type]cache.SharedIndexInformer),
    }
}
func (f *sharedInformerFacotry) Apps() apps.Interface {
    return apps.New(f)
}

// staging/src/k8s.io/client-go/informers/apps/interface.go
type Interface interface {
    V1() v1.Interface
}
type group struct {
    factory internalinterfaces.SharedInformerFactory
}
func New(f internalinterfaces.SharedInformerFactory) Interface {
    return &group{factory: f}
}
func (g *group) V1() v1.Interface {
    return v1.New(g.factory)
}

// staging/src/k8s.io/client-go/informers/apps/v1/interface.go
type Interface interface {
    Deployments() DeploymentInformer
}
type version struct {
    factory internalinterfaces.SharedInformerFactory
}
func New(f internalinterfaces.SharedInformerFactory) Interface {
    return &version{factory: f}
}
func (v *version) Deployments() DeploymentInformer {
    return &deploymentInformer{factory: v.factory}
}

// staging/src/k8s.io/client-go/informers/apps/v1/deployment.go
type DeploymentInformer interface {
    Informer() cache.SharedIndexInformer
}
type deploymentInformer struct {
    factory internalinterfaces.SharedInformerFactory
}
func (f *deploymentInformer) Informer() cache.SharedIndexInformer {
    return f.factory.InformerFor(&appsv1.Deployment{}, f.DefaultInformer)
}
func (f *deploymentInformer) DefaultInformer(resyncPeriod time.Duration) cache.SharedIndexInformer {
    return NewFilteredDeploymentInformer(resyncPeriod)
}
func NewFilteredDeploymentInformer(resyncPeriod time.Duration) {
    return cache.NewSharedIndexInformer(
        &cache.ListWatch{
            ListFunc: func() {},   // 省略
            WatchFunc: func() {},  // 省略
        },
        &appsv1.Deployment{},
        resyncPeriod
    )
}

可以看到主要就是原始的sharedInformerFactory随API层级向下传递，直至deploymentInformer，在deployment controller中调用的dInformer.Informer()即是调用了deploymentInformer的Informer()函数，继续看sharedInformerFactory的InformerFor()实现：

// staging/src/k8s.io/client-go/informers/factory.go
func (f *sharedInformerFactory) InformerFor(obj runtime.Object, newFunc internalinterface.NewInformerFunc) cache.SharedIndexInformer {
    informer, ok := f.informers[informerType]
    if ok {
        return informer
    }
    informer = newFunc(resyncPeriod)
    f.informers[informerType] = informer
    return informer
}

就是根据informer类型进行map查找，如果没有使用newFunc进行创建罢了，所以叫shared。对于deployment来说，newFunc的实现就是NewFilteredDeploymentInformer()，创建了一个SharedIndexInformer接口实例，其实现为sharedIndexInformer，并且可以看到resync period这个参数一直贯穿其中：

func NewSharedIndexInformer(
	lw ListerWatcher,
	exampleObject runtime.Object,
	defaultEventHandlerResyncPeriod time.Duration,
	indexers Indexers,
) SharedIndexInformer {
	realClock := &clock.RealClock{}
	sharedIndexInformer := &sharedIndexInformer{
		processor:                       &sharedProcessor{clock: realClock},
		indexer:                         NewIndexer(DeletionHandlingMetaNamespaceKeyFunc, indexers),
		listerWatcher:                   lw,
		objectType:                      exampleObject,
		resyncCheckPeriod:               defaultEventHandlerResyncPeriod,
		defaultEventHandlerResyncPeriod: defaultEventHandlerResyncPeriod,
		cacheMutationDetector:           NewCacheMutationDetector(fmt.Sprintf("%T", exampleObject)),
		clock:                           realClock,
	}
	return sharedIndexInformer
}

再看sharedIndexInformer是怎么启动的，sharedInformerFactory在controllermanager.go中启动，其中NewControllerInitializers是一个返回了包含每个controller初始化函数的map，startDeployController()函数就被保存在其中一个entry中：

// cmd/kube-controller-manager/app/controllermanager.go
func Run() {
    run := func(initializersFunc ControllerInitializersFunc) {
        controllerContext := CreateControllerContext()
        controllerInitializers := initializersFunc()
        StartControllers(controllerContext, controllerInitializers)
        controllerContext.InformerFactory.Start(stopChan)
        select {}  // run forever
    }
    run(NewControllerInitialziers)
}

在sharedInformerFactory的Start()函数中遍历所有的informer（因为此时完成了初始化，所有informer都在InformerFor()的调用中被创建了，并调用informer.Run(stopCh)启动informer：

// staging/src/k8s.io/client-go/informers/factory.go
// Start initializes all requested informers.
func (f *sharedInformerFactory) Start(stopCh <-chan struct{}) {
	f.lock.Lock()
	defer f.lock.Unlock()

	for informerType, informer := range f.informers {
		if !f.startedInformers[informerType] {
			go informer.Run(stopCh)
			f.startedInformers[informerType] = true
		}
	}
}

进入informer的Run()函数，这部分则和这篇博客中基本一致，informer创建controller，controller创建reflector，用wait.Group启动协程调用reflector的Run()函数，继而调用ListAndWatch函数，订阅对应资源的事件，并以resync period为周期调用DeltaFIFO的Resync()函数，另一边则运行c.processLoop，c.processLoop是对c.cfg.Process，即s.HandleDeltas的封装，而HandleDeltas就是从fifo队列中不停取元素，并分发给informer中注册的各个event handler，这样一个事件就分发到了文章一开始的deploymentController中：

// k8s.io/client-go/tools/cache/shared_informer.go
func (s *sharedIndexInformer) Run() {
    fifo := NewDeltaFIFOWithOptions()
    cfg := &Config{
        Queue:            fifo,
        ListerWatcher:    s.listerWatcher,
        ObjectType:       s.objectType
        Process:          s.HandleDeltas,
        FullResyncPeriod: s.resyncCheckPeriod,
    }
    s.controller = New(cfg)
    s.controller.Run()
}
// k8s.io/client-go/tools/cache/controller.go
func (c *controller) Run() {
    r := NewReflector(
		c.config.ListerWatcher,
		c.config.ObjectType,
		c.config.Queue,
		c.config.FullResyncPeriod,
	)
    var wg wait.Group
    wg.StartWithChannel(stopCh, r.Run)
    wait.Until(c.processLoop, time.Second, stopCh)
}

DeltaFIFO的Resync()函数主要是获取已知的所有对象，并为不在queue中的对象创建一个"Sync"事件加入queue中，knownObjects则和indexer一套机制相关，这里就不展开了：

// staging/src/k8s.io/client-go/tools/cache/delta_fifo.go
// Resync adds, with a Sync type of Delta, every object listed by
// `f.knownObjects` whose key is not already queued for processing.
// If `f.knownObjects` is `nil` then Resync does nothing.
func (f *DeltaFIFO) Resync() error {
	keys := f.knownObjects.ListKeys()
	for _, k := range keys {
		if err := f.syncKeyLocked(k); err != nil {
			return err
		}
	}
	return nil
}
func (f *DeltaFIFO) syncKeyLocked(key string) error {
	obj, exists, err := f.knownObjects.GetByKey(key)
	// If we are doing Resync() and there is already an event queued for that object,
	// we ignore the Resync for it. This is to avoid the race, in which the resync
	// comes with the previous value of object (since queueing an event for the object
	// doesn't trigger changing the underlying store <knownObjects>.
	id, err := f.KeyOf(obj)
	if len(f.items[id]) > 0 {
		return nil
	}
	if err := f.queueActionLocked(Sync, obj); err != nil {
		return fmt.Errorf("couldn't queue object: %v", err)
	}
	return nil
}

因此对于一开始的问题，可以想象是这样的过程：一开始informer监听到"Added"事件，调用deployment controller的addDeployment回调进行处理，但进行到一半时进程挂掉，重启之后由于informer会周期性的resync，产生"Sync"事件，其中包含了之前未处理完成的deployment资源，调用updateDeployment回调进行处理，该函数发现资源的现状和预期状态不一致，于是创建相应的replicaset。

加一张数据流示意图（不是UML）：