Cilium CNI 工作原理解析

// part 1

func cmdAdd(args *skel.CmdArgs) (err error) {
	var (
		ipConfig *cniTypesV1.IPConfig
		routes   []*cniTypes.Route
		ipam     *models.IPAMResponse
		n        *types.NetConf
		c        *client.Client
		netNs    ns.NetNS
		conf     *models.DaemonConfigurationStatus
	)  // 一些函数内全局使用的变量

	n, err = types.LoadNetConf(args.StdinData)  // 读取 cni 网络配置：/etc/cni/net.d/05-cilium-cni.conf

	cniArgs := types.ArgsSpec{}
	cniTypes.LoadArgs(args.Args, &cniArgs)  // 加载 cni 参数

	c, err = client.NewDefaultClientWithTimeout(defaults.ClientConnectTimeout)  // 初始化一个客户端，以连接 cilium-daemon

	// ...

// pkg/client/client.go

func NewDefaultClient() (*Client, error) {
    return NewClient("")
}

func NewClient(host string) (*Client, error) {
	clientTrans, err := NewRuntime(host)
	return &Client{*clientapi.New(clientTrans, strfmt.Default)}, err
}

func NewRuntime(host string) (*runtime_client.Runtime, error) {
	if host == "" {
		host = DefaultSockPath()
	}
	tmp := strings.SplitN(host, "://", 2)  // 根据不同的协议，组织不同的地址，目前只支持 tcp 与 unix socket 两种协议
	switch tmp[0] {
	case "tcp":
		host = "http://" + tmp[1]
	case "unix":
		host = tmp[1]
	}

	transport := configureTransport(nil, tmp[0], host)
	httpClient := &http.Client{Transport: transport}
	clientTrans := runtime_client.NewWithClient(tmp[1], clientapi.DefaultBasePath, clientapi.DefaultSchemes, httpClient)
	return clientTrans, nil
}

func DefaultSockPath() string {
	e := os.Getenv(defaults.SockPathEnv)  // 从环境变量 CILIUM_SOCK 中获取 socket 地址
	if e == "" {
		e = defaults.SockPath  // 默认值为 /var/run/cilium/cilium.sock
	}
	return "unix://" + e
}

// part 2

	netNs, err = ns.GetNS(args.Netns)  // 获取网络命名空间
	defer netNs.Close()

	err = netns.RemoveIfFromNetNSIfExists(netNs, args.IfName)  // 移除已存在的网络接口
                           \
                            \
                             func RemoveIfFromNetNSIfExists(netNS ns.NetNS, ifName string) error {
                                return netNS.Do(func(_ ns.NetNS) error {
                                    l, err := netlink.LinkByName(ifName)
                                    return netlink.LinkDel(l)
                                })
                             }

	addLabels := models.Labels{}

	conf, err = getConfigFromCiliumAgent(c)  // 从 cilium-agent 获取 cilium-daemon 的配置

	// ...

// api/v1/client/daemon/daemon_client.go

func (a *Client) GetConfig(params *GetConfigParams, opts ...ClientOption) (*GetConfigOK, error) {
	if params == nil {
		params = NewGetConfigParams()
	}
	op := &runtime.ClientOperation{
		ID:                 "GetConfig",
		Method:             "GET",
		PathPattern:        "/config",  // ***
		ProducesMediaTypes: []string{"application/json"},
		ConsumesMediaTypes: []string{"application/json"},
		Schemes:            []string{"http"},
		Params:             params,
		Reader:             &GetConfigReader{formats: a.formats},
		Context:            params.Context,
		Client:             params.HTTPClient,
	}
	for _, opt := range opts {  // opts 默认情况下为空
		opt(op)
	}

	result, err := a.transport.Submit(op)  // 提交请求
	success, ok := result.(*GetConfigOK)
	if ok {
		return success, nil
	}
	// 若执行到此处说明提交未成功，直接 panic
	panic(msg)
}

// daemon/cmd/daemon_main.go
// @ func (d *Daemon) instantiateAPI :: L1887-L1888

// /config/
restAPI.DaemonGetConfigHandler = NewGetConfigHandler(d)  // 对应 GET 请求
restAPI.DaemonPatchConfigHandler = NewPatchConfigHandler(d)

type DaemonConfiguration struct {
	// 描述了 daemon 的可变配置
	Spec *DaemonConfigurationSpec `json:"spec,omitempty"`

	// 目前 daemon 配置的相关状态，包括各种地址信息、可变与不可变配置项、node monitor 等
	Status *DaemonConfigurationStatus `json:"status,omitempty"`
}

// part 3

	var releaseIPsFunc func(context.Context)
	if conf.IpamMode == ipamOption.IPAMDelegatedPlugin {  // 根据不同的 IPAM 模式来分配地址
		ipam, releaseIPsFunc, err = allocateIPsWithDelegatedPlugin(context.TODO(), conf, n, args.StdinData)
	} else {
		ipam, releaseIPsFunc, err = allocateIPsWithCiliumAgent(c, cniArgs)
	}

	// 若在地址分配时出现错误，则把分配的地址释放掉
	defer func() {
		if err != nil && releaseIPsFunc != nil {
			releaseIPsFunc(context.TODO())
		}
	}()

	// ipam.HostAddressing 记录了 Cilium 的 Internal IP
	connector.SufficientAddressing(ipam.HostAddressing)  // 检查该 IP 地址是否提供了足够的信息，即 ipv4 或 ipv6 地址至少需要一个

	// ...

// pkg/ipam/option/option.go

const (
	IPAMKubernetes = "kubernetes"  // 默认值

	IPAMCRD = "crd"
	IPAMENI = "eni"
	IPAMAzure = "azure"
	IPAMClusterPool = "cluster-pool"
	IPAMClusterPoolV2 = "cluster-pool-v2beta"
	IPAMAlibabaCloud = "alibabacloud"

	IPAMDelegatedPlugin = "delegated-plugin"  // 走 CNI plugin 委托
)

func allocateIPsWithDelegatedPlugin(ctx context.Context, conf *models.DaemonConfigurationStatus, netConf *types.NetConf, stdinData []byte,
) (*models.IPAMResponse, func(context.Context), error) {
	// netConf.IPAM.Type 描述了所委托 plugin 的名字，stdinData 描述了调用该委托所需的输入参数
	ipamRawResult, err := cniInvoke.DelegateAdd(ctx, netConf.IPAM.Type, stdinData, nil)  // 调用委托 plugin 的 CNI ADD
	if err != nil {
		// IP 地址分配失败，此时没有 IP 需要清理，故不返回 releaseFunc
		return nil, nil, fmt.Errorf("failed to invoke delegated plugin ADD for IPAM: %w", err)
	}

	// 预备好 CNI DEL 动作的闭包
	releaseFunc := func(ctx context.Context) {
		cniInvoke.DelegateDel(ctx, netConf.IPAM.Type, stdinData, nil)
	}

	ipamResult, err := cniTypesV1.NewResultFromResult(ipamRawResult)  // 上述委托调用返回的是原始结果，此处将其转换为 CNI spec v1.0 版本对应的结果
	if err != nil {
		return nil, releaseFunc, // msg
	}

	// 这里做格式统一，将委托调用的结果与通过 cilium-agent 分配 IP 的结果对齐
	ipam := &models.IPAMResponse{
		HostAddressing: conf.Addressing,
		Address:        &models.AddressPair{},
	}
	// 记录分配的每个 ipv4 或 ipv6 地址
	for _, ipConfig := range ipamResult.IPs {
		ipNet := ipConfig.Address
		if ipv4 := ipNet.IP.To4(); ipv4 != nil {
			ipam.Address.IPV4 = ipNet.String()
			ipam.IPV4 = &models.IPAMAddressResponse{IP: ipv4.String()}
		} else {
			ipam.Address.IPV6 = ipNet.String()
			ipam.IPV6 = &models.IPAMAddressResponse{IP: ipNet.IP.String()}
		}
	}

	return ipam, releaseFunc, nil
}

func allocateIPsWithCiliumAgent(client *client.Client, cniArgs types.ArgsSpec) (*models.IPAMResponse, func(context.Context), error) {
	podName := string(cniArgs.K8S_POD_NAMESPACE) + "/" + string(cniArgs.K8S_POD_NAME)  // namespaced name
	pool := ""
	ipam, err := client.IPAMAllocate("", podName, pool, true)  // 通过本地 cilium-agent 分配地址
	if err != nil {
		return nil, nil, err
	}
	if ipam.Address == nil {  // 无地址字段
		return nil, nil, err
	}

	releaseFunc := func(context.Context) {
		if ipam.Address != nil {
			releaseIP(client, ipam.Address.IPV4, pool)
			releaseIP(client, ipam.Address.IPV6, pool)
		}
	}

	return ipam, releaseFunc, nil
}

func releaseIP(client *client.Client, ip, pool string) {
	if ip != "" {
		err := client.IPAMReleaseIP(ip, pool)  // 通过本地 cilium-agent 释放地址
	}
}

// daemon/cmd/daemon_main.go
// @ func (d *Daemon) instantiateAPI :: L1955-1960

if option.Config.DatapathMode != datapathOption.DatapathModeLBOnly {
    // /ipam/{ip}/
    restAPI.IpamPostIpamHandler = NewPostIPAMHandler(d)  // 对应 IPAMAllocate
    restAPI.IpamPostIpamIPHandler = NewPostIPAMIPHandler(d)
    restAPI.IpamDeleteIpamIPHandler = NewDeleteIPAMIPHandler(d)  // 对应 IPAMReleaseIP
}

// part 4

	switch conf.DatapathMode {
	case datapathOption.DatapathModeVeth:  // veth 模式
		var (
			veth      *netlink.Veth
			peer      netlink.Link
			tmpIfName string
		)
		// 先在 host 侧创建 veth pair 接口
		veth, peer, tmpIfName, err = connector.SetupVeth(ep.ContainerID, int(conf.DeviceMTU), int(conf.GROMaxSize), int(conf.GSOMaxSize), ep)

		defer func() {
			if err != nil {
				err2 := netlink.LinkDel(veth)  // 配置失败时，删除该 veth pair 接口
			}
		}()

		err = netlink.LinkSetNsFd(peer, int(netNs.Fd()))  // 将 veth pair 的对端移动到 netns 中

		_, _, err = connector.SetupVethRemoteNs(netNs, tmpIfName, args.IfName)  // 最后在容器侧配置 veth 接口名
	}

	// ...

// part 5

	ep := &models.EndpointChangeRequest{  // 该结构体包含了 Cilium Endpoint 的所有可变元素
		ContainerID:           args.ContainerID,
		Addressing:            &models.AddressPair{},
		K8sPodName:            string(cniArgs.K8S_POD_NAME),
		K8sNamespace:          string(cniArgs.K8S_POD_NAMESPACE),
		// ...
	}

	state := CmdState{
		Endpoint: ep,
		Client:   c,
		HostAddr: ipam.HostAddressing,  // Cilium Interna IP
	}

	res := &cniTypesV1.Result{}  // 该函数最后的返回值

	if ipv4IsEnabled(ipam) {
		ep.Addressing.IPV4 = ipam.Address.IPV4
		ep.Addressing.IPV4ExpirationUUID = ipam.IPV4.ExpirationUUID

		ipConfig, routes, err = prepareIP(ep.Addressing.IPV4, &state, int(conf.RouteMTU))  // 解析 IP 格式，返回 IP 与网关地址；及对应路由

		res.IPs = append(res.IPs, ipConfig)
		res.Routes = append(res.Routes, routes...)
	}

	// if ipv6IsEnabled(ipam) { 略，其内容同上 }

	// ...

func prepareIP(ipAddr string, state *CmdState, mtu int) (*cniTypesV1.IPConfig, []*cniTypes.Route, error) {
	var (
		routes []route.Route
		gw     string
		ip     netip.Addr
	)

	// 根据 CIDR 格式解析 IP 地址
	ipPrefix, err := netip.ParsePrefix(ipAddr)
	if err != nil {
		ip, err = netip.ParseAddr(ipAddr)  // 非 CIDR 格式的 IP 地址
	} else {
		ip = ipPrefix.Addr()
	}

	if ip.Is6() {
		// 逻辑同下，略
	} else {
		state.IP4 = ip
		state.IP4routes, err = connector.IPv4Routes(state.HostAddr, mtu)  // 获取需要被安装在 Endpoint 网络命名空间内的路由
		routes = state.IP4routes
		ip = state.IP4
		gw = connector.IPv4Gateway(state.HostAddr)  // 返回 Endpoint 对应的网关地址，即 Cilium Interna IP 地址 => return addr.IPV4.IP
	}

	rt := make([]*cniTypes.Route, 0, len(routes))
	for _, r := range routes {
		rt = append(rt, newCNIRoute(r))  // 转换为 CNI 支持的 Route 类型
	}
	gwIP := net.ParseIP(gw)

	return &cniTypesV1.IPConfig{
		Address: *iputil.AddrToIPNet(ip),
		Gateway: gwIP,
	}, rt, nil
}

// pkg/datapath/connector/ipam.go

func IPv4Routes(addr *models.NodeAddressing, linkMTU int) ([]route.Route, error) {
	ip := net.ParseIP(addr.IPV4.IP)

	return []route.Route{
		{
			Prefix: net.IPNet{
				IP:   ip,
				Mask: defaults.ContainerIPv4Mask,  // 255.255.255.255
			},
		},
		{
			Prefix:  defaults.IPv4DefaultRoute,  // 0.0.0.0/32
			Nexthop: &ip,
			MTU:     linkMTU,
		},
	}, nil
}

// part 6

	var macAddrStr string
	err = netNs.Do(func(_ ns.NetNS) error {
		macAddrStr, err = configureIface(ipam, args.IfName, &state)  // 开启接口，并写入 ip 和路由，最后返回该接口的硬件 MAC 地址
		return err
	})

	res.Interfaces = append(res.Interfaces, &cniTypesV1.Interface{  // 记录网络接口
		Name:    args.IfName,
		Mac:     macAddrStr,
		Sandbox: args.Netns,
	})

	// 将接口的下标也添加至返回结果中
	for i := range res.Interfaces {
		res.IPs[i].Interface = cniTypesV1.Int(i)
	}

	// Endpoint 也必须要同步地进行重建
	ep.SyncBuildEndpoint = true
	err = c.EndpointCreate(ep)  // 创建 CiliumEndpoint

	return cniTypes.PrintResult(res, n.CNIVersion)
}

// pkg/client/endpoint.go

func (c *Client) EndpointCreate(ep *models.EndpointChangeRequest) error {
	id := pkgEndpointID.NewCiliumID(ep.ID)  // cilium-local:$id
	params := endpoint.NewPutEndpointIDParams().WithID(id).WithEndpoint(ep).WithTimeout(api.ClientTimeout)  // 构建请求参数
	_, err := c.Endpoint.PutEndpointID(params)
	return Hint(err)
}

// daemon/cmd/endpoint.go

func (h *putEndpointID) Handle(params PutEndpointIDParams) (resp middleware.Responder) {
	epTemplate := params.Endpoint

	ep, code, err := h.d.createEndpoint(params.HTTPRequest.Context(), h.d, epTemplate)  // ***

	return NewPutEndpointIDCreated()
}

// daemon/cmd/endpoint.go

func (d *Daemon) createEndpoint(ctx context.Context, owner regeneration.Owner, epTemplate *models.EndpointChangeRequest) (*endpoint.Endpoint, int, error) {

	// 解析请求参数并创建 Endpoint
	ep, err := endpoint.NewEndpointFromChangeModel(d.ctx, owner, d, d.ipcache, d.l7Proxy, d.identityAllocator, epTemplate)

	// 检查 Endpoint ID 或 Container 对应的 Endpoint 是否已经存在
	oldEp := d.endpointManager.LookupCiliumID(ep.ID)
	oldEp = d.endpointManager.LookupContainerID(ep.GetContainerID())

	// 检查 Endpoint IP 地址是否重复
	var checkIDs []string
	checkIDs = append(checkIDs, endpointid.NewID(endpointid.IPv4Prefix, ep.IPv4.String()))  // $prefix:$ip
	// ... aslo for ipv6
	for _, id := range checkIDs {
		oldEp, err := d.endpointManager.Lookup(id)
		if err != nil || oldEp != nil {
			return err
		}
	}

        addLabels := labels.NewLabelsFromModel(epTemplate.Labels)
	infoLabels := labels.NewLabelsFromModel([]string{})

	err = d.endpointManager.AddEndpoint(owner, ep, "Create endpoint from API PUT")  // ***

	regenTriggered := ep.UpdateLabels(ctx, addLabels, infoLabels, true)  // ***
	if !regenTriggered {
		regenMetadata := &regeneration.ExternalRegenerationMetadata{
			RegenerationLevel: regeneration.RegenerateWithDatapathRewrite,
			// ...
		}
		build, err := ep.SetRegenerateStateIfAlive(regenMetadata)

		if build {
			ep.Regenerate(regenMetadata)  // ***
		}
	}

	return ep, 0, nil
}

// pkg/k8s/watchers/endpointsynchronizer.go

func (epSync *EndpointSynchronizer) RunK8sCiliumEndpointSync(e *endpoint.Endpoint, conf endpoint.EndpointStatusConfiguration) {
	var (
		endpointID     = e.ID
		controllerName = endpoint.EndpointSyncControllerName(endpointID)
	)
	ciliumClient := epSync.Clientset.CiliumV2()

	var (
		localCEP *cilium_v2.CiliumEndpoint // 本地 CEP 对象的副本，可以复用
		needInit = true                    // needInit 表面可能需要去创建 CEP
		firstTry = true                    // 尝试从 k8s cache 中获取 CEP 对象
	)

	e.UpdateController(controllerName,
		controller.ControllerParams{
			RunInterval: 10 * time.Second,
			DoFunc: func(ctx context.Context) (err error) {
				podName := e.GetK8sPodName()
				namespace := e.GetK8sNamespace()

				if needInit {
					if firstTry {
						// 首先尝试从 API server cache 中获取 CEP 对象
						localCEP, err = ciliumClient.CiliumEndpoints(namespace).Get(ctx, podName, meta_v1.GetOptions{ResourceVersion: "0"})
						firstTry = false
					} else {
						localCEP, err = ciliumClient.CiliumEndpoints(namespace).Get(ctx, podName, meta_v1.GetOptions{})
					}

					switch {
					case k8serrors.IsNotFound(err):  // 对于 CEP 对象不存在的情况，那就创建新的 CEP 对象
						pod := e.GetPod()
						cep := &cilium_v2.CiliumEndpoint{  // 初始化新的 CEP 对象
							ObjectMeta: meta_v1.ObjectMeta{
								Name: podName,  // CEP 对象与 Pod 同名
								OwnerReferences: []meta_v1.OwnerReference{  // 其 owner 就是 Endpoint 对应的 Pod
									{
										APIVersion: "v1",
										Kind:       "Pod",
										Name:       pod.GetObjectMeta().GetName(),
										UID:        pod.ObjectMeta.UID,
									},
								},
								Labels: pod.GetObjectMeta().GetLabels(),
							},
							Status: *mdl,
						}
						localCEP, err = ciliumClient.CiliumEndpoints(namespace).Create(ctx, cep, meta_v1.CreateOptions{})  // 创建 CEP 对象
					default:
						return err
					}

					needInit = false
				}

				// 对于 localCEP 为 nil 的情况，先从 API server 中尝试获取最新的 CEP 对象
				if localCEP == nil {
					localCEP, err = ciliumClient.CiliumEndpoints(namespace).Get(ctx, podName, meta_v1.GetOptions{})
					switch {
					// 若没有找到，则说明 CEP 还未创建，此时先做标记，等下一次调谐时进行创建
					case k8serrors.IsNotFound(err) || k8serrors.IsInvalid(err):
						needInit = true
						return err
					}
				}
			},
			StopFunc: func(ctx context.Context) error {
				return deleteCEP(ctx, scopedLog, ciliumClient, e)  // 直接通过 ciliumClient.CiliumEndpoints(namespace).Delete 接口删除
			},
		})
}

// pkg/endpoint/endpoint.go

func (e *Endpoint) UpdateLabels(ctx context.Context, identityLabels, infoLabels labels.Labels, blocking bool) (regenTriggered bool) {
	// 替换 endpoint 中的 infomation labels
	e.replaceInformationLabels(infoLabels)
	// 替换 identity labels，若 labels 发生变化则更新 identity；若网络发生变化则返回 identityRevision，否则返回 0
	rev := e.replaceIdentityLabels(identityLabels)
	e.unlock()
	if rev != 0 {
		return e.runIdentityResolver(ctx, rev, blocking)  // 若 identity 发生变化，则重新进行解析
	}

	return false
}

// pkg/allocator/allocator.go

func (a *Allocator) Allocate(ctx context.Context, key AllocatorKey) (idpool.ID, bool, bool, error) {

	for attempt := 0; attempt < maxAllocAttempts; attempt++ {  // maxAllocAttempts 固定为 16 次
		if val := a.localKeys.use(k); val != idpool.NoID {  // identity 为 0 说明 ID 不存在
			a.mainCache.insert(key, val)
			return val, false, false, nil  // 第二个返回值表示在 kvstore 中是否有新 ID 被创建
		}

		value, isNew, firstUse, err = a.lockedAllocate(ctx, key)  // 分配新的 ID
		if err == nil {
			a.mainCache.insert(key, value)
			return value, isNew, firstUse, nil
		}
	}

	return 0, false, false, err
}

// pkg/ipcache/kvstore.go

func UpsertIPToKVStore(ctx context.Context, IP, hostIP net.IP, ID identity.NumericIdentity, key uint8, metadata, k8sNamespace, k8sPodName string, npm types.NamedPortMap) error {
	// 按命名端口名称的字典序为端口排序
	namedPorts := // ...

	ipKey := path.Join(IPIdentitiesPath,  // => "cilium/state/ip/v1"
                       AddressSpace, IP.String())
	ipIDPair := identity.IPIdentityPair{
		IP:           IP,
		ID:           ID,
		// ...
		NamedPorts:   namedPorts,
	}

	marshaledIPIDPair, err := json.Marshal(ipIDPair)

	err = globalMap.store.upsert(ctx, ipKey, string(marshaledIPIDPair), true)  // update/insert
	return err
}

// pkg/endpoint/policy.go

func (e *Endpoint) Regenerate(regenMetadata *regeneration.ExternalRegenerationMetadata) <-chan bool {
	done := make(chan bool, 1)

	regenContext := ParseExternalRegenerationMetadata(ctx, cFunc, regenMetadata)
	epEvent := eventqueue.NewEvent(&EndpointRegenerationEvent{  // 创建重新生成（regen）事件
		regenContext: regenContext,
		ep:           e,
	})

	resChan, err := e.eventQueue.Enqueue(epEvent)  // 将 regen 事件加入到事件队列中

	go func() {
		select {
		case result, ok := <-resChan:
			if ok {
				regenResult := result.(*EndpointRegenerationResult)  // 根据 regen 事件的执行结果判断是否构建成功
				buildSuccess = regenResult.err == nil
			}
		}
		done <- buildSuccess
		close(done)
	}()

	return done
}

// pkg/eventqueue/eventqueue.go

func (q *EventQueue) Run() {  // 事件消费
	go q.run()
}

func (q *EventQueue) run() {
	q.eventQueueOnce.Do(func() {
		defer close(q.eventsClosed)
		for ev := range q.events {
			select {
			default:
				ev.Metadata.Handle(ev.eventResults)  // 事件处理
				close(ev.eventResults)
			}
		}
	})
}

type EventHandler interface {
	Handle(chan interface{})
}

// pkg/endpoint/policy.go

func (e *Endpoint) regenerate(ctx *regenerationContext) (retErr error) {

	origDir := e.StateDirectoryPath()
	ctx.datapathRegenerationContext.currentDir = origDir  // $(daemonConfig.StateDir)/$(ep.StringID)

	// temporary 目录用于保存生成的头文件
	tmpDir := e.NextDirectoryPath()
	ctx.datapathRegenerationContext.nextDir = tmpDir  // ./$(ep.StringID)_next

	// 移除现有的 temporary 目录
	if err := e.removeDirectory(tmpDir); err != nil && !os.IsNotExist(err) {
		return // err
	}

	// 创建 temporary 目录
	err := os.MkdirAll(tmpDir, 0777)

	defer func() {
		e.removeDirectory(tmpDir)
	}()

	revision, stateDirComplete, err = e.regenerateBPF(ctx)  // ***

	// 将所有 verifier 的日志写入到 temporary 目录下
	var ve *ebpf.VerifierError
	if errors.As(err, &ve) {
		p := path.Join(tmpDir, "verifier.log")
		f, err := os.Create(p)
        err := fmt.Fprintf(f, "%+v\n", ve)
	}

	return e.updateRealizedState(stats, origDir, revision, stateDirComplete)
}

// pkg/datapath/loader/loader.go

func (l *Loader) CompileAndLoad(ctx context.Context, ep datapath.Endpoint, stats *metrics.SpanStat) error {
	dirs := directoryInfo{
		Library: option.Config.BpfDir,     // /var/lib/cilium/bpf，存放 BPF 模版文件
		Runtime: option.Config.StateDir,
		State:   ep.StateDir(),
		Output:  ep.StateDir(),
	}
	return l.compileAndLoad(ctx, ep, &dirs, stats)
}

func (l *Loader) compileAndLoad(ctx context.Context, ep datapath.Endpoint, dirs *directoryInfo, stats *metrics.SpanStat) error {
	err := compileDatapath(ctx, dirs, ep.IsHost(), ep.Logger(Subsystem))  // 工作1
	err = l.reloadDatapath(ctx, ep, dirs)  // 工作2
	return err
}

func compileDatapath(ctx context.Context, dirs *directoryInfo, isHost bool, logger *logrus.Entry) error {

	versionCmd := exec.CommandContext(ctx, compiler, "--version")
	compilerVersion, err := versionCmd.CombinedOutput(scopedLog, true)  // 检查编译器的状态

	versionCmd = exec.CommandContext(ctx, linker, "--version")
	linkerVersion, err := versionCmd.CombinedOutput(scopedLog, true)  // 检查链接器的状态

	// 编译新的程序
	prog := epProg  // => struct epProg = {Source: "bpf_lxc.c", Output: "bpf_lxc.o", OutputType: "obj"}
	compile(ctx, prog, dirs)

	return nil
}

func compile(ctx context.Context, prog *progInfo, dir *directoryInfo) (err error) {
	args := make([]string, 0, 16)
	if prog.OutputType == outputSource {
		args = append(args, "-E") // Preprocessor
	} else {
		args = append(args, "-emit-llvm")
		args = append(args, "-g")
	}

	// 追加各种编译参数
	args = append(args, standardCFlags...)
	args = append(args, prog.Options...)
	args = append(args, progCFlags(prog, dir)...)

	switch prog.OutputType {
	case outputSource:
		compileCmd := exec.CommandContext(ctx, compiler, args...)
		_, err = compileCmd.CombinedOutput(log, true)
	case outputObject, outputAssembly:
		err = compileAndLink(ctx, prog, dir, args...)  // 编译执行与链接
	}

	return err
}

func cmdDel(args *skel.CmdArgs) error {
	n, err := types.LoadNetConf(args.StdinData)

	cniArgs := types.ArgsSpec{}
	cniTypes.LoadArgs(args.Args, &cniArgs)  // 提取 CNI 参数

	c, err := lib.NewDeletionFallbackClient(logger)  // 初始化 client

	id := endpointid.NewID(endpointid.ContainerIdPrefix, args.ContainerID)  // Prefix: "container-id"
	c.EndpointDelete(id)  // 删除 Endpoint

	if n.IPAM.Type != "" {
		err = cniInvoke.DelegateDel(context.TODO(), n.IPAM.Type, args.StdinData, nil)  // 释放 IP
	}

	netNs, err := ns.GetNS(args.Netns)
	defer netNs.Close()
	err = netns.RemoveIfFromNetNSIfExists(netNs, args.IfName)  // 移除网络命名空间中的接口

	return nil
}