flowchart TB
    subgraph "控制平面 Control Plane"
        API[kube-apiserver<br/>RESTful API 网关<br/>认证/授权/准入控制]
        ETCD[(etcd 集群<br/>分布式 KV 存储<br/>集群状态持久化)]
        SCHED[kube-scheduler<br/>调度决策<br/>Pod→Node 绑定]
        CM[kube-controller-manager<br/>控制循环<br/>40+ 控制器]
        CCM[cloud-controller-manager<br/>云厂商集成<br/>LoadBalancer/NodeRoute]
    end
    
    subgraph "数据平面 Data Plane"
        direction LR
        subgraph "Node 1"
            KL1[kubelet<br/>Pod 生命周期管理<br/>容器运行时交互]
            KP1[kube-proxy<br/>Service 流量转发<br/>iptables/IPVS规则]
            CRI1[Container Runtime<br/>containerd/CRI-O]
        end
        subgraph "Node N"
            KLN[kubelet]
            KPN[kube-proxy]
            CRIN[Container Runtime]
        end
    end
    
    subgraph "插件系统 Add-ons"
        DNS[CoreDNS<br/>集群内DNS服务]
        CNI[CNI Plugin<br/>Calico/Flannel<br/>容器网络配置]
        CSI[CSI Driver<br/>存储卷挂载]
        METRIC[Metrics Server<br/>资源使用率采集]
    end
    
    subgraph "客户端 Clients"
        KUBECTL[kubectl<br/>命令行工具]
        CLIENT[Client-Go<br/>SDK库]
        WEB[Dashboard/Lens<br/>Web界面]
    end
    
    KUBECTL -->|HTTP/HTTPS| API
    CLIENT -->|REST API| API
    WEB -->|API Proxy| API
    
    API <-->|Watch/List<br/>gRPC| ETCD
    SCHED -->|Pod Binding| API
    CM -->|Resource CRUD| API
    CCM -->|Cloud API| API
    
    KL1 <-->|Pod Spec Sync<br/>Node Status Report| API
    KLN <-->|Watch/Update| API
    KP1 <-->|Service/Endpoint Sync| API
    KPN <-->|Watch| API
    
    KL1 -->|CRI gRPC| CRI1
    KLN -->|CRI| CRIN
    
    DNS -->|Service Discovery| API
    CNI -.->|网络配置| KL1
    CNI -.->|网络配置| KLN
    CSI -->|Volume 操作| API
    METRIC -->|Metrics 上报| API
    
    style API fill:#326CE5,stroke:#fff,color:#fff
    style ETCD fill:#419EDA,stroke:#fff,color:#fff
    style SCHED fill:#326CE5,stroke:#fff,color:#fff
    style CM fill:#326CE5,stroke:#fff,color:#fff

sequenceDiagram
    autonumber
    participant U as kubectl/Client
    participant API as kube-apiserver
    participant ETCD as etcd
    participant SCHED as kube-scheduler
    participant CM as kube-controller-manager
    participant KL as kubelet
    participant CRI as Container Runtime
    
    Note over U,CRI: 阶段1：用户提交 Deployment
    U->>+API: POST /apis/apps/v1/deployments
    API->>API: 认证（Authentication）
    API->>API: 授权（Authorization）
    API->>API: 准入控制（Admission）
    API->>+ETCD: Put /registry/deployments/default/nginx
    ETCD-->>-API: Revision=100
    API-->>-U: 201 Created (Deployment)
    
    Note over CM,ETCD: 阶段2：Deployment Controller 创建 ReplicaSet
    CM->>CM: Watch Event: Deployment Added
    CM->>+API: GET /apis/apps/v1/deployments/default/nginx
    API->>+ETCD: Range /registry/deployments/default/nginx
    ETCD-->>-API: Deployment Object
    API-->>-CM: Deployment Spec (replicas=3)
    CM->>CM: Reconcile: 计算期望状态 (需要创建 ReplicaSet)
    CM->>+API: POST /apis/apps/v1/replicasets
    API->>+ETCD: Put /registry/replicasets/default/nginx-xxxxx
    ETCD-->>-API: Revision=101
    API-->>-CM: 201 Created (ReplicaSet)
    
    Note over CM,ETCD: 阶段3：ReplicaSet Controller 创建 Pod
    CM->>CM: Watch Event: ReplicaSet Added
    CM->>CM: Reconcile: 需要创建 3 个 Pod
    loop 创建 3 个 Pod
        CM->>+API: POST /api/v1/namespaces/default/pods
        API->>+ETCD: Put /registry/pods/default/nginx-xxxxx-yyy
        ETCD-->>-API: Revision=102/103/104
        API-->>-CM: 201 Created (Pod)
    end
    
    Note over SCHED,KL: 阶段4：Scheduler 调度 Pod
    SCHED->>SCHED: Watch Event: Pod Added (NodeName="")
    SCHED->>+API: GET /api/v1/nodes
    API->>+ETCD: Range /registry/minions
    ETCD-->>-API: Node List
    API-->>-SCHED: Node Objects
    SCHED->>SCHED: 执行调度算法<br/>Filtering (预选)<br/>Scoring (优选)
    SCHED->>SCHED: 选择 Node: node-01
    SCHED->>+API: POST /api/v1/namespaces/default/pods/nginx-xxxxx-yyy/binding
    API->>+ETCD: Put /registry/pods/default/nginx-xxxxx-yyy<br/>(spec.nodeName="node-01")
    ETCD-->>-API: Revision=105
    API-->>-SCHED: 200 OK
    
    Note over KL,CRI: 阶段5：kubelet 启动容器
    KL->>KL: Watch Event: Pod Updated<br/>(spec.nodeName="node-01")
    KL->>KL: SyncLoop 处理 Pod
    KL->>KL: 创建 Pod Sandbox (网络/IPC 命名空间)
    KL->>+CRI: RunPodSandbox(PodSandboxConfig)
    CRI->>CRI: 创建 Pause 容器
    CRI-->>-KL: PodSandboxID
    
    loop 启动 Init Containers (如果有)
        KL->>+CRI: CreateContainer(InitContainerConfig)
        CRI-->>-KL: ContainerID
        KL->>+CRI: StartContainer(ContainerID)
        CRI-->>-KL: Success
    end
    
    loop 启动 App Containers
        KL->>+CRI: PullImage(ImageSpec)
        CRI-->>-KL: ImageRef
        KL->>+CRI: CreateContainer(ContainerConfig)
        CRI-->>-KL: ContainerID
        KL->>+CRI: StartContainer(ContainerID)
        CRI-->>-KL: Success
    end
    
    KL->>KL: 更新 Pod Status (Phase=Running)
    KL->>+API: PATCH /api/v1/namespaces/default/pods/nginx-xxxxx-yyy/status
    API->>+ETCD: Put /registry/pods/default/nginx-xxxxx-yyy<br/>(status.phase="Running")
    ETCD-->>-API: Revision=106
    API-->>-KL: 200 OK
    
    Note over U,CRI: Pod 创建完成，进入运行状态

graph TB
    subgraph "同步调用（RESTful API）"
        A1[kubectl] -->|HTTP POST/GET/PATCH| A2[API Server]
        A3[client-go] -->|HTTP/HTTPS| A2
        A4[Controller Manager] -->|List/Watch/Update| A2
        A5[Scheduler] -->|List/Watch/Bind| A2
        A6[kubelet] -->|Node Status Report| A2
        A7[kube-proxy] -->|Service/Endpoint Sync| A2
        A2 -->|gRPC Range/Put/Delete| A8[etcd]
    end
    
    subgraph "异步调用（Watch 机制）"
        A2 -.->|Watch Stream| A4
        A2 -.->|Watch Stream| A5
        A2 -.->|Watch Stream| A6
        A2 -.->|Watch Stream| A7
    end
    
    subgraph "gRPC 调用"
        A6 -->|CRI gRPC| A9[Container Runtime]
        A6 -->|CSI gRPC| A10[CSI Driver]
    end
    
    subgraph "进程内调用"
        A6 -.->|exec binary| A11[CNI Plugin]
        A7 -.->|iptables/ipvsadm| A12[内核网络栈]
    end
    
    style A2 fill:#326CE5,color:#fff
    style A8 fill:#419EDA,color:#fff

// 请求延迟（P50/P95/P99）
apiserver_request_duration_seconds{verb="GET|POST|PATCH|DELETE", resource="pods|nodes|services"}

// 请求 QPS
apiserver_request_total{code="200|400|500"}

// etcd 延迟
etcd_request_duration_seconds{operation="range|put|delete"}

// 控制循环延迟
workqueue_queue_duration_seconds{name="deployment|replicaset|pod"}

// 控制循环执行速率
controller_workqueue_adds_total

// 资源协调失败次数
controller_reconcile_errors_total

// 调度延迟
scheduler_scheduling_duration_seconds

// 调度成功率
scheduler_schedule_attempts_total{result="scheduled|unschedulable|error"}

// 队列深度
scheduler_pending_pods

// Pod 启动时间
kubelet_pod_start_duration_seconds

// 容器重启次数
kubelet_containers_restarts_total

// CRI 调用延迟
kubelet_runtime_operations_duration_seconds{operation_type="create_container|start_container"}

# 1. 安装 kubeadm、kubelet、kubectl
sudo apt-get update
sudo apt-get install -y kubelet kubeadm kubectl
sudo systemctl enable kubelet

# 2. 初始化控制平面
sudo kubeadm init \
  --pod-network-cidr=10.244.0.0/16 \
  --service-cidr=10.96.0.0/12 \
  --apiserver-advertise-address=<NODE_IP>

# 3. 配置 kubectl
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

# 4. 安装 CNI 插件（以 Flannel 为例）
kubectl apply -f https://github.com/flannel-io/flannel/releases/latest/download/kube-flannel.yml

# 5. 验证集群状态
kubectl get nodes
kubectl get pods -A

# nginx-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx
  labels:
    app: nginx
spec:
  replicas: 3
  selector:
    matchLabels:
      app: nginx
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
      - name: nginx
        image: nginx:1.21
        ports:
        - containerPort: 80
        resources:
          requests:
            cpu: "100m"
            memory: "128Mi"
          limits:
            cpu: "200m"
            memory: "256Mi"
---
apiVersion: v1
kind: Service
metadata:
  name: nginx
spec:
  type: ClusterIP
  selector:
    app: nginx
  ports:
  - port: 80
    targetPort: 80

# 1. 创建资源
kubectl apply -f nginx-deployment.yaml

# 2. 观察创建过程
kubectl get deployments -w  # 查看 Deployment 状态
kubectl get replicasets -w  # 查看 ReplicaSet 状态
kubectl get pods -o wide -w  # 查看 Pod 调度与启动

# 3. 验证服务可用性
kubectl get svc nginx
kubectl run test-pod --image=busybox --rm -it --restart=Never -- wget -O- nginx

// 实现 FilterPlugin 接口
package custom

import (
    "context"
    "k8s.io/kubernetes/pkg/scheduler/framework"
)

type GPUFilter struct{}

func (g *GPUFilter) Name() string {
    return "GPUFilter"
}

// Filter 预选阶段过滤不符合条件的节点
func (g *GPUFilter) Filter(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodeInfo *framework.NodeInfo) *framework.Status {
    // 检查 Pod 是否需要 GPU
    requiredGPU, ok := pod.Labels["gpu-type"]
    if !ok {
        return nil // 不需要 GPU，跳过检查
    }
    
    // 检查 Node 是否有匹配的 GPU
    nodeGPU, ok := nodeInfo.Node().Labels["gpu-type"]
    if !ok || nodeGPU != requiredGPU {
        return framework.NewStatus(framework.Unschedulable, "GPU type mismatch")
    }
    
    return nil // 通过预选
}

// 注册插件
func New(ctx context.Context, obj runtime.Object, handle framework.Handle) (framework.Plugin, error) {
    return &GPUFilter{}, nil
}

# scheduler-config.yaml
apiVersion: kubescheduler.config.k8s.io/v1
kind: KubeSchedulerConfiguration
profiles:
- schedulerName: custom-scheduler
  plugins:
    filter:
      enabled:
      - name: GPUFilter
    score:
      disabled:
      - name: NodeResourcesBalancedAllocation  # 禁用默认插件

# 1. 编译自定义调度器
go build -o custom-scheduler main.go

# 2. 以 Deployment 运行
kubectl create deployment custom-scheduler --image=my-repo/custom-scheduler:v1
kubectl set env deployment/custom-scheduler KUBECONFIG=/etc/kubernetes/scheduler.conf

# 3. Pod 指定调度器
kubectl run gpu-pod --image=gpu-app --overrides='{"spec":{"schedulerName":"custom-scheduler"}}'

// 实现 MutatingWebhook
package webhook

import (
    admissionv1 "k8s.io/api/admission/v1"
    corev1 "k8s.io/api/core/v1"
    "encoding/json"
)

func MutatePod(ar *admissionv1.AdmissionReview) *admissionv1.AdmissionResponse {
    pod := corev1.Pod{}
    json.Unmarshal(ar.Request.Object.Raw, &pod)
    
    // 检查是否需要注入 Sidecar
    if pod.Annotations["inject-sidecar"] != "true" {
        return &admissionv1.AdmissionResponse{Allowed: true}
    }
    
    // 构造 Patch（JSON Patch 格式）
    patch := []map[string]interface{}{
        {
            "op": "add",
            "path": "/spec/containers/-",
            "value": map[string]interface{}{
                "name":  "sidecar",
                "image": "my-sidecar:v1",
                "ports": []map[string]interface{}{
                    {"containerPort": 8080},
                },
            },
        },
    }
    patchBytes, _ := json.Marshal(patch)
    
    return &admissionv1.AdmissionResponse{
        Allowed: true,
        Patch:   patchBytes,
        PatchType: func() *admissionv1.PatchType {
            pt := admissionv1.PatchTypeJSONPatch
            return &pt
        }(),
    }
}

# mutating-webhook.yaml
apiVersion: admissionregistration.k8s.io/v1
kind: MutatingWebhookConfiguration
metadata:
  name: sidecar-injector
webhooks:
- name: sidecar.example.com
  clientConfig:
    service:
      name: sidecar-injector
      namespace: default
      path: /mutate
    caBundle: <BASE64_ENCODED_CA_CERT>
  rules:
  - operations: ["CREATE"]
    apiGroups: [""]
    apiVersions: ["v1"]
    resources: ["pods"]
  admissionReviewVersions: ["v1"]
  sideEffects: None