1
apiVersion: apps/v1
2
kind: Deployment
3
metadata:
4
  name: higress-gateway
5
  namespace: higress-system
6
spec:
7
  replicas: 3                      # 生产至少 3 副本
8
  strategy:
9
    type: RollingUpdate
10
    rollingUpdate:
11
      maxUnavailable: 0            # 滚动更新时不允许不可用
12
      maxSurge: 1
13
  template:
14
    spec:
15
      affinity:
16
        # 节点反亲和：副本分散到不同节点
17
        podAntiAffinity:
18
          requiredDuringSchedulingIgnoredDuringExecution:
19
            - labelSelector:
20
                matchLabels:
21
                  app: higress-gateway
22
              topologyKey: kubernetes.io/hostname
23
        # 可用区反亲和（软约束）：尽量跨 AZ
24
        podAntiAffinity:
25
          preferredDuringSchedulingIgnoredDuringExecution:
26
            - weight: 100
27
              podAffinityTerm:
28
                labelSelector:
29
                  matchLabels:
30
                    app: higress-gateway
31
                topologyKey: topology.kubernetes.io/zone
32
      topologySpreadConstraints:
33
        # 拓扑分布约束：跨 AZ 均衡
34
        - maxSkew: 1
35
          topologyKey: topology.kubernetes.io/zone
36
          whenUnsatisfiable: ScheduleAnyway
37
          labelSelector:
38
            matchLabels:
39
              app: higress-gateway
40
      containers:
41
        - name: gateway
42
          resources:
43
            requests:
44
              cpu: "1000m"
45
              memory: "1Gi"
46
            limits:
47
              cpu: "2000m"
48
              memory: "2Gi"

1
apiVersion: policy/v1
2
kind: PodDisruptionBudget
3
metadata:
4
  name: higress-gateway-pdb
5
  namespace: higress-system
6
spec:
7
  minAvailable: 2                  # 任意时刻至少 2 个可用
8
  selector:
9
    matchLabels:
10
      app: higress-gateway

1
apiVersion: autoscaling/v2
2
kind: HorizontalPodAutoscaler
3
metadata:
4
  name: higress-gateway-hpa
5
  namespace: higress-system
6
spec:
7
  scaleTargetRef:
8
    apiVersion: apps/v1
9
    kind: Deployment
10
    name: higress-gateway
11
  minReplicas: 3
12
  maxReplicas: 20
13
  metrics:
14
    # CPU 使用率
15
    - type: Resource
16
      resource:
17
        name: cpu
18
        target:
19
          type: Utilization
20
          averageUtilization: 70
21
    # 内存使用率
22
    - type: Resource
23
      resource:
24
        name: memory
25
        target:
26
          type: Utilization
27
          averageUtilization: 80
28
  behavior:
29
    scaleUp:
30
      stabilizationWindowSeconds: 30   # 快速扩容
31
      policies:
32
        - type: Percent
33
          value: 100
34
          periodSeconds: 30
35
    scaleDown:
36
      stabilizationWindowSeconds: 300  # 缓慢缩容，防抖动
37
      policies:
38
        - type: Percent
39
          value: 10
40
          periodSeconds: 60

1
# helm values
2
global:
3
  replicas: 3                    # Controller 副本数
4
  priorityClassName: system-cluster-critical

1
apiVersion: v1
2
kind: ConfigMap
3
metadata:
4
  name: higress-config
5
  namespace: higress-system
6
data:
7
  higress: |
8
    concurrency: 0               # 0 = 自动等于 CPU 核数
9
    # 每个 worker 独立处理连接，避免锁竞争

1
# 上游连接池（针对后端服务）
2
apiVersion: networking.k8s.io/v1
3
kind: Ingress
4
metadata:
5
  name: api-route
6
  annotations:
7
    # 上游 keepalive 连接池
8
    higress.io/upstream-keepalive-connections: "256"
9
    higress.io/upstream-keepalive-time: "60s"
10
    # 单连接最大请求数（防止连接老化）
11
    higress.io/upstream-keepalive-requests: "1000"
12
    # 连接/请求超时
13
    higress.io/connect-timeout: "5s"
14
    higress.io/request-timeout: "30s"

1
# Gateway 监听器优化
2
apiVersion: v1
3
kind: ConfigMap
4
metadata:
5
  name: higress-gateway-tuning
6
  namespace: higress-system
7
data:
8
  # 开启 HTTP/2、HTTP/3(QUIC)
9
  enable-http2: "true"
10
  enable-http3: "false"          # 按需开启
11
  # 空闲连接超时
12
  downstream-idle-timeout: "180s"
13
  # 最大请求头大小
14
  max-request-headers-kb: "60"
15
  # 每连接最大请求数（0 = 无限）
16
  max-requests-per-connection: "0"

1
1. 副本数 & HPA        → 最直接，横向扩展
2
2. 资源 requests/limits → 避免 CPU 节流（throttling）
3
3. 上游连接池 keepalive → 减少握手开销（收益大）
4
4. 内核参数            → 高并发下的天花板
5
5. Wasm 插件精简       → 每个插件都有开销，按需启用
6
6. 日志采样            → 高 QPS 下全量日志很贵

1
┌─────────────────────────────────────────┐
2
│  边缘层：TLS 终止、WAF、DDoS、IP 黑名单  │
3
├─────────────────────────────────────────┤
4
│  认证层：JWT / OAuth2 / mTLS / API Key   │
5
├─────────────────────────────────────────┤
6
│  授权层：RBAC、Consumer 权限、路由级策略 │
7
├─────────────────────────────────────────┤
8
│  传输层：东西向 mTLS、加密               │
9
├─────────────────────────────────────────┤
10
│  数据层：敏感信息脱敏、日志审计          │
11
└─────────────────────────────────────────┘

1
# ❌ 反例：明文写在 annotation
2
metadata:
3
  annotations:
4
    higress.io/request-header-authorization: "Bearer sk-1234567890abcdef"
5

6
# ✅ 正例：引用 K8s Secret
7
apiVersion: v1
8
kind: Secret
9
metadata:
10
  name: llm-api-key
11
  namespace: higress-system
12
type: Opaque
13
stringData:
14
  api-key: sk-xxxx          # 生产用 External Secrets / Vault 注入
15
---
16
# 通过 Wasm 插件从 Secret 读取

1
# 使用 External Secrets Operator 从 Vault/AWS SM 同步
2
apiVersion: external-secrets.io/v1beta1
3
kind: ExternalSecret
4
metadata:
5
  name: llm-api-key
6
  namespace: higress-system
7
spec:
8
  refreshInterval: 1h
9
  secretStoreRef:
10
    name: vault-backend
11
    kind: ClusterSecretStore
12
  target:
13
    name: llm-api-key
14
  data:
15
    - secretKey: api-key
16
      remoteRef:
17
        key: secret/llm/openai
18
        property: api-key

1
# 限制谁能修改 Higress CRD
2
apiVersion: rbac.authorization.k8s.io/v1
3
kind: Role
4
metadata:
5
  name: higress-route-editor
6
  namespace: default
7
rules:
8
  - apiGroups: ["networking.k8s.io"]
9
    resources: ["ingresses"]
10
    verbs: ["get", "list", "watch", "create", "update"]
11
  # 注意：不给 delete，防误删；不给 WasmPlugin（更高权限单独授予）
12
---
13
apiVersion: rbac.authorization.k8s.io/v1
14
kind: RoleBinding
15
metadata:
16
  name: dev-team-route-editor
17
  namespace: default
18
subjects:
19
  - kind: Group
20
    name: dev-team
21
    apiGroup: rbac.authorization.k8s.io
22
roleRef:
23
  kind: Role
24
  name: higress-route-editor
25
  apiGroup: rbac.authorization.k8s.io

1
# 只允许 Gateway 访问后端服务，限制东西向流量
2
apiVersion: networking.k8s.io/v1
3
kind: NetworkPolicy
4
metadata:
5
  name: allow-gateway-only
6
  namespace: production
7
spec:
8
  podSelector:
9
    matchLabels:
10
      tier: backend
11
  policyTypes:
12
    - Ingress
13
  ingress:
14
    - from:
15
        - namespaceSelector:
16
            matchLabels:
17
              kubernetes.io/metadata.name: higress-system
18
          podSelector:
19
            matchLabels:
20
              app: higress-gateway
21
      ports:
22
        - protocol: TCP
23
          port: 8080

1
Git 仓库（唯一事实来源）
2
├── base/                      # 基础配置
3
│   ├── mcpbridge.yaml
4
│   └── wasmplugin-jwt.yaml
5
├── overlays/
6
│   ├── staging/               # 预发环境
7
│   │   ├── kustomization.yaml
8
│   │   └── ingress-patch.yaml
9
│   └── production/            # 生产环境
10
│       ├── kustomization.yaml
11
│       └── ingress-patch.yaml
12
└── argocd/
13
    └── application.yaml

1
# ArgoCD Application
2
apiVersion: argoproj.io/v1alpha1
3
kind: Application
4
metadata:
5
  name: higress-production
6
  namespace: argocd
7
spec:
8
  project: default
9
  source:
10
    repoURL: https://github.com/example/higress-config.git
11
    targetRevision: main
12
    path: overlays/production
13
  destination:
14
    server: https://kubernetes.default.svc
15
    namespace: higress-system
16
  syncPolicy:
17
    automated:
18
      prune: true              # 自动删除 Git 中移除的资源
19
      selfHeal: true           # 自动修复漂移
20
    syncOptions:
21
      - CreateNamespace=true
22
    retry:
23
      limit: 5
24
      backoff:
25
        duration: 5s
26
        factor: 2
27
        maxDuration: 3m

1
代码合并 → 构建 Wasm → 推送 OCI
2
                          ↓
3
              部署到 staging（自动）
4
                          ↓
5
              自动化测试 + 冒烟
6
                          ↓
7
        ┌─────────────────┴─────────────────┐
8
        │  灰度：5% → 20% → 50% → 100%       │
9
        │  每步观察 Metrics（错误率/延迟）    │
10
        │  异常自动回滚                        │
11
        └─────────────────────────────────────┘

1
# 用 Argo Rollouts 做金丝雀
2
apiVersion: argoproj.io/v1alpha1
3
kind: Rollout
4
metadata:
5
  name: my-service
6
spec:
7
  strategy:
8
    canary:
9
      steps:
10
        - setWeight: 5
11
        - pause: { duration: 5m }
12
        - analysis:              # 自动分析错误率
13
            templates:
14
              - templateName: success-rate
15
        - setWeight: 20
16
        - pause: { duration: 5m }
17
        - setWeight: 50
18
        - pause: { duration: 10m }

1
模式 A：每集群独立 Higress（推荐）
2
┌──────────────┐   ┌──────────────┐
3
│  Cluster 1   │   │  Cluster 2   │
4
│  Higress     │   │  Higress     │
5
│  + 业务服务  │   │  + 业务服务  │
6
└──────┬───────┘   └──────┬───────┘
7
       │                  │
8
       └───────┬──────────┘
9
               │
10
        全局负载均衡（DNS/GSLB/L4 LB）
11

12
优点：故障隔离、就近访问、独立扩缩容

1
Git 仓库（单一事实来源）
2
        ↓
3
   ArgoCD ApplicationSet（一键部署到 N 个集群）
4
        ↓
5
┌───────────┬───────────┬───────────┐
6
│ Cluster 1 │ Cluster 2 │ Cluster 3 │
7
│ Higress   │ Higress   │ Higress   │
8
└───────────┴───────────┴───────────┘

1
# ArgoCD ApplicationSet：一套配置分发到所有集群
2
apiVersion: argoproj.io/v1alpha1
3
kind: ApplicationSet
4
metadata:
5
  name: higress-multi-cluster
6
  namespace: argocd
7
spec:
8
  generators:
9
    - clusters: {}               # 匹配所有已注册集群
10
  template:
11
    metadata:
12
      name: 'higress-{{name}}'
13
    spec:
14
      source:
15
        repoURL: https://github.com/example/higress-config.git
16
        targetRevision: main
17
        path: overlays/production
18
      destination:
19
        server: '{{server}}'
20
        namespace: higress-system
21
      syncPolicy:
22
        automated:
23
          prune: true
24
          selfHeal: true

1
# 集群 A 的 Higress 通过 DNS/McpBridge 访问集群 B 的服务
2
apiVersion: networking.higress.io/v1
3
kind: McpBridge
4
metadata:
5
  name: cross-cluster
6
  namespace: higress-system
7
spec:
8
  registries:
9
    - name: cluster-b-svc
10
      type: dns
11
      domain: svc-b.cluster-b.example.com   # 集群 B 的服务暴露地址
12
      port: 8080

1
副本数 = ceil(峰值QPS / 单副本QPS × 冗余系数)
2
冗余系数建议 1.5~2（留缓冲应对突发 + 滚动更新）
3

4
示例：
5
峰值 100,000 QPS，单副本（2C4G）30,000 QPS
6
副本数 = ceil(100000 / 30000 × 1.5) = ceil(5) = 5 副本
7
再加跨 AZ 冗余 → 6 副本（每 AZ 3 个）

1
# Chaos Mesh：模拟 Gateway Pod 故障
2
apiVersion: chaos-mesh.org/v1alpha1
3
kind: PodChaos
4
metadata:
5
  name: kill-higress-gateway
6
spec:
7
  action: pod-kill
8
  mode: one                      # 每次杀一个，验证 HA
9
  selector:
10
    namespaces: [higress-system]
11
    labelSelectors:
12
      app: higress-gateway
13
  scheduler:
14
    cron: '@every 2m'