Kubernetesコントローラーのパフォーマンスチューニング

Kubernetes Meetup Tokyo #56
2023/03/16
https://k8sjp.connpass.com/event/275280/

Transcript

2. ◆ ◆ ◆ ◼ ◼ ◼ ◼ ◼ ◼ ◼
3. ◆ ◼ ◆ ◼ ◆ ◼ ⚫ ⚫ ◼
5. ◆ ◆ ◼ ◆ ◼ ◼ ◼
6. ✓ ✓ ✓ ✓ ✓ ✓
7. ◆ ◆ ◼ ◼ ◆ ◼ ◼
8. Controller Workers Workers Workers Workers Reconciler Informer
9. Controller Workers Workers Workers Workers Reconciler Informer
10. ◆ ◼ ◆ ◼ ◼ ◆ ◼ ◼
11. ◆ ◼ ◼ ◆ ◼ ◼ ◆ ◼ ◼
12. ◆ ◼ ◼ ◆
13. ◆ ◼ https://github.com/kubernetes/enhancements/issues/1602 ◆ ◼ https://kubernetes.io/docs/reference/instrumentation/metrics/ ◆ ◼ https://kubernetes.io/docs/concepts/cluster-administration/system-traces/
14. ◆ ◼ ◼ ◼ ⚫ ⚫ https://cybozu-go.github.io/moco/metrics.html ⚫
15. ◆ ◼ ◼ ⚫ ⚫ ⚫ ◼
16. ◆ ◼ ◼ ◼ ◆ ◼ ◼ ◼ https://github.com/cybozu-go/moco/pull/500
17. ◆ ◼ ◼ ◼ ◼ ◼ ◆
18. ◆ ◼ ◼ ⚫ ◆ ◼
19. ◆ ◆ ◼ ◼ ◼ ◼ ◼ ◼
20. ◆ ◼ ◆ ◆ ◼ ◆
22. ◆ ◼ ◼ ◆ ◼ ◼ ◆ ◼
23. Kubernetes Cluster Application Controller ArgoCD Server Repo Server Application Resource Application Resource
24. application-controller Workers Workers Workers Workers Status Processors Workers Workers Operation Processors Application Resource Informer Informer watch Events Application Resource
25. ◆ ◆ ◼ ◼ ◼
26. ◆ ◼ ◆ ◼ ◆ ◼ ◆ ◼
27. ◆ ◼ ◼ ◆
28. ◆ ◼ ◼
29. application-controller Workers Workers Workers Workers Status Processors Workers Workers Operation Processors Application Resource Informer Informer watch Events
30. ◆ ◼ ◼ ◆ ◼ ◆ ◼
31. ◆ ◆
32. ◆ ◼ ◼ ◼ ◆
33. workqueue_depth{job="kube-controller-manager",name="volumes"}
34. histogram_quantile(0.99, sum(rate( rest_client_rate_limiter_duration_seconds_bucket{ job="kube-controller-manager" }[1m] )) by (le))
35. kube-controller-manager PersistentVolume Controller
36. ◆ ◼ --kube-api-qps ◆ ◼ ◆ ◼ ◼
38. ◆ ◆ ◆
40. ◆ ◼ https://github.com/zoetrope/kubbernecker ◼ ◼ ⚫ ◼ ⚫ ⚫
42. # Reconcile 99 histogram_quantile(0.99, sum( rate(controller_runtime_reconcile_time_seconds_bucket[1m]) ) by(job, controller, le) ) # Reconcile sum(rate(controller_runtime_reconcile_total[1m]))by(job, controller, result)
43. # 99 histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket[1m])) by(job, name, le)) # sum(workqueue_depth) by (job, name)
44. ◆ ◆ import ( "context" "net/url" "time" "github.com/prometheus/client_golang/prometheus" clmetrics "k8s.io/client-go/tools/metrics" crmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" ) var ( rateLimiterDelay = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Name: "rest_client_rate_limiter_duration_seconds", Help: "client-go rate limiter delay in seconds. Broken down by verb, and host.", Buckets: []float64{0.005, 0.025, 0.1, 0.25, 0.5, 1.0, 2.0, 4.0, 8.0, 15.0, 30.0, 60.0}, }, []string{"verb", "host"}, ) _ clmetrics.LatencyMetric = &latencyAdapter{} ) func init() { crmetrics.Registry.MustRegister(rateLimiterDelay) adapter := latencyAdapter{ metric: rateLimiterDelay, } clmetrics.RateLimiterLatency = &adapter } type latencyAdapter struct { metric *prometheus.HistogramVec } func (c *latencyAdapter) Observe(_ context.Context, verb string, u url.URL, latency time.Duration) { c.metric.WithLabelValues(verb, u.Host).Observe(latency.Seconds()) }
45. # Rate Limiter 99 histogram_quantile(0.99, sum( rate(rest_client_rate_limiter_duration_seconds_bucket[1m]) ) by(job, verb, le) )
46. # Application Reconcile Status Processor {job=~"argocd/argocd-application-controller"} | logfmt | msg ="Reconciliation completed" | line_format "{{.application}}: {{.time_ms}}" # Application Reconcile Operation Processor {job=~"argocd/argocd-application-controller"} | logfmt | msg = "sync/terminate complete" | line_format "{{.application}}: {{.duration}}"
47. # {job=~"argocd/argocd-application-controller"} | logfmt | level = "debug" msg =~ "Refreshing app .*" apiVersion: v1 kind: ConfigMap metadata: name: argocd-cmd-params-cm data: # Application Controller debug default "info" controller.log.level: "debug"
48. ◆
49. ◆ $ kubectl port-forward svc/argocd-application-controller-metrics -n argocd 8082:8082 # 30 $ curl localhost:8082/debug/pprof/profile > cpu.pprof # goroutine $ curl localhost:8082/debug/pprof/goroutine?debug=1
50. ◆ ◆ --otlp-address ◆
51. apiVersion: v1 kind: ConfigMap metadata: name: argocd-cmd-params-cm data: # Number of application status processors (default 20) controller.status.processors: "20" # Number of application operation processors (default 10) controller.operation.processors: "10" ◆ ◆
52. import ctrl "sigs.k8s.io/controller-runtime" // ・・・途中省略・・・ cfg, err := ctrl.GetConfig() if err != nil { return err } cfg.QPS = 50 cfg.Burst = int(cfg.QPS * 1.5) mgr, err := ctrl.NewManager(cfg, ctrl.Options{ ... })