Upgrade to Pro — share decks privately, control downloads, hide ads and more …

Kubernetesコントローラーのパフォーマンスチューニング

 Kubernetesコントローラーのパフォーマンスチューニング

Kubernetes Meetup Tokyo #56
2023/03/16
https://k8sjp.connpass.com/event/275280/

Akihiro Ikezoe

March 16, 2023
Tweet

More Decks by Akihiro Ikezoe

Other Decks in Programming

Transcript

  1. application-controller Workers Workers Workers Workers Status Processors Workers Workers Operation

    Processors Application Resource Informer Informer watch Events Application Resource
  2. # Reconcile 99 histogram_quantile(0.99, sum( rate(controller_runtime_reconcile_time_seconds_bucket[1m]) ) by(job, controller, le)

    ) # Reconcile sum(rate(controller_runtime_reconcile_total[1m]))by(job, controller, result)
  3. ◆ ◆ import ( "context" "net/url" "time" "github.com/prometheus/client_golang/prometheus" clmetrics "k8s.io/client-go/tools/metrics"

    crmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" ) var ( rateLimiterDelay = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Name: "rest_client_rate_limiter_duration_seconds", Help: "client-go rate limiter delay in seconds. Broken down by verb, and host.", Buckets: []float64{0.005, 0.025, 0.1, 0.25, 0.5, 1.0, 2.0, 4.0, 8.0, 15.0, 30.0, 60.0}, }, []string{"verb", "host"}, ) _ clmetrics.LatencyMetric = &latencyAdapter{} ) func init() { crmetrics.Registry.MustRegister(rateLimiterDelay) adapter := latencyAdapter{ metric: rateLimiterDelay, } clmetrics.RateLimiterLatency = &adapter } type latencyAdapter struct { metric *prometheus.HistogramVec } func (c *latencyAdapter) Observe(_ context.Context, verb string, u url.URL, latency time.Duration) { c.metric.WithLabelValues(verb, u.Host).Observe(latency.Seconds()) }
  4. # Application Reconcile Status Processor {job=~"argocd/argocd-application-controller"} | logfmt | msg

    ="Reconciliation completed" | line_format "{{.application}}: {{.time_ms}}" # Application Reconcile Operation Processor {job=~"argocd/argocd-application-controller"} | logfmt | msg = "sync/terminate complete" | line_format "{{.application}}: {{.duration}}"
  5. # {job=~"argocd/argocd-application-controller"} | logfmt | level = "debug" msg =~

    "Refreshing app .*" apiVersion: v1 kind: ConfigMap metadata: name: argocd-cmd-params-cm data: # Application Controller debug default "info" controller.log.level: "debug"
  6. ◆ $ kubectl port-forward svc/argocd-application-controller-metrics -n argocd 8082:8082 # 30

    $ curl localhost:8082/debug/pprof/profile > cpu.pprof # goroutine $ curl localhost:8082/debug/pprof/goroutine?debug=1
  7. apiVersion: v1 kind: ConfigMap metadata: name: argocd-cmd-params-cm data: # Number

    of application status processors (default 20) controller.status.processors: "20" # Number of application operation processors (default 10) controller.operation.processors: "10" ◆ ◆
  8. import ctrl "sigs.k8s.io/controller-runtime" // ・・・途中省略・・・ cfg, err := ctrl.GetConfig() if

    err != nil { return err } cfg.QPS = 50 cfg.Burst = int(cfg.QPS * 1.5) mgr, err := ctrl.NewManager(cfg, ctrl.Options{ ... })