From 76d2f74743d10092281f553136fe711bce13f436 Mon Sep 17 00:00:00 2001 From: Mikkel Oscar Lyderik Larsen Date: Sat, 27 Jul 2019 10:42:46 +0200 Subject: [PATCH] Add support for averageValue for request-per-second Skipper metric This adds support for `averageValue` for the `request-per-second` metric based on Ingress Objects. This is only supported from Kubernetes `>=v1.14` (https://github.com/kubernetes/kubernetes/pull/72872). When defining the HPA with `autoscaling/v2beta1` you still need to define `targetValue` even though it won't be used when `averageValue` is set. Once we default to `autoscaling/v2beta2` this akward API will be gone. Signed-off-by: Mikkel Oscar Lyderik Larsen --- README.md | 58 +++++++++++++++------- example/deploy/hpa.yaml | 3 +- pkg/collector/collector.go | 2 + pkg/collector/skipper_collector.go | 37 +++++++------- pkg/collector/skipper_collector_test.go | 64 +++++++++++++++++++++++-- 5 files changed, 125 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 8e7ee51..275bc68 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,13 @@ # kube-metrics-adapter [![Build Status](https://travis-ci.org/zalando-incubator/kube-metrics-adapter.svg?branch=master)](https://travis-ci.org/zalando-incubator/kube-metrics-adapter) [![Coverage Status](https://coveralls.io/repos/github/zalando-incubator/kube-metrics-adapter/badge.svg?branch=master)](https://coveralls.io/github/zalando-incubator/kube-metrics-adapter?branch=master) + Kube Metrics Adapter is a general purpose metrics adapter for Kubernetes that can collect and serve custom and external metrics for Horizontal Pod Autoscaling. +It supports scaling based on [Prometheus metrics](https://prometheus.io/), [SQS queues](https://aws.amazon.com/sqs/) and others out of the box. + It discovers Horizontal Pod Autoscaling resources and starts to collect the requested metrics and stores them in memory. It's implemented using the [custom-metrics-apiserver](https://github.com/kubernetes-incubator/custom-metrics-apiserver) @@ -41,6 +44,18 @@ The `metric-config.*` annotations are used by the `kube-metrics-adapter` to configure a collector for getting the metrics. In the above example it configures a *json-path pod collector*. +## Kubernetes compatibility + +Like the [support +policy](https://kubernetes.io/docs/setup/release/version-skew-policy/) offered +for Kubernetes, this project aims to support the latest three minor releases of +Kubernetes. + +Currently the default supported API is `autoscaling/v2beta1`. However we aim to +move to `autoscaling/v2beta2` (available since `v1.12`) in the near future as +this adds a lot of improvements over `v2beta1`. The move to `v2beta2` will most +likely happen as soon as [GKE adds support for it](https://issuetracker.google.com/issues/135624588). + ## Building This project uses [Go modules](https://github.com/golang/go/wiki/Modules) as @@ -71,9 +86,9 @@ Currently only `json-path` collection is supported. ### Supported metrics -| Metric | Description | Type | -| ------------ | -------------- | ------- | -| *custom* | No predefined metrics. Metrics are generated from user defined queries. | Pods | +| Metric | Description | Type | K8s Versions | +| ------------ | -------------- | ------- | -- | +| *custom* | No predefined metrics. Metrics are generated from user defined queries. | Pods | `>=1.10` | ### Example @@ -151,10 +166,10 @@ the trade-offs between the two approaches. ### Supported metrics -| Metric | Description | Type | Kind | -| ------------ | -------------- | ------- | -- | -| `prometheus-query` | Generic metric which requires a user defined query. | External | | -| *custom* | No predefined metrics. Metrics are generated from user defined queries. | Object | *any* | +| Metric | Description | Type | Kind | K8s Versions | +| ------------ | -------------- | ------- | -- | -- | +| `prometheus-query` | Generic metric which requires a user defined query. | External | | `>=1.10` | +| *custom* | No predefined metrics. Metrics are generated from user defined queries. | Object | *any* | `>=1.10` | ### Example: External Metric @@ -259,9 +274,9 @@ box so users don't have to define those manually. ### Supported metrics -| Metric | Description | Type | Kind | -| ----------- | -------------- | ------ | ---- | -| `requests-per-second` | Scale based on requests per second for a certain ingress. | Object | `Ingress` | +| Metric | Description | Type | Kind | K8s Versions | +| ----------- | -------------- | ------ | ---- | ---- | +| `requests-per-second` | Scale based on requests per second for a certain ingress. | Object | `Ingress` | `>=1.14` (can work with `>=1.10`) | ### Example @@ -288,7 +303,10 @@ spec: apiVersion: extensions/v1beta1 kind: Ingress name: myapp - targetValue: 10 # this will be treated as targetAverageValue + averageValue: 10 # Only works with Kubernetes >=1.14 + # for Kubernetes <1.14 you can use `targetValue` instead: + targetValue: 10 # this must be set, but has no effect if `averageValue` is defined. + # Otherwise it will be treated as targetAverageValue ``` ### Metric weighting based on backend @@ -302,13 +320,17 @@ return the requests-per-second being sent to the `backend1`. The ingress annotat the backend weights can be obtained can be specified through the flag `--skipper-backends-annotation`. -**Note:** As of Kubernetes v1.10 the HPA does not support `targetAverageValue` for +**Note:** For Kubernetes `=1.10` | ### Example @@ -388,9 +410,9 @@ The ZMON collector allows scaling based on external metrics exposed by ### Supported metrics -| Metric | Description | Type | -| ------------ | ------- | -- | -| `zmon-check` | Scale based on any ZMON check results | External | +| Metric | Description | Type | K8s Versions | +| ------------ | ------- | -- | -- | +| `zmon-check` | Scale based on any ZMON check results | External | `>=1.10` | ### Example diff --git a/example/deploy/hpa.yaml b/example/deploy/hpa.yaml index fc7401f..cc6e513 100644 --- a/example/deploy/hpa.yaml +++ b/example/deploy/hpa.yaml @@ -37,7 +37,8 @@ spec: apiVersion: extensions/v1beta1 kind: Ingress name: custom-metrics-consumer - targetValue: 10 # this will be treated as targetAverageValue + averageValue: 10 + targetValue: 10 # this must be set, but has no effect if `averageValue` is defined. - type: External external: metricName: sqs-queue-length diff --git a/pkg/collector/collector.go b/pkg/collector/collector.go index ab1f0fa..a2953af 100644 --- a/pkg/collector/collector.go +++ b/pkg/collector/collector.go @@ -165,6 +165,7 @@ type MetricConfig struct { ObjectReference custom_metrics.ObjectReference PerReplica bool Interval time.Duration + MetricSpec autoscalingv2.MetricSpec } // ParseHPAMetrics parses the HPA object into a list of metric configurations. @@ -206,6 +207,7 @@ func ParseHPAMetrics(hpa *autoscalingv2.HorizontalPodAutoscaler) ([]*MetricConfi MetricTypeName: typeName, ObjectReference: ref, Config: map[string]string{}, + MetricSpec: metric, } if metric.Type == autoscalingv2.ExternalMetricSourceType && diff --git a/pkg/collector/skipper_collector.go b/pkg/collector/skipper_collector.go index cf7482a..d660021 100644 --- a/pkg/collector/skipper_collector.go +++ b/pkg/collector/skipper_collector.go @@ -72,8 +72,7 @@ type SkipperCollector struct { } // NewSkipperCollector initializes a new SkipperCollector. -func NewSkipperCollector(client kubernetes.Interface, plugin CollectorPlugin, hpa *autoscalingv2.HorizontalPodAutoscaler, - config *MetricConfig, interval time.Duration, backendAnnotations []string, backend string) (*SkipperCollector, error) { +func NewSkipperCollector(client kubernetes.Interface, plugin CollectorPlugin, hpa *autoscalingv2.HorizontalPodAutoscaler, config *MetricConfig, interval time.Duration, backendAnnotations []string, backend string) (*SkipperCollector, error) { return &SkipperCollector{ client: client, objectReference: config.ObjectReference, @@ -178,22 +177,26 @@ func (c *SkipperCollector) GetMetrics() ([]CollectedMetric, error) { return nil, fmt.Errorf("expected to only get one metric value, got %d", len(values)) } - // get current replicas for the targeted scale object. This is used to - // calculate an average metric instead of total. - // targetAverageValue will be available in Kubernetes v1.12 - // https://github.com/kubernetes/kubernetes/pull/64097 - replicas, err := targetRefReplicas(c.client, c.hpa) - if err != nil { - return nil, err - } - - if replicas < 1 { - return nil, fmt.Errorf("unable to get average value for %d replicas", replicas) - } - value := values[0] - avgValue := float64(value.Custom.Value.MilliValue()) / float64(replicas) - value.Custom.Value = *resource.NewMilliQuantity(int64(avgValue), resource.DecimalSI) + + // For Kubernetes