From 6b0afe51804614dd3b62e2ab212015ac1b639a43 Mon Sep 17 00:00:00 2001 From: Mikkel Oscar Lyderik Larsen Date: Sat, 31 Jul 2021 17:31:13 +0200 Subject: [PATCH] Add support for scaling based on RouteGroup resource Signed-off-by: Mikkel Oscar Lyderik Larsen --- README.md | 61 ++++- docs/rbac.yaml | 11 + go.mod | 1 + go.sum | 50 +++++ pkg/collector/skipper_collector.go | 81 +++++-- pkg/collector/skipper_collector_test.go | 282 +++++++++++++++++++++--- pkg/provider/metric_store.go | 10 + pkg/server/start.go | 30 ++- 8 files changed, 465 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index ff6b6f0..b72a946 100644 --- a/README.md +++ b/README.md @@ -306,20 +306,27 @@ for this object. But to satisfy the schema we specify a dummy pod called `dummy- ## Skipper collector The skipper collector is a simple wrapper around the Prometheus collector to -make it easy to define an HPA for scaling based on ingress metrics when +make it easy to define an HPA for scaling based on [Ingress][ingress] or +[RouteGroup][routegroup] metrics when [skipper](https://github.com/zalando/skipper) is used as the ingress implementation in your cluster. It assumes you are collecting Prometheus metrics from skipper and it provides the correct Prometheus queries out of the box so users don't have to define those manually. +[ingress]: https://kubernetes.io/docs/concepts/services-networking/ingress/ +[routegroup]: https://opensource.zalando.com/skipper/kubernetes/routegroups/ + ### Supported metrics | Metric | Description | Type | Kind | K8s Versions | | ----------- | -------------- | ------ | ---- | ---- | -| `requests-per-second` | Scale based on requests per second for a certain ingress. | Object | `Ingress` | `>=1.14` | +| `requests-per-second` | Scale based on requests per second for a certain ingress or routegroup. | Object | `Ingress`, `RouteGroup` | `>=1.14` | ### Example + +#### Ingress + This is an example of an HPA that will scale based on `requests-per-second` for an ingress called `myapp`. @@ -339,7 +346,7 @@ spec: - type: Object object: describedObject: - apiVersion: extensions/v1beta1 + apiVersion: networking.k8s.io/v1 kind: Ingress name: myapp metric: @@ -349,17 +356,49 @@ spec: type: AverageValue ``` +#### RouteGroup + +This is an example of an HPA that will scale based on `requests-per-second` for +a routegroup called `myapp`. + +```yaml +apiVersion: autoscaling/v2beta2 +kind: HorizontalPodAutoscaler +metadata: + name: myapp-hpa +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: myapp + minReplicas: 1 + maxReplicas: 10 + metrics: + - type: Object + object: + describedObject: + apiVersion: zalando.org/v1 + kind: RouteGroup + name: myapp + metric: + name: requests-per-second + target: + averageValue: "10" + type: AverageValue +``` + ### Metric weighting based on backend Skipper supports sending traffic to different backend based on annotations -present on the `Ingress` object. When the metric name is specified without a -backend as `requests-per-second` then the number of replicas will be calculated -based on the full traffic served by that ingress. If however only the traffic -being routed to a specific backend should be used then the backend name can be -specified as a metric name like `requests-per-second,backend1` which would -return the requests-per-second being sent to the `backend1`. The ingress -annotation where the backend weights can be obtained can be specified through -the flag `--skipper-backends-annotation`. +present on the `Ingress` object, or weights on the RouteGroup backends. When +the metric name is specified without a backend as `requests-per-second` then +the number of replicas will be calculated based on the full traffic served by +that ingress/routegroup. If however only the traffic being routed to a +specific backend should be used then the backend name can be specified as a +metric name like `requests-per-second,backend1` which would return the +requests-per-second being sent to the `backend1`. The ingress annotation where +the backend weights can be obtained can be specified through the flag +`--skipper-backends-annotation`. ## InfluxDB collector diff --git a/docs/rbac.yaml b/docs/rbac.yaml index e5ae229..67eab2b 100644 --- a/docs/rbac.yaml +++ b/docs/rbac.yaml @@ -64,12 +64,23 @@ rules: - statefulsets verbs: - get +# only relevant if running with the flag: +# --skipper-ingress-metrics - apiGroups: - extensions + - networking.k8s.io resources: - ingresses verbs: - get +# only relevant if running with the flag: +# --skipper-routegroup-metrics +- apiGroups: + - zalando.org + resources: + - routegroups + verbs: + - get - apiGroups: - autoscaling resources: diff --git a/go.mod b/go.mod index 569e2b4..b8b0fd5 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/spf13/cobra v1.1.3 github.com/spyzhov/ajson v0.4.2 github.com/stretchr/testify v1.7.0 + github.com/szuecs/routegroup-client v0.18.3 github.com/zalando-incubator/cluster-lifecycle-manager v0.0.0-20180921141935-824b77fb1f84 go.uber.org/zap v1.13.0 // indirect golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392 // indirect diff --git a/go.sum b/go.sum index 7a91010..fb0591c 100644 --- a/go.sum +++ b/go.sum @@ -24,13 +24,20 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= +github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -41,8 +48,10 @@ github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb0 github.com/NYTimes/gziphandler v1.0.1 h1:iLrQrdwjDd52kHDA5op2UBJFjmOb9g+7scBan4RN8F0= github.com/NYTimes/gziphandler v1.0.1/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= @@ -130,6 +139,7 @@ github.com/emicklei/go-restful-swagger12 v0.0.0-20201014110547-68ccff494617/go.m github.com/emirpasic/gods v1.9.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -155,18 +165,22 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0 h1:QvGt2nLcHH0WK9orKa+ppBPAxREcH364nPUedEpK0TY= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/jsonreference v0.19.5 h1:1WJP/wi4OjB4iV8KVbH73rQaoialJrqv8gitZLxGLtM= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= +github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= github.com/go-openapi/spec v0.20.0/go.mod h1:+81FIL1JwC5P3/Iuuozq3pPE9dXdIEGxFutcFKaVbmU= github.com/go-openapi/spec v0.20.3 h1:uH9RQ6vdyPSs2pSy9fL8QPspDF2AMIMPtmK5coSSjtQ= github.com/go-openapi/spec v0.20.3/go.mod h1:gG4F8wdEDN+YPBMVnzE85Rbhf+Th2DTvA9nFPQ5AYEg= +github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.12/go.mod h1:eFdyEBkTdoAf/9RXBvj4cr1nH7GD8Kzo5HTt47gr72M= @@ -194,6 +208,7 @@ github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -242,8 +257,11 @@ github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= +github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/goreleaser/goreleaser v0.94.0 h1:2CFMxMTLODjYfNOx2sADNzpgCwH9ltMqvQYtj+ntK1Q= github.com/goreleaser/goreleaser v0.94.0/go.mod h1:OjbYR2NhOI6AEUWCowMSBzo9nP1aRif3sYtx+rhp+Zo= @@ -314,6 +332,7 @@ github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2E github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -341,6 +360,7 @@ github.com/kubernetes-sigs/custom-metrics-apiserver v0.0.0-20201216091021-1b9fa9 github.com/kubernetes-sigs/custom-metrics-apiserver v0.0.0-20201216091021-1b9fa998bbaa/go.mod h1:o4psv/D+LJC+NGyL66BoKWXLkzlJeUqhL6/3rjFKXw0= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= @@ -497,6 +517,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/szuecs/routegroup-client v0.18.3 h1:8iEUhi0m1cOfkLEE+t/BMiY0rclJMWr7257tMw+F8mA= +github.com/szuecs/routegroup-client v0.18.3/go.mod h1:A3JsFP6Ql67IvVk6QdlCyi94bUB/SoGzobHLawnYiUg= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -534,11 +556,13 @@ go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392 h1:xYJJ3S178yv++9zXV/hnr29plCAGO9vAFG9dorqaFQc= @@ -577,6 +601,7 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -596,6 +621,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -604,6 +630,7 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200528225125-3c3fba18258b/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -626,6 +653,7 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -636,6 +664,7 @@ golang.org/x/sys v0.0.0-20181030150119-7e31e0c00fa0/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -651,6 +680,7 @@ golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -665,6 +695,7 @@ golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -677,6 +708,7 @@ golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -695,6 +727,7 @@ golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181221154417-3ad2d988d5e2/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -713,6 +746,7 @@ golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -848,12 +882,14 @@ honnef.co/go/tools v0.0.1-2019.2.2/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.18.3/go.mod h1:UOaMwERbqJMfeeeHc8XJKawj4P9TgDRnViIqqBeH2QA= k8s.io/api v0.20.0/go.mod h1:HyLC5l5eoS/ygQYl1BXBgFzWNlkHiAuyNAbevIn+FKg= k8s.io/api v0.20.2/go.mod h1:d7n6Ehyzx+S+cE3VhTGfVNNqtGc/oL9DCdYYahlurV8= k8s.io/api v0.20.9 h1:BKGV+rmHIyEVaJvWDYfqyfvHqdL/PhZalNr4mM/932Q= k8s.io/api v0.20.9/go.mod h1:wTKbf3LIlu+vuXqOk4Bi5drnSUtB10ou5XYrlgOuCdQ= k8s.io/apiextensions-apiserver v0.20.2 h1:rfrMWQ87lhd8EzQWRnbQ4gXrniL/yTRBgYH1x1+BLlo= k8s.io/apiextensions-apiserver v0.20.2/go.mod h1:F6TXp389Xntt+LUq3vw6HFOLttPa0V8821ogLGwb6Zs= +k8s.io/apimachinery v0.18.3/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko= k8s.io/apimachinery v0.20.0/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.2/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.9 h1:3e+0NczVYKqsEDiuxA+0a50t27fCyhyI7vSrmfaMpLg= @@ -862,10 +898,14 @@ k8s.io/apiserver v0.20.0/go.mod h1:6gRIWiOkvGvQt12WTYmsiYoUyYW0FXSiMdNl4m+sxY8= k8s.io/apiserver v0.20.2/go.mod h1:2nKd93WyMhZx4Hp3RfgH2K5PhwyTrprrkWYnI7id7jA= k8s.io/apiserver v0.20.9 h1:N8X4pCLES2KiWlnOSeBfhRrN4g9BST2GfN7eKcILmyg= k8s.io/apiserver v0.20.9/go.mod h1:3+0YiSQoofmDd/MaZ/3v6Kc4jpJ7J84lrOwKK95mnJQ= +k8s.io/client-go v0.18.3/go.mod h1:4a/dpQEvzAhT1BbuWW09qvIaGw6Gbu1gZYiQZIi1DMw= k8s.io/client-go v0.20.0/go.mod h1:4KWh/g+Ocd8KkCwKF8vUNnmqgv+EVnQDK4MBF4oB5tY= +k8s.io/client-go v0.20.0/go.mod h1:4KWh/g+Ocd8KkCwKF8vUNnmqgv+EVnQDK4MBF4oB5tY= +k8s.io/client-go v0.20.2/go.mod h1:kH5brqWqp7HDxUFKoEgiI4v8G1xzbe9giaCenUWJzgE= k8s.io/client-go v0.20.2/go.mod h1:kH5brqWqp7HDxUFKoEgiI4v8G1xzbe9giaCenUWJzgE= k8s.io/client-go v0.20.9 h1:Th0Ccrpq8nEuVOzvgn7GhdjUSLCGB92AUxSfUURstDU= k8s.io/client-go v0.20.9/go.mod h1:SjslwSB3f2wb/RwvGMfPIwsiBTPnD/Hp1xBGlz6U3t8= +k8s.io/code-generator v0.18.2/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc= k8s.io/code-generator v0.20.0/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= k8s.io/code-generator v0.20.2/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= k8s.io/code-generator v0.20.9 h1:zOk9CzTi4zVcIJG4Azf9EOhTnS43FmxnQStSU+UfCm4= @@ -874,20 +914,28 @@ k8s.io/component-base v0.20.0/go.mod h1:wKPj+RHnAr8LW2EIBIK7AxOHPde4gme2lzXwVSoR k8s.io/component-base v0.20.2/go.mod h1:pzFtCiwe/ASD0iV7ySMu8SYVJjCapNM9bjvk7ptpKh0= k8s.io/component-base v0.20.9 h1:xd8Mc1iR7l/wmDDmU1nGcxcZ88k8WOoLAcOUcVNj76U= k8s.io/component-base v0.20.9/go.mod h1:uieRi3vo5XxE0xrR/dNXmLSNIsdyNalBf+nBROmw6dc= +k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20201113003025-83324d819ded h1:JApXBKYyB7l9xx+DK7/+mFjC7A9Bt5A93FPvFD0HIFE= k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.4.0 h1:7+X0fUguPyrKEC4WjH8iGDg3laWgMo5tMnRTIGTTxGQ= k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= +k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd h1:sOHNzJIkytDF6qadMNKhhDRpc6ODik8lVC6nOur7B2c= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= k8s.io/metrics v0.20.0/go.mod h1:9yiRhfr8K8sjdj2EthQQE9WvpYDvsXIV3CjN4Ruq4Jw= k8s.io/metrics v0.20.9 h1:1CNMWYZhcZVkYz0cTladpGxeRyawf423TKIfKEd6P6Y= k8s.io/metrics v0.20.9/go.mod h1:2pG37GvYJWdfsIqqNX7CDm2W4W71JAg+OiQAEsfpXVs= +k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= +k8s.io/utils v0.0.0-20200520001619-278ece378a50/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= @@ -898,6 +946,8 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19 h1:0jaDAAxtqIrrq sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/controller-tools v0.5.0 h1:3u2RCwOlp0cjCALAigpOcbAf50pE+kHSdueUosrC/AE= sigs.k8s.io/controller-tools v0.5.0/go.mod h1:JTsstrMpxs+9BUj6eGuAaEb6SDSPTeVtUyp0jmnAM/I= +sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= +sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno= sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= diff --git a/pkg/collector/skipper_collector.go b/pkg/collector/skipper_collector.go index 3ffffd9..6a6d5fa 100644 --- a/pkg/collector/skipper_collector.go +++ b/pkg/collector/skipper_collector.go @@ -10,6 +10,8 @@ import ( "strings" "time" + rgv1 "github.com/szuecs/routegroup-client/apis/zalando.org/v1" + rginterface "github.com/szuecs/routegroup-client/client/clientset/versioned" autoscalingv2 "k8s.io/api/autoscaling/v2beta2" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -31,14 +33,16 @@ var ( // collectors for getting skipper ingress metrics. type SkipperCollectorPlugin struct { client kubernetes.Interface + rgClient rginterface.Interface plugin CollectorPlugin backendAnnotations []string } // NewSkipperCollectorPlugin initializes a new SkipperCollectorPlugin. -func NewSkipperCollectorPlugin(client kubernetes.Interface, prometheusPlugin *PrometheusCollectorPlugin, backendAnnotations []string) (*SkipperCollectorPlugin, error) { +func NewSkipperCollectorPlugin(client kubernetes.Interface, rgClient rginterface.Interface, prometheusPlugin *PrometheusCollectorPlugin, backendAnnotations []string) (*SkipperCollectorPlugin, error) { return &SkipperCollectorPlugin{ client: client, + rgClient: rgClient, plugin: prometheusPlugin, backendAnnotations: backendAnnotations, }, nil @@ -54,7 +58,7 @@ func (c *SkipperCollectorPlugin) NewCollector(hpa *autoscalingv2.HorizontalPodAu backend = metricNameParts[1] } } - return NewSkipperCollector(c.client, c.plugin, hpa, config, interval, c.backendAnnotations, backend) + return NewSkipperCollector(c.client, c.rgClient, c.plugin, hpa, config, interval, c.backendAnnotations, backend) } return nil, fmt.Errorf("metric '%s' not supported", config.Metric.Name) } @@ -63,6 +67,7 @@ func (c *SkipperCollectorPlugin) NewCollector(hpa *autoscalingv2.HorizontalPodAu // It depends on the prometheus collector for getting the metrics. type SkipperCollector struct { client kubernetes.Interface + rgClient rginterface.Interface metric autoscalingv2.MetricIdentifier objectReference custom_metrics.ObjectReference hpa *autoscalingv2.HorizontalPodAutoscaler @@ -74,9 +79,10 @@ type SkipperCollector struct { } // NewSkipperCollector initializes a new SkipperCollector. -func NewSkipperCollector(client kubernetes.Interface, plugin CollectorPlugin, hpa *autoscalingv2.HorizontalPodAutoscaler, config *MetricConfig, interval time.Duration, backendAnnotations []string, backend string) (*SkipperCollector, error) { +func NewSkipperCollector(client kubernetes.Interface, rgClient rginterface.Interface, plugin CollectorPlugin, hpa *autoscalingv2.HorizontalPodAutoscaler, config *MetricConfig, interval time.Duration, backendAnnotations []string, backend string) (*SkipperCollector, error) { return &SkipperCollector{ client: client, + rgClient: rgClient, objectReference: config.ObjectReference, hpa: hpa, metric: config.Metric, @@ -128,26 +134,65 @@ func getWeights(ingressAnnotations map[string]string, backendAnnotations []strin return 0.0, errBackendNameMissing } -// getCollector returns a collector for getting the metrics. -func (c *SkipperCollector) getCollector() (Collector, error) { - ingress, err := c.client.NetworkingV1beta1().Ingresses(c.objectReference.Namespace).Get(context.TODO(), c.objectReference.Name, metav1.GetOptions{}) - if err != nil { - return nil, err +func getRouteGroupWeight(backends []rgv1.RouteGroupBackendReference, backendName string) (float64, error) { + if len(backends) <= 1 { + return 1.0, nil } - backendWeight, err := getWeights(ingress.Annotations, c.backendAnnotations, c.backend) - if err != nil { - return nil, err + for _, backend := range backends { + if backend.BackendName == backendName { + return float64(backend.Weight) / 100.0, nil + } } + + if backendName != "" { + return 0.0, nil + } + + return 0.0, errBackendNameMissing +} + +// getCollector returns a collector for getting the metrics. +func (c *SkipperCollector) getCollector(ctx context.Context) (Collector, error) { + var escapedHostnames []string + var backendWeight float64 + switch c.objectReference.Kind { + case "Ingress": + ingress, err := c.client.NetworkingV1beta1().Ingresses(c.objectReference.Namespace).Get(ctx, c.objectReference.Name, metav1.GetOptions{}) + if err != nil { + return nil, err + } + + backendWeight, err = getWeights(ingress.Annotations, c.backendAnnotations, c.backend) + if err != nil { + return nil, err + } + + for _, rule := range ingress.Spec.Rules { + escapedHostnames = append(escapedHostnames, regexp.QuoteMeta(strings.Replace(rule.Host, ".", "_", -1))) + } + case "RouteGroup": + routegroup, err := c.rgClient.ZalandoV1().RouteGroups(c.objectReference.Namespace).Get(ctx, c.objectReference.Name, metav1.GetOptions{}) + if err != nil { + return nil, err + } + + backendWeight, err = getRouteGroupWeight(routegroup.Spec.DefaultBackends, c.backend) + if err != nil { + return nil, err + } + + for _, host := range routegroup.Spec.Hosts { + escapedHostnames = append(escapedHostnames, regexp.QuoteMeta(strings.Replace(host, ".", "_", -1))) + } + default: + return nil, fmt.Errorf("unknown skipper resource kind %s for resource %s/%s", c.objectReference.Kind, c.objectReference.Namespace, c.objectReference.Name) + } + config := c.config - var escapedHostnames []string - for _, rule := range ingress.Spec.Rules { - escapedHostnames = append(escapedHostnames, regexp.QuoteMeta(strings.Replace(rule.Host, ".", "_", -1))) - } - if len(escapedHostnames) == 0 { - return nil, fmt.Errorf("no hosts defined on ingress %s/%s, unable to create collector", c.objectReference.Namespace, c.objectReference.Name) + return nil, fmt.Errorf("no hosts defined on %s %s/%s, unable to create collector", c.objectReference.Kind, c.objectReference.Namespace, c.objectReference.Name) } config.Config = map[string]string{ @@ -165,7 +210,7 @@ func (c *SkipperCollector) getCollector() (Collector, error) { // GetMetrics gets skipper metrics from prometheus. func (c *SkipperCollector) GetMetrics() ([]CollectedMetric, error) { - collector, err := c.getCollector() + collector, err := c.getCollector(context.TODO()) if err != nil { return nil, err } diff --git a/pkg/collector/skipper_collector_test.go b/pkg/collector/skipper_collector_test.go index c31d739..8f610a0 100644 --- a/pkg/collector/skipper_collector_test.go +++ b/pkg/collector/skipper_collector_test.go @@ -8,6 +8,9 @@ import ( "time" "github.com/stretchr/testify/require" + rgv1 "github.com/szuecs/routegroup-client/apis/zalando.org/v1" + rginterface "github.com/szuecs/routegroup-client/client/clientset/versioned" + rgfake "github.com/szuecs/routegroup-client/client/clientset/versioned/fake" appsv1 "k8s.io/api/apps/v1" autoscalingv2 "k8s.io/api/autoscaling/v2beta2" corev1 "k8s.io/api/core/v1" @@ -100,12 +103,12 @@ func newStatefulSet(client *fake.Clientset, namespace string, name string) (*app }, metav1.CreateOptions{}) } -func TestSkipperCollector(t *testing.T) { +func TestSkipperCollectorIngress(t *testing.T) { for _, tc := range []struct { msg string metric int backend string - ingressName string + resourceName string hostnames []string expectedQuery string collectedMetric int @@ -120,7 +123,7 @@ func TestSkipperCollector(t *testing.T) { { msg: "test unweighted hpa", metric: 1000, - ingressName: "dummy-ingress", + resourceName: "dummy-ingress", hostnames: []string{"example.org"}, expectedQuery: `scalar(sum(rate(skipper_serve_host_duration_seconds_count{host=~"example_org"}[1m])) * 1.0000)`, collectedMetric: 1000, @@ -132,7 +135,7 @@ func TestSkipperCollector(t *testing.T) { { msg: "test weighted backend", metric: 1000, - ingressName: "dummy-ingress", + resourceName: "dummy-ingress", hostnames: []string{"example.org"}, expectedQuery: `scalar(sum(rate(skipper_serve_host_duration_seconds_count{host=~"example_org"}[1m])) * 0.4000)`, collectedMetric: 1000, @@ -146,7 +149,7 @@ func TestSkipperCollector(t *testing.T) { { msg: "test multiple hostnames", metric: 1000, - ingressName: "dummy-ingress", + resourceName: "dummy-ingress", hostnames: []string{"example.org", "foo.bar.com", "test.org"}, expectedQuery: `scalar(sum(rate(skipper_serve_host_duration_seconds_count{host=~"example_org|foo_bar_com|test_org"}[1m])) * 0.4000)`, collectedMetric: 1000, @@ -160,7 +163,7 @@ func TestSkipperCollector(t *testing.T) { { msg: "test multiple replicas", metric: 1000, - ingressName: "dummy-ingress", + resourceName: "dummy-ingress", hostnames: []string{"example.org"}, expectedQuery: `scalar(sum(rate(skipper_serve_host_duration_seconds_count{host=~"example_org"}[1m])) * 0.5000)`, collectedMetric: 200, @@ -175,7 +178,7 @@ func TestSkipperCollector(t *testing.T) { { msg: "test multiple replicas not calculating average internally", metric: 1500, - ingressName: "dummy-ingress", + resourceName: "dummy-ingress", hostnames: []string{"example.org"}, expectedQuery: `scalar(sum(rate(skipper_serve_host_duration_seconds_count{host=~"example_org"}[1m])) * 0.5000)`, collectedMetric: 1500, @@ -189,7 +192,7 @@ func TestSkipperCollector(t *testing.T) { { msg: "test zero weight backends", metric: 0, - ingressName: "dummy-ingress", + resourceName: "dummy-ingress", hostnames: []string{"example.org"}, expectedQuery: `scalar(sum(rate(skipper_serve_host_duration_seconds_count{host=~"example_org"}[1m])) * 0.0000)`, collectedMetric: 0, @@ -203,7 +206,7 @@ func TestSkipperCollector(t *testing.T) { { msg: "test multiple backend annotation", metric: 1500, - ingressName: "dummy-ingress", + resourceName: "dummy-ingress", hostnames: []string{"example.org"}, expectedQuery: `scalar(sum(rate(skipper_serve_host_duration_seconds_count{host=~"example_org"}[1m])) * 1.0000)`, collectedMetric: 300, @@ -221,7 +224,7 @@ func TestSkipperCollector(t *testing.T) { { msg: "test multiple backend annotation not calculating average internally", metric: 1500, - ingressName: "dummy-ingress", + resourceName: "dummy-ingress", hostnames: []string{"example.org"}, expectedQuery: `scalar(sum(rate(skipper_serve_host_duration_seconds_count{host=~"example_org"}[1m])) * 1.0000)`, collectedMetric: 1500, @@ -238,7 +241,7 @@ func TestSkipperCollector(t *testing.T) { { msg: "test backend is not set", metric: 0, - ingressName: "dummy-ingress", + resourceName: "dummy-ingress", hostnames: []string{"example.org"}, expectedQuery: `scalar(sum(rate(skipper_serve_host_duration_seconds_count{host=~"example_org"}[1m])) * 0.0000)`, collectedMetric: 0, @@ -252,7 +255,7 @@ func TestSkipperCollector(t *testing.T) { { msg: "test no annotations set", metric: 1500, - ingressName: "dummy-ingress", + resourceName: "dummy-ingress", hostnames: []string{"example.org"}, expectedQuery: `scalar(sum(rate(skipper_serve_host_duration_seconds_count{host=~"example_org"}[1m])) * 1.0000)`, collectedMetric: 1500, @@ -266,7 +269,7 @@ func TestSkipperCollector(t *testing.T) { { msg: "test annotations are set but backend is missing", metric: 1500, - ingressName: "dummy-ingress", + resourceName: "dummy-ingress", hostnames: []string{"example.org"}, expectedQuery: `scalar(sum(rate(skipper_serve_host_duration_seconds_count{host=~"example_org"}[1m])) * 1.0000)`, expectError: true, @@ -280,7 +283,7 @@ func TestSkipperCollector(t *testing.T) { { msg: "test annotations are missing and backend is unset", metric: 1500, - ingressName: "dummy-ingress", + resourceName: "dummy-ingress", hostnames: []string{"example.org"}, expectedQuery: `scalar(sum(rate(skipper_serve_host_duration_seconds_count{host=~"example_org"}[1m])) * 1.0000)`, collectedMetric: 1500, @@ -294,7 +297,7 @@ func TestSkipperCollector(t *testing.T) { { msg: "test partial backend annotations", metric: 1500, - ingressName: "dummy-ingress", + resourceName: "dummy-ingress", hostnames: []string{"example.org"}, expectedQuery: `scalar(sum(rate(skipper_serve_host_duration_seconds_count{host=~"example_org"}[1m])) * 0.2000)`, collectedMetric: 300, @@ -312,7 +315,7 @@ func TestSkipperCollector(t *testing.T) { { msg: "test partial backend annotations not calculating average internally", metric: 1500, - ingressName: "dummy-ingress", + resourceName: "dummy-ingress", hostnames: []string{"example.org"}, expectedQuery: `scalar(sum(rate(skipper_serve_host_duration_seconds_count{host=~"example_org"}[1m])) * 0.2050)`, collectedMetric: 1500, @@ -329,14 +332,14 @@ func TestSkipperCollector(t *testing.T) { } { t.Run(tc.msg, func(t *testing.T) { client := fake.NewSimpleClientset() - err := makeIngress(client, tc.namespace, tc.ingressName, tc.backend, tc.hostnames, tc.backendWeights) + err := makeIngress(client, tc.namespace, tc.resourceName, tc.backend, tc.hostnames, tc.backendWeights) require.NoError(t, err) - plugin := makePlugin(tc.metric) - hpa := makeIngressHPA(tc.namespace, tc.ingressName, tc.backend) - config := makeConfig(tc.ingressName, tc.namespace, tc.backend, tc.fakedAverage) + hpa := makeIngressHPA(tc.namespace, tc.resourceName, tc.backend) _, err = newDeployment(client, tc.namespace, tc.backend, tc.replicas, tc.readyReplicas) + plugin := makePlugin(tc.metric) + config := makeConfig(tc.resourceName, tc.namespace, hpa.Spec.Metrics[0].Object.DescribedObject.Kind, tc.backend, tc.fakedAverage) require.NoError(t, err) - collector, err := NewSkipperCollector(client, plugin, hpa, config, time.Minute, tc.backendAnnotations, tc.backend) + collector, err := NewSkipperCollector(client, nil, plugin, hpa, config, time.Minute, tc.backendAnnotations, tc.backend) require.NoError(t, err, "failed to create skipper collector: %v", err) collected, err := collector.GetMetrics() if tc.expectError { @@ -352,7 +355,190 @@ func TestSkipperCollector(t *testing.T) { } } -func makeIngress(client kubernetes.Interface, namespace, ingressName, backend string, hostnames []string, backendWeights map[string]map[string]float64) error { +func TestSkipperCollector(t *testing.T) { + for _, tc := range []struct { + msg string + metric int + backend string + resourceName string + hostnames []string + expectedQuery string + collectedMetric int + expectError bool + fakedAverage bool + namespace string + backendWeights map[string]float64 + replicas int32 + readyReplicas int32 + }{ + { + msg: "test unweighted hpa", + metric: 1000, + resourceName: "dummy-ingress", + hostnames: []string{"example.org"}, + expectedQuery: `scalar(sum(rate(skipper_serve_host_duration_seconds_count{host=~"example_org"}[1m])) * 1.0000)`, + collectedMetric: 1000, + namespace: "default", + backend: "dummy-backend", + replicas: 1, + readyReplicas: 1, + }, + { + msg: "test weighted backend", + metric: 1000, + resourceName: "dummy-ingress", + hostnames: []string{"example.org"}, + expectedQuery: `scalar(sum(rate(skipper_serve_host_duration_seconds_count{host=~"example_org"}[1m])) * 0.4000)`, + collectedMetric: 1000, + namespace: "default", + backend: "backend1", + backendWeights: map[string]float64{"backend2": 60.0, "backend1": 40}, + replicas: 1, + readyReplicas: 1, + }, + { + msg: "test multiple hostnames", + metric: 1000, + resourceName: "dummy-ingress", + hostnames: []string{"example.org", "foo.bar.com", "test.org"}, + expectedQuery: `scalar(sum(rate(skipper_serve_host_duration_seconds_count{host=~"example_org|foo_bar_com|test_org"}[1m])) * 0.4000)`, + collectedMetric: 1000, + namespace: "default", + backend: "backend1", + backendWeights: map[string]float64{"backend2": 60, "backend1": 40}, + replicas: 1, + readyReplicas: 1, + }, + { + msg: "test multiple replicas", + metric: 1000, + resourceName: "dummy-ingress", + hostnames: []string{"example.org"}, + expectedQuery: `scalar(sum(rate(skipper_serve_host_duration_seconds_count{host=~"example_org"}[1m])) * 0.5000)`, + collectedMetric: 200, + fakedAverage: true, + namespace: "default", + backend: "backend1", + backendWeights: map[string]float64{"backend2": 50, "backend1": 50}, + replicas: 5, + readyReplicas: 5, + }, + { + msg: "test multiple replicas not calculating average internally", + metric: 1500, + resourceName: "dummy-ingress", + hostnames: []string{"example.org"}, + expectedQuery: `scalar(sum(rate(skipper_serve_host_duration_seconds_count{host=~"example_org"}[1m])) * 0.5000)`, + collectedMetric: 1500, + namespace: "default", + backend: "backend1", + backendWeights: map[string]float64{"backend2": 50, "backend1": 50}, + replicas: 5, // this is not taken into account + readyReplicas: 5, + }, + { + msg: "test zero weight backends", + metric: 0, + resourceName: "dummy-ingress", + hostnames: []string{"example.org"}, + expectedQuery: `scalar(sum(rate(skipper_serve_host_duration_seconds_count{host=~"example_org"}[1m])) * 0.0000)`, + collectedMetric: 0, + namespace: "default", + backend: "backend1", + backendWeights: map[string]float64{"backend2": 100, "backend1": 0}, + replicas: 5, + readyReplicas: 5, + }, + { + msg: "test backend is not set", + metric: 0, + resourceName: "dummy-ingress", + hostnames: []string{"example.org"}, + expectedQuery: `scalar(sum(rate(skipper_serve_host_duration_seconds_count{host=~"example_org"}[1m])) * 0.0000)`, + collectedMetric: 0, + namespace: "default", + backend: "backend3", + backendWeights: map[string]float64{"backend2": 100, "backend1": 0}, + replicas: 1, + readyReplicas: 1, + }, + { + msg: "test no annotations set", + metric: 1500, + resourceName: "dummy-ingress", + hostnames: []string{"example.org"}, + expectedQuery: `scalar(sum(rate(skipper_serve_host_duration_seconds_count{host=~"example_org"}[1m])) * 1.0000)`, + collectedMetric: 1500, + namespace: "default", + backend: "backend3", + backendWeights: map[string]float64{}, + replicas: 1, + readyReplicas: 1, + }, + { + msg: "test annotations are set but backend is missing", + metric: 1500, + resourceName: "dummy-ingress", + hostnames: []string{"example.org"}, + expectedQuery: `scalar(sum(rate(skipper_serve_host_duration_seconds_count{host=~"example_org"}[1m])) * 1.0000)`, + expectError: true, + namespace: "default", + backend: "", + backendWeights: map[string]float64{"backend2": 100, "backend1": 0}, + replicas: 1, + readyReplicas: 1, + }, + { + msg: "test annotations are missing and backend is unset", + metric: 1500, + resourceName: "dummy-ingress", + hostnames: []string{"example.org"}, + expectedQuery: `scalar(sum(rate(skipper_serve_host_duration_seconds_count{host=~"example_org"}[1m])) * 1.0000)`, + collectedMetric: 1500, + namespace: "default", + backend: "", + backendWeights: nil, + replicas: 1, + readyReplicas: 1, + }, + } { + t.Run(tc.msg, func(t *testing.T) { + client := fake.NewSimpleClientset() + backendWeights := make(map[string]map[string]float64) + if len(tc.backendWeights) > 0 { + backendWeights = map[string]map[string]float64{testBackendWeightsAnnotation: tc.backendWeights} + } + err := makeIngress(client, tc.namespace, tc.resourceName, tc.backend, tc.hostnames, backendWeights) + require.NoError(t, err) + rgClient := rgfake.NewSimpleClientset() + err = makeRoutegroup(rgClient, tc.namespace, tc.resourceName, tc.hostnames, tc.backendWeights) + require.NoError(t, err) + ingressHPA := makeIngressHPA(tc.namespace, tc.resourceName, tc.backend) + rgHPA := makeRGHPA(tc.namespace, tc.resourceName, tc.backend) + _, err = newDeployment(client, tc.namespace, tc.backend, tc.replicas, tc.readyReplicas) + for _, hpa := range []*autoscalingv2.HorizontalPodAutoscaler{ingressHPA, rgHPA} { + kind := hpa.Spec.Metrics[0].Object.DescribedObject.Kind + plugin := makePlugin(tc.metric) + config := makeConfig(tc.resourceName, tc.namespace, kind, tc.backend, tc.fakedAverage) + require.NoError(t, err) + collector, err := NewSkipperCollector(client, rgClient, plugin, hpa, config, time.Minute, []string{testBackendWeightsAnnotation}, tc.backend) + require.NoError(t, err, "failed to create skipper collector: %v", err) + collected, err := collector.GetMetrics() + if tc.expectError { + require.Error(t, err, "%s", kind) + } else { + require.NoError(t, err, "%s", kind) + require.Equal(t, map[string]string{"query": tc.expectedQuery}, plugin.config, "%s", kind) + require.NoError(t, err, "%s: failed to collect metrics: %v", kind, err) + require.Len(t, collected, 1, "%s: the number of metrics returned is not 1", kind) + require.EqualValues(t, tc.collectedMetric, collected[0].Custom.Value.Value(), "%s: the returned metric is not expected value", kind) + } + } + }) + } +} + +func makeIngress(client kubernetes.Interface, namespace, resourceName, backend string, hostnames []string, backendWeights map[string]map[string]float64) error { annotations := make(map[string]string) for anno, weights := range backendWeights { sWeights, err := json.Marshal(weights) @@ -363,7 +549,7 @@ func makeIngress(client kubernetes.Interface, namespace, ingressName, backend st } ingress := &v1beta1.Ingress{ ObjectMeta: metav1.ObjectMeta{ - Name: ingressName, + Name: resourceName, Annotations: annotations, }, Spec: v1beta1.IngressSpec{ @@ -387,7 +573,7 @@ func makeIngress(client kubernetes.Interface, namespace, ingressName, backend st return err } -func makeIngressHPA(namespace, ingressName, backend string) *autoscalingv2.HorizontalPodAutoscaler { +func makeIngressHPA(namespace, name, backend string) *autoscalingv2.HorizontalPodAutoscaler { return &autoscalingv2.HorizontalPodAutoscaler{ ObjectMeta: metav1.ObjectMeta{Namespace: namespace}, Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ @@ -399,7 +585,7 @@ func makeIngressHPA(namespace, ingressName, backend string) *autoscalingv2.Horiz { Type: autoscalingv2.ObjectMetricSourceType, Object: &autoscalingv2.ObjectMetricSource{ - DescribedObject: autoscalingv2.CrossVersionObjectReference{Name: ingressName, APIVersion: "extensions/v1", Kind: "Ingress"}, + DescribedObject: autoscalingv2.CrossVersionObjectReference{Name: name, APIVersion: "extensions/v1", Kind: "Ingress"}, Metric: autoscalingv2.MetricIdentifier{Name: fmt.Sprintf("%s,%s", rpsMetricName, backend)}, }, }, @@ -407,12 +593,54 @@ func makeIngressHPA(namespace, ingressName, backend string) *autoscalingv2.Horiz }, } } -func makeConfig(ingressName, namespace, backend string, fakedAverage bool) *MetricConfig { + +func makeRoutegroup(rgClient rginterface.Interface, namespace, resourceName string, hostnames []string, backendWeights map[string]float64) error { + var backends []rgv1.RouteGroupBackendReference + for backend, weight := range backendWeights { + backends = append(backends, rgv1.RouteGroupBackendReference{BackendName: backend, Weight: int(weight)}) + } + + rg := &rgv1.RouteGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: resourceName, + }, + Spec: rgv1.RouteGroupSpec{ + Hosts: hostnames, + DefaultBackends: backends, + }, + } + _, err := rgClient.ZalandoV1().RouteGroups(namespace).Create(context.TODO(), rg, metav1.CreateOptions{}) + return err +} + +func makeRGHPA(namespace, name, backend string) *autoscalingv2.HorizontalPodAutoscaler { + return &autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{Namespace: namespace}, + Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ + Kind: "Deployment", + Name: backend, + }, + Metrics: []autoscalingv2.MetricSpec{ + { + Type: autoscalingv2.ObjectMetricSourceType, + Object: &autoscalingv2.ObjectMetricSource{ + DescribedObject: autoscalingv2.CrossVersionObjectReference{Name: name, APIVersion: "zalando.org/v1", Kind: "RouteGroup"}, + Metric: autoscalingv2.MetricIdentifier{Name: fmt.Sprintf("%s,%s", rpsMetricName, backend)}, + }, + }, + }, + }, + } +} + +func makeConfig(resourceName, namespace, kind, backend string, fakedAverage bool) *MetricConfig { config := &MetricConfig{ MetricTypeName: MetricTypeName{Metric: autoscalingv2.MetricIdentifier{Name: fmt.Sprintf("%s,%s", rpsMetricName, backend)}}, ObjectReference: custom_metrics.ObjectReference{ - Name: ingressName, + Name: resourceName, Namespace: namespace, + Kind: kind, }, MetricSpec: autoscalingv2.MetricSpec{ Object: &autoscalingv2.ObjectMetricSource{ diff --git a/pkg/provider/metric_store.go b/pkg/provider/metric_store.go index bd26fa6..a979b98 100644 --- a/pkg/provider/metric_store.go +++ b/pkg/provider/metric_store.go @@ -82,6 +82,16 @@ func (s *MetricStore) insertCustomMetric(value custom_metrics.MetricValue) { Resource: "ingresses", Group: group, } + case "RouteGroup": + group := "zalando.org" + gv, err := schema.ParseGroupVersion(value.DescribedObject.APIVersion) + if err == nil { + group = gv.Group + } + groupResource = schema.GroupResource{ + Resource: "routegroups", + Group: group, + } case "ScalingSchedule": group := "zalando.org" gv, err := schema.ParseGroupVersion(value.DescribedObject.APIVersion) diff --git a/pkg/server/start.go b/pkg/server/start.go index cb5c3a0..11ea557 100644 --- a/pkg/server/start.go +++ b/pkg/server/start.go @@ -30,6 +30,7 @@ import ( "github.com/kubernetes-sigs/custom-metrics-apiserver/pkg/cmd/server" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/spf13/cobra" + rg "github.com/szuecs/routegroup-client/client/clientset/versioned" "github.com/zalando-incubator/cluster-lifecycle-manager/pkg/credentials-loader/platformiam" generatedopenapi "github.com/zalando-incubator/kube-metrics-adapter/pkg/api/generated/openapi" v1 "github.com/zalando-incubator/kube-metrics-adapter/pkg/apis/zalando.org/v1" @@ -113,6 +114,8 @@ func NewCommandStartAdapterServer(stopCh <-chan struct{}) *cobra.Command { "path to the credentials dir where tokens are stored") flags.BoolVar(&o.SkipperIngressMetrics, "skipper-ingress-metrics", o.SkipperIngressMetrics, ""+ "whether to enable skipper ingress metrics") + flags.BoolVar(&o.SkipperRouteGroupMetrics, "skipper-routegroup-metrics", o.SkipperRouteGroupMetrics, ""+ + "whether to enable skipper routegroup metrics") flags.StringArrayVar(&o.SkipperBackendWeightAnnotation, "skipper-backends-annotation", o.SkipperBackendWeightAnnotation, ""+ "the annotation to get backend weights so that the returned metric can be weighted") flags.BoolVar(&o.AWSExternalMetrics, "aws-external-metrics", o.AWSExternalMetrics, ""+ @@ -170,6 +173,11 @@ func (o AdapterServerOptions) RunCustomMetricsAdapterServer(stopCh <-chan struct return fmt.Errorf("failed to initialize new client: %v", err) } + rgClient, err := rg.NewForConfig(clientConfig) + if err != nil { + return fmt.Errorf("failed to initialize RouteGroup client: %v", err) + } + collectorFactory := collector.NewCollectorFactory() if o.PrometheusServer != "" { @@ -186,15 +194,24 @@ func (o AdapterServerOptions) RunCustomMetricsAdapterServer(stopCh <-chan struct collectorFactory.RegisterExternalCollector([]string{collector.PrometheusMetricType, collector.PrometheusMetricNameLegacy}, promPlugin) // skipper collector can only be enabled if prometheus is. - if o.SkipperIngressMetrics { - skipperPlugin, err := collector.NewSkipperCollectorPlugin(client, promPlugin, o.SkipperBackendWeightAnnotation) + if o.SkipperIngressMetrics || o.SkipperRouteGroupMetrics { + skipperPlugin, err := collector.NewSkipperCollectorPlugin(client, rgClient, promPlugin, o.SkipperBackendWeightAnnotation) if err != nil { return fmt.Errorf("failed to initialize skipper collector plugin: %v", err) } - err = collectorFactory.RegisterObjectCollector("Ingress", "", skipperPlugin) - if err != nil { - return fmt.Errorf("failed to register skipper collector plugin: %v", err) + if o.SkipperIngressMetrics { + err = collectorFactory.RegisterObjectCollector("Ingress", "", skipperPlugin) + if err != nil { + return fmt.Errorf("failed to register skipper Ingress collector plugin: %v", err) + } + } + + if o.SkipperRouteGroupMetrics { + err = collectorFactory.RegisterObjectCollector("RouteGroup", "", skipperPlugin) + if err != nil { + return fmt.Errorf("failed to register skipper RouteGroup collector plugin: %v", err) + } } } } @@ -390,6 +407,9 @@ type AdapterServerOptions struct { // SkipperIngressMetrics switches on support for skipper ingress based // metric collection. SkipperIngressMetrics bool + // SkipperRouteGroupMetrics switches on support for skipper routegroup + // based metric collection. + SkipperRouteGroupMetrics bool // AWSExternalMetrics switches on support for getting external metrics // from AWS. AWSExternalMetrics bool