Update SECURITY.md (#84)

Signed-off-by: Mikkel Oscar Lyderik Larsen <mikkel.larsen@zalando.de>

.travis.yml

@@ -2,14 +2,14 @@ language: go
 dist: xenial
 
 go:
-  - "1.11.x"
+  - "1.13.x"
 
 env:
 - GO111MODULE=on GOLANGCI_RELEASE="v1.16.0"
 
 before_install:
-  - go get github.com/mattn/goveralls
-  - go get github.com/lawrencewoodman/roveralls
+  - GO111MODULE=off go get github.com/mattn/goveralls
+  - GO111MODULE=off go get github.com/lawrencewoodman/roveralls
   - curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin ${GOLANGCI_RELEASE}
 
 script:

README.md

@@ -1,10 +1,13 @@
 # kube-metrics-adapter
 [![Build Status](https://travis-ci.org/zalando-incubator/kube-metrics-adapter.svg?branch=master)](https://travis-ci.org/zalando-incubator/kube-metrics-adapter)
 [![Coverage Status](https://coveralls.io/repos/github/zalando-incubator/kube-metrics-adapter/badge.svg?branch=master)](https://coveralls.io/github/zalando-incubator/kube-metrics-adapter?branch=master)
+
 Kube Metrics Adapter is a general purpose metrics adapter for Kubernetes that
 can collect and serve custom and external metrics for Horizontal Pod
 Autoscaling.
 
+It supports scaling based on [Prometheus metrics](https://prometheus.io/), [SQS queues](https://aws.amazon.com/sqs/) and others out of the box.
+
 It discovers Horizontal Pod Autoscaling resources and starts to collect the
 requested metrics and stores them in memory. It's implemented using the
 [custom-metrics-apiserver](https://github.com/kubernetes-incubator/custom-metrics-apiserver)
@@ -41,6 +44,18 @@ The `metric-config.*` annotations are used by the `kube-metrics-adapter` to
 configure a collector for getting the metrics. In the above example it
 configures a *json-path pod collector*.
 
+## Kubernetes compatibility
+
+Like the [support
+policy](https://kubernetes.io/docs/setup/release/version-skew-policy/) offered
+for Kubernetes, this project aims to support the latest three minor releases of
+Kubernetes.
+
+Currently the default supported API is `autoscaling/v2beta1`. However we aim to
+move to `autoscaling/v2beta2` (available since `v1.12`) in the near future as
+this adds a lot of improvements over `v2beta1`. The move to `v2beta2` will most
+likely happen as soon as [GKE adds support for it](https://issuetracker.google.com/issues/135624588).
+
 ## Building
 
 This project uses [Go modules](https://github.com/golang/go/wiki/Modules) as
@@ -71,9 +86,9 @@ Currently only `json-path` collection is supported.
 
 ### Supported metrics
 
-| Metric | Description | Type |
-| ------------ | -------------- | ------- |
-| *custom* | No predefined metrics. Metrics are generated from user defined queries. | Pods |
+| Metric | Description | Type | K8s Versions |
+| ------------ | -------------- | ------- | -- |
+| *custom* | No predefined metrics. Metrics are generated from user defined queries. | Pods | `>=1.10` |
 
 ### Example
 
@@ -151,10 +166,10 @@ the trade-offs between the two approaches.
 
 ### Supported metrics
 
-| Metric | Description | Type | Kind |
-| ------------ | -------------- | ------- | -- |
-| `prometheus-query` | Generic metric which requires a user defined query. | External | |
-| *custom* | No predefined metrics. Metrics are generated from user defined queries. | Object | *any* |
+| Metric | Description | Type | Kind | K8s Versions |
+| ------------ | -------------- | ------- | -- | -- |
+| `prometheus-query` | Generic metric which requires a user defined query. | External | | `>=1.10` |
+| *custom* | No predefined metrics. Metrics are generated from user defined queries. | Object | *any* | `>=1.10` |
 
 ### Example: External Metric
 
@@ -172,6 +187,10 @@ kind: HorizontalPodAutoscaler
 metadata:
   name: myapp-hpa
   annotations:
+    # This annotation is optional.
+    # If specified, then this prometheus server is used,
+    # instead of the prometheus server specified as the CLI argument `--prometheus-server`.
+    metric-config.external.prometheus-query.prometheus/prometheus-server: http://prometheus.my-namespace.svc
     # metric-config.<metricType>.<metricName>.<collectorName>/<configKey>
     # <configKey> == query-name
     metric-config.external.prometheus-query.prometheus/processed-events-per-second: |
@@ -255,9 +274,9 @@ box so users don't have to define those manually.
 
 ### Supported metrics
 
-| Metric | Description | Type | Kind |
-| ----------- | -------------- | ------ | ---- |
-| `requests-per-second` | Scale based on requests per second for a certain ingress. | Object | `Ingress` |
+| Metric | Description | Type | Kind | K8s Versions |
+| ----------- | -------------- | ------ | ---- | ---- |
+| `requests-per-second` | Scale based on requests per second for a certain ingress. | Object | `Ingress` | `>=1.14` (can work with `>=1.10`) |
 
 ### Example
 
@@ -284,7 +303,10 @@ spec:
         apiVersion: extensions/v1beta1
         kind: Ingress
         name: myapp
-      targetValue: 10 # this will be treated as targetAverageValue
+      averageValue: 10 # Only works with Kubernetes >=1.14
+      # for Kubernetes <1.14 you can use `targetValue` instead:
+      targetValue: 10 # this must be set, but has no effect if `averageValue` is defined.
+                      # Otherwise it will be treated as targetAverageValue
 ```
 
 ### Metric weighting based on backend
@@ -298,13 +320,17 @@ return the requests-per-second being sent to the `backend1`. The ingress annotat
 the backend weights can be obtained can be specified through the flag `--skipper-backends-annotation`.
 
 
-**Note:** As of Kubernetes v1.10 the HPA does not support `targetAverageValue` for
+**Note:** For Kubernetes `<v1.14` the HPA does not support `averageValue` for
 metrics of type `Object`. In case of requests per second it does not make sense
 to scale on a summed value because you can not make the total requests per
 second go down by adding more pods. For this reason the skipper collector will
 automatically treat the value you define in `targetValue` as an average per pod
 instead of a total sum.
 
+**ONLY use `targetValue` if you are on Kubernetes
+`<1.14`, it is not as percise as using `averageValue` and will not be supported
+after Kubernetes `v1.16` is released according to the [support policy](https://kubernetes.io/docs/setup/release/version-skew-policy/).**
+
 ## AWS collector
 
 The AWS collector allows scaling based on external metrics exposed by AWS
@@ -336,9 +362,9 @@ PolicyDocument:
 
 ### Supported metrics
 
-| Metric | Description | Type |
-| ------------ | ------- | -- |
-| `sqs-queue-length` | Scale based on SQS queue length | External |
+| Metric | Description | Type | K8s Versions |
+| ------------ | ------- | -- | -- |
+| `sqs-queue-length` | Scale based on SQS queue length | External | `>=1.10` |
 
 ### Example
 
@@ -384,9 +410,9 @@ The ZMON collector allows scaling based on external metrics exposed by
 
 ### Supported metrics
 
-| Metric | Description | Type |
-| ------------ | ------- | -- |
-| `zmon-check` | Scale based on any ZMON check results | External |
+| Metric | Description | Type | K8s Versions |
+| ------------ | ------- | -- | -- |
+| `zmon-check` | Scale based on any ZMON check results | External | `>=1.10` |
 
 ### Example
 

SECURITY.md

@@ -1,7 +1,8 @@
 We acknowledge that every line of code that we write may potentially contain security issues.
+We are trying to deal with it responsibly and provide patches as quickly as possible.
 
-We are trying to deal with it responsibly and provide patches as quickly as  possible. If you have anything to report to us please use the following channels:
+We host our bug bounty program on HackerOne, it is currently private, therefore if you would like to report a vulnerability and get rewarded for it, please ask to join our program by filling this form:
 
-Email: Tech-Security@zalando.de
-OR
-Submit your vulnerability report through our bug bounty program at: https://hackerone.com/zalando
+https://corporate.zalando.com/en/services-and-contact#security-form
+
+You can also send you report via this form if you do not want to join our bug bounty program and just want to report a vulnerability or security issue.

delivery.yaml

@@ -7,6 +7,8 @@ pipeline:
     - /go/pkg/mod       # pkg cache for Go modules
     - ~/.cache/go-build # Go build cache
   type: script
+  env:
+    GOFLAGS: "-mod=readonly"
   commands:
   - desc: test
     cmd: |
@@ -18,9 +20,11 @@ pipeline:
     cmd: |
       if [[ $CDP_TARGET_BRANCH == master && ! $CDP_PULL_REQUEST_NUMBER ]]; then
         IMAGE=registry-write.opensource.zalan.do/teapot/kube-metrics-adapter
-        VERSION=$(git describe --tags --always --dirty)
+        VERSION=$(git describe --tags --always)
       else
         IMAGE=registry-write.opensource.zalan.do/teapot/kube-metrics-adapter-test
         VERSION=$CDP_BUILD_VERSION
       fi
+      IMAGE=$IMAGE VERSION=$VERSION make build.docker
+      git diff --stat --exit-code
       IMAGE=$IMAGE VERSION=$VERSION make build.push

example/deploy/hpa.yaml

@@ -37,7 +37,8 @@ spec:
         apiVersion: extensions/v1beta1
         kind: Ingress
         name: custom-metrics-consumer
-      targetValue: 10 # this will be treated as targetAverageValue
+      averageValue: 10
+      targetValue: 10 # this must be set, but has no effect if `averageValue` is defined.
   - type: External
     external:
       metricName: sqs-queue-length

go.mod

@@ -1,91 +1,33 @@
 module github.com/zalando-incubator/kube-metrics-adapter
 
 require (
-	bitbucket.org/ww/goautoneg v0.0.0-20120707110453-75cd24fc2f2c // indirect
-	github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect
-	github.com/BurntSushi/toml v0.3.0 // indirect
 	github.com/NYTimes/gziphandler v1.0.1 // indirect
-	github.com/PuerkitoBio/purell v1.1.0 // indirect
-	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
-	github.com/Sirupsen/logrus v1.0.6 // indirect
 	github.com/aws/aws-sdk-go v1.16.6
-	github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect
-	github.com/boltdb/bolt v1.3.1 // indirect
-	github.com/coreos/bbolt v1.3.0 // indirect
-	github.com/coreos/etcd v3.3.9+incompatible // indirect
-	github.com/coreos/go-semver v0.2.0 // indirect
 	github.com/coreos/go-systemd v0.0.0-20180705093442-88bfeed483d3 // indirect
-	github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea // indirect
-	github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
-	github.com/docker/docker v1.13.1 // indirect
-	github.com/elazarl/go-bindata-assetfs v1.0.0 // indirect
-	github.com/emicklei/go-restful v2.8.0+incompatible // indirect
-	github.com/emicklei/go-restful-swagger12 v0.0.0-20170926063155-7524189396c6 // indirect
-	github.com/evanphx/json-patch v4.1.1-0.20190203004735-bbf30d639737+incompatible // indirect
-	github.com/fsnotify/fsnotify v1.4.7 // indirect
-	github.com/ghodss/yaml v1.0.0 // indirect
-	github.com/go-openapi/jsonpointer v0.0.0-20180322222829-3a0015ad55fa // indirect
-	github.com/go-openapi/jsonreference v0.0.0-20180322222742-3fb327e6747d // indirect
-	github.com/go-openapi/spec v0.0.0-20180801175345-384415f06ee2 // indirect
-	github.com/go-openapi/swag v0.0.0-20180715190254-becd2f08beaf // indirect
-	github.com/gogo/protobuf v1.1.1 // indirect
-	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
 	github.com/golang/groupcache v0.0.0-20180513044358-24b0969c4cb7 // indirect
-	github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c // indirect
-	github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf // indirect
 	github.com/googleapis/gnostic v0.2.0 // indirect
-	github.com/gorilla/websocket v1.3.0 // indirect
-	github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
-	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway v1.4.1 // indirect
-	github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47 // indirect
-	github.com/hpcloud/tail v1.0.0 // indirect
 	github.com/imdario/mergo v0.3.6 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
-	github.com/jonboulle/clockwork v0.1.0 // indirect
-	github.com/json-iterator/go v1.1.5 // indirect
-	github.com/kubernetes-incubator/custom-metrics-apiserver v0.0.0-20190116221620-b7016fc85e1c
-	github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
-	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/kubernetes-incubator/custom-metrics-apiserver v0.0.0-20190918110929-3d9be26a50eb
 	github.com/oliveagle/jsonpath v0.0.0-20180606110733-2e52cf6e6852
-	github.com/onsi/ginkgo v1.6.0 // indirect
-	github.com/onsi/gomega v1.4.1 // indirect
-	github.com/pborman/uuid v0.0.0-20180122190007-c65b2f87fee3 // indirect
-	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
-	github.com/prometheus/client_golang v0.9.0-pre1.0.20180824101016-4eb539fa85a2
-	github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 // indirect
-	github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e
-	github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273 // indirect
-	github.com/sirupsen/logrus v1.3.0
+	github.com/prometheus/client_golang v0.9.2
+	github.com/prometheus/common v0.0.0-20181126121408-4724e9255275
+	github.com/sirupsen/logrus v1.4.2
 	github.com/soheilhy/cmux v0.1.4 // indirect
 	github.com/spf13/cobra v0.0.3
-	github.com/spf13/pflag v1.0.2 // indirect
-	github.com/stretchr/testify v1.2.2
+	github.com/stretchr/testify v1.3.0
 	github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6 // indirect
-	github.com/ugorji/go v1.1.1 // indirect
-	github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18 // indirect
 	github.com/zalando-incubator/cluster-lifecycle-manager v0.0.0-20180921141935-824b77fb1f84
-	golang.org/x/crypto v0.0.0-20181015023909-0c41d7ab0a0e // indirect
-	golang.org/x/net v0.0.0-20180824152047-4bcd98cce591 // indirect
-	golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be
-	golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f // indirect
-	golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 // indirect
-	google.golang.org/appengine v1.2.0 // indirect
-	google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 // indirect
-	google.golang.org/grpc v1.14.0 // indirect
-	gopkg.in/airbrake/gobrake.v2 v2.0.9 // indirect
-	gopkg.in/fsnotify.v1 v1.4.7 // indirect
-	gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 // indirect
+	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
 	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/natefinch/lumberjack.v2 v2.0.0-20170531160350-a96e63847dc3 // indirect
-	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
-	gopkg.in/yaml.v2 v2.2.1 // indirect
-	k8s.io/api v0.0.0-20190226173710-145d52631d00
-	k8s.io/apimachinery v0.0.0-20190221084156-01f179d85dbc
-	k8s.io/apiserver v0.0.0-20190226174732-cf2f1d68202d
-	k8s.io/client-go v2.0.0-alpha.0.0.20190226174127-78295b709ec6+incompatible
-	k8s.io/kube-openapi v0.0.0-20180731170545-e3762e86a74c // indirect
+	k8s.io/api v0.0.0-20190918155943-95b840bb6a1f
+	k8s.io/apimachinery v0.0.0-20190913080033-27d36303b655
+	k8s.io/apiserver v0.0.0-20190918160949-bfa5e2e684ad // indirect
+	k8s.io/client-go v0.0.0-20190918160344-1fbdaa4c8d90
+	k8s.io/component-base v0.0.0-20190918160511-547f6c5d7090
+	k8s.io/klog v0.4.0
 	k8s.io/metrics v0.0.0-20190226180357-f3f09b9076d1
 )
+
+go 1.13

main.go

@@ -23,7 +23,7 @@ import (
 
 	"github.com/zalando-incubator/kube-metrics-adapter/pkg/server"
 	"k8s.io/apimachinery/pkg/util/wait"
-	"k8s.io/apiserver/pkg/util/logs"
+	"k8s.io/component-base/logs"
 )
 
 func main() {

pkg/collector/aws_collector.go

@@ -56,11 +56,11 @@ func NewAWSSQSCollector(sessions map[string]*session.Session, config *MetricConf
 		return nil, fmt.Errorf("selector for queue is not specified")
 	}
 
-	name, ok := config.Metric.Selector.MatchLabels[sqsQueueNameLabelKey]
+	name, ok := config.Config[sqsQueueNameLabelKey]
 	if !ok {
 		return nil, fmt.Errorf("sqs queue name not specified on metric")
 	}
-	region, ok := config.Metric.Selector.MatchLabels[sqsQueueRegionLabelKey]
+	region, ok := config.Config[sqsQueueRegionLabelKey]
 	if !ok {
 		return nil, fmt.Errorf("sqs queue region is not specified on metric")
 	}

pkg/collector/collector.go

@@ -165,6 +165,7 @@ type MetricConfig struct {
 	ObjectReference custom_metrics.ObjectReference
 	PerReplica      bool
 	Interval        time.Duration
+	MetricSpec      autoscalingv2.MetricSpec
 }
 
 // ParseHPAMetrics parses the HPA object into a list of metric configurations.
@@ -206,9 +207,12 @@ func ParseHPAMetrics(hpa *autoscalingv2.HorizontalPodAutoscaler) ([]*MetricConfi
 			MetricTypeName:  typeName,
 			ObjectReference: ref,
 			Config:          map[string]string{},
+			MetricSpec:      metric,
 		}
 
-		if metric.Type == autoscalingv2.ExternalMetricSourceType {
+		if metric.Type == autoscalingv2.ExternalMetricSourceType &&
+			metric.External.Metric.Selector != nil &&
+			metric.External.Metric.Selector.MatchLabels != nil {
 			config.Config = metric.External.Metric.Selector.MatchLabels
 		}
 
@@ -217,7 +221,11 @@ func ParseHPAMetrics(hpa *autoscalingv2.HorizontalPodAutoscaler) ([]*MetricConfi
 			config.CollectorName = annotationConfigs.CollectorName
 			config.Interval = annotationConfigs.Interval
 			config.PerReplica = annotationConfigs.PerReplica
-			config.Config = annotationConfigs.Configs
+			// configs specified in annotations takes precedence
+			// over labels
+			for k, v := range annotationConfigs.Configs {
+				config.Config[k] = v
+			}
 		}
 		metricConfigs = append(metricConfigs, config)
 	}

pkg/collector/max_collector.go

@@ -1,67 +0,0 @@
-package collector
-
-import (
-	"fmt"
-	"strings"
-	"time"
-
-	"k8s.io/apimachinery/pkg/api/resource"
-)
-
-// MaxWeightedCollector is a simple aggregator collector that returns the maximum value
-// of metrics from all collectors.
-type MaxWeightedCollector struct {
-	collectors []Collector
-	interval   time.Duration
-	weight     float64
-}
-
-// NewMaxWeightedCollector initializes a new MaxWeightedCollector.
-func NewMaxWeightedCollector(interval time.Duration, weight float64, collectors ...Collector) *MaxWeightedCollector {
-	return &MaxWeightedCollector{
-		collectors: collectors,
-		interval:   interval,
-		weight:     weight,
-	}
-}
-
-// GetMetrics gets metrics from all collectors and return the higest value.
-func (c *MaxWeightedCollector) GetMetrics() ([]CollectedMetric, error) {
-	errors := make([]error, 0)
-	collectedMetrics := make([]CollectedMetric, 0)
-	for _, collector := range c.collectors {
-		values, err := collector.GetMetrics()
-		if err != nil {
-			if _, ok := err.(NoResultError); ok {
-				errors = append(errors, err)
-				continue
-			}
-			return nil, err
-		}
-		collectedMetrics = append(collectedMetrics, values...)
-	}
-	if len(collectedMetrics) == 0 {
-		if len(errors) == 0 {
-			return nil, fmt.Errorf("no metrics collected, cannot determine max")
-		}
-		errorStrings := make([]string, len(errors))
-		for i, e := range errors {
-			errorStrings[i] = e.Error()
-		}
-		allErrors := strings.Join(errorStrings, ",")
-		return nil, fmt.Errorf("could not determine maximum due to errors: %s", allErrors)
-	}
-	max := collectedMetrics[0]
-	for _, value := range collectedMetrics {
-		if value.Custom.Value.MilliValue() > max.Custom.Value.MilliValue() {
-			max = value
-		}
-	}
-	max.Custom.Value = *resource.NewMilliQuantity(int64(c.weight*float64(max.Custom.Value.MilliValue())), resource.DecimalSI)
-	return []CollectedMetric{max}, nil
-}
-
-// Interval returns the interval at which the collector should run.
-func (c *MaxWeightedCollector) Interval() time.Duration {
-	return c.interval
-}

pkg/collector/max_collector_test.go

@@ -1,99 +0,0 @@
-package collector
-
-import (
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/require"
-	"k8s.io/apimachinery/pkg/api/resource"
-	"k8s.io/metrics/pkg/apis/custom_metrics"
-)
-
-type dummyCollector struct {
-	value int64
-}
-
-func (c dummyCollector) Interval() time.Duration {
-	return time.Second
-}
-
-func (c dummyCollector) GetMetrics() ([]CollectedMetric, error) {
-	switch c.value {
-	case 0:
-		return nil, NoResultError{query: "invalid query"}
-	case -1:
-		return nil, fmt.Errorf("test error")
-	default:
-		quantity := resource.NewQuantity(c.value, resource.DecimalSI)
-		return []CollectedMetric{
-			{
-				Custom: custom_metrics.MetricValue{
-					Value: *quantity,
-				},
-			},
-		}, nil
-	}
-}
-
-func TestMaxCollector(t *testing.T) {
-	for _, tc := range []struct {
-		name     string
-		values   []int64
-		expected int
-		weight   float64
-		errored  bool
-	}{
-		{
-			name:     "basic",
-			values:   []int64{100, 10, 9},
-			expected: 100,
-			weight:   1,
-			errored:  false,
-		},
-		{
-			name:     "weighted",
-			values:   []int64{100, 10, 9},
-			expected: 20,
-			weight:   0.2,
-			errored:  false,
-		},
-		{
-			name:    "with error",
-			values:  []int64{10, 9, -1},
-			weight:  0.5,
-			errored: true,
-		},
-		{
-			name:     "some invalid results",
-			values:   []int64{0, 1, 0, 10, 9},
-			expected: 5,
-			weight:   0.5,
-			errored:  false,
-		},
-		{
-			name:    "both invalid results and errors",
-			values:  []int64{0, 1, 0, -1, 10, 9},
-			weight:  0.5,
-			errored: true,
-		},
-	} {
-		t.Run(tc.name, func(t *testing.T) {
-			collectors := make([]Collector, len(tc.values))
-			for i, v := range tc.values {
-				collectors[i] = dummyCollector{value: v}
-			}
-			wc := NewMaxWeightedCollector(time.Second, tc.weight, collectors...)
-			metrics, err := wc.GetMetrics()
-			if tc.errored {
-				require.Error(t, err)
-			} else {
-				require.NoError(t, err)
-				require.Len(t, metrics, 1)
-				require.EqualValues(t, tc.expected, metrics[0].Custom.Value.Value())
-			}
-
-		})
-
-	}
-}

pkg/collector/prometheus_collector.go

@@ -3,6 +3,7 @@ package collector
 import (
 	"context"
 	"fmt"
+	"math"
 	"net/http"
 	"time"
 
@@ -18,8 +19,9 @@ import (
 )
 
 const (
-	PrometheusMetricName        = "prometheus-query"
-	prometheusQueryNameLabelKey = "query-name"
+	PrometheusMetricName          = "prometheus-query"
+	prometheusQueryNameLabelKey   = "query-name"
+	prometheusServerAnnotationKey = "prometheus-server"
 )
 
 type NoResultError struct {
@@ -38,7 +40,7 @@ type PrometheusCollectorPlugin struct {
 func NewPrometheusCollectorPlugin(client kubernetes.Interface, prometheusServer string) (*PrometheusCollectorPlugin, error) {
 	cfg := api.Config{
 		Address:      prometheusServer,
-		RoundTripper: &http.Transport{},
+		RoundTripper: http.DefaultTransport,
 	}
 
 	promClient, err := api.NewClient(cfg)
@@ -90,7 +92,11 @@ func NewPrometheusCollector(client kubernetes.Interface, promAPI promv1.API, hpa
 			return nil, fmt.Errorf("no prometheus query defined")
 		}
 	case autoscalingv2.ExternalMetricSourceType:
-		queryName, ok := config.Metric.Selector.MatchLabels[prometheusQueryNameLabelKey]
+		if config.Metric.Selector == nil {
+			return nil, fmt.Errorf("selector for prometheus query is not specified")
+		}
+
+		queryName, ok := config.Config[prometheusQueryNameLabelKey]
 		if !ok {
 			return nil, fmt.Errorf("query name not specified on metric")
 		}
@@ -101,6 +107,20 @@ func NewPrometheusCollector(client kubernetes.Interface, promAPI promv1.API, hpa
 		} else {
 			return nil, fmt.Errorf("no prometheus query defined for metric")
 		}
+
+		// Use custom Prometheus URL if defined in HPA annotation.
+		if promServer, ok := config.Config[prometheusServerAnnotationKey]; ok {
+			cfg := api.Config{
+				Address:      promServer,
+				RoundTripper: http.DefaultTransport,
+			}
+
+			promClient, err := api.NewClient(cfg)
+			if err != nil {
+				return nil, err
+			}
+			c.promAPI = promv1.NewAPI(promClient)
+		}
 	}
 
 	return c, nil
@@ -127,7 +147,7 @@ func (c *PrometheusCollector) GetMetrics() ([]CollectedMetric, error) {
 		sampleValue = scalar.Value
 	}
 
-	if sampleValue.String() == "NaN" {
+	if math.IsNaN(float64(sampleValue)) {
 		return nil, &NoResultError{query: c.query}
 	}
 

pkg/collector/skipper_collector.go

@@ -2,8 +2,10 @@ package collector
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"math"
+	"regexp"
 	"strings"
 	"time"
 
@@ -15,11 +17,15 @@ import (
 )
 
 const (
-	rpsQuery                  = `scalar(sum(rate(skipper_serve_host_duration_seconds_count{host="%s"}[1m])))`
+	rpsQuery                  = `scalar(sum(rate(skipper_serve_host_duration_seconds_count{host=~"%s"}[1m])) * %.4f)`
 	rpsMetricName             = "requests-per-second"
 	rpsMetricBackendSeparator = ","
 )
 
+var (
+	errBackendNameMissing = errors.New("backend name must be specified for requests-per-second when traffic switching is used")
+)
+
 // SkipperCollectorPlugin is a collector plugin for initializing metrics
 // collectors for getting skipper ingress metrics.
 type SkipperCollectorPlugin struct {
@@ -67,8 +73,7 @@ type SkipperCollector struct {
 }
 
 // NewSkipperCollector initializes a new SkipperCollector.
-func NewSkipperCollector(client kubernetes.Interface, plugin CollectorPlugin, hpa *autoscalingv2.HorizontalPodAutoscaler,
-	config *MetricConfig, interval time.Duration, backendAnnotations []string, backend string) (*SkipperCollector, error) {
+func NewSkipperCollector(client kubernetes.Interface, plugin CollectorPlugin, hpa *autoscalingv2.HorizontalPodAutoscaler, config *MetricConfig, interval time.Duration, backendAnnotations []string, backend string) (*SkipperCollector, error) {
 	return &SkipperCollector{
 		client:             client,
 		objectReference:    config.ObjectReference,
@@ -94,7 +99,7 @@ func getAnnotationWeight(backendWeights string, backend string) float64 {
 	return 0
 }
 
-func getWeights(ingressAnnotations map[string]string, backendAnnotations []string, backend string) float64 {
+func getWeights(ingressAnnotations map[string]string, backendAnnotations []string, backend string) (float64, error) {
 	maxWeight := 0.0
 	annotationsPresent := false
 
@@ -107,10 +112,15 @@ func getWeights(ingressAnnotations map[string]string, backendAnnotations []strin
 
 	// Fallback for ingresses that don't use traffic switching
 	if !annotationsPresent {
-		return 1.0
+		return 1.0, nil
 	}
 
-	return maxWeight
+	// Require backend name here
+	if backend != "" {
+		return maxWeight, nil
+	}
+
+	return 0.0, errBackendNameMissing
 }
 
 // getCollector returns a collector for getting the metrics.
@@ -120,32 +130,31 @@ func (c *SkipperCollector) getCollector() (Collector, error) {
 		return nil, err
 	}
 
-	backendWeight := getWeights(ingress.Annotations, c.backendAnnotations, c.backend)
+	backendWeight, err := getWeights(ingress.Annotations, c.backendAnnotations, c.backend)
+	if err != nil {
+		return nil, err
+	}
 	config := c.config
 
-	var collector Collector
-	collectors := make([]Collector, 0, len(ingress.Spec.Rules))
+	var escapedHostnames []string
 	for _, rule := range ingress.Spec.Rules {
-		host := strings.Replace(rule.Host, ".", "_", -1)
-		config.Config = map[string]string{
-			"query": fmt.Sprintf(rpsQuery, host),
-		}
-
-		config.PerReplica = false // per replica is handled outside of the prometheus collector
-		collector, err := c.plugin.NewCollector(c.hpa, &config, c.interval)
-		if err != nil {
-			return nil, err
-		}
-
-		collectors = append(collectors, collector)
+		escapedHostnames = append(escapedHostnames, regexp.QuoteMeta(strings.Replace(rule.Host, ".", "_", -1)))
 	}
 
-	if len(collectors) > 0 {
-		collector = NewMaxWeightedCollector(c.interval, backendWeight, collectors...)
-	} else {
+	if len(escapedHostnames) == 0 {
 		return nil, fmt.Errorf("no hosts defined on ingress %s/%s, unable to create collector", c.objectReference.Namespace, c.objectReference.Name)
 	}
 
+	config.Config = map[string]string{
+		"query": fmt.Sprintf(rpsQuery, strings.Join(escapedHostnames, "|"), backendWeight),
+	}
+
+	config.PerReplica = false // per replica is handled outside of the prometheus collector
+	collector, err := c.plugin.NewCollector(c.hpa, &config, c.interval)
+	if err != nil {
+		return nil, err
+	}
+
 	return collector, nil
 }
 
@@ -165,22 +174,26 @@ func (c *SkipperCollector) GetMetrics() ([]CollectedMetric, error) {
 		return nil, fmt.Errorf("expected to only get one metric value, got %d", len(values))
 	}
 
-	// get current replicas for the targeted scale object. This is used to
-	// calculate an average metric instead of total.
-	// targetAverageValue will be available in Kubernetes v1.12
-	// https://github.com/kubernetes/kubernetes/pull/64097
-	replicas, err := targetRefReplicas(c.client, c.hpa)
-	if err != nil {
-		return nil, err
-	}
-
-	if replicas < 1 {
-		return nil, fmt.Errorf("unable to get average value for %d replicas", replicas)
-	}
-
 	value := values[0]
-	avgValue := float64(value.Custom.Value.MilliValue()) / float64(replicas)
-	value.Custom.Value = *resource.NewMilliQuantity(int64(avgValue), resource.DecimalSI)
+
+	// For Kubernetes <v1.14 we have to fall back to manual average
+	if c.config.MetricSpec.Object.Target.AverageValue == nil {
+		// get current replicas for the targeted scale object. This is used to
+		// calculate an average metric instead of total.
+		// targetAverageValue will be available in Kubernetes v1.12
+		// https://github.com/kubernetes/kubernetes/pull/64097
+		replicas, err := targetRefReplicas(c.client, c.hpa)
+		if err != nil {
+			return nil, err
+		}
+
+		if replicas < 1 {
+			return nil, fmt.Errorf("unable to get average value for %d replicas", replicas)
+		}
+
+		avgValue := float64(value.Custom.Value.MilliValue()) / float64(replicas)
+		value.Custom.Value = *resource.NewMilliQuantity(int64(avgValue), resource.DecimalSI)
+	}
 
 	return []CollectedMetric{value}, nil
 }

pkg/collector/zmon_collector.go

@@ -44,11 +44,7 @@ func NewZMONCollectorPlugin(zmon zmon.ZMON) (*ZMONCollectorPlugin, error) {
 func (c *ZMONCollectorPlugin) NewCollector(hpa *autoscalingv2.HorizontalPodAutoscaler, config *MetricConfig, interval time.Duration) (Collector, error) {
 	switch config.Metric.Name {
 	case ZMONCheckMetric:
-		annotations := map[string]string{}
-		if hpa != nil {
-			annotations = hpa.Annotations
-		}
-		return NewZMONCollector(c.zmon, config, annotations, interval)
+		return NewZMONCollector(c.zmon, config, interval)
 	}
 
 	return nil, fmt.Errorf("metric '%s' not supported", config.Metric.Name)
@@ -68,7 +64,11 @@ type ZMONCollector struct {
 }
 
 // NewZMONCollector initializes a new ZMONCollector.
-func NewZMONCollector(zmon zmon.ZMON, config *MetricConfig, annotations map[string]string, interval time.Duration) (*ZMONCollector, error) {
+func NewZMONCollector(zmon zmon.ZMON, config *MetricConfig, interval time.Duration) (*ZMONCollector, error) {
+	if config.Metric.Selector == nil {
+		return nil, fmt.Errorf("selector for zmon-check is not specified")
+	}
+
 	checkIDStr, ok := config.Config[zmonCheckIDLabelKey]
 	if !ok {
 		return nil, fmt.Errorf("ZMON check ID not specified on metric")
@@ -86,11 +86,6 @@ func NewZMONCollector(zmon zmon.ZMON, config *MetricConfig, annotations map[stri
 		key = k
 	}
 
-	// annotations takes precedence over label
-	if k, ok := annotations[zmonKeyAnnotationKey]; ok {
-		key = k
-	}
-
 	duration := defaultQueryDuration
 
 	// parse optional duration value
@@ -110,16 +105,6 @@ func NewZMONCollector(zmon zmon.ZMON, config *MetricConfig, annotations map[stri
 		}
 	}
 
-	// parse tags from annotations
-	// tags defined in annotations takes precedence over tags defined in
-	// the labels.
-	for k, v := range annotations {
-		if strings.HasPrefix(k, zmonTagPrefixAnnotationKey) {
-			key := strings.TrimPrefix(k, zmonTagPrefixAnnotationKey)
-			tags[key] = v
-		}
-	}
-
 	// default aggregator is last
 	aggregators := []string{"last"}
 	if k, ok := config.Config[zmonAggregatorsLabelKey]; ok {

pkg/collector/zmon_collector_test.go

@@ -50,20 +50,6 @@ func TestZMONCollectorNewCollector(t *testing.T) {
 	require.Equal(t, []string{"max"}, zmonCollector.aggregators)
 	require.Equal(t, map[string]string{"alias": "cluster_alias"}, zmonCollector.tags)
 
-	// check that annotations overwrites labels
-	hpa.ObjectMeta = metav1.ObjectMeta{
-		Annotations: map[string]string{
-			zmonKeyAnnotationKey:                 "annotation_key",
-			zmonTagPrefixAnnotationKey + "alias": "cluster_alias_annotation",
-		},
-	}
-	collector, err = collectPlugin.NewCollector(hpa, config, 1*time.Second)
-	require.NoError(t, err)
-	require.NotNil(t, collector)
-	zmonCollector = collector.(*ZMONCollector)
-	require.Equal(t, "annotation_key", zmonCollector.key)
-	require.Equal(t, map[string]string{"alias": "cluster_alias_annotation"}, zmonCollector.tags)
-
 	// should fail if the metric name isn't ZMON
 	config.Metric = newMetricIdentifier("non-zmon-check")
 	_, err = collectPlugin.NewCollector(nil, config, 1*time.Second)
@@ -131,7 +117,7 @@ func TestZMONCollectorGetMetrics(tt *testing.T) {
 				dataPoints: ti.dataPoints,
 			}
 
-			zmonCollector, err := NewZMONCollector(z, config, nil, 1*time.Second)
+			zmonCollector, err := NewZMONCollector(z, config, 1*time.Second)
 			require.NoError(t, err)
 
 			metrics, _ := zmonCollector.GetMetrics()

pkg/provider/hpa.go

@@ -262,7 +262,7 @@ func (p *HPAProvider) collectMetrics(ctx context.Context) {
 }
 
 // GetMetricByName gets a single metric by name.
-func (p *HPAProvider) GetMetricByName(name types.NamespacedName, info provider.CustomMetricInfo) (*custom_metrics.MetricValue, error) {
+func (p *HPAProvider) GetMetricByName(name types.NamespacedName, info provider.CustomMetricInfo, metricSelector labels.Selector) (*custom_metrics.MetricValue, error) {
 	metric := p.metricStore.GetMetricsByName(name, info)
 	if metric == nil {
 		return nil, provider.NewMetricNotFoundForError(info.GroupResource, info.Metric, name.Name)
@@ -272,7 +272,7 @@ func (p *HPAProvider) GetMetricByName(name types.NamespacedName, info provider.C
 
 // GetMetricBySelector returns metrics for namespaced resources by
 // label selector.
-func (p *HPAProvider) GetMetricBySelector(namespace string, selector labels.Selector, info provider.CustomMetricInfo) (*custom_metrics.MetricValueList, error) {
+func (p *HPAProvider) GetMetricBySelector(namespace string, selector labels.Selector, info provider.CustomMetricInfo, metricSelector labels.Selector) (*custom_metrics.MetricValueList, error) {
 	return p.metricStore.GetMetricsBySelector(namespace, selector, info), nil
 }
 

pkg/server/start.go

@@ -25,7 +25,6 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/golang/glog"
 	"github.com/kubernetes-incubator/custom-metrics-apiserver/pkg/cmd/server"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/spf13/cobra"
@@ -38,6 +37,7 @@ import (
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/klog"
 )
 
 const (
@@ -111,7 +111,7 @@ func NewCommandStartAdapterServer(stopCh <-chan struct{}) *cobra.Command {
 func (o AdapterServerOptions) RunCustomMetricsAdapterServer(stopCh <-chan struct{}) error {
 	go func() {
 		http.Handle("/metrics", promhttp.Handler())
-		glog.Fatal(http.ListenAndServe(o.MetricsAddress, nil))
+		klog.Fatal(http.ListenAndServe(o.MetricsAddress, nil))
 	}()
 
 	config, err := o.Config()