Merge pull request #53 from zalando-incubator/prometheus-external-metric

Allow Prometheus metrics for External target

README.md

@@ -152,9 +152,51 @@ the trade-offs between the two approaches.
 
 | Metric | Description | Type | Kind |
 | ------------ | -------------- | ------- | -- |
+| `prometheus-query` | Generic metric which requires a user defined query. | External | |
 | *custom* | No predefined metrics. Metrics are generated from user defined queries. | Object | *any* |
 
-### Example
+### Example: External Metric
+
+This is an example of an HPA configured to get metrics based on a Prometheus
+query. The query is defined in the annotation
+`metric-config.external.prometheus-query.prometheus/processed-events-per-second`
+where `processed-events-per-second` is the query name which will be associated
+with the result of the query. A matching `query-name` label must be defined in
+the `matchLabels` of the metric definition. This allows having multiple
+prometheus queries associated with a single HPA.
+
+```yaml
+apiVersion: autoscaling/v2beta1
+kind: HorizontalPodAutoscaler
+metadata:
+  name: myapp-hpa
+  annotations:
+    # metric-config.<metricType>.<metricName>.<collectorName>/<configKey>
+    # <configKey> == query-name
+    metric-config.external.prometheus-query.prometheus/processed-events-per-second: |
+      scalar(sum(rate(event-service_events_count{application="event-service",processed="true"}[1m])))
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: custom-metrics-consumer
+  minReplicas: 1
+  maxReplicas: 10
+  metrics:
+  - type: External
+    external:
+      metricName: prometheus-query
+      metricSelector:
+        matchLabels:
+          query-name: processed-events-per-second
+      targetAverageValue: 10
+```
+
+### Example: Object Metric [DEPRECATED]
+
+> _Note: Prometheus Object metrics are **deprecated** and will most likely be
+> removed in the future. Use the Prometheus External metrics instead as described
+> above._
 
 This is an example of an HPA configured to get metrics based on a Prometheus
 query. The query is defined in the annotation
@@ -200,6 +242,7 @@ spec:
 _Note:_ The HPA object requires an `Object` to be specified. However when a Prometheus metric is used there is no need
 for this object. But to satisfy the schema we specify a dummy pod called `dummy-pod`.
 
+
 ## Skipper collector
 
 The skipper collector is a simple wrapper around the Prometheus collector to

pkg/collector/prometheus_collector.go

@@ -14,6 +14,12 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/metrics/pkg/apis/custom_metrics"
+	"k8s.io/metrics/pkg/apis/external_metrics"
+)
+
+const (
+	PrometheusMetricName        = "prometheus-query"
+	prometheusQueryNameLabelKey = "query-name"
 )
 
 type NoResultError struct {
@@ -64,21 +70,37 @@ type PrometheusCollector struct {
 
 func NewPrometheusCollector(client kubernetes.Interface, promAPI promv1.API, hpa *autoscalingv2.HorizontalPodAutoscaler, config *MetricConfig, interval time.Duration) (*PrometheusCollector, error) {
 	c := &PrometheusCollector{
-		client:          client,
+		client:     client,
-		objectReference: config.ObjectReference,
+		promAPI:    promAPI,
-		metric:          config.Metric,
+		interval:   interval,
-		metricType:      config.Type,
+		hpa:        hpa,
-		interval:        interval,
+		metric:     config.Metric,
-		promAPI:         promAPI,
+		metricType: config.Type,
-		perReplica:      config.PerReplica,
-		hpa:             hpa,
 	}
 
-	if v, ok := config.Config["query"]; ok {
+	switch config.Type {
-		// TODO: validate query
+	case autoscalingv2.ObjectMetricSourceType:
-		c.query = v
+		c.objectReference = config.ObjectReference
-	} else {
+		c.perReplica = config.PerReplica
-		return nil, fmt.Errorf("no prometheus query defined")
+
+		if v, ok := config.Config["query"]; ok {
+			// TODO: validate query
+			c.query = v
+		} else {
+			return nil, fmt.Errorf("no prometheus query defined")
+		}
+	case autoscalingv2.ExternalMetricSourceType:
+		queryName, ok := config.Metric.Selector.MatchLabels[prometheusQueryNameLabelKey]
+		if !ok {
+			return nil, fmt.Errorf("query name not specified on metric")
+		}
+
+		if v, ok := config.Config[queryName]; ok {
+			// TODO: validate query
+			c.query = v
+		} else {
+			return nil, fmt.Errorf("no prometheus query defined for metric")
+		}
 	}
 
 	return c, nil
@@ -121,14 +143,28 @@ func (c *PrometheusCollector) GetMetrics() ([]CollectedMetric, error) {
 		sampleValue = model.SampleValue(float64(sampleValue) / float64(replicas))
 	}
 
-	metricValue := CollectedMetric{
+	var metricValue CollectedMetric
-		Type: c.metricType,
+	switch c.metricType {
-		Custom: custom_metrics.MetricValue{
+	case autoscalingv2.ObjectMetricSourceType:
-			DescribedObject: c.objectReference,
+		metricValue = CollectedMetric{
-			Metric:          custom_metrics.MetricIdentifier{Name: c.metric.Name, Selector: c.metric.Selector},
+			Type: c.metricType,
-			Timestamp:       metav1.Time{Time: time.Now().UTC()},
+			Custom: custom_metrics.MetricValue{
-			Value:           *resource.NewMilliQuantity(int64(sampleValue*1000), resource.DecimalSI),
+				DescribedObject: c.objectReference,
-		},
+				Metric:          custom_metrics.MetricIdentifier{Name: c.metric.Name, Selector: c.metric.Selector},
+				Timestamp:       metav1.Time{Time: time.Now().UTC()},
+				Value:           *resource.NewMilliQuantity(int64(sampleValue*1000), resource.DecimalSI),
+			},
+		}
+	case autoscalingv2.ExternalMetricSourceType:
+		metricValue = CollectedMetric{
+			Type: c.metricType,
+			External: external_metrics.ExternalMetricValue{
+				MetricName:   c.metric.Name,
+				MetricLabels: c.metric.Selector.MatchLabels,
+				Timestamp:    metav1.Time{Time: time.Now().UTC()},
+				Value:        *resource.NewMilliQuantity(int64(sampleValue*1000), resource.DecimalSI),
+			},
+		}
 	}
 
 	return []CollectedMetric{metricValue}, nil

pkg/server/start.go

@@ -156,9 +156,11 @@ func (o AdapterServerOptions) RunCustomMetricsAdapterServer(stopCh <-chan struct
 
 		err = collectorFactory.RegisterObjectCollector("", "prometheus", promPlugin)
 		if err != nil {
-			return fmt.Errorf("failed to register prometheus collector plugin: %v", err)
+			return fmt.Errorf("failed to register prometheus object collector plugin: %v", err)
 		}
 
+		collectorFactory.RegisterExternalCollector([]string{collector.PrometheusMetricName}, promPlugin)
+
 		// skipper collector can only be enabled if prometheus is.
 		if o.SkipperIngressMetrics {
 			skipperPlugin, err := collector.NewSkipperCollectorPlugin(client, promPlugin, o.SkipperBackendWeightAnnotation)