From 3e7b8ba9640043fea3a5cad44fded9d6c3fc3069 Mon Sep 17 00:00:00 2001
From: Martin Linkhorst <linki@users.noreply.github.com>
Date: Fri, 19 Oct 2018 13:40:31 +0200
Subject: [PATCH] Expose basic go metrics via prometheus (#11)

* feat: expose basic go metrics via prometheus

Signed-off-by: Martin Linkhorst <martin.linkhorst@zalando.de>

* feat: expose basic counter of successful and failed collections and updates

Signed-off-by: Martin Linkhorst <martin.linkhorst@zalando.de>
---
 pkg/provider/hpa.go | 31 +++++++++++++++++++++++++++++++
 pkg/server/start.go | 16 ++++++++++++++--
 2 files changed, 45 insertions(+), 2 deletions(-)

diff --git a/pkg/provider/hpa.go b/pkg/provider/hpa.go
index a85cddd..5e6ac45 100644
--- a/pkg/provider/hpa.go
+++ b/pkg/provider/hpa.go
@@ -8,6 +8,8 @@ import (
 
 	"github.com/golang/glog"
 	"github.com/kubernetes-incubator/custom-metrics-apiserver/pkg/provider"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/zalando-incubator/kube-metrics-adapter/pkg/collector"
 	autoscalingv2beta1 "k8s.io/api/autoscaling/v2beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -18,6 +20,29 @@ import (
 	"k8s.io/metrics/pkg/apis/external_metrics"
 )
 
+var (
+	// CollectionSuccesses is the total number of successful collections.
+	CollectionSuccesses = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "kube_metrics_adapter_collections_success",
+		Help: "The total number of successful collections",
+	})
+	// CollectionErrors is the total number of failed collection attempts.
+	CollectionErrors = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "kube_metrics_adapter_collections_error",
+		Help: "The total number of failed collection attempts",
+	})
+	// UpdateSuccesses is the total number of successful HPA updates.
+	UpdateSuccesses = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "kube_metrics_adapter_updates_success",
+		Help: "The total number of successful HPA updates",
+	})
+	// UpdateErrors is the total number of failed HPA update attempts.
+	UpdateErrors = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "kube_metrics_adapter_updates_error",
+		Help: "The total number of failed HPA update attempts",
+	})
+)
+
 type objectCollector struct {
 	ObjectReference *autoscalingv2beta1.CrossVersionObjectReference
 }
@@ -66,6 +91,9 @@ func (p *HPAProvider) Run(ctx context.Context) {
 		err := p.updateHPAs()
 		if err != nil {
 			glog.Error(err)
+			UpdateErrors.Inc()
+		} else {
+			UpdateSuccesses.Inc()
 		}
 
 		select {
@@ -180,6 +208,9 @@ func (p *HPAProvider) collectMetrics(ctx context.Context) {
 		case collection := <-p.metricSink:
 			if collection.Error != nil {
 				glog.Errorf("Failed to collect metrics: %v", collection.Error)
+				CollectionErrors.Inc()
+			} else {
+				CollectionSuccesses.Inc()
 			}
 
 			glog.Infof("Collected %d new metric(s)", len(collection.Values))
diff --git a/pkg/server/start.go b/pkg/server/start.go
index 9432099..041b4ab 100644
--- a/pkg/server/start.go
+++ b/pkg/server/start.go
@@ -19,14 +19,17 @@ package server
 import (
 	"context"
 	"fmt"
-	"github.com/aws/aws-sdk-go/aws"
+	"net/http"
 	"time"
 
+	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/golang/glog"
 	"github.com/kubernetes-incubator/custom-metrics-apiserver/pkg/cmd/server"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/spf13/cobra"
 	"github.com/zalando-incubator/kube-metrics-adapter/pkg/collector"
 	"github.com/zalando-incubator/kube-metrics-adapter/pkg/provider"
-	"github.com/spf13/cobra"
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
@@ -44,6 +47,7 @@ func NewCommandStartAdapterServer(stopCh <-chan struct{}) *cobra.Command {
 		CustomMetricsAdapterServerOptions: baseOpts,
 		EnableCustomMetricsAPI:            true,
 		EnableExternalMetricsAPI:          true,
+		MetricsAddress:                    ":7979",
 	}
 
 	cmd := &cobra.Command{
@@ -83,11 +87,17 @@ func NewCommandStartAdapterServer(stopCh <-chan struct{}) *cobra.Command {
 	flags.BoolVar(&o.AWSExternalMetrics, "aws-external-metrics", o.AWSExternalMetrics, ""+
 		"whether to enable AWS external metrics")
 	flags.StringSliceVar(&o.AWSRegions, "aws-region", o.AWSRegions, "the AWS regions which should be monitored. eg: eu-central, eu-west-1")
+	flags.StringVar(&o.MetricsAddress, "metrics-address", o.MetricsAddress, "The address where to serve prometheus metrics")
 
 	return cmd
 }
 
 func (o AdapterServerOptions) RunCustomMetricsAdapterServer(stopCh <-chan struct{}) error {
+	go func() {
+		http.Handle("/metrics", promhttp.Handler())
+		glog.Fatal(http.ListenAndServe(o.MetricsAddress, nil))
+	}()
+
 	config, err := o.Config()
 	if err != nil {
 		return err
@@ -210,4 +220,6 @@ type AdapterServerOptions struct {
 	AWSExternalMetrics bool
 	// AWSRegions the AWS regions which are supported for monitoring.
 	AWSRegions []string
+	// MetricsAddress is the address where to serve prometheus metrics.
+	MetricsAddress string
 }