yusheng-guo/kube-metrics-adapter
1
0
Fork 1
mirror of https://github.com/zalando-incubator/kube-metrics-adapter.git synced 2025-05-31 09:31:10 +00:00
Code Issues 1 Packages Projects Releases Wiki Activity

Simple ZMON collector implementation (#2)

Browse Source 
* Simple ZMON collector implementation

Signed-off-by: Mikkel Oscar Lyderik Larsen <mikkel.larsen@zalando.de>

* Add tests for ZMON client

Signed-off-by: Mikkel Oscar Lyderik Larsen <mikkel.larsen@zalando.de>

* Add tests for zmon collector

Signed-off-by: Mikkel Oscar Lyderik Larsen <mikkel.larsen@zalando.de>

* Update ZMON collector docs

Signed-off-by: Mikkel Oscar Lyderik Larsen <mikkel.larsen@zalando.de>

* Expose tags instead of entities for queries

Signed-off-by: Mikkel Oscar Lyderik Larsen <mikkel.larsen@zalando.de>

* Remove unused function

Signed-off-by: Mikkel Oscar Lyderik Larsen <mikkel.larsen@zalando.de>
This commit is contained in:
Mikkel Oscar Lyderik Larsen
2018-10-29 14:26:25 +01:00
committed by Arjun
parent b18acf3ed0
commit c86a82ca88
8 changed files with 964 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

102
pkg/server/start.go
View File

@ -19,6 +19,7 @@ package server
import (
"context"
"fmt"
"net"
"net/http"
"time"
@ -28,8 +29,11 @@ import (
"github.com/kubernetes-incubator/custom-metrics-apiserver/pkg/cmd/server"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/spf13/cobra"
"github.com/zalando-incubator/cluster-lifecycle-manager/pkg/credentials-loader/platformiam"
"github.com/zalando-incubator/kube-metrics-adapter/pkg/collector"
"github.com/zalando-incubator/kube-metrics-adapter/pkg/provider"
"github.com/zalando-incubator/kube-metrics-adapter/pkg/zmon"
"golang.org/x/oauth2"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
@ -48,6 +52,8 @@ func NewCommandStartAdapterServer(stopCh <-chan struct{}) *cobra.Command {
EnableCustomMetricsAPI: true,
EnableExternalMetricsAPI: true,
MetricsAddress: ":7979",
ZMONTokenName: "zmon",
CredentialsDir: "/meta/credentials",
}
cmd := &cobra.Command{
@ -82,6 +88,14 @@ func NewCommandStartAdapterServer(stopCh <-chan struct{}) *cobra.Command {
"whether to enable External Metrics API")
flags.StringVar(&o.PrometheusServer, "prometheus-server", o.PrometheusServer, ""+
"url of prometheus server to query")
flags.StringVar(&o.ZMONKariosDBEndpoint, "zmon-kariosdb-endpoint", o.ZMONKariosDBEndpoint, ""+
"url of ZMON KariosDB endpoint to query for ZMON checks")
flags.StringVar(&o.ZMONTokenName, "zmon-token-name", o.ZMONTokenName, ""+
"name of the token used to query ZMON")
flags.StringVar(&o.Token, "token", o.Token, ""+
"static oauth2 token to use when calling external services like ZMON")
flags.StringVar(&o.CredentialsDir, "credentials-dir", o.CredentialsDir, ""+
"path to the credentials dir where tokens are stored")
flags.BoolVar(&o.SkipperIngressMetrics, "skipper-ingress-metrics", o.SkipperIngressMetrics, ""+
"whether to enable skipper ingress metrics")
flags.BoolVar(&o.AWSExternalMetrics, "aws-external-metrics", o.AWSExternalMetrics, ""+
@ -116,6 +130,13 @@ func (o AdapterServerOptions) RunCustomMetricsAdapterServer(stopCh <-chan struct
return fmt.Errorf("unable to construct lister client config to initialize provider: %v", err)
}
// convert stop channel to a context
ctx, cancel := context.WithCancel(context.Background())
go func() {
<-stopCh
cancel()
}()
clientConfig.Timeout = defaultClientGOTimeout
client, err := kubernetes.NewForConfig(clientConfig)
@ -153,7 +174,28 @@ func (o AdapterServerOptions) RunCustomMetricsAdapterServer(stopCh <-chan struct
// register generic pod collector
err = collectorFactory.RegisterPodsCollector("", collector.NewPodCollectorPlugin(client))
if err != nil {
return fmt.Errorf("failed to register skipper collector plugin: %v", err)
return fmt.Errorf("failed to register pod collector plugin: %v", err)
}
// enable ZMON based metrics
if o.ZMONKariosDBEndpoint != "" {
var tokenSource oauth2.TokenSource
if o.Token != "" {
tokenSource = oauth2.StaticTokenSource(&oauth2.Token{AccessToken: o.Token})
} else {
tokenSource = platformiam.NewTokenSource(o.ZMONTokenName, o.CredentialsDir)
}
httpClient := newOauth2HTTPClient(ctx, tokenSource)
zmonClient := zmon.NewZMONClient(o.ZMONKariosDBEndpoint, httpClient)
zmonPlugin, err := collector.NewZMONCollectorPlugin(zmonClient)
if err != nil {
return fmt.Errorf("failed to initialize ZMON collector plugin: %v", err)
}
collectorFactory.RegisterExternalCollector([]string{collector.ZMONCheckMetric}, zmonPlugin)
}
awsSessions := make(map[string]*session.Session, len(o.AWSRegions))
@ -170,13 +212,6 @@ func (o AdapterServerOptions) RunCustomMetricsAdapterServer(stopCh <-chan struct
hpaProvider := provider.NewHPAProvider(client, 30*time.Second, 1*time.Minute, collectorFactory)
// convert stop channel to a context
ctx, cancel := context.WithCancel(context.Background())
go func() {
<-stopCh
cancel()
}()
go hpaProvider.Run(ctx)
customMetricsProvider := hpaProvider
@ -200,6 +235,45 @@ func (o AdapterServerOptions) RunCustomMetricsAdapterServer(stopCh <-chan struct
return server.GenericAPIServer.PrepareRun().Run(ctx.Done())
}
// newInstrumentedOauth2HTTPClient creates an HTTP client with automatic oauth2
// token injection. Additionally it will spawn a go-routine for closing idle
// connections every 20 seconds on the http.Transport. This solves the problem
// of re-resolving DNS when the endpoint backend changes.
// https://github.com/golang/go/issues/23427
func newOauth2HTTPClient(ctx context.Context, tokenSource oauth2.TokenSource) *http.Client {
transport := &http.Transport{
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}).DialContext,
TLSHandshakeTimeout: 10 * time.Second,
ResponseHeaderTimeout: 10 * time.Second,
IdleConnTimeout: 20 * time.Second,
MaxIdleConns: 10,
MaxIdleConnsPerHost: 2,
}
go func(transport *http.Transport, duration time.Duration) {
for {
select {
case <-time.After(duration):
transport.CloseIdleConnections()
case <-ctx.Done():
return
}
}
}(transport, 20*time.Second)
client := &http.Client{
Transport: transport,
}
// add HTTP client to context (this is how the oauth2 lib gets it).
ctx = context.WithValue(ctx, oauth2.HTTPClient, client)
// instantiate an http.Client containg the token source.
return oauth2.NewClient(ctx, tokenSource)
}
type AdapterServerOptions struct {
*server.CustomMetricsAdapterServerOptions
@ -210,8 +284,18 @@ type AdapterServerOptions struct {
// EnableExternalMetricsAPI switches on sample apiserver for External Metrics API
EnableExternalMetricsAPI bool
// PrometheusServer enables prometheus queries to the specified
// server.
// server
PrometheusServer string
// ZMONKariosDBEndpoint enables ZMON check queries to the specified
// kariosDB endpoint
ZMONKariosDBEndpoint string
// ZMONTokenName is the name of the token used to query ZMON
ZMONTokenName string
// Token is an oauth2 token used to authenticate with services like
// ZMON.
Token string
// CredentialsDir is the path to the dir where tokens are stored
CredentialsDir string
// SkipperIngressMetrics switches on support for skipper ingress based
// metric collection.
SkipperIngressMetrics bool