yusheng-guo/kube-metrics-adapter
1
0
Fork 1
mirror of https://github.com/zalando-incubator/kube-metrics-adapter.git synced 2025-06-18 01:35:53 +00:00
Code Issues 2 Packages Projects Releases Wiki Activity

Renamed min-pod-age to min-pod-ready-age. Considering LastTransitionTime for pod ready age.

Browse Source 
Signed-off-by: Anatolii Dutchak <adutchak-x@tunein.com>
This commit is contained in:
Anatolii Dutchak
2021-04-19 18:51:18 +03:00
parent 6d15a1635a
commit 5747b6c9de
6 changed files with 61 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@ -111,7 +111,7 @@ metadata:
metric-config.pods.requests-per-second.json-path/scheme: "https"
metric-config.pods.requests-per-second.json-path/aggregator: "max"
metric-config.pods.requests-per-second.json-path/interval: "60s" # optional
metric-config.pods.requests-per-second.json-path/min-pod-age: "30s" # optional
metric-config.pods.requests-per-second.json-path/min-pod-ready-age: "30s" # optional
spec:
scaleTargetRef:
apiVersion: apps/v1
@ -176,7 +176,7 @@ metric-config.pods.requests-per-second.json-path/connect-timeout: 500ms
The default for both of the above values is 15 seconds.
The `min-pod-age` configuration option instructs the service to start collecting metrics from the pods only if they are "older" than the specified amount of time.
The `min-pod-ready-age` configuration option instructs the service to start collecting metrics from the pods only if they are "older" (time elapsed after pod reached "Ready" state) than the specified amount of time.
This is handy when pods need to warm up before HPAs will start tracking their metrics.
The default value is 0 seconds.

20
pkg/annotations/parser.go
View File

@ -12,15 +12,15 @@ const (
customMetricsPrefix = "metric-config."
perReplicaMetricsConfKey = "per-replica"
intervalMetricsConfKey = "interval"
minPodAgeConfKey = "min-pod-age"
minPodReadyAgeConfKey = "min-pod-ready-age"
)
type AnnotationConfigs struct {
CollectorType string
Configs map[string]string
PerReplica bool
Interval time.Duration
MinPodAge time.Duration
CollectorType string
Configs map[string]string
PerReplica bool
Interval time.Duration
MinPodReadyAge time.Duration
}
type MetricConfigKey struct {
@ -91,12 +91,12 @@ func (m AnnotationConfigMap) Parse(annotations map[string]string) error {
continue
}
if parts[1] == minPodAgeConfKey {
minPodAge, err := time.ParseDuration(val)
if parts[1] == minPodReadyAgeConfKey {
minPodReadyAge, err := time.ParseDuration(val)
if err != nil {
return fmt.Errorf("failed to parse min-pod-age value %s for %s: %v", val, key, err)
return fmt.Errorf("failed to parse min-pod-ready-age value %s for %s: %v", val, key, err)
}
config.MinPodAge = minPodAge
config.MinPodReadyAge = minPodReadyAge
continue
}

10
pkg/annotations/parser_test.go
View File

@ -24,11 +24,11 @@ func TestParser(t *testing.T) {
{
Name: "pod metrics",
Annotations: map[string]string{
"metric-config.pods.requests-per-second.json-path/json-key": "$.http_server.rps",
"metric-config.pods.requests-per-second.json-path/path": "/metrics",
"metric-config.pods.requests-per-second.json-path/port": "9090",
"metric-config.pods.requests-per-second.json-path/scheme": "https",
"metric-config.pods.requests-per-second.json-path/min-pod-age": "30s",
"metric-config.pods.requests-per-second.json-path/json-key": "$.http_server.rps",
"metric-config.pods.requests-per-second.json-path/path": "/metrics",
"metric-config.pods.requests-per-second.json-path/port": "9090",
"metric-config.pods.requests-per-second.json-path/scheme": "https",
"metric-config.pods.requests-per-second.json-path/min-pod-ready-age": "30s",
},
MetricName: "requests-per-second",
MetricType: autoscalingv2.PodsMetricSourceType,

4
pkg/collector/collector.go
View File

@ -200,7 +200,7 @@ type MetricConfig struct {
ObjectReference custom_metrics.ObjectReference
PerReplica bool
Interval time.Duration
MinPodAge time.Duration
MinPodReadyAge time.Duration
MetricSpec autoscalingv2.MetricSpec
}
@ -259,7 +259,7 @@ func ParseHPAMetrics(hpa *autoscalingv2.HorizontalPodAutoscaler) ([]*MetricConfi
config.CollectorType = annotationConfigs.CollectorType
config.Interval = annotationConfigs.Interval
config.PerReplica = annotationConfigs.PerReplica
config.MinPodAge = annotationConfigs.MinPodAge
config.MinPodReadyAge = annotationConfigs.MinPodReadyAge
// configs specified in annotations takes precedence
// over labels
for k, v := range annotationConfigs.Configs {

35
pkg/collector/pod_collector.go
View File

@ -7,7 +7,6 @@ import (
"time"
log "github.com/sirupsen/logrus"
"github.com/zalando-incubator/kube-metrics-adapter/pkg/collector/httpmetrics"
autoscalingv2 "k8s.io/api/autoscaling/v2beta2"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
@ -15,6 +14,8 @@ import (
"k8s.io/apimachinery/pkg/labels"
"k8s.io/client-go/kubernetes"
"k8s.io/metrics/pkg/apis/custom_metrics"
"github.com/zalando-incubator/kube-metrics-adapter/pkg/collector/httpmetrics"
)
type PodCollectorPlugin struct {
@ -38,7 +39,7 @@ type PodCollector struct {
namespace string
metric autoscalingv2.MetricIdentifier
metricType autoscalingv2.MetricSourceType
minPodAge time.Duration
minPodReadyAge time.Duration
interval time.Duration
logger *log.Entry
httpClient *http.Client
@ -56,7 +57,7 @@ func NewPodCollector(client kubernetes.Interface, hpa *autoscalingv2.HorizontalP
namespace: hpa.Namespace,
metric: config.Metric,
metricType: config.Type,
minPodAge: config.MinPodAge,
minPodReadyAge: config.MinPodReadyAge,
interval: interval,
podLabelSelector: selector,
logger: log.WithFields(log.Fields{"Collector": "Pod"}),
@ -94,19 +95,19 @@ func (c *PodCollector) GetMetrics() ([]CollectedMetric, error) {
skippedPodsCount := 0
for _, pod := range pods.Items {
t := time.Now()
podAge := time.Duration(t.Sub(pod.ObjectMeta.CreationTimestamp.Time).Nanoseconds())
if podAge > c.minPodAge {
if IsPodReady(pod) {
isPodReady, podReadyAge := GetPodReadyAge(pod)
if isPodReady {
if podReadyAge > c.minPodReadyAge {
go c.getPodMetric(pod, ch, errCh)
} else {
skippedPodsCount++
c.logger.Warnf("Skipping metrics collection for pod %s because it's status is not Ready.", pod.Name)
c.logger.Warnf("Skipping metrics collection for pod %s because it's ready age is %s and min-pod-ready-age is set to %s", pod.Name, podReadyAge, c.minPodReadyAge)
}
} else {
skippedPodsCount++
c.logger.Warnf("Skipping metrics collection for pod %s because it's age is %s and min-pod-age is set to %s", pod.Name, podAge, c.minPodAge)
c.logger.Warnf("Skipping metrics collection for pod %s because it's status is not Ready.", pod.Name)
}
}
@ -170,17 +171,21 @@ func getPodLabelSelector(client kubernetes.Interface, hpa *autoscalingv2.Horizon
return nil, fmt.Errorf("unable to get pod label selector for scale target ref '%s'", hpa.Spec.ScaleTargetRef.Kind)
}
// IsPodReady extracts corev1.PodReady condition from the given pod object and
// returns the true if the condition corev1.PodReady is found. Returns -1 and false if the condition is not present.
func IsPodReady(pod corev1.Pod) bool {
// GetPodReadyAge extracts corev1.PodReady condition from the given pod object and
// returns true, time.Duration() for pod.LastTransitionTime if the condition corev1.PodReady is found. Returns time.Duration(0s), false if the condition is not present.
func GetPodReadyAge(pod corev1.Pod) (bool, time.Duration) {
t := time.Now()
podReadyAge := time.Duration(0 * time.Second)
conditions := pod.Status.Conditions
if conditions == nil {
return false
return false, podReadyAge
}
for i := range conditions {
if conditions[i].Type == corev1.PodReady {
return true
podReadyAge = time.Duration(t.Sub(conditions[i].LastTransitionTime.Time).Nanoseconds())
return true, podReadyAge
}
}
return false
return false, podReadyAge
}

45
pkg/collector/pod_collector_test.go
View File

@ -45,12 +45,12 @@ func TestPodCollector(t *testing.T) {
plugin := NewPodCollectorPlugin(client)
makeTestDeployment(t, client)
host, port, metricsHandler := makeTestHTTPServer(t, tc.metrics)
creationTimestamp := v1.NewTime(time.Now().Add(time.Duration(-30) * time.Second))
minPodAge := time.Duration(0 * time.Second)
podCondition := corev1.PodCondition{Type: corev1.PodReady, Status: corev1.ConditionStatus(corev1.PodRunning)}
makeTestPods(t, host, port, "test-metric", client, 5, creationTimestamp, podCondition)
lastReadyTransitionTimeTimestamp := v1.NewTime(time.Now().Add(time.Duration(-30) * time.Second))
minPodReadyAge := time.Duration(0 * time.Second)
podCondition := corev1.PodCondition{Type: corev1.PodReady, Status: corev1.ConditionStatus(corev1.PodRunning), LastTransitionTime: lastReadyTransitionTimeTimestamp}
makeTestPods(t, host, port, "test-metric", client, 5, podCondition)
testHPA := makeTestHPA(t, client)
testConfig := makeTestConfig(port, minPodAge)
testConfig := makeTestConfig(port, minPodReadyAge)
collector, err := plugin.NewCollector(testHPA, testConfig, testInterval)
require.NoError(t, err)
metrics, err := collector.GetMetrics()
@ -65,14 +65,14 @@ func TestPodCollector(t *testing.T) {
}
}
func TestPodCollectorWithMinPodAge(t *testing.T) {
func TestPodCollectorWithMinPodReadyAge(t *testing.T) {
for _, tc := range []struct {
name string
metrics [][]int64
result []int64
}{
{
name: "simple-with-min-pod-age",
name: "simple-with-min-pod-ready-age",
metrics: [][]int64{{1}, {3}, {8}, {5}, {2}},
result: []int64{},
},
@ -83,13 +83,13 @@ func TestPodCollectorWithMinPodAge(t *testing.T) {
makeTestDeployment(t, client)
host, port, metricsHandler := makeTestHTTPServer(t, tc.metrics)
// Setting pods age to 30 seconds
creationTimestamp := v1.NewTime(time.Now().Add(time.Duration(-30) * time.Second))
lastReadyTransitionTimeTimestamp := v1.NewTime(time.Now().Add(time.Duration(-30) * time.Second))
// Pods that are not older that 60 seconds (all in this case) should not be processed
minPodAge := time.Duration(60 * time.Second)
podCondition := corev1.PodCondition{Type: corev1.PodReady, Status: corev1.ConditionStatus(corev1.PodRunning)}
makeTestPods(t, host, port, "test-metric", client, 5, creationTimestamp, podCondition)
minPodReadyAge := time.Duration(60 * time.Second)
podCondition := corev1.PodCondition{Type: corev1.PodReady, Status: corev1.ConditionStatus(corev1.PodRunning), LastTransitionTime: lastReadyTransitionTimeTimestamp}
makeTestPods(t, host, port, "test-metric", client, 5, podCondition)
testHPA := makeTestHPA(t, client)
testConfig := makeTestConfig(port, minPodAge)
testConfig := makeTestConfig(port, minPodReadyAge)
collector, err := plugin.NewCollector(testHPA, testConfig, testInterval)
require.NoError(t, err)
metrics, err := collector.GetMetrics()
@ -121,13 +121,13 @@ func TestPodCollectorWithPodCondition(t *testing.T) {
plugin := NewPodCollectorPlugin(client)
makeTestDeployment(t, client)
host, port, metricsHandler := makeTestHTTPServer(t, tc.metrics)
creationTimestamp := v1.NewTime(time.Now().Add(time.Duration(-30) * time.Second))
minPodAge := time.Duration(0 * time.Second)
lastScheduledTransitionTimeTimestamp := v1.NewTime(time.Now().Add(time.Duration(-30) * time.Second))
minPodReadyAge := time.Duration(0 * time.Second)
//Pods in state corev1.PodScheduled should not be processed
podCondition := corev1.PodCondition{Type: corev1.PodScheduled, Status: corev1.ConditionStatus(corev1.PodRunning)}
makeTestPods(t, host, port, "test-metric", client, 5, creationTimestamp, podCondition)
podCondition := corev1.PodCondition{Type: corev1.PodScheduled, Status: corev1.ConditionStatus(corev1.PodRunning), LastTransitionTime: lastScheduledTransitionTimeTimestamp}
makeTestPods(t, host, port, "test-metric", client, 5, podCondition)
testHPA := makeTestHPA(t, client)
testConfig := makeTestConfig(port, minPodAge)
testConfig := makeTestConfig(port, minPodReadyAge)
collector, err := plugin.NewCollector(testHPA, testConfig, testInterval)
require.NoError(t, err)
metrics, err := collector.GetMetrics()
@ -175,15 +175,15 @@ func makeTestHTTPServer(t *testing.T, values [][]int64) (string, string, *testMe
return url.Hostname(), url.Port(), metricsHandler
}
func makeTestConfig(port string, minPodAge time.Duration) *MetricConfig {
func makeTestConfig(port string, minPodReadyAge time.Duration) *MetricConfig {
return &MetricConfig{
CollectorType: "json-path",
Config: map[string]string{"json-key": "$.values", "port": port, "path": "/metrics", "aggregator": "sum"},
MinPodAge: minPodAge,
CollectorType: "json-path",
Config: map[string]string{"json-key": "$.values", "port": port, "path": "/metrics", "aggregator": "sum"},
MinPodReadyAge: minPodReadyAge,
}
}
func makeTestPods(t *testing.T, testServer string, metricName string, port string, client kubernetes.Interface, replicas int, creationTimestamp v1.Time, podCondition corev1.PodCondition) {
func makeTestPods(t *testing.T, testServer string, metricName string, port string, client kubernetes.Interface, replicas int, podCondition corev1.PodCondition) {
for i := 0; i < replicas; i++ {
testPod := &corev1.Pod{
ObjectMeta: v1.ObjectMeta{
@ -192,7 +192,6 @@ func makeTestPods(t *testing.T, testServer string, metricName string, port strin
Annotations: map[string]string{
fmt.Sprintf("metric-config.pods.%s.json-path/port", metricName): port,
},
CreationTimestamp: creationTimestamp,
},
Status: corev1.PodStatus{
PodIP: testServer,