From f0b817629cf892910409541fa197fec94c815fe7 Mon Sep 17 00:00:00 2001 From: Abe Friesen <2319792+doyshinda@users.noreply.github.com> Date: Fri, 15 May 2020 20:48:46 -0600 Subject: [PATCH] Add support for custom timeouts Allow configuration per metric for connect and request timeouts when querying pods for JSON metrics. Signed-off-by: Abe Friesen <2319792+doyshinda@users.noreply.github.com> --- README.md | 8 +++ go.sum | 4 ++ pkg/collector/httpmetrics/json_path.go | 13 +++- pkg/collector/httpmetrics/pod_metrics.go | 29 ++++++++- pkg/collector/httpmetrics/pod_metrics_test.go | 65 +++++++++++++++++++ 5 files changed, 115 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 9056b1d7..eec3ef01 100644 --- a/README.md +++ b/README.md @@ -166,6 +166,14 @@ will create a URL like this: http://:9090/metrics?foo=bar&baz=bop ``` +There are also configuration options for custom (connect and request) timeouts when querying pods for metrics: +```yaml +metric-config.pods.requests-per-second.json-path/request-timeout: 2s +metric-config.pods.requests-per-second.json-path/connect-timeout: 500ms +``` + +The default for both of the above values is 15 seconds. + ## Prometheus collector The Prometheus collector is a generic collector which can map Prometheus diff --git a/go.sum b/go.sum index 8f14b6f2..0b6d7087 100644 --- a/go.sum +++ b/go.sum @@ -166,6 +166,7 @@ github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -354,6 +355,7 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.0.11 h1:DhHlBtkHWPYi8O2y31JkK0TF+DGM+51OopZjH/Ia5qI= github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= @@ -504,6 +506,7 @@ golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgmDSfjglYZ3vStQ/gSCU= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f h1:gWF768j/LaZugp8dyS4UwsslYCYz9XgFxvlgsn0n9H8= golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -563,6 +566,7 @@ google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLY google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pkg/collector/httpmetrics/json_path.go b/pkg/collector/httpmetrics/json_path.go index 2c576fd4..bb0244dc 100644 --- a/pkg/collector/httpmetrics/json_path.go +++ b/pkg/collector/httpmetrics/json_path.go @@ -26,21 +26,28 @@ func NewJSONPathMetricsGetter(httpClient *http.Client, aggregatorFunc Aggregator return &JSONPathMetricsGetter{client: httpClient, aggregator: aggregatorFunc, jsonPath: compiledPath} } -func DefaultMetricsHTTPClient() *http.Client { +var DefaultRequestTimeout = 15 * time.Second +var DefaultConnectTimeout = 15 * time.Second + +func CustomMetricsHTTPClient(requestTimeout time.Duration, connectTimeout time.Duration) *http.Client { client := &http.Client{ Transport: &http.Transport{ DialContext: (&net.Dialer{ - Timeout: 15 * time.Second, + Timeout: connectTimeout, }).DialContext, MaxIdleConns: 50, IdleConnTimeout: 90 * time.Second, ExpectContinueTimeout: 1 * time.Second, }, - Timeout: 15 * time.Second, + Timeout: requestTimeout, } return client } +func DefaultMetricsHTTPClient() *http.Client { + return CustomMetricsHTTPClient(DefaultRequestTimeout, DefaultConnectTimeout) +} + // GetMetric gets metric from pod by fetching json metrics from the pods metric // endpoint and extracting the desired value using the specified json path // query. diff --git a/pkg/collector/httpmetrics/pod_metrics.go b/pkg/collector/httpmetrics/pod_metrics.go index ae08d854..baa421ab 100644 --- a/pkg/collector/httpmetrics/pod_metrics.go +++ b/pkg/collector/httpmetrics/pod_metrics.go @@ -4,6 +4,7 @@ import ( "fmt" "net/url" "strconv" + "time" "github.com/oliveagle/jsonpath" v1 "k8s.io/api/core/v1" @@ -72,7 +73,33 @@ func NewPodMetricsJSONPathGetter(config map[string]string) (*PodMetricsJSONPathG return nil, err } } - getter.metricGetter = NewJSONPathMetricsGetter(DefaultMetricsHTTPClient(), aggregator, jsonPath) + + requestTimeout := DefaultRequestTimeout + connectTimeout := DefaultConnectTimeout + + if v, ok := config["request-timeout"]; ok { + d, err := time.ParseDuration(v) + if err != nil { + return nil, err + } + if d < 0 { + return nil, fmt.Errorf("Invalid request-timeout config value: %s", v) + } + requestTimeout = d + } + + if v, ok := config["connect-timeout"]; ok { + d, err := time.ParseDuration(v) + if err != nil { + return nil, err + } + if d < 0 { + return nil, fmt.Errorf("Invalid connect-timeout config value: %s", v) + } + connectTimeout = d + } + + getter.metricGetter = NewJSONPathMetricsGetter(CustomMetricsHTTPClient(requestTimeout, connectTimeout), aggregator, jsonPath) return &getter, nil } diff --git a/pkg/collector/httpmetrics/pod_metrics_test.go b/pkg/collector/httpmetrics/pod_metrics_test.go index 350365e4..232ac75f 100644 --- a/pkg/collector/httpmetrics/pod_metrics_test.go +++ b/pkg/collector/httpmetrics/pod_metrics_test.go @@ -3,6 +3,7 @@ package httpmetrics import ( "fmt" "testing" + "time" "github.com/oliveagle/jsonpath" "github.com/stretchr/testify/require" @@ -131,3 +132,67 @@ func TestBuildMetricsURL(t *testing.T) { receivedURLNoQuery := getterWithNoQuery.buildMetricsURL(ip) require.Equal(t, receivedURLNoQuery.String(), expectedURLNoQuery) } + +func TestCustomTimeouts(t *testing.T) { + scheme := "http" + port := "9090" + path := "/v1/test/" + + // Test no custom options results in default timeouts + defaultConfig := map[string]string{ + "json-key": "$.value", + "scheme": scheme, + "path": path, + "port": port, + } + defaultTime := time.Duration(15000) * time.Millisecond + + defaultGetter, err1 := NewPodMetricsJSONPathGetter(defaultConfig) + require.NoError(t, err1) + require.Equal(t, defaultGetter.metricGetter.client.Timeout, defaultTime) + + // Test with custom request timeout + configWithRequestTimeout := map[string]string{ + "json-key": "$.value", + "scheme": scheme, + "path": path, + "port": port, + "request-timeout": "978ms", + } + exectedTimeout := time.Duration(978) * time.Millisecond + customRequestGetter, err2 := NewPodMetricsJSONPathGetter(configWithRequestTimeout) + require.NoError(t, err2) + require.Equal(t, customRequestGetter.metricGetter.client.Timeout, exectedTimeout) + + // Test with custom connect timeout. Unfortunately, it seems there's no way to access the + // connect timeout of the client struct to actually verify it's set :/ + configWithConnectTimeout := map[string]string{ + "json-key": "$.value", + "scheme": scheme, + "path": path, + "port": port, + "connect-timeout": "512ms", + } + customRequestGetter, err3 := NewPodMetricsJSONPathGetter(configWithConnectTimeout) + require.NoError(t, err3) + + configWithInvalidTimeout := map[string]string{ + "json-key": "$.value", + "scheme": scheme, + "path": path, + "port": port, + "request-timeout": "-256ms", + } + _, err4 := NewPodMetricsJSONPathGetter(configWithInvalidTimeout) + require.Error(t, err4) + + configWithInvalidTimeout = map[string]string{ + "json-key": "$.value", + "scheme": scheme, + "path": path, + "port": port, + "connect-timeout": "-256ms", + } + _, err5 := NewPodMetricsJSONPathGetter(configWithInvalidTimeout) + require.Error(t, err5) +}