Reuse the HTTP client for scraping pods (#102)

Signed-off-by: Arjun Naik <[email protected]>

main.go

@@ -18,6 +18,7 @@ package main
 
 import (
 	"flag"
+	_ "net/http/pprof"
 	"os"
 	"runtime"
 

pkg/collector/json_path_collector.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"math"
+	"net"
 	"net/http"
 	"net/url"
 	"strconv"
@@ -23,11 +24,13 @@ type JSONPathMetricsGetter struct {
 	path       string
 	port       int
 	aggregator string
+	client     *http.Client
 }
 
 // NewJSONPathMetricsGetter initializes a new JSONPathMetricsGetter.
 func NewJSONPathMetricsGetter(config map[string]string) (*JSONPathMetricsGetter, error) {
-	getter := &JSONPathMetricsGetter{}
+	httpClient := defaultHTTPClient()
+	getter := &JSONPathMetricsGetter{client: httpClient}
 
 	if v, ok := config["json-key"]; ok {
 		path, err := jsonpath.Compile(v)
@@ -61,11 +64,26 @@ func NewJSONPathMetricsGetter(config map[string]string) (*JSONPathMetricsGetter,
 	return getter, nil
 }
 
+func defaultHTTPClient() *http.Client {
+	client := &http.Client{
+		Transport: &http.Transport{
+			DialContext: (&net.Dialer{
+				Timeout: 15 * time.Second,
+			}).DialContext,
+			MaxIdleConns:          50,
+			IdleConnTimeout:       90 * time.Second,
+			ExpectContinueTimeout: 1 * time.Second,
+		},
+		Timeout: 15 * time.Second,
+	}
+	return client
+}
+
 // GetMetric gets metric from pod by fetching json metrics from the pods metric
 // endpoint and extracting the desired value using the specified json path
 // query.
 func (g *JSONPathMetricsGetter) GetMetric(pod *corev1.Pod) (float64, error) {
-	data, err := getPodMetrics(pod, g.scheme, g.path, g.port)
+	data, err := g.getPodMetrics(pod, g.scheme, g.path, g.port)
 	if err != nil {
 		return 0, err
 	}
@@ -122,16 +140,11 @@ func castSlice(in []interface{}) ([]float64, error) {
 }
 
 // getPodMetrics returns the content of the pods metrics endpoint.
-func getPodMetrics(pod *corev1.Pod, scheme, path string, port int) ([]byte, error) {
+func (g *JSONPathMetricsGetter) getPodMetrics(pod *corev1.Pod, scheme, path string, port int) ([]byte, error) {
 	if pod.Status.PodIP == "" {
 		return nil, fmt.Errorf("pod %s/%s does not have a pod IP", pod.Namespace, pod.Namespace)
 	}
 
-	httpClient := &http.Client{
-		Timeout:   15 * time.Second,
-		Transport: &http.Transport{},
-	}
-
 	if scheme == "" {
 		scheme = "http"
 	}
@@ -147,7 +160,7 @@ func getPodMetrics(pod *corev1.Pod, scheme, path string, port int) ([]byte, erro
 		return nil, err
 	}
 
-	resp, err := httpClient.Do(request)
+	resp, err := g.client.Do(request)
 	if err != nil {
 		return nil, err
 	}

pkg/collector/json_path_collector_test.go

@@ -7,6 +7,13 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+func compareMetricsGetter(t *testing.T, first, second *JSONPathMetricsGetter) {
+	require.Equal(t, first.jsonPath, second.jsonPath)
+	require.Equal(t, first.scheme, second.scheme)
+	require.Equal(t, first.path, second.path)
+	require.Equal(t, first.port, second.port)
+}
+
 func TestNewJSONPathMetricsGetter(t *testing.T) {
 	configNoAggregator := map[string]string{
 		"json-key": "$.value",
@@ -18,7 +25,7 @@ func TestNewJSONPathMetricsGetter(t *testing.T) {
 	getterNoAggregator, err1 := NewJSONPathMetricsGetter(configNoAggregator)
 
 	require.NoError(t, err1)
-	require.Equal(t, &JSONPathMetricsGetter{
+	compareMetricsGetter(t, &JSONPathMetricsGetter{
 		jsonPath: jpath1,
 		scheme:   "http",
 		path:     "/metrics",
@@ -36,7 +43,7 @@ func TestNewJSONPathMetricsGetter(t *testing.T) {
 	getterAggregator, err2 := NewJSONPathMetricsGetter(configAggregator)
 
 	require.NoError(t, err2)
-	require.Equal(t, &JSONPathMetricsGetter{
+	compareMetricsGetter(t, &JSONPathMetricsGetter{
 		jsonPath:   jpath2,
 		scheme:     "http",
 		path:       "/metrics",

pkg/collector/pod_collector.go

@@ -2,6 +2,7 @@ package collector
 
 import (
 	"fmt"
+	"net/http"
 	"time"
 
 	log "github.com/sirupsen/logrus"
@@ -37,6 +38,7 @@ type PodCollector struct {
 	metricType       autoscalingv2.MetricSourceType
 	interval         time.Duration
 	logger           *log.Entry
+	httpClient       *http.Client
 }
 
 type PodMetricsGetter interface {