diff --git a/README.md b/README.md index eec3ef01..610b73bf 100644 --- a/README.md +++ b/README.md @@ -142,7 +142,7 @@ example the following JSON data would be expected: ``` The json-path query support depends on the -[github.com/oliveagle/jsonpath](https://github.com/oliveagle/jsonpath) library. +[github.com/spyzhov/ajson](https://github.com/spyzhov/ajson) library. See the README for possible queries. It's expected that the metric you query returns something that can be turned into a `float64`. diff --git a/go.mod b/go.mod index 4a3b0048..14190e85 100644 --- a/go.mod +++ b/go.mod @@ -8,12 +8,12 @@ require ( github.com/kubernetes-incubator/custom-metrics-apiserver v0.0.0-20200618121405-54026617ec44 github.com/lib/pq v1.2.0 // indirect github.com/mattn/go-colorable v0.1.4 // indirect - github.com/oliveagle/jsonpath v0.0.0-20180606110733-2e52cf6e6852 github.com/onsi/gomega v1.8.1 // indirect github.com/prometheus/client_golang v1.7.1 github.com/prometheus/common v0.10.0 github.com/sirupsen/logrus v1.6.0 github.com/spf13/cobra v0.0.7 + github.com/spyzhov/ajson v0.4.2 github.com/stretchr/testify v1.6.1 github.com/zalando-incubator/cluster-lifecycle-manager v0.0.0-20180921141935-824b77fb1f84 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 diff --git a/go.sum b/go.sum index fcdf5631..587f13af 100644 --- a/go.sum +++ b/go.sum @@ -302,8 +302,6 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/oliveagle/jsonpath v0.0.0-20180606110733-2e52cf6e6852 h1:Yl0tPBa8QPjGmesFh1D0rDy+q1Twx6FyU7VWHi8wZbI= -github.com/oliveagle/jsonpath v0.0.0-20180606110733-2e52cf6e6852/go.mod h1:eqOVx5Vwu4gd2mmMZvVZsgIqNSaW3xxRThUJ0k/TPk4= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= @@ -381,6 +379,8 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/spyzhov/ajson v0.4.2 h1:JMByd/jZApPKDvNsmO90X2WWGbmT2ahDFp73QhZbg3s= +github.com/spyzhov/ajson v0.4.2/go.mod h1:63V+CGM6f1Bu/p4nLIN8885ojBdt88TbLoSFzyqMuVA= github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/pkg/collector/http_collector.go b/pkg/collector/http_collector.go index 27495c4a..1e4ce578 100644 --- a/pkg/collector/http_collector.go +++ b/pkg/collector/http_collector.go @@ -7,7 +7,6 @@ import ( "github.com/zalando-incubator/kube-metrics-adapter/pkg/collector/httpmetrics" - "github.com/oliveagle/jsonpath" "k8s.io/api/autoscaling/v2beta2" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -36,14 +35,12 @@ func (p *HTTPCollectorPlugin) NewCollector(_ *v2beta2.HorizontalPodAutoscaler, c if value, ok = config.Config[HTTPJsonPathAnnotationKey]; !ok { return nil, fmt.Errorf("config value %s not found", HTTPJsonPathAnnotationKey) } - jsonPath, err := jsonpath.Compile(value) - if err != nil { - return nil, fmt.Errorf("failed to parse json path: %v", err) - } - collector.jsonPath = jsonPath + jsonPath := value + if value, ok = config.Config[HTTPEndpointAnnotationKey]; !ok { return nil, fmt.Errorf("config value %s not found", HTTPEndpointAnnotationKey) } + var err error collector.endpoint, err = url.Parse(value) if err != nil { return nil, err @@ -65,13 +62,16 @@ func (p *HTTPCollectorPlugin) NewCollector(_ *v2beta2.HorizontalPodAutoscaler, c return nil, err } } - collector.metricsGetter = httpmetrics.NewJSONPathMetricsGetter(httpmetrics.DefaultMetricsHTTPClient(), aggFunc, jsonPath) + jsonPathGetter, err := httpmetrics.NewJSONPathMetricsGetter(httpmetrics.DefaultMetricsHTTPClient(), aggFunc, jsonPath) + if err != nil { + return nil, err + } + collector.metricsGetter = jsonPathGetter return collector, nil } type HTTPCollector struct { endpoint *url.URL - jsonPath *jsonpath.Compiled interval time.Duration metricType v2beta2.MetricSourceType metricsGetter *httpmetrics.JSONPathMetricsGetter diff --git a/pkg/collector/httpmetrics/json_path.go b/pkg/collector/httpmetrics/json_path.go index bb0244dc..fdaf6345 100644 --- a/pkg/collector/httpmetrics/json_path.go +++ b/pkg/collector/httpmetrics/json_path.go @@ -1,7 +1,6 @@ package httpmetrics import ( - "encoding/json" "fmt" "io/ioutil" "net" @@ -9,21 +8,26 @@ import ( "net/url" "time" - "github.com/oliveagle/jsonpath" + "github.com/spyzhov/ajson" ) // JSONPathMetricsGetter is a metrics getter which looks up pod metrics by // querying the pods metrics endpoint and lookup the metric value as defined by // the json path query. type JSONPathMetricsGetter struct { - jsonPath *jsonpath.Compiled + jsonPath string aggregator AggregatorFunc client *http.Client } // NewJSONPathMetricsGetter initializes a new JSONPathMetricsGetter. -func NewJSONPathMetricsGetter(httpClient *http.Client, aggregatorFunc AggregatorFunc, compiledPath *jsonpath.Compiled) *JSONPathMetricsGetter { - return &JSONPathMetricsGetter{client: httpClient, aggregator: aggregatorFunc, jsonPath: compiledPath} +func NewJSONPathMetricsGetter(httpClient *http.Client, aggregatorFunc AggregatorFunc, jsonPath string) (*JSONPathMetricsGetter, error) { + // check that jsonPath parses + _, err := ajson.ParseJSONPath(jsonPath) + if err != nil { + return nil, err + } + return &JSONPathMetricsGetter{client: httpClient, aggregator: aggregatorFunc, jsonPath: jsonPath}, nil } var DefaultRequestTimeout = 15 * time.Second @@ -58,57 +62,45 @@ func (g *JSONPathMetricsGetter) GetMetric(metricsURL url.URL) (float64, error) { } // parse data - var jsonData interface{} - err = json.Unmarshal(data, &jsonData) + root, err := ajson.Unmarshal(data) if err != nil { return 0, err } - res, err := g.jsonPath.Lookup(jsonData) + nodes, err := root.JSONPath(g.jsonPath) if err != nil { return 0, err } - switch res := res.(type) { - case int: - return float64(res), nil - case float32: - return float64(res), nil - case float64: - return res, nil - case []interface{}: + if len(nodes) > 1 { + return 0, fmt.Errorf("unexpected json: expected single numeric or array value") + } + + node := nodes[0] + if node.IsArray() { if g.aggregator == nil { return 0, fmt.Errorf("no aggregator function has been specified") } - s, err := castSlice(res) - if err != nil { - return 0, err + values := make([]float64, 0, len(nodes)) + items, _ := node.GetArray() + for _, item := range items { + value, err := item.GetNumeric() + if err != nil { + return 0, fmt.Errorf("did not find numeric type: %w", err) + } + values = append(values, value) } - return g.aggregator(s...), nil - default: - return 0, fmt.Errorf("unsupported type %T", res) + return g.aggregator(values...), nil + } else if node.IsNumeric() { + res, _ := node.GetNumeric() + return res, nil } -} -// castSlice takes a slice of interface and returns a slice of float64 if all -// values in slice were castable, else returns an error -func castSlice(in []interface{}) ([]float64, error) { - var out []float64 - - for _, v := range in { - switch v := v.(type) { - case int: - out = append(out, float64(v)) - case float32: - out = append(out, float64(v)) - case float64: - out = append(out, v) - default: - return nil, fmt.Errorf("slice was returned by JSONPath, but value inside is unsupported: %T", v) - } + value, err := node.Value() + if err != nil { + return 0, fmt.Errorf("failed to check value of jsonPath result: %w", err) } - - return out, nil + return 0, fmt.Errorf("unsupported type %T", value) } func (g *JSONPathMetricsGetter) fetchMetrics(metricsURL url.URL) ([]byte, error) { diff --git a/pkg/collector/httpmetrics/json_path_test.go b/pkg/collector/httpmetrics/json_path_test.go index c6fccef2..88c59900 100644 --- a/pkg/collector/httpmetrics/json_path_test.go +++ b/pkg/collector/httpmetrics/json_path_test.go @@ -1,59 +1,20 @@ package httpmetrics import ( - "encoding/json" "fmt" "net/http" "net/http/httptest" "net/url" "testing" - "github.com/oliveagle/jsonpath" "github.com/stretchr/testify/require" ) -func TestCastSlice(t *testing.T) { - res1, err1 := castSlice([]interface{}{1, 2, 3}) - require.NoError(t, err1) - require.Equal(t, []float64{1.0, 2.0, 3.0}, res1) - - res2, err2 := castSlice([]interface{}{float32(1.0), float32(2.0), float32(3.0)}) - require.NoError(t, err2) - require.Equal(t, []float64{1.0, 2.0, 3.0}, res2) - - res3, err3 := castSlice([]interface{}{float64(1.0), float64(2.0), float64(3.0)}) - require.NoError(t, err3) - require.Equal(t, []float64{1.0, 2.0, 3.0}, res3) - - res4, err4 := castSlice([]interface{}{1, 2, "some string"}) - require.Errorf(t, err4, "slice was returned by JSONPath, but value inside is unsupported: %T", "string") - require.Equal(t, []float64(nil), res4) -} - -type testValueResponse struct { - Value int64 `json:"value"` -} - -type testValueArrayResponse struct { - Value []int64 `json:"value"` -} - -func makeTestHTTPServer(t *testing.T, values ...int64) *httptest.Server { +func makeTestHTTPServer(t *testing.T, response []byte) *httptest.Server { h := func(w http.ResponseWriter, r *http.Request) { require.Equal(t, r.URL.Path, "/metrics") w.Header().Set("Content-Type", "application/json") - var ( - response []byte - err error - ) - if len(values) == 1 { - response, err = json.Marshal(testValueResponse{Value: values[0]}) - require.NoError(t, err) - } else { - response, err = json.Marshal(testValueArrayResponse{Value: values}) - require.NoError(t, err) - } - _, err = w.Write(response) + _, err := w.Write(response) require.NoError(t, err) } return httptest.NewServer(http.HandlerFunc(h)) @@ -61,29 +22,44 @@ func makeTestHTTPServer(t *testing.T, values ...int64) *httptest.Server { func TestJSONPathMetricsGetter(t *testing.T) { for _, tc := range []struct { - name string - input []int64 - output float64 - aggregator AggregatorFunc + name string + jsonResponse []byte + jsonPath string + result float64 + aggregator AggregatorFunc }{ { - name: "basic average", - input: []int64{3, 4, 5}, - output: 4, - aggregator: Average, + name: "basic single value", + jsonResponse: []byte(`{"value":3}`), + jsonPath: "$.value", + result: 3, + aggregator: Average, + }, + { + name: "basic average", + jsonResponse: []byte(`{"value":[3,4,5]}`), + jsonPath: "$.value", + result: 4, + aggregator: Average, + }, + { + name: "dotted key", + jsonResponse: []byte(`{"metric.value":5}`), + jsonPath: "$['metric.value']", + result: 5, + aggregator: Average, }, } { t.Run(tc.name, func(t *testing.T) { - server := makeTestHTTPServer(t, tc.input...) + server := makeTestHTTPServer(t, tc.jsonResponse) defer server.Close() - path, err := jsonpath.Compile("$.value") + getter, err := NewJSONPathMetricsGetter(DefaultMetricsHTTPClient(), tc.aggregator, tc.jsonPath) require.NoError(t, err) - getter := NewJSONPathMetricsGetter(DefaultMetricsHTTPClient(), tc.aggregator, path) url, err := url.Parse(fmt.Sprintf("%s/metrics", server.URL)) require.NoError(t, err) metric, err := getter.GetMetric(*url) require.NoError(t, err) - require.Equal(t, tc.output, metric) + require.Equal(t, tc.result, metric) }) } } diff --git a/pkg/collector/httpmetrics/pod_metrics.go b/pkg/collector/httpmetrics/pod_metrics.go index baa421ab..35bc5f7d 100644 --- a/pkg/collector/httpmetrics/pod_metrics.go +++ b/pkg/collector/httpmetrics/pod_metrics.go @@ -6,7 +6,6 @@ import ( "strconv" "time" - "github.com/oliveagle/jsonpath" v1 "k8s.io/api/core/v1" ) @@ -33,18 +32,13 @@ func (g PodMetricsJSONPathGetter) GetMetric(pod *v1.Pod) (float64, error) { func NewPodMetricsJSONPathGetter(config map[string]string) (*PodMetricsJSONPathGetter, error) { getter := PodMetricsJSONPathGetter{} var ( - jsonPath *jsonpath.Compiled + jsonPath string aggregator AggregatorFunc err error ) if v, ok := config["json-key"]; ok { - path, err := jsonpath.Compile(v) - if err != nil { - return nil, fmt.Errorf("failed to parse json path definition: %v", err) - } - - jsonPath = path + jsonPath = v } if v, ok := config["scheme"]; ok { @@ -99,7 +93,11 @@ func NewPodMetricsJSONPathGetter(config map[string]string) (*PodMetricsJSONPathG connectTimeout = d } - getter.metricGetter = NewJSONPathMetricsGetter(CustomMetricsHTTPClient(requestTimeout, connectTimeout), aggregator, jsonPath) + jsonPathGetter, err := NewJSONPathMetricsGetter(CustomMetricsHTTPClient(requestTimeout, connectTimeout), aggregator, jsonPath) + if err != nil { + return nil, err + } + getter.metricGetter = jsonPathGetter return &getter, nil } diff --git a/pkg/collector/httpmetrics/pod_metrics_test.go b/pkg/collector/httpmetrics/pod_metrics_test.go index 232ac75f..202d79c9 100644 --- a/pkg/collector/httpmetrics/pod_metrics_test.go +++ b/pkg/collector/httpmetrics/pod_metrics_test.go @@ -5,7 +5,6 @@ import ( "testing" "time" - "github.com/oliveagle/jsonpath" "github.com/stretchr/testify/require" ) @@ -23,12 +22,11 @@ func TestNewPodJSONPathMetricsGetter(t *testing.T) { "path": "/metrics", "port": "9090", } - jpath1, _ := jsonpath.Compile(configNoAggregator["json-key"]) getterNoAggregator, err1 := NewPodMetricsJSONPathGetter(configNoAggregator) require.NoError(t, err1) compareMetricsGetter(t, &PodMetricsJSONPathGetter{ - metricGetter: &JSONPathMetricsGetter{jsonPath: jpath1}, + metricGetter: &JSONPathMetricsGetter{jsonPath: configNoAggregator["json-key"]}, scheme: "http", path: "/metrics", port: 9090, @@ -41,12 +39,11 @@ func TestNewPodJSONPathMetricsGetter(t *testing.T) { "port": "9090", "aggregator": "avg", } - jpath2, _ := jsonpath.Compile(configAggregator["json-key"]) getterAggregator, err2 := NewPodMetricsJSONPathGetter(configAggregator) require.NoError(t, err2) compareMetricsGetter(t, &PodMetricsJSONPathGetter{ - metricGetter: &JSONPathMetricsGetter{jsonPath: jpath2, aggregator: Average}, + metricGetter: &JSONPathMetricsGetter{jsonPath: configAggregator["json-key"], aggregator: Average}, scheme: "http", path: "/metrics", port: 9090, @@ -79,12 +76,11 @@ func TestNewPodJSONPathMetricsGetter(t *testing.T) { "port": "9090", "raw-query": "foo=bar&baz=bop", } - jpath5, _ := jsonpath.Compile(configWithRawQuery["json-key"]) getterWithRawQuery, err5 := NewPodMetricsJSONPathGetter(configWithRawQuery) require.NoError(t, err5) compareMetricsGetter(t, &PodMetricsJSONPathGetter{ - metricGetter: &JSONPathMetricsGetter{jsonPath: jpath5}, + metricGetter: &JSONPathMetricsGetter{jsonPath: configWithRawQuery["json-key"]}, scheme: "http", path: "/metrics", port: 9090, @@ -107,8 +103,6 @@ func TestBuildMetricsURL(t *testing.T) { "port": port, "raw-query": rawQuery, } - _, err := jsonpath.Compile(configWithRawQuery["json-key"]) - require.NoError(t, err) getterWithRawQuery, err1 := NewPodMetricsJSONPathGetter(configWithRawQuery) require.NoError(t, err1) @@ -123,8 +117,6 @@ func TestBuildMetricsURL(t *testing.T) { "path": path, "port": port, } - _, err2 := jsonpath.Compile(configWithNoQuery["json-key"]) - require.NoError(t, err2) getterWithNoQuery, err3 := NewPodMetricsJSONPathGetter(configWithNoQuery) require.NoError(t, err3) @@ -140,10 +132,10 @@ func TestCustomTimeouts(t *testing.T) { // Test no custom options results in default timeouts defaultConfig := map[string]string{ - "json-key": "$.value", - "scheme": scheme, - "path": path, - "port": port, + "json-key": "$.value", + "scheme": scheme, + "path": path, + "port": port, } defaultTime := time.Duration(15000) * time.Millisecond @@ -153,10 +145,10 @@ func TestCustomTimeouts(t *testing.T) { // Test with custom request timeout configWithRequestTimeout := map[string]string{ - "json-key": "$.value", - "scheme": scheme, - "path": path, - "port": port, + "json-key": "$.value", + "scheme": scheme, + "path": path, + "port": port, "request-timeout": "978ms", } exectedTimeout := time.Duration(978) * time.Millisecond @@ -167,30 +159,30 @@ func TestCustomTimeouts(t *testing.T) { // Test with custom connect timeout. Unfortunately, it seems there's no way to access the // connect timeout of the client struct to actually verify it's set :/ configWithConnectTimeout := map[string]string{ - "json-key": "$.value", - "scheme": scheme, - "path": path, - "port": port, + "json-key": "$.value", + "scheme": scheme, + "path": path, + "port": port, "connect-timeout": "512ms", } - customRequestGetter, err3 := NewPodMetricsJSONPathGetter(configWithConnectTimeout) + _, err3 := NewPodMetricsJSONPathGetter(configWithConnectTimeout) require.NoError(t, err3) configWithInvalidTimeout := map[string]string{ - "json-key": "$.value", - "scheme": scheme, - "path": path, - "port": port, + "json-key": "$.value", + "scheme": scheme, + "path": path, + "port": port, "request-timeout": "-256ms", } _, err4 := NewPodMetricsJSONPathGetter(configWithInvalidTimeout) require.Error(t, err4) configWithInvalidTimeout = map[string]string{ - "json-key": "$.value", - "scheme": scheme, - "path": path, - "port": port, + "json-key": "$.value", + "scheme": scheme, + "path": path, + "port": port, "connect-timeout": "-256ms", } _, err5 := NewPodMetricsJSONPathGetter(configWithInvalidTimeout)