diff --git a/parser/csv.go b/parser/csv.go index e92b4d1a..5f75311e 100644 --- a/parser/csv.go +++ b/parser/csv.go @@ -18,7 +18,6 @@ import ( "bytes" "encoding/csv" "fmt" - "strconv" "strings" "sync" "time" @@ -26,6 +25,7 @@ import ( "github.com/housepower/clickhouse_sinker/model" "github.com/housepower/clickhouse_sinker/util" "github.com/pkg/errors" + "github.com/valyala/fastjson/fastfloat" ) var _ Parser = (*CsvParser)(nil) @@ -87,7 +87,7 @@ func (c *CsvMetric) GetFloat(key string, nullable bool) (val interface{}) { val = float64(0.0) return } - val, _ = strconv.ParseFloat(c.values[idx], 64) + val = fastfloat.ParseBestEffort(c.values[idx]) return } @@ -102,7 +102,7 @@ func (c *CsvMetric) GetInt(key string, nullable bool) (val interface{}) { val = int64(0) return } - val, _ = strconv.ParseInt(c.values[idx], 10, 64) + val = fastfloat.ParseInt64BestEffort(c.values[idx]) return } @@ -148,20 +148,16 @@ func (c *CsvMetric) GetArray(key string, typ int) (val interface{}) { switch typ { case model.Int: results := make([]int64, 0, len(array)) - var v int64 for _, e := range array { - if v, err = strconv.ParseInt(e, 10, 64); err == nil { - results = append(results, v) - } + v := fastfloat.ParseInt64BestEffort(e) + results = append(results, v) } val = results case model.Float: results := make([]float64, 0, len(array)) - var v float64 for _, e := range array { - if v, err = strconv.ParseFloat(e, 64); err == nil { - results = append(results, v) - } + v := fastfloat.ParseBestEffort(e) + results = append(results, v) } val = results case model.String: diff --git a/parser/gjson.go b/parser/gjson.go index 2ca04118..eb91396b 100644 --- a/parser/gjson.go +++ b/parser/gjson.go @@ -82,12 +82,7 @@ func (c *GjsonMetric) GetInt(key string, nullable bool) (val interface{}) { val = int64(0) return } - switch r.Type { - case gjson.Number: - val = int64(r.Num) - default: - val = int64(0) - } + val = r.Int() return } diff --git a/parser/parser_test.go b/parser/parser_test.go index 5eccb8a9..3bedd578 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -46,7 +46,7 @@ var jsonSample = []byte(`{ "time_ms_rfc3339_1": "2019-12-16T12:10:30.123Z", "time_ms_rfc3339_2": "2019-12-16T12:10:30.123+08:00", "time_ms_clickhouse_1": "2019-12-16 12:10:30.123", - "array_int": [1,2,3], + "array_int": [-9223372036854775808,9223372036854775807], "array_float": [1.1,2.2,3.3], "array_string": ["aa","bb","cc"], "array_date": ["2000-01-01","2000-01-02","2000-01-03"], @@ -85,7 +85,7 @@ var jsonSchema = map[string]string{ "bool_false": "false", } -var csvSample = []byte(`1536813227,"0.11","escaped_""ws","{""i"":[1,2,3],""f"":[1.1,2.2,3.3],""s"":[""aa"",""bb"",""cc""],""e"":[]}",2019-12-16,2019-12-16T12:10:30Z,2019-12-16T12:10:30+08:00,2019-12-16 12:10:30,2019-12-16T12:10:30.123Z,2019-12-16T12:10:30.123+08:00,2019-12-16 12:10:30.123,"[1,2,3]","[1.1,2.2,3.3]","[""aa"",""bb"",""cc""]","[""2000-01-01"",""2000-01-02"",""2000-01-03""]","[{""i"":[1,2,3],""f"":[1.1,2.2,3.3]},{""s"":[""aa"",""bb"",""cc""],""e"":[]}]","[]","true","false"`) +var csvSample = []byte(`1536813227,"0.11","escaped_""ws","{""i"":[1,2,3],""f"":[1.1,2.2,3.3],""s"":[""aa"",""bb"",""cc""],""e"":[]}",2019-12-16,2019-12-16T12:10:30Z,2019-12-16T12:10:30+08:00,2019-12-16 12:10:30,2019-12-16T12:10:30.123Z,2019-12-16T12:10:30.123+08:00,2019-12-16 12:10:30.123,"[-9223372036854775808,9223372036854775807]","[1.1,2.2,3.3]","[""aa"",""bb"",""cc""]","[""2000-01-01"",""2000-01-02"",""2000-01-03""]","[{""i"":[1,2,3],""f"":[1.1,2.2,3.3]},{""s"":[""aa"",""bb"",""cc""],""e"":[]}]","[]","true","false"`) var csvSchema = []string{ "its", @@ -221,7 +221,7 @@ func TestParserString(t *testing.T) { {"not_exist", false, ""}, {"not_exist", true, nil}, {"its", false, "1536813227"}, - {"array_int", false, "[1,2,3]"}, + {"array_int", false, "[-9223372036854775808,9223372036854775807]"}, {"array_string", false, `["aa","bb","cc"]`}, {"mp", false, `{"i":[1,2,3],"f":[1.1,2.2,3.3],"s":["aa","bb","cc"],"e":[]}`}, } @@ -261,7 +261,8 @@ func TestParserArray(t *testing.T) { ts = append(ts, t) } testCases := []ArrayCase{ - {"array_int", model.Int, []int64{1, 2, 3}}, + {"array_int", model.Float, []float64{-9223372036854775808, 9223372036854775807}}, + {"array_int", model.Int, []int64{-9223372036854775808, 9223372036854775807}}, {"array_float", model.Float, []float64{1.1, 2.2, 3.3}}, {"array_string", model.String, []string{"aa", "bb", "cc"}}, {"array_date", model.DateTime, ts},