test MinInt64, MaxInt64

housepower · May 12, 2021 · 533bbb0 · 533bbb0
1 parent e5b466e
commit 533bbb0
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 21 deletions.
diff --git a/parser/csv.go b/parser/csv.go
@@ -18,14 +18,14 @@ import (
 	"bytes"
 	"encoding/csv"
 	"fmt"
-	"strconv"
 	"strings"
 	"sync"
 	"time"
 
 	"github.com/housepower/clickhouse_sinker/model"
 	"github.com/housepower/clickhouse_sinker/util"
 	"github.com/pkg/errors"
+	"github.com/valyala/fastjson/fastfloat"
 )
 
 var _ Parser = (*CsvParser)(nil)
@@ -87,7 +87,7 @@ func (c *CsvMetric) GetFloat(key string, nullable bool) (val interface{}) {
 		val = float64(0.0)
 		return
 	}
-	val, _ = strconv.ParseFloat(c.values[idx], 64)
+	val = fastfloat.ParseBestEffort(c.values[idx])
 	return
 }
 
@@ -102,7 +102,7 @@ func (c *CsvMetric) GetInt(key string, nullable bool) (val interface{}) {
 		val = int64(0)
 		return
 	}
-	val, _ = strconv.ParseInt(c.values[idx], 10, 64)
+	val = fastfloat.ParseInt64BestEffort(c.values[idx])
 	return
 }
 
@@ -148,20 +148,16 @@ func (c *CsvMetric) GetArray(key string, typ int) (val interface{}) {
 	switch typ {
 	case model.Int:
 		results := make([]int64, 0, len(array))
-		var v int64
 		for _, e := range array {
-			if v, err = strconv.ParseInt(e, 10, 64); err == nil {
-				results = append(results, v)
-			}
+			v := fastfloat.ParseInt64BestEffort(e)
+			results = append(results, v)
 		}
 		val = results
 	case model.Float:
 		results := make([]float64, 0, len(array))
-		var v float64
 		for _, e := range array {
-			if v, err = strconv.ParseFloat(e, 64); err == nil {
-				results = append(results, v)
-			}
+			v := fastfloat.ParseBestEffort(e)
+			results = append(results, v)
 		}
 		val = results
 	case model.String:

diff --git a/parser/gjson.go b/parser/gjson.go
@@ -82,12 +82,7 @@ func (c *GjsonMetric) GetInt(key string, nullable bool) (val interface{}) {
 		val = int64(0)
 		return
 	}
-	switch r.Type {
-	case gjson.Number:
-		val = int64(r.Num)
-	default:
-		val = int64(0)
-	}
+	val = r.Int()
 	return
 }
 

diff --git a/parser/parser_test.go b/parser/parser_test.go
@@ -46,7 +46,7 @@ var jsonSample = []byte(`{
 	"time_ms_rfc3339_1":     "2019-12-16T12:10:30.123Z",
 	"time_ms_rfc3339_2":     "2019-12-16T12:10:30.123+08:00",
 	"time_ms_clickhouse_1":  "2019-12-16 12:10:30.123",
-	"array_int": [1,2,3],
+	"array_int": [-9223372036854775808,9223372036854775807],
 	"array_float": [1.1,2.2,3.3],
 	"array_string": ["aa","bb","cc"],
 	"array_date": ["2000-01-01","2000-01-02","2000-01-03"],
@@ -85,7 +85,7 @@ var jsonSchema = map[string]string{
 	"bool_false":            "false",
 }
 
-var csvSample = []byte(`1536813227,"0.11","escaped_""ws","{""i"":[1,2,3],""f"":[1.1,2.2,3.3],""s"":[""aa"",""bb"",""cc""],""e"":[]}",2019-12-16,2019-12-16T12:10:30Z,2019-12-16T12:10:30+08:00,2019-12-16 12:10:30,2019-12-16T12:10:30.123Z,2019-12-16T12:10:30.123+08:00,2019-12-16 12:10:30.123,"[1,2,3]","[1.1,2.2,3.3]","[""aa"",""bb"",""cc""]","[""2000-01-01"",""2000-01-02"",""2000-01-03""]","[{""i"":[1,2,3],""f"":[1.1,2.2,3.3]},{""s"":[""aa"",""bb"",""cc""],""e"":[]}]","[]","true","false"`)
+var csvSample = []byte(`1536813227,"0.11","escaped_""ws","{""i"":[1,2,3],""f"":[1.1,2.2,3.3],""s"":[""aa"",""bb"",""cc""],""e"":[]}",2019-12-16,2019-12-16T12:10:30Z,2019-12-16T12:10:30+08:00,2019-12-16 12:10:30,2019-12-16T12:10:30.123Z,2019-12-16T12:10:30.123+08:00,2019-12-16 12:10:30.123,"[-9223372036854775808,9223372036854775807]","[1.1,2.2,3.3]","[""aa"",""bb"",""cc""]","[""2000-01-01"",""2000-01-02"",""2000-01-03""]","[{""i"":[1,2,3],""f"":[1.1,2.2,3.3]},{""s"":[""aa"",""bb"",""cc""],""e"":[]}]","[]","true","false"`)
 
 var csvSchema = []string{
 	"its",
@@ -221,7 +221,7 @@ func TestParserString(t *testing.T) {
 		{"not_exist", false, ""},
 		{"not_exist", true, nil},
 		{"its", false, "1536813227"},
-		{"array_int", false, "[1,2,3]"},
+		{"array_int", false, "[-9223372036854775808,9223372036854775807]"},
 		{"array_string", false, `["aa","bb","cc"]`},
 		{"mp", false, `{"i":[1,2,3],"f":[1.1,2.2,3.3],"s":["aa","bb","cc"],"e":[]}`},
 	}
@@ -261,7 +261,8 @@ func TestParserArray(t *testing.T) {
 		ts = append(ts, t)
 	}
 	testCases := []ArrayCase{
-		{"array_int", model.Int, []int64{1, 2, 3}},
+		{"array_int", model.Float, []float64{-9223372036854775808, 9223372036854775807}},
+		{"array_int", model.Int, []int64{-9223372036854775808, 9223372036854775807}},
 		{"array_float", model.Float, []float64{1.1, 2.2, 3.3}},
 		{"array_string", model.String, []string{"aa", "bb", "cc"}},
 		{"array_date", model.DateTime, ts},