From f5e3aae3e34b30c947d4821c0e75692d4b4daebb Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Wed, 11 Nov 2020 16:36:46 +0100 Subject: [PATCH] Fixed profiles, added 1y continous timing and better hashing algo. (#31) Signed-off-by: Bartlomiej Plotka --- cmd/thanosbench/block.go | 2 +- go.mod | 1 + pkg/blockgen/blockgen.go | 33 ++++++++++++++++++++++++------ pkg/blockgen/profiles.go | 43 ++++++++++++++++++++++++++++++++-------- 4 files changed, 64 insertions(+), 15 deletions(-) diff --git a/cmd/thanosbench/block.go b/cmd/thanosbench/block.go index 2118e70..f10f54d 100644 --- a/cmd/thanosbench/block.go +++ b/cmd/thanosbench/block.go @@ -133,7 +133,7 @@ Example plan with generation: ./thanosbench block plan -p --labels 'cluster="one"' --max-time 2019-10-18T00:00:00Z | ./thanosbench block gen --output.dir ./genblocks --workers 20`) profile := cmd.Flag("profile", "Name of the harcoded profile to use").Required().Short('p').Enum(blockgen.Profiles.Keys()...) maxTime := model.TimeOrDuration(cmd.Flag("max-time", "If empty current time - 30m (usual consistency delay) is used.").Default("30m")) - extLset := cmd.Flag("labels", "External labels for block stream (repeated).").PlaceHolder("=\"\"").Required().Strings() + extLset := cmd.Flag("labels", "External labels for block stream (repeated).").PlaceHolder("=\"\"").Strings() m["block plan"] = func(g *run.Group, _ log.Logger) error { ctx, cancel := context.WithCancel(context.Background()) g.Add(func() error { diff --git a/go.mod b/go.mod index 4373c2b..4d1248c 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,7 @@ module github.com/thanos-io/thanosbench require ( github.com/bwplotka/mimic v0.0.0-20190730202618-06ab9976e8ef + github.com/cespare/xxhash/v2 v2.1.1 github.com/fatih/structtag v1.1.0 github.com/go-kit/kit v0.10.0 github.com/go-openapi/swag v0.19.9 diff --git a/pkg/blockgen/blockgen.go b/pkg/blockgen/blockgen.go index 7976ba0..d65987d 100644 --- a/pkg/blockgen/blockgen.go +++ b/pkg/blockgen/blockgen.go @@ -7,6 +7,7 @@ import ( "path" "time" + "github.com/cespare/xxhash/v2" "github.com/pkg/errors" "github.com/go-kit/kit/log" @@ -76,7 +77,12 @@ func Generate(ctx context.Context, logger log.Logger, goroutines int, dir string if err != nil { return ulid.ULID{}, err } - set := &blockSeriesSet{config: block} + + extLset := block.Thanos.Labels + if extLset == nil { + extLset = map[string]string{} + } + set := &blockSeriesSet{config: block, extLset: labels.FromMap(extLset)} if err := seriesgen.Append(ctx, goroutines, w, set); err != nil { return ulid.ULID{}, errors.Wrap(err, "append") } @@ -98,10 +104,11 @@ func Generate(ctx context.Context, logger log.Logger, goroutines int, dir string } type blockSeriesSet struct { - config BlockSpec - i int - target int - err error + config BlockSpec + extLset labels.Labels + i int + target int + err error curr seriesgen.Series } @@ -122,9 +129,23 @@ func (s *blockSeriesSet) Next() bool { series := s.config.Series[s.i-1] lset := labels.Labels(append([]labels.Label{{Name: "__blockgen_target__", Value: fmt.Sprintf("%v", s.target)}}, series.Labels...)) + b := make([]byte, 0, 1024) + for _, v := range lset { + b = append(b, v.Name...) + b = append(b, '\xff') + b = append(b, v.Value...) + b = append(b, '\xff') + } + for _, v := range s.extLset { + b = append(b, v.Name...) + b = append(b, '\xff') + b = append(b, v.Value...) + b = append(b, '\xff') + } + // Stable random per series name. iter, err := series.Type.Create( - rand.New(rand.NewSource(int64(lset.Hash()))), + rand.New(rand.NewSource(int64(xxhash.Sum64(b)))), series.MinTime, series.MaxTime, series.Characteristics, diff --git a/pkg/blockgen/profiles.go b/pkg/blockgen/profiles.go index db82cf1..38e3903 100644 --- a/pkg/blockgen/profiles.go +++ b/pkg/blockgen/profiles.go @@ -52,7 +52,34 @@ var ( 48 * time.Hour, 2 * time.Hour, }, 1*time.Hour, 100, 50), - + "realistic-k8s-30d-tiny": realisticK8s([]time.Duration{ + // 30 days, from newest to oldest. + 2 * time.Hour, + 2 * time.Hour, + 2 * time.Hour, + 8 * time.Hour, + 176 * time.Hour, + 176 * time.Hour, + 176 * time.Hour, + 176 * time.Hour, + 2 * time.Hour, + }, 1*time.Hour, 1, 5), + "realistic-k8s-365d-tiny": realisticK8s([]time.Duration{ + // 1y days, from newest to oldest. + 2 * time.Hour, + 2 * time.Hour, + 2 * time.Hour, + 8 * time.Hour, + 176 * time.Hour, + 176 * time.Hour, + 176 * time.Hour, + 176 * time.Hour, + 67 * 24 * time.Hour, + 67 * 24 * time.Hour, + 67 * 24 * time.Hour, + 67 * 24 * time.Hour, + 67 * 24 * time.Hour, + }, 1*time.Hour, 1, 5), "continuous-1w-small": continuous([]time.Duration{ // One week, from newest to oldest, in the same way Thanos compactor would do. 2 * time.Hour, @@ -66,8 +93,7 @@ var ( 2 * time.Hour, // 10,000 series per block. }, 100, 100), - - "key-k8s-30d-tiny": realisticK8s([]time.Duration{ + "continuous-30d-tiny": continuous([]time.Duration{ // 30 days, from newest to oldest. 2 * time.Hour, 2 * time.Hour, @@ -78,9 +104,8 @@ var ( 176 * time.Hour, 176 * time.Hour, 2 * time.Hour, - }, 1*time.Hour, 1, 5), - - "key-k8s-365d-tiny": realisticK8s([]time.Duration{ + }, 1, 5), + "continuous-365d-tiny": continuous([]time.Duration{ // 1y days, from newest to oldest. 2 * time.Hour, 2 * time.Hour, @@ -93,7 +118,9 @@ var ( 67 * 24 * time.Hour, 67 * 24 * time.Hour, 67 * 24 * time.Hour, - }, 1*time.Hour, 1, 5), + 67 * 24 * time.Hour, + 67 * 24 * time.Hour, + }, 1, 5), } ) @@ -201,7 +228,7 @@ func continuous(ranges []time.Duration, apps int, metricsPerApp int) PlanFn { } for _, r := range ranges { - mint := maxt - durToMilis(r) + mint := maxt - durToMilis(r) + 1 if ctx.Err() != nil { return ctx.Err()