Fixed profiles, added 1y continous timing and better hashing algo. (#31)

Signed-off-by: Bartlomiej Plotka <[email protected]>

cmd/thanosbench/block.go

@@ -133,7 +133,7 @@ Example plan with generation:
 ./thanosbench block plan -p <profile> --labels 'cluster="one"' --max-time 2019-10-18T00:00:00Z | ./thanosbench block gen --output.dir ./genblocks --workers 20`)
  profile := cmd.Flag("profile", "Name of the harcoded profile to use").Required().Short('p').Enum(blockgen.Profiles.Keys()...)
  maxTime := model.TimeOrDuration(cmd.Flag("max-time", "If empty current time - 30m (usual consistency delay) is used.").Default("30m"))
- extLset := cmd.Flag("labels", "External labels for block stream (repeated).").PlaceHolder("<name>=\"<value>\"").Required().Strings()
+ extLset := cmd.Flag("labels", "External labels for block stream (repeated).").PlaceHolder("<name>=\"<value>\"").Strings()
  m["block plan"] = func(g *run.Group, _ log.Logger) error {
  ctx, cancel := context.WithCancel(context.Background())
  g.Add(func() error {

go.mod

@@ -2,6 +2,7 @@ module github.com/thanos-io/thanosbench
 
 require (
  github.com/bwplotka/mimic v0.0.0-20190730202618-06ab9976e8ef
+ github.com/cespare/xxhash/v2 v2.1.1
  github.com/fatih/structtag v1.1.0
  github.com/go-kit/kit v0.10.0
  github.com/go-openapi/swag v0.19.9

pkg/blockgen/blockgen.go

@@ -7,6 +7,7 @@ import (
  "path"
  "time"
 
+ "github.com/cespare/xxhash/v2"
  "github.com/pkg/errors"
 
  "github.com/go-kit/kit/log"
@@ -76,7 +77,12 @@ func Generate(ctx context.Context, logger log.Logger, goroutines int, dir string
  if err != nil {
  return ulid.ULID{}, err
  }
- set := &blockSeriesSet{config: block}
+
+ extLset := block.Thanos.Labels
+ if extLset == nil {
+ extLset = map[string]string{}
+ }
+ set := &blockSeriesSet{config: block, extLset: labels.FromMap(extLset)}
  if err := seriesgen.Append(ctx, goroutines, w, set); err != nil {
  return ulid.ULID{}, errors.Wrap(err, "append")
  }
@@ -98,10 +104,11 @@ func Generate(ctx context.Context, logger log.Logger, goroutines int, dir string
 }
 
 type blockSeriesSet struct {
- config BlockSpec
- i int
- target int
- err error
+ config BlockSpec
+ extLset labels.Labels
+ i int
+ target int
+ err error
 
  curr seriesgen.Series
 }
@@ -122,9 +129,23 @@ func (s *blockSeriesSet) Next() bool {
  series := s.config.Series[s.i-1]
  lset := labels.Labels(append([]labels.Label{{Name: "__blockgen_target__", Value: fmt.Sprintf("%v", s.target)}}, series.Labels...))
 
+ b := make([]byte, 0, 1024)
+ for _, v := range lset {
+ b = append(b, v.Name...)
+ b = append(b, '\xff')
+ b = append(b, v.Value...)
+ b = append(b, '\xff')
+ }
+ for _, v := range s.extLset {
+ b = append(b, v.Name...)
+ b = append(b, '\xff')
+ b = append(b, v.Value...)
+ b = append(b, '\xff')
+ }
+
  // Stable random per series name.
  iter, err := series.Type.Create(
- rand.New(rand.NewSource(int64(lset.Hash()))),
+ rand.New(rand.NewSource(int64(xxhash.Sum64(b)))),
  series.MinTime,
  series.MaxTime,
  series.Characteristics,

pkg/blockgen/profiles.go

@@ -52,7 +52,34 @@ var (
  48 * time.Hour,
  2 * time.Hour,
  }, 1*time.Hour, 100, 50),
-
+ "realistic-k8s-30d-tiny": realisticK8s([]time.Duration{
+ // 30 days, from newest to oldest.
+ 2 * time.Hour,
+ 2 * time.Hour,
+ 2 * time.Hour,
+ 8 * time.Hour,
+ 176 * time.Hour,
+ 176 * time.Hour,
+ 176 * time.Hour,
+ 176 * time.Hour,
+ 2 * time.Hour,
+ }, 1*time.Hour, 1, 5),
+ "realistic-k8s-365d-tiny": realisticK8s([]time.Duration{
+ // 1y days, from newest to oldest.
+ 2 * time.Hour,
+ 2 * time.Hour,
+ 2 * time.Hour,
+ 8 * time.Hour,
+ 176 * time.Hour,
+ 176 * time.Hour,
+ 176 * time.Hour,
+ 176 * time.Hour,
+ 67 * 24 * time.Hour,
+ 67 * 24 * time.Hour,
+ 67 * 24 * time.Hour,
+ 67 * 24 * time.Hour,
+ 67 * 24 * time.Hour,
+ }, 1*time.Hour, 1, 5),
  "continuous-1w-small": continuous([]time.Duration{
  // One week, from newest to oldest, in the same way Thanos compactor would do.
  2 * time.Hour,
@@ -66,8 +93,7 @@ var (
  2 * time.Hour,
  // 10,000 series per block.
  }, 100, 100),
-
- "key-k8s-30d-tiny": realisticK8s([]time.Duration{
+ "continuous-30d-tiny": continuous([]time.Duration{
  // 30 days, from newest to oldest.
  2 * time.Hour,
  2 * time.Hour,
@@ -78,9 +104,8 @@ var (
  176 * time.Hour,
  176 * time.Hour,
  2 * time.Hour,
- }, 1*time.Hour, 1, 5),
-
- "key-k8s-365d-tiny": realisticK8s([]time.Duration{
+ }, 1, 5),
+ "continuous-365d-tiny": continuous([]time.Duration{
  // 1y days, from newest to oldest.
  2 * time.Hour,
  2 * time.Hour,
@@ -93,7 +118,9 @@ var (
  67 * 24 * time.Hour,
  67 * 24 * time.Hour,
  67 * 24 * time.Hour,
- }, 1*time.Hour, 1, 5),
+ 67 * 24 * time.Hour,
+ 67 * 24 * time.Hour,
+ }, 1, 5),
  }
 )
 
@@ -201,7 +228,7 @@ func continuous(ranges []time.Duration, apps int, metricsPerApp int) PlanFn {
  }
 
  for _, r := range ranges {
- mint := maxt - durToMilis(r)
+ mint := maxt - durToMilis(r) + 1
 
  if ctx.Err() != nil {
  return ctx.Err()