From f8462ee7b5838f3ac5cd77fc13000425cbdef450 Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar Date: Wed, 27 Nov 2024 13:59:11 -0500 Subject: [PATCH] Add `blocks-storage.tsdb.bigger-out-of-order-blocks-for-old-samples` flag (#10035) * Add blocks-storage.tsdb.bigger-out-of-order-blocks-for-old-samples flag (#9981) * Add blocks-storage.tsdb.bigger-out-of-order-blocks-for-old-samples flag Signed-off-by: Ganesh Vernekar * Update pkg/storage/tsdb/config.go Co-authored-by: chencs * Make flag experimental Signed-off-by: Ganesh Vernekar --------- Signed-off-by: Ganesh Vernekar Co-authored-by: chencs * CHANGELOG Signed-off-by: Ganesh Vernekar * CHANGELOG Signed-off-by: Ganesh Vernekar --------- Signed-off-by: Ganesh Vernekar Co-authored-by: chencs (cherry picked from commit 832d02b54e4e83c8da3ecc47586cf3a13e77dd94) --- CHANGELOG.md | 3 ++ cmd/mimir/config-descriptor.json | 11 ++++++ cmd/mimir/help-all.txt.tmpl | 2 + .../configuration-parameters/index.md | 5 +++ pkg/ingester/ingester.go | 1 + pkg/storage/tsdb/config.go | 38 ++++++++++--------- 6 files changed, 42 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b90ee10c562..e88546f3eac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,6 +69,9 @@ * [ENHANCEMENT] Distributor: Initialize ha_tracker cache before ha_tracker and distributor reach running state and begin serving writes. #9826 #9976 * [ENHANCEMENT] Ingester: `-ingest-storage.kafka.max-buffered-bytes` to limit the memory for buffered records when using concurrent fetching. #9892 * [ENHANCEMENT] Querier: improve performance and memory consumption of queries that select many series. #9914 +* [ENHANCEMENT] Ruler: Support OAuth2 and proxies in Alertmanager client #9945 +* [ENHANCEMENT] Ingester: Add `-blocks-storage.tsdb.bigger-out-of-order-blocks-for-old-samples` to build 24h blocks for out-of-order data belonging to the previous days instead of building smaller 2h blocks. This reduces pressure on compactors and ingesters when the out-of-order samples span multiple days in the past. #9844 #10033 #10035 +* [ENHANCEMENT] Distributor: allow a different limit for info series (series ending in `_info`) label count, via `-validation.max-label-names-per-info-series`. #10028 * [BUGFIX] Fix issue where functions such as `rate()` over native histograms could return incorrect values if a float stale marker was present in the selected range. #9508 * [BUGFIX] Fix issue where negation of native histograms (eg. `-some_native_histogram_series`) did nothing. #9508 * [BUGFIX] Fix issue where `metric might not be a counter, name does not end in _total/_sum/_count/_bucket` annotation would be emitted even if `rate` or `increase` did not have enough samples to compute a result. #9508 diff --git a/cmd/mimir/config-descriptor.json b/cmd/mimir/config-descriptor.json index 4c19d3c988a..5d1b43b147d 100644 --- a/cmd/mimir/config-descriptor.json +++ b/cmd/mimir/config-descriptor.json @@ -10210,6 +10210,17 @@ "fieldType": "int", "fieldCategory": "advanced" }, + { + "kind": "field", + "name": "bigger_out_of_order_blocks_for_old_samples", + "required": false, + "desc": "When enabled, ingester produces 24h blocks for out-of-order data that is before the current day, instead of the usual 2h blocks.", + "fieldValue": null, + "fieldDefaultValue": false, + "fieldFlag": "blocks-storage.tsdb.bigger-out-of-order-blocks-for-old-samples", + "fieldType": "boolean", + "fieldCategory": "experimental" + }, { "kind": "field", "name": "series_hash_cache_max_size_bytes", diff --git a/cmd/mimir/help-all.txt.tmpl b/cmd/mimir/help-all.txt.tmpl index 7b26bb35436..c5cfbb7db5f 100644 --- a/cmd/mimir/help-all.txt.tmpl +++ b/cmd/mimir/help-all.txt.tmpl @@ -911,6 +911,8 @@ Usage of ./cmd/mimir/mimir: OpenStack Swift user ID. -blocks-storage.swift.username string OpenStack Swift username. + -blocks-storage.tsdb.bigger-out-of-order-blocks-for-old-samples + [experimental] When enabled, ingester produces 24h blocks for out-of-order data that is before the current day, instead of the usual 2h blocks. -blocks-storage.tsdb.block-postings-for-matchers-cache-force [experimental] Force the cache to be used for postings for matchers in compacted blocks, even if it's not a concurrent (query-sharding) call. -blocks-storage.tsdb.block-postings-for-matchers-cache-max-bytes int diff --git a/docs/sources/mimir/configure/configuration-parameters/index.md b/docs/sources/mimir/configure/configuration-parameters/index.md index 3be5fb721d6..ed698cd00ab 100644 --- a/docs/sources/mimir/configure/configuration-parameters/index.md +++ b/docs/sources/mimir/configure/configuration-parameters/index.md @@ -4400,6 +4400,11 @@ tsdb: # CLI flag: -blocks-storage.tsdb.head-chunks-write-queue-size [head_chunks_write_queue_size: | default = 1000000] + # (experimental) When enabled, ingester produces 24h blocks for out-of-order + # data that is before the current day, instead of the usual 2h blocks. + # CLI flag: -blocks-storage.tsdb.bigger-out-of-order-blocks-for-old-samples + [bigger_out_of_order_blocks_for_old_samples: | default = false] + # (advanced) Max size - in bytes - of the in-memory series hash cache. The # cache is shared across all tenants and it's used only when query sharding is # enabled. diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 46a79abf4b4..a9dc07713f1 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -2687,6 +2687,7 @@ func (i *Ingester) createTSDB(userID string, walReplayConcurrency int) (*userTSD MaxExemplars: int64(i.limiter.maxExemplarsPerUser(userID)), SeriesHashCache: i.seriesHashCache, EnableMemorySnapshotOnShutdown: i.cfg.BlocksStorageConfig.TSDB.MemorySnapshotOnShutdown, + EnableBiggerOOOBlockForOldSamples: i.cfg.BlocksStorageConfig.TSDB.BiggerOutOfOrderBlocksForOldSamples, IsolationDisabled: true, HeadChunksWriteQueueSize: i.cfg.BlocksStorageConfig.TSDB.HeadChunksWriteQueueSize, EnableOverlappingCompaction: false, // always false since Mimir only uploads lvl 1 compacted blocks diff --git a/pkg/storage/tsdb/config.go b/pkg/storage/tsdb/config.go index 025c0eec51f..6c7dfd2d573 100644 --- a/pkg/storage/tsdb/config.go +++ b/pkg/storage/tsdb/config.go @@ -205,24 +205,25 @@ func (cfg *BlocksStorageConfig) Validate(activeSeriesCfg activeseries.Config) er // //nolint:revive type TSDBConfig struct { - Dir string `yaml:"dir"` - BlockRanges DurationList `yaml:"block_ranges_period" category:"experimental" doc:"hidden"` - Retention time.Duration `yaml:"retention_period"` - ShipInterval time.Duration `yaml:"ship_interval" category:"advanced"` - ShipConcurrency int `yaml:"ship_concurrency" category:"advanced"` - HeadCompactionInterval time.Duration `yaml:"head_compaction_interval" category:"advanced"` - HeadCompactionConcurrency int `yaml:"head_compaction_concurrency" category:"advanced"` - HeadCompactionIdleTimeout time.Duration `yaml:"head_compaction_idle_timeout" category:"advanced"` - HeadChunksWriteBufferSize int `yaml:"head_chunks_write_buffer_size_bytes" category:"advanced"` - HeadChunksEndTimeVariance float64 `yaml:"head_chunks_end_time_variance" category:"experimental"` - StripeSize int `yaml:"stripe_size" category:"advanced"` - WALCompressionEnabled bool `yaml:"wal_compression_enabled" category:"advanced"` - WALSegmentSizeBytes int `yaml:"wal_segment_size_bytes" category:"advanced"` - WALReplayConcurrency int `yaml:"wal_replay_concurrency" category:"advanced"` - FlushBlocksOnShutdown bool `yaml:"flush_blocks_on_shutdown" category:"advanced"` - CloseIdleTSDBTimeout time.Duration `yaml:"close_idle_tsdb_timeout" category:"advanced"` - MemorySnapshotOnShutdown bool `yaml:"memory_snapshot_on_shutdown" category:"experimental"` - HeadChunksWriteQueueSize int `yaml:"head_chunks_write_queue_size" category:"advanced"` + Dir string `yaml:"dir"` + BlockRanges DurationList `yaml:"block_ranges_period" category:"experimental" doc:"hidden"` + Retention time.Duration `yaml:"retention_period"` + ShipInterval time.Duration `yaml:"ship_interval" category:"advanced"` + ShipConcurrency int `yaml:"ship_concurrency" category:"advanced"` + HeadCompactionInterval time.Duration `yaml:"head_compaction_interval" category:"advanced"` + HeadCompactionConcurrency int `yaml:"head_compaction_concurrency" category:"advanced"` + HeadCompactionIdleTimeout time.Duration `yaml:"head_compaction_idle_timeout" category:"advanced"` + HeadChunksWriteBufferSize int `yaml:"head_chunks_write_buffer_size_bytes" category:"advanced"` + HeadChunksEndTimeVariance float64 `yaml:"head_chunks_end_time_variance" category:"experimental"` + StripeSize int `yaml:"stripe_size" category:"advanced"` + WALCompressionEnabled bool `yaml:"wal_compression_enabled" category:"advanced"` + WALSegmentSizeBytes int `yaml:"wal_segment_size_bytes" category:"advanced"` + WALReplayConcurrency int `yaml:"wal_replay_concurrency" category:"advanced"` + FlushBlocksOnShutdown bool `yaml:"flush_blocks_on_shutdown" category:"advanced"` + CloseIdleTSDBTimeout time.Duration `yaml:"close_idle_tsdb_timeout" category:"advanced"` + MemorySnapshotOnShutdown bool `yaml:"memory_snapshot_on_shutdown" category:"experimental"` + HeadChunksWriteQueueSize int `yaml:"head_chunks_write_queue_size" category:"advanced"` + BiggerOutOfOrderBlocksForOldSamples bool `yaml:"bigger_out_of_order_blocks_for_old_samples" category:"experimental"` // Series hash cache. SeriesHashCacheMaxBytes uint64 `yaml:"series_hash_cache_max_size_bytes" category:"advanced"` @@ -327,6 +328,7 @@ func (cfg *TSDBConfig) RegisterFlags(f *flag.FlagSet) { f.Int64Var(&cfg.EarlyHeadCompactionMinInMemorySeries, "blocks-storage.tsdb.early-head-compaction-min-in-memory-series", 0, fmt.Sprintf("When the number of in-memory series in the ingester is equal to or greater than this setting, the ingester tries to compact the TSDB Head. The early compaction removes from the memory all samples and inactive series up until -%s time ago. After an early compaction, the ingester will not accept any sample with a timestamp older than -%s time ago (unless out of order ingestion is enabled). The ingester checks every -%s whether an early compaction is required. Use 0 to disable it.", activeseries.IdleTimeoutFlag, activeseries.IdleTimeoutFlag, headCompactionIntervalFlag)) f.IntVar(&cfg.EarlyHeadCompactionMinEstimatedSeriesReductionPercentage, "blocks-storage.tsdb.early-head-compaction-min-estimated-series-reduction-percentage", 15, "When the early compaction is enabled, the early compaction is triggered only if the estimated series reduction is at least the configured percentage (0-100).") f.BoolVar(&cfg.TimelyHeadCompaction, "blocks-storage.tsdb.timely-head-compaction-enabled", false, "Allows head compaction to happen when the min block range can no longer be appended, without requiring 1.5x the chunk range worth of data in the head.") + f.BoolVar(&cfg.BiggerOutOfOrderBlocksForOldSamples, "blocks-storage.tsdb.bigger-out-of-order-blocks-for-old-samples", false, "When enabled, ingester produces 24h blocks for out-of-order data that is before the current day, instead of the usual 2h blocks.") cfg.HeadCompactionIntervalJitterEnabled = true cfg.HeadCompactionIntervalWhileStarting = 30 * time.Second