-
Notifications
You must be signed in to change notification settings - Fork 3.4k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[bloom-compactor] Add configs to enable compactor per tenant #11235
Changes from 4 commits
ca33de6
45dfb5c
92f6e92
5e99d7b
5db8704
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -293,6 +293,11 @@ func (c *Compactor) compactUsers(ctx context.Context, logger log.Logger, sc stor | |
return fmt.Errorf("interrupting compaction of tenants: %w", err) | ||
} | ||
|
||
// Skip tenant if compaction is not enabled | ||
if !c.limits.BloomCompactorEnabled(tenant) { | ||
return level.Info(tenantLogger).Log("msg", "compaction disabled for tenant") | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Not sure about returning the return value of There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. it wasn't suppose to end the loop for this. it has a |
||
} | ||
|
||
// Skip this table if it is too new/old for the tenant limits. | ||
now := model.Now() | ||
tableMinAge := c.limits.BloomCompactorMinTableAge(tenant) | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -185,6 +185,7 @@ type Limits struct { | |
BloomCompactorShardSize int `yaml:"bloom_compactor_shard_size" json:"bloom_compactor_shard_size"` | ||
BloomCompactorMaxTableAge time.Duration `yaml:"bloom_compactor_max_table_age" json:"bloom_compactor_max_table_age"` | ||
BloomCompactorMinTableAge time.Duration `yaml:"bloom_compactor_min_table_age" json:"bloom_compactor_min_table_age"` | ||
BloomCompactorEnabled bool `yaml:"bloom_compactor_enable_compaction" json:"bloom_compactor_enable_compaction"` | ||
|
||
AllowStructuredMetadata bool `yaml:"allow_structured_metadata,omitempty" json:"allow_structured_metadata,omitempty" doc:"description=Allow user to send structured metadata in push payload."` | ||
MaxStructuredMetadataSize flagext.ByteSize `yaml:"max_structured_metadata_size" json:"max_structured_metadata_size" doc:"description=Maximum size accepted for structured metadata per log line."` | ||
|
@@ -301,6 +302,7 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { | |
f.IntVar(&l.BloomCompactorShardSize, "bloom-compactor.shard-size", 1, "The shard size defines how many bloom compactors should be used by a tenant when computing blooms. If it's set to 0, shuffle sharding is disabled.") | ||
f.DurationVar(&l.BloomCompactorMaxTableAge, "bloom-compactor.max-table-age", 7*24*time.Hour, "The maximum age of a table before it is compacted. Do not compact tables older than the the configured time. Default to 7 days. 0s means no limit.") | ||
f.DurationVar(&l.BloomCompactorMinTableAge, "bloom-compactor.min-table-age", 1*time.Hour, "The minimum age of a table before it is compacted. Do not compact tables newer than the the configured time. Default to 1 hour. 0s means no limit. This is useful to avoid compacting tables that will be updated with out-of-order writes.") | ||
f.BoolVar(&l.BloomCompactorEnabled, "bloom-compactor.enable-compaction", false, "Whether to compact chunks into bloom filters.") | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. IIUC we want this to be false by default at the beginning so we selectively enable it for testing, right? But eventually, this will be true by default. Wondering because the log line about skipping the tenant can get a bit noisy. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes correct, we want to selectively enable at first. Probably true for large cells, but i don't think for dev. |
||
|
||
l.ShardStreams = &shardstreams.Config{} | ||
l.ShardStreams.RegisterFlagsWithPrefix("shard-streams", f) | ||
|
@@ -796,6 +798,10 @@ func (o *Overrides) BloomCompactorMinTableAge(userID string) time.Duration { | |
return o.getOverridesForUser(userID).BloomCompactorMinTableAge | ||
} | ||
|
||
func (o *Overrides) BloomCompactorEnabled(userID string) bool { | ||
return o.getOverridesForUser(userID).BloomCompactorEnabled | ||
} | ||
|
||
func (o *Overrides) AllowStructuredMetadata(userID string) bool { | ||
return o.getOverridesForUser(userID).AllowStructuredMetadata | ||
} | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I think this should be
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
oof end of day silliness. sorry