apply low selectivity matchers lazily in ingester

Signed-off-by: yeya24 <[email protected]>

Author: yeya24

docs/configuration/config-file-reference.md

@@ -3733,6 +3733,12 @@ instance_limits:
 # CLI flag: -ingester.skip-metadata-limits
 [skip_metadata_limits: <boolean> | default = true]
 
+# Enable optimization of label matchers when query chunks. When enabled,
+# matchers with low selectivity such as =~.+ are applied lazily during series
+# scanning instead of being used for postings matching.
+# CLI flag: -ingester.enable-matcher-optimization
+[enable_matcher_optimization: <boolean> | default = false]
+
 query_protection:
   rejection:
     threshold:

pkg/ingester/ingester.go

@@ -162,6 +162,10 @@ type Config struct {
 	// If enabled, the metadata API returns all metadata regardless of the limits.
 	SkipMetadataLimits bool `yaml:"skip_metadata_limits"`
 
+	// When enabled, matchers with low selectivity are applied lazily during series scanning
+	// instead of being used for postings selection.
+	EnableMatcherOptimization bool `yaml:"enable_matcher_optimization"`
+
 	QueryProtection configs.QueryProtection `yaml:"query_protection"`
 }
 
@@ -185,6 +189,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
 	f.BoolVar(&cfg.DisableChunkTrimming, "ingester.disable-chunk-trimming", false, "Disable trimming of matching series chunks based on query Start and End time. When disabled, the result may contain samples outside the queried time range but select performances may be improved. Note that certain query results might change by changing this option.")
 	f.IntVar(&cfg.MatchersCacheMaxItems, "ingester.matchers-cache-max-items", 0, "Maximum number of entries in the regex matchers cache. 0 to disable.")
 	f.BoolVar(&cfg.SkipMetadataLimits, "ingester.skip-metadata-limits", true, "If enabled, the metadata API returns all metadata regardless of the limits.")
+	f.BoolVar(&cfg.EnableMatcherOptimization, "ingester.enable-matcher-optimization", false, "Enable optimization of label matchers when query chunks. When enabled, matchers with low selectivity such as =~.+ are applied lazily during series scanning instead of being used for postings matching.")
 
 	cfg.DefaultLimits.RegisterFlagsWithPrefix(f, "ingester.")
 	cfg.QueryProtection.RegisterFlagsWithPrefix(f, "ingester.")
@@ -2295,6 +2300,10 @@ func (i *Ingester) queryStreamChunks(ctx context.Context, db *userTSDB, from, th
 		End:             through,
 		DisableTrimming: i.cfg.DisableChunkTrimming,
 	}
+	var lazyMatchers []*labels.Matcher
+	if i.cfg.EnableMatcherOptimization {
+		matchers, lazyMatchers = optimizeMatchers(matchers)
+	}
 	// It's not required to return sorted series because series are sorted by the Cortex querier.
 	ss := q.Select(ctx, false, hints, matchers...)
 	c()
@@ -2308,14 +2317,19 @@ func (i *Ingester) queryStreamChunks(ctx context.Context, db *userTSDB, from, th
 	var it chunks.Iterator
 	for ss.Next() {
 		series := ss.At()
+		lbls := series.Labels()
+
+		if !labelsMatches(lbls, lazyMatchers) {
+			continue
+		}
 
-		if sm.IsSharded() && !sm.MatchesLabels(series.Labels()) {
+		if sm.IsSharded() && !sm.MatchesLabels(lbls) {
 			continue
 		}
 
 		// convert labels to LabelAdapter
 		ts := client.TimeSeriesChunk{
-			Labels: cortexpb.FromLabelsToLabelAdapters(series.Labels()),
+			Labels: cortexpb.FromLabelsToLabelAdapters(lbls),
 		}
 
 		it := series.Iterator(it)

pkg/ingester/matchers.go

@@ -0,0 +1,61 @@
+package ingester
+
+import "slices"
+
+import "github.com/prometheus/prometheus/model/labels"
+
+// optimizeMatchers categorizes input matchers to matchers used in select and matchers applied lazily
+// when scanning series.
+func optimizeMatchers(matchers []*labels.Matcher) ([]*labels.Matcher, []*labels.Matcher) {
+	// If there is only 1 matcher, use it for select.
+	// If there is no matcher to optimize, also return early.
+	if len(matchers) < 2 || !canOptimizeMatchers(matchers) {
+		return matchers, nil
+	}
+	selectMatchers := make([]*labels.Matcher, 0, len(matchers))
+	lazyMatchers := make([]*labels.Matcher, 0)
+	for _, m := range matchers {
+		// =~.* is a noop as it matches everything.
+		if m.Type == labels.MatchRegexp && m.Value == ".*" {
+			continue
+		}
+		if lazyMatcher(m) {
+			lazyMatchers = append(lazyMatchers, m)
+			continue
+		}
+		selectMatchers = append(selectMatchers, m)
+	}
+
+	// We need at least 1 select matcher.
+	if len(selectMatchers) == 0 {
+		selectMatchers = lazyMatchers[:1]
+		lazyMatchers = lazyMatchers[1:]
+	}
+
+	return selectMatchers, lazyMatchers
+}
+
+func canOptimizeMatchers(matchers []*labels.Matcher) bool {
+	return slices.ContainsFunc(matchers, lazyMatcher)
+}
+
+func labelsMatches(lbls labels.Labels, matchers []*labels.Matcher) bool {
+	for _, m := range matchers {
+		if !m.Matches(lbls.Get(m.Name)) {
+			return false
+		}
+	}
+	return true
+}
+
+// lazyMatcher checks if the label matcher should be applied lazily when scanning series instead of fetching postings
+// for matcher. The matchers to apply lazily are matchers that are known to have low selectivity.
+func lazyMatcher(matcher *labels.Matcher) bool {
+	if matcher.Value == ".+" && matcher.Type == labels.MatchRegexp {
+		return true
+	}
+	if matcher.Value == "" && (matcher.Type == labels.MatchNotEqual || matcher.Type == labels.MatchNotRegexp) {
+		return true
+	}
+	return false
+}