feat: add hook for optimizing series reads based on authorizer (#25207)

internal/authorizer.go

@@ -12,6 +12,7 @@ type AuthorizerMock struct {
 	AuthorizeQueryFn       func(database string, query *influxql.Query) error
 	AuthorizeSeriesReadFn  func(database string, measurement []byte, tags models.Tags) bool
 	AuthorizeSeriesWriteFn func(database string, measurement []byte, tags models.Tags) bool
+	OptimizeSeriesReadFn   func(database string, measurement []byte, expr influxql.Expr) (influxql.Expr, query.FineAuthorizer, error)
 }
 
 // AuthorizeDatabase determines if the provided privilege is sufficient to
@@ -32,6 +33,14 @@ func (a *AuthorizerMock) AuthorizeSeriesRead(database string, measurement []byte
 	return a.AuthorizeSeriesReadFn(database, measurement, tags)
 }
 
+// OptimizeSeriesRead optimizes series read queries based on the authorizer.
+func (a *AuthorizerMock) OptimizeSeriesRead(database string, measurement []byte, expr influxql.Expr) (influxql.Expr, query.FineAuthorizer, error) {
+	if a.OptimizeSeriesReadFn != nil {
+		return a.OptimizeSeriesReadFn(database, measurement, expr)
+	}
+	return expr, a, nil
+}
+
 // AuthorizeSeriesWrite determines if the series comprising measurement and tags
 // can be written to, on the provided database.
 func (a *AuthorizerMock) AuthorizeSeriesWrite(database string, measurement []byte, tags models.Tags) bool {
@@ -41,3 +50,7 @@ func (a *AuthorizerMock) AuthorizeSeriesWrite(database string, measurement []byt
 func (a *AuthorizerMock) IsOpen() bool {
 	return false
 }
+
+func (a *AuthorizerMock) IsVoid() bool {
+	return false
+}

query/executor.go

@@ -90,11 +90,17 @@ type FineAuthorizer interface {
 	// AuthorizeSeriesRead determines if a series is authorized for reading
 	AuthorizeSeriesRead(database string, measurement []byte, tags models.Tags) bool
 
+	// OptimizeSeriesRead produces an optimized authorizer-aware WHERE expression and updated authorizer.
+	OptimizeSeriesRead(database string, measurement []byte, expr influxql.Expr) (influxql.Expr, FineAuthorizer, error)
+
 	// AuthorizeSeriesWrite determines if a series is authorized for writing
 	AuthorizeSeriesWrite(database string, measurement []byte, tags models.Tags) bool
 
 	// IsOpen guarantees that the other methods of a FineAuthorizer always return true.
 	IsOpen() bool
+
+	// IsVoid guarantees that Authorize methods of a FineAuthorizer always return false.
+	IsVoid() bool
 }
 
 // OpenAuthorizer is the Authorizer used when authorization is disabled.
@@ -109,16 +115,49 @@ func (a openAuthorizer) AuthorizeSeriesRead(database string, measurement []byte,
 	return true
 }
 
+// OptimizeSeriesRead is a no-op for openAuthorizer.
+func (a openAuthorizer) OptimizeSeriesRead(database string, measurement []byte, expr influxql.Expr) (influxql.Expr, FineAuthorizer, error) {
+	return expr, a, nil
+}
+
 // AuthorizeSeriesWrite allows access to any series.
 func (a openAuthorizer) AuthorizeSeriesWrite(database string, measurement []byte, tags models.Tags) bool {
 	return true
 }
 
 func (a openAuthorizer) IsOpen() bool { return true }
 
+func (a openAuthorizer) IsVoid() bool { return false }
+
 // AuthorizeSeriesRead allows any query to execute.
 func (a openAuthorizer) AuthorizeQuery(_ string, _ *influxql.Query) error { return nil }
 
+// VoidAuthorizer is the Authorizer used when no access is possible.
+// It disallows all operations.
+type voidAuthorizer struct{}
+
+// VoidAuthorizer can be shared by all goroutines.
+var VoidAuthorizer = voidAuthorizer{}
+
+// AuthorizeSeriesRead allows access to no series.
+func (a voidAuthorizer) AuthorizeSeriesRead(database string, measurement []byte, tags models.Tags) bool {
+	return false
+}
+
+// OptimizeSeriesRead is a no-op for voidAuthorizer.
+func (a voidAuthorizer) OptimizeSeriesRead(database string, measurement []byte, expr influxql.Expr) (influxql.Expr, FineAuthorizer, error) {
+	return expr, a, nil
+}
+
+// AuthorizeSeriesWrite allows access no series.
+func (a voidAuthorizer) AuthorizeSeriesWrite(database string, measurement []byte, tags models.Tags) bool {
+	return false
+}
+
+func (a voidAuthorizer) IsOpen() bool { return false }
+
+func (a voidAuthorizer) IsVoid() bool { return true }
+
 // AuthorizerIsOpen returns true if the provided Authorizer is guaranteed to
 // authorize anything. A nil Authorizer returns true for this function, and this
 // function should be preferred over directly checking if an Authorizer is nil
@@ -127,6 +166,13 @@ func AuthorizerIsOpen(a FineAuthorizer) bool {
 	return a == nil || a.IsOpen()
 }
 
+// AuthorizerIsVoid returns true if the provided Authorizer is guaranteed to
+// not authorize anything. A nil Authorizer acts as an openAuthorizer, and thus
+// not a void authorizer.
+func AuthorizerIsVoid(a FineAuthorizer) bool {
+	return a != nil && a.IsVoid()
+}
+
 // ExecutionOptions contains the options for executing a query.
 type ExecutionOptions struct {
 	// The database the query is running against.

query/subquery_test.go

@@ -399,12 +399,18 @@ func (*openAuthorizer) AuthorizeQuery(database string, query *influxql.Query) er
 func (*openAuthorizer) AuthorizeSeriesRead(database string, measurement []byte, tags models.Tags) bool {
 	return true
 }
+func (o *openAuthorizer) OptimizeSeriesRead(database string, measurement []byte, expr influxql.Expr) (influxql.Expr, query.FineAuthorizer, error) {
+	return expr, o, nil
+}
 func (*openAuthorizer) AuthorizeSeriesWrite(database string, measurement []byte, tags models.Tags) bool {
 	return true
 }
 func (*openAuthorizer) IsOpen() bool {
 	return true
 }
+func (*openAuthorizer) IsVoid() bool {
+	return false
+}
 
 // Ensure that the subquery gets passed the query authorizer.
 func TestSubquery_Authorizer(t *testing.T) {

services/meta/data.go

@@ -1669,6 +1669,11 @@ func (u *UserInfo) AuthorizeSeriesRead(database string, measurement []byte, tags
 	return true
 }
 
+// OptimizeSeriesRead optimizes series read based on the authorizer.
+func (u *UserInfo) OptimizeSeriesRead(database string, measurement []byte, expr influxql.Expr) (influxql.Expr, query.FineAuthorizer, error) {
+	return expr, u, nil
+}
+
 // AuthorizeSeriesWrite is used to limit access per-series (enterprise only)
 func (u *UserInfo) AuthorizeSeriesWrite(database string, measurement []byte, tags models.Tags) bool {
 	return true
@@ -1679,6 +1684,11 @@ func (u *UserInfo) IsOpen() bool {
 	return true
 }
 
+// IsVoid is a method on FineAuthorizer to indicate all fine auth is permitted and short circuit some checks.
+func (u *UserInfo) IsVoid() bool {
+	return false
+}
+
 // AuthorizeUnrestricted identifies the admin user
 //
 // Only the pprof endpoint uses this, we should prefer to have explicit permissioning instead.

tsdb/index.go

@@ -2908,6 +2908,7 @@ func (is IndexSet) tagValuesByKeyAndExpr(auth query.FineAuthorizer, name []byte,
 	}
 
 	// Iterate all series to collect tag values.
+	authIsOpen := query.AuthorizerIsOpen(auth)
 	for {
 		e, err := itr.Next()
 		if err != nil {
@@ -2933,7 +2934,7 @@ func (is IndexSet) tagValuesByKeyAndExpr(auth query.FineAuthorizer, name []byte,
 			continue
 		}
 
-		if auth != nil {
+		if !authIsOpen {
 			name, tags := ParseSeriesKey(buf)
 			if len(name) == 0 {
 				continue
@@ -2970,7 +2971,7 @@ func (is IndexSet) tagValuesByKeyAndExpr(auth query.FineAuthorizer, name []byte,
 }
 
 // MeasurementTagKeyValuesByExpr returns a set of tag values filtered by an expression.
-func (is IndexSet) MeasurementTagKeyValuesByExpr(auth query.FineAuthorizer, name []byte, keys []string, expr influxql.Expr, keysSorted bool) ([][]string, error) {
+func (is IndexSet) MeasurementTagKeyValuesByExpr(auth query.FineAuthorizer, name []byte, keys []string, expr influxql.Expr, keysSorted bool, log *zap.Logger) ([][]string, error) {
 	if len(keys) == 0 {
 		return nil, nil
 	}
@@ -2981,6 +2982,23 @@ func (is IndexSet) MeasurementTagKeyValuesByExpr(auth query.FineAuthorizer, name
 		sort.Strings(keys)
 	}
 
+	if auth != nil {
+		// OptimizeSeriesRead is a new step. In the extremely unlikely case it returns
+		// an error, we don't want to abort the query. This would mean tha a query that worked on
+		// a previous version suddenly breaks on a newer version, all because we tried and failed
+		// to speed it up. The original expr and auth will still yield the correct answer in the
+		// same time as the previous versions.
+		if newExpr, newAuth, err := auth.OptimizeSeriesRead(is.Database(), name, expr); err == nil {
+			auth = newAuth
+			expr = newExpr
+		} else {
+			log.Error("MeasurementTagKeyValuesByExpr: error in OptimizeSeriesRead, using unoptimized expr and auth", zap.Error(err))
+		}
+	}
+	if query.AuthorizerIsVoid(auth) {
+		return results, nil
+	}
+
 	release := is.SeriesFile.Retain()
 	defer release()
 

tsdb/store.go

@@ -1857,7 +1857,7 @@ func (s *Store) TagKeys(ctx context.Context, auth query.FineAuthorizer, shardIDs
 		sort.Strings(keys)
 
 		// Filter against tag values, skip if no values exist.
-		values, err := is.MeasurementTagKeyValuesByExpr(auth, name, keys, filterExpr, true)
+		values, err := is.MeasurementTagKeyValuesByExpr(auth, name, keys, filterExpr, true, s.Logger)
 		if err != nil {
 			return nil, err
 		}
@@ -2079,7 +2079,7 @@ func (s *Store) TagValues(ctx context.Context, auth query.FineAuthorizer, shardI
 		// get all the tag values for each key in the keyset.
 		// Each slice in the results contains the sorted values associated
 		// associated with each tag key for the measurement from the key set.
-		if result.values, err = is.MeasurementTagKeyValuesByExpr(auth, name, result.keys, filterExpr, true); err != nil {
+		if result.values, err = is.MeasurementTagKeyValuesByExpr(auth, name, result.keys, filterExpr, true, s.Logger); err != nil {
 			return nil, err
 		}