Merge pull request #686 from grafana/njpm/upstream-preparation

Merge Grafana's fork changes

.deepsource.toml

@@ -15,5 +15,5 @@ name = "go"
 enabled = true
 
   [analyzers.meta]
-  import_root = "github.com/go-graphite/carbonapi"
+  import_path = "github.com/go-graphite/carbonapi"
   dependencies_vendored = true

.github/workflows/tests.yml

@@ -9,7 +9,7 @@ jobs:
 
   tests:
     name: Test code
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
         go:

COMPATIBILITY.md

@@ -109,28 +109,10 @@ _When `format=png`_ (default if not specified)
 ### Unsupported functions
 | Function                                                                  |
 | :------------------------------------------------------------------------ |
-| aggregateWithWildcards |
 | aliasQuery |
-| averageOutsidePercentile |
 | events |
-| exp |
-| exponentialMovingAverage |
 | holtWintersConfidenceArea |
-| identity |
-| logit |
-| minMax |
-| movingWindow |
 | pct |
-| powSeries |
-| removeBetweenPercentile |
-| setXFilesFactor |
-| sigmoid |
-| sin |
-| sinFunction |
-| unique |
-| verticalLine |
-| xFilesFactor |
-
 
 ### Partly supported functions
 | Function                 | Incompatibilities                              |
@@ -187,6 +169,7 @@ reverse: default value mismatch: got (empty), should be false |
 | add(seriesList, constant) | no |
 | aggregate(seriesList, func, xFilesFactor=None) | no |
 | aggregateLine(seriesList, func='average', keepStep=False) | no |
+| aggregateWithWildcards(seriesList, func, *positions) | no |
 | alias(seriesList, newName) | no |
 | aliasByMetric(seriesList) | no |
 | aliasByNode(seriesList, *nodes) | no |
@@ -198,6 +181,7 @@ reverse: default value mismatch: got (empty), should be false |
 | asPercent(seriesList, total=None, *nodes) | no |
 | averageAbove(seriesList, n) | no |
 | averageBelow(seriesList, n) | no |
+| averageOutsidePercentile(seriesList, n) | no |
 | averageSeries(*seriesLists) | no |
 | averageSeriesWithWildcards(seriesList, *position) | no |
 | avg(*seriesLists) | no |
@@ -218,6 +202,8 @@ reverse: default value mismatch: got (empty), should be false |
 | divideSeriesLists(dividendSeriesList, divisorSeriesList) | no |
 | drawAsInfinite(seriesList) | no |
 | exclude(seriesList, pattern) | no |
+| exp(seriesList) | no |
+| exponentialMovingAverage(seriesList, windowSize) | no |
 | fallbackSeries(seriesList, fallback) | no |
 | filterSeries(seriesList, func, operator, threshold) | no |
 | grep(seriesList, pattern) | no |
@@ -233,6 +219,8 @@ reverse: default value mismatch: got (empty), should be false |
 | holtWintersAberration(seriesList, delta=3, bootstrapInterval='7d') | no |
 | holtWintersConfidenceBands(seriesList, delta=3, bootstrapInterval='7d') | no |
 | holtWintersForecast(seriesList, bootstrapInterval='7d') | no |
+| logit(seriesList) | no |
+| identity(name) | no |
 | integral(seriesList) | no |
 | integralByInterval(seriesList, intervalString) | no |
 | interpolate(seriesList, limit) | no |
@@ -255,12 +243,14 @@ reverse: default value mismatch: got (empty), should be false |
 | minSeries(*seriesLists) | no |
 | minimumAbove(seriesList, n) | no |
 | minimumBelow(seriesList, n) | no |
+| minMax(seriesList) | no |
 | mostDeviant(seriesList, n) | no |
 | movingAverage(seriesList, windowSize, xFilesFactor=None) | no |
 | movingMax(seriesList, windowSize, xFilesFactor=None) | no |
 | movingMedian(seriesList, windowSize, xFilesFactor=None) | no |
 | movingMin(seriesList, windowSize, xFilesFactor=None) | no |
 | movingSum(seriesList, windowSize, xFilesFactor=None) | no |
+| movingWindow(seriesList, windowSize, func='average', xFilesFactor=None) | no |
 | multiplySeries(*seriesLists) | no |
 | multiplySeriesWithWildcards(seriesList, *position) | no |
 | nPercentile(seriesList, n) | no |
@@ -270,6 +260,7 @@ reverse: default value mismatch: got (empty), should be false |
 | perSecond(seriesList, maxValue=None) | no |
 | percentileOfSeries(seriesList, n, interpolate=False) | no |
 | pow(seriesList, factor) | no |
+| powSeries(*seriesLists) | no |
 | randomWalk(name, step=60) | no |
 | randomWalkFunction(name, step=60) | no |
 | rangeOfSeries(*seriesLists) | no |
@@ -279,12 +270,16 @@ reverse: default value mismatch: got (empty), should be false |
 | removeAboveValue(seriesList, n) | no |
 | removeBelowPercentile(seriesList, n) | no |
 | removeBelowValue(seriesList, n) | no |
+| removeBelowValue(seriesList, n) | no |
 | removeEmptySeries(seriesList, xFilesFactor=None) | no |
 | round(seriesList, precision) | no |
 | scale(seriesList, factor) | no |
 | scaleToSeconds(seriesList, seconds) | no |
 | secondYAxis(seriesList) | no |
 | seriesByTag(*tagExpressions) | no |
+| setXFilesFactor(seriesList, xFilesFactor) | no |
+| sigmoid(seriesList) | no |
+| sinFunction(seriesList, amplitude=1, step=60) | no |
 | smartSummarize(seriesList, intervalString, func='sum', alignTo=None) | no |
 | sortBy(seriesList, func='average', reverse=False) | no |
 | sortByMaxima(seriesList) | no |
@@ -307,8 +302,10 @@ reverse: default value mismatch: got (empty), should be false |
 | timeSlice(seriesList, startSliceAt, endSliceAt='now') | no |
 | timeStack(seriesList, timeShiftUnit='1d', timeShiftStart=0, timeShiftEnd=7) | no |
 | transformNull(seriesList, default=0, referenceSeries=None) | no |
+| unique(*seriesLists) | no |
 | useSeriesAbove(seriesList, value, search, replace) | no |
 | weightedAverage(seriesListAvg, seriesListWeight, *nodes) | no |
+| verticalLine(ts, label=None, color=None) | no |
 | aliasByBase64(seriesList) | yes |
 | aliasByPostgres(seriesList, *nodes) | yes |
 | aliasByRedis(seriesList. keyName) | yes |

cmd/carbonapi/config/init.go

@@ -12,10 +12,10 @@ import (
 	"time"
 	"unicode"
 
+	"github.com/go-graphite/carbonapi/util/pidfile"
 	zipperConfig "github.com/go-graphite/carbonapi/zipper/config"
 
 	"github.com/ansel1/merry"
-	"github.com/facebookgo/pidfile"
 	"github.com/go-graphite/carbonapi/cache"
 	"github.com/go-graphite/carbonapi/expr/functions"
 	"github.com/go-graphite/carbonapi/expr/functions/cairo/png"
@@ -241,10 +241,9 @@ func SetUpConfig(logger *zap.Logger, BuildVersion string) {
 	}
 
 	if Config.PidFile != "" {
-		pidfile.SetPidfilePath(Config.PidFile)
-		err := pidfile.Write()
+		err := pidfile.WritePidFile(Config.PidFile)
 		if err != nil {
-			logger.Fatal("error during pidfile.Write()",
+			logger.Fatal("error when writing pidfile",
 				zap.Error(err),
 			)
 		}

cmd/carbonzipper/main.go

@@ -20,7 +20,6 @@ import (
 	"github.com/ansel1/merry"
 
 	"github.com/dgryski/httputil"
-	"github.com/facebookgo/pidfile"
 	protov2 "github.com/go-graphite/protocol/carbonapi_v2_pb"
 	protov3 "github.com/go-graphite/protocol/carbonapi_v3_pb"
 	"github.com/lomik/zapwriter"
@@ -29,6 +28,7 @@ import (
 	"github.com/go-graphite/carbonapi/intervalset"
 	"github.com/go-graphite/carbonapi/mstats"
 	util "github.com/go-graphite/carbonapi/util/ctx"
+	"github.com/go-graphite/carbonapi/util/pidfile"
 	"github.com/go-graphite/carbonapi/zipper"
 	zipperConfig "github.com/go-graphite/carbonapi/zipper/config"
 	"github.com/go-graphite/carbonapi/zipper/types"
@@ -793,10 +793,9 @@ func main() {
 	}
 
 	if *pidFile != "" {
-		pidfile.SetPidfilePath(*pidFile)
-		err = pidfile.Write()
+		err := pidfile.WritePidFile(*pidFile)
 		if err != nil {
-			logger.Fatal("error during pidfile.Write()",
+			logger.Fatal("error when writing pidfile",
 				zap.Error(err),
 			)
 		}

expr/consolidations/consolidations.go

@@ -24,13 +24,16 @@ var ConsolidationToFunc = map[string]func([]float64) float64{
 	"minimum":  AggMin,
 	"multiply": summarizeToAggregate("multiply"),
 	"range":    summarizeToAggregate("range"),
+	"rangeOf":  summarizeToAggregate("rangeOf"),
 	"sum":      AggSum,
+	"total":    AggSum,
 	"stddev":   summarizeToAggregate("stddev"),
 	"first":    AggFirst,
 	"last":     AggLast,
+	"current":  AggLast,
 }
 
-var AvailableSummarizers = []string{"sum", "total", "avg", "average", "avg_zero", "max", "min", "last", "range", "median", "multiply", "diff", "count", "stddev"}
+var AvailableSummarizers = []string{"sum", "total", "avg", "average", "avg_zero", "max", "min", "last", "current", "range", "rangeOf", "median", "multiply", "diff", "count", "stddev"}
 
 // AvgValue returns average of list of values
 func AvgValue(f64s []float64) float64 {
@@ -167,10 +170,10 @@ func SummarizeValues(f string, values []float64, XFilesFactor float32) float64 {
 				}
 			}
 		}
-	case "last":
+	case "last", "current":
 		rv = values[len(values)-1]
 		total = notNans(values)
-	case "range":
+	case "range", "rangeOf":
 		vMax := math.Inf(-1)
 		vMin := math.Inf(1)
 		isNaN := true
@@ -195,11 +198,14 @@ func SummarizeValues(f string, values []float64, XFilesFactor float32) float64 {
 		rv = Percentile(values, 50, true)
 		total = notNans(values)
 	case "multiply":
-		rv = values[0]
-		for _, av := range values[1:] {
-			if !math.IsNaN(av) {
+		rv = 1.0
+		for _, v := range values {
+			if math.IsNaN(v) {
+				rv = math.NaN()
+				break
+			} else {
 				total++
-				rv *= av
+				rv *= v
 			}
 		}
 	case "diff":
@@ -375,19 +381,28 @@ func AggCount(v []float64) float64 {
 	return float64(n)
 }
 
-// AggCount counts non-NaN points
 func AggDiff(v []float64) float64 {
-	res := v[0]
-	if len(v) == 1 {
-		return res
-	}
-	for _, vv := range v[1:] {
+	safeValues := make([]float64, 0, len(v))
+	for _, vv := range v {
 		if !math.IsNaN(vv) {
-			res -= vv
+			safeValues = append(safeValues, vv)
 		}
 	}
 
-	return res
+	if len(safeValues) > 0 {
+		res := safeValues[0]
+		if len(safeValues) == 1 {
+			return res
+		}
+
+		for _, vv := range safeValues[1:] {
+			res -= vv
+		}
+
+		return res
+	} else {
+		return math.NaN()
+	}
 }
 
 // MaxValue returns maximum from the list

expr/expr.go

@@ -169,7 +169,6 @@ func EvalExpr(ctx context.Context, e parser.Expr, from, until int64, values map[
 				parser.ErrBadType,
 				parser.ErrMissingArgument,
 				parser.ErrMissingTimeseries,
-				parser.ErrSeriesDoesNotExist,
 				parser.ErrUnknownTimeUnits,
 			) {
 				err = merry.WithHTTPCode(err, 400)

expr/functions/absolute/function.go

@@ -36,6 +36,7 @@ func (f *absolute) Do(ctx context.Context, e parser.Expr, from, until int64, val
 			}
 			r.Values[i] = math.Abs(v)
 		}
+		r.Tags["absolute"] = "1"
 		return r
 	})
 }

expr/functions/aggregate/function.go

@@ -41,6 +41,7 @@ func New(configFile string) []interfaces.FunctionMetadata {
 // aggregate(*seriesLists)
 func (f *aggregate) Do(ctx context.Context, e parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) ([]*types.MetricData, error) {
 	var args []*types.MetricData
+	var xFilesFactor float64
 	isAggregateFunc := true
 
 	callback, err := e.GetStringArg(1)
@@ -52,23 +53,28 @@ func (f *aggregate) Do(ctx context.Context, e parser.Expr, from, until int64, va
 			if err != nil {
 				return nil, err
 			}
+			if len(args) == 0 {
+				return []*types.MetricData{}, nil
+			}
 			callback = strings.Replace(e.Target(), "Series", "", 1)
 			isAggregateFunc = false
+			xFilesFactor = -1 // xFilesFactor is not used by the ...Series functions
 		}
 	} else {
 		args, err = helper.GetSeriesArg(ctx, e.Arg(0), from, until, values)
 		if err != nil {
 			return nil, err
 		}
-	}
+		if len(args) == 0 {
+			return []*types.MetricData{}, nil
+		}
 
-	// TODO: Implement xFilesFactor
-	/*
-		xFilesFactor, err := e.GetFloatArgDefault(2, 0)
+		xFilesFactor, err = e.GetFloatArgDefault(2, float64(args[0].XFilesFactor)) // If set by setXFilesFactor, all series in a list will have the same value
 		if err != nil {
-			return false, nil, err
+			return nil, err
 		}
-	*/
+	}
+
 	aggFunc, ok := consolidations.ConsolidationToFunc[callback]
 	if !ok {
 		return nil, fmt.Errorf("unsupported consolidation function %s", callback)
@@ -79,7 +85,17 @@ func (f *aggregate) Do(ctx context.Context, e parser.Expr, from, until int64, va
 	if isAggregateFunc {
 		e.SetRawArgs(e.Arg(0).Target())
 	}
-	return helper.AggregateSeries(e, args, aggFunc, fconfig.Config.ExtractTagsFromArgs)
+
+	results, err := helper.AggregateSeries(e, args, aggFunc, xFilesFactor, fconfig.Config.ExtractTagsFromArgs)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, result := range results {
+		result.Tags["aggregatedBy"] = callback
+	}
+
+	return results, nil
 }
 
 // Description is auto-generated description, based on output of https://github.com/graphite-project/graphite-web
@@ -105,12 +121,12 @@ func (f *aggregate) Description() map[string]types.FunctionDescription {
 					Required: true,
 					Options:  types.StringsToSuggestionList(consolidations.AvailableConsolidationFuncs()),
 				},
-				/*
-					{
-						Name: "xFilesFactor",
-						Type: types.Float,
-					},
-				*/
+				{
+
+					Name:     "xFilesFactor",
+					Type:     types.Float,
+					Required: false,
+				},
 			},
 			SeriesChange: true, // function aggregate metrics or change series items count
 			NameChange:   true, // name changed