From 9bc36152d9fc15b41d72dae19d51fc71b84ac20f Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Fri, 1 Dec 2023 13:22:01 +0100 Subject: [PATCH 01/11] intermediate save - DOES NOT COMPILE --- api/schema.graphqls | 8 +- internal/graph/generated/generated.go | 370 ++++++++++++++++++++- internal/graph/model/models_gen.go | 38 ++- internal/graph/schema.resolvers.go | 15 +- internal/repository/stats.go | 96 ++++++ pkg/archive/clusterConfig.go | 2 +- web/frontend/src/HistogramSelection.svelte | 73 ++++ web/frontend/src/Status.root.svelte | 2 + web/frontend/src/User.root.svelte | 41 ++- 9 files changed, 605 insertions(+), 40 deletions(-) create mode 100644 web/frontend/src/HistogramSelection.svelte diff --git a/api/schema.graphqls b/api/schema.graphqls index 69e32e2b..537dc2eb 100644 --- a/api/schema.graphqls +++ b/api/schema.graphqls @@ -198,7 +198,7 @@ type Query { jobsFootprints(filter: [JobFilter!], metrics: [String!]!): Footprints jobs(filter: [JobFilter!], page: PageRequest, order: OrderByInput): JobResultList! - jobsStatistics(filter: [JobFilter!], page: PageRequest, sortBy: SortByAggregate, groupBy: Aggregate): [JobsStatistics!]! + jobsStatistics(filter: [JobFilter!], metrics: [String!], page: PageRequest, sortBy: SortByAggregate, groupBy: Aggregate): [JobsStatistics!]! rooflineHeatmap(filter: [JobFilter!]!, rows: Int!, cols: Int!, minX: Float!, minY: Float!, maxX: Float!, maxY: Float!): [[Float!]!]! @@ -286,6 +286,11 @@ type HistoPoint { value: Int! } +type MetricHistoPoints { + metric: String! + data: [HistoPoint!] +} + type JobsStatistics { id: ID! # If `groupBy` was used, ID of the user/project/cluster name: String! # if User-Statistics: Given Name of Account (ID) Owner @@ -303,6 +308,7 @@ type JobsStatistics { histNumNodes: [HistoPoint!]! # value: number of nodes, count: number of jobs with that number of nodes histNumCores: [HistoPoint!]! # value: number of cores, count: number of jobs with that number of cores histNumAccs: [HistoPoint!]! # value: number of accs, count: number of jobs with that number of accs + histMetrics: [MetricHistoPoints!]! # metric: metricname, data array of histopoints: value: metric average bin, count: number of jobs with that metric average } input PageRequest { diff --git a/internal/graph/generated/generated.go b/internal/graph/generated/generated.go index f29e2a03..fd503ce2 100644 --- a/internal/graph/generated/generated.go +++ b/internal/graph/generated/generated.go @@ -25,6 +25,7 @@ import ( // NewExecutableSchema creates an ExecutableSchema from the ResolverRoot interface. func NewExecutableSchema(cfg Config) graphql.ExecutableSchema { return &executableSchema{ + schema: cfg.Schema, resolvers: cfg.Resolvers, directives: cfg.Directives, complexity: cfg.Complexity, @@ -32,6 +33,7 @@ func NewExecutableSchema(cfg Config) graphql.ExecutableSchema { } type Config struct { + Schema *ast.Schema Resolvers ResolverRoot Directives DirectiveRoot Complexity ComplexityRoot @@ -141,6 +143,7 @@ type ComplexityRoot struct { JobsStatistics struct { HistDuration func(childComplexity int) int + HistMetrics func(childComplexity int) int HistNumAccs func(childComplexity int) int HistNumCores func(childComplexity int) int HistNumNodes func(childComplexity int) int @@ -176,6 +179,11 @@ type ComplexityRoot struct { Metric func(childComplexity int) int } + MetricHistoPoints struct { + Data func(childComplexity int) int + Metric func(childComplexity int) int + } + MetricStatistics struct { Avg func(childComplexity int) int Max func(childComplexity int) int @@ -208,7 +216,7 @@ type ComplexityRoot struct { JobMetrics func(childComplexity int, id string, metrics []string, scopes []schema.MetricScope) int Jobs func(childComplexity int, filter []*model.JobFilter, page *model.PageRequest, order *model.OrderByInput) int JobsFootprints func(childComplexity int, filter []*model.JobFilter, metrics []string) int - JobsStatistics func(childComplexity int, filter []*model.JobFilter, page *model.PageRequest, sortBy *model.SortByAggregate, groupBy *model.Aggregate) int + JobsStatistics func(childComplexity int, filter []*model.JobFilter, metrics []string, page *model.PageRequest, sortBy *model.SortByAggregate, groupBy *model.Aggregate) int NodeMetrics func(childComplexity int, cluster string, nodes []string, scopes []schema.MetricScope, metrics []string, from time.Time, to time.Time) int RooflineHeatmap func(childComplexity int, filter []*model.JobFilter, rows int, cols int, minX float64, minY float64, maxX float64, maxY float64) int Tags func(childComplexity int) int @@ -322,7 +330,7 @@ type QueryResolver interface { JobMetrics(ctx context.Context, id string, metrics []string, scopes []schema.MetricScope) ([]*model.JobMetricWithName, error) JobsFootprints(ctx context.Context, filter []*model.JobFilter, metrics []string) (*model.Footprints, error) Jobs(ctx context.Context, filter []*model.JobFilter, page *model.PageRequest, order *model.OrderByInput) (*model.JobResultList, error) - JobsStatistics(ctx context.Context, filter []*model.JobFilter, page *model.PageRequest, sortBy *model.SortByAggregate, groupBy *model.Aggregate) ([]*model.JobsStatistics, error) + JobsStatistics(ctx context.Context, filter []*model.JobFilter, metrics []string, page *model.PageRequest, sortBy *model.SortByAggregate, groupBy *model.Aggregate) ([]*model.JobsStatistics, error) RooflineHeatmap(ctx context.Context, filter []*model.JobFilter, rows int, cols int, minX float64, minY float64, maxX float64, maxY float64) ([][]float64, error) NodeMetrics(ctx context.Context, cluster string, nodes []string, scopes []schema.MetricScope, metrics []string, from time.Time, to time.Time) ([]*model.NodeMetrics, error) } @@ -331,12 +339,16 @@ type SubClusterResolver interface { } type executableSchema struct { + schema *ast.Schema resolvers ResolverRoot directives DirectiveRoot complexity ComplexityRoot } func (e *executableSchema) Schema() *ast.Schema { + if e.schema != nil { + return e.schema + } return parsedSchema } @@ -730,6 +742,13 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.JobsStatistics.HistDuration(childComplexity), true + case "JobsStatistics.histMetrics": + if e.complexity.JobsStatistics.HistMetrics == nil { + break + } + + return e.complexity.JobsStatistics.HistMetrics(childComplexity), true + case "JobsStatistics.histNumAccs": if e.complexity.JobsStatistics.HistNumAccs == nil { break @@ -919,6 +938,20 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.MetricFootprints.Metric(childComplexity), true + case "MetricHistoPoints.data": + if e.complexity.MetricHistoPoints.Data == nil { + break + } + + return e.complexity.MetricHistoPoints.Data(childComplexity), true + + case "MetricHistoPoints.metric": + if e.complexity.MetricHistoPoints.Metric == nil { + break + } + + return e.complexity.MetricHistoPoints.Metric(childComplexity), true + case "MetricStatistics.avg": if e.complexity.MetricStatistics.Avg == nil { break @@ -1112,7 +1145,7 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return 0, false } - return e.complexity.Query.JobsStatistics(childComplexity, args["filter"].([]*model.JobFilter), args["page"].(*model.PageRequest), args["sortBy"].(*model.SortByAggregate), args["groupBy"].(*model.Aggregate)), true + return e.complexity.Query.JobsStatistics(childComplexity, args["filter"].([]*model.JobFilter), args["metrics"].([]string), args["page"].(*model.PageRequest), args["sortBy"].(*model.SortByAggregate), args["groupBy"].(*model.Aggregate)), true case "Query.nodeMetrics": if e.complexity.Query.NodeMetrics == nil { @@ -1587,14 +1620,14 @@ func (ec *executionContext) introspectSchema() (*introspection.Schema, error) { if ec.DisableIntrospection { return nil, errors.New("introspection disabled") } - return introspection.WrapSchema(parsedSchema), nil + return introspection.WrapSchema(ec.Schema()), nil } func (ec *executionContext) introspectType(name string) (*introspection.Type, error) { if ec.DisableIntrospection { return nil, errors.New("introspection disabled") } - return introspection.WrapTypeFromDef(parsedSchema, parsedSchema.Types[name]), nil + return introspection.WrapTypeFromDef(ec.Schema(), ec.Schema().Types[name]), nil } var sources = []*ast.Source{ @@ -1798,7 +1831,7 @@ type Query { jobsFootprints(filter: [JobFilter!], metrics: [String!]!): Footprints jobs(filter: [JobFilter!], page: PageRequest, order: OrderByInput): JobResultList! - jobsStatistics(filter: [JobFilter!], page: PageRequest, sortBy: SortByAggregate, groupBy: Aggregate): [JobsStatistics!]! + jobsStatistics(filter: [JobFilter!], metrics: [String!], page: PageRequest, sortBy: SortByAggregate, groupBy: Aggregate): [JobsStatistics!]! rooflineHeatmap(filter: [JobFilter!]!, rows: Int!, cols: Int!, minX: Float!, minY: Float!, maxX: Float!, maxY: Float!): [[Float!]!]! @@ -1886,6 +1919,11 @@ type HistoPoint { value: Int! } +type MetricHistoPoints { + metric: String! + data: [HistoPoint!] +} + type JobsStatistics { id: ID! # If ` + "`" + `groupBy` + "`" + ` was used, ID of the user/project/cluster name: String! # if User-Statistics: Given Name of Account (ID) Owner @@ -1903,6 +1941,7 @@ type JobsStatistics { histNumNodes: [HistoPoint!]! # value: number of nodes, count: number of jobs with that number of nodes histNumCores: [HistoPoint!]! # value: number of cores, count: number of jobs with that number of cores histNumAccs: [HistoPoint!]! # value: number of accs, count: number of jobs with that number of accs + histMetrics: [MetricHistoPoints!]! # value: metric average bin, count: number of jobs with that metric average } input PageRequest { @@ -2142,33 +2181,42 @@ func (ec *executionContext) field_Query_jobsStatistics_args(ctx context.Context, } } args["filter"] = arg0 - var arg1 *model.PageRequest + var arg1 []string + if tmp, ok := rawArgs["metrics"]; ok { + ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("metrics")) + arg1, err = ec.unmarshalOString2ᚕstringᚄ(ctx, tmp) + if err != nil { + return nil, err + } + } + args["metrics"] = arg1 + var arg2 *model.PageRequest if tmp, ok := rawArgs["page"]; ok { ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("page")) - arg1, err = ec.unmarshalOPageRequest2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐPageRequest(ctx, tmp) + arg2, err = ec.unmarshalOPageRequest2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐPageRequest(ctx, tmp) if err != nil { return nil, err } } - args["page"] = arg1 - var arg2 *model.SortByAggregate + args["page"] = arg2 + var arg3 *model.SortByAggregate if tmp, ok := rawArgs["sortBy"]; ok { ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("sortBy")) - arg2, err = ec.unmarshalOSortByAggregate2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐSortByAggregate(ctx, tmp) + arg3, err = ec.unmarshalOSortByAggregate2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐSortByAggregate(ctx, tmp) if err != nil { return nil, err } } - args["sortBy"] = arg2 - var arg3 *model.Aggregate + args["sortBy"] = arg3 + var arg4 *model.Aggregate if tmp, ok := rawArgs["groupBy"]; ok { ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("groupBy")) - arg3, err = ec.unmarshalOAggregate2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐAggregate(ctx, tmp) + arg4, err = ec.unmarshalOAggregate2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐAggregate(ctx, tmp) if err != nil { return nil, err } } - args["groupBy"] = arg3 + args["groupBy"] = arg4 return args, nil } @@ -5640,6 +5688,56 @@ func (ec *executionContext) fieldContext_JobsStatistics_histNumAccs(ctx context. return fc, nil } +func (ec *executionContext) _JobsStatistics_histMetrics(ctx context.Context, field graphql.CollectedField, obj *model.JobsStatistics) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_JobsStatistics_histMetrics(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return obj.HistMetrics, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.([]*model.MetricHistoPoints) + fc.Result = res + return ec.marshalNMetricHistoPoints2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐMetricHistoPointsᚄ(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_JobsStatistics_histMetrics(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "JobsStatistics", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + switch field.Name { + case "metric": + return ec.fieldContext_MetricHistoPoints_metric(ctx, field) + case "data": + return ec.fieldContext_MetricHistoPoints_data(ctx, field) + } + return nil, fmt.Errorf("no field named %q was found under type MetricHistoPoints", field.Name) + }, + } + return fc, nil +} + func (ec *executionContext) _MetricConfig_name(ctx context.Context, field graphql.CollectedField, obj *schema.MetricConfig) (ret graphql.Marshaler) { fc, err := ec.fieldContext_MetricConfig_name(ctx, field) if err != nil { @@ -6185,6 +6283,97 @@ func (ec *executionContext) fieldContext_MetricFootprints_data(ctx context.Conte return fc, nil } +func (ec *executionContext) _MetricHistoPoints_metric(ctx context.Context, field graphql.CollectedField, obj *model.MetricHistoPoints) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_MetricHistoPoints_metric(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return obj.Metric, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(string) + fc.Result = res + return ec.marshalNString2string(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_MetricHistoPoints_metric(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "MetricHistoPoints", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + return nil, errors.New("field of type String does not have child fields") + }, + } + return fc, nil +} + +func (ec *executionContext) _MetricHistoPoints_data(ctx context.Context, field graphql.CollectedField, obj *model.MetricHistoPoints) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_MetricHistoPoints_data(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return obj.Data, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + return graphql.Null + } + res := resTmp.([]*model.HistoPoint) + fc.Result = res + return ec.marshalOHistoPoint2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐHistoPointᚄ(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_MetricHistoPoints_data(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "MetricHistoPoints", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + switch field.Name { + case "count": + return ec.fieldContext_HistoPoint_count(ctx, field) + case "value": + return ec.fieldContext_HistoPoint_value(ctx, field) + } + return nil, fmt.Errorf("no field named %q was found under type HistoPoint", field.Name) + }, + } + return fc, nil +} + func (ec *executionContext) _MetricStatistics_avg(ctx context.Context, field graphql.CollectedField, obj *schema.MetricStatistics) (ret graphql.Marshaler) { fc, err := ec.fieldContext_MetricStatistics_avg(ctx, field) if err != nil { @@ -7374,7 +7563,7 @@ func (ec *executionContext) _Query_jobsStatistics(ctx context.Context, field gra }() resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { ctx = rctx // use context from middleware stack in children - return ec.resolvers.Query().JobsStatistics(rctx, fc.Args["filter"].([]*model.JobFilter), fc.Args["page"].(*model.PageRequest), fc.Args["sortBy"].(*model.SortByAggregate), fc.Args["groupBy"].(*model.Aggregate)) + return ec.resolvers.Query().JobsStatistics(rctx, fc.Args["filter"].([]*model.JobFilter), fc.Args["metrics"].([]string), fc.Args["page"].(*model.PageRequest), fc.Args["sortBy"].(*model.SortByAggregate), fc.Args["groupBy"].(*model.Aggregate)) }) if err != nil { ec.Error(ctx, err) @@ -7431,6 +7620,8 @@ func (ec *executionContext) fieldContext_Query_jobsStatistics(ctx context.Contex return ec.fieldContext_JobsStatistics_histNumCores(ctx, field) case "histNumAccs": return ec.fieldContext_JobsStatistics_histNumAccs(ctx, field) + case "histMetrics": + return ec.fieldContext_JobsStatistics_histMetrics(ctx, field) } return nil, fmt.Errorf("no field named %q was found under type JobsStatistics", field.Name) }, @@ -12910,6 +13101,11 @@ func (ec *executionContext) _JobsStatistics(ctx context.Context, sel ast.Selecti if out.Values[i] == graphql.Null { out.Invalids++ } + case "histMetrics": + out.Values[i] = ec._JobsStatistics_histMetrics(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } default: panic("unknown field " + strconv.Quote(field.Name)) } @@ -13058,6 +13254,47 @@ func (ec *executionContext) _MetricFootprints(ctx context.Context, sel ast.Selec return out } +var metricHistoPointsImplementors = []string{"MetricHistoPoints"} + +func (ec *executionContext) _MetricHistoPoints(ctx context.Context, sel ast.SelectionSet, obj *model.MetricHistoPoints) graphql.Marshaler { + fields := graphql.CollectFields(ec.OperationContext, sel, metricHistoPointsImplementors) + + out := graphql.NewFieldSet(fields) + deferred := make(map[string]*graphql.FieldSet) + for i, field := range fields { + switch field.Name { + case "__typename": + out.Values[i] = graphql.MarshalString("MetricHistoPoints") + case "metric": + out.Values[i] = ec._MetricHistoPoints_metric(ctx, field, obj) + if out.Values[i] == graphql.Null { + out.Invalids++ + } + case "data": + out.Values[i] = ec._MetricHistoPoints_data(ctx, field, obj) + default: + panic("unknown field " + strconv.Quote(field.Name)) + } + } + out.Dispatch(ctx) + if out.Invalids > 0 { + return graphql.Null + } + + atomic.AddInt32(&ec.deferred, int32(len(deferred))) + + for label, dfs := range deferred { + ec.processDeferredGroup(graphql.DeferredGroup{ + Label: label, + Path: graphql.GetPath(ctx), + FieldSet: dfs, + Context: ctx, + }) + } + + return out +} + var metricStatisticsImplementors = []string{"MetricStatistics"} func (ec *executionContext) _MetricStatistics(ctx context.Context, sel ast.SelectionSet, obj *schema.MetricStatistics) graphql.Marshaler { @@ -15310,6 +15547,60 @@ func (ec *executionContext) marshalNMetricFootprints2ᚖgithubᚗcomᚋClusterCo return ec._MetricFootprints(ctx, sel, v) } +func (ec *executionContext) marshalNMetricHistoPoints2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐMetricHistoPointsᚄ(ctx context.Context, sel ast.SelectionSet, v []*model.MetricHistoPoints) graphql.Marshaler { + ret := make(graphql.Array, len(v)) + var wg sync.WaitGroup + isLen1 := len(v) == 1 + if !isLen1 { + wg.Add(len(v)) + } + for i := range v { + i := i + fc := &graphql.FieldContext{ + Index: &i, + Result: &v[i], + } + ctx := graphql.WithFieldContext(ctx, fc) + f := func(i int) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = nil + } + }() + if !isLen1 { + defer wg.Done() + } + ret[i] = ec.marshalNMetricHistoPoints2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐMetricHistoPoints(ctx, sel, v[i]) + } + if isLen1 { + f(i) + } else { + go f(i) + } + + } + wg.Wait() + + for _, e := range ret { + if e == graphql.Null { + return graphql.Null + } + } + + return ret +} + +func (ec *executionContext) marshalNMetricHistoPoints2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐMetricHistoPoints(ctx context.Context, sel ast.SelectionSet, v *model.MetricHistoPoints) graphql.Marshaler { + if v == nil { + if !graphql.HasFieldError(ctx, graphql.GetFieldContext(ctx)) { + ec.Errorf(ctx, "the requested element is null which the schema does not allow") + } + return graphql.Null + } + return ec._MetricHistoPoints(ctx, sel, v) +} + func (ec *executionContext) unmarshalNMetricScope2githubᚗcomᚋClusterCockpitᚋccᚑbackendᚋpkgᚋschemaᚐMetricScope(ctx context.Context, v interface{}) (schema.MetricScope, error) { var res schema.MetricScope err := res.UnmarshalGQL(v) @@ -16117,6 +16408,53 @@ func (ec *executionContext) marshalOFootprints2ᚖgithubᚗcomᚋClusterCockpit return ec._Footprints(ctx, sel, v) } +func (ec *executionContext) marshalOHistoPoint2ᚕᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐHistoPointᚄ(ctx context.Context, sel ast.SelectionSet, v []*model.HistoPoint) graphql.Marshaler { + if v == nil { + return graphql.Null + } + ret := make(graphql.Array, len(v)) + var wg sync.WaitGroup + isLen1 := len(v) == 1 + if !isLen1 { + wg.Add(len(v)) + } + for i := range v { + i := i + fc := &graphql.FieldContext{ + Index: &i, + Result: &v[i], + } + ctx := graphql.WithFieldContext(ctx, fc) + f := func(i int) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = nil + } + }() + if !isLen1 { + defer wg.Done() + } + ret[i] = ec.marshalNHistoPoint2ᚖgithubᚗcomᚋClusterCockpitᚋccᚑbackendᚋinternalᚋgraphᚋmodelᚐHistoPoint(ctx, sel, v[i]) + } + if isLen1 { + f(i) + } else { + go f(i) + } + + } + wg.Wait() + + for _, e := range ret { + if e == graphql.Null { + return graphql.Null + } + } + + return ret +} + func (ec *executionContext) unmarshalOID2ᚕstringᚄ(ctx context.Context, v interface{}) ([]string, error) { if v == nil { return nil, nil diff --git a/internal/graph/model/models_gen.go b/internal/graph/model/models_gen.go index 050784b9..abf7a2f1 100644 --- a/internal/graph/model/models_gen.go +++ b/internal/graph/model/models_gen.go @@ -85,22 +85,23 @@ type JobResultList struct { } type JobsStatistics struct { - ID string `json:"id"` - Name string `json:"name"` - TotalJobs int `json:"totalJobs"` - RunningJobs int `json:"runningJobs"` - ShortJobs int `json:"shortJobs"` - TotalWalltime int `json:"totalWalltime"` - TotalNodes int `json:"totalNodes"` - TotalNodeHours int `json:"totalNodeHours"` - TotalCores int `json:"totalCores"` - TotalCoreHours int `json:"totalCoreHours"` - TotalAccs int `json:"totalAccs"` - TotalAccHours int `json:"totalAccHours"` - HistDuration []*HistoPoint `json:"histDuration"` - HistNumNodes []*HistoPoint `json:"histNumNodes"` - HistNumCores []*HistoPoint `json:"histNumCores"` - HistNumAccs []*HistoPoint `json:"histNumAccs"` + ID string `json:"id"` + Name string `json:"name"` + TotalJobs int `json:"totalJobs"` + RunningJobs int `json:"runningJobs"` + ShortJobs int `json:"shortJobs"` + TotalWalltime int `json:"totalWalltime"` + TotalNodes int `json:"totalNodes"` + TotalNodeHours int `json:"totalNodeHours"` + TotalCores int `json:"totalCores"` + TotalCoreHours int `json:"totalCoreHours"` + TotalAccs int `json:"totalAccs"` + TotalAccHours int `json:"totalAccHours"` + HistDuration []*HistoPoint `json:"histDuration"` + HistNumNodes []*HistoPoint `json:"histNumNodes"` + HistNumCores []*HistoPoint `json:"histNumCores"` + HistNumAccs []*HistoPoint `json:"histNumAccs"` + HistMetrics []*MetricHistoPoints `json:"histMetrics"` } type MetricFootprints struct { @@ -108,6 +109,11 @@ type MetricFootprints struct { Data []schema.Float `json:"data"` } +type MetricHistoPoints struct { + Metric string `json:"metric"` + Data []*HistoPoint `json:"data,omitempty"` +} + type NodeMetrics struct { Host string `json:"host"` SubCluster string `json:"subCluster"` diff --git a/internal/graph/schema.resolvers.go b/internal/graph/schema.resolvers.go index 9e5e1112..82bf026d 100644 --- a/internal/graph/schema.resolvers.go +++ b/internal/graph/schema.resolvers.go @@ -2,7 +2,7 @@ package graph // This file will be automatically regenerated based on the schema, any resolver implementations // will be copied through when generating and any unknown code will be moved to the end. -// Code generated by github.com/99designs/gqlgen version v0.17.36 +// Code generated by github.com/99designs/gqlgen version v0.17.40 import ( "context" @@ -244,7 +244,7 @@ func (r *queryResolver) Jobs(ctx context.Context, filter []*model.JobFilter, pag } // JobsStatistics is the resolver for the jobsStatistics field. -func (r *queryResolver) JobsStatistics(ctx context.Context, filter []*model.JobFilter, page *model.PageRequest, sortBy *model.SortByAggregate, groupBy *model.Aggregate) ([]*model.JobsStatistics, error) { +func (r *queryResolver) JobsStatistics(ctx context.Context, filter []*model.JobFilter, metrics []string, page *model.PageRequest, sortBy *model.SortByAggregate, groupBy *model.Aggregate) ([]*model.JobsStatistics, error) { var err error var stats []*model.JobsStatistics @@ -291,6 +291,17 @@ func (r *queryResolver) JobsStatistics(ctx context.Context, filter []*model.JobF } } + if requireField(ctx, "histMetrics") { + if groupBy == nil { + stats[0], err = r.Repo.AddMetricHistograms(ctx, filter, metrics, stats[0]) + if err != nil { + return nil, err + } + } else { + return nil, errors.New("metric histograms only implemented without groupBy argument") + } + } + return stats, nil } diff --git a/internal/repository/stats.go b/internal/repository/stats.go index 80845539..3ea089ec 100644 --- a/internal/repository/stats.go +++ b/internal/repository/stats.go @@ -12,7 +12,9 @@ import ( "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/internal/graph/model" + "github.com/ClusterCockpit/cc-backend/pkg/archive" "github.com/ClusterCockpit/cc-backend/pkg/log" + "github.com/ClusterCockpit/cc-backend/pkg/schema" sq "github.com/Masterminds/squirrel" ) @@ -450,6 +452,32 @@ func (r *JobRepository) AddHistograms( return stat, nil } +// Requires thresholds for metric from config for cluster? Of all clusters and use largest? split to 10 + 1 for artifacts? +func (r *JobRepository) AddMetricHistograms( + ctx context.Context, + filter []*model.JobFilter, + metrics []string, + stat *model.JobsStatistics) (*model.JobsStatistics, error) { + start := time.Now() + + for i, m := range metrics { + // DEBUG + fmt.Println(i, m) + var err error + var metricHisto *model.MetricHistoPoints + + metricHisto, err = r.jobsMetricStatisticsHistogram(ctx, m, filter) + if err != nil { + log.Warnf("Error while loading job metric statistics histogram: %s", m) + continue + } + stat.HistMetrics = append(stat.HistMetrics, metricHisto) + } + + log.Debugf("Timer AddMetricHistograms %s", time.Since(start)) + return stat, nil +} + // `value` must be the column grouped by, but renamed to "value" func (r *JobRepository) jobsStatisticsHistogram( ctx context.Context, @@ -487,3 +515,71 @@ func (r *JobRepository) jobsStatisticsHistogram( log.Debugf("Timer jobsStatisticsHistogram %s", time.Since(start)) return points, nil } + +func (r *JobRepository) jobsMetricStatisticsHistogram( + ctx context.Context, + metric string, + filters []*model.JobFilter) (*model.MetricHistoPoints, error) { + + // "job.load_avg as value" + + // switch m { + // case "cpu_load": + + // Get specific Peak or largest Peak + var metricConfig *schema.MetricConfig + var peak float64 = 0.0 + for _, f := range filters { + if f.Cluster != nil { + metricConfig = archive.GetMetricConfig(*f.Cluster.Eq, metric) + peak = metricConfig.Peak + } else { + for _, c := range archive.Clusters { + for _, m := range c.MetricConfig { + if m.Name == metric { + if m.Peak > peak { + peak = m.Peak + } + } + } + } + } + } + + // Make bins + + start := time.Now() + query, qerr := SecurityCheck(ctx, + sq.Select(value, "COUNT(job.id) AS count").From("job")) + + if qerr != nil { + return nil, qerr + } + + for _, f := range filters { + if f.Cluster != nil { + metricConfig = archive.GetMetricConfig(*f.Cluster.Eq, metric) + peak = metricConfig.Peak + } + query = BuildWhereClause(f, query) + } + + rows, err := query.GroupBy("value").RunWith(r.DB).Query() + if err != nil { + log.Error("Error while running query") + return nil, err + } + + points := make([]*model.HistoPoint, 0) + for rows.Next() { + point := model.HistoPoint{} + if err := rows.Scan(&point.Value, &point.Count); err != nil { + log.Warn("Error while scanning rows") + return nil, err + } + + points = append(points, &point) + } + log.Debugf("Timer jobsStatisticsHistogram %s", time.Since(start)) + return points, nil +} diff --git a/pkg/archive/clusterConfig.go b/pkg/archive/clusterConfig.go index 0b1c43bb..b1bad0ac 100644 --- a/pkg/archive/clusterConfig.go +++ b/pkg/archive/clusterConfig.go @@ -8,8 +8,8 @@ import ( "errors" "fmt" - "github.com/ClusterCockpit/cc-backend/pkg/schema" "github.com/ClusterCockpit/cc-backend/pkg/log" + "github.com/ClusterCockpit/cc-backend/pkg/schema" ) var Clusters []*schema.Cluster diff --git a/web/frontend/src/HistogramSelection.svelte b/web/frontend/src/HistogramSelection.svelte new file mode 100644 index 00000000..e2100795 --- /dev/null +++ b/web/frontend/src/HistogramSelection.svelte @@ -0,0 +1,73 @@ + + + + + (isHistogramConfigOpen = !isHistogramConfigOpen)}> + + Select metrics presented in histograms + + + + + {#each availableMetrics as metric (metric)} + + updateConfiguration({ + name: cluster ? `user_view_histogramMetrics:${cluster}` : 'user_view_histogramMetrics', + value: metricsInHistograms + })} /> + + {metric} + + {/each} + + + + + + diff --git a/web/frontend/src/Status.root.svelte b/web/frontend/src/Status.root.svelte index fffbfde1..563978dd 100644 --- a/web/frontend/src/Status.root.svelte +++ b/web/frontend/src/Status.root.svelte @@ -63,6 +63,8 @@ option.key == ccconfig.status_view_selectedTopUserCategory ); + let metricsInHistograms = ccconfig[`status_view_histogramMetrics:${cluster}`] || ccconfig.status_view_histogramMetrics + const client = getContextClient(); $: mainQuery = queryStore({ client: client, diff --git a/web/frontend/src/User.root.svelte b/web/frontend/src/User.root.svelte index 3871f601..34c56154 100644 --- a/web/frontend/src/User.root.svelte +++ b/web/frontend/src/User.root.svelte @@ -1,7 +1,7 @@ - + function closeAndApply() { + metricsInHistograms = [...pendingMetrics] // Set for parent + + updateConfiguration({ + name: cluster ? `user_view_histogramMetrics:${cluster}` : 'user_view_histogramMetrics', + value: metricsInHistograms + }) - (isHistogramConfigOpen = !isHistogramConfigOpen)}> + isOpen = false + } + + + (isOpen = !isOpen)}> Select metrics presented in histograms - {#each availableMetrics as metric (metric)} - updateConfiguration({ - name: cluster ? `user_view_histogramMetrics:${cluster}` : 'user_view_histogramMetrics', - value: metricsInHistograms - })} /> - + {metric} {/each} - + diff --git a/web/frontend/src/User.root.svelte b/web/frontend/src/User.root.svelte index 16f22d63..e216aa6f 100644 --- a/web/frontend/src/User.root.svelte +++ b/web/frontend/src/User.root.svelte @@ -25,9 +25,10 @@ let jobFilters = []; let sorting = { field: 'startTime', order: 'DESC' }, isSortingOpen = false let metrics = ccconfig.plot_list_selectedMetrics, isMetricsSelectionOpen = false - let w1, w2, histogramHeight = 250 + let w1, w2, histogramHeight = 250, isHistogramSelectionOpen = false let selectedCluster = filterPresets?.cluster ? filterPresets.cluster : null - let metricsInHistograms = ccconfig[`user_view_histogramMetrics:${selectedCluster}`] || ccconfig.user_view_histogramMetrics || [] + + $: metricsInHistograms = selectedCluster ? ccconfig[`user_view_histogramMetrics:${selectedCluster}`] : (ccconfig.user_view_histogramMetrics || []) const client = getContextClient(); $: stats = queryStore({ @@ -73,9 +74,11 @@ Metrics - + {/key} @@ -219,4 +222,9 @@ bind:cluster={selectedCluster} configName="plot_list_selectedMetrics" bind:metrics={metrics} - bind:isOpen={isMetricsSelectionOpen} /> \ No newline at end of file + bind:isOpen={isMetricsSelectionOpen} /> + + \ No newline at end of file From 1185737eaa1b48713bae45556dc77b79e9675454 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Fri, 8 Dec 2023 12:03:04 +0100 Subject: [PATCH 06/11] Add metrics to histoselect, add userfilters - edit struct to make only count return required --- api/schema.graphqls | 6 +- internal/graph/generated/generated.go | 88 +++++++++------------- internal/graph/model/models_gen.go | 8 +- internal/repository/query.go | 2 +- internal/repository/stats.go | 74 +++++++++++------- web/frontend/src/HistogramSelection.svelte | 2 +- web/frontend/src/User.root.svelte | 2 +- web/frontend/src/utils.js | 18 +++-- 8 files changed, 103 insertions(+), 97 deletions(-) diff --git a/api/schema.graphqls b/api/schema.graphqls index 21a9ad2e..8a43a54c 100644 --- a/api/schema.graphqls +++ b/api/schema.graphqls @@ -293,10 +293,10 @@ type MetricHistoPoints { } type MetricHistoPoint { - min: Int! - max: Int! + bin: Int count: Int! - bin: Int! + min: Int + max: Int } type JobsStatistics { diff --git a/internal/graph/generated/generated.go b/internal/graph/generated/generated.go index f3d4f8a4..12d829a8 100644 --- a/internal/graph/generated/generated.go +++ b/internal/graph/generated/generated.go @@ -1969,10 +1969,10 @@ type MetricHistoPoints { } type MetricHistoPoint { - min: Int! - max: Int! + bin: Int count: Int! - bin: Int! + min: Int + max: Int } type JobsStatistics { @@ -6336,8 +6336,8 @@ func (ec *executionContext) fieldContext_MetricFootprints_data(ctx context.Conte return fc, nil } -func (ec *executionContext) _MetricHistoPoint_min(ctx context.Context, field graphql.CollectedField, obj *model.MetricHistoPoint) (ret graphql.Marshaler) { - fc, err := ec.fieldContext_MetricHistoPoint_min(ctx, field) +func (ec *executionContext) _MetricHistoPoint_bin(ctx context.Context, field graphql.CollectedField, obj *model.MetricHistoPoint) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_MetricHistoPoint_bin(ctx, field) if err != nil { return graphql.Null } @@ -6350,24 +6350,21 @@ func (ec *executionContext) _MetricHistoPoint_min(ctx context.Context, field gra }() resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { ctx = rctx // use context from middleware stack in children - return obj.Min, nil + return obj.Bin, nil }) if err != nil { ec.Error(ctx, err) return graphql.Null } if resTmp == nil { - if !graphql.HasFieldError(ctx, fc) { - ec.Errorf(ctx, "must not be null") - } return graphql.Null } - res := resTmp.(int) + res := resTmp.(*int) fc.Result = res - return ec.marshalNInt2int(ctx, field.Selections, res) + return ec.marshalOInt2ᚖint(ctx, field.Selections, res) } -func (ec *executionContext) fieldContext_MetricHistoPoint_min(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { +func (ec *executionContext) fieldContext_MetricHistoPoint_bin(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { fc = &graphql.FieldContext{ Object: "MetricHistoPoint", Field: field, @@ -6380,8 +6377,8 @@ func (ec *executionContext) fieldContext_MetricHistoPoint_min(ctx context.Contex return fc, nil } -func (ec *executionContext) _MetricHistoPoint_max(ctx context.Context, field graphql.CollectedField, obj *model.MetricHistoPoint) (ret graphql.Marshaler) { - fc, err := ec.fieldContext_MetricHistoPoint_max(ctx, field) +func (ec *executionContext) _MetricHistoPoint_count(ctx context.Context, field graphql.CollectedField, obj *model.MetricHistoPoint) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_MetricHistoPoint_count(ctx, field) if err != nil { return graphql.Null } @@ -6394,7 +6391,7 @@ func (ec *executionContext) _MetricHistoPoint_max(ctx context.Context, field gra }() resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { ctx = rctx // use context from middleware stack in children - return obj.Max, nil + return obj.Count, nil }) if err != nil { ec.Error(ctx, err) @@ -6411,7 +6408,7 @@ func (ec *executionContext) _MetricHistoPoint_max(ctx context.Context, field gra return ec.marshalNInt2int(ctx, field.Selections, res) } -func (ec *executionContext) fieldContext_MetricHistoPoint_max(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { +func (ec *executionContext) fieldContext_MetricHistoPoint_count(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { fc = &graphql.FieldContext{ Object: "MetricHistoPoint", Field: field, @@ -6424,8 +6421,8 @@ func (ec *executionContext) fieldContext_MetricHistoPoint_max(ctx context.Contex return fc, nil } -func (ec *executionContext) _MetricHistoPoint_count(ctx context.Context, field graphql.CollectedField, obj *model.MetricHistoPoint) (ret graphql.Marshaler) { - fc, err := ec.fieldContext_MetricHistoPoint_count(ctx, field) +func (ec *executionContext) _MetricHistoPoint_min(ctx context.Context, field graphql.CollectedField, obj *model.MetricHistoPoint) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_MetricHistoPoint_min(ctx, field) if err != nil { return graphql.Null } @@ -6438,24 +6435,21 @@ func (ec *executionContext) _MetricHistoPoint_count(ctx context.Context, field g }() resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { ctx = rctx // use context from middleware stack in children - return obj.Count, nil + return obj.Min, nil }) if err != nil { ec.Error(ctx, err) return graphql.Null } if resTmp == nil { - if !graphql.HasFieldError(ctx, fc) { - ec.Errorf(ctx, "must not be null") - } return graphql.Null } - res := resTmp.(int) + res := resTmp.(*int) fc.Result = res - return ec.marshalNInt2int(ctx, field.Selections, res) + return ec.marshalOInt2ᚖint(ctx, field.Selections, res) } -func (ec *executionContext) fieldContext_MetricHistoPoint_count(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { +func (ec *executionContext) fieldContext_MetricHistoPoint_min(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { fc = &graphql.FieldContext{ Object: "MetricHistoPoint", Field: field, @@ -6468,8 +6462,8 @@ func (ec *executionContext) fieldContext_MetricHistoPoint_count(ctx context.Cont return fc, nil } -func (ec *executionContext) _MetricHistoPoint_bin(ctx context.Context, field graphql.CollectedField, obj *model.MetricHistoPoint) (ret graphql.Marshaler) { - fc, err := ec.fieldContext_MetricHistoPoint_bin(ctx, field) +func (ec *executionContext) _MetricHistoPoint_max(ctx context.Context, field graphql.CollectedField, obj *model.MetricHistoPoint) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_MetricHistoPoint_max(ctx, field) if err != nil { return graphql.Null } @@ -6482,24 +6476,21 @@ func (ec *executionContext) _MetricHistoPoint_bin(ctx context.Context, field gra }() resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { ctx = rctx // use context from middleware stack in children - return obj.Bin, nil + return obj.Max, nil }) if err != nil { ec.Error(ctx, err) return graphql.Null } if resTmp == nil { - if !graphql.HasFieldError(ctx, fc) { - ec.Errorf(ctx, "must not be null") - } return graphql.Null } - res := resTmp.(int) + res := resTmp.(*int) fc.Result = res - return ec.marshalNInt2int(ctx, field.Selections, res) + return ec.marshalOInt2ᚖint(ctx, field.Selections, res) } -func (ec *executionContext) fieldContext_MetricHistoPoint_bin(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { +func (ec *executionContext) fieldContext_MetricHistoPoint_max(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { fc = &graphql.FieldContext{ Object: "MetricHistoPoint", Field: field, @@ -6636,14 +6627,14 @@ func (ec *executionContext) fieldContext_MetricHistoPoints_data(ctx context.Cont IsResolver: false, Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { switch field.Name { + case "bin": + return ec.fieldContext_MetricHistoPoint_bin(ctx, field) + case "count": + return ec.fieldContext_MetricHistoPoint_count(ctx, field) case "min": return ec.fieldContext_MetricHistoPoint_min(ctx, field) case "max": return ec.fieldContext_MetricHistoPoint_max(ctx, field) - case "count": - return ec.fieldContext_MetricHistoPoint_count(ctx, field) - case "bin": - return ec.fieldContext_MetricHistoPoint_bin(ctx, field) } return nil, fmt.Errorf("no field named %q was found under type MetricHistoPoint", field.Name) }, @@ -13542,26 +13533,17 @@ func (ec *executionContext) _MetricHistoPoint(ctx context.Context, sel ast.Selec switch field.Name { case "__typename": out.Values[i] = graphql.MarshalString("MetricHistoPoint") - case "min": - out.Values[i] = ec._MetricHistoPoint_min(ctx, field, obj) - if out.Values[i] == graphql.Null { - out.Invalids++ - } - case "max": - out.Values[i] = ec._MetricHistoPoint_max(ctx, field, obj) - if out.Values[i] == graphql.Null { - out.Invalids++ - } - case "count": - out.Values[i] = ec._MetricHistoPoint_count(ctx, field, obj) - if out.Values[i] == graphql.Null { - out.Invalids++ - } case "bin": out.Values[i] = ec._MetricHistoPoint_bin(ctx, field, obj) + case "count": + out.Values[i] = ec._MetricHistoPoint_count(ctx, field, obj) if out.Values[i] == graphql.Null { out.Invalids++ } + case "min": + out.Values[i] = ec._MetricHistoPoint_min(ctx, field, obj) + case "max": + out.Values[i] = ec._MetricHistoPoint_max(ctx, field, obj) default: panic("unknown field " + strconv.Quote(field.Name)) } diff --git a/internal/graph/model/models_gen.go b/internal/graph/model/models_gen.go index eb35bda2..7b8ebd21 100644 --- a/internal/graph/model/models_gen.go +++ b/internal/graph/model/models_gen.go @@ -110,10 +110,10 @@ type MetricFootprints struct { } type MetricHistoPoint struct { - Min int `json:"min"` - Max int `json:"max"` - Count int `json:"count"` - Bin int `json:"bin"` + Bin *int `json:"bin,omitempty"` + Count int `json:"count"` + Min *int `json:"min,omitempty"` + Max *int `json:"max,omitempty"` } type MetricHistoPoints struct { diff --git a/internal/repository/query.go b/internal/repository/query.go index 84b80481..317302bc 100644 --- a/internal/repository/query.go +++ b/internal/repository/query.go @@ -96,7 +96,7 @@ func SecurityCheck(ctx context.Context, query sq.SelectBuilder) (sq.SelectBuilde user := GetUserFromContext(ctx) if user == nil { var qnil sq.SelectBuilder - return qnil, fmt.Errorf("user context is nil!") + return qnil, fmt.Errorf("user context is nil") } else if user.HasAnyRole([]schema.Role{schema.RoleAdmin, schema.RoleSupport, schema.RoleApi}) { // Admin & Co. : All jobs return query, nil } else if user.HasRole(schema.RoleManager) { // Manager : Add filter for managed projects' jobs only + personal jobs diff --git a/internal/repository/stats.go b/internal/repository/stats.go index bd870a45..ab70427e 100644 --- a/internal/repository/stats.go +++ b/internal/repository/stats.go @@ -460,13 +460,8 @@ func (r *JobRepository) AddMetricHistograms( stat *model.JobsStatistics) (*model.JobsStatistics, error) { start := time.Now() - for i, m := range metrics { - // DEBUG - fmt.Println(i, m) - var err error - var metricHisto *model.MetricHistoPoints - - metricHisto, err = r.jobsMetricStatisticsHistogram(ctx, m, filter) + for _, m := range metrics { + metricHisto, err := r.jobsMetricStatisticsHistogram(ctx, m, filter) if err != nil { log.Warnf("Error while loading job metric statistics histogram: %s", m) continue @@ -529,6 +524,12 @@ func (r *JobRepository) jobsMetricStatisticsHistogram( dbMetric = "flops_any_avg" case "mem_bw": dbMetric = "mem_bw_avg" + case "mem_used": + dbMetric = "mem_used_max" + case "net_bw": + dbMetric = "net_bw_avg" + case "file_bw": + dbMetric = "file_bw_avg" default: return nil, fmt.Errorf("%s not implemented", metric) } @@ -562,46 +563,67 @@ func (r *JobRepository) jobsMetricStatisticsHistogram( } } + // log.Debugf("Metric %s: DB %s, Peak %f, Unit %s", metric, dbMetric, peak, unit) // Make bins, see https://jereze.com/code/sql-histogram/ - // Diffs: - // CAST(X AS INTEGER) instead of floor(X), used also for for Min , Max selection - // renamed to bin for simplicity and model struct - // Ditched rename from job to data, as it conflicts with security check afterwards + start := time.Now() - prepQuery := sq.Select( - fmt.Sprintf(`CAST(min(job.%s) as INTEGER) as min`, dbMetric), - fmt.Sprintf(`CAST(max(job.%s) as INTEGER) as max`, dbMetric), + + crossJoinQuery := sq.Select( + fmt.Sprintf(`max(%s) as max`, dbMetric), + fmt.Sprintf(`min(%s) as min`, dbMetric), + ).From("job").Where( + fmt.Sprintf(`%s is not null`, dbMetric), + ).Where( + fmt.Sprintf(`%s <= %f`, dbMetric, peak), + ) + + crossJoinQuery, cjqerr := SecurityCheck(ctx, crossJoinQuery) + if cjqerr != nil { + return nil, cjqerr + } + + crossJoinQuerySql, _, sqlerr := crossJoinQuery.ToSql() + if sqlerr != nil { + return nil, sqlerr + } + + bins := 10 + binQuery := fmt.Sprintf(`CAST( (case when job.%s = value.max then value.max*0.999999999 else job.%s end - value.min) / (value.max - value.min) * %d as INTEGER )`, dbMetric, dbMetric, bins) + + mainQuery := sq.Select( + fmt.Sprintf(`%s + 1 as bin`, binQuery), fmt.Sprintf(`count(job.%s) as count`, dbMetric), - fmt.Sprintf(`CAST((case when job.%s = value.max then value.max*0.999999999 else job.%s end - value.min) / (value.max - value.min) * 10 as INTEGER) +1 as bin`, dbMetric, dbMetric)) - prepQuery = prepQuery.From("job") - prepQuery = prepQuery.CrossJoin(fmt.Sprintf(`(select max(%s) as max, min(%s) as min from job where %s is not null and %s < %f) as value`, dbMetric, dbMetric, dbMetric, dbMetric, peak)) - prepQuery = prepQuery.Where(fmt.Sprintf(`job.%s is not null and job.%s < %f`, dbMetric, dbMetric, peak)) + fmt.Sprintf(`CAST(((value.max / %d) * (%s )) as INTEGER ) as min`, bins, binQuery), + fmt.Sprintf(`CAST(((value.max / %d) * (%s + 1 )) as INTEGER ) as max`, bins, binQuery), + ).From("job").CrossJoin( + fmt.Sprintf(`(%s) as value`, crossJoinQuerySql), + ).Where(fmt.Sprintf(`job.%s is not null and job.%s <= %f`, dbMetric, dbMetric, peak)) - query, qerr := SecurityCheck(ctx, prepQuery) + mainQuery, qerr := SecurityCheck(ctx, mainQuery) if qerr != nil { return nil, qerr } for _, f := range filters { - query = BuildWhereClause(f, query) + mainQuery = BuildWhereClause(f, mainQuery) } // Finalize query with Grouping and Ordering - query = query.GroupBy("bin").OrderBy("bin") + mainQuery = mainQuery.GroupBy("bin").OrderBy("bin") - rows, err := query.RunWith(r.DB).Query() + rows, err := mainQuery.RunWith(r.DB).Query() if err != nil { - log.Errorf("Error while running query: %s", err) + log.Errorf("Error while running mainQuery: %s", err) return nil, err } points := make([]*model.MetricHistoPoint, 0) for rows.Next() { point := model.MetricHistoPoint{} - if err := rows.Scan(&point.Min, &point.Max, &point.Count, &point.Bin); err != nil { - log.Warn("Error while scanning rows") - return nil, err + if err := rows.Scan(&point.Bin, &point.Count, &point.Min, &point.Max); err != nil { + log.Warnf("Error while scanning rows for %s", metric) + return nil, err // Totally bricks cc-backend if returned and if all metrics requested? } points = append(points, &point) diff --git a/web/frontend/src/HistogramSelection.svelte b/web/frontend/src/HistogramSelection.svelte index afef8c70..142f6789 100644 --- a/web/frontend/src/HistogramSelection.svelte +++ b/web/frontend/src/HistogramSelection.svelte @@ -4,10 +4,10 @@ import { gql, getContextClient , mutationStore } from '@urql/svelte' export let cluster - export let availableMetrics = ['cpu_load', 'flops_any', 'mem_bw'] export let metricsInHistograms export let isOpen + let availableMetrics = ['cpu_load', 'flops_any', 'mem_used', 'mem_bw', 'net_bw', 'file_bw'] let pendingMetrics = [...metricsInHistograms] // Copy const client = getContextClient() diff --git a/web/frontend/src/User.root.svelte b/web/frontend/src/User.root.svelte index e216aa6f..a26c1aa6 100644 --- a/web/frontend/src/User.root.svelte +++ b/web/frontend/src/User.root.svelte @@ -44,7 +44,7 @@ histNumNodes { count, value } histMetrics { metric, unit, data { min, max, count, bin } } }}`, - variables: { jobFilters, metricsInHistograms} + variables: { jobFilters, metricsInHistograms } }) onMount(() => filterComponent.update()) diff --git a/web/frontend/src/utils.js b/web/frontend/src/utils.js index 537ad3fc..794a23a3 100644 --- a/web/frontend/src/utils.js +++ b/web/frontend/src/utils.js @@ -316,16 +316,18 @@ export function checkMetricDisabled(m, c, s) { //[m]etric, [c]luster, [s]ubclust } export function convert2uplot(canvasData) { - // initial use: Canvas Histogram Data to Uplot + // Prep: Uplot Data Structure let uplotData = [[],[]] // [X, Y1, Y2, ...] + // MetricHisto Only: Check if 1st bin not-null -> Set 0-Value bin for scaling + // Else: Only Single 0-Value bin returned -> No reset required + if (canvasData[0]?.bin) { + uplotData[0].push(0) + uplotData[1].push(0) + } + // Iterate canvasData.forEach( cd => { - if (cd.bin) { // MetricHisto Datafromat - // Force Zero Entry for scaling - if (uplotData[0].length == 0) { - uplotData[0].push(0) - uplotData[1].push(0) - } - uplotData[0].push(cd.max) + if (Object.keys(cd).length == 4) { // MetricHisto Datafromat + uplotData[0].push(cd?.max ? cd.max : 0) uplotData[1].push(cd.count) } else { // Default uplotData[0].push(cd.value) From ee4097a2ddcbd3459da03724777a8de57970eb75 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Mon, 11 Dec 2023 13:55:56 +0100 Subject: [PATCH 07/11] Add missing filters to crossjoinquery --- internal/repository/stats.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/repository/stats.go b/internal/repository/stats.go index ab70427e..b5813b9e 100644 --- a/internal/repository/stats.go +++ b/internal/repository/stats.go @@ -578,10 +578,15 @@ func (r *JobRepository) jobsMetricStatisticsHistogram( ) crossJoinQuery, cjqerr := SecurityCheck(ctx, crossJoinQuery) + if cjqerr != nil { return nil, cjqerr } + for _, f := range filters { + crossJoinQuery = BuildWhereClause(f, crossJoinQuery) + } + crossJoinQuerySql, _, sqlerr := crossJoinQuery.ToSql() if sqlerr != nil { return nil, sqlerr From 119637cb9bc8e857a1cc2ca0d08ee1c385a7e99c Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Tue, 12 Dec 2023 15:07:23 +0100 Subject: [PATCH 08/11] Fix using crossjoin arguments not used --- internal/repository/stats.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/repository/stats.go b/internal/repository/stats.go index b5813b9e..3ac04901 100644 --- a/internal/repository/stats.go +++ b/internal/repository/stats.go @@ -587,7 +587,7 @@ func (r *JobRepository) jobsMetricStatisticsHistogram( crossJoinQuery = BuildWhereClause(f, crossJoinQuery) } - crossJoinQuerySql, _, sqlerr := crossJoinQuery.ToSql() + crossJoinQuerySql, crossJoinQueryArgs, sqlerr := crossJoinQuery.ToSql() if sqlerr != nil { return nil, sqlerr } @@ -601,7 +601,7 @@ func (r *JobRepository) jobsMetricStatisticsHistogram( fmt.Sprintf(`CAST(((value.max / %d) * (%s )) as INTEGER ) as min`, bins, binQuery), fmt.Sprintf(`CAST(((value.max / %d) * (%s + 1 )) as INTEGER ) as max`, bins, binQuery), ).From("job").CrossJoin( - fmt.Sprintf(`(%s) as value`, crossJoinQuerySql), + fmt.Sprintf(`(%s) as value`, crossJoinQuerySql), crossJoinQueryArgs..., ).Where(fmt.Sprintf(`job.%s is not null and job.%s <= %f`, dbMetric, dbMetric, peak)) mainQuery, qerr := SecurityCheck(ctx, mainQuery) From ee6d286cd78b16309e237b50e68d52d3db9a0965 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Tue, 12 Dec 2023 15:42:14 +0100 Subject: [PATCH 09/11] Small corrections --- web/frontend/src/HistogramSelection.svelte | 5 ++--- web/frontend/src/User.root.svelte | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/web/frontend/src/HistogramSelection.svelte b/web/frontend/src/HistogramSelection.svelte index 142f6789..00f558a2 100644 --- a/web/frontend/src/HistogramSelection.svelte +++ b/web/frontend/src/HistogramSelection.svelte @@ -35,13 +35,11 @@ function closeAndApply() { metricsInHistograms = [...pendingMetrics] // Set for parent - + isOpen = !isOpen updateConfiguration({ name: cluster ? `user_view_histogramMetrics:${cluster}` : 'user_view_histogramMetrics', value: metricsInHistograms }) - - isOpen = false } @@ -62,5 +60,6 @@ + diff --git a/web/frontend/src/User.root.svelte b/web/frontend/src/User.root.svelte index a26c1aa6..5d9c597b 100644 --- a/web/frontend/src/User.root.svelte +++ b/web/frontend/src/User.root.svelte @@ -224,7 +224,7 @@ bind:metrics={metrics} bind:isOpen={isMetricsSelectionOpen} /> - \ No newline at end of file From 07073e290a6f02e00929a285fe0080a76e51e2eb Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Tue, 12 Dec 2023 16:46:03 +0100 Subject: [PATCH 10/11] feat: add selectable histograms to status view --- internal/repository/stats.go | 113 ++++++++++++++++++++++++++++ web/frontend/src/Status.root.svelte | 59 ++++++++++++++- 2 files changed, 169 insertions(+), 3 deletions(-) diff --git a/internal/repository/stats.go b/internal/repository/stats.go index 3ac04901..4d7be089 100644 --- a/internal/repository/stats.go +++ b/internal/repository/stats.go @@ -8,10 +8,12 @@ import ( "context" "database/sql" "fmt" + "math" "time" "github.com/ClusterCockpit/cc-backend/internal/config" "github.com/ClusterCockpit/cc-backend/internal/graph/model" + "github.com/ClusterCockpit/cc-backend/internal/metricdata" "github.com/ClusterCockpit/cc-backend/pkg/archive" "github.com/ClusterCockpit/cc-backend/pkg/log" "github.com/ClusterCockpit/cc-backend/pkg/schema" @@ -460,6 +462,18 @@ func (r *JobRepository) AddMetricHistograms( stat *model.JobsStatistics) (*model.JobsStatistics, error) { start := time.Now() + // Running Jobs Only: First query jobdata from sqlite, then query data and make bins + for _, f := range filter { + if f.State != nil { + if len(f.State) == 1 && f.State[0] == "running" { + stat.HistMetrics = r.runningJobsMetricStatisticsHistogram(ctx, metrics, filter) + log.Debugf("Timer AddMetricHistograms %s", time.Since(start)) + return stat, nil + } + } + } + + // All other cases: Query and make bins in sqlite directly for _, m := range metrics { metricHisto, err := r.jobsMetricStatisticsHistogram(ctx, m, filter) if err != nil { @@ -639,3 +653,102 @@ func (r *JobRepository) jobsMetricStatisticsHistogram( log.Debugf("Timer jobsStatisticsHistogram %s", time.Since(start)) return &result, nil } + +func (r *JobRepository) runningJobsMetricStatisticsHistogram( + ctx context.Context, + metrics []string, + filters []*model.JobFilter) []*model.MetricHistoPoints { + + // Get Jobs + jobs, err := r.QueryJobs(ctx, filters, &model.PageRequest{Page: 1, ItemsPerPage: 500 + 1}, nil) + if err != nil { + log.Errorf("Error while querying jobs for footprint: %s", err) + return nil + } + if len(jobs) > 500 { + log.Errorf("too many jobs matched (max: %d)", 500) + return nil + } + + // Get AVGs from metric repo + avgs := make([][]schema.Float, len(metrics)) + for i := range avgs { + avgs[i] = make([]schema.Float, 0, len(jobs)) + } + + for _, job := range jobs { + if job.MonitoringStatus == schema.MonitoringStatusDisabled || job.MonitoringStatus == schema.MonitoringStatusArchivingFailed { + continue + } + + if err := metricdata.LoadAverages(job, metrics, avgs, ctx); err != nil { + log.Errorf("Error while loading averages for histogram: %s", err) + return nil + } + } + + // Iterate metrics to fill endresult + data := make([]*model.MetricHistoPoints, 0) + for idx, metric := range metrics { + // Get specific Peak or largest Peak + var metricConfig *schema.MetricConfig + var peak float64 = 0.0 + var unit string = "" + + for _, f := range filters { + if f.Cluster != nil { + metricConfig = archive.GetMetricConfig(*f.Cluster.Eq, metric) + peak = metricConfig.Peak + unit = metricConfig.Unit.Prefix + metricConfig.Unit.Base + log.Debugf("Cluster %s filter found with peak %f for %s", *f.Cluster.Eq, peak, metric) + } + } + + if peak == 0.0 { + for _, c := range archive.Clusters { + for _, m := range c.MetricConfig { + if m.Name == metric { + if m.Peak > peak { + peak = m.Peak + } + if unit == "" { + unit = m.Unit.Prefix + m.Unit.Base + } + } + } + } + } + + // Make and fill bins + bins := 10.0 + peakBin := peak / bins + + points := make([]*model.MetricHistoPoint, 0) + for b := 0; b < 10; b++ { + count := 0 + bindex := b + 1 + bmin := math.Round(peakBin * float64(b)) + bmax := math.Round(peakBin * (float64(b) + 1.0)) + + // Iterate AVG values for indexed metric and count for bins + for _, val := range avgs[idx] { + if float64(val) >= bmin && float64(val) < bmax { + count += 1 + } + } + + bminint := int(bmin) + bmaxint := int(bmax) + + // Append Bin to Metric Result Array + point := model.MetricHistoPoint{Bin: &bindex, Count: count, Min: &bminint, Max: &bmaxint} + points = append(points, &point) + } + + // Append Metric Result Array to final results array + result := model.MetricHistoPoints{Metric: metric, Unit: unit, Data: points} + data = append(data, &result) + } + + return data +} diff --git a/web/frontend/src/Status.root.svelte b/web/frontend/src/Status.root.svelte index 563978dd..95fc98cf 100644 --- a/web/frontend/src/Status.root.svelte +++ b/web/frontend/src/Status.root.svelte @@ -15,6 +15,7 @@ Table, Progress, Icon, + Button } from "sveltestrap"; import { init, convert2uplot, transformPerNodeDataForRoofline } from "./utils.js"; import { scaleNumbers } from "./units.js"; @@ -24,6 +25,8 @@ getContextClient, mutationStore, } from "@urql/svelte"; + import PlotTable from './PlotTable.svelte' + import HistogramSelection from './HistogramSelection.svelte' const { query: initq } = init(); const ccconfig = getContext("cc-config"); @@ -63,7 +66,8 @@ option.key == ccconfig.status_view_selectedTopUserCategory ); - let metricsInHistograms = ccconfig[`status_view_histogramMetrics:${cluster}`] || ccconfig.status_view_histogramMetrics + let isHistogramSelectionOpen = false + $: metricsInHistograms = cluster ? ccconfig[`user_view_histogramMetrics:${cluster}`] : (ccconfig.user_view_histogramMetrics || []) const client = getContextClient(); $: mainQuery = queryStore({ @@ -75,6 +79,7 @@ $metrics: [String!] $from: Time! $to: Time! + $metricsInHistograms: [String!] ) { nodeMetrics( cluster: $cluster @@ -100,7 +105,7 @@ } } - stats: jobsStatistics(filter: $filter) { + stats: jobsStatistics(filter: $filter, metrics: $metricsInHistograms) { histDuration { count value @@ -117,6 +122,16 @@ count value } + histMetrics { + metric + unit + data { + min + max + count + bin + } + } } allocatedNodes(cluster: $cluster) { @@ -131,6 +146,7 @@ from: from.toISOString(), to: to.toISOString(), filter: [{ state: ["running"] }, { cluster: { eq: cluster } }], + metricsInHistograms: metricsInHistograms }, }); @@ -313,7 +329,7 @@

Current utilization of cluster "{cluster}"

- + {#if $initq.fetching || $mainQuery.fetching} {:else if $initq.error} @@ -323,6 +339,13 @@ {/if} + + + { @@ -668,4 +691,34 @@ {/key} +
+ {#if metricsInHistograms} + + + {#key $mainQuery.data.stats[0].histMetrics} + + + + + {/key} + + + {/if} {/if} + + From b829a5aafeb3998c663e2afdf1b51286a8b19fa7 Mon Sep 17 00:00:00 2001 From: Christoph Kluge Date: Wed, 13 Dec 2023 11:58:14 +0100 Subject: [PATCH 11/11] Improve binned data histogram legends --- web/frontend/src/Analysis.root.svelte | 5 +++-- web/frontend/src/Status.root.svelte | 5 +++-- web/frontend/src/User.root.svelte | 5 +++-- web/frontend/src/plots/Histogram.svelte | 9 +++++++++ web/frontend/src/utils.js | 6 ------ 5 files changed, 18 insertions(+), 12 deletions(-) diff --git a/web/frontend/src/Analysis.root.svelte b/web/frontend/src/Analysis.root.svelte index aa4ae379..163d5115 100644 --- a/web/frontend/src/Analysis.root.svelte +++ b/web/frontend/src/Analysis.root.svelte @@ -389,9 +389,10 @@ diff --git a/web/frontend/src/User.root.svelte b/web/frontend/src/User.root.svelte index 5d9c597b..ad08bc6b 100644 --- a/web/frontend/src/User.root.svelte +++ b/web/frontend/src/User.root.svelte @@ -192,9 +192,10 @@ diff --git a/web/frontend/src/plots/Histogram.svelte b/web/frontend/src/plots/Histogram.svelte index d3e1aaaf..499ea4fa 100644 --- a/web/frontend/src/plots/Histogram.svelte +++ b/web/frontend/src/plots/Histogram.svelte @@ -11,6 +11,7 @@ import { Card } from 'sveltestrap' export let data + export let usesBins = false export let width = 500 export let height = 300 export let title = '' @@ -160,6 +161,14 @@ series: [ { label: xunit !== '' ? xunit : null, + value: (u, ts, sidx, didx) => { + if (usesBins) { + const min = u.data[sidx][didx - 1] ? u.data[sidx][didx - 1] : 0 + const max = u.data[sidx][didx] + ts = min + ' - ' + max // narrow spaces + } + return ts + } }, Object.assign({ label: yunit !== '' ? yunit : null, diff --git a/web/frontend/src/utils.js b/web/frontend/src/utils.js index 794a23a3..53462086 100644 --- a/web/frontend/src/utils.js +++ b/web/frontend/src/utils.js @@ -318,12 +318,6 @@ export function checkMetricDisabled(m, c, s) { //[m]etric, [c]luster, [s]ubclust export function convert2uplot(canvasData) { // Prep: Uplot Data Structure let uplotData = [[],[]] // [X, Y1, Y2, ...] - // MetricHisto Only: Check if 1st bin not-null -> Set 0-Value bin for scaling - // Else: Only Single 0-Value bin returned -> No reset required - if (canvasData[0]?.bin) { - uplotData[0].push(0) - uplotData[1].push(0) - } // Iterate canvasData.forEach( cd => { if (Object.keys(cd).length == 4) { // MetricHisto Datafromat