From e87bb243a9e94faa3d19f8bbf750e18a178790f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20L=C3=A9pine?= Date: Sat, 30 Nov 2024 13:59:21 +0100 Subject: [PATCH 1/4] Option to profile code --- .gitignore | 10 +++++++++- Makefile | 6 ++++++ go.mod | 3 +++ go.sum | 9 +++++++++ main.go | 34 ++++++++++++++++++++++++++++++++++ 5 files changed, 61 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 48efe53..a07e05b 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,12 @@ dist/ build bindata.go include -coverage.txt \ No newline at end of file +coverage.txt + +# Profiling +ast-metrics.cpu +ast-metrics.mem + +# CI +ast-metrics-report.json +metrics.txt \ No newline at end of file diff --git a/Makefile b/Makefile index e304a79..08cf0e0 100644 --- a/Makefile +++ b/Makefile @@ -48,3 +48,9 @@ monkey-test: @echo "\e[34m\033[1m-> Monkey testing\033[0m\e[39m\n" bash scripts/monkey-test.sh @echo "\e[34m\033[1mDONE \033[0m\e[39m\n" + +# profiling +profile: + go run . a --non-interactive --profile src + go tool pprof -png ast-metrics.cpu + go tool pprof -png ast-metrics.mem \ No newline at end of file diff --git a/go.mod b/go.mod index ea8f889..a4b4ac7 100644 --- a/go.mod +++ b/go.mod @@ -39,14 +39,17 @@ require ( github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/bsm/openmetrics v0.3.1 // indirect + github.com/chzyer/readline v1.5.1 // indirect github.com/containerd/console v1.0.4 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dlclark/regexp2 v1.11.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/google/go-cmp v0.6.0 // indirect + github.com/google/pprof v0.0.0-20241128161848-dc51965c6481 // indirect github.com/gookit/color v1.5.4 // indirect github.com/gorilla/css v1.0.1 // indirect + github.com/ianlancetaylor/demangle v0.0.0-20240312041847-bd984b5ce465 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/lithammer/fuzzysearch v1.1.8 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect diff --git a/go.sum b/go.sum index c198867..8dd452e 100644 --- a/go.sum +++ b/go.sum @@ -39,6 +39,10 @@ github.com/charmbracelet/glamour v0.6.0 h1:wi8fse3Y7nfcabbbDuwolqTqMQPMnVPeZhDM2 github.com/charmbracelet/glamour v0.6.0/go.mod h1:taqWV4swIMMbWALc0m7AfE9JkPSU8om2538k9ITBxOc= github.com/charmbracelet/lipgloss v0.10.0 h1:KWeXFSexGcfahHX+54URiZGkBFazf70JNMtwg/AFW3s= github.com/charmbracelet/lipgloss v0.10.0/go.mod h1:Wig9DSfvANsxqkRsqj6x87irdy123SR4dOXlKa91ciE= +github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ= +github.com/chzyer/readline v1.5.1 h1:upd/6fQk4src78LMRzh5vItIt361/o4uq553V8B5sGI= +github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk= +github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8= github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= github.com/containerd/console v1.0.4 h1:F2g4+oChYvBTsASRTz8NP6iIAi97J3TtSAsLbIFn4ro= github.com/containerd/console v1.0.4/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk= @@ -68,6 +72,8 @@ github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/pprof v0.0.0-20241128161848-dc51965c6481 h1:yudKIrXagAOl99WQzrP1gbz5HLB9UjhcOFnPzdd6Qec= +github.com/google/pprof v0.0.0-20241128161848-dc51965c6481/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/gookit/color v1.4.2/go.mod h1:fqRyamkC1W8uxl+lxCQxOT09l/vYfZ+QeiX3rKQHCoQ= github.com/gookit/color v1.5.0/go.mod h1:43aQb+Zerm/BWh2GnrgOQm7ffz7tvQXEKV6BFMl7wAo= github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0= @@ -75,6 +81,8 @@ github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/Q github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= +github.com/ianlancetaylor/demangle v0.0.0-20240312041847-bd984b5ce465 h1:KwWnWVWCNtNq/ewIX7HIKnELmEx2nDP42yskD/pi7QE= +github.com/ianlancetaylor/demangle v0.0.0-20240312041847-bd984b5ce465/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= github.com/inancgumus/screen v0.0.0-20190314163918-06e984b86ed3 h1:fO9A67/izFYFYky7l1pDP5Dr0BTCRkaQJUG6Jm5ehsk= github.com/inancgumus/screen v0.0.0-20190314163918-06e984b86ed3/go.mod h1:Ey4uAp+LvIl+s5jRbOHLcZpUDnkjLBROl15fZLwPlTM= github.com/jarcoal/httpmock v1.3.1 h1:iUx3whfZWVf3jT01hQTO/Eo5sAYtB2/rqaUuOtpInww= @@ -213,6 +221,7 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/main.go b/main.go index 7c9b404..57f88c2 100644 --- a/main.go +++ b/main.go @@ -4,6 +4,8 @@ import ( "bufio" "fmt" "os" + "runtime" + "runtime/pprof" "github.com/charmbracelet/lipgloss" "github.com/halleck45/ast-metrics/src/Cli" @@ -119,6 +121,12 @@ func main() { Usage: "Compare with another Git branch or commit", Category: "Global options", }, + // Profiling (with pprof) + &cli.BoolFlag{ + Name: "profile", + Usage: "Generate a profiling reports into files ast-metrics.cpu and ast-metrics.mem", + Category: "Global options", + }, }, Action: func(cCtx *cli.Context) error { @@ -127,6 +135,32 @@ func main() { log.SetLevel(log.DebugLevel) } + // get option --profile + profile := cCtx.Bool("profile") + if profile { + cpufile := "ast-metrics.cpu" + memfile := "ast-metrics.mem" + f, err := os.Create(cpufile) + if err != nil { + log.Fatal("could not create CPU profile: ", err) + } + defer f.Close() // error handling omitted for example + if err := pprof.StartCPUProfile(f); err != nil { + log.Fatal("could not start CPU profile: ", err) + } + defer pprof.StopCPUProfile() + + f, err = os.Create(memfile) + if err != nil { + log.Fatal("could not create memory profile: ", err) + } + defer f.Close() // error handling omitted for example + runtime.GC() // get up-to-date statistics + if err := pprof.WriteHeapProfile(f); err != nil { + log.Fatal("could not write memory profile: ", err) + } + } + // get option --non-interactive isInteractive := true if cCtx.Bool("non-interactive") || cCtx.Bool("ci") { From 72c98790b0222c35fc589fd90e03695c2adb3417 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20L=C3=A9pine?= Date: Sun, 1 Dec 2024 07:08:54 +0100 Subject: [PATCH 2/4] make sure no oop code has complexity --- src/Analyzer/Aggregator.go | 2 +- src/Engine/NodeTypeEnsurer.go | 25 +++++++++++++++++++ src/Report/HtmlReportGenerator.go | 5 +++- .../componentChartRadiusBarComplexity.html | 14 +++-------- 4 files changed, 34 insertions(+), 12 deletions(-) diff --git a/src/Analyzer/Aggregator.go b/src/Analyzer/Aggregator.go index 574eeb0..0b10bcf 100644 --- a/src/Analyzer/Aggregator.go +++ b/src/Analyzer/Aggregator.go @@ -141,7 +141,7 @@ func newAggregated() Aggregated { LocPerClass: NewAggregateResult(), LocPerMethod: NewAggregateResult(), ClocPerMethod: NewAggregateResult(), - CyclomaticComplexity: NewAggregateResult(), + CyclomaticComplexity: NewAggregateResult(), CyclomaticComplexityPerMethod: NewAggregateResult(), CyclomaticComplexityPerClass: NewAggregateResult(), HalsteadEffort: NewAggregateResult(), diff --git a/src/Engine/NodeTypeEnsurer.go b/src/Engine/NodeTypeEnsurer.go index 1140a05..88ae5f1 100644 --- a/src/Engine/NodeTypeEnsurer.go +++ b/src/Engine/NodeTypeEnsurer.go @@ -29,6 +29,31 @@ func EnsureNodeTypeIsComplete(file *pb.File) { } } + // Transfert complexity from classes and functions to file itself + classes := GetClassesInFile(file) + if len(classes) == 0 { + functions := GetFunctionsInFile(file) + for _, function := range functions { + if function.Stmts.Analyze == nil || function.Stmts.Analyze.Complexity == nil || function.Stmts.Analyze.Complexity.Cyclomatic == nil { + continue + } + + // increment complexity of file itself + ccn := *function.Stmts.Analyze.Complexity.Cyclomatic + *file.Stmts.Analyze.Complexity.Cyclomatic + file.Stmts.Analyze.Complexity.Cyclomatic = &ccn + } + } else { + for _, class := range classes { + if class.Stmts.Analyze == nil || class.Stmts.Analyze.Complexity == nil || class.Stmts.Analyze.Complexity.Cyclomatic == nil { + continue + } + + // increment complexity of file itself + ccn := *class.Stmts.Analyze.Complexity.Cyclomatic + *file.Stmts.Analyze.Complexity.Cyclomatic + file.Stmts.Analyze.Complexity.Cyclomatic = &ccn + } + } + if file.Stmts.Analyze.Coupling == nil { file.Stmts.Analyze.Coupling = &pb.Coupling{ Afferent: 0, diff --git a/src/Report/HtmlReportGenerator.go b/src/Report/HtmlReportGenerator.go index 778611e..7b6fd97 100644 --- a/src/Report/HtmlReportGenerator.go +++ b/src/Report/HtmlReportGenerator.go @@ -400,6 +400,9 @@ func (v *HtmlReportGenerator) RegisterFilters() { return pongo2.AsValue(Engine.GetClassesInFile(file)), nil } - return pongo2.AsValue(file.Stmts), nil + collection := make([]*pb.StmtFunction, 0) + collection = append(collection, file.Stmts.StmtFunction...) + + return pongo2.AsValue(collection), nil }) } diff --git a/src/Report/templates/html/componentChartRadiusBarComplexity.html b/src/Report/templates/html/componentChartRadiusBarComplexity.html index c044c82..69f8b16 100644 --- a/src/Report/templates/html/componentChartRadiusBarComplexity.html +++ b/src/Report/templates/html/componentChartRadiusBarComplexity.html @@ -3,20 +3,14 @@ {% set separator = "," %} {%- set files = currentView.ConcernedFiles -%} {%- for file in files -%} - {%- if len(file.Stmts.StmtClass) == 0 -%} - {% set elements = file|convertOneFileToCollection -%} - {% set name = file.Path %} - {%- else %} - {% set elements = file.Stmts.StmtClass -%} - {% set name = "" -%} - {%- endif -%} + {% set elements = file|toCollectionOfParsableComponents %} {%if forloop.last %} {% set separator = "" %} {% endif %} - {%- for class in elements -%} + {%- for item in elements -%} { - "name": "{{ name|default:class.Name.Qualified|addslashes }}", - "cyclomatic": {{ class.Stmts.Analyze.Complexity.Cyclomatic|floatformat:0 }} + "name": "{{ item.Name.Qualified|default:item.Path|addslashes }}", + "cyclomatic": {{ item.Stmts.Analyze.Complexity.Cyclomatic|floatformat:0 }} }{{ separator }} {%- endfor -%} {%- endfor -%} From 007a59941e7a6929d8a99bef9a728e3ad773829e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20L=C3=A9pine?= Date: Sun, 1 Dec 2024 07:23:23 +0100 Subject: [PATCH 3/4] markdown report uses the new aggregator --- scripts/monkey-test.sh | 56 ++++++++++++++++---------- src/Report/templates/markdown/index.md | 12 +++--- 2 files changed, 41 insertions(+), 27 deletions(-) diff --git a/scripts/monkey-test.sh b/scripts/monkey-test.sh index 7f5ae27..0061e75 100644 --- a/scripts/monkey-test.sh +++ b/scripts/monkey-test.sh @@ -1,14 +1,22 @@ set -e # number of packages to download -PACKAGES_COUNT=100 +PACKAGES_COUNT=$1 +if [ -z "$PACKAGES_COUNT" ]; then + PACKAGES_COUNT=100 +fi -workdir=$(mktemp -d) +# keep always the same workdir, to avoid download time for each package +workdir="build/monkey-test" echo "Working in $workdir" if [ -z "$workdir" ]; then echo "Workdir not found" exit 1 fi +if [ ! -d "$workdir" ]; then + echo "Workdir not found, creating it" + mkdir -p $workdir +fi # cleanup reports rm -f ast-metrics-report.json @@ -25,28 +33,34 @@ echo "Downloading $PACKAGES_COUNT packages" for package in $packages; do echo " Downloading $package" - repository=$(curl -s https://packagist.org/packages/$package.json | jq -r '.package.repository') - zipUrl="$repository/archive/refs/heads/master.zip" - # generate random name for destination - name=$(uuidgen) - destination="$workdir/$name" - echo " Downloading $zipUrl to $destination" - curl -s -L -o $destination.zip $zipUrl - - # if zip contains HTML, like "Just a moment...", then skip - if grep -q " /dev/null + rm $destination.zip + else + echo " Skipping $package because it already exists" fi - unzip $destination.zip -d $destination > /dev/null - rm $destination.zip done echo "Analyzing $workdir" diff --git a/src/Report/templates/markdown/index.md b/src/Report/templates/markdown/index.md index 0925ab7..526202f 100644 --- a/src/Report/templates/markdown/index.md +++ b/src/Report/templates/markdown/index.md @@ -3,24 +3,24 @@ AST Metrisc report ## Overview {% set mi="🔴" -%} -{%- if projectAggregated.Combined.AverageMI > 84 -%} +{%- if projectAggregated.Combined.MaintainabilityIndex.Avg > 84 -%} {% set mi="🟢" %} -{%- elif projectAggregated.Combined.AverageMI > 64 -%} +{%- elif projectAggregated.Combined.MaintainabilityIndex.Avg > 64 -%} {% set mi="🟡" -%} {%- endif -%} -> Maintainability index: {{ mi }} {{ projectAggregated.Combined.AverageMI|floatformat:0 }} +> Maintainability index: {{ mi }} {{ projectAggregated.Combined.MaintainabilityIndex.Avg|floatformat:0 }} | Language | LOC | Maintainability | Complexity per method | Average lines per method | | --- | --- | --- | --- | --- | {% for languageName,language in projectAggregated.ByProgrammingLanguage -%} {%- set mi="🔴" -%} -{%- if language.AverageMI > 84 -%} +{%- if language.MaintainabilityIndex.Avg > 84 -%} {% set mi="🟢" %} -{%- elif language.AverageMI > 64 -%} +{%- elif language.MaintainabilityIndex.Avg > 64 -%} {% set mi="🟡" -%} {%- endif -%} -| **{{ languageName }}** | {{ language.Loc|stringifyNumber }} | {{ mi }} {{ language.AverageMI | floatformat:0 }} | {{ language.AverageCyclomaticComplexityPerMethod | floatformat:2 }} | {{ language.AverageLocPerMethod | floatformat:0 }} | +| **{{ languageName }}** | {{ language.Loc.Sum|stringifyNumber }} | {{ mi }} {{ language.MaintainabilityIndex.Avg | floatformat:0 }} | {{ language.CyclomaticComplexityPerMethod.Avg | floatformat:2 }} | {{ language.LocPerMethod.Avg | floatformat:0 }} | {%- endfor %} > 💡 Help From 1783965b42f69958c73ec8f3d697483f81ff5668 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20L=C3=A9pine?= Date: Sun, 1 Dec 2024 07:36:33 +0100 Subject: [PATCH 4/4] limit parallelisation to number of available CPU when parsing PHP files --- src/Command/AnalyzeCommand.go | 1 + src/Engine/Php/PhpRunner.go | 27 +++++++++++++++++++++------ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/Command/AnalyzeCommand.go b/src/Command/AnalyzeCommand.go index c42d208..f1062f9 100644 --- a/src/Command/AnalyzeCommand.go +++ b/src/Command/AnalyzeCommand.go @@ -71,6 +71,7 @@ func (v *AnalyzeCommand) Execute() error { v.spinner = nil } + // Convert source code to ASTs (each source code is converted to a binary protobuf file) err := v.ExecuteRunnerAnalysis(v.configuration) if err != nil { return err diff --git a/src/Engine/Php/PhpRunner.go b/src/Engine/Php/PhpRunner.go index 50d7e67..772b3fd 100644 --- a/src/Engine/Php/PhpRunner.go +++ b/src/Engine/Php/PhpRunner.go @@ -2,6 +2,7 @@ package Php import ( "os" + "runtime" "strings" "sync" @@ -60,17 +61,31 @@ func (r PhpRunner) Finish() error { // DumpAST dumps the AST of python files in protobuf format func (r PhpRunner) DumpAST() { + cpuCount := runtime.NumCPU() var wg sync.WaitGroup cnt := 0 + filesChan := make(chan string, cpuCount) + + for i := 0; i < cpuCount; i++ { + go func() { + for filePath := range filesChan { + cnt++ + if r.progressbar != nil { + r.progressbar.UpdateText("Dumping AST of PHP files (" + fmt.Sprintf("%d", cnt) + "/" + fmt.Sprintf("%d", len(r.getFileList().Files)) + ")") + } + wg.Add(1) + go r.dumpOneAst(&wg, filePath) + } + }() + } + + // split files between workers for _, filePath := range r.getFileList().Files { - cnt++ - if r.progressbar != nil { - r.progressbar.UpdateText("Dumping AST of PHP files (" + fmt.Sprintf("%d", cnt) + "/" + fmt.Sprintf("%d", len(r.getFileList().Files)) + ")") - } - wg.Add(1) - go r.dumpOneAst(&wg, filePath) + filesChan <- filePath } + // wait for all workers to finish + close(filesChan) wg.Wait() if r.progressbar != nil {