Merge pull request #77 from Halleck45/optimize_performance

Option to profile code

.gitignore

@@ -9,4 +9,12 @@ dist/
 build
 bindata.go
 include
-coverage.txt
+coverage.txt
+
+# Profiling
+ast-metrics.cpu
+ast-metrics.mem
+
+# CI
+ast-metrics-report.json
+metrics.txt

Makefile

@@ -48,3 +48,9 @@ monkey-test:
 	@echo "\e[34m\033[1m-> Monkey testing\033[0m\e[39m\n"
 	bash scripts/monkey-test.sh
 	@echo "\e[34m\033[1mDONE \033[0m\e[39m\n"
+
+# profiling
+profile:
+	go run . a --non-interactive --profile src
+	go tool pprof -png  ast-metrics.cpu
+	go tool pprof -png  ast-metrics.mem

go.mod

@@ -39,14 +39,17 @@ require (
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/aymerick/douceur v0.2.0 // indirect
 	github.com/bsm/openmetrics v0.3.1 // indirect
+	github.com/chzyer/readline v1.5.1 // indirect
 	github.com/containerd/console v1.0.4 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dlclark/regexp2 v1.11.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/pprof v0.0.0-20241128161848-dc51965c6481 // indirect
 	github.com/gookit/color v1.5.4 // indirect
 	github.com/gorilla/css v1.0.1 // indirect
+	github.com/ianlancetaylor/demangle v0.0.0-20240312041847-bd984b5ce465 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/lithammer/fuzzysearch v1.1.8 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect

go.sum

@@ -39,6 +39,10 @@ github.com/charmbracelet/glamour v0.6.0 h1:wi8fse3Y7nfcabbbDuwolqTqMQPMnVPeZhDM2
 github.com/charmbracelet/glamour v0.6.0/go.mod h1:taqWV4swIMMbWALc0m7AfE9JkPSU8om2538k9ITBxOc=
 github.com/charmbracelet/lipgloss v0.10.0 h1:KWeXFSexGcfahHX+54URiZGkBFazf70JNMtwg/AFW3s=
 github.com/charmbracelet/lipgloss v0.10.0/go.mod h1:Wig9DSfvANsxqkRsqj6x87irdy123SR4dOXlKa91ciE=
+github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ=
+github.com/chzyer/readline v1.5.1 h1:upd/6fQk4src78LMRzh5vItIt361/o4uq553V8B5sGI=
+github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk=
+github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8=
 github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U=
 github.com/containerd/console v1.0.4 h1:F2g4+oChYvBTsASRTz8NP6iIAi97J3TtSAsLbIFn4ro=
 github.com/containerd/console v1.0.4/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk=
@@ -68,13 +72,17 @@ github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/pprof v0.0.0-20241128161848-dc51965c6481 h1:yudKIrXagAOl99WQzrP1gbz5HLB9UjhcOFnPzdd6Qec=
+github.com/google/pprof v0.0.0-20241128161848-dc51965c6481/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
 github.com/gookit/color v1.4.2/go.mod h1:fqRyamkC1W8uxl+lxCQxOT09l/vYfZ+QeiX3rKQHCoQ=
 github.com/gookit/color v1.5.0/go.mod h1:43aQb+Zerm/BWh2GnrgOQm7ffz7tvQXEKV6BFMl7wAo=
 github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0=
 github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w=
 github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c=
 github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8=
 github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0=
+github.com/ianlancetaylor/demangle v0.0.0-20240312041847-bd984b5ce465 h1:KwWnWVWCNtNq/ewIX7HIKnELmEx2nDP42yskD/pi7QE=
+github.com/ianlancetaylor/demangle v0.0.0-20240312041847-bd984b5ce465/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw=
 github.com/inancgumus/screen v0.0.0-20190314163918-06e984b86ed3 h1:fO9A67/izFYFYky7l1pDP5Dr0BTCRkaQJUG6Jm5ehsk=
 github.com/inancgumus/screen v0.0.0-20190314163918-06e984b86ed3/go.mod h1:Ey4uAp+LvIl+s5jRbOHLcZpUDnkjLBROl15fZLwPlTM=
 github.com/jarcoal/httpmock v1.3.1 h1:iUx3whfZWVf3jT01hQTO/Eo5sAYtB2/rqaUuOtpInww=
@@ -213,6 +221,7 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

main.go

@@ -4,6 +4,8 @@ import (
 	"bufio"
 	"fmt"
 	"os"
+	"runtime"
+	"runtime/pprof"
 
 	"github.com/charmbracelet/lipgloss"
 	"github.com/halleck45/ast-metrics/src/Cli"
@@ -119,6 +121,12 @@ func main() {
 						Usage:    "Compare with another Git branch or commit",
 						Category: "Global options",
 					},
+					// Profiling (with pprof)
+					&cli.BoolFlag{
+						Name:     "profile",
+						Usage:    "Generate a profiling reports into files ast-metrics.cpu and ast-metrics.mem",
+						Category: "Global options",
+					},
 				},
 				Action: func(cCtx *cli.Context) error {
 
@@ -127,6 +135,32 @@ func main() {
 						log.SetLevel(log.DebugLevel)
 					}
 
+					// get option --profile
+					profile := cCtx.Bool("profile")
+					if profile {
+						cpufile := "ast-metrics.cpu"
+						memfile := "ast-metrics.mem"
+						f, err := os.Create(cpufile)
+						if err != nil {
+							log.Fatal("could not create CPU profile: ", err)
+						}
+						defer f.Close() // error handling omitted for example
+						if err := pprof.StartCPUProfile(f); err != nil {
+							log.Fatal("could not start CPU profile: ", err)
+						}
+						defer pprof.StopCPUProfile()
+
+						f, err = os.Create(memfile)
+						if err != nil {
+							log.Fatal("could not create memory profile: ", err)
+						}
+						defer f.Close() // error handling omitted for example
+						runtime.GC()    // get up-to-date statistics
+						if err := pprof.WriteHeapProfile(f); err != nil {
+							log.Fatal("could not write memory profile: ", err)
+						}
+					}
+
 					// get option --non-interactive
 					isInteractive := true
 					if cCtx.Bool("non-interactive") || cCtx.Bool("ci") {

scripts/monkey-test.sh

@@ -1,14 +1,22 @@
 set -e
 
 # number of packages to download
-PACKAGES_COUNT=100
+PACKAGES_COUNT=$1
+if [ -z "$PACKAGES_COUNT" ]; then
+    PACKAGES_COUNT=100
+fi
 
-workdir=$(mktemp -d)
+# keep always the same workdir, to avoid download time for each package
+workdir="build/monkey-test"
 echo "Working in $workdir"
 if [ -z "$workdir" ]; then
     echo "Workdir not found"
     exit 1
 fi
+if [ ! -d "$workdir" ]; then
+    echo "Workdir not found, creating it"
+    mkdir -p $workdir
+fi
 
 # cleanup reports
 rm -f ast-metrics-report.json
@@ -25,28 +33,34 @@ echo "Downloading $PACKAGES_COUNT packages"
 for package in $packages; 
 do 
     echo "  Downloading $package"
-	repository=$(curl -s https://packagist.org/packages/$package.json | jq -r '.package.repository')
-    zipUrl="$repository/archive/refs/heads/master.zip"
-    # generate random name for destination
-    name=$(uuidgen)
-    destination="$workdir/$name"
-    echo "    Downloading $zipUrl to $destination"
-    curl -s -L -o $destination.zip $zipUrl
-
-    # if zip contains HTML, like "Just a moment...", then skip
-    if grep -q "<html" $destination.zip; then
-        echo "  Skipping $package because it contains HTML (probably rate limited)"
-        continue
-    fi
+
+    # generate md5 checksum for destination
+    checksum=$(echo $package | md5sum | awk '{ print $1 }')
+    destination="$workdir/$checksum"
+
+    if [ ! -d $destination ]; then
+        repository=$(curl -s https://packagist.org/packages/$package.json | jq -r '.package.repository')
+        zipUrl="$repository/archive/refs/heads/master.zip"
+        echo "    Downloading $zipUrl to $destination"
+        curl -s -L -o $destination.zip $zipUrl
+        # if zip contains HTML, like "Just a moment...", then skip
+        if grep -q "<html" $destination.zip; then
+            echo "  Skipping $package because it contains HTML (probably rate limited)"
+            continue
+        fi
+
+        # if contains 404, then skip
+        if grep -q "404" $destination.zip; then
+            echo "  Skipping $package because it contains 404"
+            continue
+        fi
 
-    # if contains 404, then skip
-    if grep -q "404" $destination.zip; then
-        echo "  Skipping $package because it contains 404"
-        continue
+        unzip $destination.zip -d $destination > /dev/null
+        rm $destination.zip
+    else
+        echo "  Skipping $package because it already exists"
     fi
 
-    unzip $destination.zip -d $destination > /dev/null
-    rm $destination.zip
 done
 
 echo "Analyzing $workdir"

src/Analyzer/Aggregator.go

@@ -141,7 +141,7 @@ func newAggregated() Aggregated {
 		LocPerClass:                             NewAggregateResult(),
 		LocPerMethod:                            NewAggregateResult(),
 		ClocPerMethod:                           NewAggregateResult(),
-		CyclomaticComplexity:           		 NewAggregateResult(),
+		CyclomaticComplexity:                    NewAggregateResult(),
 		CyclomaticComplexityPerMethod:           NewAggregateResult(),
 		CyclomaticComplexityPerClass:            NewAggregateResult(),
 		HalsteadEffort:                          NewAggregateResult(),

src/Command/AnalyzeCommand.go

@@ -71,6 +71,7 @@ func (v *AnalyzeCommand) Execute() error {
 		v.spinner = nil
 	}
 
+	// Convert source code to ASTs (each source code is converted to a binary protobuf file)
 	err := v.ExecuteRunnerAnalysis(v.configuration)
 	if err != nil {
 		return err

src/Engine/NodeTypeEnsurer.go

@@ -29,6 +29,31 @@ func EnsureNodeTypeIsComplete(file *pb.File) {
 		}
 	}
 
+	// Transfert complexity from classes and functions to file itself
+	classes := GetClassesInFile(file)
+	if len(classes) == 0 {
+		functions := GetFunctionsInFile(file)
+		for _, function := range functions {
+			if function.Stmts.Analyze == nil || function.Stmts.Analyze.Complexity == nil || function.Stmts.Analyze.Complexity.Cyclomatic == nil {
+				continue
+			}
+
+			// increment complexity of file itself
+			ccn := *function.Stmts.Analyze.Complexity.Cyclomatic + *file.Stmts.Analyze.Complexity.Cyclomatic
+			file.Stmts.Analyze.Complexity.Cyclomatic = &ccn
+		}
+	} else {
+		for _, class := range classes {
+			if class.Stmts.Analyze == nil || class.Stmts.Analyze.Complexity == nil || class.Stmts.Analyze.Complexity.Cyclomatic == nil {
+				continue
+			}
+
+			// increment complexity of file itself
+			ccn := *class.Stmts.Analyze.Complexity.Cyclomatic + *file.Stmts.Analyze.Complexity.Cyclomatic
+			file.Stmts.Analyze.Complexity.Cyclomatic = &ccn
+		}
+	}
+
 	if file.Stmts.Analyze.Coupling == nil {
 		file.Stmts.Analyze.Coupling = &pb.Coupling{
 			Afferent: 0,

src/Engine/Php/PhpRunner.go

@@ -2,6 +2,7 @@ package Php
 
 import (
 	"os"
+	"runtime"
 	"strings"
 	"sync"
 
@@ -60,17 +61,31 @@ func (r PhpRunner) Finish() error {
 // DumpAST dumps the AST of python files in protobuf format
 func (r PhpRunner) DumpAST() {
 
+	cpuCount := runtime.NumCPU()
 	var wg sync.WaitGroup
 	cnt := 0
+	filesChan := make(chan string, cpuCount)
+
+	for i := 0; i < cpuCount; i++ {
+		go func() {
+			for filePath := range filesChan {
+				cnt++
+				if r.progressbar != nil {
+					r.progressbar.UpdateText("Dumping AST of PHP files (" + fmt.Sprintf("%d", cnt) + "/" + fmt.Sprintf("%d", len(r.getFileList().Files)) + ")")
+				}
+				wg.Add(1)
+				go r.dumpOneAst(&wg, filePath)
+			}
+		}()
+	}
+
+	// split files between workers
 	for _, filePath := range r.getFileList().Files {
-		cnt++
-		if r.progressbar != nil {
-			r.progressbar.UpdateText("Dumping AST of PHP files (" + fmt.Sprintf("%d", cnt) + "/" + fmt.Sprintf("%d", len(r.getFileList().Files)) + ")")
-		}
-		wg.Add(1)
-		go r.dumpOneAst(&wg, filePath)
+		filesChan <- filePath
 	}
 
+	// wait for all workers to finish
+	close(filesChan)
 	wg.Wait()
 
 	if r.progressbar != nil {

src/Report/HtmlReportGenerator.go

@@ -400,6 +400,9 @@ func (v *HtmlReportGenerator) RegisterFilters() {
 			return pongo2.AsValue(Engine.GetClassesInFile(file)), nil
 		}
 
-		return pongo2.AsValue(file.Stmts), nil
+		collection := make([]*pb.StmtFunction, 0)
+		collection = append(collection, file.Stmts.StmtFunction...)
+
+		return pongo2.AsValue(collection), nil
 	})
 }

src/Report/templates/html/componentChartRadiusBarComplexity.html

@@ -3,20 +3,14 @@
         {% set separator = "," %}
         {%- set files = currentView.ConcernedFiles -%}
         {%- for file in files -%}
-        {%- if len(file.Stmts.StmtClass) == 0 -%}
-        {% set elements = file|convertOneFileToCollection -%}
-        {% set name = file.Path %}
-        {%- else %}
-        {% set elements = file.Stmts.StmtClass -%}
-        {% set name = "" -%}
-        {%- endif -%}
+        {% set elements = file|toCollectionOfParsableComponents %}
         {%if forloop.last %}
             {% set separator = "" %}
         {% endif %}
-        {%- for class in elements -%}
+        {%- for item in elements -%}
         {
-            "name": "{{ name|default:class.Name.Qualified|addslashes }}",
-            "cyclomatic": {{ class.Stmts.Analyze.Complexity.Cyclomatic|floatformat:0 }}
+            "name": "{{ item.Name.Qualified|default:item.Path|addslashes }}",
+            "cyclomatic": {{ item.Stmts.Analyze.Complexity.Cyclomatic|floatformat:0 }}
         }{{ separator }}
         {%- endfor -%}
         {%- endfor -%}

src/Report/templates/markdown/index.md

@@ -3,24 +3,24 @@ AST Metrisc report
 ## Overview
 
 {% set mi="🔴" -%}
-{%- if projectAggregated.Combined.AverageMI > 84 -%}
+{%- if projectAggregated.Combined.MaintainabilityIndex.Avg > 84 -%}
     {% set mi="🟢" %}
-{%- elif projectAggregated.Combined.AverageMI > 64 -%}
+{%- elif projectAggregated.Combined.MaintainabilityIndex.Avg > 64 -%}
     {% set mi="🟡" -%}
 {%- endif -%}
 
-> Maintainability index: {{ mi }} {{ projectAggregated.Combined.AverageMI|floatformat:0 }}
+> Maintainability index: {{ mi }} {{ projectAggregated.Combined.MaintainabilityIndex.Avg|floatformat:0 }}
 
 | Language | LOC | Maintainability | Complexity per method | Average lines per method |
 | --- | --- | --- | --- | --- |
 {% for languageName,language in projectAggregated.ByProgrammingLanguage -%}
 {%- set mi="🔴" -%}
-{%- if language.AverageMI > 84 -%}
+{%- if language.MaintainabilityIndex.Avg > 84 -%}
     {% set mi="🟢" %}
-{%- elif language.AverageMI > 64 -%}
+{%- elif language.MaintainabilityIndex.Avg > 64 -%}
     {% set mi="🟡" -%}
 {%- endif -%}
-| **{{ languageName }}** | {{ language.Loc|stringifyNumber }} | {{ mi }} {{ language.AverageMI | floatformat:0 }} | {{ language.AverageCyclomaticComplexityPerMethod | floatformat:2 }} | {{ language.AverageLocPerMethod | floatformat:0 }} |
+| **{{ languageName }}** | {{ language.Loc.Sum|stringifyNumber }} | {{ mi }} {{ language.MaintainabilityIndex.Avg | floatformat:0 }} | {{ language.CyclomaticComplexityPerMethod.Avg | floatformat:2 }} | {{ language.LocPerMethod.Avg | floatformat:0 }} |
 {%- endfor %}
 
 > 💡 Help