benchmark cleanup and selfhosting (#210)

.github/workflows/benchmark-comment.yml

@@ -18,6 +18,7 @@ on:
 jobs:
   comment:
     runs-on: ubuntu-latest
+    #runs-on: self-hosted
     if: >
       ${{ github.event.workflow_run.event == 'pull_request' &&
           github.event.workflow_run.conclusion == 'success' }}

.github/workflows/benchmark.yml

@@ -1,11 +1,11 @@
 name: Performance tracking
-
 on:
   pull_request:
 
 jobs:
   performance-tracking:
     runs-on: ubuntu-latest
+    #runs-on: self-hosted
     steps:
       # setup
       - uses: actions/checkout@v4

Project.toml

@@ -45,7 +45,7 @@ ILog2 = "0.2.3"
 InteractiveUtils = "1"
 LinearAlgebra = "1"
 MacroTools = "0.5.9"
-Makie = "0.19.7"
+Makie = "0.20"
 Nemo = "0.38"
 Plots = "1.38.0"
 PrecompileTools = "1.2"

benchmark/plotbenchmarks.jl

@@ -24,6 +24,8 @@ function results_to_dataframe(res)
     DataFrame(recs)
 end
 
+##
+cd("QuantumCliffordBenchmarksLog/logs")
 ##
 
 files = glob("*.benchmarkresults")
@@ -35,22 +37,29 @@ for f in files
         continue
     end
     sdf = results_to_dataframe(readresults(f))
-    sdf[!,:tag] .= m.captures[1]
+    #sdf[!,:tag] .= m.captures[1]
+    sdf[!,:version] .= VersionNumber(m.captures[1][2:end])
     sdf[!,:threads] .= m.captures[2]
-    sdf[!,:julia] .= m.captures[3]
+    #sdf[!,:julia] .= m.captures[3]
+    sdf[!,:julia] .= m.captures[3]=="nightly" ? v"1.999" : VersionNumber(m.captures[3])
     push!(subframes, sdf)
 end
 
 ##
 
 df = vcat(subframes...)
+maxjversion = maximum(df[df.julia .!= v"1.999",:julia])
+df[df.julia .== v"1.999", :julia] .= VersionNumber(maxjversion.major, maxjversion.minor+1, 0)
 df[!,:fullname] .= df[:,:taxon1] .* "-" .* df[:,:taxon2] .* "-" .* df[:,:taxon3]
 df[!,:groupname] .= df[:,:taxon1] .* "-" .* df[:,:taxon2]
 df[!,:logtime] .= log10.(df[:,:time])
+df_min = combine(groupby(df, [:taxon3, :taxon2, :taxon1, :julia, :threads]), :logtime=>minimum=>:minlogtime)
+df = outerjoin(df, df_min, on=[:taxon3, :taxon2, :taxon1, :julia, :threads])
+df[!,:normlogtime] .= df[:,:logtime] .- df[:,:minlogtime]
 
-df2 = combine(groupby(df, [:groupname, :taxon1, :tag, :julia, :threads]), :logtime=>mean=>:logtime)
-df1 = combine(groupby(df2, [:taxon1, :tag, :julia, :threads]), :logtime=>mean=>:logtime)
-df0 = combine(groupby(df1, [:tag, :julia, :threads]), :logtime=>mean=>:logtime)
+df2 = combine(groupby(df, [:groupname, :taxon1, :version, :julia, :threads]), :normlogtime=>mean=>:logtime)
+df1 = combine(groupby(df2, [:taxon1, :version, :julia, :threads]), :logtime=>mean=>:logtime)
+df0 = combine(groupby(df1, [:version, :julia, :threads]), :logtime=>mean=>:logtime)
 
 layers = visual(Stairs, step=:center)
 axis = (;width = 500, height = 400, xticklabelrotation=-pi/3)
@@ -59,25 +68,25 @@ facet = (;linkyaxes=:minimal)
 ##
 
 benchmark_data3 = data(df)
-mappings3 = mapping(:tag, :time, color=:julia, row=:fullname, col=:threads)
+mappings3 = mapping(:version, :time, color=:julia, row=:fullname, col=:threads)
 f3 = draw(benchmark_data3 * layers * mappings3; facet, axis)
 save("../benchmarks3.png",f3)
 
 benchmark_data2 = data(df2)
-mappings2 = mapping(:tag, :logtime, color=:julia, row=:groupname, col=:threads)
+mappings2 = mapping(:version, :logtime, color=:julia, row=:groupname, col=:threads)
 f2 = draw(benchmark_data2 * layers * mappings2; facet, axis)
 save("../benchmarks2.png",f2)
 
 benchmark_data1 = data(df1)
-mappings1 = mapping(:tag, :logtime, color=:julia, row=:taxon1, col=:threads)
+mappings1 = mapping(:version, :logtime, color=:julia, row=:taxon1, col=:threads)
 f1 = draw(benchmark_data1 * layers * mappings1; facet, axis)
 save("../benchmarks1.png",f1)
 
 benchmark_data0 = data(df0)
-mappings0 = mapping(:tag, :logtime, color=:julia, col=:threads)
+mappings0 = mapping(:version, :logtime, color=:julia, col=:threads)
 f0 = draw(benchmark_data0 * layers * mappings0; facet, axis)
 save("../benchmarks0.png",f0)
 
-mappings3a = mapping(:tag, :allocs, color=:julia, row=:fullname, col=:threads)
+mappings3a = mapping(:version, :allocs, color=:julia, row=:fullname, col=:threads)
 f3a = draw(benchmark_data3 * layers * mappings3a; facet, axis)
 save("../benchmarks3_allocs.png",f3a)

benchmark/plotttfxbenchmarks.jl

@@ -8,7 +8,8 @@ using Statistics
 using Glob
 
 ##
-
+cd("QuantumCliffordBenchmarksLog/ttfxlogs")
+##
 files = glob("using_*.ttfxresults")
 records = []
 for f in files
@@ -18,26 +19,38 @@ for f in files
         continue
     end
     tag = m.captures[1]
+    version = VersionNumber(tag[2:end])
     threads = m.captures[2]
-    julia = m.captures[3]
-    pkgimport = minimum(parse.(Float64, readlines(f)))
-    task = minimum(parse.(Float64, readlines("task_tag=$(tag)-nthreads=$(threads)-julia=$(julia).ttfxresults")))
-    push!(records, (;tag, threads, julia, pkgimport, task))
+    juliastr = m.captures[3]
+    julia = juliastr=="nightly" ? v"1.999" : VersionNumber(m.captures[3])
+    pkgimportsamples = parse.(Float64, readlines(f))
+    pkgimport = isempty(pkgimportsamples) ? missing : minimum(parse.(Float64, readlines(f)))
+    tasksamples = parse.(Float64, readlines("task_tag=$(tag)-nthreads=$(threads)-julia=$(juliastr).ttfxresults"))
+    task = isempty(tasksamples) ? missing : minimum(tasksamples)
+    push!(records, (;version, threads, julia, pkgimport, task))
 end
 
 ##
 
 df = DataFrame(records)
+maxjversion = maximum(df[df.julia .!= v"1.999",:julia])
+df[df.julia .== v"1.999", :julia] .= VersionNumber(maxjversion.major, maxjversion.minor+1, 0)
 df[!,:total] .= df[:,:task] .+ df[:,:pkgimport]
 df = DataFrames.stack(df, [:total, :task, :pkgimport], variable_name=:x, value_name=:time)
+sort!(df) # by first column (version)
+
+df = df[df.time .!== missing,:] # otherwise the ticks are messed up, probably a bug
 
 layers = visual(Stairs, step=:center)
-axis = (;width = 500, height = 400, xticklabelrotation=-pi/3)
-facet = (;linkyaxes=:minimal)
+maxy = ceil(maximum(skipmissing(df.time)))
+axis = (;width = 800, height = 400, xticklabelrotation=-pi/2, yticks=(0:0.5:maxy))
+facet = (;
+    linkyaxes=:minimal,
+    )
 
 ##
 
 benchmark_data = data(df)
-mappings = mapping(:tag, :time, color=:julia, row=:x, col=:threads)
+mappings = mapping(:version, :time, color=:julia, row=:x, col=:threads)
 f = draw(benchmark_data * layers * mappings; facet, axis)
 save("../benchmarks_ttfx.png",f)

benchmark/run_benchmark_matrix.fish

@@ -0,0 +1,12 @@
+#!/usr/bin/env fish
+for jlversion in nightly 1.10.0 1.9.0 1.8.0 1.7.0 1.6.0
+    for tag in (git for-each-ref --sort=creatordate --format '%(tag)' refs/tags | tail -n+15)
+        for nthreads in 1 4
+            for retune in true
+                ./run_ttfx_bechmarks.sh -v$jlversion -t$tag -n$nthreads
+                ./run_bechmarks.sh -v$jlversion -t$tag -n$nthreads -r$retune
+            end
+        end
+    end
+end
+