Merge pull request #150 from countvajhula/gather-new-benchmark-data

Gather new benchmark data

.github/workflows/competitive-benchmarks.yml

@@ -0,0 +1,43 @@
+name: competitive-benchmarks
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'qi-sdk/benchmarks/competitive/**'
+      - 'qi-lib/**'
+      - '.github/workflows/competitive-benchmarks.yml'
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  deploy-benchmarks:
+    runs-on: ubuntu-latest
+    name: Generate and deploy competitive benchmarks
+    steps:
+      - name: Checkout
+        uses: actions/checkout@master
+      - name: Install Racket
+        uses: Bogdanp/[email protected]
+        with:
+          architecture: 'x64'
+          distribution: 'full'
+          variant: 'CS'
+          version: 'stable'
+      - name: Install Package and its Dependencies
+        run: make install
+      - name: Install SDK
+        run: make install-sdk
+      - name: Run benchmark
+        shell: 'bash --noprofile --norc -eo pipefail {0}'
+        run: make new-benchmarks
+      - name: Push to GitHub Pages
+        uses: JamesIves/[email protected]
+        with:
+          folder: qi-sdk/benchmarks/competitive/results
+          target-folder: competitive-benchmarks
+          branch: gh-pages
+          clean: true

.gitignore

@@ -12,3 +12,6 @@ coverage
 *.js
 MathJax
 katex
+
+# benchmarks output
+qi-sdk/benchmarks/competitive/results

Makefile

@@ -40,6 +40,7 @@ help:
 	@echo "coverage-check - Run test coverage checker"
 	@echo "coverage-report - View test coverage report"
 	@echo "docs - view docs in a browser"
+	@echo "new-benchmarks - Run new benchmarks and generate an HTML report"
 	@echo "profile - Get a performance profile of any Racket command"
 	@echo "benchmark - Run comprehensive performance benchmarks"
 	@echo "benchmark-competitive - Run competitive benchmarks"
@@ -189,6 +190,17 @@ profile:
 	@echo "For more options:"
 	@echo "  raco profile --help"
 
+new-benchmarks:
+	cd qi-sdk/benchmarks/competitive && \
+	scribble \
+		++convert svg \
+		++arg -p \
+		++arg github \
+		--html \
+		--dest results \
+		--dest-name index.html \
+		report.scrbl
+
 benchmark-local:
 	racket $(PACKAGE-NAME)-sdk/benchmarks/local/report.rkt
 
@@ -212,4 +224,4 @@ performance-report:
 performance-regression-report:
 	@racket $(PACKAGE-NAME)-sdk/benchmarks/report.rkt -r $(REF)
 
-.PHONY:	help install remove build build-docs build-all clean check-deps test test-flow test-on test-threading test-switch test-definitions test-macro test-util test-expander test-compiler test-probe test-with-errortrace errortrace errortrace-flow errortrace-on errortrace-threading errortrace-switch errortrace-definitions errortrace-macro errortrace-util errortrace-probe docs cover coverage-check coverage-report cover-coveralls profile benchmark-local benchmark-loading benchmark-selected-forms benchmark-competitive benchmark-nonlocal benchmark performance-report performance-regression-report
+.PHONY:	help install remove build build-docs build-all clean check-deps test test-flow test-on test-threading test-switch test-definitions test-macro test-util test-expander test-compiler test-probe test-with-errortrace errortrace errortrace-flow errortrace-on errortrace-threading errortrace-switch errortrace-definitions errortrace-macro errortrace-util errortrace-probe docs cover coverage-check coverage-report cover-coveralls profile new-benchmarks benchmark-local benchmark-loading benchmark-selected-forms benchmark-competitive benchmark-nonlocal benchmark performance-report performance-regression-report

qi-sdk/benchmarks/competitive/report.scrbl

@@ -0,0 +1,169 @@
+#lang scribble/manual
+
+@require[scribble-math/dollar
+	 srfi/19
+	 vlibench
+	 (for-syntax racket/base)
+	 racket/cmdline
+	 racket/string
+	 racket/function]
+
+@;Command-line handling ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+@(define config-profile
+  (let ()
+    (define profile-box (box 'preview))
+    (command-line
+      #:once-each
+      (("-p" "--profile")
+       name
+       "profile name to use (use 'list' to list available profiles)"
+       (when (equal? name "list")
+	 (displayln
+	  (format
+	   "Available profiles: ~a"
+	   (string-join
+	    (for/list (((k v) vlib/profiles))
+	      (symbol->string k))
+	    ", ")))
+	 (exit 0))
+       (set-box! profile-box (string->symbol name))))
+    (unbox profile-box)))
+
+@title[#:style (with-html5 manual-doc-style)]{Qi Normal/Deforested Competitive Benchmarks}
+
+@;Qi version helper ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+@(begin-for-syntax
+  (require setup/getinfo)
+  (define (get-version)
+    ((get-info '("qi")) 'version)))
+@(define-syntax (get-qi-version stx)
+  (datum->syntax stx (get-version) stx stx))
+
+@;Specification ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+@(module qi-default racket/base
+  (require qi
+           racket/math
+           racket/list
+           vlibench)
+
+  (provide qi-filter-map-prog
+           qi-filter-map-foldl-prog
+           qi-long-pipeline-prog
+           qi-range-map-car-prog)
+
+  (define impl-label "Qi")
+
+  (define qi-filter-map-prog
+    (make-vlib/prog impl-label
+                    (flow (~>> (filter odd?) (map sqr)))))
+
+  (define qi-filter-map-foldl-prog
+    (make-vlib/prog impl-label
+                    (flow (~>> (filter odd?) (map sqr) (foldl + 0)))))
+
+  (define qi-long-pipeline-prog
+    (make-vlib/prog impl-label
+                    (flow (~>> range
+                               (filter odd?)
+                               (map sqr)
+                               values
+                               (filter (lambda (v) (< (remainder v 10) 5)))
+                               (map (lambda (v) (* v 2)))
+                               (foldl + 0)))))
+
+  (define qi-range-map-car-prog
+    (make-vlib/prog impl-label
+                    (flow (~>> range (map sqr) car))))
+  )
+
+@(module qi-deforested racket/base
+  (require qi
+           qi/list
+           racket/math
+           vlibench)
+
+  (provide qi/d-filter-map-prog
+           qi/d-filter-map-foldl-prog
+           qi/d-long-pipeline-prog
+           qi/d-range-map-car-prog)
+
+  (define impl-label "Qi deforested")
+
+  (define qi/d-filter-map-prog
+    (make-vlib/prog impl-label
+                    (flow (~>> (filter odd?) (map sqr)))))
+
+  (define qi/d-filter-map-foldl-prog
+    (make-vlib/prog impl-label
+                    (flow (~>> (filter odd?) (map sqr) (foldl + 0)))))
+
+  (define qi/d-long-pipeline-prog
+    (make-vlib/prog impl-label
+                    (flow (~>> range
+                               (filter odd?)
+                               (map sqr)
+                               values
+                               (filter (lambda (v) (< (remainder v 10) 5)))
+                               (map (lambda (v) (* v 2)))
+                               (foldl + 0)))))
+
+  (define qi/d-range-map-car-prog
+    (make-vlib/prog impl-label
+                    (flow (~>> range (map sqr) car))))
+  )
+
+@(require 'qi-default
+         'qi-deforested)
+
+@(define benchmarks-specs
+  (list
+   (vlib/spec 'filter-map
+          make-random-integer-list
+          (list qi/d-filter-map-prog
+                qi-filter-map-prog))
+   (vlib/spec 'filter-map-foldl
+          make-random-integer-list
+          (list qi/d-filter-map-foldl-prog
+                qi-filter-map-foldl-prog))
+   (vlib/spec 'long-pipeline
+          identity
+          (list qi/d-long-pipeline-prog
+                qi-long-pipeline-prog))
+   (vlib/spec 'range-map-car
+          identity
+          (list qi/d-range-map-car-prog
+                qi-range-map-car-prog))))
+
+@; Processing ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+@define[profile (hash-ref vlib/profiles config-profile)]
+@(define results
+  (for/list ((spec (in-list benchmarks-specs)))
+    (run-benchmark
+      spec
+      #:profile profile)))
+
+@; Rendering ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+@section{General Information}
+
+Date and time: @(date->string (seconds->date (current-seconds)))
+
+@snippet:vlib/profile[profile]
+
+@snippet:system-information[#:more-versions `(("Qi Version: " ,(get-qi-version)))]
+
+
+@section{Summary Results}
+
+@snippet:summary-results-table[results]
+
+
+@section{Detailed Results}
+
+@snippet:benchmark/s-duration[results]
+
+Measured lengths: @(racket #,(for/list ((len (vlib/profile->steps profile))) len))
+
+@(for/list ((result (in-list results)))
+  (snippet:benchmark-result result))

qi-sdk/info.rkt

@@ -11,6 +11,7 @@
                "relation-lib"
                "csv-writing"
                "require-latency"
+               "vlibench"
                "cover"
                "cover-coveralls"))
 (define build-deps '())