From ba5eb8272e758dd58535593e0a5b24c3f0803b15 Mon Sep 17 00:00:00 2001 From: Ben Perry Date: Fri, 23 Aug 2024 11:48:40 -0500 Subject: [PATCH] Benchmark interface --- tools/benchmark/benchmark.go | 138 +++++++++++++++++++++++ tools/benchmark/stats.go | 211 +++++++++++++++++++++++++++++++++++ 2 files changed, 349 insertions(+) create mode 100644 tools/benchmark/benchmark.go create mode 100644 tools/benchmark/stats.go diff --git a/tools/benchmark/benchmark.go b/tools/benchmark/benchmark.go new file mode 100644 index 0000000..9f562f4 --- /dev/null +++ b/tools/benchmark/benchmark.go @@ -0,0 +1,138 @@ +package benchmark + +import ( + "fmt" + "slices" + "time" + + "github.com/gopxl/pixel/v2" + "github.com/gopxl/pixel/v2/backends/opengl" +) + +var Benchmarks = &Registry{benchmarks: map[string]Config{}} + +// Config defines how to run a given benchmark, along with metadata describing it +type Config struct { + Name string + Description string + + // New returns the benchmark to be executed + New func(win *opengl.Window) (Benchmark, error) + // Duration sets the maximum duration to run the benchmark + Duration time.Duration + // WindowConfig defines the input parameters to the benchmark's window + WindowConfig opengl.WindowConfig +} + +// Run executes the benchmark and calculates statistics about its performance +func (c Config) Run() (*Stats, error) { + fmt.Printf("Running benchmark %s\n", c.Name) + + windowConfig := c.WindowConfig + title := windowConfig.Title + if title == "" { + title = c.Name + } + windowConfig.Title = fmt.Sprintf("%s | FPS -", title) + + if windowConfig.Bounds.Empty() { + windowConfig.Bounds = pixel.R(0, 0, 1024, 1024) + } + if windowConfig.Position.Eq(pixel.ZV) { + windowConfig.Position = pixel.V(50, 50) + } + + duration := c.Duration + if duration == 0 { + duration = 10 * time.Second + } + + win, err := opengl.NewWindow(windowConfig) + if err != nil { + return nil, err + } + defer win.Destroy() + + benchmark, err := c.New(win) + if err != nil { + return nil, err + } + + frame := 0 + frameSeconds := make([]int, 0) + prevFrameCount := 0 + second := time.NewTicker(time.Second) + done := time.NewTicker(duration) + start := time.Now() +loop: + for frame = 0; !win.Closed(); frame++ { + benchmark.Step(win) + win.Update() + + select { + case <-second.C: + frameSeconds = append(frameSeconds, frame) + win.SetTitle(fmt.Sprintf("%s | FPS %v", title, frame-prevFrameCount)) + prevFrameCount = frame + case <-done.C: + break loop + default: + } + } + stats := NewStats(c.Name, time.Since(start), frame, frameSeconds) + + if win.Closed() { + return nil, fmt.Errorf("window closed early") + } + + return stats, err +} + +// Benchmark provides hooks into the stages of a window's lifecycle +type Benchmark interface { + Step(win *opengl.Window) +} + +// Registry is a collection of benchmark configs +type Registry struct { + benchmarks map[string]Config +} + +// List returns a copy of all registered benchmark configs +func (r *Registry) List() []Config { + configs := make([]Config, len(r.benchmarks)) + for i, name := range r.ListNames() { + configs[i] = r.benchmarks[name] + i++ + } + return configs +} + +// ListNames returns a sorted list of all registered benchmark names +func (r *Registry) ListNames() []string { + names := make([]string, len(r.benchmarks)) + i := 0 + for name := range r.benchmarks { + names[i] = name + i++ + } + slices.Sort(names) + return names +} + +// Add a benchmark config to the registry +func (r *Registry) Add(configs ...Config) { + for _, config := range configs { + r.benchmarks[config.Name] = config + } +} + +// Get a benchmark config by name +func (r *Registry) Get(name string) (Config, error) { + config, ok := r.benchmarks[name] + if !ok { + return config, fmt.Errorf("unknown benchmark %s", name) + } + + return config, nil +} diff --git a/tools/benchmark/stats.go b/tools/benchmark/stats.go new file mode 100644 index 0000000..8817af5 --- /dev/null +++ b/tools/benchmark/stats.go @@ -0,0 +1,211 @@ +package benchmark + +import ( + "encoding/json" + "fmt" + "math" + "os" + "os/user" + "runtime/debug" + "slices" + "strings" + "time" + + "github.com/olekukonko/tablewriter" +) + +var ( + machineName, pixelVersion string +) + +func init() { + machineName = getMachineName() + pixelVersion = getPixelVersion() +} + +// NewStats calculates statistics about a benchmark run +func NewStats(name string, duration time.Duration, frames int, frameSeconds []int) *Stats { + stats := &Stats{ + Name: name, + Frames: frames, + Duration: duration, + Machine: machineName, + PixelVersion: pixelVersion, + } + + milliseconds := stats.Duration.Milliseconds() + if milliseconds > 0 { + stats.AvgFPS = roundFloat(1000*float64(frames)/float64(milliseconds), 2) + } + + fps := make([]float64, 0, len(frameSeconds)) + for i, frame := range frameSeconds { + if i == 0 { + fps = append(fps, float64(frame)) + } else { + fps = append(fps, float64(frame-frameSeconds[i-1])) + } + } + if len(fps) > 0 { + stats.MinFPS = slices.Min(fps) + stats.MaxFPS = slices.Max(fps) + stats.StdevFPS = standardDeviation(fps) + } + + return stats +} + +// Stats stores data about the performance of a benchmark run +type Stats struct { + Name string `json:"name"` + AvgFPS float64 `json:"avgFPS"` + MinFPS float64 `json:"minFPS"` + MaxFPS float64 `json:"maxFPS"` + StdevFPS float64 `json:"stdevFPS"` + + Frames int `json:"frames"` + Duration time.Duration `json:"duration"` + + Machine string `json:"machine"` + PixelVersion string `json:"pixelVersion"` +} + +// Print stats to stdout in a human-readable format +func (s *Stats) Print() { + StatsCollection{s}.Print() +} + +// StatsCollection holds stats from multiple benchmark runs +type StatsCollection []*Stats + +func (sc StatsCollection) Print() { + data := make([][]string, len(sc)) + for i, stats := range sc { + data[i] = []string{ + stats.Machine, + stats.PixelVersion, + stats.Name, + roundDuration(stats.Duration, 2).String(), + toString(stats.Frames), + toString(stats.AvgFPS), + toString(stats.MinFPS), + toString(stats.MaxFPS), + toString(stats.StdevFPS), + } + } + + table := tablewriter.NewWriter(os.Stdout) + headers := []string{"Machine", "Pixel", "Benchmark", "Duration", "Frames", "FPS Avg", "FPS Min", "FPS Max", "FPS Stdev"} + widths := map[string]int{ + "Machine": 18, + "Pixel": 6, + "Benchmark": 28, + } + for i, header := range headers { + minWidth := widths[header] + if minWidth == 0 { + minWidth = 6 + } + table.SetColMinWidth(i, minWidth) + } + table.SetHeader(headers) + table.SetAutoFormatHeaders(false) + table.SetAlignment(tablewriter.ALIGN_LEFT) + table.SetHeaderAlignment(tablewriter.ALIGN_LEFT) + table.SetBorders(tablewriter.Border{Left: true, Top: false, Right: true, Bottom: false}) + table.SetCenterSeparator("|") + table.AppendBulk(data) + table.Render() +} + +// Dump writes a JSON file of all stored statistics to the given path +func (sc StatsCollection) Dump(path string) error { + bytes, err := json.Marshal(sc) + if err != nil { + return err + } + if err := os.WriteFile(path, bytes, 0666); err != nil { + return err + } + return nil +} + +// roundFloat rounds the value to the given number of decimal places +func roundFloat(val float64, precision uint) float64 { + ratio := math.Pow(10, float64(precision)) + return math.Round(val*ratio) / ratio +} + +// roundDuration rounds the duration to the given number of decimal places based on the unit +func roundDuration(duration time.Duration, precision uint) time.Duration { + durationRounding := time.Duration(math.Pow(10, float64(precision))) + switch { + case duration > time.Second: + return duration.Round(time.Second / durationRounding) + case duration > time.Millisecond: + return duration.Round(time.Millisecond / durationRounding) + case duration > time.Microsecond: + return duration.Round(time.Microsecond / durationRounding) + default: + return duration + } +} + +func toString(val any) string { + switch v := val.(type) { + case float64: + return fmt.Sprintf("%v", roundFloat(v, 2)) + case float32: + return fmt.Sprintf("%v", roundFloat(float64(v), 2)) + default: + return fmt.Sprintf("%v", v) + } +} + +// standardDeviation calulates the variation of the given values relative to the average +func standardDeviation(values []float64) float64 { + var sum, avg, stdev float64 + for _, val := range values { + sum += val + } + count := float64(len(values)) + avg = sum / count + + for _, val := range values { + stdev += math.Pow(val-avg, 2) + } + stdev = math.Sqrt(stdev / count) + return stdev +} + +func getMachineName() string { + envs := []string{"MACHINE_NAME", "USER", "USERNAME"} + var name string + for _, env := range envs { + name = os.Getenv(env) + if name != "" { + return name + } + } + if u, err := user.Current(); err == nil { + return u.Username + } + return "unknown" +} + +func getPixelVersion() string { + ver := os.Getenv("PIXEL_VERSION") + if ver != "" { + return ver + } + + bi, ok := debug.ReadBuildInfo() + if ok { + for _, dep := range bi.Deps { + if dep.Path == "github.com/gopxl/pixel/v2" { + return strings.Split(dep.Version, "-")[0] + } + } + } + return "x.y.z" +}