From 8c452b6454b6f465e6189f363900941aa2ada5ad Mon Sep 17 00:00:00 2001 From: "jeremie.desgagne.bouchard" Date: Mon, 20 Nov 2023 00:15:12 -0500 Subject: [PATCH] new benchmarks --- benchmarks/regressor-cpu.csv | 13 ++ benchmarks/regressor-gpu.csv | 13 ++ benchmarks/regressor.jl | 269 ++++++++++++++++++----------------- 3 files changed, 166 insertions(+), 129 deletions(-) create mode 100644 benchmarks/regressor-cpu.csv create mode 100644 benchmarks/regressor-gpu.csv diff --git a/benchmarks/regressor-cpu.csv b/benchmarks/regressor-cpu.csv new file mode 100644 index 0000000..c18f6fe --- /dev/null +++ b/benchmarks/regressor-cpu.csv @@ -0,0 +1,13 @@ +device,nobs,nfeats,max_depth,train_evo,train_xgb,infer_evo,infer_xgb +cpu,100000,10,6,0.571302287,0.931940322,0.045015373,0.06524121 +cpu,100000,10,11,6.11657148,1.161374098,0.084641913,0.088485924 +cpu,100000,100,6,0.961452175,1.592773516,0.115538193,0.17662686 +cpu,100000,100,11,17.849992393,3.853003881,0.121810382,0.263126441 +cpu,1000000,10,6,2.383909891,6.795574553,0.264884995,0.361242933 +cpu,1000000,10,11,11.161634211,9.32421656,0.794897806,0.714718326 +cpu,1000000,100,6,6.020354898,15.24576219,0.776566288,1.873882103 +cpu,1000000,100,11,30.228386878,19.789690573,1.154914415,2.376135847 +cpu,10000000,10,6,27.248707143,88.515808528,2.812803051,3.182972925 +cpu,10000000,10,11,62.273960418,116.989930049,6.267709913,6.473606844 +cpu,10000000,100,6,71.912413469,150.482093149,6.636390315,16.507357233 +cpu,10000000,100,11,196.855192543,194.620105633,12.155773288,24.468722938 diff --git a/benchmarks/regressor-gpu.csv b/benchmarks/regressor-gpu.csv new file mode 100644 index 0000000..5b09fd0 --- /dev/null +++ b/benchmarks/regressor-gpu.csv @@ -0,0 +1,13 @@ +device,nobs,nfeats,max_depth,train_evo,train_xgb,infer_evo,infer_xgb +gpu,100000,10,6,1.009510498,0.653007059,0.044266191, +gpu,100000,10,11,16.381585168,1.498272035,0.083979915, +gpu,100000,100,6,1.686124929,0.697764277,0.075100306, +gpu,100000,100,11,31.022717789,3.686352557,0.111697012, +gpu,1000000,10,6,1.715696068,1.146328262,0.362810704, +gpu,1000000,10,11,24.533893695,3.147383026,0.796694699, +gpu,1000000,100,6,3.450828332,3.24444115,0.633558641, +gpu,1000000,100,11,43.326096264,8.691848273,1.184580648, +gpu,10000000,10,6,7.573575617,8.384329387,3.870231138, +gpu,10000000,10,11,38.915609007,12.824271885,6.358167029, +gpu,10000000,100,6,20.566461348,26.477476765,6.024578283, +gpu,10000000,100,11,71.042451127,55.537101359,12.329862253, diff --git a/benchmarks/regressor.jl b/benchmarks/regressor.jl index cb32da2..245a502 100644 --- a/benchmarks/regressor.jl +++ b/benchmarks/regressor.jl @@ -1,4 +1,6 @@ using Revise +using DataFrames +using CSV using Statistics using StatsBase: sample using XGBoost @@ -6,7 +8,7 @@ using XGBoost using EvoTrees using BenchmarkTools using Random: seed! -# import CUDA +import CUDA ### v.0.15.1 # desktop | 1e6 | depth 11 | cpu: 37.2s @@ -19,133 +21,142 @@ using Random: seed! #threads # laptop depth 6: 12.717845 seconds (2.08 M allocations: 466.228 MiB) -nobs = Int(1e6) -num_feat = Int(100) -nrounds = 200 -max_depth = 6 -tree_type = "binary" -T = Float64 -nthread = Base.Threads.nthreads() -@info "testing with: $nobs observations | $num_feat features. nthread: $nthread | tree_type : $tree_type | max_depth : $max_depth" -seed!(123) -x_train = rand(T, nobs, num_feat) -y_train = rand(T, size(x_train, 1)) - -@info nthread -loss = "mse" -if loss == "mse" - loss_xgb = "reg:squarederror" - metric_xgb = "mae" - loss_evo = :mse - metric_evo = :mae -elseif loss == "logloss" - loss_xgb = "reg:logistic" - metric_xgb = "logloss" - loss_evo = :logloss - metric_evo = :logloss -end +# for device in ["cpu", "gpu"] +# for nobs in Int.([1e5, 1e6, 1e7]) +# for nfeats in [10, 100] +# for max_depth in [6, 11] -@info "XGBoost" -@info "train" -params_xgb = Dict( - :num_round => nrounds, - :max_depth => max_depth - 1, - :eta => 0.05, - :objective => loss_xgb, - :print_every_n => 5, - :subsample => 0.5, - :colsample_bytree => 0.5, - :tree_method => "hist", # hist/gpu_hist - :max_bin => 64, -) - -dtrain = DMatrix(x_train, y_train) -watchlist = Dict("train" => DMatrix(x_train, y_train)); -@time m_xgb = xgboost(dtrain; watchlist, nthread=nthread, verbosity=0, eval_metric=metric_xgb, params_xgb...); -# @btime m_xgb = xgboost($dtrain; watchlist, nthread=nthread, verbosity=0, eval_metric = metric_xgb, params_xgb...); -@info "predict" -@time pred_xgb = XGBoost.predict(m_xgb, x_train); -# @btime XGBoost.predict($m_xgb, $x_train); - -# @info "lightgbm train:" -# m_gbm = LGBMRegression( -# objective = "regression", -# boosting = "gbdt", -# num_iterations = 200, -# learning_rate = 0.05, -# num_leaves = 256, -# max_depth = 5, -# tree_learner = "serial", -# num_threads = Sys.CPU_THREADS, -# histogram_pool_size = -1., -# min_data_in_leaf = 1, -# min_sum_hessian_in_leaf = 0, -# max_delta_step = 0, -# min_gain_to_split = 0, -# feature_fraction = 0.5, -# feature_fraction_seed = 2, -# bagging_fraction = 0.5, -# bagging_freq = 1, -# bagging_seed = 3, -# max_bin = 64, -# bin_construct_sample_cnt = 200000, -# data_random_seed = 1, -# is_sparse = false, -# feature_pre_filter = false, -# is_unbalance = false, -# min_data_per_group = 1, -# metric = ["mae"], -# metric_freq = 10, -# # early_stopping_round = 10, -# ) -# @time gbm_results = fit!(m_gbm, x_train, y_train, (x_train, y_train)) -# @time pred_gbm = LightGBM.predict(m_gbm, x_train) |> vec - -@info "EvoTrees" -verbosity = 1 -params_evo = EvoTreeRegressor(; - loss=loss_evo, - nrounds=nrounds, - alpha=0.5, - lambda=0.0, - gamma=0.0, - eta=0.05, - max_depth=max_depth, - min_weight=1.0, - rowsample=0.5, - colsample=0.5, - nbins=64, - tree_type, - rng=123 -) - -@info "EvoTrees CPU" -device = "cpu" -# @info "init" -# @time m, cache = EvoTrees.init(params_evo, x_train, y_train); -# @time m, cache = EvoTrees.init(params_evo, x_train, y_train); -# @info "train - no eval" -# @time m_evo = fit_evotree(params_evo; x_train, y_train, device, verbosity, print_every_n=100); -# @time m_evo = fit_evotree(params_evo; x_train, y_train, device, verbosity, print_every_n=100); -@info "train - eval" -@time m_evo = fit_evotree(params_evo; x_train, y_train, x_eval=x_train, y_eval=y_train, metric=metric_evo, device, verbosity, print_every_n=100); -@time m_evo = fit_evotree(params_evo; x_train, y_train, x_eval=x_train, y_eval=y_train, metric=metric_evo, device, verbosity, print_every_n=100); -@info "predict" -@time pred_evo = m_evo(x_train); -@time pred_evo = m_evo(x_train); -# @btime m_evo($x_train); - -@info "EvoTrees GPU" +df = DataFrame() device = "gpu" -# @info "train - no eval" -# CUDA.@time m_evo = fit_evotree(params_evo; x_train, y_train, device, verbosity, print_every_n=100); -# CUDA.@time m_evo = fit_evotree(params_evo; x_train, y_train, device, verbosity, print_every_n=100); -@info "train - eval" -CUDA.@time m_evo = fit_evotree(params_evo; x_train, y_train, x_eval=x_train, y_eval=y_train, metric=metric_evo, device, verbosity, print_every_n=100); -CUDA.@time m_evo = fit_evotree(params_evo; x_train, y_train, x_eval=x_train, y_eval=y_train, metric=metric_evo, device, verbosity, print_every_n=100); -# @time m_evo = fit_evotree(params_evo; x_train, y_train); -# @btime fit_evotree($params_evo; x_train=$x_train, y_train=$y_train, x_eval=$x_train, y_eval=$y_train, metric=metric_evo, device, verbosity); -@info "predict" -CUDA.@time pred_evo = m_evo(x_train; device); -CUDA.@time pred_evo = m_evo(x_train; device); -# @btime m_evo($x_train; device); +for device in [device] + for nobs in Int.([1e5, 1e6, 1e7]) + for nfeats in [10, 100] + for max_depth in [6, 11] + + # nobs = Int(1e6) + # nfeats = Int(100) + # max_depth = 6 + max_nrounds = 200 + tree_type = "binary" + T = Float64 + nthreads = Base.Threads.nthreads() + @info "device: $device | nobs: $nobs | nfeats: $nfeats | max_depth : $max_depth | nthreads: $nthreads | tree_type : $tree_type" + seed!(123) + x_train = rand(T, nobs, nfeats) + y_train = rand(T, size(x_train, 1)) + + loss = "mse" + if loss == "mse" + loss_xgb = "reg:squarederror" + metric_xgb = "mae" + loss_evo = :mse + metric_evo = :mae + elseif loss == "logloss" + loss_xgb = "reg:logistic" + metric_xgb = "logloss" + loss_evo = :logloss + metric_evo = :logloss + end + tree_method = device == "gpu" ? "gpu_hist" : "hist" + + @info "XGBoost" + params_xgb = Dict( + :num_round => max_nrounds, + :max_depth => max_depth - 1, + :eta => 0.05, + :objective => loss_xgb, + :print_every_n => 5, + :subsample => 0.5, + :colsample_bytree => 0.5, + :tree_method => tree_method, # hist/gpu_hist + :max_bin => 64, + ) + + @info "train" + dtrain = DMatrix(x_train, y_train) + watchlist = Dict("train" => DMatrix(x_train, y_train)) + # m_xgb = xgboost(dtrain; watchlist, nthread=nthreads, verbosity=0, eval_metric=metric_xgb, params_xgb...) + t_train_xgb = @elapsed m_xgb = xgboost(dtrain; watchlist, nthread=nthreads, verbosity=0, eval_metric=metric_xgb, params_xgb...) + # @btime m_xgb = xgboost($dtrain; watchlist, nthread=nthreads, verbosity=0, eval_metric = metric_xgb, params_xgb...); + @info "predict" + # pred_xgb = XGBoost.predict(m_xgb, x_train) + # t_infer_xgb = @elapsed pred_xgb = XGBoost.predict(m_xgb, x_train) + t_infer_xgb = missing + # @btime XGBoost.predict($m_xgb, $x_train); + + # @info "lightgbm train:" + # m_gbm = LGBMRegression( + # objective = "regression", + # boosting = "gbdt", + # num_iterations = 200, + # learning_rate = 0.05, + # num_leaves = 256, + # max_depth = 5, + # tree_learner = "serial", + # num_threads = Sys.CPU_THREADS, + # histogram_pool_size = -1., + # min_data_in_leaf = 1, + # min_sum_hessian_in_leaf = 0, + # max_delta_step = 0, + # min_gain_to_split = 0, + # feature_fraction = 0.5, + # feature_fraction_seed = 2, + # bagging_fraction = 0.5, + # bagging_freq = 1, + # bagging_seed = 3, + # max_bin = 64, + # bin_construct_sample_cnt = 200000, + # data_random_seed = 1, + # is_sparse = false, + # min_data_per_group = 1, + # metric = ["mae"], + # metric_freq = 10, + # # early_stopping_round = 10, + # ) + # @time gbm_results = fit!(m_gbm, x_train, y_train, (x_train, y_train)) + # @time pred_gbm = LightGBM.predict(m_gbm, x_train) |> vec + + @info "EvoTrees" + verbosity = 1 + + params_evo = EvoTreeRegressor(; + loss=loss_evo, + nrounds=max_nrounds, + alpha=0.5, + lambda=0.0, + gamma=0.0, + eta=0.05, + max_depth=max_depth, + min_weight=1.0, + rowsample=0.5, + colsample=0.5, + nbins=64, + tree_type, + rng=123 + ) + + @info "train - eval" + @time m_evo = fit_evotree(params_evo; x_train, y_train, x_eval=x_train, y_eval=y_train, metric=metric_evo, device, verbosity, print_every_n=100); + t_train_evo = @elapsed m_evo = fit_evotree(params_evo; x_train, y_train, x_eval=x_train, y_eval=y_train, metric=metric_evo, device, verbosity, print_every_n=100); + + @info "predict" + @time pred_evo = m_evo(x_train; device) + t_infer_evo = @elapsed pred_evo = m_evo(x_train; device) + + _df = DataFrame( + :device => device, + :nobs => nobs, + :nfeats => nfeats, + :max_depth => max_depth, + :train_evo => t_train_evo, + :train_xgb => t_train_xgb, + :infer_evo => t_infer_evo, + :infer_xgb => t_infer_xgb) + append!(df, _df) + end + end + end +end + +path = joinpath(@__DIR__, "regressor-$device.csv") +CSV.write(path, df)