Skip to content

Commit

Permalink
'max_bin' != 'max_bins' 🤦
Browse files Browse the repository at this point in the history
  • Loading branch information
@rishic3
rishic3 committed Dec 11, 2024
1 parent f71b7e1 commit d21d2ff
Show file tree
Hide file tree
Showing 2 changed files with 108 additions and 65 deletions.
86 changes: 49 additions & 37 deletions examples/ML+DL-Examples/Optuna-Spark/optuna-examples/optuna-dataframe.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
Expand Down Expand Up @@ -89,9 +89,17 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 4,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Failed to download the file. Status code: 502\n"
]
}
],
"source": [
"url = \"https://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-red.csv\"\n",
"\n",
Expand Down Expand Up @@ -314,7 +322,7 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -327,12 +335,11 @@
" \"subsample\": trial.suggest_float(\"subsample\", 0.05, 1.0),\n",
" \"colsample_bytree\": trial.suggest_float(\"colsample_bytree\", 0.05, 1.0),\n",
" \"min_child_weight\": trial.suggest_int(\"min_child_weight\", 1, 20),\n",
" \"max_bins\": 64,\n",
" \"tree_method\": \"gpu_hist\",\n",
" \"device\": \"cuda\",\n",
" }\n",
"\n",
" booster = xgb.train(params, dtrain=Xy_train_qdm, num_boost_round=1000)\n",
" booster = xgb.train(params, dtrain=Xy_train_qdm, num_boost_round=100)\n",
" predictions = booster.inplace_predict(X_val)\n",
" rmse = mean_squared_error(y_val, predictions, squared=False).get()\n",
" \n",
Expand All @@ -355,17 +362,23 @@
"name": "stderr",
"output_type": "stream",
"text": [
"[I 2024-12-11 00:26:39,598] A new study created in memory with name: optuna-xgboost-local\n",
"[I 2024-12-11 00:26:41,504] Trial 0 finished with value: 0.5448661551754318 and parameters: {'learning_rate': 0.005611516415334507, 'max_depth': 10, 'subsample': 0.7453942447208348, 'colsample_bytree': 0.6187255599871848, 'min_child_weight': 4}. Best is trial 0 with value: 0.5448661551754318.\n",
"[I 2024-12-11 00:26:41,750] Trial 1 finished with value: 0.6762149201768457 and parameters: {'learning_rate': 0.002051110418843397, 'max_depth': 1, 'subsample': 0.8728673384861885, 'colsample_bytree': 0.6210592611560484, 'min_child_weight': 15}. Best is trial 0 with value: 0.5448661551754318.\n",
"[I 2024-12-11 00:26:43,524] Trial 2 finished with value: 0.6871436852105118 and parameters: {'learning_rate': 0.0010994335574766201, 'max_depth': 10, 'subsample': 0.8408205087604007, 'colsample_bytree': 0.25172215514436236, 'min_child_weight': 4}. Best is trial 0 with value: 0.5448661551754318.\n",
"[I 2024-12-11 00:26:44,126] Trial 3 finished with value: 0.6096809835807359 and parameters: {'learning_rate': 0.002327067708383781, 'max_depth': 4, 'subsample': 0.548518610050626, 'colsample_bytree': 0.46034776771000996, 'min_child_weight': 6}. Best is trial 0 with value: 0.5448661551754318.\n",
"[I 2024-12-11 00:26:44,478] Trial 4 finished with value: 0.6060284135590066 and parameters: {'learning_rate': 0.01673808578875214, 'max_depth': 2, 'subsample': 0.3275374161084572, 'colsample_bytree': 0.3980437511290071, 'min_child_weight': 10}. Best is trial 0 with value: 0.5448661551754318.\n",
"[I 2024-12-11 00:26:44,833] Trial 5 finished with value: 0.6041777615275192 and parameters: {'learning_rate': 0.037183641805732096, 'max_depth': 2, 'subsample': 0.538522716492931, 'colsample_bytree': 0.6127938404189404, 'min_child_weight': 1}. Best is trial 0 with value: 0.5448661551754318.\n",
"[I 2024-12-11 00:26:45,192] Trial 6 finished with value: 0.6012987372324395 and parameters: {'learning_rate': 0.016409286730647923, 'max_depth': 2, 'subsample': 0.11179901333601554, 'colsample_bytree': 0.9514412603906666, 'min_child_weight': 20}. Best is trial 0 with value: 0.5448661551754318.\n",
"[I 2024-12-11 00:26:45,767] Trial 7 finished with value: 0.5881042953989125 and parameters: {'learning_rate': 0.041380401125610165, 'max_depth': 4, 'subsample': 0.14278850830606465, 'colsample_bytree': 0.700021375186549, 'min_child_weight': 9}. Best is trial 0 with value: 0.5448661551754318.\n",
"[I 2024-12-11 00:26:46,466] Trial 8 finished with value: 0.6243828841244896 and parameters: {'learning_rate': 0.0017541893487450805, 'max_depth': 5, 'subsample': 0.08266909505945748, 'colsample_bytree': 0.9138543819748429, 'min_child_weight': 6}. Best is trial 0 with value: 0.5448661551754318.\n",
"[I 2024-12-11 00:26:47,058] Trial 9 finished with value: 0.5769714938263446 and parameters: {'learning_rate': 0.02113705944064573, 'max_depth': 4, 'subsample': 0.5440646201189203, 'colsample_bytree': 0.5693747653761156, 'min_child_weight': 4}. Best is trial 0 with value: 0.5448661551754318.\n"
"[I 2024-12-11 19:26:00,951] A new study created in memory with name: optuna-xgboost-local\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"[I 2024-12-11 19:26:02,816] Trial 0 finished with value: 0.5448661551754318 and parameters: {'learning_rate': 0.005611516415334507, 'max_depth': 10, 'subsample': 0.7453942447208348, 'colsample_bytree': 0.6187255599871848, 'min_child_weight': 4}. Best is trial 0 with value: 0.5448661551754318.\n",
"[I 2024-12-11 19:26:03,054] Trial 1 finished with value: 0.6762149201768457 and parameters: {'learning_rate': 0.002051110418843397, 'max_depth': 1, 'subsample': 0.8728673384861885, 'colsample_bytree': 0.6210592611560484, 'min_child_weight': 15}. Best is trial 0 with value: 0.5448661551754318.\n",
"[I 2024-12-11 19:26:04,789] Trial 2 finished with value: 0.6871436852105118 and parameters: {'learning_rate': 0.0010994335574766201, 'max_depth': 10, 'subsample': 0.8408205087604007, 'colsample_bytree': 0.25172215514436236, 'min_child_weight': 4}. Best is trial 0 with value: 0.5448661551754318.\n",
"[I 2024-12-11 19:26:05,377] Trial 3 finished with value: 0.6096809835807359 and parameters: {'learning_rate': 0.002327067708383781, 'max_depth': 4, 'subsample': 0.548518610050626, 'colsample_bytree': 0.46034776771000996, 'min_child_weight': 6}. Best is trial 0 with value: 0.5448661551754318.\n",
"[I 2024-12-11 19:26:05,720] Trial 4 finished with value: 0.6060284135590066 and parameters: {'learning_rate': 0.01673808578875214, 'max_depth': 2, 'subsample': 0.3275374161084572, 'colsample_bytree': 0.3980437511290071, 'min_child_weight': 10}. Best is trial 0 with value: 0.5448661551754318.\n",
"[I 2024-12-11 19:26:06,065] Trial 5 finished with value: 0.6041777615275192 and parameters: {'learning_rate': 0.037183641805732096, 'max_depth': 2, 'subsample': 0.538522716492931, 'colsample_bytree': 0.6127938404189404, 'min_child_weight': 1}. Best is trial 0 with value: 0.5448661551754318.\n",
"[I 2024-12-11 19:26:06,413] Trial 6 finished with value: 0.6012987372324395 and parameters: {'learning_rate': 0.016409286730647923, 'max_depth': 2, 'subsample': 0.11179901333601554, 'colsample_bytree': 0.9514412603906666, 'min_child_weight': 20}. Best is trial 0 with value: 0.5448661551754318.\n",
"[I 2024-12-11 19:26:06,982] Trial 7 finished with value: 0.5881042953989125 and parameters: {'learning_rate': 0.041380401125610165, 'max_depth': 4, 'subsample': 0.14278850830606465, 'colsample_bytree': 0.700021375186549, 'min_child_weight': 9}. Best is trial 0 with value: 0.5448661551754318.\n",
"[I 2024-12-11 19:26:07,666] Trial 8 finished with value: 0.6243828841244896 and parameters: {'learning_rate': 0.0017541893487450805, 'max_depth': 5, 'subsample': 0.08266909505945748, 'colsample_bytree': 0.9138543819748429, 'min_child_weight': 6}. Best is trial 0 with value: 0.5448661551754318.\n",
"[I 2024-12-11 19:26:08,244] Trial 9 finished with value: 0.5769714938263446 and parameters: {'learning_rate': 0.02113705944064573, 'max_depth': 4, 'subsample': 0.5440646201189203, 'colsample_bytree': 0.5693747653761156, 'min_child_weight': 4}. Best is trial 0 with value: 0.5448661551754318.\n"
]
}
],
Expand Down Expand Up @@ -419,21 +432,20 @@
"name": "stdout",
"output_type": "stream",
"text": [
"Downloading Spark Rapids jar\n",
"File 'rapids-4-spark_2.12-24.10.1.jar' downloaded and saved successfully.\n"
"Plugin file already exists. Skipping download.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"24/12/11 00:26:52 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n",
"24/12/11 00:26:52 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n",
"24/12/11 00:26:53 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n",
"24/12/11 19:26:08 WARN Utils: Your hostname, cb4ae00-lcedt resolves to a loopback address: 127.0.1.1; using 10.110.47.100 instead (on interface eno1)\n",
"24/12/11 19:26:08 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address\n",
"24/12/11 19:26:09 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n",
"Setting default log level to \"WARN\".\n",
"To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n",
"24/12/11 00:26:53 WARN RapidsPluginUtils: RAPIDS Accelerator 24.10.1 using cudf 24.10.0, private revision bd4e99e18e20234ee0c54f95f4b0bfce18a6255e\n",
"24/12/11 00:26:53 WARN RapidsPluginUtils: RAPIDS Accelerator is enabled, to disable GPU support set `spark.rapids.sql.enabled` to false.\n"
"24/12/11 19:26:09 WARN RapidsPluginUtils: RAPIDS Accelerator 24.10.1 using cudf 24.10.0, private revision bd4e99e18e20234ee0c54f95f4b0bfce18a6255e\n",
"24/12/11 19:26:09 WARN RapidsPluginUtils: RAPIDS Accelerator is enabled, to disable GPU support set `spark.rapids.sql.enabled` to false.\n"
]
}
],
Expand Down Expand Up @@ -603,7 +615,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
Expand Down Expand Up @@ -643,7 +655,7 @@
" y = data[\"quality\"].values\n",
" X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)\n",
" \n",
" if \"max_bins\" not in optuna_params.hyperparameters:\n",
" if \"max_bin\" not in optuna_params.hyperparameters:\n",
" Xy_train_qdm = xgb.QuantileDMatrix(X_train, y_train) # Precompute Quantile DMatrix to avoid repeated quantization every trial.\n",
"\n",
" def objective(trial):\n",
Expand All @@ -654,20 +666,21 @@
" \"device\": f\"cuda:{gpu_id}\",\n",
" \"seed\": seed,\n",
" }\n",
" \n",
" tuning_params = optuna_params.suggest_params(trial)\n",
" xgb_params.update(tuning_params)\n",
"\n",
" if \"max_bins\" in tuning_params:\n",
" # If tuning the max_bins param, we must recompute the QDM every trial, since the quantiles change.\n",
" if \"max_bin\" in tuning_params:\n",
" # If tuning the max_bin param, we must recompute the QDM every trial, since the quantiles change.\n",
" if \"n_estimators\" not in xgb_params:\n",
" xgb_params[\"n_estimators\"] = 1000\n",
" xgb_params[\"n_estimators\"] = 100\n",
"\n",
" model = xgb.XGBRegressor(**xgb_params)\n",
" model.fit(X_train, y_train)\n",
" booster = model.get_booster()\n",
" else:\n",
" # Train the model with xgb.train() API using the precomputed QDM.\n",
" num_boost_round = xgb_params.get(\"n_estimators\", 1000)\n",
" num_boost_round = xgb_params.get(\"n_estimators\", 100)\n",
" booster = xgb.train(params=xgb_params, dtrain=Xy_train_qdm, num_boost_round=num_boost_round)\n",
" \n",
" # Perform in-place predictions on GPU using the booster.\n",
Expand Down Expand Up @@ -755,13 +768,13 @@
"name": "stderr",
"output_type": "stream",
"text": [
"[I 2024-12-11 00:26:54,506] A new study created in RDB with name: optuna-xgboost-dataframe\n"
"[I 2024-12-11 19:26:10,386] A new study created in RDB with name: optuna-xgboost-dataframe\n"
]
},
{
"data": {
"text/plain": [
"<optuna.study.study.Study at 0x7b60e1943610>"
"<optuna.study.study.Study at 0x727355a65b40>"
]
},
"execution_count": 16,
Expand Down Expand Up @@ -816,7 +829,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 18,
"metadata": {},
"outputs": [
{
Expand All @@ -843,7 +856,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 19,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -853,7 +866,6 @@
"hyperparams.add_float_param(\"subsample\", low=0.05, high=1.0)\n",
"hyperparams.add_float_param(\"colsample_bytree\", low=0.05, high=1.0)\n",
"hyperparams.add_int_param(\"min_child_weight\", low=1, high=20)\n",
"hyperparams.add_int_param(\"max_bins\", low=32, high=256)\n",
"\n",
"out_schema = hyperparams.to_schema()"
]
Expand Down Expand Up @@ -976,8 +988,8 @@
"name": "stdout",
"output_type": "stream",
"text": [
"Best parameters: {'learning_rate': 0.09809986282255367, 'max_depth': 9.0, 'subsample': 0.7124410299808877, 'colsample_bytree': 0.5705492261881397, 'min_child_weight': 9.0, 'max_bins': 77.0}\n",
"Best value: 0.5388542301487141\n"
"Best parameters: {'learning_rate': 0.05428828893707576, 'max_depth': 9.0, 'subsample': 0.8592746224685398, 'colsample_bytree': 0.824735648285962, 'min_child_weight': 3.0}\n",
"Best value: 0.5389662409927146\n"
]
}
],
Expand Down
Loading

0 comments on commit d21d2ff

Please sign in to comment.