From b1034bfc9f6447905fb3eb61d3e60ef00095cf83 Mon Sep 17 00:00:00 2001 From: chuanlihao Date: Fri, 27 Sep 2019 14:37:57 +0800 Subject: [PATCH] Update docs for the new release (#50) - CUDF 0.9.1 - XGBoost4J 1.0.0-Beta2 --- README.md | 3 - examples/apps/scala/pom.xml | 4 +- .../spark/examples/agaricus/CPUMain.scala | 22 ++++--- .../spark/examples/agaricus/GPUMain.scala | 40 +++++------- .../spark/examples/mortgage/CPUMain.scala | 18 ++++-- .../spark/examples/mortgage/GPUMain.scala | 36 +++++------ .../spark/examples/mortgage/Mortgage.scala | 1 + .../rapids/spark/examples/taxi/CPUMain.scala | 17 +++--- .../rapids/spark/examples/taxi/GPUMain.scala | 35 +++++------ .../ai/rapids/spark/examples/taxi/Taxi.scala | 1 + .../spark/examples/utility/Benchmark.scala | 61 +++++++++++-------- examples/notebooks/scala/agaricus-gpu.ipynb | 6 +- .../scala/mortgage-gpu-databricks.scala | 6 +- examples/notebooks/scala/mortgage-gpu.ipynb | 6 +- examples/notebooks/scala/taxi-gpu.ipynb | 6 +- .../building-sample-apps/scala.md | 9 ++- getting-started-guides/csp/aws/sagemaker.md | 6 +- .../csp/databricks/databricks.md | 6 +- ...rk-xgboost-on-databricks-gpu-5.3-5.4.ipynb | 14 ++--- ...-spark-xgboost-on-databricks-gpu-5.5.ipynb | 14 ++--- getting-started-guides/csp/gcp/gcp.md | 4 +- .../notebook/python-notebook.md | 6 +- .../on-premises-cluster/kubernetes.md | 6 +- .../on-premises-cluster/standalone-python.md | 26 ++++---- .../on-premises-cluster/standalone-scala.md | 4 +- .../on-premises-cluster/yarn-python.md | 32 +++++----- .../on-premises-cluster/yarn-scala.md | 4 +- 27 files changed, 202 insertions(+), 191 deletions(-) diff --git a/README.md b/README.md index a4c20896cfa5..1d959c602488 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,5 @@ This repo provides docs and example applications that demonstrate the RAPIDS.ai GPU-accelerated XGBoost-Spark project. -*Please note: Spark 2.4.4 support is currently in development. Please run the examples with other Spark 2.4 versions, or Spark 2.3.* - ### Examples - Mortgage: [Scala](/examples/apps/scala/src/main/scala/ai/rapids/spark/examples/mortgage), [Python](/examples/apps/python/ai/rapids/spark/examples/mortgage) @@ -28,7 +26,6 @@ You can get a small size datasets for each example in the [datasets](/datasets) - [EMR](/getting-started-guides/csp/aws/emr.md) - [SageMaker](/getting-started-guides/csp/aws/sagemaker.md) - [Databricks](/getting-started-guides/csp/databricks/databricks.md) - - [Azure Databricks](/getting-started-guides/csp/azure-databricks/azure-databricks.md) - [Google Cloud Platform](/getting-started-guides/csp/gcp/gcp.md) - Getting started for Jupyter Notebook applications - [Apache Toree Notebook for Scala](/getting-started-guides/notebook/toree.md) diff --git a/examples/apps/scala/pom.xml b/examples/apps/scala/pom.xml index 52d6bb7b1a00..bb1ab7b0f86c 100644 --- a/examples/apps/scala/pom.xml +++ b/examples/apps/scala/pom.xml @@ -13,8 +13,8 @@ UTF-8 - 1.0.0-Beta - 0.9 + 1.0.0-Beta2 + 0.9.1 2.4.0 2.11.6 2.11 diff --git a/examples/apps/scala/src/main/scala/ai/rapids/spark/examples/agaricus/CPUMain.scala b/examples/apps/scala/src/main/scala/ai/rapids/spark/examples/agaricus/CPUMain.scala index 01e955310d14..18056676d9d4 100644 --- a/examples/apps/scala/src/main/scala/ai/rapids/spark/examples/agaricus/CPUMain.scala +++ b/examples/apps/scala/src/main/scala/ai/rapids/spark/examples/agaricus/CPUMain.scala @@ -32,14 +32,14 @@ object CPUMain { def schema(length: Int): StructType = StructType(featureNames(length).map(n => StructField(n, FloatType))) - val xgboostArgs = XGBoostArgs.parse(args) val dataSchema = schema(126) + val xgboostArgs = XGBoostArgs.parse(args) + val processor = this.getClass.getSimpleName.stripSuffix("$").substring(0, 3) + val appInfo = Seq("Agaricus", processor, xgboostArgs.format) // build spark session - val objName = this.getClass.getSimpleName.stripSuffix("$") - val spark = SparkSetup(args, "AgaricusAppFor$objName") - spark.sparkContext.setLogLevel("WARN") - + val spark = SparkSetup(args, appInfo.mkString("-")) + val benchmark = Benchmark(appInfo(0), appInfo(1), appInfo(2)) // === diff === // build data reader val dataReader = spark.read @@ -74,7 +74,7 @@ object CPUMain { .setFeaturesCol("features") println("\n------ Training ------") - val (model, _) = Benchmark.time(s"Agaricus CPU train ${xgboostArgs.format}") { + val (model, _) = benchmark.time("train") { xgbClassifier.fit(datasets(0).get) } // Save model if modelPath exists @@ -88,7 +88,7 @@ object CPUMain { if (xgboostArgs.isToTransform) { // start transform println("\n------ Transforming ------") - var (results, _) = Benchmark.time(s"Agaricus CPU transform ${xgboostArgs.format}") { + var (results, _) = benchmark.time("transform") { val ret = xgbClassificationModel.transform(datasets(2).get).cache() ret.foreachPartition(_ => ()) ret @@ -102,9 +102,11 @@ object CPUMain { println("\n------Accuracy of Evaluation------") val evaluator = new MulticlassClassificationEvaluator().setLabelCol(labelName) - val accuracy = evaluator.evaluate(results) - - println(s"accuracy == $accuracy") + evaluator.evaluate(results) match { + case accuracy if !accuracy.isNaN => + benchmark.value(accuracy, "Accuracy", "Accuracy for") + // Throw an exception when NaN ? + } } spark.close() diff --git a/examples/apps/scala/src/main/scala/ai/rapids/spark/examples/agaricus/GPUMain.scala b/examples/apps/scala/src/main/scala/ai/rapids/spark/examples/agaricus/GPUMain.scala index 96a913f01fba..3111e0734968 100644 --- a/examples/apps/scala/src/main/scala/ai/rapids/spark/examples/agaricus/GPUMain.scala +++ b/examples/apps/scala/src/main/scala/ai/rapids/spark/examples/agaricus/GPUMain.scala @@ -32,36 +32,26 @@ object GPUMain { def schema(length: Int): StructType = StructType(featureNames(length).map(n => StructField(n, FloatType))) - val xgboostArgs = XGBoostArgs.parse(args) val dataSchema = schema(126) + val xgboostArgs = XGBoostArgs.parse(args) + val processor = this.getClass.getSimpleName.stripSuffix("$").substring(0, 3) + val appInfo = Seq("Agaricus", processor, xgboostArgs.format) // build spark session - val objName = this.getClass.getSimpleName.stripSuffix("$") - val spark = SparkSetup(args, "AgaricusAppFor$objName") - spark.sparkContext.setLogLevel("WARN") - + val spark = SparkSetup(args, appInfo.mkString("-")) + val benchmark = Benchmark(appInfo(0), appInfo(1), appInfo(2)) // === diff === // build data reader val dataReader = new GpuDataReader(spark) + .option("asFloats", xgboostArgs.asFloats).option("maxRowsPerChunk", xgboostArgs.maxRowsPerChunk) // load datasets, the order is (train, train-eval, eval) var datasets = xgboostArgs.dataPaths.map(_.map{ path => xgboostArgs.format match { - case "csv" => dataReader - .option("header", xgboostArgs.hasHeader) - .option("asFloats", xgboostArgs.asFloats) - .option("maxRowsPerChunk", xgboostArgs.maxRowsPerChunk) - .schema(dataSchema) - .csv(path) - case "parquet" => dataReader - .option("asFloats", xgboostArgs.asFloats) - .option("maxRowsPerChunk", xgboostArgs.maxRowsPerChunk) - .parquet(path) - case "orc" => dataReader - .option("asFloats", xgboostArgs.asFloats) - .option("maxRowsPerChunk", xgboostArgs.maxRowsPerChunk) - .orc(path) + case "csv" => dataReader.option("header", xgboostArgs.hasHeader).schema(dataSchema).csv(path) + case "parquet" => dataReader.parquet(path) + case "orc" => dataReader.orc(path) case _ => throw new IllegalArgumentException("Unsupported data file format!") } }) @@ -85,7 +75,7 @@ object GPUMain { .setFeaturesCols(featureCols) println("\n------ Training ------") - val (model, _) = Benchmark.time(s"Agaricus GPU train ${xgboostArgs.format}") { + val (model, _) = benchmark.time("train") { xgbClassifier.fit(datasets(0).get) } // Save model if modelPath exists @@ -99,7 +89,7 @@ object GPUMain { if (xgboostArgs.isToTransform) { // start transform println("\n------ Transforming ------") - var (results, _) = Benchmark.time(s"Agaricus GPU transform ${xgboostArgs.format}") { + var (results, _) = benchmark.time("transform") { val ret = xgbClassificationModel.transform(datasets(2).get).cache() ret.foreachPartition(_ => ()) ret @@ -113,9 +103,11 @@ object GPUMain { println("\n------Accuracy of Evaluation------") val evaluator = new MulticlassClassificationEvaluator().setLabelCol(labelName) - val accuracy = evaluator.evaluate(results) - - println(s"accuracy == $accuracy") + evaluator.evaluate(results) match { + case accuracy if !accuracy.isNaN => + benchmark.value(accuracy, "Accuracy", "Accuracy for") + // Throw an exception when NaN ? + } } spark.close() diff --git a/examples/apps/scala/src/main/scala/ai/rapids/spark/examples/mortgage/CPUMain.scala b/examples/apps/scala/src/main/scala/ai/rapids/spark/examples/mortgage/CPUMain.scala index 3151bf2102cf..410cf4c10cc4 100644 --- a/examples/apps/scala/src/main/scala/ai/rapids/spark/examples/mortgage/CPUMain.scala +++ b/examples/apps/scala/src/main/scala/ai/rapids/spark/examples/mortgage/CPUMain.scala @@ -26,13 +26,15 @@ object CPUMain extends Mortgage { def main(args: Array[String]): Unit = { val xgboostArgs = XGBoostArgs.parse(args) + val processor = this.getClass.getSimpleName.stripSuffix("$").substring(0, 3) + val appInfo = Seq(appName, processor, xgboostArgs.format) // build spark session - val objName = this.getClass.getSimpleName.stripSuffix("$") val spark = SparkSession.builder() - .appName(s"Mortgage-$objName-${xgboostArgs.format}") + .appName(appInfo.mkString("-")) .getOrCreate() + val benchmark = Benchmark(appInfo(0), appInfo(1), appInfo(2)) // === diff === // build data reader val dataReader = spark.read @@ -66,7 +68,8 @@ object CPUMain extends Mortgage { // Start training println("\n------ Training ------") - val (model, _) = Benchmark.time(s"Mortgage CPU train ${xgboostArgs.format}") { + // Shall we not log the time if it is abnormal, which is usually caused by training failure + val (model, _) = benchmark.time("train") { xgbClassifier.fit(datasets(0).get) } // Save model if modelPath exists @@ -79,7 +82,7 @@ object CPUMain extends Mortgage { if (xgboostArgs.isToTransform) { println("\n------ Transforming ------") - var (results, _) = Benchmark.time(s"Mortgage CPU transform ${xgboostArgs.format}") { + var (results, _) = benchmark.time("transform") { val ret = xgbClassificationModel.transform(datasets(2).get).cache() // Trigger the transformation ret.foreachPartition(_ => ()) @@ -94,8 +97,11 @@ object CPUMain extends Mortgage { println("\n------Accuracy of Evaluation------") val evaluator = new MulticlassClassificationEvaluator().setLabelCol(labelColName) - val accuracy = evaluator.evaluate(results) - println("Accuracy: " + accuracy) + evaluator.evaluate(results) match { + case accuracy if !accuracy.isNaN => + benchmark.value(accuracy, "Accuracy", "Accuracy for") + // Throw an exception when NaN ? + } } spark.close() diff --git a/examples/apps/scala/src/main/scala/ai/rapids/spark/examples/mortgage/GPUMain.scala b/examples/apps/scala/src/main/scala/ai/rapids/spark/examples/mortgage/GPUMain.scala index a7a6c06a9a2b..df4cd731bbe8 100644 --- a/examples/apps/scala/src/main/scala/ai/rapids/spark/examples/mortgage/GPUMain.scala +++ b/examples/apps/scala/src/main/scala/ai/rapids/spark/examples/mortgage/GPUMain.scala @@ -26,35 +26,27 @@ object GPUMain extends Mortgage { def main(args: Array[String]): Unit = { val xgboostArgs = XGBoostArgs.parse(args) + val processor = this.getClass.getSimpleName.stripSuffix("$").substring(0, 3) + val appInfo = Seq(appName, processor, xgboostArgs.format) // build spark session - val objName = this.getClass.getSimpleName.stripSuffix("$") val spark = SparkSession.builder() - .appName(s"Mortgage-$objName-${xgboostArgs.format}") + .appName(appInfo.mkString("-")) .getOrCreate() + val benchmark = Benchmark(appInfo(0), appInfo(1), appInfo(2)) // === diff === // build data reader val dataReader = new GpuDataReader(spark) + .option("asFloats", xgboostArgs.asFloats).option("maxRowsPerChunk", xgboostArgs.maxRowsPerChunk) // load datasets, the order is (train, train-eval, eval) var datasets = xgboostArgs.dataPaths.map(_.map{ path => xgboostArgs.format match { - case "csv" => dataReader - .option("header", xgboostArgs.hasHeader) - .option("asFloats", xgboostArgs.asFloats) - .option("maxRowsPerChunk", xgboostArgs.maxRowsPerChunk) - .schema(schema) - .csv(path) - case "parquet" => dataReader - .option("asFloats", xgboostArgs.asFloats) - .option("maxRowsPerChunk", xgboostArgs.maxRowsPerChunk) - .parquet(path) - case "orc" => dataReader - .option("asFloats", xgboostArgs.asFloats) - .option("maxRowsPerChunk", xgboostArgs.maxRowsPerChunk) - .orc(path) + case "csv" => dataReader.option("header", xgboostArgs.hasHeader).schema(schema).csv(path) + case "parquet" => dataReader.parquet(path) + case "orc" => dataReader.orc(path) case _ => throw new IllegalArgumentException("Unsupported data file format!") } }) @@ -77,7 +69,8 @@ object GPUMain extends Mortgage { // Start training println("\n------ Training ------") - val (model, _) = Benchmark.time(s"Mortgage GPU train ${xgboostArgs.format}") { + // Shall we not log the time if it is abnormal, which is usually caused by training failure + val (model, _) = benchmark.time("train") { xgbClassifier.fit(datasets(0).get) } // Save model if modelPath exists @@ -90,7 +83,7 @@ object GPUMain extends Mortgage { if (xgboostArgs.isToTransform) { println("\n------ Transforming ------") - var (results, _) = Benchmark.time(s"Mortgage GPU transform ${xgboostArgs.format}") { + var (results, _) = benchmark.time("transform") { val ret = xgbClassificationModel.transform(datasets(2).get).cache() // Trigger the transformation ret.foreachPartition(_ => ()) @@ -105,8 +98,11 @@ object GPUMain extends Mortgage { println("\n------Accuracy of Evaluation------") val evaluator = new MulticlassClassificationEvaluator().setLabelCol(labelColName) - val accuracy = evaluator.evaluate(results) - println("Accuracy: " + accuracy) + evaluator.evaluate(results) match { + case accuracy if !accuracy.isNaN => + benchmark.value(accuracy, "Accuracy", "Accuracy for") + // Throw an exception when NaN ? + } } spark.close() diff --git a/examples/apps/scala/src/main/scala/ai/rapids/spark/examples/mortgage/Mortgage.scala b/examples/apps/scala/src/main/scala/ai/rapids/spark/examples/mortgage/Mortgage.scala index 356a714e0c0b..ec97666029b1 100644 --- a/examples/apps/scala/src/main/scala/ai/rapids/spark/examples/mortgage/Mortgage.scala +++ b/examples/apps/scala/src/main/scala/ai/rapids/spark/examples/mortgage/Mortgage.scala @@ -18,6 +18,7 @@ package ai.rapids.spark.examples.mortgage import org.apache.spark.sql.types.{FloatType, IntegerType, StructField, StructType} private[mortgage] trait Mortgage { + val appName = "Mortgage" val labelColName = "delinquency_12" val schema = StructType(List( diff --git a/examples/apps/scala/src/main/scala/ai/rapids/spark/examples/taxi/CPUMain.scala b/examples/apps/scala/src/main/scala/ai/rapids/spark/examples/taxi/CPUMain.scala index 9f907580c436..9c1a72b02281 100644 --- a/examples/apps/scala/src/main/scala/ai/rapids/spark/examples/taxi/CPUMain.scala +++ b/examples/apps/scala/src/main/scala/ai/rapids/spark/examples/taxi/CPUMain.scala @@ -26,13 +26,15 @@ object CPUMain extends Taxi { def main(args: Array[String]): Unit = { val xgboostArgs = XGBoostArgs.parse(args) + val processor = this.getClass.getSimpleName.stripSuffix("$").substring(0, 3) + val appInfo = Seq(appName, processor, xgboostArgs.format) // build spark session - val objName = this.getClass.getSimpleName.stripSuffix("$") val spark = SparkSession.builder() - .appName(s"Taxi-$objName-${xgboostArgs.format}") + .appName(appInfo.mkString("-")) .getOrCreate() + val benchmark = Benchmark(appInfo(0), appInfo(1), appInfo(2)) // === diff === // build data reader val dataReader = spark.read @@ -65,7 +67,8 @@ object CPUMain extends Taxi { .setFeaturesCol("features") println("\n------ Training ------") - val (model, _) = Benchmark.time(s"Taxi CPU train ${xgboostArgs.format}") { + // Shall we not log the time if it is abnormal, which is usually caused by training failure + val (model, _) = benchmark.time("train") { xgbRegressor.fit(datasets(0).get) } // Save model if modelPath exists @@ -78,7 +81,7 @@ object CPUMain extends Taxi { if (xgboostArgs.isToTransform) { println("\n------ Transforming ------") - var (prediction, _) = Benchmark.time(s"Taxi CPU transform ${xgboostArgs.format}") { + var (prediction, _) = benchmark.time("transform") { val ret = xgbRegressionModel.transform(datasets(2).get).cache() ret.foreachPartition(_ => ()) ret @@ -92,10 +95,10 @@ object CPUMain extends Taxi { println("\n------Accuracy of Evaluation------") val evaluator = new RegressionEvaluator().setLabelCol(labelColName) - val (rmse, _) = Benchmark.time(s"Taxi CPU evaluation ${xgboostArgs.format}") { - evaluator.evaluate(prediction) + evaluator.evaluate(prediction) match { + case rmse if !rmse.isNaN => benchmark.value(rmse, "RMSE", "RMSE for") + // Throw an exception when NaN ? } - println(s"RMSE == $rmse") } spark.close() diff --git a/examples/apps/scala/src/main/scala/ai/rapids/spark/examples/taxi/GPUMain.scala b/examples/apps/scala/src/main/scala/ai/rapids/spark/examples/taxi/GPUMain.scala index 97f438337d5d..7c3807f43364 100644 --- a/examples/apps/scala/src/main/scala/ai/rapids/spark/examples/taxi/GPUMain.scala +++ b/examples/apps/scala/src/main/scala/ai/rapids/spark/examples/taxi/GPUMain.scala @@ -26,35 +26,27 @@ object GPUMain extends Taxi { def main(args: Array[String]): Unit = { val xgboostArgs = XGBoostArgs.parse(args) + val processor = this.getClass.getSimpleName.stripSuffix("$").substring(0, 3) + val appInfo = Seq(appName, processor, xgboostArgs.format) // build spark session - val objName = this.getClass.getSimpleName.stripSuffix("$") val spark = SparkSession.builder() - .appName(s"Taxi-$objName-${xgboostArgs.format}") + .appName(appInfo.mkString("-")) .getOrCreate() + val benchmark = Benchmark(appInfo(0), appInfo(1), appInfo(2)) // === diff === // build data reader val dataReader = new GpuDataReader(spark) + .option("asFloats", xgboostArgs.asFloats).option("maxRowsPerChunk", xgboostArgs.maxRowsPerChunk) // load datasets, the order is (train, train-eval, eval) var datasets = xgboostArgs.dataPaths.map(_.map{ path => xgboostArgs.format match { - case "csv" => dataReader - .option("header", xgboostArgs.hasHeader) - .option("asFloats", xgboostArgs.asFloats) - .option("maxRowsPerChunk", xgboostArgs.maxRowsPerChunk) - .schema(schema) - .csv(path) - case "parquet" => dataReader - .option("asFloats", xgboostArgs.asFloats) - .option("maxRowsPerChunk", xgboostArgs.maxRowsPerChunk) - .parquet(path) - case "orc" => dataReader - .option("asFloats", xgboostArgs.asFloats) - .option("maxRowsPerChunk", xgboostArgs.maxRowsPerChunk) - .orc(path) + case "csv" => dataReader.option("header", xgboostArgs.hasHeader).schema(schema).csv(path) + case "parquet" => dataReader.parquet(path) + case "orc" => dataReader.orc(path) case _ => throw new IllegalArgumentException("Unsupported data file format!") } }) @@ -76,7 +68,8 @@ object GPUMain extends Taxi { .setFeaturesCols(featureNames) println("\n------ Training ------") - val (model, _) = Benchmark.time(s"Taxi GPU train ${xgboostArgs.format}") { + // Shall we not log the time if it is abnormal, which is usually caused by training failure + val (model, _) = benchmark.time("train") { xgbRegressor.fit(datasets(0).get) } // Save model if modelPath exists @@ -89,7 +82,7 @@ object GPUMain extends Taxi { if (xgboostArgs.isToTransform) { println("\n------ Transforming ------") - var (prediction, _) = Benchmark.time(s"Taxi GPU transform ${xgboostArgs.format}") { + var (prediction, _) = benchmark.time("transform") { val ret = xgbRegressionModel.transform(datasets(2).get).cache() ret.foreachPartition(_ => ()) ret @@ -103,10 +96,10 @@ object GPUMain extends Taxi { println("\n------Accuracy of Evaluation------") val evaluator = new RegressionEvaluator().setLabelCol(labelColName) - val (rmse, _) = Benchmark.time(s"Taxi GPU evaluation ${xgboostArgs.format}") { - evaluator.evaluate(prediction) + evaluator.evaluate(prediction) match { + case rmse if !rmse.isNaN => benchmark.value(rmse, "RMSE", "RMSE for") + // Throw an exception when NaN ? } - println(s"RMSE == $rmse") } spark.close() diff --git a/examples/apps/scala/src/main/scala/ai/rapids/spark/examples/taxi/Taxi.scala b/examples/apps/scala/src/main/scala/ai/rapids/spark/examples/taxi/Taxi.scala index 8e1e8c73bcb7..1e5249cb1dac 100644 --- a/examples/apps/scala/src/main/scala/ai/rapids/spark/examples/taxi/Taxi.scala +++ b/examples/apps/scala/src/main/scala/ai/rapids/spark/examples/taxi/Taxi.scala @@ -18,6 +18,7 @@ package ai.rapids.spark.examples.taxi import org.apache.spark.sql.types.{FloatType, IntegerType, StructField, StructType} private[taxi] trait Taxi { + val appName = "Taxi" val labelColName = "fare_amount" lazy val commParamMap = Map( diff --git a/examples/apps/scala/src/main/scala/ai/rapids/spark/examples/utility/Benchmark.scala b/examples/apps/scala/src/main/scala/ai/rapids/spark/examples/utility/Benchmark.scala index 05699873a81c..1a41e56a9aa0 100644 --- a/examples/apps/scala/src/main/scala/ai/rapids/spark/examples/utility/Benchmark.scala +++ b/examples/apps/scala/src/main/scala/ai/rapids/spark/examples/utility/Benchmark.scala @@ -15,36 +15,49 @@ */ package ai.rapids.spark.examples.utility -import ml.dmlc.xgboost4j.java.EnvironmentDetector -import sys.process._ +import scala.util.Properties -object Benchmark { - def time[R](phase: String)(block: => R): (R, Float) = { +class Benchmark( + appName: String, + processor: String, + dataFormat: String) { - val env = getRunEnvironment() + def time[R](phase: String, silent: (Any, Float) => Boolean = (_,_) => false) + (block: => R): (R, Float) = { val t0 = System.currentTimeMillis val result = block // call-by-name - val t1 = System.currentTimeMillis - println("\n--------------") - println("==> Benchmark: Elapsed time for [" + phase + " " + env + "]: " + ((t1 - t0).toFloat / 1000) + "s") - println("--------------\n") - (result, (t1 - t0).toFloat / 1000) + val elapsedTimeSec = (System.currentTimeMillis - t0).toFloat / 1000 + logging(elapsedTimeSec, phase, "Elapsed time for", "s", silent(result, elapsedTimeSec)) + (result, elapsedTimeSec) } - def getRunEnvironment(): String = { - val cudaVersion = EnvironmentDetector.getCudaVersion().orElse("9.2.0") - val cuda = if (cudaVersion.startsWith("9.")) { - "cuda9" - } else { - "cuda10" - } - try { - val os = "apt -v".! - "benchmark " + cuda + " ubuntu16" - } catch { - case x : java.io.IOException => { - "benchmark " + cuda + " centos7" - } + def value(value: Any, name: String = "value", prefix: String="", suffix: String = "") = { + logging(value, name, prefix, suffix, false) + } + + private def logging(value: Any, name: String , prefix: String, suffix: String, silent: Boolean) = { + if (!silent) { + val logString = buildLogSimple(value, prefix, suffix, buildRuntimeInfo(name)) + println("\n--------------") + println("==> Benchmark: " + logString) + println("--------------\n") } } + + private def buildRuntimeInfo(name: String): String = { + // Get runtime information from Environment + val osType = Properties.envOrElse("RAPIDS_XGB_EXAMPLE_OS_TYPE", "Unknown") + val cudaVersion = Properties.envOrElse("RAPIDS_XGB_EXAMPLE_CUDA_VERSION", "Unknown") + val sparkVersion = Properties.envOrElse("RAPIDS_XGB_EXAMPLE_SPARK_VERSION", "Unknown") + Seq(appName, processor, name, dataFormat, "stub", cudaVersion, osType, sparkVersion) + .mkString(" ") + } + + private def buildLogSimple(value: Any, prefix: String, suffix: String, runtimeInfo: String): String = + prefix + " [" + runtimeInfo + "]: " + value + suffix +} + +object Benchmark { + def apply(appName: String, processor: String, dataFormat: String) = + new Benchmark(appName, processor, dataFormat) } diff --git a/examples/notebooks/scala/agaricus-gpu.ipynb b/examples/notebooks/scala/agaricus-gpu.ipynb index e2932b4dfa2e..5447a6ebb43b 100644 --- a/examples/notebooks/scala/agaricus-gpu.ipynb +++ b/examples/notebooks/scala/agaricus-gpu.ipynb @@ -165,9 +165,9 @@ "```scala\n", "import org.apache.spark.sql.SparkSession\n", "val spark = SparkSession.builder().appName(\"agaricus-GPU\").getOrCreate\n", - "%AddJar file:/data/libs/cudf-0.9-cuda10.jar\n", - "%AddJar file:/data/libs/xgboost4j_2.11-1.0.0-Beta.jar\n", - "%AddJar file:/data/libs/xgboost4j-spark_2.11-1.0.0-Beta.jar\n", + "%AddJar file:/data/libs/cudf-0.9.1-cuda10.jar\n", + "%AddJar file:/data/libs/xgboost4j_2.11-1.0.0-Beta2.jar\n", + "%AddJar file:/data/libs/xgboost4j-spark_2.11-1.0.0-Beta2.jar\n", "// ...\n", "```" ] diff --git a/examples/notebooks/scala/mortgage-gpu-databricks.scala b/examples/notebooks/scala/mortgage-gpu-databricks.scala index 7d628b7dddd3..43421b037060 100644 --- a/examples/notebooks/scala/mortgage-gpu-databricks.scala +++ b/examples/notebooks/scala/mortgage-gpu-databricks.scala @@ -112,9 +112,9 @@ val commParamMap = Map( // MAGIC ```scala // MAGIC import org.apache.spark.sql.SparkSession // MAGIC val spark = SparkSession.builder().appName("mortgage-GPU").getOrCreate -// MAGIC %AddJar file:/data/libs/cudf-0.9-cuda10.jar -// MAGIC %AddJar file:/data/libs/xgboost4j_2.11-1.0.0-Beta_on_Rapids.jar -// MAGIC %AddJar file:/data/libs/xgboost4j-spark_2.11-1.0.0-Beta_on_Rapids.jar +// MAGIC %AddJar file:/data/libs/cudf-0.9.1-cuda10.jar +// MAGIC %AddJar file:/data/libs/xgboost4j_2.11-1.0.0-Beta2_on_Rapids.jar +// MAGIC %AddJar file:/data/libs/xgboost4j-spark_2.11-1.0.0-Beta2_on_Rapids.jar // MAGIC // ... // MAGIC ``` diff --git a/examples/notebooks/scala/mortgage-gpu.ipynb b/examples/notebooks/scala/mortgage-gpu.ipynb index 2be32827ecaf..1643c1b70bd7 100644 --- a/examples/notebooks/scala/mortgage-gpu.ipynb +++ b/examples/notebooks/scala/mortgage-gpu.ipynb @@ -196,9 +196,9 @@ "```scala\n", "import org.apache.spark.sql.SparkSession\n", "val spark = SparkSession.builder().appName(\"mortgage-GPU\").getOrCreate\n", - "%AddJar file:/data/libs/cudf-0.9-cuda10.jar\n", - "%AddJar file:/data/libs/xgboost4j_2.11-1.0.0-Beta.jar\n", - "%AddJar file:/data/libs/xgboost4j-spark_2.11-1.0.0-Beta.jar\n", + "%AddJar file:/data/libs/cudf-0.9.1-cuda10.jar\n", + "%AddJar file:/data/libs/xgboost4j_2.11-1.0.0-Beta2.jar\n", + "%AddJar file:/data/libs/xgboost4j-spark_2.11-1.0.0-Beta2.jar\n", "// ...\n", "```" ] diff --git a/examples/notebooks/scala/taxi-gpu.ipynb b/examples/notebooks/scala/taxi-gpu.ipynb index 43f653113808..471086c5b97d 100644 --- a/examples/notebooks/scala/taxi-gpu.ipynb +++ b/examples/notebooks/scala/taxi-gpu.ipynb @@ -176,9 +176,9 @@ "```scala\n", "import org.apache.spark.sql.SparkSession\n", "val spark = SparkSession.builder().appName(\"Taxi-GPU\").getOrCreate\n", - "%AddJar file:/data/libs/cudf-0.9-cuda10.jar\n", - "%AddJar file:/data/libs/xgboost4j_2.11-1.0.0-Beta.jar\n", - "%AddJar file:/data/libs/xgboost4j-spark_2.11-1.0.0-Beta.jar\n", + "%AddJar file:/data/libs/cudf-0.9.1-cuda10.jar\n", + "%AddJar file:/data/libs/xgboost4j_2.11-1.0.0-Beta2.jar\n", + "%AddJar file:/data/libs/xgboost4j-spark_2.11-1.0.0-Beta2.jar\n", "// ...\n", "```" ] diff --git a/getting-started-guides/building-sample-apps/scala.md b/getting-started-guides/building-sample-apps/scala.md index 74515b5ef58a..3886f791a489 100644 --- a/getting-started-guides/building-sample-apps/scala.md +++ b/getting-started-guides/building-sample-apps/scala.md @@ -4,12 +4,12 @@ Our examples rely on [cuDF](https://github.com/rapidsai/cudf) and [XGBoost](http ##### Build Process -Follow these steps to build the Scala jars: +Follow these steps to build the Scala jars (Here take CUDA 10.0 as an example): ``` git clone https://github.com/rapidsai/spark-examples.git cd spark-examples/examples/apps/scala -mvn package -Dcuda.classifier=cuda10 # omit cuda.classifier for cuda 9.2 +mvn package -Dcuda.classifier=cuda10 ``` ##### Generated Jars @@ -23,4 +23,7 @@ The build process generates two jars: Classifiers: -+ *cuda.classifier*: omit this classifier for CUDA 9.2 building, and set *cuda10* for CUDA 10.0 building ++ *cuda.classifier* + + For CUDA 9.2 building, omit this classifier + + For CUDA 10.0 building, specify *cuda10* + + For CUDA 10.1 building, specify *cuda10-1* diff --git a/getting-started-guides/csp/aws/sagemaker.md b/getting-started-guides/csp/aws/sagemaker.md index e349812de7b9..d51154dd46d3 100644 --- a/getting-started-guides/csp/aws/sagemaker.md +++ b/getting-started-guides/csp/aws/sagemaker.md @@ -114,9 +114,9 @@ Now Launch the GPU Mortgage Example: "driverCores": 2, "executorMemory": "16000M", "conf" : {"spark.executor.instances":2, "spark.executor.cores":8, "spark.task.cpus": 8, "spark.yarn.maxAppAttempts": 1, "spark.dynamicAllocation.enabled": false}, - "jars" : ["https://repo1.maven.org/maven2/ai/rapids/cudf/0.9/cudf-0.9.jar", - "https://repo1.maven.org/maven2/ai/rapids/xgboost4j-spark_2.11/1.0.0-Beta/xgboost4j-spark_2.11-1.0.0-Beta.jar", - "https://repo1.maven.org/maven2/ai/rapids/xgboost4j_2.11/1.0.0-Beta/xgboost4j_2.11-1.0.0-Beta.jar"] + "jars" : ["https://repo1.maven.org/maven2/ai/rapids/cudf/0.9.1/cudf-0.9.1.jar", + "https://repo1.maven.org/maven2/ai/rapids/xgboost4j-spark_2.11/1.0.0-Beta2/xgboost4j-spark_2.11-1.0.0-Beta2.jar", + "https://repo1.maven.org/maven2/ai/rapids/xgboost4j_2.11/1.0.0-Beta2/xgboost4j_2.11-1.0.0-Beta2.jar"] } sc.listJars.foreach(println) diff --git a/getting-started-guides/csp/databricks/databricks.md b/getting-started-guides/csp/databricks/databricks.md index 8c8fb79489c2..835b306fcf81 100644 --- a/getting-started-guides/csp/databricks/databricks.md +++ b/getting-started-guides/csp/databricks/databricks.md @@ -10,8 +10,8 @@ Prerequisites * Multi-node clusters with homogenous GPU configuration * Software Requirements * Ubuntu 16.04/CentOS - * NVIDIA driver 410.48+ - * CUDA V10.0/9.2 + * CUDA V10.1/10.0/9.2 + * NVIDIA driver compatible with your CUDA * NCCL 2.4.7 The number of GPUs per node dictates the number of Spark executors that can run in that node. Each executor should only be allowed to run 1 task at any given time. @@ -48,7 +48,7 @@ Add cluster initialization script Install the xgboost4j_spark jar in the cluster --------------------------- 1. See [Libraries](https://docs.databricks.com/user-guide/libraries.html) for how to install jars from DBFS -2. Go to "Libraries" tab under your cluster and install dbfs:/FileStore/jars/xgboost4j-spark_2.11-1.0.0-Beta.jar in your cluster by selecting the "DBFS" option for installing jars +2. Go to "Libraries" tab under your cluster and install dbfs:/FileStore/jars/xgboost4j-spark_2.11-1.0.0-Beta2.jar in your cluster by selecting the "DBFS" option for installing jars These steps will ensure you have a GPU Cluster ready for importing XGBoost notebooks or create your own XGBoost Application for training. diff --git a/getting-started-guides/csp/databricks/init-notebook-for-rapids-spark-xgboost-on-databricks-gpu-5.3-5.4.ipynb b/getting-started-guides/csp/databricks/init-notebook-for-rapids-spark-xgboost-on-databricks-gpu-5.3-5.4.ipynb index a2d3509ab371..887531a2c271 100644 --- a/getting-started-guides/csp/databricks/init-notebook-for-rapids-spark-xgboost-on-databricks-gpu-5.3-5.4.ipynb +++ b/getting-started-guides/csp/databricks/init-notebook-for-rapids-spark-xgboost-on-databricks-gpu-5.3-5.4.ipynb @@ -24,9 +24,9 @@ "source": [ "%sh\n", "cd ../../dbfs/FileStore/jars/\n", - "wget -O cudf-0.9.jar https://search.maven.org/remotecontent?filepath=ai/rapids/cudf/0.9/cudf-0.9.jar\n", - "wget -O xgboost4j_2.11-1.0.0-Beta.jar https://search.maven.org/remotecontent?filepath=ai/rapids/xgboost4j_2.11/1.0.0-Beta/xgboost4j_2.11-1.0.0-Beta.jar\n", - "wget -O xgboost4j-spark_2.11-1.0.0-Beta.jar https://search.maven.org/remotecontent?filepath=ai/rapids/xgboost4j-spark_2.11/1.0.0-Beta/xgboost4j-spark_2.11-1.0.0-Beta.jar\n", + "wget -O cudf-0.9.1.jar https://search.maven.org/remotecontent?filepath=ai/rapids/cudf/0.9.1/cudf-0.9.1.jar\n", + "wget -O xgboost4j_2.11-1.0.0-Beta2.jar https://search.maven.org/remotecontent?filepath=ai/rapids/xgboost4j_2.11/1.0.0-Beta2/xgboost4j_2.11-1.0.0-Beta2.jar\n", + "wget -O xgboost4j-spark_2.11-1.0.0-Beta2.jar https://search.maven.org/remotecontent?filepath=ai/rapids/xgboost4j-spark_2.11/1.0.0-Beta2/xgboost4j-spark_2.11-1.0.0-Beta2.jar\n", "ls -ltr\n", "\n", "# Your Jars are downloaded in dbfs:/FileStore/jars directory" @@ -56,9 +56,9 @@ "source": [ "dbutils.fs.put(\"/databricks/init_scripts/init.sh\",\"\"\"\n", "#!/bin/bash\n", - "sudo cp /dbfs/FileStore/jars/xgboost4j_2.11-1.0.0-Beta.jar /databricks/jars/spark--maven-trees--ml--xgboost--ml.dmlc--xgboost4j--ml.dmlc__xgboost4j__0.81.jar\n", - "sudo cp /dbfs/FileStore/jars/cudf-0.9.jar /databricks/jars/\n", - "sudo cp /dbfs/FileStore/jars/xgboost4j-spark_2.11-1.0.0-Beta.jar /databricks/jars/spark--maven-trees--ml--xgboost--ml.dmlc--xgboost4j-spark--ml.dmlc__xgboost4j-spark__0.81.jar\"\"\", True)" + "sudo cp /dbfs/FileStore/jars/xgboost4j_2.11-1.0.0-Beta2.jar /databricks/jars/spark--maven-trees--ml--xgboost--ml.dmlc--xgboost4j--ml.dmlc__xgboost4j__0.81.jar\n", + "sudo cp /dbfs/FileStore/jars/cudf-0.9.1.jar /databricks/jars/\n", + "sudo cp /dbfs/FileStore/jars/xgboost4j-spark_2.11-1.0.0-Beta2.jar /databricks/jars/spark--maven-trees--ml--xgboost--ml.dmlc--xgboost4j-spark--ml.dmlc__xgboost4j-spark__0.81.jar\"\"\", True)" ] }, { @@ -129,7 +129,7 @@ "\n", "1. Edit your cluster, adding an initialization script from `dbfs:/databricks/init_scripts/init.sh` in the \"Advanced Options\" under \"Init Scripts\" tab\n", "2. Reboot the cluster\n", - "3. Go to \"Libraries\" tab under your cluster and install `dbfs:/FileStore/jars/xgboost4j-spark_2.11-1.0.0-Beta.jar` in your cluster by selecting the \"DBFS\" option for installing jars\n", + "3. Go to \"Libraries\" tab under your cluster and install `dbfs:/FileStore/jars/xgboost4j-spark_2.11-1.0.0-Beta2.jar` in your cluster by selecting the \"DBFS\" option for installing jars\n", "4. Import the mortgage example notebook from `https://github.com/rapidsai/spark-examples/blob/master/examples/notebooks/python/mortgage-gpu.ipynb`\n", "5. Inside the mortgage example notebook, update the data paths\n", " `train_data = GpuDataReader(spark).schema(schema).option('header', True).csv('dbfs:/FileStore/tables/mortgage/csv/train/mortgage_train_merged.csv')`\n", diff --git a/getting-started-guides/csp/databricks/init-notebook-for-rapids-spark-xgboost-on-databricks-gpu-5.5.ipynb b/getting-started-guides/csp/databricks/init-notebook-for-rapids-spark-xgboost-on-databricks-gpu-5.5.ipynb index 3e66172ebb3c..b94a8976df36 100644 --- a/getting-started-guides/csp/databricks/init-notebook-for-rapids-spark-xgboost-on-databricks-gpu-5.5.ipynb +++ b/getting-started-guides/csp/databricks/init-notebook-for-rapids-spark-xgboost-on-databricks-gpu-5.5.ipynb @@ -24,9 +24,9 @@ "source": [ "%sh\n", "cd ../../dbfs/FileStore/jars/\n", - "wget -O cudf-0.9-cuda10.jar https://search.maven.org/remotecontent?filepath=ai/rapids/cudf/0.9/cudf-0.9-cuda10.jar\n", - "wget -O xgboost4j_2.11-1.0.0-Beta.jar https://search.maven.org/remotecontent?filepath=ai/rapids/xgboost4j_2.11/1.0.0-Beta/xgboost4j_2.11-1.0.0-Beta.jar\n", - "wget -O xgboost4j-spark_2.11-1.0.0-Beta.jar https://search.maven.org/remotecontent?filepath=ai/rapids/xgboost4j-spark_2.11/1.0.0-Beta/xgboost4j-spark_2.11-1.0.0-Beta.jar\n", + "wget -O cudf-0.9.1-cuda10.jar https://search.maven.org/remotecontent?filepath=ai/rapids/cudf/0.9.1/cudf-0.9.1-cuda10.jar\n", + "wget -O xgboost4j_2.11-1.0.0-Beta2.jar https://search.maven.org/remotecontent?filepath=ai/rapids/xgboost4j_2.11/1.0.0-Beta2/xgboost4j_2.11-1.0.0-Beta2.jar\n", + "wget -O xgboost4j-spark_2.11-1.0.0-Beta2.jar https://search.maven.org/remotecontent?filepath=ai/rapids/xgboost4j-spark_2.11/1.0.0-Beta2/xgboost4j-spark_2.11-1.0.0-Beta2.jar\n", "ls -ltr\n", "\n", "# Your Jars are downloaded in dbfs:/FileStore/jars directory" @@ -56,9 +56,9 @@ "source": [ "dbutils.fs.put(\"/databricks/init_scripts/init.sh\",\"\"\"\n", "#!/bin/bash\n", - "sudo cp /dbfs/FileStore/jars/xgboost4j_2.11-1.0.0-Beta.jar /databricks/jars/spark--maven-trees--ml--xgboost--ml.dmlc--xgboost4j--ml.dmlc__xgboost4j__0.90.jar\n", - "sudo cp /dbfs/FileStore/jars/cudf-0.9-cuda10.jar /databricks/jars/\n", - "sudo cp /dbfs/FileStore/jars/xgboost4j-spark_2.11-1.0.0-Beta.jar /databricks/jars/spark--maven-trees--ml--xgboost--ml.dmlc--xgboost4j-spark--ml.dmlc__xgboost4j-spark__0.90.jar\"\"\", True)" + "sudo cp /dbfs/FileStore/jars/xgboost4j_2.11-1.0.0-Beta2.jar /databricks/jars/spark--maven-trees--ml--xgboost--ml.dmlc--xgboost4j--ml.dmlc__xgboost4j__0.90.jar\n", + "sudo cp /dbfs/FileStore/jars/cudf-0.9.1-cuda10.jar /databricks/jars/\n", + "sudo cp /dbfs/FileStore/jars/xgboost4j-spark_2.11-1.0.0-Beta2.jar /databricks/jars/spark--maven-trees--ml--xgboost--ml.dmlc--xgboost4j-spark--ml.dmlc__xgboost4j-spark__0.90.jar\"\"\", True)" ] }, { @@ -129,7 +129,7 @@ "\n", "1. Edit your cluster, adding an initialization script from `dbfs:/databricks/init_scripts/init.sh` in the \"Advanced Options\" under \"Init Scripts\" tab\n", "2. Reboot the cluster\n", - "3. Go to \"Libraries\" tab under your cluster and install `dbfs:/FileStore/jars/xgboost4j-spark_2.11-1.0.0-Beta.jar` in your cluster by selecting the \"DBFS\" option for installing jars\n", + "3. Go to \"Libraries\" tab under your cluster and install `dbfs:/FileStore/jars/xgboost4j-spark_2.11-1.0.0-Beta2.jar` in your cluster by selecting the \"DBFS\" option for installing jars\n", "4. Import the mortgage example notebook from `https://github.com/rapidsai/spark-examples/blob/master/examples/notebooks/python/mortgage-gpu.ipynb`\n", "5. Inside the mortgage example notebook, update the data paths\n", " `train_data = GpuDataReader(spark).schema(schema).option('header', True).csv('dbfs:/FileStore/tables/mortgage/csv/train/mortgage_train_merged.csv')`\n", diff --git a/getting-started-guides/csp/gcp/gcp.md b/getting-started-guides/csp/gcp/gcp.md index 1630ce306673..23a6c353b8d7 100644 --- a/getting-started-guides/csp/gcp/gcp.md +++ b/getting-started-guides/csp/gcp/gcp.md @@ -49,7 +49,7 @@ Use the following command to submit spark jobs on this GPU cluster. CLUSTER_NAME=raghav-cluster ROOT_FOLDER=gs://rapidsai-test-1/spark MAIN_CLASS=ai.rapids.spark.examples.agaricus.GPUMain -RAPIDS_JARS=$ROOT_FOLDER/jars/sample_xgboost_apps-0.1.4.jar,$ROOT_FOLDER/jars/cudf-0.9-cuda10.jar,$ROOT_FOLDER/jars/xgboost4j_2.11-1.0.0-Beta.jar,$ROOT_FOLDER/jars/xgboost4j-spark_2.11-1.0.0-Beta.jar +RAPIDS_JARS=$ROOT_FOLDER/jars/sample_xgboost_apps-0.1.4.jar,$ROOT_FOLDER/jars/cudf-0.9.1-cuda10.jar,$ROOT_FOLDER/jars/xgboost4j_2.11-1.0.0-Beta2.jar,$ROOT_FOLDER/jars/xgboost4j-spark_2.11-1.0.0-Beta2.jar TRAIN_DATA_PATH=$ROOT_FOLDER/data/agaricus/csv/train TEST_DATA_PATH=$ROOT_FOLDER/data/agaricus/csv/test MODEL_PATH=$ROOT_FOLDER/models @@ -87,7 +87,7 @@ Submitting a CPU job on this cluster is very similar. Below's an example command CLUSTER_NAME=raghav-cluster ROOT_FOLDER=gs://rapidsai-test-1/spark MAIN_CLASS=ai.rapids.spark.examples.agaricus.CPUMain -RAPIDS_JARS=$ROOT_FOLDER/jars/sample_xgboost_apps-0.1.4.jar,$ROOT_FOLDER/jars/cudf-0.9-cuda10.jar,$ROOT_FOLDER/jars/xgboost4j_2.11-1.0.0-Beta.jar,$ROOT_FOLDER/jars/xgboost4j-spark_2.11-1.0.0-Beta.jar +RAPIDS_JARS=$ROOT_FOLDER/jars/sample_xgboost_apps-0.1.4.jar,$ROOT_FOLDER/jars/cudf-0.9.1-cuda10.jar,$ROOT_FOLDER/jars/xgboost4j_2.11-1.0.0-Beta2.jar,$ROOT_FOLDER/jars/xgboost4j-spark_2.11-1.0.0-Beta2.jar TRAIN_DATA_PATH=$ROOT_FOLDER/data/agaricus/csv/train TEST_DATA_PATH=$ROOT_FOLDER/data/agaricus/csv/test MODEL_PATH=$ROOT_FOLDER/models diff --git a/getting-started-guides/notebook/python-notebook.md b/getting-started-guides/notebook/python-notebook.md index c052e5576f66..7fb304a1a1cc 100644 --- a/getting-started-guides/notebook/python-notebook.md +++ b/getting-started-guides/notebook/python-notebook.md @@ -8,11 +8,11 @@ It is assumed that the `SPARK_MASTER` and `SPARK_HOME` environment variables are 1. Make sure you have [Jupyter notebook installed](https://jupyter.org/install.html). If you install it with conda, please makes sure your Python version is consistent. -2. Make sure you have `SPARK_JARS` and `SPARK_PY_FILES` set properly. You may need to update these env variables because the working directory will be changed: +2. Make sure you have `SPARK_JARS` and `SPARK_PY_FILES` set properly. Please note, here *cudf-0.9.1-cuda10.jar* is used as an example. Please choose other *cudf-0.9.1* jars based on your environment. You may need to update these env variables because the working directory will be changed: ``` export LIBS_PATH=[full path to xgboost4j_spark/libs] - export SPARK_JARS=${LIBS_PATH}/cudf-0.9-cuda10.jar,${LIBS_PATH}/xgboost4j_2.11-1.0.0-Beta.jar,${LIBS_PATH}/xgboost4j-spark_2.11-1.0.0-Beta.jar - export SPARK_PY_FILES=${LIBS_PATH}/xgboost4j-spark_2.11-1.0.0-Beta.jar,${LIBS_PATH}/samples.zip + export SPARK_JARS=${LIBS_PATH}/cudf-0.9.1-cuda10.jar,${LIBS_PATH}/xgboost4j_2.11-1.0.0-Beta2.jar,${LIBS_PATH}/xgboost4j-spark_2.11-1.0.0-Beta2.jar + export SPARK_PY_FILES=${LIBS_PATH}/xgboost4j-spark_2.11-1.0.0-Beta2.jar,${LIBS_PATH}/samples.zip ``` 3. Go to the project root directory and launch the notebook: diff --git a/getting-started-guides/on-premises-cluster/kubernetes.md b/getting-started-guides/on-premises-cluster/kubernetes.md index cca305a1c493..d45fb2711dd7 100644 --- a/getting-started-guides/on-premises-cluster/kubernetes.md +++ b/getting-started-guides/on-premises-cluster/kubernetes.md @@ -11,8 +11,8 @@ Prerequisites * Multi-node clusters with homogenous GPU configuration * Software Requirements * Ubuntu 16.04/CentOS - * NVIDIA driver 410.48+ - * CUDA V10.0/9.2 + * CUDA V10.1/10.0/9.2 + * NVIDIA driver compatible with your CUDA * NCCL 2.4.7 * [Kubernetes 1.6+ cluster with NVIDIA GPUs](https://docs.nvidia.com/datacenter/kubernetes/index.html) * See official [Spark on Kubernetes](https://spark.apache.org/docs/latest/running-on-kubernetes.html#prerequisites) instructions for detailed spark-specific cluster requirements @@ -26,7 +26,7 @@ Build a GPU Docker image with Spark resources in it, this Docker image must be a 1. Locate your Spark installations. If you don't have one, you can [download](https://spark.apache.org/downloads.html) from Apache and unzip it. 2. `export SPARK_HOME=` -3. [Download the Dockerfile](https://github.com/rapidsai/spark-examples/Dockerfile) into `${SPARK_HOME}` +3. [Download the Dockerfile](https://github.com/rapidsai/spark-examples/Dockerfile) into `${SPARK_HOME}` (Here CUDA 10.0 is used as an example in the Dockerfile, you may need to update it for other CUDA versions.) 4. __(OPTIONAL)__ install any additional library jars into the `${SPARK_HOME}/jars` directory * Most public cloud file systems are not natively supported -- pulling data and jar files from S3, GCS, etc. require installing additional libraries 5. Build and push the docker image diff --git a/getting-started-guides/on-premises-cluster/standalone-python.md b/getting-started-guides/on-premises-cluster/standalone-python.md index 2e90bc011e59..d08c5325dbb7 100644 --- a/getting-started-guides/on-premises-cluster/standalone-python.md +++ b/getting-started-guides/on-premises-cluster/standalone-python.md @@ -10,8 +10,8 @@ Prerequisites * Multi-node clusters with homogenous GPU configuration * Software Requirements * Ubuntu 16.04/CentOS - * NVIDIA driver 410.48+ - * CUDA V10.0/9.2 + * CUDA V10.1/10.0/9.2 + * NVIDIA driver compatible with your CUDA * NCCL 2.4.7 * Python 2.7/3.4/3.5/3.6/3.7 * NumPy @@ -35,9 +35,11 @@ Get Application Files and Dataset ------------------------------- 1. *samples.zip* and *main.py*: Please build the files as specified in the [guide](/getting-started-guides/building-sample-apps/python.md) 2. Jars: Please download the following jars: - * [*cudf-0.9-cuda10.jar*](https://search.maven.org/remotecontent?filepath=ai/rapids/cudf/0.9/cudf-0.9-cuda10.jar) (For CUDA 9.2, please download [*cudf-0.9.jar*](https://search.maven.org/remotecontent?filepath=ai/rapids/cudf/0.9/cudf-0.9.jar) instead, and replace *cudf-0.9-cuda10.jar* with *cudf-0.9.jar* throughout this whole guide) - * [*xgboost4j_2.11-1.0.0-Beta.jar*](https://search.maven.org/remotecontent?filepath=ai/rapids/xgboost4j_2.11/1.0.0-Beta/xgboost4j_2.11-1.0.0-Beta.jar) - * [*xgboost4j-spark_2.11-1.0.0-Beta.jar*](https://search.maven.org/remotecontent?filepath=ai/rapids/xgboost4j-spark_2.11/1.0.0-Beta/xgboost4j-spark_2.11-1.0.0-Beta.jar) + * [*cudf-0.9.1-cuda10.jar*](https://search.maven.org/remotecontent?filepath=ai/rapids/cudf/0.9.1/cudf-0.9.1-cuda10.jar) (Here take CUDA 10.0 as an example) + * For CUDA 9.2, please download [*cudf-0.9.1.jar*](https://search.maven.org/remotecontent?filepath=ai/rapids/cudf/0.9.1/cudf-0.9.1.jar) instead, and replace *cudf-0.9.1-cuda10.jar* with *cudf-0.9.1.jar* throughout this whole guide + * For CUDA 10.1, please download [*cudf-0.9.1-cuda10-1.jar*](https://search.maven.org/remotecontent?filepath=ai/rapids/cudf/0.9.1/cudf-0.9.1-cuda10-1.jar) instead, and replace *cudf-0.9.1-cuda10.jar* with *cudf-0.9.1-cuda10-1.jar* throughout this whole guide + * [*xgboost4j_2.11-1.0.0-Beta2.jar*](https://search.maven.org/remotecontent?filepath=ai/rapids/xgboost4j_2.11/1.0.0-Beta2/xgboost4j_2.11-1.0.0-Beta2.jar) + * [*xgboost4j-spark_2.11-1.0.0-Beta2.jar*](https://search.maven.org/remotecontent?filepath=ai/rapids/xgboost4j-spark_2.11/1.0.0-Beta2/xgboost4j-spark_2.11-1.0.0-Beta2.jar) 3. Dataset: https://rapidsai-data.s3.us-east-2.amazonaws.com/spark/mortgage.zip Place dataset and other files in a local directory. In this example the dataset was unzipped in the `xgboost4j_spark/data` directory, and all other files in the `xgboost4j_spark/libs` directory. @@ -46,11 +48,11 @@ Place dataset and other files in a local directory. In this example the dataset [xgboost4j_spark]$ find . -type f | sort ./data/mortgage/csv/test/mortgage_eval_merged.csv ./data/mortgage/csv/train/mortgage_train_merged.csv -./libs/cudf-0.9-cuda10.jar +./libs/cudf-0.9.1-cuda10.jar ./libs/main.py ./libs/samples.zip -./libs/xgboost4j_2.11-1.0.0-Beta.jar -./libs/xgboost4j-spark_2.11-1.0.0-Beta.jar +./libs/xgboost4j_2.11-1.0.0-Beta2.jar +./libs/xgboost4j-spark_2.11-1.0.0-Beta2.jar ``` Launch a Standalone Spark Cluster @@ -114,10 +116,10 @@ export SPARK_PYTHON_ENTRYPOINT=${LIBS_PATH}/main.py export EXAMPLE_CLASS=ai.rapids.spark.examples.mortgage.gpu_main # additional jars for XGBoost4J example -export SPARK_JARS=${LIBS_PATH}/cudf-0.9-cuda10.jar,${LIBS_PATH}/xgboost4j_2.11-1.0.0-Beta.jar,${LIBS_PATH}/xgboost4j-spark_2.11-1.0.0-Beta.jar +export SPARK_JARS=${LIBS_PATH}/cudf-0.9.1-cuda10.jar,${LIBS_PATH}/xgboost4j_2.11-1.0.0-Beta2.jar,${LIBS_PATH}/xgboost4j-spark_2.11-1.0.0-Beta2.jar # additional Python files for XGBoost4J example -export SPARK_PY_FILES=${LIBS_PATH}/xgboost4j-spark_2.11-1.0.0-Beta.jar,${LIBS_PATH}/samples.zip +export SPARK_PY_FILES=${LIBS_PATH}/xgboost4j-spark_2.11-1.0.0-Beta2.jar,${LIBS_PATH}/samples.zip # tree construction algorithm export TREE_METHOD=gpu_hist @@ -207,10 +209,10 @@ export SPARK_PYTHON_ENTRYPOINT=${LIBS_PATH}/main.py export EXAMPLE_CLASS=ai.rapids.spark.examples.mortgage.cpu_main # additional jars for XGBoost4J example -export SPARK_JARS=${LIBS_PATH}/cudf-0.9-cuda10.jar,${LIBS_PATH}/xgboost4j_2.11-1.0.0-Beta.jar,${LIBS_PATH}/xgboost4j-spark_2.11-1.0.0-Beta.jar +export SPARK_JARS=${LIBS_PATH}/cudf-0.9.1-cuda10.jar,${LIBS_PATH}/xgboost4j_2.11-1.0.0-Beta2.jar,${LIBS_PATH}/xgboost4j-spark_2.11-1.0.0-Beta2.jar # additional Python files for XGBoost4J example -export SPARK_PY_FILES=${LIBS_PATH}/xgboost4j-spark_2.11-1.0.0-Beta.jar,${LIBS_PATH}/samples.zip +export SPARK_PY_FILES=${LIBS_PATH}/xgboost4j-spark_2.11-1.0.0-Beta2.jar,${LIBS_PATH}/samples.zip # tree construction algorithm export TREE_METHOD=hist diff --git a/getting-started-guides/on-premises-cluster/standalone-scala.md b/getting-started-guides/on-premises-cluster/standalone-scala.md index 71f592394114..bc8b3227dab2 100644 --- a/getting-started-guides/on-premises-cluster/standalone-scala.md +++ b/getting-started-guides/on-premises-cluster/standalone-scala.md @@ -10,8 +10,8 @@ Prerequisites * Multi-node clusters with homogenous GPU configuration * Software Requirements * Ubuntu 16.04/CentOS - * NVIDIA driver 410.48+ - * CUDA V10.0/9.2 + * CUDA V10.1/10.0/9.2 + * NVIDIA driver compatible with your CUDA * NCCL 2.4.7 * `EXCLUSIVE_PROCESS` must be set for all GPUs in each host. This can be accomplished using the `nvidia-smi` utility: diff --git a/getting-started-guides/on-premises-cluster/yarn-python.md b/getting-started-guides/on-premises-cluster/yarn-python.md index 78dd3333e2e6..a3093311044c 100644 --- a/getting-started-guides/on-premises-cluster/yarn-python.md +++ b/getting-started-guides/on-premises-cluster/yarn-python.md @@ -10,8 +10,8 @@ Prerequisites * Multi-node clusters with homogenous GPU configuration * Software Requirements * Ubuntu 16.04/CentOS - * NVIDIA driver 410.48+ - * CUDA V10.0/9.2 + * CUDA V10.1/10.0/9.2 + * NVIDIA driver compatible with your CUDA * NCCL 2.4.7 * Python 2.7/3.4/3.5/3.6/3.7 * NumPy @@ -35,9 +35,11 @@ Get Application Jar and Dataset ------------------------------- 1. *samples.zip* and *main.py*: Please build the files as specified in the [guide](/getting-started-guides/building-sample-apps/python.md) 2. Jars: Please download the following jars: - * [*cudf-0.9-cuda10.jar*](https://search.maven.org/remotecontent?filepath=ai/rapids/cudf/0.9/cudf-0.9-cuda10.jar) (For CUDA 9.2, please download [*cudf-0.9.jar*](https://search.maven.org/remotecontent?filepath=ai/rapids/cudf/0.9/cudf-0.9.jar) instead, and replace *cudf-0.9-cuda10.jar* with *cudf-0.9.jar* throughout this whole guide) - * [*xgboost4j_2.11-1.0.0-Beta.jar*](https://search.maven.org/remotecontent?filepath=ai/rapids/xgboost4j_2.11/1.0.0-Beta/xgboost4j_2.11-1.0.0-Beta.jar) - * [*xgboost4j-spark_2.11-1.0.0-Beta.jar*](https://search.maven.org/remotecontent?filepath=ai/rapids/xgboost4j-spark_2.11/1.0.0-Beta/xgboost4j-spark_2.11-1.0.0-Beta.jar) + * [*cudf-0.9.1-cuda10.jar*](https://search.maven.org/remotecontent?filepath=ai/rapids/cudf/0.9.1/cudf-0.9.1-cuda10.jar) (Here take CUDA 10.0 as an example) + * For CUDA 9.2, please download [*cudf-0.9.1.jar*](https://search.maven.org/remotecontent?filepath=ai/rapids/cudf/0.9.1/cudf-0.9.1.jar) instead, and replace *cudf-0.9.1-cuda10.jar* with *cudf-0.9.1.jar* throughout this whole guide + * For CUDA 10.1, please download [*cudf-0.9.1-cuda10-1.jar*](https://search.maven.org/remotecontent?filepath=ai/rapids/cudf/0.9.1/cudf-0.9.1-cuda10-1.jar) instead, and replace *cudf-0.9.1-cuda10.jar* with *cudf-0.9.1-cuda10-1.jar* throughout this whole guide + * [*xgboost4j_2.11-1.0.0-Beta2.jar*](https://search.maven.org/remotecontent?filepath=ai/rapids/xgboost4j_2.11/1.0.0-Beta2/xgboost4j_2.11-1.0.0-Beta2.jar) + * [*xgboost4j-spark_2.11-1.0.0-Beta2.jar*](https://search.maven.org/remotecontent?filepath=ai/rapids/xgboost4j-spark_2.11/1.0.0-Beta2/xgboost4j-spark_2.11-1.0.0-Beta2.jar) 3. Dataset: https://rapidsai-data.s3.us-east-2.amazonaws.com/spark/mortgage.zip Place dataset and other files in a local directory. In this example the dataset was unzipped in the `xgboost4j_spark_python/data` directory, and all other files in the `xgboost4j_spark_python/libs` directory. @@ -46,11 +48,11 @@ Place dataset and other files in a local directory. In this example the dataset [xgboost4j_spark_python]$ find . -type f | sort ./data/mortgage/csv/test/mortgage_eval_merged.csv ./data/mortgage/csv/train/mortgage_train_merged.csv -./libs/cudf-0.9-cuda10.jar +./libs/cudf-0.9.1-cuda10.jar ./libs/main.py ./libs/samples.zip -./libs/xgboost4j_2.11-1.0.0-Beta.jar -./libs/xgboost4j-spark_2.11-1.0.0-Beta.jar +./libs/xgboost4j_2.11-1.0.0-Beta2.jar +./libs/xgboost4j-spark_2.11-1.0.0-Beta2.jar ``` Create a directory in HDFS, and copy: @@ -66,11 +68,11 @@ Verify that the jar and dataset are in HDFS: [xgboost4j_spark_python]$ hadoop fs -find /tmp/xgboost4j_spark_python | grep "\." | sort /tmp/xgboost4j_spark_python/data/mortgage/csv/test/mortgage_eval_merged.csv /tmp/xgboost4j_spark_python/data/mortgage/csv/train/mortgage_train_merged.csv -/tmp/xgboost4j_spark_python/libs/cudf-0.9-cuda10.jar +/tmp/xgboost4j_spark_python/libs/cudf-0.9.1-cuda10.jar /tmp/xgboost4j_spark_python/libs/main.py /tmp/xgboost4j_spark_python/libs/samples.zip -/tmp/xgboost4j_spark_python/libs/xgboost4j_2.11-1.0.0-Beta.jar -/tmp/xgboost4j_spark_python/libs/xgboost4j-spark_2.11-1.0.0-Beta.jar +/tmp/xgboost4j_spark_python/libs/xgboost4j_2.11-1.0.0-Beta2.jar +/tmp/xgboost4j_spark_python/libs/xgboost4j-spark_2.11-1.0.0-Beta2.jar ``` Launch GPU Mortgage Example @@ -104,10 +106,10 @@ export SPARK_PYTHON_ENTRYPOINT=${LIBS_PATH}/main.py export EXAMPLE_CLASS=ai.rapids.spark.examples.mortgage.gpu_main # additional jars for XGBoost4J example -export SPARK_JARS=${LIBS_PATH}/cudf-0.9-cuda10.jar,${LIBS_PATH}/xgboost4j_2.11-1.0.0-Beta.jar,${LIBS_PATH}/xgboost4j-spark_2.11-1.0.0-Beta.jar +export SPARK_JARS=${LIBS_PATH}/cudf-0.9.1-cuda10.jar,${LIBS_PATH}/xgboost4j_2.11-1.0.0-Beta2.jar,${LIBS_PATH}/xgboost4j-spark_2.11-1.0.0-Beta2.jar # additional Python files for XGBoost4J example -export SPARK_PY_FILES=${LIBS_PATH}/xgboost4j-spark_2.11-1.0.0-Beta.jar,${LIBS_PATH}/samples.zip +export SPARK_PY_FILES=${LIBS_PATH}/xgboost4j-spark_2.11-1.0.0-Beta2.jar,${LIBS_PATH}/samples.zip # tree construction algorithm export TREE_METHOD=gpu_hist @@ -189,10 +191,10 @@ export SPARK_PYTHON_ENTRYPOINT=${LIBS_PATH}/main.py export EXAMPLE_CLASS=ai.rapids.spark.examples.mortgage.cpu_main # additional jars for XGBoost4J example -export SPARK_JARS=${LIBS_PATH}/cudf-0.9-cuda10.jar,${LIBS_PATH}/xgboost4j_2.11-1.0.0-Beta.jar,${LIBS_PATH}/xgboost4j-spark_2.11-1.0.0-Beta.jar +export SPARK_JARS=${LIBS_PATH}/cudf-0.9.1-cuda10.jar,${LIBS_PATH}/xgboost4j_2.11-1.0.0-Beta2.jar,${LIBS_PATH}/xgboost4j-spark_2.11-1.0.0-Beta2.jar # additional Python files for XGBoost4J example -export SPARK_PY_FILES=${LIBS_PATH}/xgboost4j-spark_2.11-1.0.0-Beta.jar,${LIBS_PATH}/samples.zip +export SPARK_PY_FILES=${LIBS_PATH}/xgboost4j-spark_2.11-1.0.0-Beta2.jar,${LIBS_PATH}/samples.zip # tree construction algorithm export TREE_METHOD=hist diff --git a/getting-started-guides/on-premises-cluster/yarn-scala.md b/getting-started-guides/on-premises-cluster/yarn-scala.md index 24959b9bd017..5e844aa1949d 100644 --- a/getting-started-guides/on-premises-cluster/yarn-scala.md +++ b/getting-started-guides/on-premises-cluster/yarn-scala.md @@ -10,8 +10,8 @@ Prerequisites * Multi-node clusters with homogenous GPU configuration * Software Requirements * Ubuntu 16.04/CentOS - * NVIDIA driver 410.48+ - * CUDA V10.0/9.2 + * CUDA V10.1/10.0/9.2 + * NVIDIA driver compatible with your CUDA * NCCL 2.4.7 * `EXCLUSIVE_PROCESS` must be set for all GPUs in each NodeManager. This can be accomplished using the `nvidia-smi` utility: