diff --git a/docs/configs.md b/docs/configs.md index 2d757a40779..7f9544496c4 100644 --- a/docs/configs.md +++ b/docs/configs.md @@ -37,7 +37,7 @@ Name | Description | Default Value | Applicable at spark.rapids.memory.gpu.minAllocFraction|The fraction of total GPU memory that limits the minimum size of the RMM pool. The value must be less than or equal to the setting for spark.rapids.memory.gpu.allocFraction.|0.25|Startup spark.rapids.memory.host.spillStorageSize|Amount of off-heap host memory to use for buffering spilled GPU data before spilling to local disk. Use -1 to set the amount to the combined size of pinned and pageable memory pools.|-1|Startup spark.rapids.memory.pinnedPool.size|The size of the pinned memory pool in bytes unless otherwise specified. Use 0 to disable the pool.|0|Startup -spark.rapids.sql.batchSizeBytes|Set the target number of bytes for a GPU batch. Splits sizes for input data is covered by separate configs. The maximum setting is 2 GB to avoid exceeding the cudf row count limit of a column.|1073741824|Runtime +spark.rapids.sql.batchSizeBytes|Set the target number of bytes for a GPU batch. Splits sizes for input data is covered by separate configs.|1073741824|Runtime spark.rapids.sql.concurrentGpuTasks|Set the number of tasks that can execute concurrently per GPU. Tasks may temporarily block when the number of concurrent tasks in the executor exceeds this amount. Allowing too many concurrent tasks on the same GPU may lead to GPU out of memory errors.|2|Runtime spark.rapids.sql.enabled|Enable (true) or disable (false) sql operations on the GPU|true|Runtime spark.rapids.sql.explain|Explain why some parts of a query were not placed on a GPU or not. Possible values are ALL: print everything, NONE: print nothing, NOT_ON_GPU: print only parts of a query that did not go on the GPU|NOT_ON_GPU|Runtime diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuCoalesceBatches.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuCoalesceBatches.scala index 1afc03b177b..b7fea71d3ef 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuCoalesceBatches.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuCoalesceBatches.scala @@ -223,8 +223,6 @@ case class RequireSingleBatchWithFilter(filterExpression: GpuExpression) case class TargetSize(override val targetSizeBytes: Long) extends CoalesceSizeGoal with SplittableGoal { - require(targetSizeBytes <= Integer.MAX_VALUE, - "Target cannot exceed 2GB without checks for cudf row count limit") } /** diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala index a83ad716d34..ccb21ed818e 100644 --- a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala +++ b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/RapidsConf.scala @@ -553,12 +553,10 @@ val GPU_COREDUMP_PIPE_PATTERN = conf("spark.rapids.gpu.coreDump.pipePattern") val GPU_BATCH_SIZE_BYTES = conf("spark.rapids.sql.batchSizeBytes") .doc("Set the target number of bytes for a GPU batch. Splits sizes for input data " + - "is covered by separate configs. The maximum setting is 2 GB to avoid exceeding the " + - "cudf row count limit of a column.") + "is covered by separate configs.") .commonlyUsed() .bytesConf(ByteUnit.BYTE) - .checkValue(v => v >= 0 && v <= Integer.MAX_VALUE, - s"Batch size must be positive and not exceed ${Integer.MAX_VALUE} bytes.") + .checkValue(v => v > 0, "Batch size must be positive") .createWithDefault(1 * 1024 * 1024 * 1024) // 1 GiB is the default val CHUNKED_READER = conf("spark.rapids.sql.reader.chunked")