From 92f0197197df9e1defbd49903d0b3c25b071d805 Mon Sep 17 00:00:00 2001 From: Vukasin Milovanovic Date: Mon, 9 Sep 2024 16:16:17 -0700 Subject: [PATCH] Simplify the nvCOMP adapter (#16762) This PR removes the adapter code that allow running with older nvCOMP versions. Feature status checking has been significantly simplified, and compile-time checks for newer compression types have been removed. Also removed the fallback to the old version of get_temp_size, since we are now guaranteed to have access to the extended version. Authors: - Vukasin Milovanovic (https://github.com/vuule) Approvers: - Bradley Dice (https://github.com/bdice) - Nghia Truong (https://github.com/ttnghia) - MithunR (https://github.com/mythrocks) URL: https://github.com/rapidsai/cudf/pull/16762 --- cpp/include/cudf/io/nvcomp_adapter.hpp | 24 +- cpp/src/io/comp/nvcomp_adapter.cpp | 334 +++------------------ cpp/src/io/comp/nvcomp_adapter.hpp | 14 +- cpp/src/io/orc/writer_impl.cu | 8 +- cpp/src/io/parquet/writer_impl_helpers.cpp | 2 +- cpp/tests/io/comp/decomp_test.cpp | 46 ++- 6 files changed, 79 insertions(+), 349 deletions(-) diff --git a/cpp/include/cudf/io/nvcomp_adapter.hpp b/cpp/include/cudf/io/nvcomp_adapter.hpp index f3260d0cb53..e7fe3cc7214 100644 --- a/cpp/include/cudf/io/nvcomp_adapter.hpp +++ b/cpp/include/cudf/io/nvcomp_adapter.hpp @@ -36,33 +36,20 @@ struct feature_status_parameters { int lib_patch_version; ///< patch version bool are_all_integrations_enabled; ///< all integrations bool are_stable_integrations_enabled; ///< stable integrations - int compute_capability_major; ///< cuda compute major version /** - * @brief Default Constructor + * @brief Default constructor using the current version of nvcomp and current environment + * variables */ feature_status_parameters(); /** - * @brief feature_status_parameters Constructor + * @brief Constructor using the current version of nvcomp * - * @param major positive integer representing major value of nvcomp - * @param minor positive integer representing minor value of nvcomp - * @param patch positive integer representing patch value of nvcomp * @param all_enabled if all integrations are enabled * @param stable_enabled if stable integrations are enabled - * @param cc_major CUDA compute capability */ - feature_status_parameters( - int major, int minor, int patch, bool all_enabled, bool stable_enabled, int cc_major) - : lib_major_version{major}, - lib_minor_version{minor}, - lib_patch_version{patch}, - are_all_integrations_enabled{all_enabled}, - are_stable_integrations_enabled{stable_enabled}, - compute_capability_major{cc_major} - { - } + feature_status_parameters(bool all_enabled, bool stable_enabled); }; /** @@ -74,8 +61,7 @@ inline bool operator==(feature_status_parameters const& lhs, feature_status_para lhs.lib_minor_version == rhs.lib_minor_version and lhs.lib_patch_version == rhs.lib_patch_version and lhs.are_all_integrations_enabled == rhs.are_all_integrations_enabled and - lhs.are_stable_integrations_enabled == rhs.are_stable_integrations_enabled and - lhs.compute_capability_major == rhs.compute_capability_major; + lhs.are_stable_integrations_enabled == rhs.are_stable_integrations_enabled; } /** diff --git a/cpp/src/io/comp/nvcomp_adapter.cpp b/cpp/src/io/comp/nvcomp_adapter.cpp index 5d0c6a8c83b..261a8eb401d 100644 --- a/cpp/src/io/comp/nvcomp_adapter.cpp +++ b/cpp/src/io/comp/nvcomp_adapter.cpp @@ -22,95 +22,44 @@ #include #include +#include #include #include +#include #include -#define NVCOMP_DEFLATE_HEADER -#if __has_include(NVCOMP_DEFLATE_HEADER) -#include NVCOMP_DEFLATE_HEADER -#endif - -#define NVCOMP_ZSTD_HEADER -#if __has_include(NVCOMP_ZSTD_HEADER) -#include NVCOMP_ZSTD_HEADER -#endif - -// When building with nvcomp 4.0 or newer, map the new version macros to the old ones -#ifndef NVCOMP_MAJOR_VERSION -#define NVCOMP_MAJOR_VERSION NVCOMP_VER_MAJOR -#define NVCOMP_MINOR_VERSION NVCOMP_VER_MINOR -#define NVCOMP_PATCH_VERSION NVCOMP_VER_PATCH -#endif - -#define NVCOMP_HAS_ZSTD_DECOMP(MAJOR, MINOR, PATCH) (MAJOR > 2 or (MAJOR == 2 and MINOR >= 3)) - -#define NVCOMP_HAS_ZSTD_COMP(MAJOR, MINOR, PATCH) (MAJOR > 2 or (MAJOR == 2 and MINOR >= 4)) - -#define NVCOMP_HAS_DEFLATE(MAJOR, MINOR, PATCH) (MAJOR > 2 or (MAJOR == 2 and MINOR >= 5)) - -#define NVCOMP_HAS_DECOMP_TEMPSIZE_EX(MAJOR, MINOR, PATCH) \ - (MAJOR > 2 or (MAJOR == 2 and MINOR > 3) or (MAJOR == 2 and MINOR == 3 and PATCH >= 1)) - -#define NVCOMP_HAS_COMP_TEMPSIZE_EX(MAJOR, MINOR, PATCH) (MAJOR > 2 or (MAJOR == 2 and MINOR >= 6)) - -// ZSTD is stable for nvcomp 2.3.2 or newer -#define NVCOMP_ZSTD_DECOMP_IS_STABLE(MAJOR, MINOR, PATCH) \ - (MAJOR > 2 or (MAJOR == 2 and MINOR > 3) or (MAJOR == 2 and MINOR == 3 and PATCH >= 2)) - namespace cudf::io::nvcomp { // Dispatcher for nvcompBatchedDecompressGetTempSizeEx template -std::optional batched_decompress_get_temp_size_ex(compression_type compression, - Args&&... args) +auto batched_decompress_get_temp_size_ex(compression_type compression, Args&&... args) { -#if NVCOMP_HAS_DECOMP_TEMPSIZE_EX(NVCOMP_MAJOR_VERSION, NVCOMP_MINOR_VERSION, NVCOMP_PATCH_VERSION) switch (compression) { case compression_type::SNAPPY: return nvcompBatchedSnappyDecompressGetTempSizeEx(std::forward(args)...); case compression_type::ZSTD: -#if NVCOMP_HAS_ZSTD_DECOMP(NVCOMP_MAJOR_VERSION, NVCOMP_MINOR_VERSION, NVCOMP_PATCH_VERSION) return nvcompBatchedZstdDecompressGetTempSizeEx(std::forward(args)...); -#else - return std::nullopt; -#endif case compression_type::LZ4: return nvcompBatchedLZ4DecompressGetTempSizeEx(std::forward(args)...); - case compression_type::DEFLATE: [[fallthrough]]; - default: return std::nullopt; - } -#endif - return std::nullopt; -} - -// Dispatcher for nvcompBatchedDecompressGetTempSize -template -auto batched_decompress_get_temp_size(compression_type compression, Args&&... args) -{ - switch (compression) { - case compression_type::SNAPPY: - return nvcompBatchedSnappyDecompressGetTempSize(std::forward(args)...); - case compression_type::ZSTD: -#if NVCOMP_HAS_ZSTD_DECOMP(NVCOMP_MAJOR_VERSION, NVCOMP_MINOR_VERSION, NVCOMP_PATCH_VERSION) - return nvcompBatchedZstdDecompressGetTempSize(std::forward(args)...); -#else - CUDF_FAIL("Decompression error: " + - nvcomp::is_decompression_disabled(nvcomp::compression_type::ZSTD).value()); -#endif case compression_type::DEFLATE: -#if NVCOMP_HAS_DEFLATE(NVCOMP_MAJOR_VERSION, NVCOMP_MINOR_VERSION, NVCOMP_PATCH_VERSION) - return nvcompBatchedDeflateDecompressGetTempSize(std::forward(args)...); -#else - CUDF_FAIL("Decompression error: " + - nvcomp::is_decompression_disabled(nvcomp::compression_type::DEFLATE).value()); -#endif - case compression_type::LZ4: - return nvcompBatchedLZ4DecompressGetTempSize(std::forward(args)...); + return nvcompBatchedDeflateDecompressGetTempSizeEx(std::forward(args)...); default: CUDF_FAIL("Unsupported compression type"); } } +size_t batched_decompress_temp_size(compression_type compression, + size_t num_chunks, + size_t max_uncomp_chunk_size, + size_t max_total_uncomp_size) +{ + size_t temp_size = 0; + nvcompStatus_t nvcomp_status = batched_decompress_get_temp_size_ex( + compression, num_chunks, max_uncomp_chunk_size, &temp_size, max_total_uncomp_size); + + CUDF_EXPECTS(nvcomp_status == nvcompStatus_t::nvcompSuccess, + "Unable to get scratch size for decompression"); + return temp_size; +} // Dispatcher for nvcompBatchedDecompressAsync template @@ -120,19 +69,9 @@ auto batched_decompress_async(compression_type compression, Args&&... args) case compression_type::SNAPPY: return nvcompBatchedSnappyDecompressAsync(std::forward(args)...); case compression_type::ZSTD: -#if NVCOMP_HAS_ZSTD_DECOMP(NVCOMP_MAJOR_VERSION, NVCOMP_MINOR_VERSION, NVCOMP_PATCH_VERSION) return nvcompBatchedZstdDecompressAsync(std::forward(args)...); -#else - CUDF_FAIL("Decompression error: " + - nvcomp::is_decompression_disabled(nvcomp::compression_type::ZSTD).value()); -#endif case compression_type::DEFLATE: -#if NVCOMP_HAS_DEFLATE(NVCOMP_MAJOR_VERSION, NVCOMP_MINOR_VERSION, NVCOMP_PATCH_VERSION) return nvcompBatchedDeflateDecompressAsync(std::forward(args)...); -#else - CUDF_FAIL("Decompression error: " + - nvcomp::is_decompression_disabled(nvcomp::compression_type::DEFLATE).value()); -#endif case compression_type::LZ4: return nvcompBatchedLZ4DecompressAsync(std::forward(args)...); default: CUDF_FAIL("Unsupported compression type"); } @@ -149,27 +88,6 @@ std::string compression_type_name(compression_type compression) return "compression_type(" + std::to_string(static_cast(compression)) + ")"; } -size_t batched_decompress_temp_size(compression_type compression, - size_t num_chunks, - size_t max_uncomp_chunk_size, - size_t max_total_uncomp_size) -{ - size_t temp_size = 0; - auto nvcomp_status = batched_decompress_get_temp_size_ex( - compression, num_chunks, max_uncomp_chunk_size, &temp_size, max_total_uncomp_size); - - if (nvcomp_status.value_or(nvcompStatus_t::nvcompErrorInternal) != - nvcompStatus_t::nvcompSuccess) { - nvcomp_status = - batched_decompress_get_temp_size(compression, num_chunks, max_uncomp_chunk_size, &temp_size); - } - - CUDF_EXPECTS(nvcomp_status == nvcompStatus_t::nvcompSuccess, - "Unable to get scratch size for decompression"); - - return temp_size; -} - void batched_decompress(compression_type compression, device_span const> inputs, device_span const> outputs, @@ -204,54 +122,10 @@ void batched_decompress(compression_type compression, update_compression_results(nvcomp_statuses, actual_uncompressed_data_sizes, results, stream); } -// Wrapper for nvcompBatchedCompressGetTempSize -auto batched_compress_get_temp_size(compression_type compression, - size_t batch_size, - size_t max_uncompressed_chunk_bytes) -{ - size_t temp_size = 0; - nvcompStatus_t nvcomp_status = nvcompStatus_t::nvcompSuccess; - switch (compression) { - case compression_type::SNAPPY: - nvcomp_status = nvcompBatchedSnappyCompressGetTempSize( - batch_size, max_uncompressed_chunk_bytes, nvcompBatchedSnappyDefaultOpts, &temp_size); - break; - case compression_type::DEFLATE: -#if NVCOMP_HAS_DEFLATE(NVCOMP_MAJOR_VERSION, NVCOMP_MINOR_VERSION, NVCOMP_PATCH_VERSION) - nvcomp_status = nvcompBatchedDeflateCompressGetTempSize( - batch_size, max_uncompressed_chunk_bytes, nvcompBatchedDeflateDefaultOpts, &temp_size); - break; -#else - CUDF_FAIL("Compression error: " + - nvcomp::is_compression_disabled(nvcomp::compression_type::DEFLATE).value()); -#endif - case compression_type::ZSTD: -#if NVCOMP_HAS_ZSTD_COMP(NVCOMP_MAJOR_VERSION, NVCOMP_MINOR_VERSION, NVCOMP_PATCH_VERSION) - nvcomp_status = nvcompBatchedZstdCompressGetTempSize( - batch_size, max_uncompressed_chunk_bytes, nvcompBatchedZstdDefaultOpts, &temp_size); - break; -#else - CUDF_FAIL("Compression error: " + - nvcomp::is_compression_disabled(nvcomp::compression_type::ZSTD).value()); -#endif - case compression_type::LZ4: - nvcomp_status = nvcompBatchedLZ4CompressGetTempSize( - batch_size, max_uncompressed_chunk_bytes, nvcompBatchedLZ4DefaultOpts, &temp_size); - break; - default: CUDF_FAIL("Unsupported compression type"); - } - - CUDF_EXPECTS(nvcomp_status == nvcompStatus_t::nvcompSuccess, - "Unable to get scratch size for compression"); - return temp_size; -} - -#if NVCOMP_HAS_COMP_TEMPSIZE_EX(NVCOMP_MAJOR_VERSION, NVCOMP_MINOR_VERSION, NVCOMP_PATCH_VERSION) -// Wrapper for nvcompBatchedCompressGetTempSizeEx -auto batched_compress_get_temp_size_ex(compression_type compression, - size_t batch_size, - size_t max_uncompressed_chunk_bytes, - size_t max_total_uncompressed_bytes) +size_t batched_compress_temp_size(compression_type compression, + size_t batch_size, + size_t max_uncompressed_chunk_bytes, + size_t max_total_uncompressed_bytes) { size_t temp_size = 0; nvcompStatus_t nvcomp_status = nvcompStatus_t::nvcompSuccess; @@ -291,28 +165,8 @@ auto batched_compress_get_temp_size_ex(compression_type compression, "Unable to get scratch size for compression"); return temp_size; } -#endif - -size_t batched_compress_temp_size(compression_type compression, - size_t num_chunks, - size_t max_uncomp_chunk_size, - size_t max_total_uncomp_size) -{ -#if NVCOMP_HAS_COMP_TEMPSIZE_EX(NVCOMP_MAJOR_VERSION, NVCOMP_MINOR_VERSION, NVCOMP_PATCH_VERSION) - try { - return batched_compress_get_temp_size_ex( - compression, num_chunks, max_uncomp_chunk_size, max_total_uncomp_size); - } catch (...) { - // Ignore errors in the expanded version; fall back to the old API in case of failure - CUDF_LOG_WARN( - "CompressGetTempSizeEx call failed, falling back to CompressGetTempSize; this may increase " - "the memory usage"); - } -#endif - - return batched_compress_get_temp_size(compression, num_chunks, max_uncomp_chunk_size); -} +// Wrapper for nvcompBatchedCompressGetMaxOutputChunkSize size_t compress_max_output_chunk_size(compression_type compression, uint32_t max_uncompressed_chunk_bytes) { @@ -328,23 +182,13 @@ size_t compress_max_output_chunk_size(compression_type compression, capped_uncomp_bytes, nvcompBatchedSnappyDefaultOpts, &max_comp_chunk_size); break; case compression_type::DEFLATE: -#if NVCOMP_HAS_DEFLATE(NVCOMP_MAJOR_VERSION, NVCOMP_MINOR_VERSION, NVCOMP_PATCH_VERSION) status = nvcompBatchedDeflateCompressGetMaxOutputChunkSize( capped_uncomp_bytes, nvcompBatchedDeflateDefaultOpts, &max_comp_chunk_size); break; -#else - CUDF_FAIL("Compression error: " + - nvcomp::is_compression_disabled(nvcomp::compression_type::DEFLATE).value()); -#endif case compression_type::ZSTD: -#if NVCOMP_HAS_ZSTD_COMP(NVCOMP_MAJOR_VERSION, NVCOMP_MINOR_VERSION, NVCOMP_PATCH_VERSION) status = nvcompBatchedZstdCompressGetMaxOutputChunkSize( capped_uncomp_bytes, nvcompBatchedZstdDefaultOpts, &max_comp_chunk_size); break; -#else - CUDF_FAIL("Compression error: " + - nvcomp::is_compression_disabled(nvcomp::compression_type::ZSTD).value()); -#endif case compression_type::LZ4: status = nvcompBatchedLZ4CompressGetMaxOutputChunkSize( capped_uncomp_bytes, nvcompBatchedLZ4DefaultOpts, &max_comp_chunk_size); @@ -384,7 +228,6 @@ static void batched_compress_async(compression_type compression, stream.value()); break; case compression_type::DEFLATE: -#if NVCOMP_HAS_DEFLATE(NVCOMP_MAJOR_VERSION, NVCOMP_MINOR_VERSION, NVCOMP_PATCH_VERSION) nvcomp_status = nvcompBatchedDeflateCompressAsync(device_uncompressed_ptrs, device_uncompressed_bytes, max_uncompressed_chunk_bytes, @@ -396,12 +239,7 @@ static void batched_compress_async(compression_type compression, nvcompBatchedDeflateDefaultOpts, stream.value()); break; -#else - CUDF_FAIL("Compression error: " + - nvcomp::is_compression_disabled(nvcomp::compression_type::DEFLATE).value()); -#endif case compression_type::ZSTD: -#if NVCOMP_HAS_ZSTD_COMP(NVCOMP_MAJOR_VERSION, NVCOMP_MINOR_VERSION, NVCOMP_PATCH_VERSION) nvcomp_status = nvcompBatchedZstdCompressAsync(device_uncompressed_ptrs, device_uncompressed_bytes, max_uncompressed_chunk_bytes, @@ -413,10 +251,6 @@ static void batched_compress_async(compression_type compression, nvcompBatchedZstdDefaultOpts, stream.value()); break; -#else - CUDF_FAIL("Compression error: " + - nvcomp::is_compression_disabled(nvcomp::compression_type::ZSTD).value()); -#endif case compression_type::LZ4: nvcomp_status = nvcompBatchedLZ4CompressAsync(device_uncompressed_ptrs, device_uncompressed_bytes, @@ -478,16 +312,18 @@ void batched_compress(compression_type compression, } feature_status_parameters::feature_status_parameters() - : lib_major_version{NVCOMP_MAJOR_VERSION}, - lib_minor_version{NVCOMP_MINOR_VERSION}, - lib_patch_version{NVCOMP_PATCH_VERSION}, - are_all_integrations_enabled{nvcomp_integration::is_all_enabled()}, - are_stable_integrations_enabled{nvcomp_integration::is_stable_enabled()} + : feature_status_parameters(nvcomp_integration::is_all_enabled(), + nvcomp_integration::is_stable_enabled()) +{ +} + +feature_status_parameters::feature_status_parameters(bool all_enabled, bool stable_enabled) + : lib_major_version{NVCOMP_VER_MAJOR}, + lib_minor_version{NVCOMP_VER_MINOR}, + lib_patch_version{NVCOMP_VER_PATCH}, + are_all_integrations_enabled{all_enabled}, + are_stable_integrations_enabled{stable_enabled} { - int device; - CUDF_CUDA_TRY(cudaGetDevice(&device)); - CUDF_CUDA_TRY( - cudaDeviceGetAttribute(&compute_capability_major, cudaDevAttrComputeCapabilityMajor, device)); } // Represents all parameters required to determine status of a compression/decompression feature @@ -510,41 +346,19 @@ std::optional is_compression_disabled_impl(compression_type compres { switch (compression) { case compression_type::DEFLATE: { - if (not NVCOMP_HAS_DEFLATE( - params.lib_major_version, params.lib_minor_version, params.lib_patch_version)) { - return "nvCOMP 2.5 or newer is required for Deflate compression"; - } if (not params.are_all_integrations_enabled) { return "DEFLATE compression is experimental, you can enable it through " "`LIBCUDF_NVCOMP_POLICY` environment variable."; } return std::nullopt; } - case compression_type::SNAPPY: { - if (not params.are_stable_integrations_enabled) { - return "Snappy compression has been disabled through the `LIBCUDF_NVCOMP_POLICY` " - "environment variable."; - } - return std::nullopt; - } - case compression_type::ZSTD: { - if (not NVCOMP_HAS_ZSTD_COMP( - params.lib_major_version, params.lib_minor_version, params.lib_patch_version)) { - return "nvCOMP 2.4 or newer is required for Zstandard compression"; - } - if (not params.are_stable_integrations_enabled) { - return "Zstandard compression is experimental, you can enable it through " - "`LIBCUDF_NVCOMP_POLICY` environment variable."; - } - return std::nullopt; - } case compression_type::LZ4: + case compression_type::SNAPPY: + case compression_type::ZSTD: if (not params.are_stable_integrations_enabled) { - return "LZ4 compression has been disabled through the `LIBCUDF_NVCOMP_POLICY` " - "environment variable."; + return "nvCOMP use is disabled through the `LIBCUDF_NVCOMP_POLICY` environment variable."; } return std::nullopt; - default: return "Unsupported compression type"; } return "Unsupported compression type"; } @@ -578,58 +392,25 @@ std::optional is_compression_disabled(compression_type compression, return reason; } -std::optional is_zstd_decomp_disabled(feature_status_parameters const& params) -{ - if (not NVCOMP_HAS_ZSTD_DECOMP( - params.lib_major_version, params.lib_minor_version, params.lib_patch_version)) { - return "nvCOMP 2.3 or newer is required for Zstandard decompression"; - } - - if (NVCOMP_ZSTD_DECOMP_IS_STABLE( - params.lib_major_version, params.lib_minor_version, params.lib_patch_version)) { - if (not params.are_stable_integrations_enabled) { - return "Zstandard decompression has been disabled through the `LIBCUDF_NVCOMP_POLICY` " - "environment variable."; - } - } else if (not params.are_all_integrations_enabled) { - return "Zstandard decompression is experimental, you can enable it through " - "`LIBCUDF_NVCOMP_POLICY` environment variable."; - } - - return std::nullopt; -} - std::optional is_decompression_disabled_impl(compression_type compression, feature_status_parameters params) { switch (compression) { case compression_type::DEFLATE: { - if (not NVCOMP_HAS_DEFLATE( - params.lib_major_version, params.lib_minor_version, params.lib_patch_version)) { - return "nvCOMP 2.5 or newer is required for Deflate decompression"; - } if (not params.are_all_integrations_enabled) { return "DEFLATE decompression is experimental, you can enable it through " "`LIBCUDF_NVCOMP_POLICY` environment variable."; } return std::nullopt; } - case compression_type::SNAPPY: { - if (not params.are_stable_integrations_enabled) { - return "Snappy decompression has been disabled through the `LIBCUDF_NVCOMP_POLICY` " - "environment variable."; - } - return std::nullopt; - } - case compression_type::ZSTD: return is_zstd_decomp_disabled(params); - case compression_type::LZ4: { + case compression_type::LZ4: + case compression_type::SNAPPY: + case compression_type::ZSTD: { if (not params.are_stable_integrations_enabled) { - return "LZ4 decompression has been disabled through the `LIBCUDF_NVCOMP_POLICY` " - "environment variable."; + return "nvCOMP use is disabled through the `LIBCUDF_NVCOMP_POLICY` environment variable."; } return std::nullopt; } - default: return "Unsupported compression type"; } return "Unsupported compression type"; } @@ -663,24 +444,13 @@ std::optional is_decompression_disabled(compression_type compressio return reason; } -size_t compress_input_alignment_bits(compression_type compression) +size_t required_alignment(compression_type compression) { switch (compression) { - case compression_type::DEFLATE: return 0; - case compression_type::SNAPPY: return 0; - case compression_type::ZSTD: return 2; - case compression_type::LZ4: return 2; - default: CUDF_FAIL("Unsupported compression type"); - } -} - -size_t compress_output_alignment_bits(compression_type compression) -{ - switch (compression) { - case compression_type::DEFLATE: return 3; - case compression_type::SNAPPY: return 0; - case compression_type::ZSTD: return 0; - case compression_type::LZ4: return 2; + case compression_type::DEFLATE: return nvcompDeflateRequiredAlignment; + case compression_type::SNAPPY: return nvcompSnappyRequiredAlignment; + case compression_type::ZSTD: return nvcompZstdRequiredAlignment; + case compression_type::LZ4: return nvcompLZ4RequiredAlignment; default: CUDF_FAIL("Unsupported compression type"); } } @@ -688,16 +458,10 @@ size_t compress_output_alignment_bits(compression_type compression) std::optional compress_max_allowed_chunk_size(compression_type compression) { switch (compression) { - case compression_type::DEFLATE: return 64 * 1024; - case compression_type::SNAPPY: return std::nullopt; - case compression_type::ZSTD: -#if NVCOMP_HAS_ZSTD_COMP(NVCOMP_MAJOR_VERSION, NVCOMP_MINOR_VERSION, NVCOMP_PATCH_VERSION) - return nvcompZstdCompressionMaxAllowedChunkSize; -#else - CUDF_FAIL("Compression error: " + - nvcomp::is_compression_disabled(nvcomp::compression_type::ZSTD).value()); -#endif - case compression_type::LZ4: return 16 * 1024 * 1024; + case compression_type::DEFLATE: return nvcompDeflateCompressionMaxAllowedChunkSize; + case compression_type::SNAPPY: return nvcompSnappyCompressionMaxAllowedChunkSize; + case compression_type::ZSTD: return nvcompZstdCompressionMaxAllowedChunkSize; + case compression_type::LZ4: return nvcompLZ4CompressionMaxAllowedChunkSize; default: return std::nullopt; } } diff --git a/cpp/src/io/comp/nvcomp_adapter.hpp b/cpp/src/io/comp/nvcomp_adapter.hpp index 43c79e32375..583bd6a3523 100644 --- a/cpp/src/io/comp/nvcomp_adapter.hpp +++ b/cpp/src/io/comp/nvcomp_adapter.hpp @@ -75,20 +75,12 @@ size_t batched_decompress_temp_size(compression_type compression, uint32_t max_uncomp_chunk_size); /** - * @brief Gets input alignment requirements for the given compression type. + * @brief Gets input and output alignment requirements for the given compression type. * * @param compression Compression type - * @returns required alignment, in bits + * @returns required alignment */ -[[nodiscard]] size_t compress_input_alignment_bits(compression_type compression); - -/** - * @brief Gets output alignment requirements for the given compression type. - * - * @param compression Compression type - * @returns required alignment, in bits - */ -[[nodiscard]] size_t compress_output_alignment_bits(compression_type compression); +[[nodiscard]] size_t required_alignment(compression_type compression); /** * @brief Maximum size of uncompressed chunks that can be compressed with nvCOMP. diff --git a/cpp/src/io/orc/writer_impl.cu b/cpp/src/io/orc/writer_impl.cu index ede9fd060b8..ebdf9f3f249 100644 --- a/cpp/src/io/orc/writer_impl.cu +++ b/cpp/src/io/orc/writer_impl.cu @@ -532,20 +532,20 @@ auto uncomp_block_alignment(CompressionKind compression_kind) { if (compression_kind == NONE or nvcomp::is_compression_disabled(to_nvcomp_compression_type(compression_kind))) { - return 1u; + return 1ul; } - return 1u << nvcomp::compress_input_alignment_bits(to_nvcomp_compression_type(compression_kind)); + return nvcomp::required_alignment(to_nvcomp_compression_type(compression_kind)); } auto comp_block_alignment(CompressionKind compression_kind) { if (compression_kind == NONE or nvcomp::is_compression_disabled(to_nvcomp_compression_type(compression_kind))) { - return 1u; + return 1ul; } - return 1u << nvcomp::compress_output_alignment_bits(to_nvcomp_compression_type(compression_kind)); + return nvcomp::required_alignment(to_nvcomp_compression_type(compression_kind)); } /** diff --git a/cpp/src/io/parquet/writer_impl_helpers.cpp b/cpp/src/io/parquet/writer_impl_helpers.cpp index e2f09f872d3..396d44c0763 100644 --- a/cpp/src/io/parquet/writer_impl_helpers.cpp +++ b/cpp/src/io/parquet/writer_impl_helpers.cpp @@ -62,7 +62,7 @@ uint32_t page_alignment(Compression codec) return 1u; } - return 1u << nvcomp::compress_input_alignment_bits(to_nvcomp_compression_type(codec)); + return nvcomp::required_alignment(to_nvcomp_compression_type(codec)); } size_t max_compression_output_size(Compression codec, uint32_t compression_blocksize) diff --git a/cpp/tests/io/comp/decomp_test.cpp b/cpp/tests/io/comp/decomp_test.cpp index 38c1a57eca9..840cf263ed9 100644 --- a/cpp/tests/io/comp/decomp_test.cpp +++ b/cpp/tests/io/comp/decomp_test.cpp @@ -176,23 +176,19 @@ TEST_F(NvcompConfigTest, Compression) using cudf::io::nvcomp::compression_type; auto const& comp_disabled = cudf::io::nvcomp::is_compression_disabled; - EXPECT_FALSE(comp_disabled(compression_type::DEFLATE, {2, 5, 0, true, true, 0})); - // version 2.5 required - EXPECT_TRUE(comp_disabled(compression_type::DEFLATE, {2, 4, 0, true, true, 0})); + EXPECT_FALSE(comp_disabled(compression_type::DEFLATE, {true, true})); // all integrations enabled required - EXPECT_TRUE(comp_disabled(compression_type::DEFLATE, {2, 5, 0, false, true, 0})); + EXPECT_TRUE(comp_disabled(compression_type::DEFLATE, {false, true})); - EXPECT_FALSE(comp_disabled(compression_type::ZSTD, {2, 4, 0, true, true, 0})); - EXPECT_FALSE(comp_disabled(compression_type::ZSTD, {2, 4, 0, false, true, 0})); - // 2.4 version required - EXPECT_TRUE(comp_disabled(compression_type::ZSTD, {2, 3, 1, false, true, 0})); + EXPECT_FALSE(comp_disabled(compression_type::ZSTD, {true, true})); + EXPECT_FALSE(comp_disabled(compression_type::ZSTD, {false, true})); // stable integrations enabled required - EXPECT_TRUE(comp_disabled(compression_type::ZSTD, {2, 4, 0, false, false, 0})); + EXPECT_TRUE(comp_disabled(compression_type::ZSTD, {false, false})); - EXPECT_FALSE(comp_disabled(compression_type::SNAPPY, {2, 5, 0, true, true, 0})); - EXPECT_FALSE(comp_disabled(compression_type::SNAPPY, {2, 4, 0, false, true, 0})); + EXPECT_FALSE(comp_disabled(compression_type::SNAPPY, {true, true})); + EXPECT_FALSE(comp_disabled(compression_type::SNAPPY, {false, true})); // stable integrations enabled required - EXPECT_TRUE(comp_disabled(compression_type::SNAPPY, {2, 3, 0, false, false, 0})); + EXPECT_TRUE(comp_disabled(compression_type::SNAPPY, {false, false})); } TEST_F(NvcompConfigTest, Decompression) @@ -200,27 +196,19 @@ TEST_F(NvcompConfigTest, Decompression) using cudf::io::nvcomp::compression_type; auto const& decomp_disabled = cudf::io::nvcomp::is_decompression_disabled; - EXPECT_FALSE(decomp_disabled(compression_type::DEFLATE, {2, 5, 0, true, true, 7})); - // version 2.5 required - EXPECT_TRUE(decomp_disabled(compression_type::DEFLATE, {2, 4, 0, true, true, 7})); + EXPECT_FALSE(decomp_disabled(compression_type::DEFLATE, {true, true})); // all integrations enabled required - EXPECT_TRUE(decomp_disabled(compression_type::DEFLATE, {2, 5, 0, false, true, 7})); - - EXPECT_FALSE(decomp_disabled(compression_type::ZSTD, {2, 4, 0, true, true, 7})); - EXPECT_FALSE(decomp_disabled(compression_type::ZSTD, {2, 3, 2, false, true, 6})); - EXPECT_FALSE(decomp_disabled(compression_type::ZSTD, {2, 3, 0, true, true, 6})); - // 2.3.1 and earlier requires all integrations to be enabled - EXPECT_TRUE(decomp_disabled(compression_type::ZSTD, {2, 3, 1, false, true, 7})); - // 2.3 version required - EXPECT_TRUE(decomp_disabled(compression_type::ZSTD, {2, 2, 0, true, true, 7})); + EXPECT_TRUE(decomp_disabled(compression_type::DEFLATE, {false, true})); + + EXPECT_FALSE(decomp_disabled(compression_type::ZSTD, {true, true})); + EXPECT_FALSE(decomp_disabled(compression_type::ZSTD, {false, true})); // stable integrations enabled required - EXPECT_TRUE(decomp_disabled(compression_type::ZSTD, {2, 4, 0, false, false, 7})); + EXPECT_TRUE(decomp_disabled(compression_type::ZSTD, {false, false})); - EXPECT_FALSE(decomp_disabled(compression_type::SNAPPY, {2, 4, 0, true, true, 7})); - EXPECT_FALSE(decomp_disabled(compression_type::SNAPPY, {2, 3, 0, false, true, 7})); - EXPECT_FALSE(decomp_disabled(compression_type::SNAPPY, {2, 2, 0, false, true, 7})); + EXPECT_FALSE(decomp_disabled(compression_type::SNAPPY, {true, true})); + EXPECT_FALSE(decomp_disabled(compression_type::SNAPPY, {false, true})); // stable integrations enabled required - EXPECT_TRUE(decomp_disabled(compression_type::SNAPPY, {2, 2, 0, false, false, 7})); + EXPECT_TRUE(decomp_disabled(compression_type::SNAPPY, {false, false})); } CUDF_TEST_PROGRAM_MAIN()