Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix slice_strings wide strings logic with multi-byte characters #16777

Merged
merged 3 commits into from
Sep 11, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions cpp/src/strings/slice.cu
Original file line number Diff line number Diff line change
Expand Up @@ -122,26 +122,28 @@ CUDF_KERNEL void substring_from_kernel(column_device_view const d_strings,
break;
}
size_type const cc = (itr < end) && is_begin_utf8_char(*itr);
size_type const bc = (itr < end);
size_type const bc = (itr < end) ? bytes_in_utf8_byte(*itr) : 0;
char_count += cg::reduce(warp, cc, cg::plus<int>());
byte_count += cg::reduce(warp, bc, cg::plus<int>());
itr += cudf::detail::warp_size;
}

__syncwarp();

if (warp.thread_rank() == 0) {
if (start >= char_count) {
d_output[str_idx] = string_index_pair{"", 0};
return;
}

// we are just below start/stop and must now increment up to it from here
// we are just below start/stop and must now increment up to them from here
auto first_byte = start_counts.second;
if (start_counts.first < start) {
auto const sub_str = string_view(d_str.data() + first_byte, d_str.size_bytes() - first_byte);
first_byte += std::get<0>(bytes_to_character_position(sub_str, start - start_counts.first));
}

stop = max(stop, char_count);
stop = min(stop, char_count);
auto last_byte = stop_counts.second;
if (stop_counts.first < stop) {
auto const sub_str = string_view(d_str.data() + last_byte, d_str.size_bytes() - last_byte);
Expand Down
19 changes: 19 additions & 0 deletions cpp/tests/strings/slice_tests.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -268,6 +268,25 @@ TEST_F(StringsSliceTest, MaxPositions)
CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected);
}

TEST_F(StringsSliceTest, MultiByteChars)
{
auto input = cudf::test::strings_column_wrapper({
// clang-format off
"quick brown fox jumped over the lazy brown dog; the fat cats jump in place without moving "
"the following code snippet demonstrates how to use search for values in an ordered range "
// this placement tests proper multi-byte chars handling ------vvvvv
"it returns the last position where value could be inserted without the ééééé ordering ",
"algorithms execution is parallelized as determined by an execution policy; this is a 12345"
"continuation of previous row to make sure string boundaries are honored 012345678901234567"
// v--- this one also
"01234567890é34567890012345678901234567890"
// clang-format on
});

auto results = cudf::strings::slice_strings(cudf::strings_column_view(input), 0);
CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, input);
}

TEST_F(StringsSliceTest, Error)
{
cudf::test::strings_column_wrapper strings{"this string intentionally left blank"};
Expand Down
Loading