diff --git a/submodules/mh_stuff b/submodules/mh_stuff index b8b6140b..559c6f6b 160000 --- a/submodules/mh_stuff +++ b/submodules/mh_stuff @@ -1 +1 @@ -Subproject commit b8b6140b4be8af7dc83eba0321dc37fd02923e73 +Subproject commit 559c6f6bdb470a4a21a413c2994e4b63a2071eac diff --git a/tf2_bot_detector/CMakeLists.txt b/tf2_bot_detector/CMakeLists.txt index 5fd39155..222e608e 100644 --- a/tf2_bot_detector/CMakeLists.txt +++ b/tf2_bot_detector/CMakeLists.txt @@ -223,6 +223,7 @@ if (TF2BD_ENABLE_TESTS) "Tests/Catch2.cpp" "Tests/ConsoleLineTests.cpp" "Tests/Tests.h" + "Tests/CharConverterTests.cpp" ) SET(TF2BD_ENABLE_CLI_EXE true) diff --git a/tf2_bot_detector/Config/ChatWrappers.cpp b/tf2_bot_detector/Config/ChatWrappers.cpp index f68ce667..df1e62b5 100644 --- a/tf2_bot_detector/Config/ChatWrappers.cpp +++ b/tf2_bot_detector/Config/ChatWrappers.cpp @@ -258,6 +258,8 @@ static bool GetChatCategory(const std::string* src, std::string_view* name, Chat static void GetChatMsgFormats(const std::string_view& debugInfo, const std::string_view& translations, ChatFormatStrings& strings) { + assert(!translations.empty()); + const char* begin = translations.data(); const char* end = begin + translations.size(); std::error_code ec; @@ -279,13 +281,26 @@ static void GetChatMsgFormats(const std::string_view& debugInfo, const std::stri if (!GetChatCategory(&attrib.first, &chatType, &cat, &isEnglish)) continue; + if (attrib.second.empty()) + { + LogWarning(MH_SOURCE_LOCATION_CURRENT(), "{}: Empty value read for {} ({})", + std::quoted(debugInfo), std::quoted(attrib.first), mh::enum_fmt(cat)); + } + (isEnglish ? strings.m_English : strings.m_Localized)[(int)cat] = attrib.second; } } } -static void ApplyChatWrappers(ChatCategory cat, std::string& translation, const ChatWrappers& wrappers) +static void ApplyChatWrappers(const std::string_view& debugInfo, ChatCategory cat, + std::string& translation, const ChatWrappers& wrappers) { + if (translation.empty()) + { + LogWarning(MH_SOURCE_LOCATION_CURRENT(), "{}: Translation empty for {}", debugInfo, mh::enum_fmt(cat)); + return; + } + static const std::basic_regex s_Regex(R"regex(([\x01-\x05]?)(.*)%s1(.*)%s2(.*))regex", std::regex::optimize | std::regex::icase); @@ -616,32 +631,41 @@ size_t ChatFmtStrLengths::Type::GetMaxWrapperLength() const } ChatWrappers tf2_bot_detector::RandomizeChatWrappers(const std::filesystem::path& tfdir, - ChatWrappersProgress* progress) + mh::status_reader* progressReader) { + mh::status_source progressSource; + if (progressReader) + *progressReader = progressSource; + + ChatWrappersProgress progress; + assert(!tfdir.empty()); if (auto path = tfdir / "custom" / "tf2_bot_detector"; std::filesystem::exists(path)) { - Log("Deleting "s << path); + DebugLog("Deleting {}", path); std::filesystem::remove_all(path); } if (auto path = tfdir / "custom" / TF2BD_CHAT_WRAPPERS_DIR; std::filesystem::exists(path)) { - Log("Deleting "s << path); - std::filesystem::remove_all(path); + DebugLog("Deleting {}", path); + std::error_code ec; + std::filesystem::remove_all(path, ec); + if (ec) + LogWarning("Failed to delete {}: {}: {}", path, ec.value(), ec.message()); } const auto outputDir = tfdir / "custom" / TF2BD_CHAT_WRAPPERS_DIR / "resource"; std::filesystem::create_directories(outputDir); - if (progress) - progress->m_MaxValue = unsigned(std::size(LANGUAGES) * 5); + progress.m_MaxValue = unsigned(std::size(LANGUAGES) * 5); + progressSource.set(progress); const auto IncrementProgress = [&] { - if (progress) - ++progress->m_Value; + ++progress.m_Value; + progressSource.set(progress); }; ChatFormatStrings translations[std::size(LANGUAGES)]; @@ -696,7 +720,7 @@ ChatWrappers tf2_bot_detector::RandomizeChatWrappers(const std::filesystem::path for (size_t i = 0; i < translationsSet.m_Localized.size(); i++) { - ApplyChatWrappers(ChatCategory(i), translationsSet.m_Localized[i], wrappers); + ApplyChatWrappers(lang, ChatCategory(i), translationsSet.m_Localized[i], wrappers); const auto key = GetChatCategoryKey(ChatCategory(i), false); tokens->attribs[std::string(key)] = translationsSet.m_Localized[i]; } @@ -705,7 +729,7 @@ ChatWrappers tf2_bot_detector::RandomizeChatWrappers(const std::filesystem::path if (translationsSet.m_English[i].empty()) continue; - ApplyChatWrappers(ChatCategory(i), translationsSet.m_English[i], wrappers); + ApplyChatWrappers(lang, ChatCategory(i), translationsSet.m_English[i], wrappers); const auto key = GetChatCategoryKey(ChatCategory(i), true); tokens->attribs[std::string(key)] = translationsSet.m_English[i]; } diff --git a/tf2_bot_detector/Config/ChatWrappers.h b/tf2_bot_detector/Config/ChatWrappers.h index 05acdd76..7a2953b4 100644 --- a/tf2_bot_detector/Config/ChatWrappers.h +++ b/tf2_bot_detector/Config/ChatWrappers.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -115,11 +116,13 @@ namespace tf2_bot_detector struct ChatWrappersProgress { - std::atomic m_Value{}; - std::atomic m_MaxValue{}; + auto operator<=>(const ChatWrappersProgress&) const = default; + + uint32_t m_Value{}; + uint32_t m_MaxValue{}; }; ChatWrappers RandomizeChatWrappers(const std::filesystem::path& tfdir, - ChatWrappersProgress* progress = nullptr); + mh::status_reader* progress = nullptr); } MH_ENUM_REFLECT_BEGIN(tf2_bot_detector::ChatCategory) diff --git a/tf2_bot_detector/SetupFlow/ChatWrappersGeneratorPage.cpp b/tf2_bot_detector/SetupFlow/ChatWrappersGeneratorPage.cpp index b27eebf7..5b96c928 100644 --- a/tf2_bot_detector/SetupFlow/ChatWrappersGeneratorPage.cpp +++ b/tf2_bot_detector/SetupFlow/ChatWrappersGeneratorPage.cpp @@ -49,11 +49,11 @@ auto ChatWrappersGeneratorPage::OnDraw(const DrawState& ds) -> OnDrawResult ImGui::NewLine(); - float progress = 0; - if (m_Progress && m_Progress->m_MaxValue > 0) - progress = m_Progress->m_Value / float(m_Progress->m_MaxValue); + float progressFloat = 0; + if (const auto progressObj = m_Progress.get(); progressObj->m_MaxValue > 0) + progressFloat = progressObj->m_Value / float(progressObj->m_MaxValue); - ImGui::ProgressBar(progress); + ImGui::ProgressBar(progressFloat); if (mh::is_future_ready(m_ChatWrappersGenerated)) return OnDrawResult::EndDrawing; @@ -114,8 +114,7 @@ void ChatWrappersGeneratorPage::Init(const InitState& is) } DebugLog("Regenerating chat wrappers..."); - auto progress = m_Progress = std::make_shared(); - m_ChatWrappersGenerated = std::async([tfDir, progress] { return RandomizeChatWrappers(tfDir, progress.get()); }); + m_ChatWrappersGenerated = std::async([this, tfDir] { return RandomizeChatWrappers(tfDir, &m_Progress); }); } } diff --git a/tf2_bot_detector/SetupFlow/ChatWrappersGeneratorPage.h b/tf2_bot_detector/SetupFlow/ChatWrappersGeneratorPage.h index 5d431520..1446646b 100644 --- a/tf2_bot_detector/SetupFlow/ChatWrappersGeneratorPage.h +++ b/tf2_bot_detector/SetupFlow/ChatWrappersGeneratorPage.h @@ -3,6 +3,8 @@ #include "Config/ChatWrappers.h" #include "ISetupFlowPage.h" +#include + #include #include @@ -25,7 +27,7 @@ namespace tf2_bot_detector static constexpr char VERIFY_CFG_FILE_NAME[] = "__tf2bd_chat_wrappers_verify.cfg"; private: - std::shared_ptr m_Progress; + mh::status_reader m_Progress; std::future m_ChatWrappersGenerated; bool m_WasInitiallyClosed = true; diff --git a/tf2_bot_detector/Tests/CharConverterTests.cpp b/tf2_bot_detector/Tests/CharConverterTests.cpp new file mode 100644 index 00000000..41aafb94 --- /dev/null +++ b/tf2_bot_detector/Tests/CharConverterTests.cpp @@ -0,0 +1,83 @@ + + +#include +#include + +#include + +using namespace std::string_view_literals; + +template +static void RequireEqual(const std::basic_string_view& a, const std::basic_string_view& b) +{ + std::vector araw(a.begin(), a.end()), braw(b.begin(), b.end()); + CAPTURE(araw, braw); + + REQUIRE(araw == braw); +} + +template +static void CompareExpected(const std::basic_string_view& v1, const std::basic_string_view& v2) +{ + RequireEqual(mh::change_encoding(v1), v2); + RequireEqual(v1, mh::change_encoding(v2)); +} + +static void CompareExpected3(const std::u8string_view& v1, const std::u16string_view& v2, const std::u32string_view& v3) +{ + CompareExpected(v1, v2); + + CompareExpected(v1, v3); + + CompareExpected(v2, v3); +} + +TEST_CASE("tf2bd_char_conversion_fundamental") +{ + CompareExpected3(u8"\U00010348", u"\U00010348", U"\U00010348"); + CompareExpected3(u8"\u0024", u"\u0024", U"\u0024"); + CompareExpected3(u8"\u00a2", u"\u00a2", U"\u00a2"); + CompareExpected3(u8"\u0939", u"\u0939", U"\u0939"); + CompareExpected3(u8"\u20ac", u"\u20ac", U"\u20ac"); + CompareExpected3(u8"\ud55c", u"\ud55c", U"\ud55c"); + CompareExpected3(u8"😐", u"😐", U"😐"); +} + +template +static void CompareRoundtrip(const std::basic_string_view& val) +{ + const auto converted = mh::change_encoding(val); + const auto convertedBack = mh::change_encoding(converted); + + REQUIRE(convertedBack.size() == val.size()); + for (size_t i = 0; i < val.size(); i++) + { + CAPTURE(i); + REQUIRE(((int64_t)convertedBack.at(i)) == ((int64_t)val.at(i))); + } +} + +template +static void CompareStringsAll(const std::basic_string_view& val) +{ + CompareRoundtrip(val); + CompareRoundtrip(val); + CompareRoundtrip(val); +} + +TEST_CASE("tf2bd_char_conversion") +{ + constexpr const std::u8string_view value_u8 = u8"😐"; + constexpr const std::u16string_view value_u16 = u"😐"; + constexpr const std::u32string_view value_u32 = U"😐"; + + CompareStringsAll(value_u8); + CompareStringsAll(value_u16); + CompareStringsAll(value_u32); + + //REQUIRE(mh::change_encoding(value_u8) == value_u16); + //auto u8Str = mh::change_encoding(value_u8); + + //auto u16Str2 = mh::change_encoding(input); + +} diff --git a/tf2_bot_detector/Util/TextUtils.cpp b/tf2_bot_detector/Util/TextUtils.cpp index 2afb4e37..d55f53bf 100644 --- a/tf2_bot_detector/Util/TextUtils.cpp +++ b/tf2_bot_detector/Util/TextUtils.cpp @@ -15,6 +15,7 @@ using namespace std::string_literals; std::u16string tf2_bot_detector::ToU16(const std::u8string_view& input) { + const char* dataTest = reinterpret_cast(input.data()); return mh::change_encoding(input); } @@ -28,17 +29,20 @@ std::u16string tf2_bot_detector::ToU16(const char* input, const char* input_end) std::u16string tf2_bot_detector::ToU16(const std::string_view& input) { - return ToU16(std::u8string_view(reinterpret_cast(input.data()), input.size())); + return mh::change_encoding(input); + //return ToU16(std::u8string_view(reinterpret_cast(input.data()), input.size())); } std::u16string tf2_bot_detector::ToU16(const std::wstring_view& input) { - return std::u16string(std::u16string_view(reinterpret_cast(input.data()), input.size())); + return mh::change_encoding(input); + //return std::u16string(std::u16string_view(reinterpret_cast(input.data()), input.size())); } std::u8string tf2_bot_detector::ToU8(const std::string_view& input) { - return std::u8string(std::u8string_view(reinterpret_cast(input.data()), input.size())); + return mh::change_encoding(input); + //return std::u8string(std::u8string_view(reinterpret_cast(input.data()), input.size())); } std::u8string tf2_bot_detector::ToU8(const std::u16string_view& input) @@ -48,22 +52,26 @@ std::u8string tf2_bot_detector::ToU8(const std::u16string_view& input) std::u8string tf2_bot_detector::ToU8(const std::wstring_view& input) { - return ToU8(std::u16string_view(reinterpret_cast(input.data()), input.size())); + return mh::change_encoding(input); + //return ToU8(std::u16string_view(reinterpret_cast(input.data()), input.size())); } std::string tf2_bot_detector::ToMB(const std::u8string_view& input) { - return std::string(std::string_view(reinterpret_cast(input.data()), input.size())); + return mh::change_encoding(input); + //return std::string(std::string_view(reinterpret_cast(input.data()), input.size())); } std::string tf2_bot_detector::ToMB(const std::u16string_view& input) { - return ToMB(ToU8(input)); + return mh::change_encoding(input); + //return ToMB(ToU8(input)); } std::string tf2_bot_detector::ToMB(const std::wstring_view& input) { - return ToMB(ToU16(input)); + return mh::change_encoding(input); + //return ToMB(ToU16(input)); } std::wstring tf2_bot_detector::ToWC(const std::string_view& input) @@ -77,26 +85,21 @@ std::u16string tf2_bot_detector::ReadWideFile(const std::filesystem::path& filen std::u16string wideFileData; { - std::ifstream file(filename, std::ios::binary); - if (!file.good()) - return {}; + std::ifstream file; + file.exceptions(std::ios::badbit | std::ios::failbit); + file.open(filename, std::ios::binary); file.seekg(0, std::ios::end); - if (!file.good()) - return {}; - const auto length = static_cast(file.tellg()); + // Length, minus BOM + const auto length = static_cast(file.tellg()) - sizeof(char16_t); // Skip BOM file.seekg(2, std::ios::beg); - if (!file.good()) - return {}; - wideFileData.resize(length / 2 - 1); + wideFileData.resize(length / sizeof(char16_t)); file.read(reinterpret_cast(wideFileData.data()), length); - if (file.bad()) - return {}; } return wideFileData; @@ -104,7 +107,9 @@ std::u16string tf2_bot_detector::ReadWideFile(const std::filesystem::path& filen void tf2_bot_detector::WriteWideFile(const std::filesystem::path& filename, const std::u16string_view& text) { - std::ofstream file(filename, std::ios::binary); + std::ofstream file; + file.exceptions(std::ios::badbit | std::ios::failbit); + file.open(filename, std::ios::binary); file << '\xFF' << '\xFE'; // BOM - UTF16LE file.write(reinterpret_cast(text.data()), text.size() * sizeof(text[0]));