i will have nice things and nobody can stop me (fixed issue with chat…

… messages becoming broken)

tf2_bot_detector/CMakeLists.txt

@@ -223,6 +223,7 @@ if (TF2BD_ENABLE_TESTS)
 		"Tests/Catch2.cpp"
 		"Tests/ConsoleLineTests.cpp"
 		"Tests/Tests.h"
+		"Tests/CharConverterTests.cpp"
 	)
 
 	SET(TF2BD_ENABLE_CLI_EXE true)

tf2_bot_detector/Config/ChatWrappers.cpp

@@ -258,6 +258,8 @@ static bool GetChatCategory(const std::string* src, std::string_view* name, Chat
 
 static void GetChatMsgFormats(const std::string_view& debugInfo, const std::string_view& translations, ChatFormatStrings& strings)
 {
+	assert(!translations.empty());
+
 	const char* begin = translations.data();
 	const char* end = begin + translations.size();
 	std::error_code ec;
@@ -279,13 +281,26 @@ static void GetChatMsgFormats(const std::string_view& debugInfo, const std::stri
 			if (!GetChatCategory(&attrib.first, &chatType, &cat, &isEnglish))
 				continue;
 
+			if (attrib.second.empty())
+			{
+				LogWarning(MH_SOURCE_LOCATION_CURRENT(), "{}: Empty value read for {} ({})",
+					std::quoted(debugInfo), std::quoted(attrib.first), mh::enum_fmt(cat));
+			}
+
 			(isEnglish ? strings.m_English : strings.m_Localized)[(int)cat] = attrib.second;
 		}
 	}
 }
 
-static void ApplyChatWrappers(ChatCategory cat, std::string& translation, const ChatWrappers& wrappers)
+static void ApplyChatWrappers(const std::string_view& debugInfo, ChatCategory cat,
+	std::string& translation, const ChatWrappers& wrappers)
 {
+	if (translation.empty())
+	{
+		LogWarning(MH_SOURCE_LOCATION_CURRENT(), "{}: Translation empty for {}", debugInfo, mh::enum_fmt(cat));
+		return;
+	}
+
 	static const std::basic_regex s_Regex(R"regex(([\x01-\x05]?)(.*)%s1(.*)%s2(.*))regex",
 		std::regex::optimize | std::regex::icase);
 
@@ -616,32 +631,41 @@ size_t ChatFmtStrLengths::Type::GetMaxWrapperLength() const
 }
 
 ChatWrappers tf2_bot_detector::RandomizeChatWrappers(const std::filesystem::path& tfdir,
-	ChatWrappersProgress* progress)
+	mh::status_reader<ChatWrappersProgress>* progressReader)
 {
+	mh::status_source<ChatWrappersProgress> progressSource;
+	if (progressReader)
+		*progressReader = progressSource;
+
+	ChatWrappersProgress progress;
+
 	assert(!tfdir.empty());
 
 	if (auto path = tfdir / "custom" / "tf2_bot_detector"; std::filesystem::exists(path))
 	{
-		Log("Deleting "s << path);
+		DebugLog("Deleting {}", path);
 		std::filesystem::remove_all(path);
 	}
 
 	if (auto path = tfdir / "custom" / TF2BD_CHAT_WRAPPERS_DIR; std::filesystem::exists(path))
 	{
-		Log("Deleting "s << path);
-		std::filesystem::remove_all(path);
+		DebugLog("Deleting {}", path);
+		std::error_code ec;
+		std::filesystem::remove_all(path, ec);
+		if (ec)
+			LogWarning("Failed to delete {}: {}: {}", path, ec.value(), ec.message());
 	}
 
 	const auto outputDir = tfdir / "custom" / TF2BD_CHAT_WRAPPERS_DIR / "resource";
 	std::filesystem::create_directories(outputDir);
 
-	if (progress)
-		progress->m_MaxValue = unsigned(std::size(LANGUAGES) * 5);
+	progress.m_MaxValue = unsigned(std::size(LANGUAGES) * 5);
+	progressSource.set(progress);
 
 	const auto IncrementProgress = [&]
 	{
-		if (progress)
-			++progress->m_Value;
+		++progress.m_Value;
+		progressSource.set(progress);
 	};
 
 	ChatFormatStrings translations[std::size(LANGUAGES)];
@@ -696,7 +720,7 @@ ChatWrappers tf2_bot_detector::RandomizeChatWrappers(const std::filesystem::path
 
 			for (size_t i = 0; i < translationsSet.m_Localized.size(); i++)
 			{
-				ApplyChatWrappers(ChatCategory(i), translationsSet.m_Localized[i], wrappers);
+				ApplyChatWrappers(lang, ChatCategory(i), translationsSet.m_Localized[i], wrappers);
 				const auto key = GetChatCategoryKey(ChatCategory(i), false);
 				tokens->attribs[std::string(key)] = translationsSet.m_Localized[i];
 			}
@@ -705,7 +729,7 @@ ChatWrappers tf2_bot_detector::RandomizeChatWrappers(const std::filesystem::path
 				if (translationsSet.m_English[i].empty())
 					continue;
 
-				ApplyChatWrappers(ChatCategory(i), translationsSet.m_English[i], wrappers);
+				ApplyChatWrappers(lang, ChatCategory(i), translationsSet.m_English[i], wrappers);
 				const auto key = GetChatCategoryKey(ChatCategory(i), true);
 				tokens->attribs[std::string(key)] = translationsSet.m_English[i];
 			}

tf2_bot_detector/Config/ChatWrappers.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <mh/error/status.hpp>
 #include <mh/reflection/enum.hpp>
 #include <nlohmann/json_fwd.hpp>
 
@@ -115,11 +116,13 @@ namespace tf2_bot_detector
 
 	struct ChatWrappersProgress
 	{
-		std::atomic<uint32_t> m_Value{};
-		std::atomic<uint32_t> m_MaxValue{};
+		auto operator<=>(const ChatWrappersProgress&) const = default;
+
+		uint32_t m_Value{};
+		uint32_t m_MaxValue{};
 	};
 	ChatWrappers RandomizeChatWrappers(const std::filesystem::path& tfdir,
-		ChatWrappersProgress* progress = nullptr);
+		mh::status_reader<ChatWrappersProgress>* progress = nullptr);
 }
 
 MH_ENUM_REFLECT_BEGIN(tf2_bot_detector::ChatCategory)

tf2_bot_detector/SetupFlow/ChatWrappersGeneratorPage.cpp

@@ -49,11 +49,11 @@ auto ChatWrappersGeneratorPage::OnDraw(const DrawState& ds) -> OnDrawResult
 
 		ImGui::NewLine();
 
-		float progress = 0;
-		if (m_Progress && m_Progress->m_MaxValue > 0)
-			progress = m_Progress->m_Value / float(m_Progress->m_MaxValue);
+		float progressFloat = 0;
+		if (const auto progressObj = m_Progress.get(); progressObj->m_MaxValue > 0)
+			progressFloat = progressObj->m_Value / float(progressObj->m_MaxValue);
 
-		ImGui::ProgressBar(progress);
+		ImGui::ProgressBar(progressFloat);
 
 		if (mh::is_future_ready(m_ChatWrappersGenerated))
 			return OnDrawResult::EndDrawing;
@@ -114,8 +114,7 @@ void ChatWrappersGeneratorPage::Init(const InitState& is)
 		}
 
 		DebugLog("Regenerating chat wrappers...");
-		auto progress = m_Progress = std::make_shared<ChatWrappersProgress>();
-		m_ChatWrappersGenerated = std::async([tfDir, progress] { return RandomizeChatWrappers(tfDir, progress.get()); });
+		m_ChatWrappersGenerated = std::async([this, tfDir] { return RandomizeChatWrappers(tfDir, &m_Progress); });
 	}
 }
 

tf2_bot_detector/SetupFlow/ChatWrappersGeneratorPage.h

@@ -3,6 +3,8 @@
 #include "Config/ChatWrappers.h"
 #include "ISetupFlowPage.h"
 
+#include <mh/error/status.hpp>
+
 #include <future>
 #include <optional>
 
@@ -25,7 +27,7 @@ namespace tf2_bot_detector
 		static constexpr char VERIFY_CFG_FILE_NAME[] = "__tf2bd_chat_wrappers_verify.cfg";
 
 	private:
-		std::shared_ptr<ChatWrappersProgress> m_Progress;
+		mh::status_reader<ChatWrappersProgress> m_Progress;
 
 		std::future<ChatWrappers> m_ChatWrappersGenerated;
 		bool m_WasInitiallyClosed = true;

tf2_bot_detector/Tests/CharConverterTests.cpp

@@ -0,0 +1,83 @@
+
+
+#include <catch2/catch.hpp>
+#include <mh/text/codecvt.hpp>
+
+#include <string_view>
+
+using namespace std::string_view_literals;
+
+template<typename T>
+static void RequireEqual(const std::basic_string_view<T>& a, const std::basic_string_view<T>& b)
+{
+	std::vector<int64_t> araw(a.begin(), a.end()), braw(b.begin(), b.end());
+	CAPTURE(araw, braw);
+
+	REQUIRE(araw == braw);
+}
+
+template<typename T1, typename T2>
+static void CompareExpected(const std::basic_string_view<T1>& v1, const std::basic_string_view<T2>& v2)
+{
+	RequireEqual<T2>(mh::change_encoding<T2>(v1), v2);
+	RequireEqual<T1>(v1, mh::change_encoding<T1>(v2));
+}
+
+static void CompareExpected3(const std::u8string_view& v1, const std::u16string_view& v2, const std::u32string_view& v3)
+{
+	CompareExpected(v1, v2);
+
+	CompareExpected(v1, v3);
+
+	CompareExpected(v2, v3);
+}
+
+TEST_CASE("tf2bd_char_conversion_fundamental")
+{
+	CompareExpected3(u8"\U00010348", u"\U00010348", U"\U00010348");
+	CompareExpected3(u8"\u0024", u"\u0024", U"\u0024");
+	CompareExpected3(u8"\u00a2", u"\u00a2", U"\u00a2");
+	CompareExpected3(u8"\u0939", u"\u0939", U"\u0939");
+	CompareExpected3(u8"\u20ac", u"\u20ac", U"\u20ac");
+	CompareExpected3(u8"\ud55c", u"\ud55c", U"\ud55c");
+	CompareExpected3(u8"😐", u"😐", U"😐");
+}
+
+template<typename TConvertTo, typename TInput>
+static void CompareRoundtrip(const std::basic_string_view<TInput>& val)
+{
+	const auto converted = mh::change_encoding<TConvertTo>(val);
+	const auto convertedBack = mh::change_encoding<TInput>(converted);
+
+	REQUIRE(convertedBack.size() == val.size());
+	for (size_t i = 0; i < val.size(); i++)
+	{
+		CAPTURE(i);
+		REQUIRE(((int64_t)convertedBack.at(i)) == ((int64_t)val.at(i)));
+	}
+}
+
+template<typename T>
+static void CompareStringsAll(const std::basic_string_view<T>& val)
+{
+	CompareRoundtrip<char8_t>(val);
+	CompareRoundtrip<char16_t>(val);
+	CompareRoundtrip<char32_t>(val);
+}
+
+TEST_CASE("tf2bd_char_conversion")
+{
+	constexpr const std::u8string_view value_u8 = u8"😐";
+	constexpr const std::u16string_view value_u16 = u"😐";
+	constexpr const std::u32string_view value_u32 = U"😐";
+
+	CompareStringsAll(value_u8);
+	CompareStringsAll(value_u16);
+	CompareStringsAll(value_u32);
+
+	//REQUIRE(mh::change_encoding<char16_t>(value_u8) == value_u16);
+	//auto u8Str = mh::change_encoding<char8_t>(value_u8);
+
+	//auto u16Str2 = mh::change_encoding<char16_t>(input);
+
+}

tf2_bot_detector/Util/TextUtils.cpp

@@ -15,6 +15,7 @@ using namespace std::string_literals;
 
 std::u16string tf2_bot_detector::ToU16(const std::u8string_view& input)
 {
+	const char* dataTest = reinterpret_cast<const char*>(input.data());
 	return mh::change_encoding<char16_t>(input);
 }
 
@@ -28,17 +29,20 @@ std::u16string tf2_bot_detector::ToU16(const char* input, const char* input_end)
 
 std::u16string tf2_bot_detector::ToU16(const std::string_view& input)
 {
-	return ToU16(std::u8string_view(reinterpret_cast<const char8_t*>(input.data()), input.size()));
+	return mh::change_encoding<char16_t>(input);
+	//return ToU16(std::u8string_view(reinterpret_cast<const char8_t*>(input.data()), input.size()));
 }
 
 std::u16string tf2_bot_detector::ToU16(const std::wstring_view& input)
 {
-	return std::u16string(std::u16string_view(reinterpret_cast<const char16_t*>(input.data()), input.size()));
+	return mh::change_encoding<char16_t>(input);
+	//return std::u16string(std::u16string_view(reinterpret_cast<const char16_t*>(input.data()), input.size()));
 }
 
 std::u8string tf2_bot_detector::ToU8(const std::string_view& input)
 {
-	return std::u8string(std::u8string_view(reinterpret_cast<const char8_t*>(input.data()), input.size()));
+	return mh::change_encoding<char8_t>(input);
+	//return std::u8string(std::u8string_view(reinterpret_cast<const char8_t*>(input.data()), input.size()));
 }
 
 std::u8string tf2_bot_detector::ToU8(const std::u16string_view& input)
@@ -48,22 +52,26 @@ std::u8string tf2_bot_detector::ToU8(const std::u16string_view& input)
 
 std::u8string tf2_bot_detector::ToU8(const std::wstring_view& input)
 {
-	return ToU8(std::u16string_view(reinterpret_cast<const char16_t*>(input.data()), input.size()));
+	return mh::change_encoding<char8_t>(input);
+	//return ToU8(std::u16string_view(reinterpret_cast<const char16_t*>(input.data()), input.size()));
 }
 
 std::string tf2_bot_detector::ToMB(const std::u8string_view& input)
 {
-	return std::string(std::string_view(reinterpret_cast<const char*>(input.data()), input.size()));
+	return mh::change_encoding<char>(input);
+	//return std::string(std::string_view(reinterpret_cast<const char*>(input.data()), input.size()));
 }
 
 std::string tf2_bot_detector::ToMB(const std::u16string_view& input)
 {
-	return ToMB(ToU8(input));
+	return mh::change_encoding<char>(input);
+	//return ToMB(ToU8(input));
 }
 
 std::string tf2_bot_detector::ToMB(const std::wstring_view& input)
 {
-	return ToMB(ToU16(input));
+	return mh::change_encoding<char>(input);
+	//return ToMB(ToU16(input));
 }
 
 std::wstring tf2_bot_detector::ToWC(const std::string_view& input)
@@ -77,34 +85,31 @@ std::u16string tf2_bot_detector::ReadWideFile(const std::filesystem::path& filen
 
 	std::u16string wideFileData;
 	{
-		std::ifstream file(filename, std::ios::binary);
-		if (!file.good())
-			return {};
+		std::ifstream file;
+		file.exceptions(std::ios::badbit | std::ios::failbit);
+		file.open(filename, std::ios::binary);
 
 		file.seekg(0, std::ios::end);
-		if (!file.good())
-			return {};
 
-		const auto length = static_cast<size_t>(file.tellg());
+		// Length, minus BOM
+		const auto length = static_cast<size_t>(file.tellg()) - sizeof(char16_t);
 
 		// Skip BOM
 		file.seekg(2, std::ios::beg);
-		if (!file.good())
-			return {};
 
-		wideFileData.resize(length / 2 - 1);
+		wideFileData.resize(length / sizeof(char16_t));
 
 		file.read(reinterpret_cast<char*>(wideFileData.data()), length);
-		if (file.bad())
-			return {};
 	}
 
 	return wideFileData;
 }
 
 void tf2_bot_detector::WriteWideFile(const std::filesystem::path& filename, const std::u16string_view& text)
 {
-	std::ofstream file(filename, std::ios::binary);
+	std::ofstream file;
+	file.exceptions(std::ios::badbit | std::ios::failbit);
+	file.open(filename, std::ios::binary);
 	file << '\xFF' << '\xFE'; // BOM - UTF16LE
 
 	file.write(reinterpret_cast<const char*>(text.data()), text.size() * sizeof(text[0]));