overlays: Allow use of extended ascii8

- Use custom string conversion to ensure overlay deals with extended ascii whenever possible
- Improves language compatibility greatly and avoids empty spaces for unknown glyphs

rpcs3/Emu/RSX/Overlays/overlay_controls.h

@@ -248,7 +248,7 @@ namespace rsx
 					return{};
 
 				stbtt_aligned_quad quad;
-				stbtt_GetPackedQuad(pack_info.data(), width, height, c, &x_advance, &y_advance, &quad, true);
+				stbtt_GetPackedQuad(pack_info.data(), width, height, u8(c), &x_advance, &y_advance, &quad, true);
 				return quad;
 			}
 
@@ -270,10 +270,11 @@ namespace rsx
 				{
 					if (char c = text[i++]; c && (i <= char_limit))
 					{
-						if ((u32)c >= char_count)
+						if (u8(c) >= char_count)
 						{
 							// Unsupported glyph, render null for now
 							c = ' ';
+							c = ' ';
 						}
 
 						switch (c)
@@ -1017,7 +1018,7 @@ namespace rsx
 						last_word = text_width;
 					}
 
-					if ((u32)c > renderer->char_count)
+					if (u8(c) > renderer->char_count)
 					{
 						// Non-existent glyph
 						text_width += renderer->em_size;
@@ -1041,7 +1042,6 @@ namespace rsx
 				max_w = std::max(max_w, text_width);
 				width = (u16)ceilf(max_w);
 			}
-
 		};
 
 		struct animation_base
@@ -1376,7 +1376,7 @@ namespace rsx
 		{
 			label() {}
 
-			label(const char *text)
+			label(const std::string& text)
 			{
 				this->text = text;
 			}

rpcs3/Emu/RSX/Overlays/overlay_osk.cpp

@@ -123,7 +123,7 @@ namespace rsx
 			else
 			{
 				m_preview.set_text(initial_text);
-				m_preview.caret_position = initial_text.length();
+				m_preview.caret_position = ::narrow<u16>(initial_text.length());
 				m_preview.fore_color.a = 1.f;
 			}
 
@@ -406,7 +406,7 @@ namespace rsx
 
 		void osk_dialog::on_text_changed()
 		{
-			const auto ws = utf8_to_utf16(m_preview.text);
+			const auto ws = ascii8_to_utf16(m_preview.text);
 			const auto length = (ws.length() + 1) * sizeof(char16_t);
 			memcpy(osk_text, ws.c_str(), length);
 
@@ -660,7 +660,7 @@ namespace rsx
 
 			// Narrow to utf-8 as native does not have support for non-ascii glyphs
 			// TODO: Full multibyte string support in all of rsx::overlays (kd-11)
-			initialize_layout(layout, utf16_to_utf8(message), utf16_to_utf8(init_text));
+			initialize_layout(layout, utf16_to_ascii8(message), utf16_to_ascii8(init_text));
 		}
 	}
 }

rpcs3/Emu/RSX/Overlays/overlays.cpp

@@ -2,54 +2,66 @@
 #include "overlays.h"
 #include "../GSRender.h"
 
-#include <locale>
+std::string utf8_to_ascii8(const std::string& utf8_string)
-#include <codecvt>
-
-#if _MSC_VER >= 1900
-// Stupid MSVC bug when T is set to char16_t
-std::string utf16_to_utf8(const std::u16string& utf16_string)
 {
-	// Strip extended codes
+	std::vector<u8> out;
-	auto tmp = utf16_string;
+	out.reserve(utf8_string.length() + 1);
-	for (auto &c : tmp)
+
+	for (u32 index = 0; index < utf8_string.length(); ++index)
 	{
-		if (c > 0xFF) c = '#';
+		const auto code = (u8)utf8_string[index];
+		if (code <= 0x7F)
+		{
+			out.push_back(code);
+			continue;
+		}
+
+		auto extra_bytes = (code <= 0xDF) ? 1u : (code <= 0xEF) ? 2u : 3u;
+		index += extra_bytes;
+
+		if (extra_bytes > 1 || (code & 0x1C))
+		{
+			// Needs more bits than we could represent with extended ASCII anyway
+			out.push_back('#');
+			continue;
+		}
+
+		u8 out_code = ((code & 0x3) << 6) | (u8(utf8_string[index]) & 0x3F);
+		out.push_back(out_code);
 	}
 
-	std::wstring_convert<std::codecvt_utf8_utf16<int16_t>, int16_t> convert;
+	out.push_back(0);
-	auto p = reinterpret_cast<const int16_t *>(tmp.data());
+	return { reinterpret_cast<char*>(out.data()) };
-	return convert.to_bytes(p, p + utf16_string.size());
 }
 
-std::u16string utf8_to_utf16(const std::string& utf8_string)
+std::string utf16_to_ascii8(const std::u16string& utf16_string)
 {
-	std::wstring_convert<std::codecvt_utf8_utf16<int16_t>, int16_t> convert;
+	// Strip extended codes, map to '#' instead (placeholder)
-	auto ws = convert.from_bytes(utf8_string);
+	std::vector<u8> out;
-	return reinterpret_cast<const char16_t*>(ws.c_str());
+	out.reserve(utf16_string.length() + 1);
-}
 
-#else
+	for (const auto& code : utf16_string)
-
-std::string utf16_to_utf8(const std::u16string& utf16_string)
-{
-	// Strip extended codes
-	auto tmp = utf16_string;
-	for (auto &c : tmp)
 	{
-		if (c > 0xFF) c = '#';
+		out.push_back(code > 0xFF ? '#': (u8)code);
 	}
 
-	std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> convert;
+	out.push_back(0);
-	return convert.to_bytes(tmp);
+	return { reinterpret_cast<char*>(out.data()) };
 }
 
-std::u16string utf8_to_utf16(const std::string& utf8_string)
+std::u16string ascii8_to_utf16(const std::string& ascii_string)
 {
-	std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> convert;
+	std::vector<char16_t> out;
-	return convert.from_bytes(utf8_string);
+	out.reserve(ascii_string.length() + 1);
-}
 
-#endif
+	for (const auto& code : ascii_string)
+	{
+		out.push_back(code > 0xFF ? '#' : (char16_t)code);
+	}
+
+	out.push_back(0);
+	return { out.data() };
+}
 
 namespace rsx
 {

rpcs3/Emu/RSX/Overlays/overlays.h

@@ -17,8 +17,9 @@
 #include "Utilities/Timer.h"
 
 // Utils
-std::string utf16_to_utf8(const std::u16string& utf16_string);
+std::string utf8_to_ascii8(const std::string& utf8_string);
-std::u16string utf8_to_utf16(const std::string& utf8_string);
+std::string utf16_to_ascii8(const std::u16string& utf16_string);
+std::u16string ascii8_to_utf16(const std::string& utf8_string);
 extern u64 get_system_time();
 
 // Definition of user interface implementations
@@ -495,7 +496,7 @@ namespace rsx
 				std::unique_ptr<image_info> icon_data;
 
 			public:
-				save_dialog_entry(const char* text1, const char* text2, u8 resource_id, const std::vector<u8>& icon_buf)
+				save_dialog_entry(const std::string& text1, const std::string& text2, u8 resource_id, const std::vector<u8>& icon_buf)
 				{
 					std::unique_ptr<overlay_element> image = std::make_unique<image_view>();
 					image->set_size(160, 110);
@@ -519,16 +520,14 @@ namespace rsx
 
 					std::unique_ptr<overlay_element> text_stack = std::make_unique<vertical_layout>();
 					std::unique_ptr<overlay_element> padding = std::make_unique<spacer>();
-					std::unique_ptr<overlay_element> header_text = std::make_unique<label>(text1);
+					std::unique_ptr<overlay_element> header_text = std::make_unique<label>(utf8_to_ascii8(text1));
-					std::unique_ptr<overlay_element> subtext = std::make_unique<label>(text2);
+					std::unique_ptr<overlay_element> subtext = std::make_unique<label>(utf8_to_ascii8(text2));
 
 					padding->set_size(1, 1);
 					header_text->set_size(800, 40);
-					header_text->set_text(text1);
 					header_text->set_font("Arial", 16);
 					header_text->set_wrap_text(true);
 					subtext->set_size(800, 40);
-					subtext->set_text(text2);
 					subtext->set_font("Arial", 14);
 					subtext->set_wrap_text(true);
 
@@ -634,7 +633,7 @@ namespace rsx
 				for (auto& entry : save_entries)
 				{
 					std::unique_ptr<overlay_element> e;
-					e = std::make_unique<save_dialog_entry>(entry.title.c_str(), (entry.subtitle + " - " + entry.details).c_str(), image_resource_id::raw_image, entry.iconBuf);
+					e = std::make_unique<save_dialog_entry>(entry.title, entry.subtitle + " - " + entry.details, image_resource_id::raw_image, entry.iconBuf);
 					entries.emplace_back(std::move(e));
 				}
 
@@ -702,12 +701,11 @@ namespace rsx
 
 				if (!m_list->m_items.size())
 				{
-					m_no_saves_text = std::make_unique<label>();
+					m_no_saves_text = std::make_unique<label>("There is no saved data.");
 					m_no_saves_text->set_font("Arial", 20);
 					m_no_saves_text->align_text(overlay_element::text_align::center);
 					m_no_saves_text->set_pos(m_list->x, m_list->y + m_list->h / 2);
 					m_no_saves_text->set_size(m_list->w, 30);
-					m_no_saves_text->set_text("There is no saved data.");
 					m_no_saves_text->back_color.a = 0;
 
 					m_no_saves = true;
@@ -909,7 +907,7 @@ namespace rsx
 				close();
 			}
 
-			s32 show(std::string text, const MsgDialogType &type, std::function<void(s32 status)> on_close)
+			s32 show(const std::string& text, const MsgDialogType &type, std::function<void(s32 status)> on_close)
 			{
 				num_progress_bars = type.progress_bar_count;
 				if (num_progress_bars)
@@ -929,7 +927,7 @@ namespace rsx
 					btn_cancel.translate(0, offset);
 				}
 
-				text_display.set_text(text.c_str());
+				text_display.set_text(utf8_to_ascii8(text));
 
 				u16 text_w, text_h;
 				text_display.measure_text(text_w, text_h);
@@ -1113,7 +1111,7 @@ namespace rsx
 				default: break;
 				}
 
-				trophy_message = "You have earned the " + trophy_message + " trophy\n" + trophy.name;
+				trophy_message = "You have earned the " + trophy_message + " trophy\n" + utf8_to_ascii8(trophy.name);
 				text_view.set_text(trophy_message);
 				text_view.auto_resize();