feat(gametext): decode UTF-8 in plain-text .str files

Core/GameEngine/Source/GameClient/GameText.cpp

@@ -665,6 +665,125 @@ void GameTextManager::reverseWord ( Char *file, Char *lp )
 
 }
 
+//-------------------------------------------------------------------------------------------------
+// UTF-8 decoding for plain-text .str files. The legacy path copies one byte per wide character,
+// which cannot represent anything outside Latin-1 and turns UTF-8 strings into mojibake. A string
+// is decoded as UTF-8 only when all of its non-ASCII bytes form well-formed sequences, so existing
+// ANSI text keeps its old byte-for-byte output.
+//-------------------------------------------------------------------------------------------------
+
+static Bool isValidUtf8String( const Char *text )
+{
+	const UnsignedByte *s = (const UnsignedByte *)text;
+	Bool hasMultiByte = FALSE;
+
+	while ( *s != 0 )
+	{
+		if ( *s < 0x80 )
+		{
+			++s;
+			continue;
+		}
+
+		Int len;
+		UnsignedInt code;
+		if ( ( *s & 0xE0 ) == 0xC0 )
+		{
+			len = 2;
+			code = *s & 0x1F;
+		}
+		else if ( ( *s & 0xF0 ) == 0xE0 )
+		{
+			len = 3;
+			code = *s & 0x0F;
+		}
+		else if ( ( *s & 0xF8 ) == 0xF0 )
+		{
+			len = 4;
+			code = *s & 0x07;
+		}
+		else
+		{
+			return FALSE;
+		}
+
+		for ( Int i = 1; i < len; ++i )
+		{
+			if ( ( s[i] & 0xC0 ) != 0x80 )
+			{
+				return FALSE;
+			}
+			code = ( code << 6 ) | ( s[i] & 0x3F );
+		}
+
+		// Reject overlong forms, UTF-16 surrogate halves and values beyond Unicode.
+		if ( ( len == 2 && code < 0x80 ) ||
+				 ( len == 3 && code < 0x800 ) ||
+				 ( len == 4 && code < 0x10000 ) ||
+				 code > 0x10FFFF ||
+				 ( code >= 0xD800 && code <= 0xDFFF ) )
+		{
+			return FALSE;
+		}
+
+		hasMultiByte = TRUE;
+		s += len;
+	}
+
+	return hasMultiByte;
+}
+
+//-------------------------------------------------------------------------------------------------
+// Writes the wide character(s) for one UTF-8 sequence and returns the number of bytes consumed.
+//-------------------------------------------------------------------------------------------------
+
+static Int decodeUtf8Char( const Char *in, WideChar **out )
+{
+	const UnsignedByte *s = (const UnsignedByte *)in;
+	UnsignedInt code;
+	Int len;
+
+	if ( ( s[0] & 0xE0 ) == 0xC0 )
+	{
+		code = s[0] & 0x1F;
+		len = 2;
+	}
+	else if ( ( s[0] & 0xF0 ) == 0xE0 )
+	{
+		code = s[0] & 0x0F;
+		len = 3;
+	}
+	else if ( ( s[0] & 0xF8 ) == 0xF0 )
+	{
+		code = s[0] & 0x07;
+		len = 4;
+	}
+	else
+	{
+		*( *out )++ = s[0];
+		return 1;
+	}
+
+	for ( Int i = 1; i < len; ++i )
+	{
+		code = ( code << 6 ) | ( s[i] & 0x3F );
+	}
+
+	// WideChar is UTF-16 on Windows, so characters outside the BMP need a surrogate pair there.
+	if ( sizeof( WideChar ) == 2 && code > 0xFFFF )
+	{
+		code -= 0x10000;
+		*( *out )++ = (WideChar)( 0xD800 + ( code >> 10 ) );
+		*( *out )++ = (WideChar)( 0xDC00 + ( code & 0x3FF ) );
+	}
+	else
+	{
+		*( *out )++ = (WideChar)code;
+	}
+
+	return len;
+}
+
 //============================================================================
 // GameTextManager::translateCopy
 //============================================================================
@@ -738,6 +857,8 @@ void GameTextManager::translateCopy( WideChar *outbuf, Char *inbuf )
 	}
 #endif
 
+	const Bool utf8 = isValidUtf8String( inbuf );
+
 	while( *inbuf != '\0' )
 	{
 		if( slash == TRUE )
@@ -776,13 +897,27 @@ void GameTextManager::translateCopy( WideChar *outbuf, Char *inbuf )
 					break;
 
 				default:
-					*outbuf++ = *inbuf & 0x00FF;
+					if ( utf8 && *(const UnsignedByte *)inbuf >= 0x80 )
+					{
+						inbuf += decodeUtf8Char( inbuf, &outbuf ) - 1;
+					}
+					else
+					{
+						*outbuf++ = *inbuf & 0x00FF;
+					}
 					break;
 			}
 		}
 		else if( *inbuf != '\\' )
 		{
-			*outbuf++ = *inbuf & 0x00FF;
+			if ( utf8 && *(const UnsignedByte *)inbuf >= 0x80 )
+			{
+				inbuf += decodeUtf8Char( inbuf, &outbuf ) - 1;
+			}
+			else
+			{
+				*outbuf++ = *inbuf & 0x00FF;
+			}
 		}
 		else
 			slash = TRUE;