diff --git a/Core/GameEngine/Source/GameClient/GameText.cpp b/Core/GameEngine/Source/GameClient/GameText.cpp index 9ada81db32c..aadab1ff03c 100644 --- a/Core/GameEngine/Source/GameClient/GameText.cpp +++ b/Core/GameEngine/Source/GameClient/GameText.cpp @@ -665,6 +665,125 @@ void GameTextManager::reverseWord ( Char *file, Char *lp ) } +//------------------------------------------------------------------------------------------------- +// UTF-8 decoding for plain-text .str files. The legacy path copies one byte per wide character, +// which cannot represent anything outside Latin-1 and turns UTF-8 strings into mojibake. A string +// is decoded as UTF-8 only when all of its non-ASCII bytes form well-formed sequences, so existing +// ANSI text keeps its old byte-for-byte output. +//------------------------------------------------------------------------------------------------- + +static Bool isValidUtf8String( const Char *text ) +{ + const UnsignedByte *s = (const UnsignedByte *)text; + Bool hasMultiByte = FALSE; + + while ( *s != 0 ) + { + if ( *s < 0x80 ) + { + ++s; + continue; + } + + Int len; + UnsignedInt code; + if ( ( *s & 0xE0 ) == 0xC0 ) + { + len = 2; + code = *s & 0x1F; + } + else if ( ( *s & 0xF0 ) == 0xE0 ) + { + len = 3; + code = *s & 0x0F; + } + else if ( ( *s & 0xF8 ) == 0xF0 ) + { + len = 4; + code = *s & 0x07; + } + else + { + return FALSE; + } + + for ( Int i = 1; i < len; ++i ) + { + if ( ( s[i] & 0xC0 ) != 0x80 ) + { + return FALSE; + } + code = ( code << 6 ) | ( s[i] & 0x3F ); + } + + // Reject overlong forms, UTF-16 surrogate halves and values beyond Unicode. + if ( ( len == 2 && code < 0x80 ) || + ( len == 3 && code < 0x800 ) || + ( len == 4 && code < 0x10000 ) || + code > 0x10FFFF || + ( code >= 0xD800 && code <= 0xDFFF ) ) + { + return FALSE; + } + + hasMultiByte = TRUE; + s += len; + } + + return hasMultiByte; +} + +//------------------------------------------------------------------------------------------------- +// Writes the wide character(s) for one UTF-8 sequence and returns the number of bytes consumed. +//------------------------------------------------------------------------------------------------- + +static Int decodeUtf8Char( const Char *in, WideChar **out ) +{ + const UnsignedByte *s = (const UnsignedByte *)in; + UnsignedInt code; + Int len; + + if ( ( s[0] & 0xE0 ) == 0xC0 ) + { + code = s[0] & 0x1F; + len = 2; + } + else if ( ( s[0] & 0xF0 ) == 0xE0 ) + { + code = s[0] & 0x0F; + len = 3; + } + else if ( ( s[0] & 0xF8 ) == 0xF0 ) + { + code = s[0] & 0x07; + len = 4; + } + else + { + *( *out )++ = s[0]; + return 1; + } + + for ( Int i = 1; i < len; ++i ) + { + code = ( code << 6 ) | ( s[i] & 0x3F ); + } + + // WideChar is UTF-16 on Windows, so characters outside the BMP need a surrogate pair there. + if ( sizeof( WideChar ) == 2 && code > 0xFFFF ) + { + code -= 0x10000; + *( *out )++ = (WideChar)( 0xD800 + ( code >> 10 ) ); + *( *out )++ = (WideChar)( 0xDC00 + ( code & 0x3FF ) ); + } + else + { + *( *out )++ = (WideChar)code; + } + + return len; +} + //============================================================================ // GameTextManager::translateCopy //============================================================================ @@ -738,6 +857,8 @@ void GameTextManager::translateCopy( WideChar *outbuf, Char *inbuf ) } #endif + const Bool utf8 = isValidUtf8String( inbuf ); + while( *inbuf != '\0' ) { if( slash == TRUE ) @@ -776,13 +897,27 @@ void GameTextManager::translateCopy( WideChar *outbuf, Char *inbuf ) break; default: - *outbuf++ = *inbuf & 0x00FF; + if ( utf8 && *(const UnsignedByte *)inbuf >= 0x80 ) + { + inbuf += decodeUtf8Char( inbuf, &outbuf ) - 1; + } + else + { + *outbuf++ = *inbuf & 0x00FF; + } break; } } else if( *inbuf != '\\' ) { - *outbuf++ = *inbuf & 0x00FF; + if ( utf8 && *(const UnsignedByte *)inbuf >= 0x80 ) + { + inbuf += decodeUtf8Char( inbuf, &outbuf ) - 1; + } + else + { + *outbuf++ = *inbuf & 0x00FF; + } } else slash = TRUE;