DS2Revival/mods/fixes/text.cpp

366 lines
19 KiB
C++

// DS2Fixes: the game's text under a UTF-8 (or otherwise unusual) system code page - Russian and other language packs.
//
// The game keeps its text as narrow strings in a legacy code page and converts them with MultiByteToWideChar /
// WideCharToMultiByte asking for CP_ACP (0), the system's "language for non-Unicode programs": through its helpers
// (Steam 0x5f8e11 / 0x5f8eff take the code page from the global at 0xbd95e0, 0 unless a LANGUAGE.DLL set it), the
// static C runtime and some direct calls. The bitmap fonts are indexed by the narrow byte, so a pack that ships its
// text as Windows-1251 bytes (the Russian pack has no LANGUAGE.DLL) works under any single-byte system code page: the
// bytes only travel narrow -> wide -> narrow and come back unchanged. What breaks it (docs/architecture/text-codepage.md):
// 1. A UTF-8 system code page (Windows 10/11 "Beta: Use Unicode UTF-8 for worldwide language support", ACP 65001):
// every byte of a Cyrillic letter is an invalid UTF-8 sequence, becomes U+FFFD and comes back as three bytes: every
// letter is drawn as three garbage glyphs, typed letters arrive as UTF-8 bytes one by one, a hero called "Ярик"
// cannot be saved ("Game ... failed to save"): the folder is made, the files in it are not.
// 2. A double-byte system code page (Japanese, Chinese, Korean) reads the same bytes as lead and trail bytes.
//
// [Fixes] TextCodepage = auto | off | <number>, default auto:
// - auto: nothing, unless the system code page is UTF-8; then 1252. The text is bytes, so the code page only has to be
// one that converts every byte both ways: 1252 is what a Western system uses anyway. (The user's own locale, 1251
// for a Russian one, would be wrong: wide characters above U+00FF make the C runtime's %S conversions fail.)
// - a number forces it, e.g. TextCodepage=1252 on a double-byte system for a Cyrillic pack. off = leave everything alone.
// With a text code page that differs from the system's:
// - MultiByteToWideChar / WideCharToMultiByte / GetACP ask for it instead of CP_ACP; CharNextA and IsDBCSLeadByte
// treat the text as single-byte;
// - the narrow file APIs, which would read a name in the system code page, go through their wide versions with the
// name read as the text code page (names that are UTF-8, from the system, are recognised per path component);
// - UTF-8 system only: a typed key is translated into a UTF-16 character by a hidden Unicode window (Windows gives an
// ANSI window the UTF-8 bytes one by one), converted into the text code page, else into the one of the keyboard
// layout (a Russian layout types Windows-1251 bytes, which the Cyrillic font draws) and posted as the single-byte
// WM_CHAR the game expects. With a single-byte system code page Windows already delivers the keyboard layout's code
// page (tested with a Russian layout on a Western system), so nothing is done there.
// A plain Western or Russian setup is not touched (the log says "game text code page 0").
#include "../common/plugin.h"
#include <string>
namespace {
UINT g_textCp; // 0 = no override
UINT g_systemAcp;
HWND g_proxy;
using MultiByteToWideChar_t = int(WINAPI*)(UINT, DWORD, LPCCH, int, LPWSTR, int);
using WideCharToMultiByte_t = int(WINAPI*)(UINT, DWORD, LPCWCH, int, LPSTR, int, LPCCH, LPBOOL);
using GetACP_t = UINT(WINAPI*)();
using CharNextA_t = LPSTR(WINAPI*)(LPCSTR);
using CharNextExA_t = LPSTR(WINAPI*)(WORD, LPCSTR, DWORD);
using IsDBCSLeadByte_t = BOOL(WINAPI*)(BYTE);
using TranslateMessage_t = BOOL(WINAPI*)(const MSG*);
MultiByteToWideChar_t g_MultiByteToWideChar;
WideCharToMultiByte_t g_WideCharToMultiByte;
GetACP_t g_GetACP;
CharNextA_t g_CharNextA;
CharNextExA_t g_CharNextExA;
IsDBCSLeadByte_t g_IsDBCSLeadByte;
TranslateMessage_t g_TranslateMessage;
bool SingleByte(UINT cp) {
CPINFO info;
return cp && cp != CP_UTF8 && cp != CP_UTF7 && GetCPInfo(cp, &info) && info.MaxCharSize == 1;
}
UINT LocaleAnsiCp(LCID lcid) {
char buf[16] = {};
if (!GetLocaleInfoA(lcid, LOCALE_IDEFAULTANSICODEPAGE, buf, sizeof(buf))) return 0;
return static_cast<UINT>(std::atoi(buf));
}
// The code page of the game's narrow text now: the one a LANGUAGE.DLL set (Steam: the global the game's conversion
// helpers read), else the text code page.
UINT TextCp() {
DWORD lang = (ds2::ExeKind() == ds2::Exe::Steam23) ? *reinterpret_cast<const DWORD*>(0xBD95E0) : 0;
return lang ? lang : g_textCp;
}
UINT Map(UINT cp) { return (cp == CP_ACP || cp == CP_THREAD_ACP) && g_textCp ? g_textCp : cp; }
int WINAPI HookMultiByteToWideChar(UINT cp, DWORD flags, LPCCH src, int n, LPWSTR dst, int cch) {
return g_MultiByteToWideChar(Map(cp), flags, src, n, dst, cch);
}
int WINAPI HookWideCharToMultiByte(UINT cp, DWORD flags, LPCWCH src, int n, LPSTR dst, int cb, LPCCH def, LPBOOL used) {
return g_WideCharToMultiByte(Map(cp), flags, src, n, dst, cb, def, used);
}
UINT WINAPI HookGetACP() { return g_textCp ? g_textCp : g_GetACP(); }
// Single-byte text has no lead bytes. (With ACP 65001 CharNext would step over whole UTF-8 sequences.)
LPSTR WINAPI HookCharNextA(LPCSTR p) {
if (!SingleByte(TextCp())) return g_CharNextA(p);
return const_cast<LPSTR>(p && *p ? p + 1 : p);
}
LPSTR WINAPI HookCharNextExA(WORD cp, LPCSTR p, DWORD flags) {
if (!SingleByte(TextCp())) return g_CharNextExA(cp, p, flags);
return const_cast<LPSTR>(p && *p ? p + 1 : p);
}
BOOL WINAPI HookIsDBCSLeadByte(BYTE b) { return SingleByte(TextCp()) ? FALSE : g_IsDBCSLeadByte(b); }
// --- Typed characters (UTF-8 system code page) ---------------------------------------------------------------
// One character message of the proxy window, forwarded to the game window as the byte(s) of the text code page.
void Forward(HWND game, const MSG& w, UINT textCp) {
wchar_t c = static_cast<wchar_t>(w.wParam);
if (c < 0x80 || w.message == WM_DEADCHAR || w.message == WM_SYSDEADCHAR) {
PostMessageA(game, w.message, w.wParam & 0xFF, w.lParam);
return;
}
// The text code page first, then the keyboard layout's, then the system's (not for UTF-8: it has no default character).
UINT tried[3] = {textCp, LocaleAnsiCp(MAKELCID(LOWORD(reinterpret_cast<UINT_PTR>(GetKeyboardLayout(0))), SORT_DEFAULT)), g_systemAcp};
BYTE out[4] = {};
for (UINT t : tried) {
if (!t || t == CP_UTF8) continue;
BOOL lost = FALSE;
int n = g_WideCharToMultiByte(t, 0, &c, 1, reinterpret_cast<LPSTR>(out), sizeof(out), nullptr, &lost);
if (n <= 0 || lost) continue;
for (int i = 0; i < n; ++i) PostMessageA(game, w.message, out[i], w.lParam);
return;
}
// a character no code page has: the game cannot show it, so it is dropped rather than typed as '?'
}
BOOL WINAPI HookTranslateMessage(const MSG* m) {
if (!m || (m->message != WM_KEYDOWN && m->message != WM_SYSKEYDOWN)) return g_TranslateMessage(m);
if (!g_proxy)
g_proxy = CreateWindowExW(0, L"STATIC", L"", 0, 0, 0, 0, 0, HWND_MESSAGE, nullptr, GetModuleHandleW(nullptr), nullptr);
if (!g_proxy) return g_TranslateMessage(m);
MSG copy = *m;
copy.hwnd = g_proxy; // a Unicode window: the character is generated as UTF-16 whatever the system code page is
BOOL r = g_TranslateMessage(&copy);
UINT cp = TextCp();
MSG w;
while (PeekMessageW(&w, g_proxy, WM_KEYFIRST, WM_KEYLAST, PM_REMOVE)) {
if (w.message == WM_CHAR || w.message == WM_SYSCHAR || w.message == WM_DEADCHAR || w.message == WM_SYSDEADCHAR)
Forward(m->hwnd, w, cp);
}
return r;
}
// --- File names ----------------------------------------------------------------------------------------------
// The game makes file names from its text (a hero called "Ярик" is a folder and a .ds2party of that name) as narrow
// strings in the text code page and hands them to the narrow file APIs, which read them in the system code page. With
// another system code page the folder is made (a wide call) and the files in it are not. The calls are redirected to
// their wide versions, reading the narrow name as the text code page, except that a path component which is valid UTF-8
// is taken as UTF-8: that is how the system itself hands out names (user profile, game folder) under ACP 65001.
// File names the game gets back (FindFirstFile, GetFullPathName...) are narrowed the same way, per component: in the
// text code page when the component fits, else UTF-8. ASCII-only names, the usual case, go to the original call.
bool AsciiOnly(const char* s) {
for (; *s; ++s)
if (static_cast<unsigned char>(*s) >= 0x80) return false;
return true;
}
bool ValidUtf8(const char* s, size_t n) {
return n == 0 || MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, s, static_cast<int>(n), nullptr, 0) > 0;
}
std::wstring Widen(const char* s) {
std::wstring out;
const char* p = s;
while (*p) {
const char* q = p;
while (*q && *q != '\\' && *q != '/') ++q;
size_t n = static_cast<size_t>(q - p);
if (n) {
UINT cp = ValidUtf8(p, n) ? CP_UTF8 : (TextCp() ? TextCp() : g_systemAcp);
int w = g_MultiByteToWideChar(cp, 0, p, static_cast<int>(n), nullptr, 0);
if (w > 0) {
size_t at = out.size();
out.resize(at + w);
g_MultiByteToWideChar(cp, 0, p, static_cast<int>(n), &out[at], w);
}
}
if (*q) out.push_back(static_cast<wchar_t>(*q++));
p = q;
}
return out;
}
std::string NarrowPath(const wchar_t* w) {
std::string out;
const wchar_t* p = w;
while (*p) {
const wchar_t* q = p;
while (*q && *q != L'\\' && *q != L'/') ++q;
int n = static_cast<int>(q - p);
if (n) {
char buf[1024];
BOOL lost = FALSE;
UINT cp = TextCp();
int len = cp && cp != CP_UTF8 ? g_WideCharToMultiByte(cp, 0, p, n, buf, sizeof(buf), nullptr, &lost) : 0;
if (len <= 0 || lost) len = g_WideCharToMultiByte(CP_UTF8, 0, p, n, buf, sizeof(buf), nullptr, nullptr);
if (len > 0) out.append(buf, static_cast<size_t>(len));
}
if (*q) out.push_back(static_cast<char>(*q++));
p = q;
}
return out;
}
using CreateFileA_t = HANDLE(WINAPI*)(LPCSTR, DWORD, DWORD, LPSECURITY_ATTRIBUTES, DWORD, DWORD, HANDLE);
using CreateDirectoryA_t = BOOL(WINAPI*)(LPCSTR, LPSECURITY_ATTRIBUTES);
using RemoveDirectoryA_t = BOOL(WINAPI*)(LPCSTR);
using DeleteFileA_t = BOOL(WINAPI*)(LPCSTR);
using MoveFileA_t = BOOL(WINAPI*)(LPCSTR, LPCSTR);
using CopyFileA_t = BOOL(WINAPI*)(LPCSTR, LPCSTR, BOOL);
using GetFileAttributesA_t = DWORD(WINAPI*)(LPCSTR);
using SetFileAttributesA_t = BOOL(WINAPI*)(LPCSTR, DWORD);
using FindFirstFileA_t = HANDLE(WINAPI*)(LPCSTR, LPWIN32_FIND_DATAA);
using FindNextFileA_t = BOOL(WINAPI*)(HANDLE, LPWIN32_FIND_DATAA);
using GetFullPathNameA_t = DWORD(WINAPI*)(LPCSTR, DWORD, LPSTR, LPSTR*);
using GetLongPathNameA_t = DWORD(WINAPI*)(LPCSTR, LPSTR, DWORD);
CreateFileA_t g_CreateFileA;
CreateDirectoryA_t g_CreateDirectoryA;
RemoveDirectoryA_t g_RemoveDirectoryA;
DeleteFileA_t g_DeleteFileA;
MoveFileA_t g_MoveFileA;
CopyFileA_t g_CopyFileA;
GetFileAttributesA_t g_GetFileAttributesA;
SetFileAttributesA_t g_SetFileAttributesA;
FindFirstFileA_t g_FindFirstFileA;
FindNextFileA_t g_FindNextFileA;
GetFullPathNameA_t g_GetFullPathNameA;
GetLongPathNameA_t g_GetLongPathNameA;
HANDLE WINAPI ShimCreateFileA(LPCSTR n, DWORD a, DWORD sh, LPSECURITY_ATTRIBUTES sa, DWORD d, DWORD f, HANDLE t) {
if (!n || AsciiOnly(n)) return g_CreateFileA(n, a, sh, sa, d, f, t);
return CreateFileW(Widen(n).c_str(), a, sh, sa, d, f, t);
}
BOOL WINAPI ShimCreateDirectoryA(LPCSTR n, LPSECURITY_ATTRIBUTES sa) {
if (!n || AsciiOnly(n)) return g_CreateDirectoryA(n, sa);
return CreateDirectoryW(Widen(n).c_str(), sa);
}
BOOL WINAPI ShimRemoveDirectoryA(LPCSTR n) {
if (!n || AsciiOnly(n)) return g_RemoveDirectoryA(n);
return RemoveDirectoryW(Widen(n).c_str());
}
BOOL WINAPI ShimDeleteFileA(LPCSTR n) {
if (!n || AsciiOnly(n)) return g_DeleteFileA(n);
return DeleteFileW(Widen(n).c_str());
}
BOOL WINAPI ShimMoveFileA(LPCSTR a, LPCSTR b) {
if (!a || !b || (AsciiOnly(a) && AsciiOnly(b))) return g_MoveFileA(a, b);
return MoveFileW(Widen(a).c_str(), Widen(b).c_str());
}
BOOL WINAPI ShimCopyFileA(LPCSTR a, LPCSTR b, BOOL fail) {
if (!a || !b || (AsciiOnly(a) && AsciiOnly(b))) return g_CopyFileA(a, b, fail);
return CopyFileW(Widen(a).c_str(), Widen(b).c_str(), fail);
}
DWORD WINAPI ShimGetFileAttributesA(LPCSTR n) {
if (!n || AsciiOnly(n)) return g_GetFileAttributesA(n);
return GetFileAttributesW(Widen(n).c_str());
}
BOOL WINAPI ShimSetFileAttributesA(LPCSTR n, DWORD attr) {
if (!n || AsciiOnly(n)) return g_SetFileAttributesA(n, attr);
return SetFileAttributesW(Widen(n).c_str(), attr);
}
DWORD CopyNarrow(const std::string& s, LPSTR buf, DWORD cch) {
if (s.size() + 1 > cch) return static_cast<DWORD>(s.size() + 1); // too small: the size needed, with the NUL
std::memcpy(buf, s.c_str(), s.size() + 1);
return static_cast<DWORD>(s.size());
}
DWORD WINAPI ShimGetFullPathNameA(LPCSTR n, DWORD cch, LPSTR buf, LPSTR* filePart) {
if (!n || AsciiOnly(n)) return g_GetFullPathNameA(n, cch, buf, filePart);
wchar_t w[2048];
DWORD len = GetFullPathNameW(Widen(n).c_str(), 2048, w, nullptr);
if (!len || len >= 2048) return 0;
std::string out = NarrowPath(w);
DWORD r = CopyNarrow(out, buf, cch);
if (r < cch && filePart) {
size_t slash = out.find_last_of("\\/");
*filePart = (slash == std::string::npos || slash + 1 >= out.size()) ? nullptr : buf + slash + 1;
}
return r;
}
DWORD WINAPI ShimGetLongPathNameA(LPCSTR n, LPSTR buf, DWORD cch) {
if (!n || AsciiOnly(n)) return g_GetLongPathNameA(n, buf, cch);
wchar_t w[2048];
DWORD len = GetLongPathNameW(Widen(n).c_str(), w, 2048);
if (!len || len >= 2048) return 0;
return CopyNarrow(NarrowPath(w), buf, cch);
}
void FillFindData(const WIN32_FIND_DATAW& w, LPWIN32_FIND_DATAA a) {
a->dwFileAttributes = w.dwFileAttributes;
a->ftCreationTime = w.ftCreationTime;
a->ftLastAccessTime = w.ftLastAccessTime;
a->ftLastWriteTime = w.ftLastWriteTime;
a->nFileSizeHigh = w.nFileSizeHigh;
a->nFileSizeLow = w.nFileSizeLow;
a->dwReserved0 = w.dwReserved0;
a->dwReserved1 = w.dwReserved1;
char alt[14] = {};
g_WideCharToMultiByte(CP_ACP, 0, w.cAlternateFileName, -1, alt, sizeof(alt) - 1, nullptr, nullptr);
lstrcpynA(a->cAlternateFileName, alt, sizeof(a->cAlternateFileName));
std::string name = NarrowPath(w.cFileName);
if (name.size() >= sizeof(a->cFileName)) name = alt; // too long for the narrow structure: the 8.3 name
lstrcpynA(a->cFileName, name.c_str(), sizeof(a->cFileName));
}
HANDLE WINAPI ShimFindFirstFileA(LPCSTR n, LPWIN32_FIND_DATAA data) {
// always through the wide call: even an ASCII pattern finds non-ASCII names, which the narrow call gives as UTF-8
WIN32_FIND_DATAW w;
HANDLE h = FindFirstFileW(Widen(n ? n : "").c_str(), &w);
if (h != INVALID_HANDLE_VALUE) FillFindData(w, data);
return h;
}
BOOL WINAPI ShimFindNextFileA(HANDLE h, LPWIN32_FIND_DATAA data) {
WIN32_FIND_DATAW w;
if (!FindNextFileW(h, &w)) return FALSE;
FillFindData(w, data);
return TRUE;
}
#define SHIM(name) g_##name = static_cast<name##_t>(ds2::HookImport("KERNEL32.dll", #name, reinterpret_cast<void*>(Shim##name), #name " (file names in the text code page)"))
void InstallFileShim() {
SHIM(CreateFileA);
SHIM(CreateDirectoryA);
SHIM(RemoveDirectoryA);
SHIM(DeleteFileA);
SHIM(MoveFileA);
SHIM(CopyFileA);
SHIM(GetFileAttributesA);
SHIM(SetFileAttributesA);
SHIM(FindFirstFileA);
SHIM(FindNextFileA);
SHIM(GetFullPathNameA);
SHIM(GetLongPathNameA);
}
} // namespace
// The code page the game's narrow text really uses: the text code page when one is set, else the system's. (The save
// folder check in fixes.cpp converts the documents path with it: under a UTF-8 system code page with TextCodepage=auto
// the game converts with 1252, so a Cyrillic or CJK user name is NOT safe for it.) Valid before InstallTextCodepage ran.
UINT GameTextCodepage() { return g_textCp ? g_textCp : (g_systemAcp ? g_systemAcp : GetACP()); }
void InstallTextCodepage(const ds2::Ini& ini) {
char setting[32];
ini.String("Fixes", "TextCodepage", "auto", setting, sizeof(setting));
g_systemAcp = GetACP();
bool off = _stricmp(setting, "off") == 0 || std::strcmp(setting, "0") == 0;
if (!off) {
if (_stricmp(setting, "auto") == 0) {
if (g_systemAcp == CP_UTF8) g_textCp = 1252;
} else {
g_textCp = static_cast<UINT>(std::atoi(setting));
if (!IsValidCodePage(g_textCp)) {
ds2::log().Write("SKIP text code page: %s is not an installed code page", setting);
g_textCp = 0;
}
}
}
ds2::log().Write("text: system code page %u, TextCodepage=%s -> game text code page %u", g_systemAcp, setting, g_textCp);
// The original conversions are needed by the redirected calls too, so they are always found.
HMODULE k = GetModuleHandleA("kernel32.dll");
g_WideCharToMultiByte = reinterpret_cast<WideCharToMultiByte_t>(GetProcAddress(k, "WideCharToMultiByte"));
g_MultiByteToWideChar = reinterpret_cast<MultiByteToWideChar_t>(GetProcAddress(k, "MultiByteToWideChar"));
if (!g_textCp) return;
auto hook = [](const char* dll, const char* name, void* fn, const char* what) { return ds2::HookImport(dll, name, fn, what); };
if (void* o = hook("KERNEL32.dll", "MultiByteToWideChar", reinterpret_cast<void*>(HookMultiByteToWideChar), "MultiByteToWideChar (text code page)"))
g_MultiByteToWideChar = static_cast<MultiByteToWideChar_t>(o);
if (void* o = hook("KERNEL32.dll", "WideCharToMultiByte", reinterpret_cast<void*>(HookWideCharToMultiByte), "WideCharToMultiByte (text code page)"))
g_WideCharToMultiByte = static_cast<WideCharToMultiByte_t>(o);
g_GetACP = static_cast<GetACP_t>(hook("KERNEL32.dll", "GetACP", reinterpret_cast<void*>(HookGetACP), "GetACP (text code page)"));
g_CharNextA = static_cast<CharNextA_t>(hook("USER32.dll", "CharNextA", reinterpret_cast<void*>(HookCharNextA), "CharNextA (single-byte text)"));
g_CharNextExA = static_cast<CharNextExA_t>(hook("USER32.dll", "CharNextExA", reinterpret_cast<void*>(HookCharNextExA), "CharNextExA (single-byte text)"));
g_IsDBCSLeadByte = static_cast<IsDBCSLeadByte_t>(hook("KERNEL32.dll", "IsDBCSLeadByte", reinterpret_cast<void*>(HookIsDBCSLeadByte), "IsDBCSLeadByte (single-byte text)"));
if (g_textCp != g_systemAcp) InstallFileShim();
if (g_systemAcp == CP_UTF8)
g_TranslateMessage = static_cast<TranslateMessage_t>(hook("USER32.dll", "TranslateMessage", reinterpret_cast<void*>(HookTranslateMessage),
"TranslateMessage (typed characters in the text code page)"));
}