#include "utils/language.h" #include #include #include #include #include #include #include "engine/assets.hpp" #include "options.h" #include "utils/algorithm/container.hpp" #include "utils/file_util.h" #include "utils/log.hpp" #include "utils/paths.h" #include "utils/string_view_hash.hpp" #ifdef USE_SDL1 #include "utils/sdl2_to_1_2_backports.h" #endif #define MO_MAGIC 0x950412de std::string forceLocale; namespace { using namespace devilution; // Translations normally come in ".gmo" files. // We also support ".mo" because that is what poedit generates // and what translators use to test their work. constexpr std::array Extensions { ".mo", ".gmo" }; std::unique_ptr translationKeys; std::unique_ptr translationValues; using TranslationRef = uint32_t; std::vector> translation = { {}, {} }; constexpr uint32_t TranslationRefOffsetBits = 19; constexpr uint32_t TranslationRefSizeBits = 32 - TranslationRefOffsetBits; // 13 constexpr uint32_t TranslationRefSizeMask = (1 << TranslationRefSizeBits) - 1; TranslationRef EncodeTranslationRef(uint32_t offset, uint32_t size) { return (offset << TranslationRefSizeBits) | size; } std::string_view GetTranslation(TranslationRef ref) { return { &translationValues[ref >> TranslationRefSizeBits], ref & TranslationRefSizeMask }; } } // namespace namespace { struct MoHead { uint32_t magic; struct { uint16_t major; uint16_t minor; } revision; uint32_t nbMappings; uint32_t srcOffset; uint32_t dstOffset; }; void SwapLE(MoHead &head) { head.magic = SDL_SwapLE32(head.magic); head.revision.major = SDL_SwapLE16(head.revision.major); head.revision.minor = SDL_SwapLE16(head.revision.minor); head.nbMappings = SDL_SwapLE32(head.nbMappings); head.srcOffset = SDL_SwapLE32(head.srcOffset); head.dstOffset = SDL_SwapLE32(head.dstOffset); } struct MoEntry { uint32_t length; uint32_t offset; }; void SwapLE(MoEntry &entry) { entry.length = SDL_SwapLE32(entry.length); entry.offset = SDL_SwapLE32(entry.offset); } std::string_view TrimLeft(std::string_view str) { str.remove_prefix(std::min(str.find_first_not_of(" \t"), str.size())); return str; } std::string_view TrimRight(std::string_view str) { str.remove_suffix(str.size() - (str.find_last_not_of(" \t") + 1)); return str; } /** plural=(n != 1); */ int PluralIfNotOne(int n) { return n != 1 ? 1 : 0; } // English, Danish, Spanish, Italian, Swedish unsigned PluralForms = 2; tl::function_ref GetLocalPluralId = PluralIfNotOne; /** * Match plural=(n != 1);" */ void SetPluralForm(std::string_view expression) { const std::string_view key = "plural="; const std::string_view::size_type keyPos = expression.find(key); if (keyPos == std::string_view::npos) return; expression.remove_prefix(keyPos + key.size()); const std::string_view::size_type semicolonPos = expression.find(';'); if (semicolonPos != std::string_view::npos) { expression.remove_suffix(expression.size() - semicolonPos); } expression = TrimLeft(TrimRight(expression)); // ko, zh_CN, zh_TW if (expression == "0") { GetLocalPluralId = [](int /*n*/) -> int { return 0; }; return; } // en, bg, da, de, es, it, sv if (expression == "(n != 1)") { GetLocalPluralId = PluralIfNotOne; return; } // fr, pt_BR if (expression == "(n > 1)") { GetLocalPluralId = [](int n) -> int { return n > 1 ? 1 : 0; }; return; } // hr, ru if (expression == "(n%10==1 && n%100!=11 ? 0 : n%10>=2 && n%10<=4 && (n%100<12 || n%100>14) ? 1 : 2)") { GetLocalPluralId = [](int n) -> int { if (n % 10 == 1 && n % 100 != 11) return 0; if (n % 10 >= 2 && n % 10 <= 4 && (n % 100 < 12 || n % 100 > 14)) return 1; return 2; }; return; } // pl if (expression == "(n==1 ? 0 : n%10>=2 && n%10<=4 && (n%100<12 || n%100>14) ? 1 : 2)") { GetLocalPluralId = [](int n) -> int { if (n == 1) return 0; if (n % 10 >= 2 && n % 10 <= 4 && (n % 100 < 12 || n % 100 > 14)) return 1; return 2; }; return; } // ro if (expression == "(n==1 ? 0 : n==0 || (n!=1 && n%100>=1 && n%100<=19) ? 1 : 2)") { GetLocalPluralId = [](int n) -> int { if (n == 1) return 0; if (n == 0 || (n != 1 && n % 100 >= 1 && n % 100 <= 19)) return 1; return 2; }; return; } // cs if (expression == "(n==1) ? 0 : (n>=2 && n<=4) ? 1 : 2") { GetLocalPluralId = [](int n) -> int { if (n == 1) return 0; if (n >= 2 && n <= 4) return 1; return 2; }; return; } LogError("Unknown plural expression: '{}'", expression); } /** * Parse "nplurals=2;" */ void ParsePluralForms(std::string_view string) { const std::string_view pluralsKey = "nplurals"; const std::string_view::size_type pluralsPos = string.find(pluralsKey); if (pluralsPos == std::string_view::npos) return; string.remove_prefix(pluralsPos + pluralsKey.size()); const std::string_view::size_type eqPos = string.find('='); if (eqPos == std::string_view::npos) return; std::string_view value = string.substr(eqPos + 1); if (value.empty() || value[0] < '0') return; const unsigned nplurals = value[0] - '0'; if (nplurals == 0) return; PluralForms = nplurals; SetPluralForm(value); } void ParseMetadata(std::string_view metadata) { std::string_view::size_type delim; while (!metadata.empty() && ((delim = metadata.find(':')) != std::string_view::npos)) { const std::string_view key = TrimLeft(std::string_view(metadata.data(), delim)); std::string_view val = TrimLeft(std::string_view(metadata.data() + delim + 1, metadata.size() - delim - 1)); if ((delim = val.find('\n')) != std::string_view::npos) { val = std::string_view(val.data(), delim); metadata.remove_prefix(val.data() - metadata.data() + val.size() + 1); } else { metadata.remove_prefix(metadata.size()); } // Match "Plural-Forms: nplurals=2; plural=(n != 1);" if (key == "Plural-Forms") { ParsePluralForms(val); break; } } } bool ReadEntry(AssetHandle &handle, const MoEntry &e, char *result) { if (!handle.seek(e.offset)) return false; result[e.length] = '\0'; return handle.read(result, e.length); } bool CopyData(void *dst, const std::byte *data, size_t dataSize, size_t offset, size_t length) { if (offset + length > dataSize) return false; memcpy(dst, data + offset, length); return true; } bool ReadEntry(const std::byte *data, size_t dataSize, const MoEntry &e, char *result) { if (!CopyData(result, data, dataSize, e.offset, e.length)) return false; result[e.length] = '\0'; return true; } } // namespace std::string_view LanguageParticularTranslate(std::string_view context, std::string_view message) { constexpr const char Glue = '\004'; std::string key = std::string(context); key.reserve(key.size() + 1 + message.size()); key += Glue; key.append(message); auto it = translation[0].find(key.c_str()); if (it == translation[0].end()) { return message; } return GetTranslation(it->second); } std::string_view LanguagePluralTranslate(const char *singular, std::string_view plural, int count) { int n = GetLocalPluralId(count); auto it = translation[n].find(singular); if (it == translation[n].end()) { if (count != 1) return plural; return singular; } return GetTranslation(it->second); } std::string_view LanguageTranslate(const char *key) { auto it = translation[0].find(key); if (it == translation[0].end()) { return key; } return GetTranslation(it->second); } bool HasTranslation(const std::string &locale) { if (locale == "en") { // the base translation is en (really en_US). No translation file will be present for this code but we want // the check to succeed to prevent further searches. return true; } return c_any_of(Extensions, [locale](const char *extension) { return FindAsset((locale + extension).c_str()).ok(); }); } std::string_view GetLanguageCode() { if (!forceLocale.empty()) return forceLocale; return *GetOptions().Language.code; } void LanguageInitialize() { translation = { {}, {} }; translationKeys = nullptr; translationValues = nullptr; const std::string lang(GetLanguageCode()); if (lang == "en") { // English does not have a translation file. GetLocalPluralId = PluralIfNotOne; return; } if (IsSmallFontTall() && !HaveExtraFonts()) { UiErrorOkDialog( "Missing fonts.mpq", StrCat("fonts.mpq is required for locale \"", GetLanguageCode(), "\"\n\n" "Please download fonts.mpq from:\n" "github.com/diasurgical/\ndevilutionx-assets/releases")); forceLocale = "en"; GetLocalPluralId = PluralIfNotOne; return; } AssetHandle handle; const uint32_t loadTranslationsStart = SDL_GetTicks(); std::string translationsPath; size_t fileSize; for (const char *ext : Extensions) { translationsPath = lang + ext; handle = OpenAsset(translationsPath.c_str(), fileSize); if (handle.ok()) break; } if (!handle.ok()) { // Reset to English, which is always available: forceLocale = "en"; GetLocalPluralId = PluralIfNotOne; return; } #ifdef UNPACKED_MPQS const bool readWholeFile = false; #else // If reading from an MPQ, it is much faster to // load the whole file instead of seeking. const bool readWholeFile = handle.handle->type == SDL_RWOPS_UNKNOWN; #endif std::unique_ptr data; if (readWholeFile) { data.reset(new std::byte[fileSize]); if (!handle.read(data.get(), fileSize)) return; handle = {}; } // Read header and do sanity checks MoHead head; if (readWholeFile ? !CopyData(&head, data.get(), fileSize, 0, sizeof(MoHead)) : !handle.read(&head, sizeof(MoHead))) { return; } SwapLE(head); if (head.magic != MO_MAGIC) { return; // not a MO file } if (head.revision.major > 1 || head.revision.minor > 1) { return; // unsupported revision } // Read entries of source strings std::unique_ptr src { new MoEntry[head.nbMappings] }; if (readWholeFile ? !CopyData(src.get(), data.get(), fileSize, head.srcOffset, head.nbMappings * sizeof(MoEntry)) : !handle.seek(head.srcOffset) || !handle.read(src.get(), head.nbMappings * sizeof(MoEntry))) { return; } for (size_t i = 0; i < head.nbMappings; ++i) { SwapLE(src[i]); } // Read entries of target strings std::unique_ptr dst { new MoEntry[head.nbMappings] }; if (readWholeFile ? !CopyData(dst.get(), data.get(), fileSize, head.dstOffset, head.nbMappings * sizeof(MoEntry)) : !handle.seek(head.dstOffset) || !handle.read(dst.get(), head.nbMappings * sizeof(MoEntry))) { return; } for (size_t i = 0; i < head.nbMappings; ++i) { SwapLE(dst[i]); } // MO header if (src[0].length != 0) { return; } { auto headerValue = std::unique_ptr { new char[dst[0].length + 1] }; if (readWholeFile ? !ReadEntry(data.get(), fileSize, dst[0], &headerValue[0]) : !ReadEntry(handle, dst[0], &headerValue[0])) { return; } ParseMetadata(&headerValue[0]); } translation.resize(PluralForms); for (unsigned i = 0; i < PluralForms; i++) translation[i] = {}; // Read strings described by entries size_t keysSize = 0; size_t valuesSize = 0; for (uint32_t i = 1; i < head.nbMappings; i++) { keysSize += src[i].length + 1; valuesSize += dst[i].length + 1; } translationKeys = std::unique_ptr { new char[keysSize] }; translationValues = std::unique_ptr { new char[valuesSize] }; char *keyPtr = &translationKeys[0]; char *valuePtr = &translationValues[0]; translation[0].reserve(head.nbMappings - 1); for (uint32_t i = 1; i < head.nbMappings; i++) { if (readWholeFile ? ReadEntry(data.get(), fileSize, src[i], keyPtr) && ReadEntry(data.get(), fileSize, dst[i], valuePtr) : ReadEntry(handle, src[i], keyPtr) && ReadEntry(handle, dst[i], valuePtr)) { // Plural keys also have a plural form but it does not participate in lookup. // Plural values are \0-terminated. std::string_view value { valuePtr, dst[i].length + 1 }; for (size_t j = 0; j < PluralForms && !value.empty(); j++) { const size_t formValueEnd = value.find('\0'); translation[j].emplace(keyPtr, EncodeTranslationRef(static_cast(value.data() - &translationValues[0]), static_cast(formValueEnd))); value.remove_prefix(formValueEnd + 1); } keyPtr += src[i].length + 1; valuePtr += dst[i].length + 1; } } LogVerbose(StrCat("Loaded translations from ", translationsPath, " in ", SDL_GetTicks() - loadTranslationsStart, "ms")); }