Add support for Unicode fonts

Source/DiabloUI/diabloui.cpp

@@ -21,7 +21,6 @@
 #include "utils/sdl_compat.h"
 #include "utils/sdl_wrap.h"
 #include "utils/stubs.h"
-#include "utils/utf8.h"
 #include "utils/language.h"
 
 #ifdef __SWITCH__
@@ -226,15 +225,13 @@ void UiFocusPageDown()
 
 void SelheroCatToName(char *inBuf, char *outBuf, int cnt)
 {
-	std::string output = utf8_to_latin1(inBuf);
-	strncat(outBuf, output.c_str(), cnt - strlen(outBuf));
+	strncat(outBuf, inBuf, cnt - strlen(outBuf));
 }
 
 #ifdef __vita__
 void selhero_SetName(char *in_buf, char *out_buf, int cnt)
 {
-	std::string output = utf8_to_latin1(in_buf);
-	strncpy(out_buf, output.c_str(), cnt);
+	strncpy(out_buf, in_buf, cnt);
 }
 #endif
 
@@ -544,14 +541,6 @@ void UiInitialize()
 {
 	LoadUiGFX();
 
-	LoadFont(GameFont12, ColorUiSilverDark);
-	LoadFont(GameFont12, ColorUiGoldDark);
-	LoadFont(GameFont24, ColorUiSilver);
-	LoadFont(GameFont24, ColorUiGold);
-	LoadFont(GameFont30, ColorUiSilver);
-	LoadFont(GameFont30, ColorUiGold);
-	LoadFont(GameFont42, ColorUiGold);
-
 	if (ArtCursor.surface != nullptr) {
 		if (SDL_ShowCursor(SDL_DISABLE) <= -1) {
 			ErrSdl();

Source/control.cpp

@@ -1662,13 +1662,13 @@ void DrawTalkPan(const Surface &out)
 		DrawPanelBox(out, { 180, sgbPlrTalkTbl + i + 70, i + 284, 1 }, { PANEL_X + 180, i + PANEL_Y + 54 });
 	}
 	DrawPanelBox(out, { 170, sgbPlrTalkTbl + 80, 310, 55 }, { PANEL_X + 170, PANEL_Y + 64 });
-	char *msg = TalkMessage;
 
 	int x = PANEL_LEFT + 200;
 	int y = PANEL_Y + 10;
 
-	int idx = DrawString(out, msg, { { x, y }, { 250, 27 } }, UiFlags::ColorWhite | UiFlags::PentaCursor, 1, 13);
-	msg[idx] = '\0';
+	int idx = DrawString(out, TalkMessage, { { x, y }, { 250, 27 } }, UiFlags::ColorWhite | UiFlags::PentaCursor, 1, 13);
+	if (idx < sizeof(TalkMessage))
+		TalkMessage[idx] = '\0';
 
 	x += 46;
 	int talkBtn = 0;

Source/diablo.cpp

@@ -1495,18 +1495,6 @@ void InitKeymapActions()
 #endif
 }
 
-void LoadGameFonts()
-{
-	LoadFont(GameFont12, ColorWhite);
-	LoadFont(GameFont12, ColorWhitegold);
-	LoadFont(GameFont12, ColorRed);
-	LoadFont(GameFont12, ColorBlue);
-	LoadFont(GameFont12, ColorBlack);
-	LoadFont(GameFont30, ColorGold);
-	LoadFont(GameFont46, ColorGold);
-	LoadFont(GameFont46, ColorBlack);
-}
-
 } // namespace
 
 void FreeGameMem()
@@ -1543,7 +1531,6 @@ bool StartGame(bool bNewGame, bool bSinglePlayer)
 		// Save 2.8 MiB of RAM by freeing all main menu resources
 		// before starting the game.
 		UiDestroy();
-		LoadGameFonts();
 
 		gbSelectProvider = false;
 

Source/engine/render/text_render.cpp

@@ -20,14 +20,14 @@
 #include "palette.h"
 #include "utils/display.h"
 #include "utils/sdl_compat.h"
+#include "utils/utf8.h"
 
 namespace devilution {
 
 namespace {
 
-std::unordered_map<uint32_t, std::unique_ptr<Art>> Fonts;
-
-std::array<std::array<uint8_t, 256>, 5> FontKerns;
+std::unordered_map<uint32_t, Art> Fonts;
+std::unordered_map<uint32_t, std::array<uint8_t, 256>> FontKerns;
 std::array<int, 5> FontSizes = { 12, 24, 30, 42, 46 };
 std::array<int, 5> LineHeights = { 12, 26, 38, 42, 50 };
 std::array<int, 5> BaseLineOffset = { -3, -2, -3, -6, -7 };
@@ -92,37 +92,59 @@ text_color GetColorFromFlags(UiFlags flags)
 	return ColorWhitegold;
 }
 
-} // namespace
+std::array<uint8_t, 256> *LoadFontKerning(GameFontTables size, uint16_t row)
+{
+	uint32_t fontId = (size << 16) | row;
+
+	auto hotKerning = FontKerns.find(fontId);
+	if (hotKerning != FontKerns.end()) {
+		return &hotKerning->second;
+	}
 
-void LoadFont(GameFontTables size, text_color color)
+	char path[32];
+	sprintf(path, "fonts\\%i-%02d.bin", FontSizes[size], row);
+
+	auto *kerning = &FontKerns[fontId];
+
+	LoadFileInMem(path, kerning);
+
+	return kerning;
+}
+
+Art *LoadFont(GameFontTables size, text_color color, uint16_t row)
 {
-	auto font = std::make_unique<Art>();
+	uint32_t fontId = (color << 24) | (size << 16) | row;
+
+	auto hotFont = Fonts.find(fontId);
+	if (hotFont != Fonts.end()) {
+		return &hotFont->second;
+	}
 
 	char path[32];
-	sprintf(path, "fonts\\%i-00.pcx", FontSizes[size]);
+	sprintf(path, "fonts\\%i-%02d.pcx", FontSizes[size], row);
+
+	auto *font = &Fonts[fontId];
 
 	if (ColorTranlations[color] != nullptr) {
 		std::array<uint8_t, 256> colorMapping;
 		LoadFileInMem(ColorTranlations[color], colorMapping);
-		LoadMaskedArt(path, font.get(), 256, 1, &colorMapping);
+		LoadMaskedArt(path, font, 256, 1, &colorMapping);
 	} else {
-		LoadMaskedArt(path, font.get(), 256, 1);
+		LoadMaskedArt(path, font, 256, 1);
 	}
 
-	uint32_t fontId = (color << 24) | (size << 16);
-	Fonts.insert(make_pair(fontId, move(font)));
-
-	sprintf(path, "fonts\\%i-00.bin", FontSizes[size]);
-	LoadFileInMem(path, FontKerns[size]);
+	return font;
 }
 
-void UnloadFont(GameFontTables size, text_color color)
+} // namespace
+
+void UnloadFonts(GameFontTables size, text_color color)
 {
-	uint32_t fontId = (color << 24) | (size << 16);
+	uint32_t fontStyle = (color << 24) | (size << 16);
 
 	for (auto font = Fonts.begin(); font != Fonts.end();) {
-		if ((font->first & 0xFFFF0000) == fontId) {
-			Fonts.erase(font++);
+		if ((font->first & 0xFFFF0000) == fontStyle) {
+			font = Fonts.erase(font);
 		} else {
 			font++;
 		}
@@ -132,19 +154,41 @@ void UnloadFont(GameFontTables size, text_color color)
 void UnloadFonts()
 {
 	Fonts.clear();
+	FontKerns.clear();
 }
 
 int GetLineWidth(string_view text, GameFontTables size, int spacing, int *charactersInLine)
 {
 	int lineWidth = 0;
 
+	std::string textBuffer(text);
+	textBuffer.resize(textBuffer.size() + 4); // Buffer must be padded before calling utf8_decode()
+	const char *textData = textBuffer.data();
+
 	size_t i = 0;
-	for (; i < text.length(); i++) {
-		if (text[i] == '\n')
+	uint32_t currentUnicodeRow = 0;
+	std::array<uint8_t, 256> *kerning = nullptr;
+	uint32_t next;
+	int error;
+	for (; *textData != '\0'; i++) {
+		textData = utf8_decode(textData, &next, &error);
+		if (error)
+			next = '?';
+
+		if (next == '\n')
 			break;
 
-		uint8_t frame = text[i] & 0xFF;
-		lineWidth += FontKerns[size][frame] + spacing;
+		uint8_t frame = next & 0xFF;
+		uint32_t unicodeRow = next >> 8;
+		if (unicodeRow != currentUnicodeRow || kerning == nullptr) {
+			kerning = LoadFontKerning(size, unicodeRow);
+			if (kerning == nullptr) {
+				continue;
+			}
+			currentUnicodeRow = unicodeRow;
+		}
+		lineWidth += (*kerning)[frame] + spacing;
+		i++;
 	}
 
 	if (charactersInLine != nullptr)
@@ -166,40 +210,56 @@ int AdjustSpacingToFitHorizontally(int &lineWidth, int maxSpacing, int character
 
 void WordWrapString(char *text, size_t width, GameFontTables size, int spacing)
 {
-	const size_t textLength = strlen(text);
-	size_t lineStart = 0;
+	int lastKnownSpaceAt = -1;
 	size_t lineWidth = 0;
-	for (unsigned i = 0; i < textLength; i++) {
-		if (text[i] == '\n') { // Existing line break, scan next line
-			lineStart = i + 1;
+
+	std::string textBuffer(text);
+	textBuffer.resize(textBuffer.size() + 4); // Buffer must be padded before calling utf8_decode()
+	const char *textData = textBuffer.data();
+
+	uint32_t currentUnicodeRow = 0;
+	std::array<uint8_t, 256> *kerning = nullptr;
+	uint32_t next;
+	int error;
+	while (*textData != '\0') {
+		textData = utf8_decode(textData, &next, &error);
+		if (error)
+			next = '?';
+
+		if (next == '\n') { // Existing line break, scan next line
+			lastKnownSpaceAt = -1;
 			lineWidth = 0;
 			continue;
 		}
 
-		uint8_t frame = text[i] & 0xFF;
-		lineWidth += FontKerns[size][frame] + spacing;
+		uint8_t frame = next & 0xFF;
+		uint32_t unicodeRow = next >> 8;
+		if (unicodeRow != currentUnicodeRow || kerning == nullptr) {
+			kerning = LoadFontKerning(size, unicodeRow);
+			if (kerning == nullptr) {
+				continue;
+			}
+			currentUnicodeRow = unicodeRow;
+		}
+		lineWidth += (*kerning)[frame] + spacing;
 
-		if (lineWidth - spacing <= width) {
-			continue; // String is still within the limit, continue to the next line
+		if (next == ' ') {
+			lastKnownSpaceAt = textData - textBuffer.data() - 1;
+			continue;
 		}
 
-		size_t j; // Backtrack to the previous space
-		for (j = i; j > lineStart; j--) {
-			if (text[j] == ' ') {
-				break;
-			}
+		if (lineWidth - spacing <= width) {
+			continue; // String is still within the limit, continue to the next symbol
 		}
 
-		if (j == lineStart) { // Single word longer than width
-			if (i == textLength)
-				break;
-			j = i;
+		if (lastKnownSpaceAt == -1) { // Single word longer than width
+			continue;
 		}
 
 		// Break line and continue to next line
-		i = j;
-		text[i] = '\n';
-		lineStart = i + 1;
+		text[lastKnownSpaceAt] = '\n';
+		textData = &textBuffer.data()[lastKnownSpaceAt + 1];
+		lastKnownSpaceAt = -1;
 		lineWidth = 0;
 	}
 }
@@ -240,52 +300,62 @@ uint32_t DrawString(const Surface &out, string_view text, const Rectangle &rect,
 
 	characterPosition.y += BaseLineOffset[size];
 
-	uint32_t fontId = (color << 24) | (size << 16);
-	auto font = Fonts.find(fontId);
-	if (font == Fonts.end()) {
-		Log("Font: size {} and color {} not loaded ", size, color);
-		return 0;
-	}
-
-	const auto &activeFont = font->second;
+	Art *font = nullptr;
+	std::array<uint8_t, 256> *kerning = nullptr;
+
+	std::string textBuffer(text);
+	textBuffer.resize(textBuffer.size() + 4); // Buffer must be padded before calling utf8_decode()
+	const char *textData = textBuffer.data();
+	const char *previousPosition = textData;
+
+	uint32_t next;
+	uint32_t currentUnicodeRow = 0;
+	int error;
+	for (; *textData != '\0'; previousPosition = textData) {
+		textData = utf8_decode(textData, &next, &error);
+		if (error)
+			next = '?';
+
+		uint32_t unicodeRow = next >> 8;
+		if (unicodeRow != currentUnicodeRow || font == nullptr) {
+			kerning = LoadFontKerning(size, unicodeRow);
+			font = LoadFont(size, color, unicodeRow);
+			currentUnicodeRow = unicodeRow;
+		}
 
-	uint32_t i = 0;
-	for (; i < text.length(); i++) {
-		uint8_t frame = text[i] & 0xFF;
-		if (text[i] == '\n' || characterPosition.x > rightMargin) {
+		uint8_t frame = next & 0xFF;
+		if (next == '\n' || characterPosition.x > rightMargin) {
 			if (characterPosition.y + lineHeight >= bottomMargin)
 				break;
+			characterPosition.x = rect.position.x;
 			characterPosition.y += lineHeight;
 
-			if (HasAnyOf(flags, (UiFlags::AlignCenter | UiFlags::AlignRight | UiFlags::KerningFitSpacing))) {
-				std::size_t nextLineIndex = text[i] == '\n' ? i + 1 : i;
-				if (nextLineIndex < text.length())
-					lineWidth = GetLineWidth(&text[nextLineIndex], size, spacing, &charactersInLine);
-				else
-					lineWidth = 0;
+			if (HasAnyOf(flags, (UiFlags::AlignCenter | UiFlags::AlignRight))) {
+				lineWidth = (*kerning)[frame];
+				if (*textData != '\0')
+					lineWidth += spacing + GetLineWidth(textData, size, spacing);
 			}
 
-			if (HasAnyOf(flags, UiFlags::KerningFitSpacing))
-				spacing = AdjustSpacingToFitHorizontally(lineWidth, maxSpacing, charactersInLine, rect.size.width);
-
-			characterPosition.x = rect.position.x;
 			if (HasAnyOf(flags, UiFlags::AlignCenter))
 				characterPosition.x += (rect.size.width - lineWidth) / 2;
 			else if (HasAnyOf(flags, UiFlags::AlignRight))
 				characterPosition.x += rect.size.width - lineWidth;
+
+			if (next == '\n')
+				continue;
 		}
-		DrawArt(out, characterPosition, activeFont.get(), frame);
-		if (text[i] != '\n')
-			characterPosition.x += FontKerns[size][frame] + spacing;
+
+		DrawArt(out, characterPosition, font, frame);
+		characterPosition.x += (*kerning)[frame] + spacing;
 	}
 
 	if (HasAnyOf(flags, UiFlags::PentaCursor)) {
 		CelDrawTo(out, characterPosition + Displacement { 0, lineHeight - BaseLineOffset[size] }, *pSPentSpn2Cels, PentSpn2Spin());
 	} else if (HasAnyOf(flags, UiFlags::TextCursor) && GetAnimationFrame(2, 500) != 0) {
-		DrawArt(out, characterPosition, activeFont.get(), '|');
+		DrawArt(out, characterPosition, LoadFont(size, color, 0), '|');
 	}
 
-	return i;
+	return previousPosition - textBuffer.data();
 }
 
 uint8_t PentSpn2Spin()

Source/engine/render/text_render.hpp

@@ -46,8 +46,7 @@ enum text_color : uint8_t {
 
 extern std::optional<CelSprite> pSPentSpn2Cels;
 
-void LoadFont(GameFontTables size, text_color color);
-void UnloadFont(GameFontTables size, text_color color);
+void UnloadFonts(GameFontTables size, text_color color);
 
 /**
  * @brief Calculate pixel width of first line of text, respecting kerning
@@ -77,7 +76,7 @@ void WordWrapString(char *text, size_t width, GameFontTables size = GameFont12,
  * @param spacing Additional space to add between characters.
  *                This value may be adjusted if the flag UIS_FIT_SPACING is passed in the flags parameter.
  * @param lineHeight Allows overriding the default line height, useful for multi-line strings.
- * @return The number of characters rendered, including characters "drawn" outside the buffer.
+ * @return The number of bytes rendered, including characters "drawn" outside the buffer.
  */
 uint32_t DrawString(const Surface &out, string_view text, const Rectangle &rect, UiFlags flags = UiFlags::None, int spacing = 1, int lineHeight = -1);
 

Source/help.cpp

@@ -101,7 +101,7 @@ std::vector<std::string> HelpTextLines;
 void InitHelp()
 {
 	HelpFlag = false;
-	char tempString[512];
+	char tempString[1024];
 
 	for (const auto *text : HelpText) {
 		strcpy(tempString, _(text));

Source/miniwin/misc_msg.cpp

@@ -538,8 +538,7 @@ bool FetchMessage_Real(tagMSG *lpMsg)
 		return FalseAvail("SDL_TEXTEDITING", e.edit.length);
 	case SDL_TEXTINPUT:
 		if (gbRunGame) {
-			std::string output = utf8_to_latin1(e.text.text);
-			control_new_text(output);
+			control_new_text(e.text.text);
 			break;
 		}
 		return FalseAvail("SDL_TEXTINPUT", e.text.windowID);

Source/panels/charpanel.cpp

@@ -203,7 +203,7 @@ void DrawShadowString(const Surface &out, const PanelEntry &entry)
 		return;
 
 	std::string text_tmp = _(entry.label.c_str());
-	char buffer[32];
+	char buffer[64];
 	int spacing = 0;
 	strcpy(buffer, text_tmp.c_str());
 	if (entry.labelLength > 0)

Source/panels/mainpanel.cpp

@@ -81,9 +81,6 @@ void LoadMainPanel()
 	if (SDLC_SetSurfaceColors(pBtmBuff->surface, PalSurface->format->palette) <= -1)
 		ErrSdl();
 
-	LoadFont(GameFont12, ColorButtonface);
-	LoadFont(GameFont12, ColorButtonpushed);
-
 	RenderMainButton(0, _("char"), 0);
 	RenderMainButton(1, _("quests"), 1);
 	RenderMainButton(2, _("map"), 1);
@@ -111,8 +108,8 @@ void LoadMainPanel()
 		DrawButtonText(talkSurface, _("voice"), { { 0, 33 }, { TalkButton.w(), 0 } }, UiFlags::ColorButtonpushed);
 	}
 
-	UnloadFont(GameFont12, ColorButtonface);
-	UnloadFont(GameFont12, ColorButtonpushed);
+	UnloadFonts(GameFont12, ColorButtonface);
+	UnloadFonts(GameFont12, ColorButtonpushed);
 
 	PanelButton.Unload();
 	PanelButtonGrime.Unload();

Source/utils/language.cpp

@@ -8,7 +8,6 @@
 #include "options.h"
 #include "utils/file_util.h"
 #include "utils/paths.h"
-#include "utils/utf8.h"
 
 using namespace devilution;
 #define MO_MAGIC 0x950412de
@@ -65,8 +64,6 @@ char *StrTrimRight(char *s)
 	return s;
 }
 
-bool IsUTF8 = true;
-
 // English, Danish, Spanish, Italian, Swedish
 int PluralForms = 2;
 std::function<int(int n)> GetLocalPluralId = [](int n) -> int { return n != 1 ? 1 : 0; };
@@ -176,9 +173,10 @@ void ParseMetadata(char *ptr)
 		val = StrTrimRight(val);
 		meta[key] = val;
 
-		// Match "Content-Type: text/plain; charset=UTF-8"
 		if ((strcmp("Content-Type", key) == 0) && ((delim = strstr(val, "=")) != nullptr)) {
-			IsUTF8 = (strcasecmp(delim + 1, "utf-8") == 0);
+			if (strcasecmp(delim + 1, "utf-8") != 0) {
+				Log("Translation is now UTF-8 encoded!");
+			}
 			continue;
 		}
 
@@ -211,7 +209,7 @@ const std::string &LanguageParticularTranslate(const char *context, const char *
 
 	auto it = translation[0].find(key);
 	if (it == translation[0].end()) {
-		it = translation[0].insert({ key, utf8_to_latin1(message) }).first;
+		it = translation[0].insert({ key, message }).first;
 	}
 
 	return it->second;
@@ -224,9 +222,9 @@ const std::string &LanguagePluralTranslate(const char *singular, const char *plu
 	auto it = translation[n].find(singular);
 	if (it == translation[n].end()) {
 		if (count != 1)
-			it = translation[1].insert({ singular, utf8_to_latin1(plural) }).first;
+			it = translation[1].insert({ singular, plural }).first;
 		else
-			it = translation[0].insert({ singular, utf8_to_latin1(singular) }).first;
+			it = translation[0].insert({ singular, singular }).first;
 	}
 
 	return it->second;
@@ -236,7 +234,7 @@ const std::string &LanguageTranslate(const char *key)
 {
 	auto it = translation[0].find(key);
 	if (it == translation[0].end()) {
-		it = translation[0].insert({ key, utf8_to_latin1(key) }).first;
+		it = translation[0].insert({ key, key }).first;
 	}
 
 	return it->second;
@@ -327,7 +325,7 @@ void LanguageInitialize()
 			size_t offset = 0;
 			for (int j = 0; j < PluralForms; j++) {
 				const char *text = value.data() + offset;
-				translation[j].emplace(key.data(), IsUTF8 ? utf8_to_latin1(text) : text);
+				translation[j].emplace(key.data(), text);
 
 				if (dst[i].length <= offset + strlen(value.data()))
 					break;

Source/utils/utf8.h

@@ -24,8 +24,7 @@
  * occurs, this pointer will be a guess that depends on the particular
  * error, but it will always advance at least one byte.
  */
-inline const unsigned char *
-utf8_decode(const unsigned char *buf, uint32_t *c, int *e)
+inline const char *utf8_decode(const char *buf, uint32_t *c, int *e)
 {
 	static const char lengths[] = {
 		1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
@@ -36,7 +35,7 @@ utf8_decode(const unsigned char *buf, uint32_t *c, int *e)
 	static const int shiftc[] = { 0, 18, 12, 6, 0 };
 	static const int shifte[] = { 0, 6, 4, 2, 0 };
 
-	const unsigned char *s = buf;
+	auto s = reinterpret_cast<const unsigned char *>(buf);
 	int len = lengths[s[0] >> 3];
 
 	/* Compute the pointer to the next character early so that the next
@@ -64,23 +63,5 @@ utf8_decode(const unsigned char *buf, uint32_t *c, int *e)
 	*e ^= 0x2a; // top two bits of each tail byte correct?
 	*e >>= shifte[len];
 
-	return next;
-}
-
-inline std::string utf8_to_latin1(const char *in)
-{
-	std::string instr(in);
-	instr.resize(instr.size() + 4);
-	const unsigned char *buf = reinterpret_cast<const unsigned char *>(instr.data());
-	std::string ret;
-	uint32_t next;
-	int error;
-	while (*buf) {
-		buf = utf8_decode(buf, &next, &error);
-		if (!error && next <= 255)
-			ret.push_back(static_cast<char>(next));
-		else
-			ret.push_back('?');
-	}
-	return ret;
+	return reinterpret_cast<const char *>(next);
 }