Support mods with UNPACKED_MPQS

Source/CMakeLists.txt

@@ -316,6 +316,7 @@ add_devilutionx_object_library(libdevilutionx_init
 target_link_dependencies(libdevilutionx_init PUBLIC
   libdevilutionx_assets
   libdevilutionx_config
+  libdevilutionx_mpq
   libdevilutionx_options
 )
 

Source/engine/assets.cpp

@@ -24,37 +24,23 @@
 namespace devilution {
 
 std::vector<std::string> OverridePaths;
-
+std::map<int, MpqArchiveT> MpqArchives;
-#ifdef UNPACKED_MPQS
-std::optional<std::string> spawn_data_path;
-std::optional<std::string> diabdat_data_path;
-std::optional<std::string> hellfire_data_path;
-std::optional<std::string> font_data_path;
-std::optional<std::string> lang_data_path;
-#else
-std::map<int, std::unique_ptr<MpqArchive>> MpqArchives;
 bool HasHellfireMpq;
-#endif
 
 namespace {
 
 #ifdef UNPACKED_MPQS
 char *FindUnpackedMpqFile(char *relativePath)
 {
+	// Iterate over archives in reverse order to prefer files from high priority archives
 	char *path = nullptr;
-	const auto at = [&](const std::optional<std::string> &unpackedDir) -> bool {
+	for (auto rit = MpqArchives.rbegin(); rit != MpqArchives.rend(); ++rit) {
-		if (!unpackedDir)
+		const std::string_view unpackedDir = rit->second;
-			return false;
+		path = relativePath - unpackedDir.size();
-		path = relativePath - unpackedDir->size();
+		std::memcpy(path, unpackedDir.data(), unpackedDir.size());
-		std::memcpy(path, unpackedDir->data(), unpackedDir->size());
+		if (FileExists(path)) break;
-		if (FileExists(path))
-			return true;
 		path = nullptr;
-		return false;
+	}
-	};
-	at(font_data_path) || at(lang_data_path)
-	    || (gbIsHellfire && at(hellfire_data_path))
-	    || at(spawn_data_path) || at(diabdat_data_path);
 	return path;
 }
 #else
@@ -78,8 +64,8 @@ bool FindMpqFile(std::string_view filename, MpqArchive **archive, uint32_t *file
 
 	// Iterate over archives in reverse order to prefer files from high priority archives
 	for (auto rit = MpqArchives.rbegin(); rit != MpqArchives.rend(); ++rit) {
-		if (rit->second->GetFileNumber(fileHash, *fileNumber)) {
+		if (rit->second.GetFileNumber(fileHash, *fileNumber)) {
-			*archive = rit->second.get();
+			*archive = &rit->second;
 			return true;
 		}
 	}
@@ -262,7 +248,7 @@ std::string FailedToOpenFileErrorMessage(std::string_view path, std::string_view
 
 namespace {
 #ifdef UNPACKED_MPQS
-std::optional<std::string> FindUnpackedMpqData(const std::vector<std::string> &paths, std::string_view mpqName)
+std::optional<std::string> FindUnpackedMpqData(std::span<const std::string> paths, std::string_view mpqName)
 {
 	std::string targetPath;
 	for (const std::string &path : paths) {
@@ -276,12 +262,28 @@ std::optional<std::string> FindUnpackedMpqData(const std::vector<std::string> &p
 	}
 	return std::nullopt;
 }
+
+bool FindMPQ(std::span<const std::string> paths, std::string_view mpqName)
+{
+	return FindUnpackedMpqData(paths, mpqName).has_value();
+}
+
+bool LoadMPQ(std::span<const std::string> paths, std::string_view mpqName, int priority)
+{
+	std::optional<std::string> mpqPath = FindUnpackedMpqData(paths, mpqName);
+	if (!mpqPath.has_value()) {
+		LogVerbose("Missing: {}", mpqName);
+		return false;
+	}
+	MpqArchives[priority] = *std::move(mpqPath);
+	return true;
+}
 #else
-bool FindMPQ(const std::vector<std::string> &paths, std::string_view mpqName)
+bool FindMPQ(std::span<const std::string> paths, std::string_view mpqName)
 {
 	std::string mpqAbsPath;
 	for (const auto &path : paths) {
-		mpqAbsPath = path + mpqName.data();
+		mpqAbsPath = StrCat(path, mpqName, ".mpq");
 		if (FileExists(mpqAbsPath)) {
 			LogVerbose("  Found: {} in {}", mpqName, path);
 			return true;
@@ -290,16 +292,21 @@ bool FindMPQ(const std::vector<std::string> &paths, std::string_view mpqName)
 
 	return false;
 }
-bool LoadMPQ(const std::vector<std::string> &paths, std::string_view mpqName, int priority)
+
+bool LoadMPQ(std::span<const std::string> paths, std::string_view mpqName, int priority, std::string_view ext = ".mpq")
 {
 	std::optional<MpqArchive> archive;
 	std::string mpqAbsPath;
 	std::int32_t error = 0;
 	for (const auto &path : paths) {
-		mpqAbsPath = path + mpqName.data();
+		mpqAbsPath = StrCat(path, mpqName, ext);
-		if ((archive = MpqArchive::Open(mpqAbsPath.c_str(), error))) {
+		archive = MpqArchive::Open(mpqAbsPath.c_str(), error);
+		if (archive.has_value()) {
 			LogVerbose("  Found: {} in {}", mpqName, path);
-			MpqArchives[priority] = std::make_unique<MpqArchive>(std::move(*archive));
+			auto [it, inserted] = MpqArchives.emplace(priority, *std::move(archive));
+			if (!inserted) {
+				LogError("MPQ with priority {} is already registered, skipping {}", priority, mpqName);
+			}
 			return true;
 		}
 		if (error != 0) {
@@ -376,92 +383,78 @@ void LoadCoreArchives()
 {
 	auto paths = GetMPQSearchPaths();
 
-#ifdef UNPACKED_MPQS
-	font_data_path = FindUnpackedMpqData(paths, "fonts");
-#else // !UNPACKED_MPQS
 #if !defined(__ANDROID__) && !defined(__APPLE__) && !defined(__3DS__) && !defined(__SWITCH__)
 	// Load devilutionx.mpq first to get the font file for error messages
-	LoadMPQ(paths, "devilutionx.mpq", DevilutionXMpqPriority);
+	LoadMPQ(paths, "devilutionx", DevilutionXMpqPriority);
-#endif
-	LoadMPQ(paths, "fonts.mpq", FontMpqPriority); // Extra fonts
-	HasHellfireMpq = FindMPQ(paths, "hellfire.mpq");
 #endif
+	LoadMPQ(paths, "fonts", FontMpqPriority); // Extra fonts
+	HasHellfireMpq = FindMPQ(paths, "hellfire");
 }
 
 void LoadLanguageArchive()
 {
-#ifdef UNPACKED_MPQS
+	const std::string_view code = GetLanguageCode();
-	lang_data_path = std::nullopt;
-#endif
-
-	std::string_view code = GetLanguageCode();
 	if (code != "en") {
-		std::string langMpqName { code };
+		LoadMPQ(GetMPQSearchPaths(), code, LangMpqPriority);
-#ifdef UNPACKED_MPQS
-		lang_data_path = FindUnpackedMpqData(GetMPQSearchPaths(), langMpqName);
-#else
-		langMpqName.append(".mpq");
-		LoadMPQ(GetMPQSearchPaths(), langMpqName, LangMpqPriority);
-#endif
 	}
 }
 
 void LoadGameArchives()
 {
-	auto paths = GetMPQSearchPaths();
+	const std::vector<std::string> paths = GetMPQSearchPaths();
-#ifdef UNPACKED_MPQS
+	bool haveDiabdat = false;
-	diabdat_data_path = FindUnpackedMpqData(paths, "diabdat");
+	bool haveSpawn = false;
-	if (!diabdat_data_path) {
+
-		spawn_data_path = FindUnpackedMpqData(paths, "spawn");
+#ifndef UNPACKED_MPQS
-		if (spawn_data_path)
+	// DIABDAT.MPQ is uppercase on the original CD and the GOG version.
-			gbIsSpawn = true;
+	haveDiabdat = LoadMPQ(paths, "DIABDAT", MainMpqPriority, ".MPQ");
-	}
+#endif
-	if (!HeadlessMode) {
+
-		AssetRef ref = FindAsset("ui_art\\title.clx");
+	if (!haveDiabdat) {
-		if (!ref.ok()) {
+		haveDiabdat = LoadMPQ(paths, "diabdat", MainMpqPriority);
-			LogError("{}", SDL_GetError());
+		if (!haveDiabdat) {
-			InsertCDDlg(_("diabdat.mpq or spawn.mpq"));
+			gbIsSpawn = haveSpawn = LoadMPQ(paths, "spawn", MainMpqPriority);
 		}
 	}
-	hellfire_data_path = FindUnpackedMpqData(paths, "hellfire");
-	if (forceHellfire && !hellfire_data_path)
-		InsertCDDlg("hellfire");
-#else // !UNPACKED_MPQS
-	if (!LoadMPQ(paths, "DIABDAT.MPQ", MainMpqPriority)) {
-		// DIABDAT.MPQ is uppercase on the original CD and the GOG version.
-		if (!LoadMPQ(paths, "diabdat.mpq", MainMpqPriority))
-			gbIsSpawn = LoadMPQ(paths, "spawn.mpq", MainMpqPriority);
-	}
 
 	if (!HeadlessMode) {
-		AssetRef ref = FindAsset("ui_art\\title.pcx");
+		if (!haveDiabdat && !haveSpawn) {
-		if (!ref.ok()) {
 			LogError("{}", SDL_GetError());
 			InsertCDDlg(_("diabdat.mpq or spawn.mpq"));
 		}
 	}
 
-	if (forceHellfire && !HasHellfireMpq)
+	if (forceHellfire && !HasHellfireMpq) {
+#ifdef UNPACKED_MPQS
+		InsertCDDlg("hellfire");
+#else
 		InsertCDDlg("hellfire.mpq");
-	LoadMPQ(paths, "hfbard.mpq", 8110);
+#endif
-	LoadMPQ(paths, "hfbarb.mpq", 8120);
+	}
+
+#ifndef UNPACKED_MPQS
+	// In unpacked mode, all the hellfire data is in the hellfire directory.
+	LoadMPQ(paths, "hfbard", 8110);
+	LoadMPQ(paths, "hfbarb", 8120);
 #endif
 }
 
 void LoadHellfireArchives()
 {
-	auto paths = GetMPQSearchPaths();
+	const std::vector<std::string> paths = GetMPQSearchPaths();
+	LoadMPQ(paths, "hellfire", 8000);
+
 #ifdef UNPACKED_MPQS
-	const bool hasMonk = FileExists(*hellfire_data_path + "plrgfx/monk/mha/mhaas.clx");
+	const std::string &hellfireDataPath = MpqArchives.at(8000);
-	const bool hasMusic = FileExists(*hellfire_data_path + "music/dlvlf.wav")
+	const bool hasMonk = FileExists(hellfireDataPath + "plrgfx/monk/mha/mhaas.clx");
-	    || FileExists(*hellfire_data_path + "music/dlvlf.mp3");
+	const bool hasMusic = FileExists(hellfireDataPath + "music/dlvlf.wav")
-	const bool hasVoice = FileExists(*hellfire_data_path + "sfx/hellfire/cowsut1.wav")
+	    || FileExists(hellfireDataPath + "music/dlvlf.mp3");
-	    || FileExists(*hellfire_data_path + "sfx/hellfire/cowsut1.mp3");
+	const bool hasVoice = FileExists(hellfireDataPath + "sfx/hellfire/cowsut1.wav")
-#else // !UNPACKED_MPQS
+	    || FileExists(hellfireDataPath + "sfx/hellfire/cowsut1.mp3");
-	LoadMPQ(paths, "hellfire.mpq", 8000);
+#else
-	const bool hasMonk = LoadMPQ(paths, "hfmonk.mpq", 8100);
+	const bool hasMonk = LoadMPQ(paths, "hfmonk", 8100);
-	const bool hasMusic = LoadMPQ(paths, "hfmusic.mpq", 8200);
+	const bool hasMusic = LoadMPQ(paths, "hfmusic", 8200);
-	const bool hasVoice = LoadMPQ(paths, "hfvoice.mpq", 8500);
+	const bool hasVoice = LoadMPQ(paths, "hfvoice", 8500);
 #endif
 
 	if (!hasMonk || !hasMusic || !hasVoice)
@@ -498,14 +491,12 @@ void LoadModArchives(std::span<const std::string_view> modnames)
 	}
 	OverridePaths.emplace_back(paths::PrefPath());
 
-#ifndef UNPACKED_MPQS
 	int priority = 10000;
 	auto paths = GetMPQSearchPaths();
 	for (std::string_view modname : modnames) {
-		LoadMPQ(paths, StrCat("mods" DIRECTORY_SEPARATOR_STR, modname, ".mpq"), priority);
+		LoadMPQ(paths, StrCat("mods" DIRECTORY_SEPARATOR_STR, modname), priority);
 		priority++;
 	}
-#endif
 }
 
 } // namespace devilution

Source/engine/assets.hpp

@@ -280,19 +280,17 @@ struct AssetData {
 tl::expected<AssetData, std::string> LoadAsset(std::string_view path);
 
 #ifdef UNPACKED_MPQS
-extern DVL_API_FOR_TEST std::optional<std::string> spawn_data_path;
+using MpqArchiveT = std::string;
-extern DVL_API_FOR_TEST std::optional<std::string> diabdat_data_path;
-extern std::optional<std::string> hellfire_data_path;
-extern std::optional<std::string> font_data_path;
-extern std::optional<std::string> lang_data_path;
 #else
-extern DVL_API_FOR_TEST std::map<int, std::unique_ptr<MpqArchive>> MpqArchives;
+using MpqArchiveT = MpqArchive;
+#endif
+
+extern DVL_API_FOR_TEST std::map<int, MpqArchiveT> MpqArchives;
 constexpr int MainMpqPriority = 1000;
 constexpr int DevilutionXMpqPriority = 9000;
 constexpr int LangMpqPriority = 9100;
 constexpr int FontMpqPriority = 9200;
 extern bool HasHellfireMpq;
-#endif
 
 void LoadCoreArchives();
 void LoadLanguageArchive();
@@ -301,19 +299,11 @@ void LoadHellfireArchives();
 void UnloadModArchives();
 void LoadModArchives(std::span<const std::string_view> modnames);
 
-#ifdef UNPACKED_MPQS
-#ifdef BUILD_TESTING
-[[nodiscard]] inline bool HaveMainData() { return diabdat_data_path.has_value() || spawn_data_path.has_value(); }
-#endif
-[[nodiscard]] inline bool HaveHellfire() { return hellfire_data_path.has_value(); }
-[[nodiscard]] inline bool HaveExtraFonts() { return font_data_path.has_value(); }
-#else
 #ifdef BUILD_TESTING
 [[nodiscard]] inline bool HaveMainData() { return MpqArchives.find(MainMpqPriority) != MpqArchives.end(); }
 #endif
-[[nodiscard]] inline bool HaveHellfire() { return HasHellfireMpq; }
 [[nodiscard]] inline bool HaveExtraFonts() { return MpqArchives.find(FontMpqPriority) != MpqArchives.end(); }
-#endif
+[[nodiscard]] inline bool HaveHellfire() { return HasHellfireMpq; }
 [[nodiscard]] inline bool HaveIntro() { return FindAsset("gendata\\diablo1.smk").ok(); }
 [[nodiscard]] inline bool HaveFullMusic() { return FindAsset("music\\dintro.wav").ok() || FindAsset("music\\dintro.mp3").ok(); }
 [[nodiscard]] inline bool HaveBardAssets() { return FindAsset("plrgfx\\bard\\bha\\bhaas.clx").ok(); }

Source/init.cpp

@@ -80,9 +80,9 @@ bool IsDevilutionXMpqOutOfDate()
 }
 
 #ifdef UNPACKED_MPQS
-bool AreExtraFontsOutOfDate(const std::string &path)
+bool AreExtraFontsOutOfDate(std::string_view path)
 {
-	const std::string versionPath = path + "fonts" DIRECTORY_SEPARATOR_STR "VERSION";
+	const std::string versionPath = StrCat(path, "fonts" DIRECTORY_SEPARATOR_STR "VERSION");
 	if (versionPath.size() + 1 > AssetRef::PathBufSize)
 		app_fatal("Path too long");
 	AssetRef ref;
@@ -105,6 +105,12 @@ bool AreExtraFontsOutOfDate(MpqArchive &archive)
 }
 #endif
 
+bool AreExtraFontsOutOfDate()
+{
+	const auto it = MpqArchives.find(FontMpqPriority);
+	return it != MpqArchives.end() && AreExtraFontsOutOfDate(it->second);
+}
+
 void init_cleanup()
 {
 	if (gbIsMultiplayer && gbRunGame) {
@@ -112,16 +118,8 @@ void init_cleanup()
 		sfile_write_stash();
 	}
 
-#ifdef UNPACKED_MPQS
-	lang_data_path = std::nullopt;
-	font_data_path = std::nullopt;
-	hellfire_data_path = std::nullopt;
-	diabdat_data_path = std::nullopt;
-	spawn_data_path = std::nullopt;
-#else
 	MpqArchives.clear();
 	HasHellfireMpq = false;
-#endif
 
 	NetClose();
 }

Source/init.h

@@ -5,13 +5,10 @@
  */
 #pragma once
 
-#include <optional>
+#include <SDL.h>
-
-#include "engine/assets.hpp"
-#include "utils/attributes.h"
 
 #ifdef UNPACKED_MPQS
-#include <string>
+#include <string_view>
 #else
 #include "mpq/mpq_reader.hpp"
 #endif
@@ -21,27 +18,16 @@ namespace devilution {
 /** True if the game is the current active window */
 extern bool gbActive;
 
+[[nodiscard]] bool IsDevilutionXMpqOutOfDate();
+
 #ifdef UNPACKED_MPQS
-bool AreExtraFontsOutOfDate(const std::string &path);
+bool AreExtraFontsOutOfDate(std::string_view path);
 #else
 bool AreExtraFontsOutOfDate(MpqArchive &archive);
 #endif
 
-inline bool AreExtraFontsOutOfDate()
+[[nodiscard]] bool AreExtraFontsOutOfDate();
-{
-#ifdef UNPACKED_MPQS
-	return font_data_path && AreExtraFontsOutOfDate(*font_data_path);
-#else
-	auto it = MpqArchives.find(FontMpqPriority);
-	if (it == MpqArchives.end()) {
-		return false;
-	}
-
-	return AreExtraFontsOutOfDate(*it->second);
-#endif
-}
 
-bool IsDevilutionXMpqOutOfDate();
 void init_cleanup();
 void init_create_window();
 void MainWndProc(const SDL_Event &event);

Source/menu.cpp

@@ -8,6 +8,7 @@
 
 #include "DiabloUI/diabloui.h"
 #include "DiabloUI/settingsmenu.h"
+#include "engine/assets.hpp"
 #include "engine/demomode.h"
 #include "game_mode.hpp"
 #include "init.h"

Source/options.cpp

@@ -949,6 +949,7 @@ void OptionEntryLanguageCode::CheckLanguagesAreInitialized() const
 {
 	if (!languages.empty())
 		return;
+	const bool haveExtraFonts = HaveExtraFonts();
 
 	// Add well-known supported languages
 	languages.emplace_back("bg", "Български");
@@ -964,7 +965,7 @@ void OptionEntryLanguageCode::CheckLanguagesAreInitialized() const
 	languages.emplace_back("hu", "Magyar");
 	languages.emplace_back("it", "Italiano");
 
-	if (HaveExtraFonts()) {
+	if (haveExtraFonts) {
 		languages.emplace_back("ja", "日本語");
 		languages.emplace_back("ko", "한국어");
 	}
@@ -977,7 +978,7 @@ void OptionEntryLanguageCode::CheckLanguagesAreInitialized() const
 	languages.emplace_back("tr", "Türkçe");
 	languages.emplace_back("uk", "Українська");
 
-	if (HaveExtraFonts()) {
+	if (haveExtraFonts) {
 		languages.emplace_back("zh_CN", "汉语");
 		languages.emplace_back("zh_TW", "漢語");
 	}

test/player_test.cpp

@@ -3,6 +3,7 @@
 #include <gtest/gtest.h>
 
 #include "cursor.h"
+#include "engine/assets.hpp"
 #include "init.h"
 #include "playerdat.hpp"
 

test/writehero_test.cpp

@@ -9,6 +9,7 @@
 #include <picosha2.h>
 
 #include "cursor.h"
+#include "engine/assets.hpp"
 #include "game_mode.hpp"
 #include "init.h"
 #include "loadsave.h"