Deduplicate some of the MPQ handling

Reuse libmpq functions where possible instead of our own.

.github/workflows/s390x_qemu_big_endian_tests.yml

@@ -34,7 +34,7 @@ jobs:
 
     - name: Run tests
       run: >
-        docker run --rm --interactive --mount type=bind,source=$(pwd),target=/host s390x/alpine sh -c
+        docker run run --platform linux/s390x --rm --interactive --mount type=bind,source=$(pwd),target=/host s390x/alpine sh -c
         "
         apk add --update-cache g++ ninja cmake ccache sdl2-dev sdl2_image-dev fmt-dev libpng-dev bzip2-dev gtest-dev wget &&
         cd /host &&

3rdParty/libmpq/CMakeLists.txt

@@ -10,8 +10,8 @@ include(functions/FetchContent_MakeAvailableExcludeFromAll)
 
 include(FetchContent)
 FetchContent_Declare(libmpq
-    URL https://github.com/diasurgical/libmpq/archive/34ace76ecb18c3c72300a6b8352ca0c96333488d.tar.gz
-    URL_HASH MD5=e76ab7e4f9dfd7f96c152dd7d8418fee
+    URL https://github.com/diasurgical/libmpq/archive/2cf61ebd1ce78082e20dc1e23bd9d66aec7fc0d5.tar.gz
+    URL_HASH MD5=073d634575465e57d679a7498a666fe3
 )
 FetchContent_MakeAvailableExcludeFromAll(libmpq)
 

Source/CMakeLists.txt

@@ -187,6 +187,7 @@ endif()
 if(SUPPORTS_MPQ)
   list(APPEND libdevilutionx_DEPS libmpq)
   list(APPEND libdevilutionx_SRCS
+    mpq/mpq_common.cpp
     mpq/mpq_reader.cpp
     mpq/mpq_sdl_rwops.cpp
     mpq/mpq_writer.cpp)

Source/encrypt.cpp

@@ -18,6 +18,14 @@ namespace devilution {
 
 namespace {
 
+struct TDataInfo {
+	std::byte *srcData;
+	uint32_t srcOffset;
+	std::byte *destData;
+	uint32_t destOffset;
+	uint32_t size;
+};
+
 unsigned int PkwareBufferRead(char *buf, unsigned int *size, void *param) // NOLINT(readability-non-const-parameter)
 {
 	auto *pInfo = reinterpret_cast<TDataInfo *>(param);
@@ -43,65 +51,8 @@ void PkwareBufferWrite(char *buf, unsigned int *size, void *param) // NOLINT(rea
 	pInfo->destOffset += *size;
 }
 
-const std::array<std::array<uint32_t, 256>, 5> hashtable = []() {
-	uint32_t seed = 0x00100001;
-	std::array<std::array<uint32_t, 256>, 5> ret = {};
-
-	for (int i = 0; i < 256; i++) {
-		for (int j = 0; j < 5; j++) { // NOLINT(modernize-loop-convert)
-			seed = (125 * seed + 3) % 0x2AAAAB;
-			uint32_t ch = (seed & 0xFFFF);
-			seed = (125 * seed + 3) % 0x2AAAAB;
-			ret[j][i] = ch << 16 | (seed & 0xFFFF);
-		}
-	}
-	return ret;
-}();
-
 } // namespace
 
-void Decrypt(uint32_t *castBlock, uint32_t size, uint32_t key)
-{
-	uint32_t seed = 0xEEEEEEEE;
-	for (uint32_t i = 0; i < (size >> 2); i++) {
-		uint32_t t = SDL_SwapLE32(*castBlock);
-		seed += hashtable[4][(key & 0xFF)];
-		t ^= seed + key;
-		*castBlock = t;
-		seed += t + (seed << 5) + 3;
-		castBlock++;
-		key = (((key << 0x15) ^ 0xFFE00000) + 0x11111111) | (key >> 0x0B);
-	}
-}
-
-void Encrypt(uint32_t *castBlock, uint32_t size, uint32_t key)
-{
-	uint32_t seed = 0xEEEEEEEE;
-	for (unsigned i = 0; i < (size >> 2); i++) {
-		uint32_t ch = *castBlock;
-		uint32_t t = ch;
-		seed += hashtable[4][(key & 0xFF)];
-		t ^= seed + key;
-		*castBlock = SDL_SwapLE32(t);
-		castBlock++;
-		seed += ch + (seed << 5) + 3;
-		key = (((key << 0x15) ^ 0xFFE00000) + 0x11111111) | (key >> 0x0B);
-	}
-}
-
-uint32_t Hash(const char *s, int type)
-{
-	uint32_t seed1 = 0x7FED7FED;
-	uint32_t seed2 = 0xEEEEEEEE;
-	while (s != nullptr && (*s != '\0')) {
-		int8_t ch = *s++;
-		ch = toupper(ch);
-		seed1 = hashtable[type][ch] ^ (seed1 + seed2);
-		seed2 += ch + seed1 + (seed2 << 5) + 3;
-	}
-	return seed1;
-}
-
 uint32_t PkwareCompress(std::byte *srcData, uint32_t size)
 {
 	std::unique_ptr<char[]> ptr = std::make_unique<char[]>(CMP_BUFFER_SIZE);

Source/encrypt.h

@@ -10,17 +10,6 @@
 
 namespace devilution {
 
-struct TDataInfo {
-	std::byte *srcData;
-	uint32_t srcOffset;
-	std::byte *destData;
-	uint32_t destOffset;
-	uint32_t size;
-};
-
-void Decrypt(uint32_t *castBlock, uint32_t size, uint32_t key);
-void Encrypt(uint32_t *castBlock, uint32_t size, uint32_t key);
-uint32_t Hash(const char *s, int type);
 uint32_t PkwareCompress(std::byte *srcData, uint32_t size);
 void PkwareDecompress(std::byte *inBuff, uint32_t recvSize, int maxBytes);
 

Source/engine/assets.cpp

@@ -55,7 +55,7 @@ SDL_RWops *OpenOptionalRWops(const std::string &path)
 
 bool FindMpqFile(std::string_view filename, MpqArchive **archive, uint32_t *fileNumber)
 {
-	const MpqArchive::FileHash fileHash = MpqArchive::CalculateFileHash(filename);
+	const MpqFileHash fileHash = CalculateMpqFileHash(filename);
 	const auto at = [=](std::optional<MpqArchive> &src) -> bool {
 		if (src && src->GetFileNumber(fileHash, *fileNumber)) {
 			*archive = &(*src);

Source/init.cpp

@@ -33,6 +33,7 @@
 #include "utils/utf8.hpp"
 
 #ifndef UNPACKED_MPQS
+#include "mpq/mpq_common.hpp"
 #include "mpq/mpq_reader.hpp"
 #endif
 
@@ -201,7 +202,7 @@ bool AreExtraFontsOutOfDate(const std::string &path)
 bool AreExtraFontsOutOfDate(MpqArchive &archive)
 {
 	const char filename[] = "fonts\\VERSION";
-	const MpqArchive::FileHash fileHash = MpqArchive::CalculateFileHash(filename);
+	const MpqFileHash fileHash = CalculateMpqFileHash(filename);
 	uint32_t fileNumber;
 	if (!archive.GetFileNumber(fileHash, fileNumber))
 		return true;

Source/mpq/mpq_common.cpp

@@ -0,0 +1,18 @@
+#include "mpq/mpq_common.hpp"
+
+#include <string_view>
+
+#include <libmpq/mpq.h>
+
+namespace devilution {
+
+#if !defined(UNPACKED_MPQS) || !defined(UNPACKED_SAVES)
+MpqFileHash CalculateMpqFileHash(std::string_view filename)
+{
+	MpqFileHash fileHash;
+	libmpq__file_hash_s(filename.data(), filename.size(), &fileHash[0], &fileHash[1], &fileHash[2]);
+	return fileHash;
+}
+#endif
+
+} // namespace devilution

Source/mpq/mpq_common.hpp

@@ -1,7 +1,9 @@
 #pragma once
 
+#include <array>
 #include <cstddef>
 #include <cstdint>
+#include <string_view>
 
 #include "utils/endian.hpp"
 
@@ -87,4 +89,10 @@ struct MpqBlockEntry {
 };
 #pragma pack(pop)
 
+using MpqFileHash = std::array<std::uint32_t, 3>;
+
+#if !defined(UNPACKED_MPQS) || !defined(UNPACKED_SAVES)
+MpqFileHash CalculateMpqFileHash(std::string_view filename);
+#endif
+
 } // namespace devilution

Source/mpq/mpq_reader.cpp

@@ -34,13 +34,6 @@ const char *MpqArchive::ErrorMessage(int32_t errorCode)
 	return libmpq__strerror(errorCode);
 }
 
-MpqArchive::FileHash MpqArchive::CalculateFileHash(std::string_view filename)
-{
-	FileHash fileHash;
-	libmpq__file_hash_s(filename.data(), filename.size(), &fileHash[0], &fileHash[1], &fileHash[2]);
-	return fileHash;
-}
-
 MpqArchive &MpqArchive::operator=(MpqArchive &&other) noexcept
 {
 	path_ = std::move(other.path_);
@@ -57,7 +50,7 @@ MpqArchive::~MpqArchive()
 		libmpq__archive_close(archive_);
 }
 
-bool MpqArchive::GetFileNumber(MpqArchive::FileHash fileHash, uint32_t &fileNumber)
+bool MpqArchive::GetFileNumber(MpqFileHash fileHash, uint32_t &fileNumber)
 {
 	return libmpq__file_number_from_hash(archive_, fileHash[0], fileHash[1], fileHash[2], &fileNumber) == 0;
 }

Source/mpq/mpq_reader.hpp

@@ -9,6 +9,8 @@
 #include <string_view>
 #include <vector>
 
+#include "mpq/mpq_common.hpp"
+
 // Forward-declare so that we can avoid exposing libmpq.
 struct mpq_archive;
 using mpq_archive_s = struct mpq_archive;
@@ -24,9 +26,6 @@ public:
 
 	static const char *ErrorMessage(int32_t errorCode);
 
-	using FileHash = std::array<std::uint32_t, 3>;
-	static FileHash CalculateFileHash(std::string_view filename);
-
 	MpqArchive(MpqArchive &&other) noexcept
 	    : path_(std::move(other.path_))
 	    , archive_(other.archive_)
@@ -40,7 +39,7 @@ public:
 	~MpqArchive();
 
 	// Returns false if the file does not exit.
-	bool GetFileNumber(FileHash fileHash, uint32_t &fileNumber);
+	bool GetFileNumber(MpqFileHash fileHash, uint32_t &fileNumber);
 
 	std::unique_ptr<std::byte[]> ReadFile(std::string_view filename, std::size_t &fileSize, int32_t &error);
 

Source/mpq/mpq_writer.cpp

@@ -6,6 +6,8 @@
 #include <memory>
 #include <type_traits>
 
+#include <libmpq/mpq.h>
+
 #include "appfat.h"
 #include "encrypt.h"
 #include "engine.h"
@@ -125,8 +127,7 @@ MpqWriter::MpqWriter(const char *path)
 				error = "Failed to read block table";
 				goto on_error;
 			}
-			uint32_t key = Hash("(block table)", 3);
-			Decrypt(reinterpret_cast<uint32_t *>(blockTable_.get()), fhdr.blockEntriesCount * sizeof(MpqBlockEntry), key);
+			libmpq__decrypt_block(reinterpret_cast<uint32_t *>(blockTable_.get()), fhdr.blockEntriesCount * sizeof(MpqBlockEntry), LIBMPQ_BLOCK_TABLE_HASH_KEY);
 		}
 		hashTable_ = std::make_unique<MpqHashEntry[]>(HashEntriesCount);
 
@@ -138,8 +139,7 @@ MpqWriter::MpqWriter(const char *path)
 				error = "Failed to read hash entries";
 				goto on_error;
 			}
-			uint32_t key = Hash("(hash table)", 3);
-			Decrypt(reinterpret_cast<uint32_t *>(hashTable_.get()), fhdr.hashEntriesCount * sizeof(MpqHashEntry), key);
+			libmpq__decrypt_block(reinterpret_cast<uint32_t *>(hashTable_.get()), fhdr.hashEntriesCount * sizeof(MpqHashEntry), LIBMPQ_HASH_TABLE_HASH_KEY);
 		}
 
 #ifndef CAN_SEEKP_BEYOND_EOF
@@ -179,9 +179,9 @@ MpqWriter::~MpqWriter()
 		LogVerbose("Closing failed {}", name_);
 }
 
-uint32_t MpqWriter::FetchHandle(const char *filename) const
+uint32_t MpqWriter::FetchHandle(std::string_view filename) const
 {
-	return GetHashIndex(Hash(filename, 0), Hash(filename, 1), Hash(filename, 2));
+	return GetHashIndex(CalculateMpqFileHash(filename));
 }
 
 void MpqWriter::InitDefaultMpqHeader(MpqFileHeader *hdr)
@@ -305,15 +305,15 @@ uint32_t MpqWriter::FindFreeBlock(uint32_t size)
 	return result;
 }
 
-uint32_t MpqWriter::GetHashIndex(uint32_t index, uint32_t hashA, uint32_t hashB) const // NOLINT(bugprone-easily-swappable-parameters)
+uint32_t MpqWriter::GetHashIndex(MpqFileHash fileHash) const // NOLINT(bugprone-easily-swappable-parameters)
 {
 	uint32_t i = HashEntriesCount;
-	for (unsigned idx = index & 0x7FF; hashTable_[idx].block != MpqHashEntry::NullBlock; idx = (idx + 1) & 0x7FF) {
+	for (unsigned idx = fileHash[0] & 0x7FF; hashTable_[idx].block != MpqHashEntry::NullBlock; idx = (idx + 1) & 0x7FF) {
 		if (i-- == 0)
 			break;
-		if (hashTable_[idx].hashA != hashA)
+		if (hashTable_[idx].hashA != fileHash[1])
 			continue;
-		if (hashTable_[idx].hashB != hashB)
+		if (hashTable_[idx].hashB != fileHash[2])
 			continue;
 		if (hashTable_[idx].block == MpqHashEntry::DeletedBlock)
 			continue;
@@ -329,14 +329,12 @@ bool MpqWriter::WriteHeaderAndTables()
 	return WriteHeader() && WriteBlockTable() && WriteHashTable();
 }
 
-MpqBlockEntry *MpqWriter::AddFile(const char *filename, MpqBlockEntry *block, uint32_t blockIndex)
+MpqBlockEntry *MpqWriter::AddFile(std::string_view filename, MpqBlockEntry *block, uint32_t blockIndex)
 {
-	uint32_t h1 = Hash(filename, 0);
-	uint32_t h2 = Hash(filename, 1);
-	uint32_t h3 = Hash(filename, 2);
-	if (GetHashIndex(h1, h2, h3) != HashEntryNotFound)
+	const MpqFileHash fileHash = CalculateMpqFileHash(filename);
+	if (GetHashIndex(fileHash) != HashEntryNotFound)
 		app_fatal(StrCat("Hash collision between \"", filename, "\" and existing file\n"));
-	unsigned int hIdx = h1 & 0x7FF;
+	unsigned int hIdx = fileHash[0] & 0x7FF;
 
 	bool hasSpace = false;
 	for (unsigned i = 0; i < HashEntriesCount; ++i) {
@@ -353,8 +351,8 @@ MpqBlockEntry *MpqWriter::AddFile(const char *filename, MpqBlockEntry *block, ui
 		block = NewBlock(&blockIndex);
 
 	MpqHashEntry &entry = hashTable_[hIdx];
-	entry.hashA = h2;
-	entry.hashB = h3;
+	entry.hashA = fileHash[1];
+	entry.hashB = fileHash[2];
 	entry.locale = 0;
 	entry.platform = 0;
 	entry.block = blockIndex;
@@ -362,15 +360,8 @@ MpqBlockEntry *MpqWriter::AddFile(const char *filename, MpqBlockEntry *block, ui
 	return block;
 }
 
-bool MpqWriter::WriteFileContents(const char *filename, const std::byte *fileData, size_t fileSize, MpqBlockEntry *block)
+bool MpqWriter::WriteFileContents(const std::byte *fileData, size_t fileSize, MpqBlockEntry *block)
 {
-	const char *tmp;
-	while ((tmp = strchr(filename, ':')) != nullptr)
-		filename = tmp + 1;
-	while ((tmp = strchr(filename, '\\')) != nullptr)
-		filename = tmp + 1;
-	Hash(filename, 3);
-
 	const uint32_t numSectors = (fileSize + (BlockSize - 1)) / BlockSize;
 	const uint32_t offsetTableByteSize = sizeof(uint32_t) * (numSectors + 1);
 	block->offset = FindFreeBlock(fileSize + offsetTableByteSize);
@@ -465,23 +456,23 @@ bool MpqWriter::WriteHeader()
 
 bool MpqWriter::WriteBlockTable()
 {
-	Encrypt(reinterpret_cast<uint32_t *>(blockTable_.get()), BlockEntrySize, Hash("(block table)", 3));
+	libmpq__encrypt_block(reinterpret_cast<uint32_t *>(blockTable_.get()), BlockEntrySize, LIBMPQ_BLOCK_TABLE_HASH_KEY);
 	const bool success = stream_.Write(reinterpret_cast<const char *>(blockTable_.get()), BlockEntrySize);
-	Decrypt(reinterpret_cast<uint32_t *>(blockTable_.get()), BlockEntrySize, Hash("(block table)", 3));
+	libmpq__decrypt_block(reinterpret_cast<uint32_t *>(blockTable_.get()), BlockEntrySize, LIBMPQ_BLOCK_TABLE_HASH_KEY);
 	return success;
 }
 
 bool MpqWriter::WriteHashTable()
 {
-	Encrypt(reinterpret_cast<uint32_t *>(hashTable_.get()), HashEntrySize, Hash("(hash table)", 3));
+	libmpq__encrypt_block(reinterpret_cast<uint32_t *>(hashTable_.get()), HashEntrySize, LIBMPQ_HASH_TABLE_HASH_KEY);
 	const bool success = stream_.Write(reinterpret_cast<const char *>(hashTable_.get()), HashEntrySize);
-	Decrypt(reinterpret_cast<uint32_t *>(hashTable_.get()), HashEntrySize, Hash("(hash table)", 3));
+	libmpq__decrypt_block(reinterpret_cast<uint32_t *>(hashTable_.get()), HashEntrySize, LIBMPQ_HASH_TABLE_HASH_KEY);
 	return success;
 }
 
-void MpqWriter::RemoveHashEntry(const char *filename)
+void MpqWriter::RemoveHashEntry(std::string_view filename)
 {
-	uint32_t hIdx = FetchHandle(filename);
+	const uint32_t hIdx = FetchHandle(filename);
 	if (hIdx == HashEntryNotFound) {
 		return;
 	}
@@ -504,20 +495,20 @@ void MpqWriter::RemoveHashEntries(bool (*fnGetName)(uint8_t, char *))
 	}
 }
 
-bool MpqWriter::WriteFile(const char *filename, const std::byte *data, size_t size)
+bool MpqWriter::WriteFile(std::string_view filename, const std::byte *data, size_t size)
 {
 	MpqBlockEntry *blockEntry;
 
 	RemoveHashEntry(filename);
 	blockEntry = AddFile(filename, nullptr, 0);
-	if (!WriteFileContents(filename, data, size, blockEntry)) {
+	if (!WriteFileContents(data, size, blockEntry)) {
 		RemoveHashEntry(filename);
 		return false;
 	}
 	return true;
 }
 
-void MpqWriter::RenameFile(const char *name, const char *newName) // NOLINT(bugprone-easily-swappable-parameters)
+void MpqWriter::RenameFile(std::string_view name, std::string_view newName) // NOLINT(bugprone-easily-swappable-parameters)
 {
 	uint32_t index = FetchHandle(name);
 	if (index == HashEntryNotFound) {
@@ -531,7 +522,7 @@ void MpqWriter::RenameFile(const char *name, const char *newName) // NOLINT(bugp
 	AddFile(newName, blockEntry, block);
 }
 
-bool MpqWriter::HasFile(const char *name) const
+bool MpqWriter::HasFile(std::string_view name) const
 {
 	return FetchHandle(name) != HashEntryNotFound;
 }

Source/mpq/mpq_writer.hpp

@@ -7,6 +7,7 @@
 
 #include <cstddef>
 #include <cstdint>
+#include <string_view>
 
 #include "mpq/mpq_common.hpp"
 #include "utils/logged_fstream.hpp"
@@ -23,21 +24,21 @@ public:
 	MpqWriter &operator=(MpqWriter &&other) = default;
 	~MpqWriter();
 
-	bool HasFile(const char *name) const;
+	bool HasFile(std::string_view name) const;
 
-	void RemoveHashEntry(const char *filename);
+	void RemoveHashEntry(std::string_view filename);
 	void RemoveHashEntries(bool (*fnGetName)(uint8_t, char *));
-	bool WriteFile(const char *filename, const std::byte *data, size_t size);
-	void RenameFile(const char *name, const char *newName);
+	bool WriteFile(std::string_view filename, const std::byte *data, size_t size);
+	void RenameFile(std::string_view name, std::string_view newName);
 
 private:
 	bool IsValidMpqHeader(MpqFileHeader *hdr) const;
-	uint32_t GetHashIndex(uint32_t index, uint32_t hashA, uint32_t hashB) const;
-	uint32_t FetchHandle(const char *filename) const;
+	uint32_t GetHashIndex(MpqFileHash fileHash) const;
+	uint32_t FetchHandle(std::string_view filename) const;
 
 	bool ReadMPQHeader(MpqFileHeader *hdr);
-	MpqBlockEntry *AddFile(const char *filename, MpqBlockEntry *block, uint32_t blockIndex);
-	bool WriteFileContents(const char *filename, const std::byte *fileData, size_t fileSize, MpqBlockEntry *block);
+	MpqBlockEntry *AddFile(std::string_view filename, MpqBlockEntry *block, uint32_t blockIndex);
+	bool WriteFileContents(const std::byte *fileData, size_t fileSize, MpqBlockEntry *block);
 
 	// Returns an unused entry in the block entry table.
 	MpqBlockEntry *NewBlock(uint32_t *blockIndex = nullptr);

tools/run_big_endian_tests.sh

@@ -12,7 +12,7 @@ fi
 
 # We disable ASAN and UBSAN for now because of:
 # https://gitlab.alpinelinux.org/alpine/aports/-/issues/14435
-docker run -u "$(id -u "$USER"):$(id -g "$USER")" --rm --mount "type=bind,source=${PWD},target=/host" devilutionx-s390x-test sh -c "cd /host && \
+docker run --platform linux/s390x -u "$(id -u "$USER"):$(id -g "$USER")" --rm --mount "type=bind,source=${PWD},target=/host" devilutionx-s390x-test sh -c "cd /host && \
 export CCACHE_DIR=/host/.s390x-ccache && \
 cmake -S. -Bbuild-s390x-test -G Ninja -DASAN=OFF -DUBSAN=OFF -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_COMPILER=/usr/bin/clang -DCMAKE_CXX_COMPILER=/usr/bin/clang++ \
   -DNONET=ON -DNOSOUND=ON && \