diff --git a/.github/workflows/s390x_qemu_big_endian_tests.yml b/.github/workflows/s390x_qemu_big_endian_tests.yml index 821276e3a..71b42f9cd 100644 --- a/.github/workflows/s390x_qemu_big_endian_tests.yml +++ b/.github/workflows/s390x_qemu_big_endian_tests.yml @@ -34,7 +34,7 @@ jobs: - name: Run tests run: > - docker run --rm --interactive --mount type=bind,source=$(pwd),target=/host s390x/alpine sh -c + docker run run --platform linux/s390x --rm --interactive --mount type=bind,source=$(pwd),target=/host s390x/alpine sh -c " apk add --update-cache g++ ninja cmake ccache sdl2-dev sdl2_image-dev fmt-dev libpng-dev bzip2-dev gtest-dev wget && cd /host && diff --git a/3rdParty/libmpq/CMakeLists.txt b/3rdParty/libmpq/CMakeLists.txt index d4aef7564..0717f753f 100644 --- a/3rdParty/libmpq/CMakeLists.txt +++ b/3rdParty/libmpq/CMakeLists.txt @@ -10,8 +10,8 @@ include(functions/FetchContent_MakeAvailableExcludeFromAll) include(FetchContent) FetchContent_Declare(libmpq - URL https://github.com/diasurgical/libmpq/archive/34ace76ecb18c3c72300a6b8352ca0c96333488d.tar.gz - URL_HASH MD5=e76ab7e4f9dfd7f96c152dd7d8418fee + URL https://github.com/diasurgical/libmpq/archive/2cf61ebd1ce78082e20dc1e23bd9d66aec7fc0d5.tar.gz + URL_HASH MD5=073d634575465e57d679a7498a666fe3 ) FetchContent_MakeAvailableExcludeFromAll(libmpq) diff --git a/Source/CMakeLists.txt b/Source/CMakeLists.txt index 3a2a415ed..f83fe5e42 100644 --- a/Source/CMakeLists.txt +++ b/Source/CMakeLists.txt @@ -187,6 +187,7 @@ endif() if(SUPPORTS_MPQ) list(APPEND libdevilutionx_DEPS libmpq) list(APPEND libdevilutionx_SRCS + mpq/mpq_common.cpp mpq/mpq_reader.cpp mpq/mpq_sdl_rwops.cpp mpq/mpq_writer.cpp) diff --git a/Source/encrypt.cpp b/Source/encrypt.cpp index 7bbe61480..3af23e2af 100644 --- a/Source/encrypt.cpp +++ b/Source/encrypt.cpp @@ -18,6 +18,14 @@ namespace devilution { namespace { +struct TDataInfo { + std::byte *srcData; + uint32_t srcOffset; + std::byte *destData; + uint32_t destOffset; + uint32_t size; +}; + unsigned int PkwareBufferRead(char *buf, unsigned int *size, void *param) // NOLINT(readability-non-const-parameter) { auto *pInfo = reinterpret_cast(param); @@ -43,65 +51,8 @@ void PkwareBufferWrite(char *buf, unsigned int *size, void *param) // NOLINT(rea pInfo->destOffset += *size; } -const std::array, 5> hashtable = []() { - uint32_t seed = 0x00100001; - std::array, 5> ret = {}; - - for (int i = 0; i < 256; i++) { - for (int j = 0; j < 5; j++) { // NOLINT(modernize-loop-convert) - seed = (125 * seed + 3) % 0x2AAAAB; - uint32_t ch = (seed & 0xFFFF); - seed = (125 * seed + 3) % 0x2AAAAB; - ret[j][i] = ch << 16 | (seed & 0xFFFF); - } - } - return ret; -}(); - } // namespace -void Decrypt(uint32_t *castBlock, uint32_t size, uint32_t key) -{ - uint32_t seed = 0xEEEEEEEE; - for (uint32_t i = 0; i < (size >> 2); i++) { - uint32_t t = SDL_SwapLE32(*castBlock); - seed += hashtable[4][(key & 0xFF)]; - t ^= seed + key; - *castBlock = t; - seed += t + (seed << 5) + 3; - castBlock++; - key = (((key << 0x15) ^ 0xFFE00000) + 0x11111111) | (key >> 0x0B); - } -} - -void Encrypt(uint32_t *castBlock, uint32_t size, uint32_t key) -{ - uint32_t seed = 0xEEEEEEEE; - for (unsigned i = 0; i < (size >> 2); i++) { - uint32_t ch = *castBlock; - uint32_t t = ch; - seed += hashtable[4][(key & 0xFF)]; - t ^= seed + key; - *castBlock = SDL_SwapLE32(t); - castBlock++; - seed += ch + (seed << 5) + 3; - key = (((key << 0x15) ^ 0xFFE00000) + 0x11111111) | (key >> 0x0B); - } -} - -uint32_t Hash(const char *s, int type) -{ - uint32_t seed1 = 0x7FED7FED; - uint32_t seed2 = 0xEEEEEEEE; - while (s != nullptr && (*s != '\0')) { - int8_t ch = *s++; - ch = toupper(ch); - seed1 = hashtable[type][ch] ^ (seed1 + seed2); - seed2 += ch + seed1 + (seed2 << 5) + 3; - } - return seed1; -} - uint32_t PkwareCompress(std::byte *srcData, uint32_t size) { std::unique_ptr ptr = std::make_unique(CMP_BUFFER_SIZE); diff --git a/Source/encrypt.h b/Source/encrypt.h index cf53d3f31..66f841a3f 100644 --- a/Source/encrypt.h +++ b/Source/encrypt.h @@ -10,17 +10,6 @@ namespace devilution { -struct TDataInfo { - std::byte *srcData; - uint32_t srcOffset; - std::byte *destData; - uint32_t destOffset; - uint32_t size; -}; - -void Decrypt(uint32_t *castBlock, uint32_t size, uint32_t key); -void Encrypt(uint32_t *castBlock, uint32_t size, uint32_t key); -uint32_t Hash(const char *s, int type); uint32_t PkwareCompress(std::byte *srcData, uint32_t size); void PkwareDecompress(std::byte *inBuff, uint32_t recvSize, int maxBytes); diff --git a/Source/engine/assets.cpp b/Source/engine/assets.cpp index 19f0f7045..2af12afee 100644 --- a/Source/engine/assets.cpp +++ b/Source/engine/assets.cpp @@ -55,7 +55,7 @@ SDL_RWops *OpenOptionalRWops(const std::string &path) bool FindMpqFile(std::string_view filename, MpqArchive **archive, uint32_t *fileNumber) { - const MpqArchive::FileHash fileHash = MpqArchive::CalculateFileHash(filename); + const MpqFileHash fileHash = CalculateMpqFileHash(filename); const auto at = [=](std::optional &src) -> bool { if (src && src->GetFileNumber(fileHash, *fileNumber)) { *archive = &(*src); diff --git a/Source/init.cpp b/Source/init.cpp index ce3330599..654a96991 100644 --- a/Source/init.cpp +++ b/Source/init.cpp @@ -33,6 +33,7 @@ #include "utils/utf8.hpp" #ifndef UNPACKED_MPQS +#include "mpq/mpq_common.hpp" #include "mpq/mpq_reader.hpp" #endif @@ -201,7 +202,7 @@ bool AreExtraFontsOutOfDate(const std::string &path) bool AreExtraFontsOutOfDate(MpqArchive &archive) { const char filename[] = "fonts\\VERSION"; - const MpqArchive::FileHash fileHash = MpqArchive::CalculateFileHash(filename); + const MpqFileHash fileHash = CalculateMpqFileHash(filename); uint32_t fileNumber; if (!archive.GetFileNumber(fileHash, fileNumber)) return true; diff --git a/Source/mpq/mpq_common.cpp b/Source/mpq/mpq_common.cpp new file mode 100644 index 000000000..9a11a1179 --- /dev/null +++ b/Source/mpq/mpq_common.cpp @@ -0,0 +1,18 @@ +#include "mpq/mpq_common.hpp" + +#include + +#include + +namespace devilution { + +#if !defined(UNPACKED_MPQS) || !defined(UNPACKED_SAVES) +MpqFileHash CalculateMpqFileHash(std::string_view filename) +{ + MpqFileHash fileHash; + libmpq__file_hash_s(filename.data(), filename.size(), &fileHash[0], &fileHash[1], &fileHash[2]); + return fileHash; +} +#endif + +} // namespace devilution diff --git a/Source/mpq/mpq_common.hpp b/Source/mpq/mpq_common.hpp index 1fe0fbe5e..1eacece67 100644 --- a/Source/mpq/mpq_common.hpp +++ b/Source/mpq/mpq_common.hpp @@ -1,7 +1,9 @@ #pragma once +#include #include #include +#include #include "utils/endian.hpp" @@ -87,4 +89,10 @@ struct MpqBlockEntry { }; #pragma pack(pop) +using MpqFileHash = std::array; + +#if !defined(UNPACKED_MPQS) || !defined(UNPACKED_SAVES) +MpqFileHash CalculateMpqFileHash(std::string_view filename); +#endif + } // namespace devilution diff --git a/Source/mpq/mpq_reader.cpp b/Source/mpq/mpq_reader.cpp index 0d1a1c604..3f2144182 100644 --- a/Source/mpq/mpq_reader.cpp +++ b/Source/mpq/mpq_reader.cpp @@ -34,13 +34,6 @@ const char *MpqArchive::ErrorMessage(int32_t errorCode) return libmpq__strerror(errorCode); } -MpqArchive::FileHash MpqArchive::CalculateFileHash(std::string_view filename) -{ - FileHash fileHash; - libmpq__file_hash_s(filename.data(), filename.size(), &fileHash[0], &fileHash[1], &fileHash[2]); - return fileHash; -} - MpqArchive &MpqArchive::operator=(MpqArchive &&other) noexcept { path_ = std::move(other.path_); @@ -57,7 +50,7 @@ MpqArchive::~MpqArchive() libmpq__archive_close(archive_); } -bool MpqArchive::GetFileNumber(MpqArchive::FileHash fileHash, uint32_t &fileNumber) +bool MpqArchive::GetFileNumber(MpqFileHash fileHash, uint32_t &fileNumber) { return libmpq__file_number_from_hash(archive_, fileHash[0], fileHash[1], fileHash[2], &fileNumber) == 0; } diff --git a/Source/mpq/mpq_reader.hpp b/Source/mpq/mpq_reader.hpp index cbd9f7189..7a359fb40 100644 --- a/Source/mpq/mpq_reader.hpp +++ b/Source/mpq/mpq_reader.hpp @@ -9,6 +9,8 @@ #include #include +#include "mpq/mpq_common.hpp" + // Forward-declare so that we can avoid exposing libmpq. struct mpq_archive; using mpq_archive_s = struct mpq_archive; @@ -24,9 +26,6 @@ public: static const char *ErrorMessage(int32_t errorCode); - using FileHash = std::array; - static FileHash CalculateFileHash(std::string_view filename); - MpqArchive(MpqArchive &&other) noexcept : path_(std::move(other.path_)) , archive_(other.archive_) @@ -40,7 +39,7 @@ public: ~MpqArchive(); // Returns false if the file does not exit. - bool GetFileNumber(FileHash fileHash, uint32_t &fileNumber); + bool GetFileNumber(MpqFileHash fileHash, uint32_t &fileNumber); std::unique_ptr ReadFile(std::string_view filename, std::size_t &fileSize, int32_t &error); diff --git a/Source/mpq/mpq_writer.cpp b/Source/mpq/mpq_writer.cpp index dc6dc0c03..8a4c02a28 100644 --- a/Source/mpq/mpq_writer.cpp +++ b/Source/mpq/mpq_writer.cpp @@ -6,6 +6,8 @@ #include #include +#include + #include "appfat.h" #include "encrypt.h" #include "engine.h" @@ -125,8 +127,7 @@ MpqWriter::MpqWriter(const char *path) error = "Failed to read block table"; goto on_error; } - uint32_t key = Hash("(block table)", 3); - Decrypt(reinterpret_cast(blockTable_.get()), fhdr.blockEntriesCount * sizeof(MpqBlockEntry), key); + libmpq__decrypt_block(reinterpret_cast(blockTable_.get()), fhdr.blockEntriesCount * sizeof(MpqBlockEntry), LIBMPQ_BLOCK_TABLE_HASH_KEY); } hashTable_ = std::make_unique(HashEntriesCount); @@ -138,8 +139,7 @@ MpqWriter::MpqWriter(const char *path) error = "Failed to read hash entries"; goto on_error; } - uint32_t key = Hash("(hash table)", 3); - Decrypt(reinterpret_cast(hashTable_.get()), fhdr.hashEntriesCount * sizeof(MpqHashEntry), key); + libmpq__decrypt_block(reinterpret_cast(hashTable_.get()), fhdr.hashEntriesCount * sizeof(MpqHashEntry), LIBMPQ_HASH_TABLE_HASH_KEY); } #ifndef CAN_SEEKP_BEYOND_EOF @@ -179,9 +179,9 @@ MpqWriter::~MpqWriter() LogVerbose("Closing failed {}", name_); } -uint32_t MpqWriter::FetchHandle(const char *filename) const +uint32_t MpqWriter::FetchHandle(std::string_view filename) const { - return GetHashIndex(Hash(filename, 0), Hash(filename, 1), Hash(filename, 2)); + return GetHashIndex(CalculateMpqFileHash(filename)); } void MpqWriter::InitDefaultMpqHeader(MpqFileHeader *hdr) @@ -305,15 +305,15 @@ uint32_t MpqWriter::FindFreeBlock(uint32_t size) return result; } -uint32_t MpqWriter::GetHashIndex(uint32_t index, uint32_t hashA, uint32_t hashB) const // NOLINT(bugprone-easily-swappable-parameters) +uint32_t MpqWriter::GetHashIndex(MpqFileHash fileHash) const // NOLINT(bugprone-easily-swappable-parameters) { uint32_t i = HashEntriesCount; - for (unsigned idx = index & 0x7FF; hashTable_[idx].block != MpqHashEntry::NullBlock; idx = (idx + 1) & 0x7FF) { + for (unsigned idx = fileHash[0] & 0x7FF; hashTable_[idx].block != MpqHashEntry::NullBlock; idx = (idx + 1) & 0x7FF) { if (i-- == 0) break; - if (hashTable_[idx].hashA != hashA) + if (hashTable_[idx].hashA != fileHash[1]) continue; - if (hashTable_[idx].hashB != hashB) + if (hashTable_[idx].hashB != fileHash[2]) continue; if (hashTable_[idx].block == MpqHashEntry::DeletedBlock) continue; @@ -329,14 +329,12 @@ bool MpqWriter::WriteHeaderAndTables() return WriteHeader() && WriteBlockTable() && WriteHashTable(); } -MpqBlockEntry *MpqWriter::AddFile(const char *filename, MpqBlockEntry *block, uint32_t blockIndex) +MpqBlockEntry *MpqWriter::AddFile(std::string_view filename, MpqBlockEntry *block, uint32_t blockIndex) { - uint32_t h1 = Hash(filename, 0); - uint32_t h2 = Hash(filename, 1); - uint32_t h3 = Hash(filename, 2); - if (GetHashIndex(h1, h2, h3) != HashEntryNotFound) + const MpqFileHash fileHash = CalculateMpqFileHash(filename); + if (GetHashIndex(fileHash) != HashEntryNotFound) app_fatal(StrCat("Hash collision between \"", filename, "\" and existing file\n")); - unsigned int hIdx = h1 & 0x7FF; + unsigned int hIdx = fileHash[0] & 0x7FF; bool hasSpace = false; for (unsigned i = 0; i < HashEntriesCount; ++i) { @@ -353,8 +351,8 @@ MpqBlockEntry *MpqWriter::AddFile(const char *filename, MpqBlockEntry *block, ui block = NewBlock(&blockIndex); MpqHashEntry &entry = hashTable_[hIdx]; - entry.hashA = h2; - entry.hashB = h3; + entry.hashA = fileHash[1]; + entry.hashB = fileHash[2]; entry.locale = 0; entry.platform = 0; entry.block = blockIndex; @@ -362,15 +360,8 @@ MpqBlockEntry *MpqWriter::AddFile(const char *filename, MpqBlockEntry *block, ui return block; } -bool MpqWriter::WriteFileContents(const char *filename, const std::byte *fileData, size_t fileSize, MpqBlockEntry *block) +bool MpqWriter::WriteFileContents(const std::byte *fileData, size_t fileSize, MpqBlockEntry *block) { - const char *tmp; - while ((tmp = strchr(filename, ':')) != nullptr) - filename = tmp + 1; - while ((tmp = strchr(filename, '\\')) != nullptr) - filename = tmp + 1; - Hash(filename, 3); - const uint32_t numSectors = (fileSize + (BlockSize - 1)) / BlockSize; const uint32_t offsetTableByteSize = sizeof(uint32_t) * (numSectors + 1); block->offset = FindFreeBlock(fileSize + offsetTableByteSize); @@ -465,23 +456,23 @@ bool MpqWriter::WriteHeader() bool MpqWriter::WriteBlockTable() { - Encrypt(reinterpret_cast(blockTable_.get()), BlockEntrySize, Hash("(block table)", 3)); + libmpq__encrypt_block(reinterpret_cast(blockTable_.get()), BlockEntrySize, LIBMPQ_BLOCK_TABLE_HASH_KEY); const bool success = stream_.Write(reinterpret_cast(blockTable_.get()), BlockEntrySize); - Decrypt(reinterpret_cast(blockTable_.get()), BlockEntrySize, Hash("(block table)", 3)); + libmpq__decrypt_block(reinterpret_cast(blockTable_.get()), BlockEntrySize, LIBMPQ_BLOCK_TABLE_HASH_KEY); return success; } bool MpqWriter::WriteHashTable() { - Encrypt(reinterpret_cast(hashTable_.get()), HashEntrySize, Hash("(hash table)", 3)); + libmpq__encrypt_block(reinterpret_cast(hashTable_.get()), HashEntrySize, LIBMPQ_HASH_TABLE_HASH_KEY); const bool success = stream_.Write(reinterpret_cast(hashTable_.get()), HashEntrySize); - Decrypt(reinterpret_cast(hashTable_.get()), HashEntrySize, Hash("(hash table)", 3)); + libmpq__decrypt_block(reinterpret_cast(hashTable_.get()), HashEntrySize, LIBMPQ_HASH_TABLE_HASH_KEY); return success; } -void MpqWriter::RemoveHashEntry(const char *filename) +void MpqWriter::RemoveHashEntry(std::string_view filename) { - uint32_t hIdx = FetchHandle(filename); + const uint32_t hIdx = FetchHandle(filename); if (hIdx == HashEntryNotFound) { return; } @@ -504,20 +495,20 @@ void MpqWriter::RemoveHashEntries(bool (*fnGetName)(uint8_t, char *)) } } -bool MpqWriter::WriteFile(const char *filename, const std::byte *data, size_t size) +bool MpqWriter::WriteFile(std::string_view filename, const std::byte *data, size_t size) { MpqBlockEntry *blockEntry; RemoveHashEntry(filename); blockEntry = AddFile(filename, nullptr, 0); - if (!WriteFileContents(filename, data, size, blockEntry)) { + if (!WriteFileContents(data, size, blockEntry)) { RemoveHashEntry(filename); return false; } return true; } -void MpqWriter::RenameFile(const char *name, const char *newName) // NOLINT(bugprone-easily-swappable-parameters) +void MpqWriter::RenameFile(std::string_view name, std::string_view newName) // NOLINT(bugprone-easily-swappable-parameters) { uint32_t index = FetchHandle(name); if (index == HashEntryNotFound) { @@ -531,7 +522,7 @@ void MpqWriter::RenameFile(const char *name, const char *newName) // NOLINT(bugp AddFile(newName, blockEntry, block); } -bool MpqWriter::HasFile(const char *name) const +bool MpqWriter::HasFile(std::string_view name) const { return FetchHandle(name) != HashEntryNotFound; } diff --git a/Source/mpq/mpq_writer.hpp b/Source/mpq/mpq_writer.hpp index 6a6d25452..5114c51fb 100644 --- a/Source/mpq/mpq_writer.hpp +++ b/Source/mpq/mpq_writer.hpp @@ -7,6 +7,7 @@ #include #include +#include #include "mpq/mpq_common.hpp" #include "utils/logged_fstream.hpp" @@ -23,21 +24,21 @@ public: MpqWriter &operator=(MpqWriter &&other) = default; ~MpqWriter(); - bool HasFile(const char *name) const; + bool HasFile(std::string_view name) const; - void RemoveHashEntry(const char *filename); + void RemoveHashEntry(std::string_view filename); void RemoveHashEntries(bool (*fnGetName)(uint8_t, char *)); - bool WriteFile(const char *filename, const std::byte *data, size_t size); - void RenameFile(const char *name, const char *newName); + bool WriteFile(std::string_view filename, const std::byte *data, size_t size); + void RenameFile(std::string_view name, std::string_view newName); private: bool IsValidMpqHeader(MpqFileHeader *hdr) const; - uint32_t GetHashIndex(uint32_t index, uint32_t hashA, uint32_t hashB) const; - uint32_t FetchHandle(const char *filename) const; + uint32_t GetHashIndex(MpqFileHash fileHash) const; + uint32_t FetchHandle(std::string_view filename) const; bool ReadMPQHeader(MpqFileHeader *hdr); - MpqBlockEntry *AddFile(const char *filename, MpqBlockEntry *block, uint32_t blockIndex); - bool WriteFileContents(const char *filename, const std::byte *fileData, size_t fileSize, MpqBlockEntry *block); + MpqBlockEntry *AddFile(std::string_view filename, MpqBlockEntry *block, uint32_t blockIndex); + bool WriteFileContents(const std::byte *fileData, size_t fileSize, MpqBlockEntry *block); // Returns an unused entry in the block entry table. MpqBlockEntry *NewBlock(uint32_t *blockIndex = nullptr); diff --git a/tools/run_big_endian_tests.sh b/tools/run_big_endian_tests.sh index 7c50654e8..3458a6cf8 100755 --- a/tools/run_big_endian_tests.sh +++ b/tools/run_big_endian_tests.sh @@ -12,7 +12,7 @@ fi # We disable ASAN and UBSAN for now because of: # https://gitlab.alpinelinux.org/alpine/aports/-/issues/14435 -docker run -u "$(id -u "$USER"):$(id -g "$USER")" --rm --mount "type=bind,source=${PWD},target=/host" devilutionx-s390x-test sh -c "cd /host && \ +docker run --platform linux/s390x -u "$(id -u "$USER"):$(id -g "$USER")" --rm --mount "type=bind,source=${PWD},target=/host" devilutionx-s390x-test sh -c "cd /host && \ export CCACHE_DIR=/host/.s390x-ccache && \ cmake -S. -Bbuild-s390x-test -G Ninja -DASAN=OFF -DUBSAN=OFF -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_COMPILER=/usr/bin/clang -DCMAKE_CXX_COMPILER=/usr/bin/clang++ \ -DNONET=ON -DNOSOUND=ON && \