#include #include "data/file.hpp" #include "data/parser.hpp" #include #include #include "utils/paths.h" namespace devilution { auto LoadDataFile(std::string_view file) { std::string unitTestFolderCompletePath = paths::BasePath() + "/test/fixtures/"; paths::SetAssetsPath(unitTestFolderCompletePath); return DataFile::load(file); } void TestFileContents( const DataFile &dataFile, std::vector> expectedContent, GetFieldResult::Status expectedEndOfRecordStatus = GetFieldResult::Status::EndOfRecord, GetFieldResult::Status expectedEndOfFileStatus = GetFieldResult::Status::EndOfFile) { GetFieldResult result { dataFile.begin() }; unsigned row = 0; do { ASSERT_LT(row, expectedContent.size()) << "Too many records"; unsigned col = 0; do { ASSERT_LT(col, expectedContent[row].size()) << "Too many fields in record " << row; result = GetNextField(result.next, dataFile.end()); EXPECT_EQ(result.value, expectedContent[row][col]) << "Unexpected value at record " << row << " and field " << col; col++; } while (!result.endOfRecord()); if (!result.endOfFile()) EXPECT_EQ(result.status, expectedEndOfRecordStatus) << "Unexpected status when parsing the end of record " << row; EXPECT_EQ(col, expectedContent[row].size()) << "Parsing returned fewer fields than expected in record " << row; row++; } while (!result.endOfFile()); EXPECT_EQ(result.status, expectedEndOfFileStatus) << "Unexpected status when parsing the end of the file"; EXPECT_EQ(row, expectedContent.size()) << "Parsing returned fewer records than expected"; } TEST(DataFileTest, TryLoadMissingFile) { auto result = LoadDataFile("txtdata\\not_found.tsv"); EXPECT_FALSE(result.has_value()) << "Trying to load a non-existent file should return an unexpected/error response"; EXPECT_EQ(result.error(), DataFile::Error::NotFound) << "The error code should indicate the file was not found"; } TEST(DataFileTest, LoadCRFile) { auto result = LoadDataFile("txtdata\\cr.tsv"); ASSERT_TRUE(result.has_value()) << "Unable to load cr.tsv"; DataFile &dataFile = result.value(); EXPECT_EQ(dataFile.size(), 33) << "File size should be reported in code units"; std::vector> expectedFields { { "Test", "Empty", "Values" }, { "", "2", "3" }, { "1", "2", "" }, { "1", "", "3" } // because the file does not end with a newline parsing stops at the previous record }; TestFileContents(dataFile, expectedFields, GetFieldResult::Status::BadRecordSeparator, GetFieldResult::Status::FileTruncated); } TEST(DataFileTest, LoadWindowsFile) { auto result = LoadDataFile("txtdata\\crlf.tsv"); ASSERT_TRUE(result.has_value()) << "Unable to load crlf.tsv"; DataFile &dataFile = result.value(); EXPECT_EQ(dataFile.size(), 37) << "File size should be reported in code units"; std::vector> expectedFields { { "Test", "Empty", "Values" }, { "", "2", "3" }, { "1", "2", "" }, { "1", "", "3" }, { "" } // file ends with a newline, parsing returns a single empty field }; TestFileContents(dataFile, expectedFields); } TEST(DataFileTest, LoadTypicalFile) { auto result = LoadDataFile("txtdata\\lf.tsv"); ASSERT_TRUE(result.has_value()) << "Unable to load lf.tsv"; DataFile &dataFile = result.value(); EXPECT_EQ(dataFile.size(), 33) << "File size should be reported in code units"; std::vector> expectedFields { { "Test", "Empty", "Values" }, { "", "2", "3" }, { "1", "2", "" }, { "1", "", "3" }, { "" } // file ends with a newline, parsing returns a single empty field }; TestFileContents(dataFile, expectedFields); } TEST(DataFileTest, LoadUtf8WithBOM) { auto result = LoadDataFile("txtdata\\utf8_bom.tsv"); ASSERT_TRUE(result.has_value()) << "Unable to load utf8_bom.tsv"; DataFile &dataFile = result.value(); EXPECT_EQ(dataFile.size(), 33) << "Loading a file containing a UTF8 byte order marker should strip that prefix"; std::vector> expectedFields { { "Test", "Empty", "Values" }, { "", "2", "3" }, { "1", "2", "" }, { "1", "", "3" }, { "" } // file ends with a newline, parsing returns a single empty field }; TestFileContents(dataFile, expectedFields); } TEST(DataFileTest, IterateOverRecords) { auto result = LoadDataFile("txtdata\\lf.tsv"); ASSERT_TRUE(result.has_value()) << "Unable to load lf.tsv"; DataFile &dataFile = result.value(); std::vector> expectedFields { { "Test", "Empty", "Values" }, { "", "2", "3" }, { "1", "2", "" }, { "1", "", "3" }, { "" } // file ends with a newline, parsing returns a single empty field }; unsigned row = 0; for (FieldsInRecordRange record : dataFile.records()) { ASSERT_LT(row, expectedFields.size()) << "Too many records"; unsigned col = 0; for (std::string_view field : record) { if (col < expectedFields[row].size()) EXPECT_EQ(field, expectedFields[row][col]) << "Unexpected value at record " << row << " and field " << col; else ADD_FAILURE() << "Extra value '" << field << "' in record " << row << " at field " << col; col++; } EXPECT_GE(col, expectedFields[row].size()) << "Parsing returned fewer fields than expected in record " << row; row++; } EXPECT_EQ(row, expectedFields.size()) << "Parsing returned fewer records than expected"; } TEST(DataFileTest, DiscardAllAfterFirstField) { auto loadDataResult = LoadDataFile("txtdata\\lf.tsv"); ASSERT_TRUE(loadDataResult.has_value()) << "Unable to load lf.tsv"; DataFile &dataFile = loadDataResult.value(); std::array expectedFields { "Test", "", "1", "1", "" }; GetFieldResult result { dataFile.begin() }; unsigned row = 0; do { ASSERT_LT(row, expectedFields.size()) << "Too many records"; result = GetNextField(result.next, dataFile.end()); EXPECT_EQ(result.value, expectedFields[row]) << "Unexpected first value at record " << row; if (result.endOfRecord() && !result.endOfFile()) ADD_FAILURE() << "Parsing returned fewer fields than expected in record " << row; else result = DiscardRemainingFields(result.next, dataFile.end()); row++; } while (!result.endOfFile()); EXPECT_EQ(row, expectedFields.size()) << "Parsing returned fewer records than expected"; } TEST(DataFileTest, DiscardAllAfterFirstFieldIterator) { auto result = LoadDataFile("txtdata\\lf.tsv"); ASSERT_TRUE(result.has_value()) << "Unable to load lf.tsv"; DataFile &dataFile = result.value(); std::array expectedFields { "Test", "", "1", "1", "" }; unsigned row = 0; for (FieldsInRecordRange record : dataFile.records()) { ASSERT_LT(row, expectedFields.size()) << "Too many records"; for (std::string_view field : record) { EXPECT_EQ(field, expectedFields[row]) << "Unexpected first value at record " << row; break; } row++; } } TEST(DataFileTest, DiscardAllUpToLastField) { auto loadDataResult = LoadDataFile("txtdata\\lf.tsv"); ASSERT_TRUE(loadDataResult.has_value()) << "Unable to load lf.tsv"; DataFile &dataFile = loadDataResult.value(); std::array expectedFields { "Values", "3", "", "3", "" }; GetFieldResult result { dataFile.begin() }; unsigned row = 0; do { ASSERT_LT(row, expectedFields.size()) << "Too many records"; result = DiscardMultipleFields(result.next, dataFile.end(), 2); if (row < expectedFields.size() - 1) { EXPECT_FALSE(result.endOfRecord()) << "Parsing returned fewer fields than expected in record " << row; if (!result.endOfRecord()) result = GetNextField(result.next, dataFile.end()); } EXPECT_EQ(result.value, expectedFields[row]) << "Unexpected last value at record " << row; if (!result.endOfRecord()) { ADD_FAILURE() << "Parsing returned fewer fields than expected in record " << row; result = DiscardRemainingFields(result.next, dataFile.end()); } row++; } while (!result.endOfFile()); EXPECT_EQ(row, expectedFields.size()) << "Parsing returned fewer records than expected"; } TEST(DataFileTest, SkipFieldIterator) { auto loadDataResult = LoadDataFile("txtdata\\lf.tsv"); ASSERT_TRUE(loadDataResult.has_value()) << "Unable to load lf.tsv"; DataFile &dataFile = loadDataResult.value(); // we don't actually test the last value as incrementing the fields-in-record iterator past // the last field invalidates the value, we can't recover the same way the procedural interface does. std::array expectedFields { "Values", "3", "", "3", "" }; RecordsRange recordsRange = dataFile.records(); auto record = recordsRange.begin(); auto endRecord = recordsRange.end(); unsigned row = 0; while (record != endRecord) { ASSERT_LT(row, expectedFields.size()) << "Too many records"; FieldsInRecordRange fieldRange = *record; auto field = fieldRange.begin(); auto endField = fieldRange.end(); field += 2; if (row < expectedFields.size() - 1) { if (field == endField) { ADD_FAILURE() << "Parsing returned fewer fields than expected in record " << row; } else { EXPECT_EQ(*field, expectedFields[row]) << "Unexpected last value at record " << row; ++field; } } EXPECT_EQ(field, endField) << "Parsing returned more fields than expected in record " << row; ++row; ++record; } EXPECT_EQ(row, expectedFields.size()) << "Parsing returned fewer records than expected"; } TEST(DataFileTest, SkipRowIterator) { auto loadDataResult = LoadDataFile("txtdata\\lf.tsv"); ASSERT_TRUE(loadDataResult.has_value()) << "Unable to load lf.tsv"; DataFile &dataFile = loadDataResult.value(); std::vector> expectedFields { // skipping the first two lines { "1", "2", "" }, { "1", "", "3" }, { "" } // file ends with a newline, parsing returns a single empty field }; RecordsRange recordsRange = dataFile.records(); auto record = recordsRange.begin(); auto endRecord = recordsRange.end(); record += 2; unsigned row = 0; while (record != endRecord) { ASSERT_LT(row, expectedFields.size()) << "Too many records"; unsigned col = 0; for (std::string_view field : *record) { if (col < expectedFields[row].size()) EXPECT_EQ(field, expectedFields[row][col]) << "Unexpected value at record " << row << " and field " << col; else ADD_FAILURE() << "Extra value '" << field << "' in record " << row << " at field " << col; col++; } EXPECT_GE(col, expectedFields[row].size()) << "Parsing returned fewer fields than expected in record " << row; ++row; ++record; } EXPECT_EQ(row, expectedFields.size()) << "Parsing returned fewer records than expected"; } } // namespace devilution