diff --git a/Source/qol/itemlabels.cpp b/Source/qol/itemlabels.cpp
index 7f749c3c6..ca97e1dfa 100644
--- a/Source/qol/itemlabels.cpp
+++ b/Source/qol/itemlabels.cpp
@@ -1,7 +1,8 @@
 #include "itemlabels.h"
 
+#include <algorithm>
+#include <limits>
 #include <string>
-#include <unordered_set>
 #include <vector>
 
 #include <fmt/format.h>
@@ -12,7 +13,6 @@
 #include "engine/render/clx_render.hpp"
 #include "gmenu.h"
 #include "inv.h"
-#include "itemlabels.h"
 #include "options.h"
 #include "qol/stash.h"
 #include "utils/format_int.hpp"
@@ -41,6 +41,31 @@ const int MarginX = 2;               // horizontal margins between text and edge
 const int MarginY = 1;               // vertical margins between text and edges of the label
 const int Height = 11 + MarginY * 2; // going above 13 scatters labels of items that are next to each other
 
+/**
+ * @brief The set of used X coordinates for a certain Y coordinate.
+ */
+class UsedX {
+public:
+	[[nodiscard]] bool contains(int val) const
+	{
+		return std::find(data_.begin(), data_.end(), val) != data_.end();
+	}
+
+	void insert(int val)
+	{
+		if (!contains(val))
+			data_.push_back(val);
+	}
+
+	void clear()
+	{
+		data_.clear();
+	}
+
+private:
+	std::vector<int> data_;
+};
+
 } // namespace
 
 void ToggleItemLabelHighlight()
@@ -116,33 +141,36 @@ void FillRect(const Surface &out, int x, int y, int width, int height, Uint8 col
 void DrawItemNameLabels(const Surface &out)
 {
 	isLabelHighlighted = false;
+	if (labelQueue.empty())
+		return;
+	UsedX usedX;
 
-	for (unsigned int i = 0; i < labelQueue.size(); ++i) {
-		std::unordered_set<int> backtrace;
+	for (unsigned i = 0; i < labelQueue.size(); ++i) {
+		usedX.clear();
 
 		bool canShow;
 		do {
 			canShow = true;
-			for (unsigned int j = 0; j < i; ++j) {
+			for (unsigned j = 0; j < i; ++j) {
 				ItemLabel &a = labelQueue[i];
 				ItemLabel &b = labelQueue[j];
 				if (abs(b.pos.y - a.pos.y) < Height + BorderY) {
-					int widthA = a.width + BorderX + MarginX * 2;
-					int widthB = b.width + BorderX + MarginX * 2;
+					const int widthA = a.width + BorderX + MarginX * 2;
+					const int widthB = b.width + BorderX + MarginX * 2;
 					int newpos = b.pos.x;
 					if (b.pos.x >= a.pos.x && b.pos.x - a.pos.x < widthA) {
 						newpos -= widthA;
-						if (backtrace.find(newpos) != backtrace.end())
+						if (usedX.contains(newpos))
 							newpos = b.pos.x + widthB;
 					} else if (b.pos.x < a.pos.x && a.pos.x - b.pos.x < widthB) {
 						newpos += widthB;
-						if (backtrace.find(newpos) != backtrace.end())
+						if (usedX.contains(newpos))
 							newpos = b.pos.x - widthA;
 					} else
 						continue;
 					canShow = false;
 					a.pos.x = newpos;
-					backtrace.insert(newpos);
+					usedX.insert(newpos);
 				}
 			}
 		} while (!canShow);