diff --git a/Source/controls/touch/event_handlers.cpp b/Source/controls/touch/event_handlers.cpp
index ee0b4e8e4..8b2ce17cf 100644
--- a/Source/controls/touch/event_handlers.cpp
+++ b/Source/controls/touch/event_handlers.cpp
@@ -8,6 +8,7 @@
 #include "diablo.h"
 #include "engine.h"
 #include "gmenu.h"
+#include "inv.h"
 #include "scrollrt.h"
 #include "stores.h"
 #include "utils/ui_fwd.h"
@@ -30,9 +31,13 @@ Point ScaleToScreenCoordinates(float x, float y)
 
 void SimulateMouseMovement(const SDL_Event &event)
 {
-	float x = event.tfinger.x;
-	float y = event.tfinger.y;
-	MousePosition = ScaleToScreenCoordinates(x, y);
+	Point position = ScaleToScreenCoordinates(event.tfinger.x, event.tfinger.y);
+
+	if (!spselflag && invflag && !MainPanel.Contains(position) && !RightPanel.Contains(position))
+		return;
+
+	MousePosition = position;
+
 	sgbControllerActive = false;
 	InvalidateInventorySlot();
 }