From 7a5ad4792aa85bd6e39dd2a5ee11fbb993ddb3a3 Mon Sep 17 00:00:00 2001
From: Gleb Mazovetskiy <glex.spb@gmail.com>
Date: Fri, 25 Nov 2022 14:31:04 +0000
Subject: [PATCH] A tool to measure performance using the time demo
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Example:

```
$ tools/measure_timedemo_performance.py -n 32 --binary build-reld-sdl1-nonet-nosound-unpk/devilutionx
Run  1 of 32: 	 6.02 seconds	170.8 FPS
Run  2 of 32: 	 6.22 seconds	165.5 FPS
Run  3 of 32: 	 6.01 seconds	171.1 FPS
Run  4 of 32: 	 6.03 seconds	170.6 FPS
Run  5 of 32: 	 6.05 seconds	170.2 FPS
Run  6 of 32: 	 6.04 seconds	170.3 FPS
Run  7 of 32: 	 6.03 seconds	170.5 FPS
Run  8 of 32: 	 6.03 seconds	170.5 FPS
Run  9 of 32: 	 6.01 seconds	171.1 FPS
Run 10 of 32: 	 6.04 seconds	170.3 FPS
Run 11 of 32: 	 6.03 seconds	170.7 FPS
Run 12 of 32: 	 6.03 seconds	170.7 FPS
Run 13 of 32: 	 6.04 seconds	170.3 FPS
Run 14 of 32: 	 6.03 seconds	170.6 FPS
Run 15 of 32: 	 6.04 seconds	170.3 FPS
Run 16 of 32: 	 6.04 seconds	170.4 FPS
Run 17 of 32: 	 6.03 seconds	170.6 FPS
Run 18 of 32: 	 6.03 seconds	170.5 FPS
Run 19 of 32: 	 6.06 seconds	169.9 FPS
Run 20 of 32: 	 6.04 seconds	170.3 FPS
Run 21 of 32: 	 6.03 seconds	170.7 FPS
Run 22 of 32: 	 6.02 seconds	171.0 FPS
Run 23 of 32: 	 6.02 seconds	170.8 FPS
Run 24 of 32: 	 6.02 seconds	170.8 FPS
Run 25 of 32: 	 6.04 seconds	170.3 FPS
Run 26 of 32: 	 6.03 seconds	170.8 FPS
Run 27 of 32: 	 6.04 seconds	170.4 FPS
Run 28 of 32: 	 6.07 seconds	169.4 FPS
Run 29 of 32: 	 6.03 seconds	170.7 FPS
Run 30 of 32: 	 5.99 seconds	171.7 FPS
Run 31 of 32: 	 6.01 seconds	171.1 FPS
Run 32 of 32: 	 6.06 seconds	169.9 FPS
6.038 ± 0.037 seconds, 170.400 ± 0.988 FPS
```
---
 tools/measure_timedemo_performance.py | 43 +++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)
 create mode 100755 tools/measure_timedemo_performance.py

diff --git a/tools/measure_timedemo_performance.py b/tools/measure_timedemo_performance.py
new file mode 100755
index 000000000..e2a053acc
--- /dev/null
+++ b/tools/measure_timedemo_performance.py
@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+
+import argparse
+import re
+import sys
+import statistics
+import subprocess
+from typing import NamedTuple
+
+_TIME_AND_FPS_REGEX = re.compile(rb'\d+ frames, (\d+(?:\.\d+)?) seconds: (\d+(?:\.\d+)?) fps')
+
+class RunMetrics(NamedTuple):
+	time: float
+	fps: float
+
+def measure(binary: str) -> RunMetrics:
+	result: subprocess.CompletedProcess = subprocess.run(
+		[binary, '--diablo', '--spawn', '--demo', '0', '--timedemo'], capture_output=True)
+	match = _TIME_AND_FPS_REGEX.search(result.stderr)
+	if not match:
+		raise Exception(f"Failed to parse output in:\n{result.stderr}")
+	return RunMetrics(float(match.group(1)), float(match.group(2)))
+
+
+def main():
+	parser = argparse.ArgumentParser()
+	parser.add_argument('--binary', help='Path to the devilutionx binary', required=True)
+	parser.add_argument('-n', '--num-runs', type=int, default=16, metavar='N')
+	args = parser.parse_args()
+
+	num_runs = args.num_runs
+	metrics = []
+	for i in range(1, num_runs + 1):
+		print(f"Run {i:>2} of {num_runs}: ", end='', file=sys.stderr, flush=True)
+		run_metrics = measure(args.binary)
+		print(f"\t{run_metrics.time:>5.2f} seconds\t{run_metrics.fps:>5.1f} FPS", file=sys.stderr, flush=True)
+		metrics.append(run_metrics)
+
+	mean = RunMetrics(statistics.mean(m.time for m in metrics), statistics.mean(m.fps for m in metrics))
+	stdev = RunMetrics(statistics.stdev((m.time for m in metrics), mean.time), statistics.stdev((m.fps for m in metrics), mean.fps))
+	print(f"{mean.time:.3f} ± {stdev.time:.3f} seconds, {mean.fps:.3f} ± {stdev.fps:.3f} FPS")
+
+main()