From 82a4b993be6c9563b4823a05c6e7fcdd5bb17888 Mon Sep 17 00:00:00 2001 From: Jan Bylicki Date: Fri, 15 Mar 2024 18:16:49 +0100 Subject: [PATCH] [#55970] graph.py: watch.py: Added psutil-based RAM data collection Signed-off-by: Jan Bylicki --- graph.py | 54 ++++++++++++++++++++++++++++++++++------------- requirements.txt | 3 ++- watch.py | 55 ++++++++++++++++++++++++++++++++++++------------ 3 files changed, 82 insertions(+), 30 deletions(-) diff --git a/graph.py b/graph.py index 06cc40f..4ce2dcf 100644 --- a/graph.py +++ b/graph.py @@ -48,6 +48,7 @@ HOST = socket.gethostname() # The number of plots on the graph NUMBER_OF_PLOTS = 5 +RAM_DATA_POSITION = 1 # The default format OUTPUT_TYPE = "pngcairo" @@ -114,7 +115,7 @@ def plot_stacked(ylabel, title, session, column, tmpfs_color, other_cache_color, else: g("unset xdata") g("set yrange [0:*]") - g(f"stats '{session}.txt' using {column}") + g(f"stats '{session}_ramdata.txt' using {column}") g(f"set yrange [0:STATS_max*{autoscale}]") g(f"set cbrange [0:STATS_max*{autoscale}]") g("set xdata time") @@ -123,9 +124,9 @@ def plot_stacked(ylabel, title, session, column, tmpfs_color, other_cache_color, g('set style data histograms') g('set style histogram rowstacked') g('set key reverse below Left width -25') - g(f"plot '{session}.txt' using 1:($8 + ${column}):{column} title 'RAM' with boxes palette, \ - '' using 1:8 with boxes title 'Shared mem' lc rgb '{tmpfs_color}', \ - '' using 1:($8 - $7) with boxes title 'Other cache (freed automatically)' lc rgb '{other_cache_color}'") + g(f"plot '{session}_ramdata.txt' using 1:($3 + ${column}):{column} title 'RAM' with boxes palette, \ + '' using 1:5 with boxes title 'Shared mem' lc rgb '{tmpfs_color}', \ + '' using 1:($3 - $5) with boxes title 'Other cache (freed automatically)' lc rgb '{other_cache_color}'") g('unset key') # Read additional information from 'data.txt' comments def read_comments(session): @@ -424,23 +425,23 @@ def graph(session, tmpfs_color, other_cache_color, fname='plot'): plot("CPU load (%)", f"CPU load (average = {AVERAGE_LOAD:.2f} %)", session, 2, space=space) plot_stacked(f"RAM usage (100% = {TOTAL_RAM})", - f"RAM usage (max = {MAX_USED_RAM})", session, 3, tmpfs_color, other_cache_color, space=space) + f"RAM usage (max = {MAX_USED_RAM})", session, 4, tmpfs_color, other_cache_color, space=space) plot(f"FS usage (100% = {TOTAL_FS})", f"{NAME_FS} usage (max = {MAX_USED_FS})", - session, 4, space=space) + session, 3, space=space) plot(f"{NAME_IFACE} received (Mb/s)", f"{NAME_IFACE} data received (max = {MAX_RX}, total = {TOTAL_RX})", - session, 5, space=space, autoscale=1.2) + session, 4, space=space, autoscale=1.2) plot(f"{NAME_IFACE} sent (Mb/s)", f"{NAME_IFACE} data sent (max = {MAX_TX}, total = {TOTAL_TX})", - session, 6, space=space, autoscale=1.2) + session, 5, space=space, autoscale=1.2) # GPU params if TOTAL_GPU_RAM != 0: plot("GPU load (%)", - f"GPU load (average = {AVERAGE_GPU_LOAD} %)", session, 9, space=space) + f"GPU load (average = {AVERAGE_GPU_LOAD} %)", session, 6, space=space) plot(f"GPU RAM usage (100% = {TOTAL_GPU_RAM})", - f"GPU RAM usage (max = {MAX_USED_GPU_RAM})", session, 10, space=space) + f"GPU RAM usage (max = {MAX_USED_GPU_RAM})", session, 7, space=space) g("unset multiplot") g("unset output") @@ -449,6 +450,7 @@ def graph(session, tmpfs_color, other_cache_color, fname='plot'): def read_data(session): xdata = list() + xdata_ram = list() ydata = [[] for _ in range(NUMBER_OF_PLOTS)] with open(f"{session}.txt", "r") as f: for line in f: @@ -457,8 +459,17 @@ def read_data(session): date = datetime.datetime.strptime(line[0], '%Y-%m-%d-%H:%M:%S') xdata.append(date) for i in range(NUMBER_OF_PLOTS): - ydata[i].append(stof(line[i+1])) - return (xdata, ydata) + if i != RAM_DATA_POSITION: + ydata[i].append(stof(line[i+1 - int(i > RAM_DATA_POSITION)])) + with open(f"{session}_ramdata.txt", 'r') as f: + for line in f: + if(line[0] != '#'): + line = line.split(" ") + date = datetime.datetime.strptime(line[0], '%Y-%m-%d-%H:%M:%S.%f') + xdata_ram.append(date) + ydata[RAM_DATA_POSITION].append(100-stof(line[1])) + + return (xdata, xdata_ram, ydata) def convert_labels_to_tags(labels): @@ -475,7 +486,7 @@ def convert_labels_to_tags(labels): def servis_graph(session, fname='plot', output_ext='ascii'): read_comments(session) - xdata, ydata = read_data(session) + xdata, xdata_ram, ydata = read_data(session) titles = [f"""CPU load (average = {AVERAGE_LOAD} %)""", f"""RAM usage (max = {MAX_USED_RAM})""", f"""{NAME_FS} usage (max = {MAX_USED_FS})""", @@ -526,9 +537,16 @@ def servis_graph(session, fname='plot', output_ext='ascii'): from servis import render_multiple_time_series_plot if output_ext == 'ascii': + xdatas = [[xdata_to_int]] * (NUMBER_OF_PLOTS - 1) + xdatas.insert(1, [[ + int(timestamp.replace( + tzinfo=datetime.timezone.utc).timestamp()*1000)/1000 + for timestamp in xdata_ram + ]]) + render_multiple_time_series_plot( ydatas=[[yd] for yd in ydata], - xdatas=[[xdata_to_int]] * NUMBER_OF_PLOTS, + xdatas=xdatas, title=summary, subtitles=titles, xtitles=['time'] * NUMBER_OF_PLOTS, @@ -543,9 +561,15 @@ def servis_graph(session, fname='plot', output_ext='ascii'): ) elif output_ext == 'html': converted_labels = convert_labels_to_tags(labels) + xdatas = [ + int(timestamp.replace( + tzinfo=datetime.timezone.utc).timestamp()*1000)/1000 + for timestamp in xdata_ram + ] + xdatas = [xdata_to_int] + [xdatas] + [xdata_to_int * (NUMBER_OF_PLOTS - 2)] render_multiple_time_series_plot( ydatas=ydata, - xdatas=[xdata_to_int] * NUMBER_OF_PLOTS, + xdatas=xdatas, title=summary, subtitles=titles, xtitles=['time'] * NUMBER_OF_PLOTS, diff --git a/requirements.txt b/requirements.txt index 4407111..cac6603 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ -git+https://github.com/antmicro/servis \ No newline at end of file +git+https://github.com/antmicro/servis +psutil diff --git a/watch.py b/watch.py index 09a95c0..312812c 100644 --- a/watch.py +++ b/watch.py @@ -15,6 +15,9 @@ import signal import subprocess import sys import time +import psutil +import sched +from threading import Thread import graph @@ -42,6 +45,7 @@ END_RX = 0 TOTAL_GPU_LOAD = 0.0 TOTAL_GPU_RAM = 0 MAX_USED_GPU_RAM = 0 +RAM_DATA_FILE_HANDLE = None FS_NAME = None FS_SAR_INDEX = None @@ -136,6 +140,9 @@ def initialize(session, machine): with open(f"{session}.txt", "w") as f: print(*header, sep=", ", file=f) + # clear the contents of the ramdata file + open(f"{session}_ramdata.txt", 'w').close() + # Add a summary comment to 'data.txt' def summarize(session): @@ -182,6 +189,24 @@ def summarize(session): with open(f"{session}.txt", "a") as f: print(*summary, sep=", ", file=f) +def get_meminfo(scheduler): + global MAX_USED_RAM + global RAM_DATA_FILE_HANDLE + scheduler.enter(0.1, 1, get_meminfo, (scheduler,)) + now = datetime.datetime.now() + date = now.strftime("%Y-%m-%d") + daytime = now.strftime("%H:%M:%S.%f") + ram_data = psutil.virtual_memory() + if (ram_data.total - ram_data.free) // 1024 > MAX_USED_RAM: + MAX_USED_RAM = (ram_data.total - ram_data.free) // 1024 + line = [ + date + "-" + daytime, + 100 * ram_data.free / ram_data.total, + 100 * ram_data.cached / ram_data.total, + 100 * ram_data.used / ram_data.total, + 100 * ram_data.shared / ram_data.total, + ] + print(*line, file=RAM_DATA_FILE_HANDLE) # Run sar and gather data from it def watch(session, fsdev, iface, tmpfs_color, other_cache_color): @@ -206,6 +231,11 @@ def watch(session, fsdev, iface, tmpfs_color, other_cache_color): global TOTAL_GPU_LOAD global TOTAL_GPU_RAM global MAX_USED_GPU_RAM + global RAM_DATA_FILE_HANDLE + + if RAM_DATA_FILE_HANDLE == None: + RAM_DATA_FILE_HANDLE = open(f"{session}_ramdata.txt", 'a'); + global die @@ -214,8 +244,13 @@ def watch(session, fsdev, iface, tmpfs_color, other_cache_color): my_env = os.environ my_env["S_TIME_FORMAT"] = "ISO" - p = run_or_fail("sar", "-F", "-u", "-r", "ALL", "-n", "DEV", "1", stdout=subprocess.PIPE, env=my_env) - p2 = run_or_fail("while true; do cat /proc/meminfo | grep Shmem\: | awk '{print $2}'; sleep 1; done", shell=True, stdout=subprocess.PIPE, env=my_env) + p = run_or_fail("sar", "-F", "-u", "-n", "DEV", "1", stdout=subprocess.PIPE, env=my_env) + + s = sched.scheduler(time.time, time.sleep) + mem_ev = s.enter(0, 1, get_meminfo, (s,)) + thread = Thread(target = s.run) + thread.start() + # subprocess for GPU data fetching in the background try: pgpu = subprocess.Popen( @@ -293,16 +328,8 @@ def watch(session, fsdev, iface, tmpfs_color, other_cache_color): TOTAL_LOAD += stof(cpu_data["%user"][0]) SAMPLE_NUMBER += 1 - # Read and process RAM data - ram_data = read_table(p.stdout) - tmpfs_data = p2.stdout.readline().decode('utf-8') - while p2.poll(): - tmpfs_data = p2.stdout.readline().decode('utf-8') - if TOTAL_RAM == 0: - TOTAL_RAM = (int(ram_data['kbmemused'][0]) + int(ram_data['kbmemfree'][0])) - if MAX_USED_RAM < int(ram_data['kbmemused'][0]) + int(ram_data['kbcached'][0]): - MAX_USED_RAM = int(ram_data['kbmemused'][0]) + int(ram_data['kbcached'][0]) + TOTAL_RAM = psutil.virtual_memory().total // 1024 # Read and process network data net_data = read_table(p.stdout) @@ -360,12 +387,9 @@ def watch(session, fsdev, iface, tmpfs_color, other_cache_color): line = [ timestamp, cpu_data['%user'][0], - ram_data['%memused'][0], fs_data['%fsused'][FS_SAR_INDEX], stof(net_data['rxkB/s'][IFACE_SAR_INDEX])/128, # kB/s to Mb/s stof(net_data['txkB/s'][IFACE_SAR_INDEX])/128, # kB/s to Mb/s - f'{100*int(tmpfs_data)/TOTAL_RAM:.2f}', - f'{100*int(ram_data["kbcached"][0])/TOTAL_RAM:.2f}' ] if pgpu and TOTAL_GPU_RAM != 0: line.extend([ @@ -377,6 +401,9 @@ def watch(session, fsdev, iface, tmpfs_color, other_cache_color): if die: break + list(map(s.cancel, s.queue)) + thread.join() + # This runs if we were stopped by SIGTERM and no plot was made so far if not dont_plot: summarize(session)