mirror
/
sargraph
mirror of https://github.com/antmicro/sargraph.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

[#55970] graph.py: watch.py: Added psutil-based RAM data collection 

Signed-off-by: Jan Bylicki <jbylicki@antmicro.com>
main
Jan Bylicki 2 years ago
parent
4a9ea12e5f
commit
82a4b993be
3 changed files with 82 additions and 30 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 54
      graph.py
  2. 3
      requirements.txt
  3. 55
      watch.py

54
graph.py
Unescape View File

@ -48,6 +48,7 @@ HOST = socket.gethostname()
# The number of plots on the graph
NUMBER_OF_PLOTS = 5
RAM_DATA_POSITION = 1
# The default format
OUTPUT_TYPE = "pngcairo"
@ -114,7 +115,7 @@ def plot_stacked(ylabel, title, session, column, tmpfs_color, other_cache_color,
else:
g("unset xdata")
g("set yrange [0:*]")
g(f"stats '{session}.txt' using {column}")
g(f"stats '{session}_ramdata.txt' using {column}")
g(f"set yrange [0:STATS_max*{autoscale}]")
g(f"set cbrange [0:STATS_max*{autoscale}]")
g("set xdata time")
@ -123,9 +124,9 @@ def plot_stacked(ylabel, title, session, column, tmpfs_color, other_cache_color,
g('set style data histograms')
g('set style histogram rowstacked')
g('set key reverse below Left width -25')
g(f"plot '{session}.txt' using 1:($8 + ${column}):{column} title 'RAM' with boxes palette, \
'' using 1:8 with boxes title 'Shared mem' lc rgb '{tmpfs_color}', \
'' using 1:($8 - $7) with boxes title 'Other cache (freed automatically)' lc rgb '{other_cache_color}'")
g(f"plot '{session}_ramdata.txt' using 1:($3 + ${column}):{column} title 'RAM' with boxes palette, \
'' using 1:5 with boxes title 'Shared mem' lc rgb '{tmpfs_color}', \
'' using 1:($3 - $5) with boxes title 'Other cache (freed automatically)' lc rgb '{other_cache_color}'")
g('unset key')
# Read additional information from 'data.txt' comments
def read_comments(session):
@ -424,23 +425,23 @@ def graph(session, tmpfs_color, other_cache_color, fname='plot'):
plot("CPU load (%)",
f"CPU load (average = {AVERAGE_LOAD:.2f} %)", session, 2, space=space)
plot_stacked(f"RAM usage (100% = {TOTAL_RAM})",
f"RAM usage (max = {MAX_USED_RAM})", session, 3, tmpfs_color, other_cache_color, space=space)
f"RAM usage (max = {MAX_USED_RAM})", session, 4, tmpfs_color, other_cache_color, space=space)
plot(f"FS usage (100% = {TOTAL_FS})", f"{NAME_FS} usage (max = {MAX_USED_FS})",
session, 4, space=space)
session, 3, space=space)
plot(f"{NAME_IFACE} received (Mb/s)",
f"{NAME_IFACE} data received (max = {MAX_RX}, total = {TOTAL_RX})",
session, 5, space=space, autoscale=1.2)
session, 4, space=space, autoscale=1.2)
plot(f"{NAME_IFACE} sent (Mb/s)",
f"{NAME_IFACE} data sent (max = {MAX_TX}, total = {TOTAL_TX})",
session, 6, space=space, autoscale=1.2)
session, 5, space=space, autoscale=1.2)
# GPU params
if TOTAL_GPU_RAM != 0:
plot("GPU load (%)",
f"GPU load (average = {AVERAGE_GPU_LOAD} %)", session, 9, space=space)
f"GPU load (average = {AVERAGE_GPU_LOAD} %)", session, 6, space=space)
plot(f"GPU RAM usage (100% = {TOTAL_GPU_RAM})",
f"GPU RAM usage (max = {MAX_USED_GPU_RAM})", session, 10, space=space)
f"GPU RAM usage (max = {MAX_USED_GPU_RAM})", session, 7, space=space)
g("unset multiplot")
g("unset output")
@ -449,6 +450,7 @@ def graph(session, tmpfs_color, other_cache_color, fname='plot'):
def read_data(session):
xdata = list()
xdata_ram = list()
ydata = [[] for _ in range(NUMBER_OF_PLOTS)]
with open(f"{session}.txt", "r") as f:
for line in f:
@ -457,8 +459,17 @@ def read_data(session):
date = datetime.datetime.strptime(line[0], '%Y-%m-%d-%H:%M:%S')
xdata.append(date)
for i in range(NUMBER_OF_PLOTS):
ydata[i].append(stof(line[i+1]))
return (xdata, ydata)
if i != RAM_DATA_POSITION:
ydata[i].append(stof(line[i+1 - int(i > RAM_DATA_POSITION)]))
with open(f"{session}_ramdata.txt", 'r') as f:
for line in f:
if(line[0] != '#'):
line = line.split(" ")
date = datetime.datetime.strptime(line[0], '%Y-%m-%d-%H:%M:%S.%f')
xdata_ram.append(date)
ydata[RAM_DATA_POSITION].append(100-stof(line[1]))
return (xdata, xdata_ram, ydata)
def convert_labels_to_tags(labels):
@ -475,7 +486,7 @@ def convert_labels_to_tags(labels):
def servis_graph(session, fname='plot', output_ext='ascii'):
read_comments(session)
xdata, ydata = read_data(session)
xdata, xdata_ram, ydata = read_data(session)
titles = [f"""CPU load (average = {AVERAGE_LOAD} %)""",
f"""RAM usage (max = {MAX_USED_RAM})""",
f"""{NAME_FS} usage (max = {MAX_USED_FS})""",
@ -526,9 +537,16 @@ def servis_graph(session, fname='plot', output_ext='ascii'):
from servis import render_multiple_time_series_plot
if output_ext == 'ascii':
xdatas = [[xdata_to_int]] * (NUMBER_OF_PLOTS - 1)
xdatas.insert(1, [[
int(timestamp.replace(
tzinfo=datetime.timezone.utc).timestamp()*1000)/1000
for timestamp in xdata_ram
]])
render_multiple_time_series_plot(
ydatas=[[yd] for yd in ydata],
xdatas=[[xdata_to_int]] * NUMBER_OF_PLOTS,
xdatas=xdatas,
title=summary,
subtitles=titles,
xtitles=['time'] * NUMBER_OF_PLOTS,
@ -543,9 +561,15 @@ def servis_graph(session, fname='plot', output_ext='ascii'):
)
elif output_ext == 'html':
converted_labels = convert_labels_to_tags(labels)
xdatas = [
int(timestamp.replace(
tzinfo=datetime.timezone.utc).timestamp()*1000)/1000
for timestamp in xdata_ram
]
xdatas = [xdata_to_int] + [xdatas] + [xdata_to_int * (NUMBER_OF_PLOTS - 2)]
render_multiple_time_series_plot(
ydatas=ydata,
xdatas=[xdata_to_int] * NUMBER_OF_PLOTS,
xdatas=xdatas,
title=summary,
subtitles=titles,
xtitles=['time'] * NUMBER_OF_PLOTS,

3
requirements.txt
Unescape View File

@ -1 +1,2 @@
git+https://github.com/antmicro/servis
git+https://github.com/antmicro/servis
psutil

55
watch.py
Unescape View File

@ -15,6 +15,9 @@ import signal
import subprocess
import sys
import time
import psutil
import sched
from threading import Thread
import graph
@ -42,6 +45,7 @@ END_RX = 0
TOTAL_GPU_LOAD = 0.0
TOTAL_GPU_RAM = 0
MAX_USED_GPU_RAM = 0
RAM_DATA_FILE_HANDLE = None
FS_NAME = None
FS_SAR_INDEX = None
@ -136,6 +140,9 @@ def initialize(session, machine):
with open(f"{session}.txt", "w") as f:
print(*header, sep=", ", file=f)
# clear the contents of the ramdata file
open(f"{session}_ramdata.txt", 'w').close()
# Add a summary comment to 'data.txt'
def summarize(session):
@ -182,6 +189,24 @@ def summarize(session):
with open(f"{session}.txt", "a") as f:
print(*summary, sep=", ", file=f)
def get_meminfo(scheduler):
global MAX_USED_RAM
global RAM_DATA_FILE_HANDLE
scheduler.enter(0.1, 1, get_meminfo, (scheduler,))
now = datetime.datetime.now()
date = now.strftime("%Y-%m-%d")
daytime = now.strftime("%H:%M:%S.%f")
ram_data = psutil.virtual_memory()
if (ram_data.total - ram_data.free) // 1024 > MAX_USED_RAM:
MAX_USED_RAM = (ram_data.total - ram_data.free) // 1024
line = [
date + "-" + daytime,
100 * ram_data.free / ram_data.total,
100 * ram_data.cached / ram_data.total,
100 * ram_data.used / ram_data.total,
100 * ram_data.shared / ram_data.total,
]
print(*line, file=RAM_DATA_FILE_HANDLE)
# Run sar and gather data from it
def watch(session, fsdev, iface, tmpfs_color, other_cache_color):
@ -206,6 +231,11 @@ def watch(session, fsdev, iface, tmpfs_color, other_cache_color):
global TOTAL_GPU_LOAD
global TOTAL_GPU_RAM
global MAX_USED_GPU_RAM
global RAM_DATA_FILE_HANDLE
if RAM_DATA_FILE_HANDLE == None:
RAM_DATA_FILE_HANDLE = open(f"{session}_ramdata.txt", 'a');
global die
@ -214,8 +244,13 @@ def watch(session, fsdev, iface, tmpfs_color, other_cache_color):
my_env = os.environ
my_env["S_TIME_FORMAT"] = "ISO"
p = run_or_fail("sar", "-F", "-u", "-r", "ALL", "-n", "DEV", "1", stdout=subprocess.PIPE, env=my_env)
p2 = run_or_fail("while true; do cat /proc/meminfo | grep Shmem\: | awk '{print $2}'; sleep 1; done", shell=True, stdout=subprocess.PIPE, env=my_env)
p = run_or_fail("sar", "-F", "-u", "-n", "DEV", "1", stdout=subprocess.PIPE, env=my_env)
s = sched.scheduler(time.time, time.sleep)
mem_ev = s.enter(0, 1, get_meminfo, (s,))
thread = Thread(target = s.run)
thread.start()
# subprocess for GPU data fetching in the background
try:
pgpu = subprocess.Popen(
@ -293,16 +328,8 @@ def watch(session, fsdev, iface, tmpfs_color, other_cache_color):
TOTAL_LOAD += stof(cpu_data["%user"][0])
SAMPLE_NUMBER += 1
# Read and process RAM data
ram_data = read_table(p.stdout)
tmpfs_data = p2.stdout.readline().decode('utf-8')
while p2.poll():
tmpfs_data = p2.stdout.readline().decode('utf-8')
if TOTAL_RAM == 0:
TOTAL_RAM = (int(ram_data['kbmemused'][0]) + int(ram_data['kbmemfree'][0]))
if MAX_USED_RAM < int(ram_data['kbmemused'][0]) + int(ram_data['kbcached'][0]):
MAX_USED_RAM = int(ram_data['kbmemused'][0]) + int(ram_data['kbcached'][0])
TOTAL_RAM = psutil.virtual_memory().total // 1024
# Read and process network data
net_data = read_table(p.stdout)
@ -360,12 +387,9 @@ def watch(session, fsdev, iface, tmpfs_color, other_cache_color):
line = [
timestamp,
cpu_data['%user'][0],
ram_data['%memused'][0],
fs_data['%fsused'][FS_SAR_INDEX],
stof(net_data['rxkB/s'][IFACE_SAR_INDEX])/128, # kB/s to Mb/s
stof(net_data['txkB/s'][IFACE_SAR_INDEX])/128, # kB/s to Mb/s
f'{100*int(tmpfs_data)/TOTAL_RAM:.2f}',
f'{100*int(ram_data["kbcached"][0])/TOTAL_RAM:.2f}'
]
if pgpu and TOTAL_GPU_RAM != 0:
line.extend([
@ -377,6 +401,9 @@ def watch(session, fsdev, iface, tmpfs_color, other_cache_color):
if die:
break
list(map(s.cancel, s.queue))
thread.join()
# This runs if we were stopped by SIGTERM and no plot was made so far
if not dont_plot:
summarize(session)

Write Preview
Loading…
Cancel
Save