mirror
/
sargraph
mirror of https://github.com/antmicro/sargraph.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

[#85496] Use sockets for communication

main
Przemysław Barcicki 3 months ago
parent
5368bc5063
commit
669c9bfebd
4 changed files with 605 additions and 540 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 5
      README.md
  2. 6
      common.py
  3. 159
      sargraph.py
  4. 407
      watch.py

5
README.md
Unescape View File

@ -1,6 +1,6 @@
# sargraph # sargraph
Copyright (c) 2019-2023 [Antmicro](https://www.antmicro.com) Copyright (c) 2019-2026 [Antmicro](https://www.antmicro.com)
This is a simple python tool that uses "sysstat" ("sar") to save information on CPU, RAM and disk usage. This is a simple python tool that uses "sysstat" ("sar") to save information on CPU, RAM and disk usage.
The process runs in background and can be controlled with a set of sargraph sub-commands. The process runs in background and can be controlled with a set of sargraph sub-commands.
@ -10,7 +10,7 @@ Supported plot formats are PNG, SVG and ASCII, they are determined by filename e
# Install requirements # Install requirements
The sargraph requires `gnuplot`, `sysstat` (`sar`), `python3`, `coreutils` and `screen` to operate. The sargraph requires `gnuplot`, `sysstat` (`sar`), `python3` and `coreutils` to operate.
In Debian you can install them with: In Debian you can install them with:
``` ```
@ -22,7 +22,6 @@ apt-get install -qqy --no-install-recommends \
gnuplot-nox \ gnuplot-nox \
python3 \ python3 \
python3-pip \ python3-pip \
screen \
sysstat sysstat
# install Python dependencies # install Python dependencies

6
common.py
Unescape View File

@ -39,7 +39,11 @@ def run_or_fail(*argv, **kwargs):
# Check if a process is running # Check if a process is running
def pid_running(pid): def pid_running(pid):
return os.path.exists(f"/proc/{pid}") return file_exists(f"/proc/{pid}")
# Check whether path exists
def file_exists(filename: str):
return os.path.exists(filename)
# Convert a string to float, also when the separator is a comma # Convert a string to float, also when the separator is a comma

159
sargraph.py
Unescape View File

@ -11,12 +11,13 @@ import time
import graph import graph
import watch import watch
import warnings
from common import * from common import *
# Declare and parse command line flags # Declare and parse command line flags
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument('session', metavar='SESSION-NAME', type=str, nargs='?', default=None, help='sargraph session name') parser.add_argument('session', metavar='SESSION-NAME', type=str, nargs='?', help='sargraph session name')
parser.add_argument('command', metavar='COMMAND', type=str, nargs='*', help='send command') parser.add_argument('command', metavar='COMMAND', type=str, nargs='*', help='send command')
parser.add_argument('-f', metavar='DEVICE-NAME', type=str, nargs='?', default=None, dest='fsdev', help='observe a chosen filesystem') parser.add_argument('-f', metavar='DEVICE-NAME', type=str, nargs='?', default=None, dest='fsdev', help='observe a chosen filesystem')
parser.add_argument('-m', metavar='MOUNT-DIR', type=str, nargs='?', default=None, dest='fspath', help='observe a chosen filesystem') parser.add_argument('-m', metavar='MOUNT-DIR', type=str, nargs='?', default=None, dest='fspath', help='observe a chosen filesystem')
@ -29,23 +30,16 @@ parser.add_argument('-C', metavar='UDP_COOKIE', type=str, nargs='?', defa
parser.add_argument('-p', action='store_true', dest='psutil', help='use psutil instead of sar') parser.add_argument('-p', action='store_true', dest='psutil', help='use psutil instead of sar')
args = parser.parse_args() args = parser.parse_args()
def send(sid, msg): def send(session: str, message: str):
p = subprocess.Popen(["screen", "-S", sid, "-X", "stuff", f"{msg}\n"]) sock, socket_path = watch.get_socket(session)
while p.poll() is None: if not file_exists(socket_path):
time.sleep(0.1) fail(f"Session '{session}' does not exist")
# Check if sar is available
if not is_darwin():
p = run_or_fail("sar", "-V", stdout=subprocess.PIPE)
# Check if screen is available sock.connect(socket_path)
p = run_or_fail("screen", "-v", stdout=subprocess.PIPE) sock.send(message.encode("utf-8"))
version = scan("Screen version (\\d+)", int, p.stdout.readline().decode()) sock.close()
if version is None:
fail("'screen' tool returned unknown output")
# If the script was run with no parameters, run in background and gather data def create_session():
if args.session is None:
# Find requested disk device # Find requested disk device
if args.fspath: if args.fspath:
args.fspath = os.path.realpath(args.fspath) args.fspath = os.path.realpath(args.fspath)
@ -53,69 +47,98 @@ if args.session is None:
while args.fsdev is None: while args.fsdev is None:
args.fsdev = scan(f"^(/dev/\\S+)\\s+{re.escape(args.fspath)}\\s+", str, f.readline()) args.fsdev = scan(f"^(/dev/\\S+)\\s+{re.escape(args.fspath)}\\s+", str, f.readline())
if not args.fsdev: if not args.fsdev:
fail(f"no device is mounted on {args.fspath}") fail(f"No device is mounted on {args.fspath}")
params = (args.session, args.fsdev, args.iface, args.tmpfs, args.cache, args.udp, args.udp_cookie)
if is_darwin() or args.psutil:
watcher = watch.PsUtilWatcher(*params)
else:
watcher = watch.SarWatcher(*params)
watch.watch(args.name, args.fsdev, args.iface, args.tmpfs, args.cache, args.psutil, args.udp, args.udp_cookie) watcher.start()
sys.exit(0) sys.exit(0)
# Now handle the commands # Check if sar is available
if not is_darwin():
p = run_or_fail("sar", "-V", stdout=subprocess.PIPE)
if args.name != "data":
warnings.warn("'-o' is deprecated, session name is default output base name")
# Check if a command was provided # Check if a command was provided, if that session exists, yell at user for lack of commands, else spawn
if len(args.command) <= 0: if len(args.command) == 0:
fail("command not provided") sock, socket_path = watch.get_socket(args.session)
if file_exists(socket_path):
fail("Command not provided")
# Get session name and command name else:
sid = args.session print(f"Starting sargraph session '{args.session}'")
cmd = args.command create_session()
if args.command[0] == "start":
sock, socket_path = watch.get_socket(args.session)
if file_exists(socket_path):
fail("Session with this name already exists")
# Start watcher process
p = subprocess.Popen(
args=[sys.executable, os.path.realpath(__file__), args.session, *sys.argv[3:]],
stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
start_new_session=True
)
# Spinloop to see whether the subprocess even starts
attempts = 5
while attempts:
time.sleep(0.1)
if file_exists(socket_path):
print(f"Session '{args.session}' started")
sys.exit(0)
attempts -= 1
if cmd[0] == "start": fail("Session did not start")
print(f"Starting sargraph session '{sid}'")
# Spawn watcher process, *sys.argv[3:] is all arguments after 'chart start' + '-o [log name]' if not given elif args.command[0] == "stop":
if "-o" not in sys.argv: _, socket_path = watch.get_socket(args.session)
sys.argv += ["-o", sid]
p = subprocess.Popen(["screen", "-Logfile", f"{sid}.log", "-dmSL", sid, os.path.realpath(__file__), *sys.argv[3:]])
while p.poll() is None: print(f"Terminating sargraph session '{args.session}'")
time.sleep(0.1) if len(args.command) < 2:
gpid = 0 send(args.session, "command:q:")
j = 0
time.sleep(1)
print(f"Session '{sid}' started")
elif cmd[0] == "stop":
print(f"Terminating sargraph session '{sid}'")
try:
gpid = int(os.popen(f"screen -ls | grep '.{sid}' | tr -d ' \t' | cut -f 1 -d '.'").read())
except:
print("Warning: cannot find pid.")
gpid = -1
if len(cmd) < 2:
send(sid, "command:q:")
else: else:
send(sid, f"command:q:{cmd[1]}") send(args.session, f"command:q:{args.command[1]}")
if gpid == -1:
print("Waiting 3 seconds.") # Spinloop to see whether the subprocess even dies
time.sleep(3) attempts = 5
else: while attempts:
while pid_running(gpid): time.sleep(0.5)
time.sleep(0.25) if not file_exists(socket_path):
elif cmd[0] == "label": print(f"Session '{args.session}' killed")
sys.exit(0)
attempts -= 1
fail("Session did not respond")
elif args.command[0] == "label":
# Check if the label name was provided # Check if the label name was provided
if len(cmd) < 2: if len(args.command) < 2:
fail("label command requires an additional parameter") fail("label command requires an additional parameter")
print(f"Adding label '{cmd[1]}' to sargraph session '{sid}'.")
send(sid, f"label:{cmd[1]}") print(f"Adding label '{args.command[1]}' to sargraph session '{args.session}'.")
elif cmd[0] == 'save': send(args.session, f"label:{args.command[1]}")
print(f"Saving graph from session '{sid}'.")
if len(cmd) < 2:
send(sid, "command:s:") elif args.command[0] == 'save':
print(f"Saving graph from session '{args.session}'.")
if len(args.command) < 2:
send(args.session, "command:s:")
else: else:
send(sid, f"command:s:{cmd[1]}") send(args.session, f"command:s:{args.command[1]}")
elif cmd[0] == 'plot':
if len(cmd) < 2: elif args.command[0] == 'plot':
graph.graph(sid, args.tmpfs, args.cache) if len(args.command) < 2:
graph.graph(args.session, args.tmpfs, args.cache)
else: else:
graph.graph(sid, args.tmpfs, args.cache, cmd[1]) graph.graph(args.session, args.tmpfs, args.cache, args.command[1])
else: else:
fail(f"unknown command '{cmd[0]}'") fail(f"unknown command '{args.command[0]}'")

407
watch.py
Unescape View File

@ -7,29 +7,25 @@
import datetime import datetime
import fcntl
import os import os
import re
import select import select
import signal import signal
import subprocess import subprocess
import sys
import time import time
import psutil import psutil
import sched import sched
import platform import platform
import logging import logging
from threading import Thread, Lock import socket
import threading import abc
import traceback
from threading import Thread
from logging.handlers import DatagramHandler from logging.handlers import DatagramHandler
import graph import graph
from common import * from common import *
die = 0
# Initialize summary variables # Initialize summary variables
SAMPLE_NUMBER = 0 SAMPLE_NUMBER = 0
TOTAL_RAM = 0 TOTAL_RAM = 0
@ -56,11 +52,6 @@ FS_SAR_INDEX = None
IFACE_NAME = None IFACE_NAME = None
IFACE_SAR_INDEX = None IFACE_SAR_INDEX = None
# Handle SIGTERM
def kill_handler(a, b):
global die
die = 1
class UDPHandler(DatagramHandler): class UDPHandler(DatagramHandler):
def emit(self, msg): def emit(self, msg):
try: try:
@ -70,10 +61,6 @@ class UDPHandler(DatagramHandler):
except Exception as e: except Exception as e:
pass pass
logger = logging.getLogger("sargraph")
logger.setLevel(logging.INFO)
# Read a single table from sar output # Read a single table from sar output
def read_table(psar): def read_table(psar):
# Find the header # Find the header
@ -114,74 +101,68 @@ def read_iface_stats(iface):
tx = scan(r"(\d+)", int, f.readline()) tx = scan(r"(\d+)", int, f.readline())
return rx, tx return rx, tx
def get_socket(session):
# TODO: when using on other platforms make sure this path exist; namely windows
path = f"/tmp/sargraph-{session}.sock"
sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
return sock, path
# Initialize 'data.txt' where the data is dumped
def initialize(session, machine):
global TOTAL_RAM
global TOTAL_GPU_RAM
with open("/proc/meminfo") as f: class Watcher(abc.ABC):
TOTAL_RAM = int(scan("MemTotal:\s+(\d+)", float, f.read()))
uname = machine.split(" ")[0:2] def __init__(self, session, fsdev, iface, tmpfs_color, other_cache_color, udp=None, udp_cookie=None):
uname = f"{uname[0]} {uname[1]}" super().__init__()
cpus = int(machine.split(" CPU)")[0].split("(")[-1]) self.session = session
cpu_name = "unknown" self.fsdev = fsdev
self.iface = iface
self.tmpfs_color = tmpfs_color
self.other_cache_color = other_cache_color
with open("/proc/cpuinfo") as f: self.logger = logging.getLogger("sargraph")
for line in f: self.logger.setLevel(logging.INFO)
if "model name" in line:
cpu_name = line.replace("\n", "").split(": ")[1]
break
header = [
f"# sargraph version: {SARGRAPH_VERSION}",
f"pid: {os.getpid()}",
f"machine: {uname}",
f"cpu count: {cpus}",
f"cpu: {cpu_name}"
]
try:
pgpu = subprocess.run(
'nvidia-smi --query-gpu=name,driver_version,memory.total --format=csv,noheader,nounits'.split(' '),
capture_output=True
)
if pgpu.returncode == 0:
gpuname, gpudriver, memory_total = pgpu.stdout.decode('utf-8').rsplit(', ', 2)
header.extend([
f"gpu: {gpuname}",
f"gpu driver: {gpudriver}"
])
TOTAL_GPU_RAM = int(memory_total)
except Exception as e:
print(e)
pass
logger.info(", ".join(header)) file_handler = logging.FileHandler(f"{session}.txt")
file_handler.setFormatter(logging.Formatter("%(message)s"))
self.logger.addHandler(file_handler)
def initialize_darwin(session): self.socket, self.socket_path = get_socket(self.session)
global TOTAL_RAM
global TOTAL_GPU_RAM
TOTAL_RAM = int(psutil.virtual_memory().total / 1024) # Was a graph alreay produced by save command from sargraph?
self.dont_plot = False
cpus = psutil.cpu_count(logical=True) # Should we die?
self.die = False
cpu_name = platform.processor() or "unknown" if udp is not None:
spl = udp.rsplit(':', 1)
udp_handler = UDPHandler(spl[0], int(spl[1]))
if udp_cookie is None:
udp_handler.setFormatter(logging.Formatter("%(message)s\n"))
else:
udp_handler.setFormatter(logging.Formatter(f"[{udp_cookie}] %(message)s\n"))
self.logger.addHandler(udp_handler)
header = [ # Initialize 'data.txt' where the data is dumped
f"# psutil version: {psutil.__version__}", @abc.abstractmethod
f"pid: {os.getpid()}", def initialize(self, machine):
f"machine: {platform.system()}", pass
f"cpu count: {cpus}",
f"cpu: {cpu_name}"
]
logger.info(", ".join(header))
# Implement the watcher method
@abc.abstractmethod
def watch(self):
pass
def kill_handler(self, *_):
self.die = True
def recv_data(self) -> str:
data = self.socket.recv(1 << 10) # 1024 bytes should be enough
return data.decode("utf-8").replace("\n", "").strip()
# Add a summary comment to 'data.txt' # Add a summary comment to 'data.txt'
def summarize(session): def summarize(self):
# Is there anything to be summarized? # Is there anything to be summarized?
if SAMPLE_NUMBER == 0: if SAMPLE_NUMBER == 0:
return return
@ -222,11 +203,11 @@ def summarize(session):
f"average gpu load: {TOTAL_GPU_LOAD / SAMPLE_NUMBER:.2f} %" f"average gpu load: {TOTAL_GPU_LOAD / SAMPLE_NUMBER:.2f} %"
]) ])
logger.info(", ".join([str(i) for i in summary])) self.logger.info(", ".join([str(i) for i in summary]))
def get_meminfo(scheduler): def get_meminfo(self, scheduler):
global MAX_USED_RAM global MAX_USED_RAM
scheduler.enter(0.1, 1, get_meminfo, (scheduler,)) scheduler.enter(0.1, 1, self.get_meminfo, (scheduler,))
now = datetime.datetime.now() now = datetime.datetime.now()
date = now.strftime("%Y-%m-%d") date = now.strftime("%Y-%m-%d")
daytime = now.strftime("%H:%M:%S.%f") daytime = now.strftime("%H:%M:%S.%f")
@ -251,29 +232,109 @@ def get_meminfo(scheduler):
100 * ram_data.shared / ram_data.total 100 * ram_data.shared / ram_data.total
] ]
msg = " ".join(["psu"]+[str(i) for i in line]) msg = " ".join(["psu"]+[str(i) for i in line])
logger.info(msg) self.logger.info(msg)
def start(self):
self.socket.bind(self.socket_path)
signal.signal(signal.SIGTERM, self.kill_handler)
def watch(session, fsdev, iface, tmpfs_color, other_cache_color, use_psutil, udp=None, udp_cookie=None): try:
file_handler = logging.FileHandler(f"{session}.txt") self.watch()
file_handler.setFormatter(logging.Formatter("%(message)s")) except Exception as e:
logger.addHandler(file_handler) # make sure we prepend '#' to every line, to make reading file work
self.logger.error("# Exception while watching!")
for line in traceback.format_exception(type(e), e, e.__traceback__):
self.logger.error(f"# {line}")
try: # clean up after ourselves
os.unlink(self.socket_path)
except OSError:
pass
if udp is not None: return
spl = udp.rsplit(':', 1)
udp_handler = UDPHandler(spl[0], int(spl[1])) def handle_command(self, label_line: str, s: sched.scheduler, now: datetime.datetime):
if udp_cookie is None: if label_line.startswith("command:"):
udp_handler.setFormatter(logging.Formatter("%(message)s\n")) label_line = label_line[len("command:"):]
if label_line.startswith("q:"):
label_line = label_line[len("q:"):]
list(map(s.cancel, s.queue))
self.summarize()
if label_line == "none":
pass
elif label_line:
graph.graph(self.session, self.tmpfs_color, self.other_cache_color, label_line)
elif not self.dont_plot:
graph.graph(self.session, self.tmpfs_color, self.other_cache_color)
self.dont_plot = True
self.die = 1
return True
elif label_line.startswith("s:"):
label_line = label_line[len("s:"):]
self.dont_plot = True
if label_line != "none":
self.summarize()
if not label_line:
graph.graph(self.session, self.tmpfs_color, self.other_cache_color)
else: else:
udp_handler.setFormatter(logging.Formatter(f"[{udp_cookie}] %(message)s\n")) graph.graph(self.session, self.tmpfs_color, self.other_cache_color, label_line)
logger.addHandler(udp_handler) elif label_line.startswith('label:'):
label_line = label_line[len('label:'):]
with open(f"{self.session}.txt", "a") as f:
timestamp = now.strftime("%Y-%m-%d-%H:%M:%S")
print(f"# {timestamp} label: {label_line}", file=f)
return False
class SarWatcher(Watcher):
def initialize(self, machine):
global TOTAL_RAM
global TOTAL_GPU_RAM
with open("/proc/meminfo") as f:
TOTAL_RAM = int(scan("MemTotal:\s+(\d+)", float, f.read()))
uname = machine.split(" ")[0:2]
uname = f"{uname[0]} {uname[1]}"
cpus = int(machine.split(" CPU)")[0].split("(")[-1])
cpu_name = "unknown"
with open("/proc/cpuinfo") as f:
for line in f:
if "model name" in line:
cpu_name = line.replace("\n", "").split(": ")[1]
break
header = [
f"# sargraph version: {SARGRAPH_VERSION}",
f"pid: {os.getpid()}",
f"machine: {uname}",
f"cpu count: {cpus}",
f"cpu: {cpu_name}"
]
try:
pgpu = subprocess.run(
'nvidia-smi --query-gpu=name,driver_version,memory.total --format=csv,noheader,nounits'.split(' '),
capture_output=True
)
if pgpu.returncode == 0:
gpuname, gpudriver, memory_total = pgpu.stdout.decode('utf-8').rsplit(', ', 2)
header.extend([
f"gpu: {gpuname}",
f"gpu driver: {gpudriver}"
])
TOTAL_GPU_RAM = int(memory_total)
except Exception as e:
print(e)
pass
if is_darwin() or use_psutil: self.logger.info(", ".join(header))
return watch_psutil(session, fsdev, iface, tmpfs_color, other_cache_color)
return watch_sar(session, fsdev, iface, tmpfs_color, other_cache_color)
# Run sar and gather data from it def watch(self):
def watch_sar(session, fsdev, iface, tmpfs_color, other_cache_color):
global SAMPLE_NUMBER global SAMPLE_NUMBER
global START_DATE global START_DATE
global END_DATE global END_DATE
@ -296,18 +357,13 @@ def watch_sar(session, fsdev, iface, tmpfs_color, other_cache_color):
global TOTAL_GPU_RAM global TOTAL_GPU_RAM
global MAX_USED_GPU_RAM global MAX_USED_GPU_RAM
global die
# Was a graph alreay produced by save command from sargraph?
dont_plot = False
my_env = os.environ my_env = os.environ
my_env["S_TIME_FORMAT"] = "ISO" my_env["S_TIME_FORMAT"] = "ISO"
psar = run_or_fail("sar", "-F", "-u", "-n", "DEV", "1", stdout=subprocess.PIPE, env=my_env) psar = run_or_fail("sar", "-F", "-u", "-n", "DEV", "1", stdout=subprocess.PIPE, env=my_env)
s = sched.scheduler(time.time, time.sleep) s = sched.scheduler(time.time, time.sleep)
mem_ev = s.enter(0, 1, get_meminfo, (s,)) mem_ev = s.enter(0, 1, self.get_meminfo, (s,))
thread = Thread(target = s.run) thread = Thread(target = s.run)
thread.start() thread.start()
@ -322,32 +378,33 @@ def watch_sar(session, fsdev, iface, tmpfs_color, other_cache_color):
pgpu = None pgpu = None
machine = psar.stdout.readline().decode() machine = psar.stdout.readline().decode()
initialize(session, machine) self.initialize(machine)
psar.stdout.readline() psar.stdout.readline()
signal.signal(signal.SIGTERM, kill_handler)
# Make stdin nonblocking to continue working when no command is sent
flags = fcntl.fcntl(sys.stdin, fcntl.F_GETFL)
fcntl.fcntl(sys.stdin, fcntl.F_SETFL, flags | os.O_NONBLOCK)
# Gather data from sar output # Gather data from sar output
curr_gpu_util = 0 curr_gpu_util = 0
curr_gpu_mem = 0 curr_gpu_mem = 0
socket_fd = self.socket.fileno()
while 1: while 1:
# Await sar output or a command sent from command handler in sargraph.py # Await sar output or a command sent from command handler in sargraph.py
readlist = [psar.stdout, sys.stdin] readlist = [psar.stdout, socket_fd]
if pgpu: if pgpu:
readlist.append(pgpu.stdout) readlist.append(pgpu.stdout)
rlist, _, _ = select.select(readlist, [], [], 0.25) rlist, _, _ = select.select(readlist, [], [], 0.25)
now = datetime.datetime.now() now = datetime.datetime.now()
if sys.stdin in rlist:
if handle_command(session, s, dont_plot, tmpfs_color, other_cache_color, now):
break
if psar.stdout not in rlist: if psar.stdout not in rlist:
continue continue
if socket_fd in rlist:
data = self.recv_data()
now = datetime.datetime.now()
if self.handle_command(data, s, now):
break
date = now.strftime("%Y-%m-%d") date = now.strftime("%Y-%m-%d")
daytime = now.strftime("%H:%M:%S") daytime = now.strftime("%H:%M:%S")
@ -365,8 +422,8 @@ def watch_sar(session, fsdev, iface, tmpfs_color, other_cache_color):
# Read and process network data # Read and process network data
net_data = read_table(psar) net_data = read_table(psar)
if IFACE_SAR_INDEX is None: if IFACE_SAR_INDEX is None:
if iface: if self.iface:
IFACE_SAR_INDEX = net_data['IFACE'].index(iface) IFACE_SAR_INDEX = net_data['IFACE'].index(self.iface)
else: else:
maxj, maxv = 0, 0 maxj, maxv = 0, 0
for j, used in enumerate(net_data['IFACE']): for j, used in enumerate(net_data['IFACE']):
@ -387,8 +444,8 @@ def watch_sar(session, fsdev, iface, tmpfs_color, other_cache_color):
# Read and process FS data # Read and process FS data
fs_data = read_table(psar) fs_data = read_table(psar)
if FS_SAR_INDEX is None: if FS_SAR_INDEX is None:
if fsdev: if self.fsdev:
FS_SAR_INDEX = fs_data['FILESYSTEM'].index(fsdev) FS_SAR_INDEX = fs_data['FILESYSTEM'].index(self.fsdev)
else: else:
maxj, maxv = 0, 0 maxj, maxv = 0, 0
for j, free in enumerate(fs_data['MBfsfree']): for j, free in enumerate(fs_data['MBfsfree']):
@ -447,92 +504,42 @@ def watch_sar(session, fsdev, iface, tmpfs_color, other_cache_color):
f'{curr_gpu_util:.2f}', f'{curr_gpu_util:.2f}',
f'{curr_gpu_mem / TOTAL_GPU_RAM * 100.0:.2f}' f'{curr_gpu_mem / TOTAL_GPU_RAM * 100.0:.2f}'
]) ])
logger.info(" ".join(["sar"]+[str(i) for i in line])) self.logger.info(" ".join(["sar"]+[str(i) for i in line]))
if die: if self.die:
break break
list(map(s.cancel, s.queue)) list(map(s.cancel, s.queue))
thread.join() thread.join()
# This runs if we were stopped by SIGTERM and no plot was made so far # This runs if we were stopped by SIGTERM and no plot was made so far
if not dont_plot: if not self.dont_plot:
summarize(session) self.summarize()
graph.graph(session, tmpfs_color, other_cache_color) graph.graph(self.session, self.tmpfs_color, self.other_cache_color)
def watch_psutil(session, fsdev, iface, tmpfs_color, other_cache_color): class PsUtilWatcher(Watcher):
# Was a graph already produced by save command from sargraph?
dont_plot = False
s = sched.scheduler(time.time, time.sleep)
sar_ev = s.enter(0, 1, psutil_sar_simulation, (s,))
mem_ev = s.enter(0, 1, get_meminfo, (s,))
thread = Thread(target = s.run)
thread.start()
initialize_darwin(session)
signal.signal(signal.SIGTERM, kill_handler)
# Make stdin nonblocking to continue working when no command is sent def initialize(self, _ = None):
flags = fcntl.fcntl(sys.stdin, fcntl.F_GETFL) global TOTAL_RAM
fcntl.fcntl(sys.stdin, fcntl.F_SETFL, flags | os.O_NONBLOCK) global TOTAL_GPU_RAM
while 1:
# Await sar output or a command sent from command handler in sargraph.py
readlist = [sys.stdin]
rlist, _, _ = select.select(readlist, [], [], 0.25)
now = datetime.datetime.now()
if handle_command(session, s, dont_plot, tmpfs_color, other_cache_color, now):
break
list(map(s.cancel, s.queue))
thread.join()
# This runs if we were stopped by SIGTERM and no plot was made so far
if not dont_plot:
summarize(session)
graph.graph(session, tmpfs_color, other_cache_color)
def handle_command(session, s, dont_plot, tmpfs_color, other_cache_color, now): TOTAL_RAM = int(psutil.virtual_memory().total / 1024)
global die
label_line = sys.stdin.readline().replace("\n", "")
if label_line.startswith("command:"):
label_line = label_line[len("command:"):]
if label_line.startswith("q:"):
label_line = label_line[len("q:"):]
list(map(s.cancel, s.queue)) cpus = psutil.cpu_count(logical=True)
summarize(session)
if label_line == "none":
pass
elif label_line:
graph.graph(session, tmpfs_color, other_cache_color, label_line)
elif not dont_plot:
graph.graph(session, tmpfs_color, other_cache_color)
dont_plot = True
die = 1
return True
elif label_line.startswith("s:"):
label_line = label_line[len("s:"):]
dont_plot = True cpu_name = platform.processor() or "unknown"
if label_line != "none": header = [
summarize(session) f"# psutil version: {psutil.__version__}",
if not label_line: f"pid: {os.getpid()}",
graph.graph(session, tmpfs_color, other_cache_color) f"machine: {platform.system()}",
else: f"cpu count: {cpus}",
graph.graph(session, tmpfs_color, other_cache_color, label_line) f"cpu: {cpu_name}"
elif label_line.startswith('label:'): ]
label_line = label_line[len('label:'):] self.logger.info(", ".join(header))
with open(f"{session}.txt", "a") as f:
timestamp = now.strftime("%Y-%m-%d-%H:%M:%S")
print(f"# {timestamp} label: {label_line}", file=f)
return False
# sar is not available on macOS. This function creates the sar behavior, but use psutil instead. # sar is not available on macOS. This function creates the sar behavior, but use psutil instead.
def psutil_sar_simulation(scheduler): def psutil_sar_simulation(self, scheduler: sched.scheduler):
global START_DATE global START_DATE
global TOTAL_LOAD global TOTAL_LOAD
global SAMPLE_NUMBER global SAMPLE_NUMBER
@ -549,7 +556,7 @@ def psutil_sar_simulation(scheduler):
global FS_NAME global FS_NAME
global END_DATE global END_DATE
scheduler.enter(1, 1, psutil_sar_simulation, (scheduler,)) scheduler.enter(1, 1, self.psutil_sar_simulation, (scheduler,))
now = datetime.datetime.now() now = datetime.datetime.now()
date = now.strftime("%Y-%m-%d") date = now.strftime("%Y-%m-%d")
daytime = now.strftime("%H:%M:%S") daytime = now.strftime("%H:%M:%S")
@ -599,4 +606,36 @@ def psutil_sar_simulation(scheduler):
curr_tx / 128, curr_tx / 128,
] ]
logger.info(" ".join(["sar"]+[str(i) for i in line])) self.logger.info(" ".join(["sar"]+[str(i) for i in line]))
def watch(self):
# Was a graph already produced by save command from sargraph?
dont_plot = False
s = sched.scheduler(time.time, time.sleep)
sar_ev = s.enter(0, 1, self.psutil_sar_simulation, (s,))
mem_ev = s.enter(0, 1, self.get_meminfo, (s,))
thread = Thread(target = s.run)
thread.start()
self.initialize()
socket_fd = self.socket.fileno()
while 1:
# Await sar output or a command sent from command handler in sargraph.py
readlist = [socket_fd]
rlist, _, _ = select.select(readlist, [], [], 0.25)
if socket_fd in rlist:
data = self.recv_data()
now = datetime.datetime.now()
if self.handle_command(data, s, now):
break
list(map(s.cancel, s.queue))
thread.join()
# This runs if we were stopped by SIGTERM and no plot was made so far
if not dont_plot:
self.summarize()
graph.graph(self.session, self.tmpfs_color, self.other_cache_color)
Write Preview
Loading…
Cancel
Save