Browse Source

Implement the matches sub-command for bar charts

pull/2/head
JuanLeon Lahoz 5 years ago
parent
commit
c8edd3de1e
  1. 12
      README.md
  2. BIN
      resources/matches-example.png
  3. 67
      src/app.rs
  4. 27
      src/histogram.rs
  5. 102
      src/main.rs
  6. 107
      src/matchbar.rs
  7. 8
      src/plot.rs
  8. 86
      src/reader.rs
  9. 8
      src/stats.rs

12
README.md

@ -20,15 +20,21 @@ terminal.
Type `lowcharts --help`, or `lowcharts PLOT-TYPE --help` for a complete list of
options.
[![Sample histogram with lowcharts](resources/histogram-example.png)](resources/histogram-example.png)
Currently three basic types of plots are supported:
#### Bar chart for matches in the input
Since `grep -c` does not aggregate counts per pattern, this is maybe my most frequent use case.
Currently two basic types of plots are supported:
This chart is generated using `lowcharts matches database.log SELECT UPDATE DELETE INSERT DROP`:
[![Simple bar chart with lowcharts](resources/matches-example.png)](resources/matches-example.png)
#### Histogram
This chart is generated using `python3 -c 'import random; [print(random.normalvariate(5, 5)) for _ in range(100000)]' | lowcharts hist`:
[![Sample histogram with lowcharts](resources/histogram-example.png)](resources/histogram-example.png)
This was inspired by [data-hacks](https://github.com/bitly/data_hacks).
However, for some big log files I found that project was slower of what I would

BIN
resources/matches-example.png

Binary file not shown.

After

Width:  |  Height:  |  Size: 9.9 KiB

67
src/app.rs

@ -1,7 +1,6 @@
use clap::{self, App, Arg, AppSettings};
use clap::{self, App, AppSettings, Arg};
fn add_common_options(app: App) -> App {
const LONG_RE_ABOUT: &str = "\
A regular expression used for capturing the values to be plotted inside input
lines.
@ -17,20 +16,19 @@ group) and 'a(a)? (?P<value>[0-9.]+)' (where there are two capture groups, and
the named one will be used).
";
app
.arg(
app.arg(
Arg::new("max")
.long("max")
.short('M')
.about("Filter out values bigger than this")
.takes_value(true)
.takes_value(true),
)
.arg(
Arg::new("min")
.long("min")
.short('m')
.about("Filter out values smaller than this")
.takes_value(true)
.takes_value(true),
)
.arg(
Arg::new("width")
@ -38,7 +36,7 @@ the named one will be used).
.short('w')
.about("Use this many characters as terminal width")
.default_value("110")
.takes_value(true)
.takes_value(true),
)
.arg(
Arg::new("regex")
@ -46,19 +44,17 @@ the named one will be used).
.short('R')
.about("Use a regex to capture input values")
.long_about(LONG_RE_ABOUT)
.takes_value(true)
.takes_value(true),
)
.arg(
Arg::new("input")
.about("Input file")
.default_value("-")
.long_about("If not present or a single dash, standard input will be used")
.long_about("If not present or a single dash, standard input will be used"),
)
}
pub fn get_app() -> App<'static> {
let mut hist = App::new("hist")
.version(clap::crate_version!())
.setting(AppSettings::ColoredHelp)
@ -69,7 +65,7 @@ pub fn get_app() -> App<'static> {
.short('i')
.about("Use no more than this amount of buckets to classify data")
.default_value("20")
.takes_value(true)
.takes_value(true),
);
hist = add_common_options(hist);
@ -77,17 +73,44 @@ pub fn get_app() -> App<'static> {
let mut plot = App::new("plot")
.version(clap::crate_version!())
.setting(AppSettings::ColoredHelp)
.about("Plot an 2d plot where y-values are averages of input values")
.about("Plot an 2d x-y graph where y-values are averages of input values")
.arg(
Arg::new("height")
.long("height")
.short('h')
.about("Use that many `rows` for the plot")
.default_value("40")
.takes_value(true)
.takes_value(true),
);
plot = add_common_options(plot);
let matches = App::new("matches")
.version(clap::crate_version!())
.setting(AppSettings::ColoredHelp)
.setting(AppSettings::AllowMissingPositional)
.about("Plot barchar with counts of occurences of matches params")
.arg(
Arg::new("width")
.long("width")
.short('w')
.about("Use this many characters as terminal width")
.default_value("110")
.takes_value(true),
)
.arg(
Arg::new("input")
.about("Input file")
.required(true)
.long_about("If not present or a single dash, standard input will be used"),
)
.arg(
Arg::new("match")
.about("Count maches for those strings")
.required(true)
.takes_value(true)
.multiple(true),
);
App::new("lowcharts")
.author(clap::crate_authors!())
.version(clap::crate_version!())
@ -101,17 +124,18 @@ pub fn get_app() -> App<'static> {
.long("color")
.about("Use colors in the output")
.possible_values(&["auto", "no", "yes"])
.takes_value(true)
.takes_value(true),
)
.arg(
Arg::new("verbose")
.short('v')
.long("verbose")
.about("Be more verbose")
.takes_value(false)
.takes_value(false),
)
.subcommand(hist)
.subcommand(plot)
.subcommand(matches)
}
#[cfg(test)]
@ -138,7 +162,16 @@ mod tests {
#[test]
fn plot_subcommand_arg_parsing() {
let arg_vec = vec!["lowcharts", "plot", "--max", "1.1", "-m", "0.9", "--height", "11"];
let arg_vec = vec![
"lowcharts",
"plot",
"--max",
"1.1",
"-m",
"0.9",
"--height",
"11",
];
let m = get_app().get_matches_from(arg_vec);
assert!(!m.is_present("verbose"));
if let Some(sub_m) = m.subcommand_matches("plot") {

27
src/histogram.rs

@ -6,7 +6,7 @@ use yansi::Color::{Blue, Green, Red};
use crate::stats::Stats;
#[derive(Debug)]
pub struct Bucket {
struct Bucket {
range: Range<f64>,
count: usize,
}
@ -33,20 +33,20 @@ pub struct Histogram {
impl Histogram {
pub fn new(size: usize, step: f64, stats: Stats) -> Histogram {
let mut b = Histogram {
vec: Vec::with_capacity(size),
let mut vec = Vec::<Bucket>::with_capacity(size);
let mut lower = stats.min;
for _ in 0..size {
vec.push(Bucket::new(lower..lower + step));
lower += step;
}
Histogram {
vec,
max: stats.min + (step * size as f64),
step,
top: 0,
last: size - 1,
stats,
};
let mut lower = b.stats.min;
for _ in 0..size {
b.vec.push(Bucket::new(lower..lower + step));
lower += step;
}
b
}
pub fn load(&mut self, vec: &[f64]) {
@ -110,7 +110,6 @@ impl HistWriter {
width: usize,
width_count: usize,
) -> fmt::Result {
let bar = Red.paint(format!("{:∎<width$}", "", width = bucket.count / divisor));
writeln!(
f,
"[{range}] [{count}] {bar}",
@ -121,7 +120,7 @@ impl HistWriter {
width = width,
)),
count = Green.paint(format!("{:width$}", bucket.count, width = width_count)),
bar = bar
bar = Red.paint(format!("{:∎<width$}", "", width = bucket.count / divisor)),
)
}
@ -172,8 +171,8 @@ mod tests {
]);
Paint::disable();
let display = format!("{}", hist);
assert!(display.find("[-2.000 .. 0.500] [3] ∎∎∎\n").is_some());
assert!(display.find("[ 0.500 .. 3.000] [8] ∎∎∎∎∎∎∎∎\n").is_some());
assert!(display.find("[10.500 .. 13.000] [2] ∎∎\n").is_some());
assert!(display.contains("[-2.000 .. 0.500] [3] ∎∎∎\n"));
assert!(display.contains("[ 0.500 .. 3.000] [8] ∎∎∎∎∎∎∎∎\n"));
assert!(display.contains("[10.500 .. 13.000] [2] ∎∎\n"));
}
}

102
src/main.rs

@ -3,18 +3,18 @@ use std::env;
use isatty::stdout_isatty;
use regex::Regex;
use yansi::Color::Red;
use yansi::Color::Yellow;
use yansi::Color::{Red, Yellow};
use yansi::Paint;
#[macro_use]
extern crate derive_builder;
mod app;
mod histogram;
mod matchbar;
mod plot;
mod reader;
mod stats;
mod app;
fn disable_color_if_needed(option: &str) {
match option {
@ -38,7 +38,10 @@ fn get_reader(matches: &ArgMatches, verbose: bool) -> reader::DataReader {
let min = matches.value_of_t("min").unwrap_or(f64::NEG_INFINITY);
let max = matches.value_of_t("max").unwrap_or(f64::INFINITY);
if min > max {
eprintln!("[{}] Minimum should be smaller than maximum", Red.paint("ERROR"));
eprintln!(
"[{}] Minimum should be smaller than maximum",
Red.paint("ERROR")
);
std::process::exit(1);
}
builder.range(min..max);
@ -57,52 +60,69 @@ fn get_reader(matches: &ArgMatches, verbose: bool) -> reader::DataReader {
builder.build().unwrap()
}
fn main() {
let matches = app::get_app().get_matches();
if let Some(c) = matches.value_of("color") {
disable_color_if_needed(c);
fn histogram(matches: &ArgMatches, verbose: bool) {
let reader = get_reader(&matches, verbose);
let vec = reader.read(matches.value_of("input").unwrap());
if vec.is_empty() {
eprintln!("[{}] No data to process", Yellow.paint("WARN"));
std::process::exit(0);
}
let stats = stats::Stats::new(&vec);
let width = matches.value_of_t("width").unwrap();
let mut intervals: usize = matches.value_of_t("intervals").unwrap();
let sub_matches = match matches.subcommand_name() {
Some("hist") => {
matches.subcommand_matches("hist").unwrap()
},
Some("plot") => {
matches.subcommand_matches("plot").unwrap()
},
_ => {
eprintln!("[{}] Invalid subcommand", Red.paint("ERROR"));
std::process::exit(1);
intervals = intervals.min(vec.len());
let mut histogram =
histogram::Histogram::new(intervals, (stats.max - stats.min) / intervals as f64, stats);
histogram.load(&vec);
println!("{:width$}", histogram, width = width);
}
};
let reader = get_reader(&sub_matches, matches.is_present("verbose"));
let vec = reader.read(sub_matches.value_of("input").unwrap_or("-"));
fn plot(matches: &ArgMatches, verbose: bool) {
let reader = get_reader(&matches, verbose);
let vec = reader.read(matches.value_of("input").unwrap());
if vec.is_empty() {
eprintln!("[{}]: No data to process", Yellow.paint("WARN"));
eprintln!("[{}] No data to process", Yellow.paint("WARN"));
std::process::exit(0);
}
let stats = stats::Stats::new(&vec);
let width = sub_matches.value_of_t("width").unwrap();
match matches.subcommand_name() {
Some("hist") => {
let mut intervals: usize = sub_matches.value_of_t("intervals").unwrap();
intervals = intervals.min(vec.len());
let mut histogram = histogram::Histogram::new(
intervals,
(stats.max - stats.min) / intervals as f64,
stats,
let mut plot = plot::Plot::new(
matches.value_of_t("width").unwrap(),
matches.value_of_t("height").unwrap(),
stats::Stats::new(&vec),
);
histogram.load(&vec);
println!("{:width$}", histogram, width = width);
},
Some("plot") => {
let mut plot = plot::Plot::new(width, sub_matches.value_of_t("height").unwrap(), stats);
plot.load(&vec);
print!("{}", plot);
},
_ => ()
}
fn matchbar(matches: &ArgMatches) {
let reader = reader::DataReader::default();
let width = matches.value_of_t("width").unwrap();
print!(
"{:width$}",
reader.read_matches(
matches.value_of("input").unwrap(),
matches.values_of("match").unwrap().collect()
),
width = width
);
}
fn main() {
let matches = app::get_app().get_matches();
let verbose = matches.is_present("verbose");
if let Some(c) = matches.value_of("color") {
disable_color_if_needed(c);
}
match matches.subcommand() {
Some(("hist", subcommand_matches)) => {
histogram(subcommand_matches, verbose);
}
Some(("plot", subcommand_matches)) => {
plot(subcommand_matches, verbose);
}
Some(("matches", subcommand_matches)) => {
matchbar(subcommand_matches);
}
_ => unreachable!("Invalid subcommand"),
};
}

107
src/matchbar.rs

@ -0,0 +1,107 @@
use std::fmt;
use yansi::Color::{Blue, Green, Red};
#[derive(Debug)]
pub struct MatchBarRow {
pub label: String,
pub count: usize,
}
impl MatchBarRow {
pub fn new(string: &str) -> MatchBarRow {
MatchBarRow {
label: string.to_string(),
count: 0,
}
}
pub fn inc_if_matches(&mut self, line: &str) {
if line.contains(&self.label) {
self.count += 1;
}
}
}
#[derive(Debug)]
pub struct MatchBar {
pub vec: Vec<MatchBarRow>,
top_values: usize,
top_lenght: usize,
}
impl MatchBar {
pub fn new(vec: Vec<MatchBarRow>) -> MatchBar {
let mut top_lenght: usize = 0;
let mut top_values: usize = 0;
for row in vec.iter() {
top_lenght = top_lenght.max(row.label.len());
top_values = top_values.max(row.count);
}
MatchBar {
vec,
top_lenght,
top_values,
}
}
}
impl fmt::Display for MatchBar {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
let width = f.width().unwrap_or(100);
let divisor = 1.max(self.top_values / width);
let width_count = format!("{}", self.top_values).len();
writeln!(
f,
"Matches: {}.",
Blue.paint(format!(
"{}",
self.vec.iter().map(|r| r.count).sum::<usize>()
)),
)?;
writeln!(
f,
"Each {} represents a count of {}",
Red.paint("∎"),
Blue.paint(divisor.to_string()),
)?;
for row in self.vec.iter() {
writeln!(
f,
"[{label}] [{count}] {bar}",
label = Blue.paint(format!("{:width$}", row.label, width = self.top_lenght)),
count = Green.paint(format!("{:width$}", row.count, width = width_count)),
bar = Red.paint(format!("{:∎<width$}", "", width = row.count / divisor))
)?;
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use yansi::Paint;
#[test]
fn test_matchbar() {
let mut row0 = MatchBarRow::new("label1");
row0.inc_if_matches("labelN");
row0.inc_if_matches("label1");
row0.inc_if_matches("label1");
row0.inc_if_matches("label11");
let mut row1 = MatchBarRow::new("label2");
row1.inc_if_matches("label2");
let mb = MatchBar::new(vec![row0, row1, MatchBarRow::new("label333")]);
assert_eq!(mb.top_lenght, 8);
assert_eq!(mb.top_values, 3);
Paint::disable();
let display = format!("{}", mb);
assert!(display.contains("[label1 ] [3] ∎∎∎\n"));
assert!(display.contains("[label2 ] [1] ∎\n"));
assert!(display.contains("[label333] [0] \n"));
assert!(display.contains("represents a count of 1"));
assert!(display.contains("Matches: 4"));
}
}

8
src/plot.rs

@ -104,9 +104,9 @@ mod tests {
plot.load(&[-1.0, 0.0, 1.0, 2.0, 3.0, 4.0, -1.0]);
Paint::disable();
let display = format!("{}", plot);
assert!(display.find("[3] ● ").is_some());
assert!(display.find("[2] ").is_some());
assert!(display.find("[1] ● ").is_some());
assert!(display.find("[-1] ● ●").is_some());
assert!(display.contains("[3] ● "));
assert!(display.contains("[2] "));
assert!(display.contains("[1] ● "));
assert!(display.contains("[-1] ● ●"));
}
}

86
src/reader.rs

@ -1,10 +1,12 @@
use std::fs::File;
use std::io::{self, BufRead};
use std::io::{self, BufRead, BufReader};
use std::ops::Range;
use regex::Regex;
use yansi::Color::{Magenta, Red};
use crate::matchbar::{MatchBar, MatchBarRow};
#[derive(Debug, Default, Builder)]
pub struct DataReader {
#[builder(setter(strip_option), default)]
@ -17,31 +19,12 @@ pub struct DataReader {
impl DataReader {
pub fn read(&self, path: &str) -> Vec<f64> {
let mut vec: Vec<f64> = vec![];
match path {
"-" => {
vec = self.read_data(io::stdin().lock().lines());
}
_ => {
let file = File::open(path);
match file {
Ok(fd) => {
vec = self.read_data(io::BufReader::new(fd).lines());
}
Err(error) => eprintln!("[{}]: {}", Red.paint("ERROR"), error),
}
}
}
vec
}
fn read_data<T: BufRead>(&self, lines: std::io::Lines<T>) -> Vec<f64> {
let mut vec: Vec<f64> = Vec::new();
let line_parser = match self.regex {
Some(_) => Self::parse_regex,
None => Self::parse_float,
};
for line in lines {
for line in open_file(path).lines() {
match line {
Ok(as_string) => {
if let Some(n) = line_parser(&self, &as_string) {
@ -101,6 +84,42 @@ impl DataReader {
}
}
}
pub fn read_matches(&self, path: &str, strings: Vec<&str>) -> MatchBar {
let mut rows = Vec::<MatchBarRow>::with_capacity(strings.len());
for s in strings {
rows.push(MatchBarRow::new(s));
}
for line in open_file(path).lines() {
match line {
Ok(as_string) => {
for row in rows.iter_mut() {
row.inc_if_matches(&as_string);
}
}
Err(error) => eprintln!("[{}]: {}", Red.paint("ERROR"), error),
}
}
MatchBar::new(rows)
}
}
fn open_file(path: &str) -> Box<dyn io::BufRead> {
match path {
"-" => Box::new(BufReader::new(io::stdin())),
_ => match File::open(path) {
Ok(fd) => Box::new(io::BufReader::new(fd)),
Err(error) => {
eprintln!(
"[{}] Could not open {}: {}",
Red.paint("ERROR"),
path,
error
);
std::process::exit(0);
}
},
}
}
#[cfg(test)]
@ -194,4 +213,29 @@ mod tests {
Err(_) => assert!(false, "Could not create temp file"),
}
}
#[test]
fn basic_match_reader() {
let reader = DataReader::default();
match NamedTempFile::new() {
Ok(ref mut file) => {
writeln!(file, "foobar").unwrap();
writeln!(file, "data data foobar").unwrap();
writeln!(file, "data data").unwrap();
writeln!(file, "foobar").unwrap();
writeln!(file, "none").unwrap();
let mb = reader.read_matches(
file.path().to_str().unwrap(),
vec!["random", "foobar", "data"],
);
assert_eq!(mb.vec[0].label, "random");
assert_eq!(mb.vec[0].count, 0);
assert_eq!(mb.vec[1].label, "foobar");
assert_eq!(mb.vec[1].count, 3);
assert_eq!(mb.vec[2].label, "data");
assert_eq!(mb.vec[2].count, 2);
}
Err(_) => assert!(false, "Could not create temp file"),
}
}
}

8
src/stats.rs

@ -81,9 +81,9 @@ mod tests {
let stats = Stats::new(&[1.1, 3.3, 2.2]);
Paint::disable();
let display = format!("{}", stats);
assert!(display.find("Samples = 3").is_some());
assert!(display.find("Min = 1.1").is_some());
assert!(display.find("Max = 3.3").is_some());
assert!(display.find("Average = 2.2").is_some());
assert!(display.contains("Samples = 3"));
assert!(display.contains("Min = 1.1"));
assert!(display.contains("Max = 3.3"));
assert!(display.contains("Average = 2.2"));
}
}

Loading…
Cancel
Save