diff --git a/README.md b/README.md index 67877e0..975ff96 100644 --- a/README.md +++ b/README.md @@ -20,15 +20,21 @@ terminal. Type `lowcharts --help`, or `lowcharts PLOT-TYPE --help` for a complete list of options. -[![Sample histogram with lowcharts](resources/histogram-example.png)](resources/histogram-example.png) +Currently three basic types of plots are supported: + +#### Bar chart for matches in the input + +Since `grep -c` does not aggregate counts per pattern, this is maybe my most frequent use case. -Currently two basic types of plots are supported: +This chart is generated using `lowcharts matches database.log SELECT UPDATE DELETE INSERT DROP`: + +[![Simple bar chart with lowcharts](resources/matches-example.png)](resources/matches-example.png) #### Histogram This chart is generated using `python3 -c 'import random; [print(random.normalvariate(5, 5)) for _ in range(100000)]' | lowcharts hist`: - +[![Sample histogram with lowcharts](resources/histogram-example.png)](resources/histogram-example.png) This was inspired by [data-hacks](https://github.com/bitly/data_hacks). However, for some big log files I found that project was slower of what I would diff --git a/resources/matches-example.png b/resources/matches-example.png new file mode 100644 index 0000000..8fd5ee9 Binary files /dev/null and b/resources/matches-example.png differ diff --git a/src/app.rs b/src/app.rs index 6100dcf..6ccc893 100644 --- a/src/app.rs +++ b/src/app.rs @@ -1,7 +1,6 @@ -use clap::{self, App, Arg, AppSettings}; - -fn add_common_options (app: App) -> App { +use clap::{self, App, AppSettings, Arg}; +fn add_common_options(app: App) -> App { const LONG_RE_ABOUT: &str = "\ A regular expression used for capturing the values to be plotted inside input lines. @@ -17,48 +16,45 @@ group) and 'a(a)? (?P[0-9.]+)' (where there are two capture groups, and the named one will be used). "; - app - .arg( - Arg::new("max") - .long("max") - .short('M') - .about("Filter out values bigger than this") - .takes_value(true) - ) - .arg( - Arg::new("min") - .long("min") - .short('m') - .about("Filter out values smaller than this") - .takes_value(true) - ) - .arg( - Arg::new("width") - .long("width") - .short('w') - .about("Use this many characters as terminal width") - .default_value("110") - .takes_value(true) - ) - .arg( - Arg::new("regex") - .long("regex") - .short('R') - .about("Use a regex to capture input values") - .long_about(LONG_RE_ABOUT) - .takes_value(true) - ) - .arg( - Arg::new("input") - .about("Input file") - .default_value("-") - .long_about("If not present or a single dash, standard input will be used") - ) - + app.arg( + Arg::new("max") + .long("max") + .short('M') + .about("Filter out values bigger than this") + .takes_value(true), + ) + .arg( + Arg::new("min") + .long("min") + .short('m') + .about("Filter out values smaller than this") + .takes_value(true), + ) + .arg( + Arg::new("width") + .long("width") + .short('w') + .about("Use this many characters as terminal width") + .default_value("110") + .takes_value(true), + ) + .arg( + Arg::new("regex") + .long("regex") + .short('R') + .about("Use a regex to capture input values") + .long_about(LONG_RE_ABOUT) + .takes_value(true), + ) + .arg( + Arg::new("input") + .about("Input file") + .default_value("-") + .long_about("If not present or a single dash, standard input will be used"), + ) } pub fn get_app() -> App<'static> { - let mut hist = App::new("hist") .version(clap::crate_version!()) .setting(AppSettings::ColoredHelp) @@ -69,7 +65,7 @@ pub fn get_app() -> App<'static> { .short('i') .about("Use no more than this amount of buckets to classify data") .default_value("20") - .takes_value(true) + .takes_value(true), ); hist = add_common_options(hist); @@ -77,17 +73,44 @@ pub fn get_app() -> App<'static> { let mut plot = App::new("plot") .version(clap::crate_version!()) .setting(AppSettings::ColoredHelp) - .about("Plot an 2d plot where y-values are averages of input values") + .about("Plot an 2d x-y graph where y-values are averages of input values") .arg( Arg::new("height") .long("height") .short('h') .about("Use that many `rows` for the plot") .default_value("40") - .takes_value(true) + .takes_value(true), ); plot = add_common_options(plot); + let matches = App::new("matches") + .version(clap::crate_version!()) + .setting(AppSettings::ColoredHelp) + .setting(AppSettings::AllowMissingPositional) + .about("Plot barchar with counts of occurences of matches params") + .arg( + Arg::new("width") + .long("width") + .short('w') + .about("Use this many characters as terminal width") + .default_value("110") + .takes_value(true), + ) + .arg( + Arg::new("input") + .about("Input file") + .required(true) + .long_about("If not present or a single dash, standard input will be used"), + ) + .arg( + Arg::new("match") + .about("Count maches for those strings") + .required(true) + .takes_value(true) + .multiple(true), + ); + App::new("lowcharts") .author(clap::crate_authors!()) .version(clap::crate_version!()) @@ -101,17 +124,18 @@ pub fn get_app() -> App<'static> { .long("color") .about("Use colors in the output") .possible_values(&["auto", "no", "yes"]) - .takes_value(true) + .takes_value(true), ) .arg( Arg::new("verbose") .short('v') .long("verbose") .about("Be more verbose") - .takes_value(false) + .takes_value(false), ) .subcommand(hist) .subcommand(plot) + .subcommand(matches) } #[cfg(test)] @@ -138,7 +162,16 @@ mod tests { #[test] fn plot_subcommand_arg_parsing() { - let arg_vec = vec!["lowcharts", "plot", "--max", "1.1", "-m", "0.9", "--height", "11"]; + let arg_vec = vec![ + "lowcharts", + "plot", + "--max", + "1.1", + "-m", + "0.9", + "--height", + "11", + ]; let m = get_app().get_matches_from(arg_vec); assert!(!m.is_present("verbose")); if let Some(sub_m) = m.subcommand_matches("plot") { diff --git a/src/histogram.rs b/src/histogram.rs index 10cc677..c5e521c 100644 --- a/src/histogram.rs +++ b/src/histogram.rs @@ -6,7 +6,7 @@ use yansi::Color::{Blue, Green, Red}; use crate::stats::Stats; #[derive(Debug)] -pub struct Bucket { +struct Bucket { range: Range, count: usize, } @@ -33,20 +33,20 @@ pub struct Histogram { impl Histogram { pub fn new(size: usize, step: f64, stats: Stats) -> Histogram { - let mut b = Histogram { - vec: Vec::with_capacity(size), + let mut vec = Vec::::with_capacity(size); + let mut lower = stats.min; + for _ in 0..size { + vec.push(Bucket::new(lower..lower + step)); + lower += step; + } + Histogram { + vec, max: stats.min + (step * size as f64), step, top: 0, last: size - 1, stats, - }; - let mut lower = b.stats.min; - for _ in 0..size { - b.vec.push(Bucket::new(lower..lower + step)); - lower += step; } - b } pub fn load(&mut self, vec: &[f64]) { @@ -110,7 +110,6 @@ impl HistWriter { width: usize, width_count: usize, ) -> fmt::Result { - let bar = Red.paint(format!("{:∎ reader::DataReader { let min = matches.value_of_t("min").unwrap_or(f64::NEG_INFINITY); let max = matches.value_of_t("max").unwrap_or(f64::INFINITY); if min > max { - eprintln!("[{}] Minimum should be smaller than maximum", Red.paint("ERROR")); + eprintln!( + "[{}] Minimum should be smaller than maximum", + Red.paint("ERROR") + ); std::process::exit(1); } builder.range(min..max); @@ -57,52 +60,69 @@ fn get_reader(matches: &ArgMatches, verbose: bool) -> reader::DataReader { builder.build().unwrap() } +fn histogram(matches: &ArgMatches, verbose: bool) { + let reader = get_reader(&matches, verbose); + let vec = reader.read(matches.value_of("input").unwrap()); + if vec.is_empty() { + eprintln!("[{}] No data to process", Yellow.paint("WARN")); + std::process::exit(0); + } + let stats = stats::Stats::new(&vec); + let width = matches.value_of_t("width").unwrap(); + let mut intervals: usize = matches.value_of_t("intervals").unwrap(); + + intervals = intervals.min(vec.len()); + let mut histogram = + histogram::Histogram::new(intervals, (stats.max - stats.min) / intervals as f64, stats); + histogram.load(&vec); + println!("{:width$}", histogram, width = width); +} + +fn plot(matches: &ArgMatches, verbose: bool) { + let reader = get_reader(&matches, verbose); + let vec = reader.read(matches.value_of("input").unwrap()); + if vec.is_empty() { + eprintln!("[{}] No data to process", Yellow.paint("WARN")); + std::process::exit(0); + } + let mut plot = plot::Plot::new( + matches.value_of_t("width").unwrap(), + matches.value_of_t("height").unwrap(), + stats::Stats::new(&vec), + ); + plot.load(&vec); + print!("{}", plot); +} + +fn matchbar(matches: &ArgMatches) { + let reader = reader::DataReader::default(); + let width = matches.value_of_t("width").unwrap(); + print!( + "{:width$}", + reader.read_matches( + matches.value_of("input").unwrap(), + matches.values_of("match").unwrap().collect() + ), + width = width + ); +} fn main() { let matches = app::get_app().get_matches(); - + let verbose = matches.is_present("verbose"); if let Some(c) = matches.value_of("color") { disable_color_if_needed(c); } - - let sub_matches = match matches.subcommand_name() { - Some("hist") => { - matches.subcommand_matches("hist").unwrap() - }, - Some("plot") => { - matches.subcommand_matches("plot").unwrap() - }, - _ => { - eprintln!("[{}] Invalid subcommand", Red.paint("ERROR")); - std::process::exit(1); + match matches.subcommand() { + Some(("hist", subcommand_matches)) => { + histogram(subcommand_matches, verbose); } - }; - let reader = get_reader(&sub_matches, matches.is_present("verbose")); - - let vec = reader.read(sub_matches.value_of("input").unwrap_or("-")); - if vec.is_empty() { - eprintln!("[{}]: No data to process", Yellow.paint("WARN")); - std::process::exit(0); - } - let stats = stats::Stats::new(&vec); - let width = sub_matches.value_of_t("width").unwrap(); - match matches.subcommand_name() { - Some("hist") => { - let mut intervals: usize = sub_matches.value_of_t("intervals").unwrap(); - intervals = intervals.min(vec.len()); - let mut histogram = histogram::Histogram::new( - intervals, - (stats.max - stats.min) / intervals as f64, - stats, - ); - histogram.load(&vec); - println!("{:width$}", histogram, width = width); - }, - Some("plot") => { - let mut plot = plot::Plot::new(width, sub_matches.value_of_t("height").unwrap(), stats); - plot.load(&vec); - print!("{}", plot); - }, - _ => () + Some(("plot", subcommand_matches)) => { + plot(subcommand_matches, verbose); + } + Some(("matches", subcommand_matches)) => { + matchbar(subcommand_matches); + } + _ => unreachable!("Invalid subcommand"), }; } diff --git a/src/matchbar.rs b/src/matchbar.rs new file mode 100644 index 0000000..73863df --- /dev/null +++ b/src/matchbar.rs @@ -0,0 +1,107 @@ +use std::fmt; + +use yansi::Color::{Blue, Green, Red}; + +#[derive(Debug)] +pub struct MatchBarRow { + pub label: String, + pub count: usize, +} + +impl MatchBarRow { + pub fn new(string: &str) -> MatchBarRow { + MatchBarRow { + label: string.to_string(), + count: 0, + } + } + + pub fn inc_if_matches(&mut self, line: &str) { + if line.contains(&self.label) { + self.count += 1; + } + } +} + +#[derive(Debug)] +pub struct MatchBar { + pub vec: Vec, + top_values: usize, + top_lenght: usize, +} + +impl MatchBar { + pub fn new(vec: Vec) -> MatchBar { + let mut top_lenght: usize = 0; + let mut top_values: usize = 0; + for row in vec.iter() { + top_lenght = top_lenght.max(row.label.len()); + top_values = top_values.max(row.count); + } + MatchBar { + vec, + top_lenght, + top_values, + } + } +} + +impl fmt::Display for MatchBar { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let width = f.width().unwrap_or(100); + let divisor = 1.max(self.top_values / width); + let width_count = format!("{}", self.top_values).len(); + writeln!( + f, + "Matches: {}.", + Blue.paint(format!( + "{}", + self.vec.iter().map(|r| r.count).sum::() + )), + )?; + writeln!( + f, + "Each {} represents a count of {}", + Red.paint("∎"), + Blue.paint(divisor.to_string()), + )?; + for row in self.vec.iter() { + writeln!( + f, + "[{label}] [{count}] {bar}", + label = Blue.paint(format!("{:width$}", row.label, width = self.top_lenght)), + count = Green.paint(format!("{:width$}", row.count, width = width_count)), + bar = Red.paint(format!("{:∎ Vec { - let mut vec: Vec = vec![]; - match path { - "-" => { - vec = self.read_data(io::stdin().lock().lines()); - } - _ => { - let file = File::open(path); - match file { - Ok(fd) => { - vec = self.read_data(io::BufReader::new(fd).lines()); - } - Err(error) => eprintln!("[{}]: {}", Red.paint("ERROR"), error), - } - } - } - vec - } - - fn read_data(&self, lines: std::io::Lines) -> Vec { let mut vec: Vec = Vec::new(); let line_parser = match self.regex { Some(_) => Self::parse_regex, None => Self::parse_float, }; - for line in lines { + for line in open_file(path).lines() { match line { Ok(as_string) => { if let Some(n) = line_parser(&self, &as_string) { @@ -101,6 +84,42 @@ impl DataReader { } } } + + pub fn read_matches(&self, path: &str, strings: Vec<&str>) -> MatchBar { + let mut rows = Vec::::with_capacity(strings.len()); + for s in strings { + rows.push(MatchBarRow::new(s)); + } + for line in open_file(path).lines() { + match line { + Ok(as_string) => { + for row in rows.iter_mut() { + row.inc_if_matches(&as_string); + } + } + Err(error) => eprintln!("[{}]: {}", Red.paint("ERROR"), error), + } + } + MatchBar::new(rows) + } +} + +fn open_file(path: &str) -> Box { + match path { + "-" => Box::new(BufReader::new(io::stdin())), + _ => match File::open(path) { + Ok(fd) => Box::new(io::BufReader::new(fd)), + Err(error) => { + eprintln!( + "[{}] Could not open {}: {}", + Red.paint("ERROR"), + path, + error + ); + std::process::exit(0); + } + }, + } } #[cfg(test)] @@ -194,4 +213,29 @@ mod tests { Err(_) => assert!(false, "Could not create temp file"), } } + + #[test] + fn basic_match_reader() { + let reader = DataReader::default(); + match NamedTempFile::new() { + Ok(ref mut file) => { + writeln!(file, "foobar").unwrap(); + writeln!(file, "data data foobar").unwrap(); + writeln!(file, "data data").unwrap(); + writeln!(file, "foobar").unwrap(); + writeln!(file, "none").unwrap(); + let mb = reader.read_matches( + file.path().to_str().unwrap(), + vec!["random", "foobar", "data"], + ); + assert_eq!(mb.vec[0].label, "random"); + assert_eq!(mb.vec[0].count, 0); + assert_eq!(mb.vec[1].label, "foobar"); + assert_eq!(mb.vec[1].count, 3); + assert_eq!(mb.vec[2].label, "data"); + assert_eq!(mb.vec[2].count, 2); + } + Err(_) => assert!(false, "Could not create temp file"), + } + } } diff --git a/src/stats.rs b/src/stats.rs index 2b4d5b0..0c7d781 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -81,9 +81,9 @@ mod tests { let stats = Stats::new(&[1.1, 3.3, 2.2]); Paint::disable(); let display = format!("{}", stats); - assert!(display.find("Samples = 3").is_some()); - assert!(display.find("Min = 1.1").is_some()); - assert!(display.find("Max = 3.3").is_some()); - assert!(display.find("Average = 2.2").is_some()); + assert!(display.contains("Samples = 3")); + assert!(display.contains("Min = 1.1")); + assert!(display.contains("Max = 3.3")); + assert!(display.contains("Average = 2.2")); } }