From 58aae8c0929428f528a3d8274f2a429f2da30e25 Mon Sep 17 00:00:00 2001 From: WebeWizard Date: Tue, 9 Feb 2021 22:44:44 -0700 Subject: [PATCH] Added command line options to override benchmark file. Use types better to protect against bad user input --- README.md | 12 +++-- src/benchmark.rs | 43 ++++++++++++--- src/checker.rs | 9 +--- src/config.rs | 137 +++++++++++++++++++++++++---------------------- src/main.rs | 62 +++++++++++++++------ 5 files changed, 165 insertions(+), 98 deletions(-) diff --git a/README.md b/README.md index 5aa2a58..db465f4 100644 --- a/README.md +++ b/README.md @@ -217,10 +217,14 @@ FLAGS: -V, --version Prints version information OPTIONS: - -b, --benchmark Sets the benchmark file - -c, --compare Sets a compare file - -r, --report Sets a report file - -t, --threshold Sets a threshold value in ms amongst the compared file + -b, --benchmark Sets the benchmark file + -c, --compare Sets a compare file + -p, --concurrency Number of concurrent requests + -i, --iterations Total number of requests to perform + -e, --rampup Amount of time it takes to reach full concurrency + -r, --report Sets a report file + -t, --threshold Sets a threshold value in ms amongst the compared file + -u, --url Base URL for requests ``` diff --git a/src/benchmark.rs b/src/benchmark.rs index 69a257c..fa3c564 100644 --- a/src/benchmark.rs +++ b/src/benchmark.rs @@ -6,8 +6,9 @@ use futures::stream::{self, StreamExt}; use serde_json::{json, Value}; use tokio::{runtime, time::delay_for}; +use yaml_rust::{yaml, Yaml}; -use crate::actions::{Report, Runnable}; +use crate::actions::{Report, Request, Runnable}; use crate::config::Config; use crate::expandable::include; use crate::writer; @@ -22,12 +23,29 @@ pub type Reports = Vec; pub type PoolStore = HashMap; pub type Pool = Arc>; +// represents un-validated user inputs +pub struct BenchmarkOptions<'a> { + pub benchmark_path_option: Option<&'a str>, + pub report_path_option: Option<&'a str>, + pub relaxed_interpolations: bool, + pub no_check_certificate: bool, + pub stats: bool, + pub compare_path_option: Option<&'a str>, + pub threshold_option: Option, + pub quiet: bool, + pub nanosec: bool, + pub concurrency_option: Option, + pub iterations_option: Option, + pub base_url_option: Option<&'a str>, + pub rampup_option: Option, +} + pub struct BenchmarkResult { pub reports: Vec, pub duration: f64, } -async fn run_iteration(benchmark: Arc, pool: Pool, config: Arc, iteration: i64) -> Vec { +async fn run_iteration(benchmark: Arc, pool: Pool, config: Arc, iteration: usize) -> Vec { if config.rampup > 0 { let delay = config.rampup / config.iterations; delay_for(Duration::new((delay * iteration) as u64, 0)).await; @@ -53,10 +71,11 @@ fn join(l: Vec, sep: &str) -> String { ) } -pub fn execute(benchmark_path: &str, report_path_option: Option<&str>, relaxed_interpolations: bool, no_check_certificate: bool, quiet: bool, nanosec: bool) -> BenchmarkResult { - let config = Arc::new(Config::new(benchmark_path, relaxed_interpolations, no_check_certificate, quiet, nanosec)); +pub fn execute(options: &BenchmarkOptions) -> BenchmarkResult { + // prepare config + let config = Arc::new(Config::new(&options)); - if report_path_option.is_some() { + if options.report_path_option.is_some() { println!("{}: {}. Ignoring {} and {} properties...", "Report mode".yellow(), "on".purple(), "concurrency".yellow(), "iterations".yellow()); } else { println!("{} {}", "Concurrency".yellow(), config.concurrency.to_string().purple()); @@ -73,12 +92,22 @@ pub fn execute(benchmark_path: &str, report_path_option: Option<&str>, relaxed_i let mut benchmark: Benchmark = Benchmark::new(); let pool_store: PoolStore = PoolStore::new(); - include::expand_from_filepath(benchmark_path, &mut benchmark, Some("plan")); + if let Some(benchmark_path) = options.benchmark_path_option { + include::expand_from_filepath(benchmark_path, &mut benchmark, Some("plan")); + } else { + // if no benchmark plan is provided. then default to requesting the baseUrl itself. + let mut default_item = yaml::Hash::new(); + default_item.insert(Yaml::from_str("name"), Yaml::from_str("Default")); + let mut url_item = yaml::Hash::new(); + url_item.insert(Yaml::from_str("url"), Yaml::from_str("/")); + default_item.insert(Yaml::from_str("request"), Yaml::Hash(url_item)); + benchmark.push(Box::new(Request::new(&Yaml::Hash(default_item), None, None))); + } let benchmark = Arc::new(benchmark); let pool = Arc::new(Mutex::new(pool_store)); - if let Some(report_path) = report_path_option { + if let Some(report_path) = options.report_path_option { let reports = run_iteration(benchmark.clone(), pool.clone(), config, 0).await; writer::write_file(report_path, join(reports, "")); diff --git a/src/checker.rs b/src/checker.rs index 9ee3009..ce9833d 100644 --- a/src/checker.rs +++ b/src/checker.rs @@ -7,12 +7,7 @@ use yaml_rust::YamlLoader; use crate::actions::Report; -pub fn compare(list_reports: &[Vec], filepath: &str, threshold: &str) -> Result<(), i32> { - let threshold_value = match threshold.parse::() { - Ok(v) => v, - _ => panic!("arrrgh"), - }; - +pub fn compare(list_reports: &[Vec], filepath: &str, threshold: f64) -> Result<(), i32> { // Create a path to the desired file let path = Path::new(filepath); let display = path.display(); @@ -41,7 +36,7 @@ pub fn compare(list_reports: &[Vec], filepath: &str, threshold: &str) -> let recorded_duration = items[i]["duration"].as_f64().unwrap(); let delta_ms = report_item.duration - recorded_duration; - if delta_ms > threshold_value { + if delta_ms > threshold { println!("{:width$} is {}{} slower than before", report_item.name.green(), delta_ms.round().to_string().red(), "ms".red(), width = 25); slow_counter += 1; diff --git a/src/config.rs b/src/config.rs index 0a91cbf..ac40c01 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,99 +1,108 @@ +use std::convert::TryFrom; + use yaml_rust::{Yaml, YamlLoader}; -use crate::benchmark::Context; +use crate::benchmark::{BenchmarkOptions, Context}; use crate::interpolator; use crate::reader; -const NITERATIONS: i64 = 1; -const NRAMPUP: i64 = 0; +const NCONCURRENCY: usize = 1; +const NITERATIONS: usize = 1; +const NRAMPUP: usize = 0; pub struct Config { pub base: String, - pub concurrency: i64, - pub iterations: i64, + pub concurrency: usize, + pub iterations: usize, pub relaxed_interpolations: bool, pub no_check_certificate: bool, - pub rampup: i64, + pub rampup: usize, pub quiet: bool, pub nanosec: bool, } impl Config { - pub fn new(path: &str, relaxed_interpolations: bool, no_check_certificate: bool, quiet: bool, nanosec: bool) -> Config { - let config_file = reader::read_file(path); + pub fn new(options: &BenchmarkOptions) -> Config { + let mut config = Config { + base: "".to_owned(), + concurrency: NCONCURRENCY, + iterations: NITERATIONS, + relaxed_interpolations: options.relaxed_interpolations, + no_check_certificate: options.no_check_certificate, + rampup: NRAMPUP, + quiet: options.quiet, + nanosec: options.nanosec, + }; + // load options from benchmark file + if options.benchmark_path_option.is_some() { + let config_file = reader::read_file(options.benchmark_path_option.unwrap()); - let config_docs = YamlLoader::load_from_str(config_file.as_str()).unwrap(); - let config_doc = &config_docs[0]; + let config_docs = YamlLoader::load_from_str(config_file.as_str()).unwrap(); + let config_doc = &config_docs[0]; - let context: Context = Context::new(); - let interpolator = interpolator::Interpolator::new(&context); + let context: Context = Context::new(); + let interpolator = interpolator::Interpolator::new(&context); - let iterations = read_i64_configuration(config_doc, &interpolator, "iterations", NITERATIONS); - let concurrency = read_i64_configuration(config_doc, &interpolator, "concurrency", iterations); - let rampup = read_i64_configuration(config_doc, &interpolator, "rampup", NRAMPUP); - let base = read_str_configuration(config_doc, &interpolator, "base", ""); + if let Some(value) = read_i64_configuration(config_doc, &interpolator, "iterations") { + config.iterations = usize::try_from(value).expect("Expecting a positive integer value for 'iterations' parameter"); + } + if let Some(value) = read_i64_configuration(config_doc, &interpolator, "concurrency") { + config.concurrency = usize::try_from(value).expect("Expecting a positive integer value for 'concurrency' parameter"); + } + if let Some(value) = read_i64_configuration(config_doc, &interpolator, "rampup") { + config.rampup = usize::try_from(value).expect("Expecting a positive integer value for 'rampup' parameter"); + } + if let Some(value) = read_str_configuration(config_doc, &interpolator, "base") { + config.base = value; + } + } + // overwrite defaults and options from benchmark file with those from command line (BenchmarkOptions struct) + if let Some(value) = options.base_url_option { + config.base = value.to_owned(); + } + if let Some(value) = options.concurrency_option { + config.concurrency = value; + } + if let Some(value) = options.iterations_option { + config.iterations = value; + } + if let Some(value) = options.rampup_option { + config.rampup = value; + } - if concurrency > iterations { + if config.concurrency > config.iterations { panic!("The concurrency can not be higher than the number of iterations") } - Config { - base, - concurrency, - iterations, - relaxed_interpolations, - no_check_certificate, - rampup, - quiet, - nanosec, - } + return config; } } -fn read_str_configuration(config_doc: &Yaml, interpolator: &interpolator::Interpolator, name: &str, default: &str) -> String { - match config_doc[name].as_str() { - Some(value) => { - if value.contains('{') { - interpolator.resolve(&value, true).to_owned() - } else { - value.to_owned() - } - } - None => { - if config_doc[name].as_str().is_some() { - println!("Invalid {} value!", name); - } - - default.to_owned() +fn read_str_configuration(config_doc: &Yaml, interpolator: &interpolator::Interpolator, name: &str) -> Option { + if let Some(value) = config_doc[name].as_str() { + if value.contains('{') { + Some(interpolator.resolve(&value, true).to_owned()) + } else { + Some(value.to_owned()) } + } else { + if config_doc[name].as_str().is_some() { + println!("Invalid {} value!", name) + }; + None } } -fn read_i64_configuration(config_doc: &Yaml, interpolator: &interpolator::Interpolator, name: &str, default: i64) -> i64 { - let value = if let Some(value) = config_doc[name].as_i64() { +// Note: yaml_rust can't parse directly into usize yet, so must be i64 for now +fn read_i64_configuration(config_doc: &Yaml, interpolator: &interpolator::Interpolator, name: &str) -> Option { + if let Some(value) = config_doc[name].as_i64() { Some(value) } else if let Some(key) = config_doc[name].as_str() { - interpolator.resolve(&key, false).parse::().ok() + Some(interpolator.resolve(&key, false).parse::().expect(format!("Unable to parse benchmark option '{}' into i64", name).as_str())) } else { + if config_doc[name].as_str().is_some() { + println!("Invalid {} value!", name) + }; None - }; - - match value { - Some(value) => { - if value < 0 { - println!("Invalid negative {} value!", name); - - default - } else { - value - } - } - None => { - if config_doc[name].as_str().is_some() { - println!("Invalid {} value!", name); - } - - default - } } } diff --git a/src/main.rs b/src/main.rs index fc60ffc..b893d11 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,6 +8,7 @@ mod reader; mod writer; use crate::actions::Report; +use crate::benchmark::BenchmarkOptions; use clap::crate_version; use clap::{App, Arg}; use colored::*; @@ -17,23 +18,48 @@ use std::f64; use std::process; fn main() { + // validate user command line input let matches = app_args(); - let benchmark_file = matches.value_of("benchmark").unwrap(); - let report_path_option = matches.value_of("report"); - let stats_option = matches.is_present("stats"); - let compare_path_option = matches.value_of("compare"); - let threshold_option = matches.value_of("threshold"); - let no_check_certificate = matches.is_present("no-check-certificate"); - let relaxed_interpolations = matches.is_present("relaxed-interpolations"); - let quiet = matches.is_present("quiet"); - let nanosec = matches.is_present("nanosec"); - - let benchmark_result = benchmark::execute(benchmark_file, report_path_option, relaxed_interpolations, no_check_certificate, quiet, nanosec); + let options = BenchmarkOptions { + benchmark_path_option: matches.value_of("benchmark"), + report_path_option: matches.value_of("report"), + stats: matches.is_present("stats"), + compare_path_option: matches.value_of("compare"), + threshold_option: if let Some(threshold) = matches.value_of("threshold") { + Some(threshold.parse::().expect("Command line parameter 'threshold' value must be a positive numerical value")) + } else { + None + }, + no_check_certificate: matches.is_present("no-check-certificate"), + relaxed_interpolations: matches.is_present("relaxed-interpolations"), + quiet: matches.is_present("quiet"), + nanosec: matches.is_present("nanosec"), + concurrency_option: if let Some(concurrency) = matches.value_of("concurrency") { + Some(concurrency.parse::().expect("Command line parameter 'concurrency' value must be a positive integer")) + } else { + None + }, + base_url_option: matches.value_of("url"), + iterations_option: if let Some(iterations) = matches.value_of("iterations") { + Some(iterations.parse::().expect("Command line parameter 'iterations' value must be a positive integer")) + } else { + None + }, + rampup_option: if let Some(rampup) = matches.value_of("rampup") { + Some(rampup.parse::().expect("Command line parameter 'rampup' value must be a positive integer")) + } else { + None + }, + }; + + // run the benchmark + let benchmark_result = benchmark::execute(&options); + + // process reports and statistics let list_reports = benchmark_result.reports; let duration = benchmark_result.duration; - - show_stats(&list_reports, stats_option, nanosec, duration); - compare_benchmark(&list_reports, compare_path_option, threshold_option); + show_stats(&list_reports, options.stats, options.nanosec, duration); + compare_benchmark(&list_reports, options.compare_path_option, options.threshold_option); process::exit(0) } @@ -42,7 +68,7 @@ fn app_args<'a>() -> clap::ArgMatches<'a> { App::new("drill") .version(crate_version!()) .about("HTTP load testing application written in Rust inspired by Ansible syntax") - .arg(Arg::with_name("benchmark").help("Sets the benchmark file").long("benchmark").short("b").required(true).takes_value(true)) + .arg(Arg::with_name("benchmark").help("Sets the benchmark file").long("benchmark").short("b").required_unless("url").takes_value(true)) .arg(Arg::with_name("stats").short("s").long("stats").help("Shows request statistics").takes_value(false).conflicts_with("compare")) .arg(Arg::with_name("report").short("r").long("report").help("Sets a report file").takes_value(true).conflicts_with("compare")) .arg(Arg::with_name("compare").short("c").long("compare").help("Sets a compare file").takes_value(true).conflicts_with("report")) @@ -51,6 +77,10 @@ fn app_args<'a>() -> clap::ArgMatches<'a> { .arg(Arg::with_name("no-check-certificate").long("no-check-certificate").help("Disables SSL certification check. (Not recommended)").takes_value(false)) .arg(Arg::with_name("quiet").short("q").long("quiet").help("Disables output").takes_value(false)) .arg(Arg::with_name("nanosec").short("n").long("nanosec").help("Shows statistics in nanoseconds").takes_value(false)) + .arg(Arg::with_name("concurrency").short("p").long("concurrency").help("Sets the number of parallel/concurrent requests (overrides benchmark file)").takes_value(true)) + .arg(Arg::with_name("iterations").short("i").long("iterations").help("Sets the total number of requests to perform (overrides benchmark file)").takes_value(true)) + .arg(Arg::with_name("url").short("u").long("url").help("Sets the base URL for requests (overrides benchmark file)").required_unless("benchmark").takes_value(true)) + .arg(Arg::with_name("rampup").short("e").long("rampup").help("Sets the amount of time it takes to reach full concurrency (overrides benchmark file)").takes_value(true)) .get_matches() } @@ -146,7 +176,7 @@ fn show_stats(list_reports: &[Vec], stats_option: bool, nanosec: bool, d println!("{:width2$} {}", "Sample standard deviation".yellow(), format_time(global_stats.stdev_duration, nanosec).purple(), width2 = 25); } -fn compare_benchmark(list_reports: &[Vec], compare_path_option: Option<&str>, threshold_option: Option<&str>) { +fn compare_benchmark(list_reports: &[Vec], compare_path_option: Option<&str>, threshold_option: Option) { if let Some(compare_path) = compare_path_option { if let Some(threshold) = threshold_option { let compare_result = checker::compare(&list_reports, compare_path, threshold);