Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added command line options to override those in benchmark file. #99

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Added command line options to override those in benchmark file. #99

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -217,10 +217,14 @@ FLAGS:
-V, --version Prints version information

OPTIONS:
-b, --benchmark <benchmark> Sets the benchmark file
-c, --compare <compare> Sets a compare file
-r, --report <report> Sets a report file
-t, --threshold <threshold> Sets a threshold value in ms amongst the compared file
-b, --benchmark <benchmark> Sets the benchmark file
-c, --compare <compare> Sets a compare file
-p, --concurrency <concurrency> Number of concurrent requests
-i, --iterations <iterations> Total number of requests to perform
-e, --rampup <rampup> Amount of time it takes to reach full concurrency
-r, --report <report> Sets a report file
-t, --threshold <threshold> Sets a threshold value in ms amongst the compared file
-u, --url <url> Base URL for requests

```

Expand Down
43 changes: 36 additions & 7 deletions src/benchmark.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,9 @@ use futures::stream::{self, StreamExt};

use serde_json::{json, Value};
use tokio::{runtime, time::delay_for};
use yaml_rust::{yaml, Yaml};

use crate::actions::{Report, Runnable};
use crate::actions::{Report, Request, Runnable};
use crate::config::Config;
use crate::expandable::include;
use crate::writer;
Expand All @@ -22,12 +23,29 @@ pub type Reports = Vec<Report>;
pub type PoolStore = HashMap<String, Client>;
pub type Pool = Arc<Mutex<PoolStore>>;

// represents un-validated user inputs
pub struct BenchmarkOptions<'a> {
pub benchmark_path_option: Option<&'a str>,
pub report_path_option: Option<&'a str>,
pub relaxed_interpolations: bool,
pub no_check_certificate: bool,
pub stats: bool,
pub compare_path_option: Option<&'a str>,
pub threshold_option: Option<f64>,
pub quiet: bool,
pub nanosec: bool,
pub concurrency_option: Option<usize>,
pub iterations_option: Option<usize>,
pub base_url_option: Option<&'a str>,
pub rampup_option: Option<usize>,
}

pub struct BenchmarkResult {
pub reports: Vec<Reports>,
pub duration: f64,
}

async fn run_iteration(benchmark: Arc<Benchmark>, pool: Pool, config: Arc<Config>, iteration: i64) -> Vec<Report> {
async fn run_iteration(benchmark: Arc<Benchmark>, pool: Pool, config: Arc<Config>, iteration: usize) -> Vec<Report> {
if config.rampup > 0 {
let delay = config.rampup / config.iterations;
delay_for(Duration::new((delay * iteration) as u64, 0)).await;
Expand All @@ -53,10 +71,11 @@ fn join<S: ToString>(l: Vec<S>, sep: &str) -> String {
)
}

pub fn execute(benchmark_path: &str, report_path_option: Option<&str>, relaxed_interpolations: bool, no_check_certificate: bool, quiet: bool, nanosec: bool) -> BenchmarkResult {
let config = Arc::new(Config::new(benchmark_path, relaxed_interpolations, no_check_certificate, quiet, nanosec));
pub fn execute(options: &BenchmarkOptions) -> BenchmarkResult {
// prepare config
let config = Arc::new(Config::new(&options));

if report_path_option.is_some() {
if options.report_path_option.is_some() {
println!("{}: {}. Ignoring {} and {} properties...", "Report mode".yellow(), "on".purple(), "concurrency".yellow(), "iterations".yellow());
} else {
println!("{} {}", "Concurrency".yellow(), config.concurrency.to_string().purple());
Expand All @@ -73,12 +92,22 @@ pub fn execute(benchmark_path: &str, report_path_option: Option<&str>, relaxed_i
let mut benchmark: Benchmark = Benchmark::new();
let pool_store: PoolStore = PoolStore::new();

include::expand_from_filepath(benchmark_path, &mut benchmark, Some("plan"));
if let Some(benchmark_path) = options.benchmark_path_option {
include::expand_from_filepath(benchmark_path, &mut benchmark, Some("plan"));
} else {
// if no benchmark plan is provided. then default to requesting the baseUrl itself.
let mut default_item = yaml::Hash::new();
default_item.insert(Yaml::from_str("name"), Yaml::from_str("Default"));
let mut url_item = yaml::Hash::new();
url_item.insert(Yaml::from_str("url"), Yaml::from_str("/"));
default_item.insert(Yaml::from_str("request"), Yaml::Hash(url_item));
benchmark.push(Box::new(Request::new(&Yaml::Hash(default_item), None, None)));
}

let benchmark = Arc::new(benchmark);
let pool = Arc::new(Mutex::new(pool_store));

if let Some(report_path) = report_path_option {
if let Some(report_path) = options.report_path_option {
let reports = run_iteration(benchmark.clone(), pool.clone(), config, 0).await;

writer::write_file(report_path, join(reports, ""));
Expand Down
9 changes: 2 additions & 7 deletions src/checker.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,7 @@ use yaml_rust::YamlLoader;

use crate::actions::Report;

pub fn compare(list_reports: &[Vec<Report>], filepath: &str, threshold: &str) -> Result<(), i32> {
let threshold_value = match threshold.parse::<f64>() {
Ok(v) => v,
_ => panic!("arrrgh"),
};

pub fn compare(list_reports: &[Vec<Report>], filepath: &str, threshold: f64) -> Result<(), i32> {
// Create a path to the desired file
let path = Path::new(filepath);
let display = path.display();
Expand Down Expand Up @@ -41,7 +36,7 @@ pub fn compare(list_reports: &[Vec<Report>], filepath: &str, threshold: &str) ->
let recorded_duration = items[i]["duration"].as_f64().unwrap();
let delta_ms = report_item.duration - recorded_duration;

if delta_ms > threshold_value {
if delta_ms > threshold {
println!("{:width$} is {}{} slower than before", report_item.name.green(), delta_ms.round().to_string().red(), "ms".red(), width = 25);

slow_counter += 1;
Expand Down
137 changes: 73 additions & 64 deletions src/config.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,99 +1,108 @@
use std::convert::TryFrom;

use yaml_rust::{Yaml, YamlLoader};

use crate::benchmark::Context;
use crate::benchmark::{BenchmarkOptions, Context};
use crate::interpolator;
use crate::reader;

const NITERATIONS: i64 = 1;
const NRAMPUP: i64 = 0;
const NCONCURRENCY: usize = 1;
const NITERATIONS: usize = 1;
const NRAMPUP: usize = 0;

pub struct Config {
pub base: String,
pub concurrency: i64,
pub iterations: i64,
pub concurrency: usize,
pub iterations: usize,
pub relaxed_interpolations: bool,
pub no_check_certificate: bool,
pub rampup: i64,
pub rampup: usize,
pub quiet: bool,
pub nanosec: bool,
}

impl Config {
pub fn new(path: &str, relaxed_interpolations: bool, no_check_certificate: bool, quiet: bool, nanosec: bool) -> Config {
let config_file = reader::read_file(path);
pub fn new(options: &BenchmarkOptions) -> Config {
let mut config = Config {
base: "".to_owned(),
concurrency: NCONCURRENCY,
iterations: NITERATIONS,
relaxed_interpolations: options.relaxed_interpolations,
no_check_certificate: options.no_check_certificate,
rampup: NRAMPUP,
quiet: options.quiet,
nanosec: options.nanosec,
};
// load options from benchmark file
if options.benchmark_path_option.is_some() {
let config_file = reader::read_file(options.benchmark_path_option.unwrap());

let config_docs = YamlLoader::load_from_str(config_file.as_str()).unwrap();
let config_doc = &config_docs[0];
let config_docs = YamlLoader::load_from_str(config_file.as_str()).unwrap();
let config_doc = &config_docs[0];

let context: Context = Context::new();
let interpolator = interpolator::Interpolator::new(&context);
let context: Context = Context::new();
let interpolator = interpolator::Interpolator::new(&context);

let iterations = read_i64_configuration(config_doc, &interpolator, "iterations", NITERATIONS);
let concurrency = read_i64_configuration(config_doc, &interpolator, "concurrency", iterations);
let rampup = read_i64_configuration(config_doc, &interpolator, "rampup", NRAMPUP);
let base = read_str_configuration(config_doc, &interpolator, "base", "");
if let Some(value) = read_i64_configuration(config_doc, &interpolator, "iterations") {
config.iterations = usize::try_from(value).expect("Expecting a positive integer value for 'iterations' parameter");
}
if let Some(value) = read_i64_configuration(config_doc, &interpolator, "concurrency") {
config.concurrency = usize::try_from(value).expect("Expecting a positive integer value for 'concurrency' parameter");
}
if let Some(value) = read_i64_configuration(config_doc, &interpolator, "rampup") {
config.rampup = usize::try_from(value).expect("Expecting a positive integer value for 'rampup' parameter");
}
if let Some(value) = read_str_configuration(config_doc, &interpolator, "base") {
config.base = value;
}
}
// overwrite defaults and options from benchmark file with those from command line (BenchmarkOptions struct)
if let Some(value) = options.base_url_option {
config.base = value.to_owned();
}
if let Some(value) = options.concurrency_option {
config.concurrency = value;
}
if let Some(value) = options.iterations_option {
config.iterations = value;
}
if let Some(value) = options.rampup_option {
config.rampup = value;
}

if concurrency > iterations {
if config.concurrency > config.iterations {
panic!("The concurrency can not be higher than the number of iterations")
}

Config {
base,
concurrency,
iterations,
relaxed_interpolations,
no_check_certificate,
rampup,
quiet,
nanosec,
}
return config;
}
}

fn read_str_configuration(config_doc: &Yaml, interpolator: &interpolator::Interpolator, name: &str, default: &str) -> String {
match config_doc[name].as_str() {
Some(value) => {
if value.contains('{') {
interpolator.resolve(&value, true).to_owned()
} else {
value.to_owned()
}
}
None => {
if config_doc[name].as_str().is_some() {
println!("Invalid {} value!", name);
}

default.to_owned()
fn read_str_configuration(config_doc: &Yaml, interpolator: &interpolator::Interpolator, name: &str) -> Option<String> {
if let Some(value) = config_doc[name].as_str() {
if value.contains('{') {
Some(interpolator.resolve(&value, true).to_owned())
} else {
Some(value.to_owned())
}
} else {
if config_doc[name].as_str().is_some() {
println!("Invalid {} value!", name)
};
None
}
}

fn read_i64_configuration(config_doc: &Yaml, interpolator: &interpolator::Interpolator, name: &str, default: i64) -> i64 {
let value = if let Some(value) = config_doc[name].as_i64() {
// Note: yaml_rust can't parse directly into usize yet, so must be i64 for now
fn read_i64_configuration(config_doc: &Yaml, interpolator: &interpolator::Interpolator, name: &str) -> Option<i64> {
if let Some(value) = config_doc[name].as_i64() {
Some(value)
} else if let Some(key) = config_doc[name].as_str() {
interpolator.resolve(&key, false).parse::<i64>().ok()
Some(interpolator.resolve(&key, false).parse::<i64>().expect(format!("Unable to parse benchmark option '{}' into i64", name).as_str()))
} else {
if config_doc[name].as_str().is_some() {
println!("Invalid {} value!", name)
};
None
};

match value {
Some(value) => {
if value < 0 {
println!("Invalid negative {} value!", name);

default
} else {
value
}
}
None => {
if config_doc[name].as_str().is_some() {
println!("Invalid {} value!", name);
}

default
}
}
}
62 changes: 46 additions & 16 deletions src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ mod reader;
mod writer;

use crate::actions::Report;
use crate::benchmark::BenchmarkOptions;
use clap::crate_version;
use clap::{App, Arg};
use colored::*;
Expand All @@ -17,23 +18,48 @@ use std::f64;
use std::process;

fn main() {
// validate user command line input
let matches = app_args();
let benchmark_file = matches.value_of("benchmark").unwrap();
let report_path_option = matches.value_of("report");
let stats_option = matches.is_present("stats");
let compare_path_option = matches.value_of("compare");
let threshold_option = matches.value_of("threshold");
let no_check_certificate = matches.is_present("no-check-certificate");
let relaxed_interpolations = matches.is_present("relaxed-interpolations");
let quiet = matches.is_present("quiet");
let nanosec = matches.is_present("nanosec");

let benchmark_result = benchmark::execute(benchmark_file, report_path_option, relaxed_interpolations, no_check_certificate, quiet, nanosec);
let options = BenchmarkOptions {
benchmark_path_option: matches.value_of("benchmark"),
report_path_option: matches.value_of("report"),
stats: matches.is_present("stats"),
compare_path_option: matches.value_of("compare"),
threshold_option: if let Some(threshold) = matches.value_of("threshold") {
Some(threshold.parse::<f64>().expect("Command line parameter 'threshold' value must be a positive numerical value"))
} else {
None
},
no_check_certificate: matches.is_present("no-check-certificate"),
relaxed_interpolations: matches.is_present("relaxed-interpolations"),
quiet: matches.is_present("quiet"),
nanosec: matches.is_present("nanosec"),
concurrency_option: if let Some(concurrency) = matches.value_of("concurrency") {
Some(concurrency.parse::<usize>().expect("Command line parameter 'concurrency' value must be a positive integer"))
} else {
None
},
base_url_option: matches.value_of("url"),
iterations_option: if let Some(iterations) = matches.value_of("iterations") {
Some(iterations.parse::<usize>().expect("Command line parameter 'iterations' value must be a positive integer"))
} else {
None
},
rampup_option: if let Some(rampup) = matches.value_of("rampup") {
Some(rampup.parse::<usize>().expect("Command line parameter 'rampup' value must be a positive integer"))
} else {
None
},
};

// run the benchmark
let benchmark_result = benchmark::execute(&options);

// process reports and statistics
let list_reports = benchmark_result.reports;
let duration = benchmark_result.duration;

show_stats(&list_reports, stats_option, nanosec, duration);
compare_benchmark(&list_reports, compare_path_option, threshold_option);
show_stats(&list_reports, options.stats, options.nanosec, duration);
compare_benchmark(&list_reports, options.compare_path_option, options.threshold_option);

process::exit(0)
}
Expand All @@ -42,7 +68,7 @@ fn app_args<'a>() -> clap::ArgMatches<'a> {
App::new("drill")
.version(crate_version!())
.about("HTTP load testing application written in Rust inspired by Ansible syntax")
.arg(Arg::with_name("benchmark").help("Sets the benchmark file").long("benchmark").short("b").required(true).takes_value(true))
.arg(Arg::with_name("benchmark").help("Sets the benchmark file").long("benchmark").short("b").required_unless("url").takes_value(true))
.arg(Arg::with_name("stats").short("s").long("stats").help("Shows request statistics").takes_value(false).conflicts_with("compare"))
.arg(Arg::with_name("report").short("r").long("report").help("Sets a report file").takes_value(true).conflicts_with("compare"))
.arg(Arg::with_name("compare").short("c").long("compare").help("Sets a compare file").takes_value(true).conflicts_with("report"))
Expand All @@ -51,6 +77,10 @@ fn app_args<'a>() -> clap::ArgMatches<'a> {
.arg(Arg::with_name("no-check-certificate").long("no-check-certificate").help("Disables SSL certification check. (Not recommended)").takes_value(false))
.arg(Arg::with_name("quiet").short("q").long("quiet").help("Disables output").takes_value(false))
.arg(Arg::with_name("nanosec").short("n").long("nanosec").help("Shows statistics in nanoseconds").takes_value(false))
.arg(Arg::with_name("concurrency").short("p").long("concurrency").help("Sets the number of parallel/concurrent requests (overrides benchmark file)").takes_value(true))
.arg(Arg::with_name("iterations").short("i").long("iterations").help("Sets the total number of requests to perform (overrides benchmark file)").takes_value(true))
.arg(Arg::with_name("url").short("u").long("url").help("Sets the base URL for requests (overrides benchmark file)").required_unless("benchmark").takes_value(true))
.arg(Arg::with_name("rampup").short("e").long("rampup").help("Sets the amount of time it takes to reach full concurrency (overrides benchmark file)").takes_value(true))
.get_matches()
}

Expand Down Expand Up @@ -146,7 +176,7 @@ fn show_stats(list_reports: &[Vec<Report>], stats_option: bool, nanosec: bool, d
println!("{:width2$} {}", "Sample standard deviation".yellow(), format_time(global_stats.stdev_duration, nanosec).purple(), width2 = 25);
}

fn compare_benchmark(list_reports: &[Vec<Report>], compare_path_option: Option<&str>, threshold_option: Option<&str>) {
fn compare_benchmark(list_reports: &[Vec<Report>], compare_path_option: Option<&str>, threshold_option: Option<f64>) {
if let Some(compare_path) = compare_path_option {
if let Some(threshold) = threshold_option {
let compare_result = checker::compare(&list_reports, compare_path, threshold);
Expand Down