From 4fa4eb10722985e5e779d52ab06c146ecaa0d233 Mon Sep 17 00:00:00 2001 From: ucyo Date: Tue, 29 Nov 2022 12:28:13 +0100 Subject: [PATCH 01/11] Add first possible annotation syntax --- examples/duplicate.toml | 4 ++++ src/core.rs | 1 + 2 files changed, 5 insertions(+) diff --git a/examples/duplicate.toml b/examples/duplicate.toml index 137c36d..163a226 100644 --- a/examples/duplicate.toml +++ b/examples/duplicate.toml @@ -41,6 +41,10 @@ setup = "which ls" shell = "none" command = "dd if={ifile} of=/tmp/Cargo.toml.dd" +[run.dd.annotations] +filesize = "ls -al /tmp/Cargo.toml.dd | awk '{{print $5}}'" +modified = "ls -al /tmp/Cargo.toml.dd | awk '{{print $6\" \"$7\" \"$8}}'" + [run.cp] command = "cp {ifile} /tmp/Cargo.toml.cp" diff --git a/src/core.rs b/src/core.rs index 7a4137d..5bf53cc 100644 --- a/src/core.rs +++ b/src/core.rs @@ -64,6 +64,7 @@ pub(crate) struct Run { prepare: Option, setup: Option, shell: Option, + annotations: Option>, command: String, } From 58f615c7373f91f77013453f09c7bf3eabaf79ed Mon Sep 17 00:00:00 2001 From: ucyo Date: Tue, 29 Nov 2022 19:52:11 +0100 Subject: [PATCH 02/11] Cleanup --- src/main.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main.rs b/src/main.rs index 364dad0..9bba215 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,13 +10,13 @@ mod util; /// Command-line Interface (CLI) for the hypcmp library #[derive(Parser, Debug)] #[clap(author, version, about, long_about = None)] -pub struct Cli { +struct Cli { /// Configuration file [*.toml] #[clap(value_parser)] - pub config: PathBuf, + config: PathBuf, #[clap(flatten)] - pub verbose: clap_verbosity_flag::Verbosity, + verbose: clap_verbosity_flag::Verbosity, } fn main() -> std::io::Result<()> { From 966be5117291702a5ae367a05c8936f80e980f38 Mon Sep 17 00:00:00 2001 From: ucyo Date: Wed, 30 Nov 2022 14:51:59 +0100 Subject: [PATCH 03/11] Add scaffolding for code --- src/core.rs | 8 +++++--- src/util.rs | 23 +++++++++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/src/core.rs b/src/core.rs index 5bf53cc..0985dcb 100644 --- a/src/core.rs +++ b/src/core.rs @@ -181,12 +181,14 @@ impl Hyperfined for Run { } None => (), } - match &self.cleanup { - Some(cmd) => { + match (&self.cleanup, &self.annotations) { + (Some(cmd), Some(hm)) => unimplemented!(), + (None, Some(hm)) => unimplemented!(), + (Some(cmd), None) => { result.push("--cleanup".to_string()); result.push(cmd.clone()); } - None => (), + (None, None) => (), } match &self.prepare { Some(cmd) => { diff --git a/src/util.rs b/src/util.rs index aceab7c..93f729e 100644 --- a/src/util.rs +++ b/src/util.rs @@ -251,3 +251,26 @@ fn move_commit_label_to_cmd_name(mut json: Value) -> std::io::Result String { + format!( + "jq --arg {0} $({1}) '.results[0].{0}=${0}' {2}", + key, cmd, json + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn jq() { + let key = "filesize"; + let cmd = "ls -al /tmp/Cargo.toml.dd | awk '{{print $5}}'"; + let json = "out.json"; + let expected = "jq --arg filesize $(ls -al /tmp/Cargo.toml.dd | awk '{{print $5}}') '.results[0].filesize=$filesize' out.json"; + + assert_eq!(expected, generate_jq_cmd(key, cmd, json)) + } +} From 09358664533c9b994986101c0aae3511ed8e3c6d Mon Sep 17 00:00:00 2001 From: ucyo Date: Tue, 6 Dec 2022 21:39:46 +0100 Subject: [PATCH 04/11] Export function for json included --- src/core.rs | 8 ++++++++ src/main.rs | 18 ++++++++++-------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/core.rs b/src/core.rs index 0985dcb..7c9fba7 100644 --- a/src/core.rs +++ b/src/core.rs @@ -12,6 +12,14 @@ use std::{collections::HashMap, fmt::Display, fs::File, io::Read, path::PathBuf} /// Transformation to hyperfine parameters pub trait Hyperfined { fn to_hyperfine(&self) -> Vec; + fn to_hyperfine_with_json(&self, json: &str) -> Vec { + let mut params = self.to_hyperfine(); + params.push("--export-json".to_string()); + params.push(json.to_string()); + params + } + // fn to_hyperfine_with_annotations(&self, annotations: &str) -> Vec; + // fn to_hyperfine_with_json_and_annotations(&self, json: &str, annotations: &str) -> Vec; } /// Configuration for a complete Benchmark set consisting of several Runs diff --git a/src/main.rs b/src/main.rs index 9bba215..5cde8ba 100644 --- a/src/main.rs +++ b/src/main.rs @@ -42,23 +42,25 @@ fn main() -> std::io::Result<()> { for (label, run) in c.run.iter() { debug!("Run: {run:?}"); + // Initiate `hyperfine` command let mut cmd = Command::new("hyperfine"); - cmd.args(c.to_hyperfine()); - let mut name = vec!["--command-name".to_string()]; - name.push(label.clone()); - cmd.args(name); - - let mut json = vec!["--export-json".to_string()]; + // Add json output to arguments let mut filename = label.clone(); filename.push_str(".json"); let output = dir.path().join(filename).display().to_string(); - json.push(output.clone()); - cmd.args(json); + cmd.args(c.to_hyperfine_with_json(&output)); + + // Add command name to arguments + let mut name = vec!["--command-name".to_string()]; + name.push(label.clone()); + cmd.args(name); + // Add run specific arguments cmd.args(run.to_hyperfine()); info!("Running: {cmd:?}"); + // Execute command let result = cmd.output()?; if result.status.success() { debug!("Benchmark run successful"); From 5148e6ad35a948d6f89b70389a1c22089e2738e9 Mon Sep 17 00:00:00 2001 From: ucyo Date: Tue, 6 Dec 2022 23:47:02 +0100 Subject: [PATCH 05/11] Reverse former edit --- src/core.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/core.rs b/src/core.rs index 7c9fba7..2f8ae7c 100644 --- a/src/core.rs +++ b/src/core.rs @@ -189,14 +189,12 @@ impl Hyperfined for Run { } None => (), } - match (&self.cleanup, &self.annotations) { - (Some(cmd), Some(hm)) => unimplemented!(), - (None, Some(hm)) => unimplemented!(), - (Some(cmd), None) => { + match &self.cleanup { + Some(cmd) => { result.push("--cleanup".to_string()); result.push(cmd.clone()); } - (None, None) => (), + None => (), } match &self.prepare { Some(cmd) => { From 78587cf03eb5d8a4f43be443a4210af9fa9b851b Mon Sep 17 00:00:00 2001 From: ucyo Date: Tue, 6 Dec 2022 23:48:50 +0100 Subject: [PATCH 06/11] Add annotations to cleanup process - In case there is a cleanup process already setup, then the annotation tasks will be prepended. Otherwise a cleanup process will be added and all annotations will be executed --- src/core.rs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/core.rs b/src/core.rs index 2f8ae7c..96e518b 100644 --- a/src/core.rs +++ b/src/core.rs @@ -18,8 +18,23 @@ pub trait Hyperfined { params.push(json.to_string()); params } - // fn to_hyperfine_with_annotations(&self, annotations: &str) -> Vec; - // fn to_hyperfine_with_json_and_annotations(&self, json: &str, annotations: &str) -> Vec; + fn to_hyperfine_with_json_and_annotations(&self, json: &str, kw: &HashMap) -> Vec { + let mut params = self.to_hyperfine_with_json(json); + let mut annotations: Vec<_> = kw.iter().map(|(k, v)| util::generate_jq_cmd(&k, &v, json)).collect(); + match ¶ms.iter().position(|x| x == &"--cleanup".to_string()) { + Some(ix) => { + // Cleanup used in command and annotations must be prepand + annotations.push(params.get(ix+1).unwrap().clone()); + params.insert(ix + 1, annotations.join("&&")); + }, + None => { + // Cleanup not used, annotations can be added as cleanup + params.push("--cleanup".to_string()); + params.push(annotations.join("&&")); + } + } + params + } } /// Configuration for a complete Benchmark set consisting of several Runs From b069342e86f5f429636f007d0d51ccd9c397a5db Mon Sep 17 00:00:00 2001 From: ucyo Date: Tue, 6 Dec 2022 23:57:31 +0100 Subject: [PATCH 07/11] Refer to annotations externally --- src/core.rs | 2 +- src/main.rs | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/core.rs b/src/core.rs index 96e518b..00484e6 100644 --- a/src/core.rs +++ b/src/core.rs @@ -87,7 +87,7 @@ pub(crate) struct Run { prepare: Option, setup: Option, shell: Option, - annotations: Option>, + pub annotations: Option>, command: String, } diff --git a/src/main.rs b/src/main.rs index 5cde8ba..e22e64c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -57,7 +57,10 @@ fn main() -> std::io::Result<()> { cmd.args(name); // Add run specific arguments - cmd.args(run.to_hyperfine()); + match &run.annotations { + Some(hm) => cmd.args(run.to_hyperfine_with_json_and_annotations(&output, hm)), + None => cmd.args(run.to_hyperfine()), + }; info!("Running: {cmd:?}"); // Execute command From 1d3afbae1a42afa0e57a8a16d245b5dd89250721 Mon Sep 17 00:00:00 2001 From: ucyo Date: Tue, 6 Dec 2022 23:59:30 +0100 Subject: [PATCH 08/11] Fix json output --- src/core.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/core.rs b/src/core.rs index 00484e6..1be12b7 100644 --- a/src/core.rs +++ b/src/core.rs @@ -19,18 +19,19 @@ pub trait Hyperfined { params } fn to_hyperfine_with_json_and_annotations(&self, json: &str, kw: &HashMap) -> Vec { - let mut params = self.to_hyperfine_with_json(json); + let mut params = self.to_hyperfine(); let mut annotations: Vec<_> = kw.iter().map(|(k, v)| util::generate_jq_cmd(&k, &v, json)).collect(); match ¶ms.iter().position(|x| x == &"--cleanup".to_string()) { Some(ix) => { // Cleanup used in command and annotations must be prepand annotations.push(params.get(ix+1).unwrap().clone()); - params.insert(ix + 1, annotations.join("&&")); + params.insert(ix + 1, annotations.join(" && ")); + params.remove(ix + 2); }, None => { // Cleanup not used, annotations can be added as cleanup params.push("--cleanup".to_string()); - params.push(annotations.join("&&")); + params.push(annotations.join(" && ")); } } params From 7b05600b3456bf624b0fde7d83121bdfd6ef7181 Mon Sep 17 00:00:00 2001 From: ucyo Date: Wed, 7 Dec 2022 00:24:53 +0100 Subject: [PATCH 09/11] No cleanup --- src/main.rs | 2 +- src/util.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main.rs b/src/main.rs index e22e64c..3c3990a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -85,7 +85,7 @@ fn main() -> std::io::Result<()> { } else { let json = util::merge_json_files(&files_to_be_merged)?; util::write_json_to_disk(json)?; - util::cleanup(files_to_be_merged, dir)?; + // util::cleanup(files_to_be_merged, dir)?; util::checkout(current_branch)?; } Ok(()) diff --git a/src/util.rs b/src/util.rs index 93f729e..4f6ee45 100644 --- a/src/util.rs +++ b/src/util.rs @@ -255,7 +255,7 @@ fn move_commit_label_to_cmd_name(mut json: Value) -> std::io::Result String { format!( - "jq --arg {0} $({1}) '.results[0].{0}=${0}' {2}", + "jq --arg {0} $({1}) '.results[0].{0}=${0}' {2} > {2}", key, cmd, json ) } From e634db0d41a1ae37a0aa4b4eeb3d8f3548799f9a Mon Sep 17 00:00:00 2001 From: ucyo Date: Tue, 20 Dec 2022 17:37:48 +0100 Subject: [PATCH 10/11] Add a python script to add annotations to hyperfine result file --- annotate.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 annotate.py diff --git a/annotate.py b/annotate.py new file mode 100644 index 0000000..5caf144 --- /dev/null +++ b/annotate.py @@ -0,0 +1,24 @@ +import sys +import subprocess as sp + + +def main(keyword, hyperfine_json, cmd): + result = execute_cmd(cmd) + print(keyword, hyperfine_json, cmd, result) + + +def execute_cmd(cmd): + try: + result = sp.check_output(cmd, shell=True) + except sp.CalledProcessError as err: + print(f"Failed w/ {err}") + raise + else: + return result + + +if __name__ == '__main__': + keyword = sys.argv[1] + json = sys.argv[2] + command = sys.argv[3:] + main(keyword = keyword, hyperfine_json = json, cmd = command) From 7886772c0a46d9d8e8c4569251461d133bf2f8ff Mon Sep 17 00:00:00 2001 From: ucyo Date: Wed, 21 Dec 2022 22:07:53 +0100 Subject: [PATCH 11/11] Update Python script --- annotate.py | 48 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/annotate.py b/annotate.py index 5caf144..3cf92f9 100644 --- a/annotate.py +++ b/annotate.py @@ -1,24 +1,46 @@ import sys import subprocess as sp +import json +from dataclasses import dataclass, field +""" Piped commands not working currently +""" -def main(keyword, hyperfine_json, cmd): - result = execute_cmd(cmd) - print(keyword, hyperfine_json, cmd, result) +@dataclass +class Annotation: + json: str + data: dict = field(init=False) + + def __post_init__(self): + with open(self.json, 'r') as f: + self.data = json.load(f) + def execute_cmd(self, key, cmd): + try: + result = sp.check_output(cmd, shell=True).decode('ascii') + except sp.CalledProcessError as err: + print(f"Failed w/ {err}") + raise + else: + tmp = self.data["results"][0] + if hasattr(tmp,"annotations"): + tmp["annotations"][key] = result + else: + tmp["annotations"] = {key:result} + self.data["results"][0] = tmp -def execute_cmd(cmd): - try: - result = sp.check_output(cmd, shell=True) - except sp.CalledProcessError as err: - print(f"Failed w/ {err}") - raise - else: - return result + def export(self): + with open('new_' + self.json, 'w') as f: + json.dump(self.data, f, indent=2) +def main(keyword, hyperfine_json, cmd): + annon = Annotation(hyperfine_json) + annon.execute_cmd(keyword, cmd) + annon.export() + if __name__ == '__main__': keyword = sys.argv[1] - json = sys.argv[2] + js = sys.argv[2] command = sys.argv[3:] - main(keyword = keyword, hyperfine_json = json, cmd = command) + main(keyword = keyword, hyperfine_json = js, cmd = command)