evobench_eval/
evobench-eval.rs

1use std::fmt::Debug;
2use std::path::PathBuf;
3
4use anyhow::Result;
5use clap::Parser;
6use mimalloc::MiMalloc;
7
8use evobench_tools::{
9    evaluator::{
10        all_fields_table::{SingleRunStats, SummaryStats},
11        all_outputs_all_fields_table::AllOutputsAllFieldsTable,
12        data::log_data_and_tree::LogDataAndTree,
13        options::{
14            CheckedOutputOptions, EvaluationAndOutputOpts, FieldSelectorDimension3Opt,
15            FieldSelectorDimension4Opt, FlameFieldOpt,
16        },
17    },
18    stats_tables::stats::StatsField,
19    utillib::{
20        get_terminal_width::get_terminal_width,
21        logging::{LogLevelOpts, set_log_level},
22    },
23};
24use rayon::iter::{ParallelBridge, ParallelIterator};
25use run_git::flattened::IntoFlattened;
26
27#[global_allocator]
28static GLOBAL: MiMalloc = MiMalloc;
29
30include!("../../include/evobench_version.rs");
31
32const PROGRAM_NAME: &str = "evobench-eval";
33
34/// How many files to process in parallel at most. This is to avoid
35/// using too much RAM.
36const NUM_FILES_IN_PARALLEL: usize = 4;
37
38#[derive(clap::Parser, Debug)]
39#[clap(next_line_help = true)]
40#[clap(term_width = get_terminal_width(4))]
41struct Opts {
42    #[clap(flatten)]
43    log_level: LogLevelOpts,
44
45    /// The subcommand to run. Use `--help` after the sub-command to
46    /// get a list of the allowed options there.
47    #[clap(subcommand)]
48    command: Command,
49}
50
51#[derive(clap::Subcommand, Debug)]
52enum Command {
53    /// Print version
54    Version,
55
56    /// Show statistics for a single benchmarking log file
57    Single {
58        #[clap(flatten)]
59        evaluation_and_output_opts: EvaluationAndOutputOpts,
60
61        /// The path that was provided via the `EVOBENCH_LOG`
62        /// environment variable to the evobench-probes library.
63        path: PathBuf,
64    },
65
66    /// Show statistics for a set of benchmarking log files, all for
67    /// the same software version.
68    Summary {
69        #[clap(flatten)]
70        evaluation_and_output_opts: EvaluationAndOutputOpts,
71        #[clap(flatten)]
72        field_selector_dimension_3: FieldSelectorDimension3Opt,
73        #[clap(flatten)]
74        flame_selector: FlameFieldOpt,
75
76        /// The paths that were provided via the `EVOBENCH_LOG`
77        /// environment variable to the evobench-probes library.
78        paths: Vec<PathBuf>,
79    },
80
81    /// Show statistics across multiple sets of benchmarking log
82    /// files, each group consisting of files for the same software
83    /// version. Each group is enclosed with square brackets, e.g.:
84    /// `trend [ a.log b.log ] [ c.log ] [ d.log e.log ]` has data for
85    /// 3 software versions, the first and third version with data
86    /// from two runs each.
87    Trend {
88        #[clap(flatten)]
89        evaluation_and_output_opts: EvaluationAndOutputOpts,
90        #[clap(flatten)]
91        field_selector_dimension_3: FieldSelectorDimension3Opt,
92        #[clap(flatten)]
93        field_selector_dimension_4: FieldSelectorDimension4Opt,
94        #[clap(flatten)]
95        flame_selector: FlameFieldOpt,
96
97        /// The paths that were provided via the `EVOBENCH_LOG`
98        /// environment variable to the evobench-probes library.
99        grouped_paths: Vec<PathBuf>,
100    },
101}
102
103fn main() -> Result<()> {
104    let Opts { log_level, command } = Opts::parse();
105
106    set_log_level(log_level.try_into()?);
107
108    match command {
109        Command::Version => println!("{PROGRAM_NAME} version {EVOBENCH_VERSION}"),
110
111        Command::Single {
112            evaluation_and_output_opts:
113                EvaluationAndOutputOpts {
114                    evaluation_opts,
115                    output_opts,
116                },
117            path,
118        } => {
119            let CheckedOutputOptions { variants } = output_opts.check()?;
120            let ldat = LogDataAndTree::read_file(&path, None)?;
121            let aoaft = AllOutputsAllFieldsTable::from_log_data_tree(
122                ldat.tree(),
123                &evaluation_opts,
124                variants,
125                true,
126            )?;
127            aoaft.write_to_files(StatsField::Sum)?;
128        }
129
130        Command::Summary {
131            evaluation_and_output_opts:
132                EvaluationAndOutputOpts {
133                    evaluation_opts,
134                    output_opts,
135                },
136            paths,
137            field_selector_dimension_3: FieldSelectorDimension3Opt { summary_field },
138            flame_selector: FlameFieldOpt { flame_field },
139        } => {
140            let CheckedOutputOptions { variants } = output_opts.check()?;
141            let chunk_size = (paths.len() + NUM_FILES_IN_PARALLEL - 1) / NUM_FILES_IN_PARALLEL;
142            let afts: Vec<Vec<AllOutputsAllFieldsTable<SingleRunStats>>> = paths
143                .chunks(chunk_size)
144                .par_bridge()
145                .map(
146                    |source_paths| -> Result<Vec<AllOutputsAllFieldsTable<SingleRunStats>>> {
147                        let mut afts = Vec::new();
148                        for source_path in source_paths {
149                            let ldat = LogDataAndTree::read_file(source_path, None)?;
150                            afts.push(AllOutputsAllFieldsTable::from_log_data_tree(
151                                ldat.tree(),
152                                &evaluation_opts,
153                                variants.clone(),
154                                false,
155                            )?);
156                        }
157                        Ok(afts)
158                    },
159                )
160                .collect::<Result<_>>()?;
161            let aft = AllOutputsAllFieldsTable::<SummaryStats>::summary_stats(
162                &afts.into_flattened(),
163                summary_field,
164                &evaluation_opts,
165                variants, // same as passed to from_log_data_tree above
166                true,
167            );
168            aft.write_to_files(flame_field)?;
169        }
170
171        #[allow(unused)]
172        Command::Trend {
173            evaluation_and_output_opts: evaluation_opts,
174            grouped_paths,
175            field_selector_dimension_3: FieldSelectorDimension3Opt { summary_field },
176            field_selector_dimension_4: FieldSelectorDimension4Opt { trend_field },
177            flame_selector: FlameFieldOpt { flame_field },
178        } => todo!(),
179    }
180
181    Ok(())
182}
evobench_eval/evobench-eval.rs

evobench_eval/
evobench-eval.rs
