blob: 2b79974f70e72ca1b5a17e417191004d00ab54a8 [file] [log] [blame]
David Brown2bc26852019-01-14 21:54:04 +00001//! Parallel testing.
2//!
3//! mcuboot simulator is strictly single threaded, as there is a lock around running the C startup
4//! code, because it contains numerous global variables.
5//!
David Brown48a4ec32021-01-04 17:02:27 -07006//! To help speed up testing, the Workflow configuration defines all of the configurations that can
David Brown2bc26852019-01-14 21:54:04 +00007//! be run in parallel. Fortunately, cargo works well this way, and these can be run by simply
8//! using subprocess for each particular thread.
David Brown48a4ec32021-01-04 17:02:27 -07009//!
10//! For now, we assume all of the features are listed under
11//! jobs->environment->strategy->matric->features
David Brown2bc26852019-01-14 21:54:04 +000012
13use chrono::Local;
David Brown87983372024-04-11 08:58:48 -060014use clap::{Parser, Subcommand};
David Brown2bc26852019-01-14 21:54:04 +000015use log::{debug, error, warn};
David Brown2bc26852019-01-14 21:54:04 +000016use std::{
17 collections::HashSet,
David Brownddd390a2021-01-12 12:04:56 -070018 env,
David Brown2bc26852019-01-14 21:54:04 +000019 fs::{self, OpenOptions},
20 io::{ErrorKind, stdout, Write},
David Brownddd390a2021-01-12 12:04:56 -070021 process::Command,
David Brown2bc26852019-01-14 21:54:04 +000022 result,
23 sync::{
24 Arc,
25 Mutex,
26 },
27 thread,
28 time::Duration,
29};
30use std_semaphore::Semaphore;
31use yaml_rust::{
32 Yaml,
33 YamlLoader,
34};
35
36type Result<T> = result::Result<T, failure::Error>;
37
38fn main() -> Result<()> {
39 env_logger::init();
40
David Brown87983372024-04-11 08:58:48 -060041 let args = Cli::parse();
42
43 match args.command {
44 Commands::Run => (),
45 }
46
David Brown48a4ec32021-01-04 17:02:27 -070047 let workflow_text = fs::read_to_string("../.github/workflows/sim.yaml")?;
48 let workflow = YamlLoader::load_from_str(&workflow_text)?;
David Brown2bc26852019-01-14 21:54:04 +000049
50 let ncpus = num_cpus::get();
51 let limiter = Arc::new(Semaphore::new(ncpus as isize));
52
David Brown91de33d2021-03-05 15:43:44 -070053 let matrix = Matrix::from_yaml(&workflow);
David Brown2bc26852019-01-14 21:54:04 +000054
55 let mut children = vec![];
56 let state = State::new(matrix.envs.len());
57 let st2 = state.clone();
58 let _status = thread::spawn(move || {
59 loop {
60 thread::sleep(Duration::new(15, 0));
61 st2.lock().unwrap().status();
62 }
63 });
64 for env in matrix.envs {
65 let state = state.clone();
66 let limiter = limiter.clone();
67
68 let child = thread::spawn(move || {
69 let _run = limiter.access();
70 state.lock().unwrap().start(&env);
71 let out = env.run();
72 state.lock().unwrap().done(&env, out);
73 });
74 children.push(child);
75 }
76
77 for child in children {
78 child.join().unwrap();
79 }
80
David Brown91de33d2021-03-05 15:43:44 -070081 println!();
David Brown2bc26852019-01-14 21:54:04 +000082
83 Ok(())
84}
85
David Brown87983372024-04-11 08:58:48 -060086/// The main Cli.
87#[derive(Debug, Parser)]
88#[command(name = "ptest")]
89#[command(about = "Run MCUboot CI tests stand alone")]
90struct Cli {
91 #[command(subcommand)]
92 command: Commands,
93}
94
95#[derive(Debug, Subcommand)]
96enum Commands {
97 /// Runs the tests.
98 Run,
99}
100
David Brown2bc26852019-01-14 21:54:04 +0000101/// State, for printing status.
102struct State {
103 running: HashSet<String>,
104 done: HashSet<String>,
105 total: usize,
106}
107
David Brownddd390a2021-01-12 12:04:56 -0700108/// Result of a test run.
109struct TestResult {
110 /// Was this run successful.
111 success: bool,
112
113 /// The captured output.
114 output: Vec<u8>,
115}
116
David Brown2bc26852019-01-14 21:54:04 +0000117impl State {
118 fn new(total: usize) -> Arc<Mutex<State>> {
119 Arc::new(Mutex::new(State {
120 running: HashSet::new(),
121 done: HashSet::new(),
David Brown91de33d2021-03-05 15:43:44 -0700122 total,
David Brown2bc26852019-01-14 21:54:04 +0000123 }))
124 }
125
126 fn start(&mut self, fs: &FeatureSet) {
127 let key = fs.textual();
128 if self.running.contains(&key) || self.done.contains(&key) {
129 warn!("Duplicate: {:?}", key);
130 }
131 debug!("Starting: {} ({} running)", key, self.running.len() + 1);
132 self.running.insert(key);
133 self.status();
134 }
135
David Brownddd390a2021-01-12 12:04:56 -0700136 fn done(&mut self, fs: &FeatureSet, output: Result<TestResult>) {
David Brown2bc26852019-01-14 21:54:04 +0000137 let key = fs.textual();
138 self.running.remove(&key);
139 self.done.insert(key.clone());
140 match output {
David Brownddd390a2021-01-12 12:04:56 -0700141 Ok(output) => {
142 if !output.success || log_all() {
143 // Write the output into a file.
144 let mut count = 1;
145 let (mut fd, logname) = loop {
146 let base = if output.success { "success" } else { "failure" };
147 let name = format!("./{}-{:04}.log", base, count);
148 count += 1;
149 match OpenOptions::new()
150 .create_new(true)
151 .write(true)
152 .open(&name)
153 {
154 Ok(file) => break (file, name),
155 Err(ref err) if err.kind() == ErrorKind::AlreadyExists => continue,
156 Err(err) => {
157 error!("Unable to write log file to current directory: {:?}", err);
158 return;
159 }
David Brown2bc26852019-01-14 21:54:04 +0000160 }
David Brownddd390a2021-01-12 12:04:56 -0700161 };
162 fd.write_all(&output.output).unwrap();
163 if !output.success {
164 error!("Failure {} log:{:?} ({} running)", key, logname,
165 self.running.len());
David Brown2bc26852019-01-14 21:54:04 +0000166 }
David Brownddd390a2021-01-12 12:04:56 -0700167 }
David Brown2bc26852019-01-14 21:54:04 +0000168 }
169 Err(err) => {
David Brownddd390a2021-01-12 12:04:56 -0700170 error!("Unable to run test {:?} ({:?}", key, err);
David Brown2bc26852019-01-14 21:54:04 +0000171 }
172 }
173 self.status();
174 }
175
176 fn status(&self) {
177 let running = self.running.len();
178 let done = self.done.len();
179 print!(" {} running ({}/{}/{} done)\r", running, done, running + done, self.total);
180 stdout().flush().unwrap();
181 }
182}
183
David Brown48a4ec32021-01-04 17:02:27 -0700184/// The extracted configurations from the workflow config
David Brown2bc26852019-01-14 21:54:04 +0000185#[derive(Debug)]
186struct Matrix {
187 envs: Vec<FeatureSet>,
188}
189
190#[derive(Debug, Eq, Hash, PartialEq)]
191struct FeatureSet {
192 // The environment variable to set.
193 env: String,
194 // The successive values to set it to.
195 values: Vec<String>,
196}
197
198impl Matrix {
David Brown91de33d2021-03-05 15:43:44 -0700199 fn from_yaml(yaml: &[Yaml]) -> Matrix {
David Brown2bc26852019-01-14 21:54:04 +0000200 let mut envs = vec![];
201
202 let mut all_tests = HashSet::new();
203
204 for y in yaml {
205 let m = match lookup_matrix(y) {
206 Some (m) => m,
207 None => continue,
208 };
209 for elt in m {
David Brown48a4ec32021-01-04 17:02:27 -0700210 let elt = match elt.as_str() {
211 None => {
212 warn!("Unexpected yaml: {:?}", elt);
213 continue;
214 }
215 Some(e) => e,
216 };
David Brown91de33d2021-03-05 15:43:44 -0700217 let fset = FeatureSet::decode(elt);
David Brown2bc26852019-01-14 21:54:04 +0000218
David Brown48a4ec32021-01-04 17:02:27 -0700219 if false {
220 // Respect the groupings in the `.workflow.yml` file.
221 envs.push(fset);
222 } else {
223 // Break each test up so we can run more in
224 // parallel.
225 let env = fset.env.clone();
226 for val in fset.values {
227 if !all_tests.contains(&val) {
228 all_tests.insert(val.clone());
229 envs.push(FeatureSet {
230 env: env.clone(),
231 values: vec![val],
232 });
233 } else {
234 warn!("Duplicate: {:?}: {:?}", env, val);
David Brown2bc26852019-01-14 21:54:04 +0000235 }
236 }
237 }
238 }
239 }
240
David Brown91de33d2021-03-05 15:43:44 -0700241 Matrix {
242 envs,
243 }
David Brown2bc26852019-01-14 21:54:04 +0000244 }
245}
246
247impl FeatureSet {
David Brown91de33d2021-03-05 15:43:44 -0700248 fn decode(text: &str) -> FeatureSet {
David Brown48a4ec32021-01-04 17:02:27 -0700249 // The github workflow is just a space separated set of values.
250 let values: Vec<_> = text
251 .split(',')
252 .map(|s| s.to_string())
253 .collect();
David Brown91de33d2021-03-05 15:43:44 -0700254 FeatureSet {
David Brown48a4ec32021-01-04 17:02:27 -0700255 env: "MULTI_FEATURES".to_string(),
David Brown91de33d2021-03-05 15:43:44 -0700256 values,
257 }
David Brown2bc26852019-01-14 21:54:04 +0000258 }
259
260 /// Run a test for this given feature set. Output is captured and will be returned if there is
261 /// an error. Each will be run successively, and the first failure will be returned.
262 /// Otherwise, it returns None, which means everything worked.
David Brownddd390a2021-01-12 12:04:56 -0700263 fn run(&self) -> Result<TestResult> {
264 let mut output = vec![];
265 let mut success = true;
David Brown2bc26852019-01-14 21:54:04 +0000266 for v in &self.values {
David Brownddd390a2021-01-12 12:04:56 -0700267 let cmdout = Command::new("bash")
David Brownb899f792019-03-14 15:21:06 -0600268 .arg("./ci/sim_run.sh")
David Brown2bc26852019-01-14 21:54:04 +0000269 .current_dir("..")
270 .env(&self.env, v)
271 .output()?;
David Brownddd390a2021-01-12 12:04:56 -0700272 // Grab the output for logging, etc.
273 writeln!(&mut output, "Test {} {}",
274 if cmdout.status.success() { "success" } else { "FAILURE" },
275 self.textual())?;
276 writeln!(&mut output, "time: {}", Local::now().to_rfc3339())?;
277 writeln!(&mut output, "----------------------------------------")?;
278 writeln!(&mut output, "stdout:")?;
279 output.extend(&cmdout.stdout);
280 writeln!(&mut output, "----------------------------------------")?;
281 writeln!(&mut output, "stderr:")?;
282 output.extend(&cmdout.stderr);
283 if !cmdout.status.success() {
284 success = false;
David Brown2bc26852019-01-14 21:54:04 +0000285 }
286 }
David Brownddd390a2021-01-12 12:04:56 -0700287 Ok(TestResult { success, output })
David Brown2bc26852019-01-14 21:54:04 +0000288 }
289
290 /// Convert this feature set into a textual representation
291 fn textual(&self) -> String {
292 use std::fmt::Write;
293
294 let mut buf = String::new();
295
296 write!(&mut buf, "{}:", self.env).unwrap();
297 for v in &self.values {
298 write!(&mut buf, " {}", v).unwrap();
299 }
300
301 buf
302 }
303}
304
305fn lookup_matrix(y: &Yaml) -> Option<&Vec<Yaml>> {
David Brown48a4ec32021-01-04 17:02:27 -0700306 let jobs = Yaml::String("jobs".to_string());
307 let environment = Yaml::String("environment".to_string());
308 let strategy = Yaml::String("strategy".to_string());
David Brown2bc26852019-01-14 21:54:04 +0000309 let matrix = Yaml::String("matrix".to_string());
David Brown48a4ec32021-01-04 17:02:27 -0700310 let features = Yaml::String("features".to_string());
311 y
312 .as_hash()?.get(&jobs)?
313 .as_hash()?.get(&environment)?
314 .as_hash()?.get(&strategy)?
315 .as_hash()?.get(&matrix)?
316 .as_hash()?.get(&features)?
317 .as_vec()
David Brown2bc26852019-01-14 21:54:04 +0000318}
David Brownddd390a2021-01-12 12:04:56 -0700319
320/// Query if we should be logging all tests and not only failures.
321fn log_all() -> bool {
322 env::var("PTEST_LOG_ALL").is_ok()
323}