use std::borrow::Cow; use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::str::FromStr; use std::sync; use std::{env, fs, io, mem}; use snapbox::cmd::{Command, OutputAssert}; use snapbox::{Assert, Substitutions}; use thiserror::Error; const CARGO_TARGET_DIR_DIRNAME: &str = "target"; const CARGO_PROFILE: &str = "debug"; /// Used to ensure the build task is only run once. static BUILD: sync::Once = sync::Once::new(); #[derive(Error, Debug)] pub enum Error { #[error("parsing failed")] Parse, #[error("invalid file path: {0:?}")] InvalidFilePath(String), #[error("unknown home {0:?}")] UnknownHome(String), #[error("test file not found: {0:?}")] TestNotFound(PathBuf), #[error("i/o: {0}")] Io(#[from] io::Error), #[error("snapbox: {0}")] Snapbox(#[from] snapbox::Error), } #[derive(Debug, PartialEq, Eq)] enum ExitStatus { Success, Failure, } /// A test which may contain multiple assertions. #[derive(Debug, Default, PartialEq, Eq)] pub struct Test { /// Human-readable context around the test. Functions as documentation. context: Vec, /// Test assertions to run. assertions: Vec, /// Whether to check stderr's output instead of stdout. stderr: bool, /// Whether to expect an error status code. fail: bool, /// Home directory under which to run this test. home: Option, /// Local env vars to use just for this test. env: HashMap, } /// An assertion is a command to run with an expected output. #[derive(Debug, PartialEq, Eq)] pub struct Assertion { /// The test file that contains this assertion. path: PathBuf, /// Name of command to run, eg. `git`. command: String, /// Command arguments, eg. `["push"]`. args: Vec, /// Expected output (stdout or stderr). expected: String, /// Expected exit status. exit: ExitStatus, /// Line number in the test file where this assertion is defined. line: usize, } #[derive(Debug, Default, PartialEq, Eq, Clone)] pub struct Home { name: Option, path: PathBuf, envs: HashMap, } #[derive(Debug)] pub struct TestRun { home: Home, env: HashMap, } impl TestRun { fn cd(&mut self, path: PathBuf) { self.home.path = path; } fn envs(&self) -> impl Iterator + '_ { self.home .envs .iter() .chain(self.env.iter()) .map(|(k, v)| (k.to_owned(), v.to_owned())) .chain(Some(( "PWD".to_owned(), self.home.path.to_string_lossy().to_string(), ))) } fn path(&self) -> PathBuf { self.home.path.clone() } } #[derive(Debug)] pub struct TestRunner<'a> { cwd: Option, homes: HashMap, formula: &'a TestFormula, } impl<'a> TestRunner<'a> { fn new(formula: &'a TestFormula) -> Self { Self { cwd: None, homes: formula.homes.clone(), formula, } } fn run(&mut self, test: &'a Test) -> TestRun { let mut env = self.formula.env.clone(); env.extend(test.env.clone()); if let Some(ref h) = test.home { if let Some(home) = self.homes.get(h) { env.insert("USER".to_owned(), h.to_owned()); return TestRun { home: home.clone(), env, }; } else { panic!("TestRunner::test: home `~{h}` does not exist"); } } TestRun { home: Home { name: None, path: self.cwd.clone().unwrap_or_else(|| self.formula.cwd.clone()), envs: HashMap::new(), }, env, } } fn finish(&mut self, run: TestRun) { if let Some(name) = &run.home.name { self.homes.insert(name.clone(), run.home); } else { self.cwd = Some(run.home.path); } } } #[derive(Debug, Default, PartialEq, Eq)] pub struct TestFormula { /// Current working directory to run the test in. cwd: PathBuf, /// User homes. homes: HashMap, /// Environment to pass to the test. env: HashMap, /// Tests to run. tests: Vec, /// Output substitutions. subs: Substitutions, } impl TestFormula { pub fn new(cwd: PathBuf) -> Self { Self { cwd: cwd.clone(), env: HashMap::new(), homes: HashMap::new(), tests: Vec::new(), subs: Substitutions::new(), } } pub fn build(&mut self, binaries: &[(&str, &str)]) -> &mut Self { // We don't need to re-build every time the `build` function is called. Once is enough. BUILD.call_once(|| { use escargot::format::Message; use radicle::logger::env_level; use radicle::logger::test::Logger; let level = env_level().unwrap_or(log::Level::Debug); let logger = Box::new(Logger::new(level)); log::set_boxed_logger(logger).expect("no other logger should have been set already"); log::set_max_level(level.to_level_filter()); for (package, binary) in binaries { log::debug!(target: "test", "Building binaries for package `{package}`.."); let results = escargot::CargoBuild::new() .package(package) .bin(binary) .manifest_path(cargo_manifest_dir().join("Cargo.toml")) .target_dir(cargo_target_dir()) .exec() .unwrap(); for result in results { match result { Ok(msg) => { if let Ok(Message::CompilerArtifact(a)) = msg.decode() { if let Some(e) = a.executable { log::debug!(target: "test", "Built {}", e.display()); } } } Err(e) => { log::error!(target: "test", "Error building package `{package}`: {e}"); } } } } }); self } pub fn env(&mut self, key: impl ToString, val: impl ToString) -> &mut Self { self.env.insert(key.to_string(), val.to_string()); self } pub fn home( &mut self, user: impl ToString, path: impl AsRef, envs: impl IntoIterator, ) -> &mut Self { self.homes.insert( user.to_string(), Home { name: Some(user.to_string()), path: path.as_ref().to_path_buf(), envs: envs .into_iter() .map(|(k, v)| (k.to_string(), v.to_string())) .collect(), }, ); self } pub fn envs( &mut self, envs: impl IntoIterator, ) -> &mut Self { for (k, v) in envs { self.env.insert(k.to_string(), v.to_string()); } self } pub fn file(&mut self, path: impl AsRef) -> Result<&mut Self, Error> { let path = path.as_ref(); let contents = match fs::read(path) { Ok(bytes) => bytes, Err(err) if err.kind() == io::ErrorKind::NotFound => { return Err(Error::TestNotFound(path.to_path_buf())); } Err(err) => return Err(err.into()), }; self.read(path, io::Cursor::new(contents)) } pub fn read(&mut self, path: &Path, r: impl io::BufRead) -> Result<&mut Self, Error> { let mut test = Test::default(); let mut fenced = false; // Whether we're inside a fenced code block. let mut file: Option<(PathBuf, String)> = None; // Path and content of file created by this test block. for (row, line) in r.lines().enumerate() { let line = line?; if line.starts_with("```") { if fenced { if let Some((ref path, ref mut content)) = file.take() { // Write file. let path = self.cwd.join(path); if let Some(dir) = path.parent() { log::debug!(target: "test", "Creating directory {}..", dir.display()); fs::create_dir_all(dir)?; } log::debug!(target: "test", "Writing {} bytes to {}..", content.len(), path.display()); fs::write(path, content)?; } else { // End existing code block. self.tests.push(mem::take(&mut test)); } } else { for token in line.split_whitespace() { if let Some(home) = token.strip_prefix('~') { test.home = Some(home.to_owned()); } else if let Some((key, val)) = token.split_once('=') { test.env.insert(key.to_owned(), val.to_owned()); } else if token.contains("stderr") { test.stderr = true; } else if token.contains("fail") { test.fail = true; } else if let Some(path) = token.strip_prefix("./") { file = Some(( PathBuf::from_str(path) .map_err(|_| Error::InvalidFilePath(token.to_owned()))?, String::new(), )); } } } fenced = !fenced; continue; } if fenced { if let Some((_, ref mut content)) = file { content.push_str(line.as_str()); content.push('\n'); } else if let Some(line) = line.strip_prefix('$') { let line = line.trim(); #[cfg(unix)] let parts = shlex::split(line).ok_or(Error::Parse)?; #[cfg(windows)] let parts = winsplit::split(line); let (cmd, args) = parts.split_first().ok_or(Error::Parse)?; test.assertions.push(Assertion { path: path.to_path_buf(), command: cmd.to_owned(), args: args.to_owned(), expected: String::new(), exit: if test.fail { ExitStatus::Failure } else { ExitStatus::Success }, line: row + 1, }); } else if let Some(a) = test.assertions.last_mut() { a.expected.push_str(line.as_str()); a.expected.push('\n'); } else { return Err(Error::Parse); } } else { test.context.push(line); } } Ok(self) } #[allow(dead_code)] pub fn substitute( &mut self, value: &'static str, other: impl Into>, ) -> Result<&mut Self, Error> { self.subs.insert(value, other)?; Ok(self) } /// Convert instances of '[.. ]' to '[..]' where the number of ' 's are arbitrary. /// /// Supporting these bracket types help support using the '[..]' pattern while preserving /// spaces important for text alignment. fn map_spaced_brackets(s: &str) -> String { let mut ret = String::new(); let mut pos = 0; for c in s.chars() { match (c, pos) { ('[', 0) => pos += 1, (' ', 1) => continue, ('.', 1) => pos += 1, ('.', 2) => pos += 1, ('.', 3) => continue, (' ', 3) => continue, (']', 3) => pos = 0, (_, _) => pos = 0, } ret.push(c); } ret } pub fn run(&mut self) -> Result { let assert = Assert::new().substitutions(self.subs.clone()); let mut runner = TestRunner::new(self); fs::create_dir_all(&self.cwd)?; // For each code block. for test in &self.tests { let mut run = runner.run(test); // For each command. for (i, assertion) in test.assertions.iter().enumerate() { let location = assertion .path .file_name() .map(|f| f.to_string_lossy().to_string()) .map(|f| f.strip_suffix(".md").unwrap_or(&f).to_owned()) .map(|f| f + ":" + assertion.line.to_string().as_str()) .unwrap_or(String::from("")); if assertion.command == "cd" { let arg = assertion.args.first().unwrap(); let dir: PathBuf = arg.into(); let dir = run.path().join(dir); // TODO: Add support for `..` and `/` // TODO: Error if more than one args are given. log::debug!(target: "test", "{location}: `cd {}`..", dir.display()); if !dir.exists() { return Err(io::Error::new( io::ErrorKind::NotFound, format!("cd: '{}' does not exist", dir.display()), )); } run.cd(dir); continue; } // Expand environment variables. let mut args = assertion.args.clone(); for arg in &mut args { for (k, v) in run.envs() { *arg = arg.replace(format!("${k}").as_str(), &v); } } if !run.path().exists() { log::warn!(target: "test", "{location}: Directory {} does not exist. Creating..", run.path().display()); fs::create_dir_all(run.path())?; } let jj_envs = if assertion.command == "jj" { vec![ ("JJ_RANDOMNESS_SEED", i.to_string()), ("JJ_TIMESTAMP", "2001-02-03T04:05:06+07:00".to_string()), ("JJ_OP_TIMESTAMP", "2001-02-03T04:05:06+07:00".to_string()), ] } else { vec![] }; let bins = std::env::join_paths(bins(self.cwd.clone())).unwrap(); let command = Command::new(assertion.command.clone()) .env_clear() .env("PATH", &bins) .env("RUST_BACKTRACE", "1") .envs(jj_envs) .envs(run.envs()) .current_dir(run.path()) .args(args.clone()) .with_assert(assert.clone()); log::debug!(target: "test", "{location}: `{} {}` @ {}", assertion.command, args.join(" "), run.path().display()); log::trace!(target: "test", "{location}: {}", run.envs().map(|(k, v)| format!("{}={}", k, v)).collect::>().join(", ")); log::logger().flush(); // Even though it would be possible to use `Command::assert` to directly obtain // `OutputAssert`, we use `Command::output` to be able to handle `io::ErrorKind::NotFound` // separately and provide a more helpful error message in that case. match command.output() { Ok(output) => { let assert = OutputAssert::new(output).with_assert(assert.clone()); let expected = Self::map_spaced_brackets(&assertion.expected); let expected = { #[cfg(windows)] const EXE: &str = ".exe"; #[cfg(unix)] const EXE: &str = ""; expected.replace("[EXE]", EXE) }; let matches = if test.stderr { assert.stderr_matches(&expected) } else { assert.stdout_matches(&expected) }; match assertion.exit { ExitStatus::Success => { matches.success(); } ExitStatus::Failure => { matches.failure(); } } } Err(err) => { if err.kind() == io::ErrorKind::NotFound { log::error!(target: "test", "{location}: Command `{}` does not exist..", assertion.command); } return Err(io::Error::new( err.kind(), format!("{location}: {err}: `{}`", assertion.command), )); } } } runner.finish(run); } Ok(true) } } fn cargo_manifest_dir() -> PathBuf { env::var("CARGO_MANIFEST_DIR").map(PathBuf::from).unwrap() } fn cargo_target_dir() -> PathBuf { env::var("CARGO_TARGET_DIR") .map(PathBuf::from) .unwrap_or(cargo_manifest_dir().join(CARGO_TARGET_DIR_DIRNAME)) } /// Get the list of binary paths to use as `$PATH` for the tests, /// starting with the current working directory. fn bins(cwd: PathBuf) -> Vec { let mut bins: Vec = Vec::new(); // Add current working directory to `$PATH`, // this makes it more convenient to execute scripts during testing. bins.push(cwd); bins.push(cargo_target_dir().join(CARGO_PROFILE)); // Add the "real" `$PATH`. if let Ok(path) = env::var("PATH") { bins.extend(env::split_paths(&path)); } #[cfg(windows)] { // Radicle CLI tests rely on various Unix coreutils // (such as `ls` and `touch`) being available. // On Windows, it is very likely that we can find them in the // following location. // Note that adding this path to the end of `$PATH` causes // no harm, even if the directory does not exist. bins.push(PathBuf::from(r#"C:\Program Files\Git\usr\bin"#)); } bins } #[cfg(test)] mod tests { use super::*; use pretty_assertions::assert_eq; #[test] fn test_parse() { let input = r#" Let's try to track @dave and @sean: ``` RAD_HINT=true $ rad track @dave Tracking relationship established for @dave. Nothing to do. $ rad track @sean Tracking relationship established for @sean. Nothing to do. ``` Super, now let's move on to the next step. ``` ~alice (stderr) $ rad sync ``` "# .trim() .as_bytes() .to_owned(); let cwd = PathBuf::from("radicle-cli-test"); let mut actual = TestFormula::new(cwd.clone()); let path = Path::new("test.md").to_path_buf(); actual .read(path.as_path(), io::BufReader::new(io::Cursor::new(input))) .unwrap(); let expected = TestFormula { homes: HashMap::new(), cwd: cwd.clone(), env: HashMap::new(), subs: Substitutions::new(), tests: vec![ Test { context: vec![String::from("Let's try to track @dave and @sean:")], home: None, assertions: vec![ Assertion { line: 3, path: path.clone(), command: String::from("rad"), args: vec![String::from("track"), String::from("@dave")], expected: String::from( "Tracking relationship established for @dave.\nNothing to do.\n\n", ), exit: ExitStatus::Success, }, Assertion { line: 7, path: path.clone(), command: String::from("rad"), args: vec![String::from("track"), String::from("@sean")], expected: String::from( "Tracking relationship established for @sean.\nNothing to do.\n", ), exit: ExitStatus::Success, }, ], fail: false, stderr: false, env: vec![("RAD_HINT".to_owned(), "true".to_owned())] .into_iter() .collect(), }, Test { context: vec![String::from("Super, now let's move on to the next step.")], home: Some("alice".to_owned()), assertions: vec![Assertion { line: 13, path: path.clone(), command: String::from("rad"), args: vec![String::from("sync")], expected: String::new(), exit: ExitStatus::Success, }], fail: false, stderr: true, env: HashMap::default(), }, ], }; assert_eq!(actual, expected); } #[test] fn test_run() { let input = r#" Running a simple command such as `head`: ``` $ head -n 2 Cargo.toml [package] name = "radicle-cli-test" ``` "# .trim() .as_bytes() .to_owned(); let mut formula = TestFormula::new(PathBuf::from_str(env!("CARGO_MANIFEST_DIR")).unwrap()); formula .read( Path::new("test.md"), io::BufReader::new(io::Cursor::new(input)), ) .unwrap(); formula.run().unwrap(); } #[test] fn test_example_spaced_brackets() { let input = r#" Running a simple command such as `head`: ``` $ echo " hello" [..]hello $ echo " hello" [.. ]hello $ echo " hello" [ ..]hello $ echo "[bug, good-first-issue]" [bug, good-first-issue] $ echo "[bug, good-first-issue]" [bug, [ .. ]-issue] $ echo "[bug, good-first-issue]" [bug, [ ... ]-issue] ``` "# .trim() .as_bytes() .to_owned(); let mut formula = TestFormula::new(PathBuf::from_str(env!("CARGO_MANIFEST_DIR")).unwrap()); formula .read( Path::new("test.md"), io::BufReader::new(io::Cursor::new(input)), ) .unwrap(); formula.run().unwrap(); } }