From 8c32d27c9ad60ebb5b7997cfc214b0c1dd6ff343 Mon Sep 17 00:00:00 2001 From: Alex Huszagh Date: Sat, 5 Nov 2022 18:00:40 -0500 Subject: [PATCH] Initial version of UI tests. --- Cargo.lock | 1 + ci/test-ui.sh | 17 ++ src/cli.rs | 2 +- xtask/Cargo.toml | 1 + xtask/src/main.rs | 14 ++ xtask/src/temp.rs | 35 ++++ xtask/src/ui.rs | 480 ++++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 549 insertions(+), 1 deletion(-) create mode 100755 ci/test-ui.sh create mode 100644 xtask/src/temp.rs create mode 100644 xtask/src/ui.rs diff --git a/Cargo.lock b/Cargo.lock index 1ad41e9c8..b49ceeb28 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1008,6 +1008,7 @@ dependencies = [ "serde", "serde_json", "shell-words", + "tempfile", "toml", "walkdir", "which", diff --git a/ci/test-ui.sh b/ci/test-ui.sh new file mode 100755 index 000000000..a6ac8e174 --- /dev/null +++ b/ci/test-ui.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# shellcheck disable=SC1091 + +set -eo pipefail + +ci_dir=$(dirname "${BASH_SOURCE[0]}") +ci_dir=$(realpath "${ci_dir}") +. "${ci_dir}"/shared.sh + +if [[ -z "${TARGET}" ]]; then + export TARGET="x86_64-unknown-linux-gnu" +fi + +cargo xtask ui-test \ + --target "${TARGET}" \ + --engine "${CROSS_ENGINE}" \ + --verbose diff --git a/src/cli.rs b/src/cli.rs index a2b90a254..4942624bf 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -54,7 +54,7 @@ pub fn fmt_subcommands(stdout: &str, msg_info: &mut MessageInfo) -> Result<()> { } if !host.is_empty() { msg_info.print("Host Commands:")?; - for line in &cross { + for line in &host { msg_info.print(line)?; } } diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 040352e47..02dda1d30 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -23,3 +23,4 @@ once_cell = "1.15" semver = "1" chrono = "0.4" wildmatch = "2.1.1" +tempfile = "3.3.0" diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 41779c61c..af4a3b903 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -8,6 +8,8 @@ pub mod crosstool; pub mod hooks; pub mod install_git_hooks; pub mod target_info; +pub mod temp; +pub mod ui; pub mod util; use ci::CiJob; @@ -23,6 +25,8 @@ use self::crosstool::ConfigureCrosstool; use self::hooks::{Check, Test}; use self::install_git_hooks::InstallGitHooks; use self::target_info::TargetInfo; +use self::temp::MakeTempDir; +use self::ui::UiTest; #[derive(Parser, Debug)] #[clap(version, about, long_about = None)] @@ -65,6 +69,10 @@ enum Commands { ValidateChangelog(ValidateChangelog), /// Code generation Codegen(Codegen), + /// Create temporary directory + MakeTempDir(MakeTempDir), + /// Run user-interface suite. + UiTest(UiTest), } fn is_toolchain(toolchain: &str) -> cross::Result { @@ -131,6 +139,12 @@ pub fn main() -> cross::Result<()> { changelog::validate_changelog(args, &mut msg_info)?; } Commands::Codegen(args) => codegen::codegen(args)?, + Commands::MakeTempDir(args) => temp::make_temp_dir(args)?, + Commands::UiTest(args) => { + let mut msg_info = get_msg_info!(args, args.verbose)?; + let engine = get_engine!(args, msg_info)?; + ui::ui_test(args, &engine, &mut msg_info)? + } } Ok(()) diff --git a/xtask/src/temp.rs b/xtask/src/temp.rs new file mode 100644 index 000000000..f2bfba74f --- /dev/null +++ b/xtask/src/temp.rs @@ -0,0 +1,35 @@ +use std::path::{Path, PathBuf}; +use std::{fs, mem}; + +use crate::util; +use clap::Args; +use cross::shell::MessageInfo; +use cross::ToUtf8; + +/// Create and print a temporary directory to stdout. +#[derive(Args, Debug)] +pub struct MakeTempDir { + /// `tmp` to create the temporary directory inside. + /// Defaults to `"${target_dir}/tmp"`. + tmpdir: Option, +} + +pub fn make_temp_dir(MakeTempDir { tmpdir }: MakeTempDir) -> cross::Result<()> { + let mut msg_info = MessageInfo::create(0, false, None)?; + let dir = temp_dir(tmpdir.as_deref(), &mut msg_info)?; + msg_info.print(dir.to_utf8()?) +} + +pub fn temp_dir(parent: Option<&Path>, msg_info: &mut MessageInfo) -> cross::Result { + let parent = match parent { + Some(parent) => parent.to_owned(), + None => util::project_dir(msg_info)?.join("target").join("tmp"), + }; + fs::create_dir_all(&parent)?; + let dir = tempfile::TempDir::new_in(&parent)?; + let path = dir.path().to_owned(); + mem::drop(dir); + + fs::create_dir(&path)?; + Ok(path) +} diff --git a/xtask/src/ui.rs b/xtask/src/ui.rs new file mode 100644 index 000000000..88087ea6a --- /dev/null +++ b/xtask/src/ui.rs @@ -0,0 +1,480 @@ +use std::path::{Path, PathBuf}; +use std::process::{Command, ExitStatus, Output, Stdio}; +use std::{env, fs}; + +use crate::{temp, util, ImageTarget}; +use clap::Args; +use cross::shell::MessageInfo; +use cross::{docker, CommandExt, ToUtf8}; + +#[derive(Args, Debug)] +pub struct UiTest { + /// Provide verbose diagnostic output. + #[clap(short, long)] + pub verbose: bool, + /// Do not print cross log messages. + #[clap(short, long)] + pub quiet: bool, + /// Coloring: auto, always, never + #[clap(long)] + pub color: Option, + /// Container engine (such as docker or podman). + #[clap(long)] + pub engine: Option, + /// Specify a tag to use instead of the derived one, eg `local` + #[clap(long)] + pub tag: Option, + /// Repository name for image. + #[clap(long, default_value = docker::CROSS_IMAGE)] + pub repository: String, + /// Do not pull the image (local image). + #[clap(long)] + pub no_pull: bool, + /// Target to run tests for. + #[clap(long)] + pub target: Option, +} + +struct Utf8Output { + command: String, + status: ExitStatus, + stdout: String, + stderr: String, +} + +impl Utf8Output { + fn bail(&self, message: &str) -> eyre::ErrReport { + eyre::eyre!( + r#"command unexpectedly {message} with exit status of {:?}: +command +======= +{} + +stdout +====== +{} + +stderr +====== +{} +"#, + self.status, + self.command, + self.stdout, + self.stderr + ) + } + + fn check(&self, message: &str, op: impl Fn(&Self) -> bool) -> cross::Result<()> { + match op(self) { + true => Ok(()), + false => Err(self.bail(message)), + } + } + + fn success(&self) -> cross::Result<()> { + self.check("failed", |x| x.status.success()) + } + + fn failure(&self) -> cross::Result<()> { + self.check("succeeded", |x| !x.status.success()) + } + + fn no_fallback(&self) -> cross::Result<()> { + self.check("fallsback", |x| { + !x.stderr + .contains("[cross] note: Falling back to `cargo` on the host.") + }) + } + + fn fallsback(&self) -> cross::Result<()> { + self.check("does not fallback", |x| { + x.stderr + .contains("[cross] note: Falling back to `cargo` on the host.") + }) + } + + fn no_metadata(&self) -> cross::Result<()> { + self.check("has metadata", |x| { + x.stderr + .contains("[cross] warning: unable to get metadata for package") + }) + } + + fn no_stdout(&self) -> cross::Result<()> { + self.check("has stdout", |x| x.stdout.is_empty()) + } + + fn outside_package(&self) -> cross::Result<()> { + self.failure()?; + self.fallsback()?; + self.no_metadata()?; + self.no_stdout()?; + self.check("did not error", |x| { + x.stderr.contains("error: could not find `Cargo.toml`") + }) + } + + fn compiles(&self) -> cross::Result<()> { + self.success()?; + self.no_fallback()?; + self.no_stdout()?; + self.check("did not compile", |x| x.stderr.contains("Finished dev")) + } + + fn runs(&self, output: &str) -> cross::Result<()> { + self.success()?; + self.no_fallback()?; + self.check("does not have output", |x| x.stdout.contains(output))?; + self.check("did not compile", |x| x.stderr.contains("Finished dev")) + } + + fn no_toolchain(&self, toolchain: &str) -> cross::Result<()> { + self.failure()?; + self.fallsback()?; + self.no_metadata()?; + self.no_stdout()?; + self.check("has toolchain", |x| { + x.stderr + .contains(&format!("error: toolchain '{toolchain}' is not installed")) + }) + } + + fn no_engine(&self) -> cross::Result<()> { + self.failure()?; + self.no_fallback()?; + self.no_stdout()?; + self.check("has engine", |x| { + x.stderr.contains("no container engine found") + }) + } + + fn no_image(&self, image: &str) -> cross::Result<()> { + // the actual messages differ, but it will contain the image name + self.failure()?; + self.no_fallback()?; + self.no_stdout()?; + self.check("has image", |x| x.stderr.contains(image)) + } +} + +trait Utf8CommandExt { + type Output; + + fn utf8_output(&mut self, msg_info: &mut MessageInfo) -> cross::Result; +} + +impl Utf8CommandExt for Command { + type Output = Utf8Output; + + fn utf8_output(&mut self, msg_info: &mut MessageInfo) -> cross::Result { + let command = self.fmt_message(msg_info); + self.run_and_get_output(msg_info)?.to_utf8(command) + } +} + +trait Utf8OutputExt { + fn to_utf8(self, command: String) -> cross::Result; +} + +impl Utf8OutputExt for Output { + fn to_utf8(self, command: String) -> cross::Result { + Ok(Utf8Output { + command, + status: self.status, + stdout: String::from_utf8(self.stdout)?, + stderr: String::from_utf8(self.stderr)?, + }) + } +} + +fn cross_ui( + target: &str, + image: &str, + path: &Path, + msg_info: &mut MessageInfo, +) -> cross::Result<()> { + // check both versions are printed + let output = cargo_bin("cross", path)? + .arg("--version") + .utf8_output(msg_info)?; + output.success()?; + output.check("does not have cross version", |x| { + x.stdout.contains("cross 0.") + })?; + output.check("does not have cargo version", |x| { + x.stdout.contains("cargo 1.") + })?; + output.fallsback()?; + + // check help falls back + let output = cargo_bin("cross", path)? + .arg("--help") + .utf8_output(msg_info)?; + output.fallsback()?; + output.check("does not have cargo help", |x| { + x.stdout.contains("Rust's package manager") + })?; + + // check that list prints both dependencies + let output = cargo_bin("cross", path)? + .arg("--list") + .utf8_output(msg_info)?; + output.fallsback()?; + output.check("does not have cross commands", |x| { + x.stdout.contains("Cross Commands:") + })?; + output.check("does not have cargo commands", |x| { + x.stdout.contains("Host Commands:") + })?; + output.check("does not have build command", |x| { + x.stdout.contains("alias: build") + })?; + + // check that we fail with invalid colors + let output = cross_command("build", target, image, "/", None)? + .args(["--color"]) + .utf8_output(msg_info)?; + output.failure()?; + output.no_fallback()?; + output.no_stdout()?; + output.check("has no error", |x| { + x.stderr.contains( + "[cross] error: The argument '--color ' requires a value but none was supplied", + ) + })?; + + // check that we fail with both quiet and verbose + let output = cross_command("build", target, image, "/", None)? + .args(["--quiet", "--verbose"]) + .utf8_output(msg_info)?; + output.failure()?; + output.no_fallback()?; + output.no_stdout()?; + output.check("has no error", |x| { + x.stderr + .contains("[cross] error: cannot set both --verbose and --quiet") + })?; + + // check building outside of workspace + // make sure we do it in root, with the target dir as the + // current path, to just ensure we don't have any remnants. + cross_command("build", target, image, "/", None)? + .args(["--target-dir", path.join("target").to_utf8()?]) + .utf8_output(msg_info)? + .outside_package()?; + + // check custom manifest path build + cross_command("build", target, image, path, None)? + .arg("--manifest-path=./workspace/Cargo.toml") + .utf8_output(msg_info)? + .compiles()?; + + // check within a workspace + let workspace_dir = path.join("workspace"); + cross_command("build", target, image, &workspace_dir, None)? + .utf8_output(msg_info)? + .compiles()?; + cross_command("run", target, image, &workspace_dir, None)? + .args(["--features", "dependencies"]) + .utf8_output(msg_info)? + .runs("Hello from test-workspace, bin/dependencies.rs")?; + check_file_exists( + &workspace_dir, + &format!("target/{target}/debug/dependencies"), + )?; + + // check we can run custom binaries + cross_command("run", target, image, &workspace_dir.join("binary"), None)? + .utf8_output(msg_info)? + .runs("Hello from binary, binary/src/main.rs")?; + check_file_exists(&workspace_dir, &format!("target/{target}/debug/binary"))?; + + // check using a custom target directory + cross_command("build", target, image, &workspace_dir, None)? + .args(["--features", "dependencies"]) + .args(["--target-dir", "custom"]) + .utf8_output(msg_info)? + .compiles()?; + check_file_exists( + &workspace_dir, + &format!("custom/{target}/debug/dependencies"), + )?; + + // check using a fake toolchain + let fake_toolchain = "cross-fake-toolchain"; + cross_command("build", target, image, &workspace_dir, Some(fake_toolchain))? + .utf8_output(msg_info)? + .no_toolchain(fake_toolchain)?; + + // test with an invalid container engine + cross_command("build", target, image, &workspace_dir, None)? + .env("CROSS_CONTAINER_ENGINE", "cross-fake-engine") + .utf8_output(msg_info)? + .no_engine()?; + + // test with an invalid image + let fake_image = "ghcr.io/cross-rs/fake-image:latest"; + cross_command("build", target, fake_image, &workspace_dir, None)? + .utf8_output(msg_info)? + .no_image(fake_image)?; + + // TODO(ahuszagh) + // Test other missing config options + // Ex: invalid platforms + + Ok(()) +} + +// TODO(ahuszagh) Needs msg_info +fn cross_util_ui() -> cross::Result<()> { + // TODO(ahuszagh) + // Test help + // Test version + // Test images, volumes, containers, clean + + Ok(()) +} + +// Adapted from +// https://github.com/rust-lang/cargo/blob/485670b3983b52289a2f353d589c57fae2f60f82/tests/testsuite/support/mod.rs#L507 +fn target_dir() -> PathBuf { + env::current_exe() + .ok() + .map(|mut path| { + path.pop(); + if path.ends_with("deps") { + path.pop(); + } + path + }) + .unwrap() +} + +// Adapted from +// https://github.com/assert-rs/assert_cmd/blob/363daac637b5f20ba3524ccf648de5f96f0dc45f/src/cargo.rs#L198-L208 +fn cargo_bin(bin: impl AsRef, cwd: impl AsRef) -> cross::Result { + let bin = bin.as_ref(); + let env_var = format!("CARGO_BIN_EXE_{}", bin); + let path = env::var_os(&env_var) + .map(|p| p.into()) + .unwrap_or_else(|| target_dir().join(format!("{}{}", bin, env::consts::EXE_SUFFIX))); + + match path.is_file() { + true => { + let mut cmd = Command::new(path); + cmd.current_dir(cwd.as_ref()); + Ok(cmd) + } + false => eyre::bail!("unable to find cargo command {bin}"), + } +} + +fn cross_command( + subcommand: &str, + target: &str, + image: &str, + path: impl AsRef, + toolchain: Option<&str>, +) -> cross::Result { + let envvar = format!( + "CROSS_TARGET_{}_IMAGE", + target.replace('-', "_").to_ascii_uppercase() + ); + let mut cmd = cargo_bin("cross", path)?; + if let Some(toolchain) = toolchain { + cmd.arg(&format!("+{toolchain}")); + } + cmd.args([subcommand, "--target", target]); + cmd.env(envvar, image); + + Ok(cmd) +} + +fn check_file_exists(base: impl AsRef, relpath: &str) -> cross::Result<()> { + let path = base.as_ref().join(relpath); + match path.exists() { + true => Ok(()), + false => eyre::bail!("path \"{relpath}\" unexpectedly does not exist"), + } +} + +fn clone(url: &str, path: impl AsRef, msg_info: &mut MessageInfo) -> cross::Result<()> { + let mut cmd = Command::new("git"); + let path = path.as_ref().to_utf8()?; + cmd.args(["clone", "--depth", "1", "--recursive", url, path]); + if !msg_info.is_verbose() { + cmd.stderr(Stdio::null()); + } + cmd.run(msg_info, !msg_info.is_verbose()) + .map_err(Into::into) +} + +fn pull(engine: &docker::Engine, image: &str, msg_info: &mut MessageInfo) -> cross::Result<()> { + msg_info.note(format_args!( + "pulling container image \"{image}\": this may take a while" + ))?; + + let mut cmd = engine.subcommand("pull"); + cmd.arg(image); + if !msg_info.is_verbose() { + cmd.stderr(Stdio::null()); + } + cmd.run(msg_info, !msg_info.is_verbose()) + .map_err(Into::into) +} + +fn build_cross(msg_info: &mut MessageInfo) -> cross::Result<()> { + msg_info.note("building cross: this might take a while")?; + + let project = util::project_dir(msg_info)?; + let mut cmd = Command::new("cargo"); + cmd.current_dir(&project); + cmd.args(["build", "--workspace"]); + if !msg_info.is_verbose() { + cmd.stderr(Stdio::null()); + } + cmd.run(msg_info, !msg_info.is_verbose()) + .map_err(Into::into) +} + +// this always has to be quiet since we capture output +pub fn ui_test( + UiTest { + target, + repository, + tag, + no_pull, + .. + }: UiTest, + engine: &docker::Engine, + msg_info: &mut MessageInfo, +) -> cross::Result<()> { + let temp_dir = temp::temp_dir(None, msg_info)?; + let test_url = "https://github.com/cross-rs/test-workspace"; + let tag = tag.as_deref().unwrap_or("main"); + let target = match target { + Some(target) => target, + None => "x86_64-unknown-linux-gnu".parse()?, + }; + let image = target.image_name(&repository, tag); + + build_cross(msg_info)?; + clone(test_url, &temp_dir, msg_info)?; + if !no_pull { + pull(engine, &image, msg_info)?; + } + + let cleanup = || fs::remove_dir_all(&temp_dir); + + if let Err(err) = cross_ui(&target.name, &image, &temp_dir, msg_info) { + cleanup().ok(); + return Err(err); + } + if let Err(err) = cross_util_ui() { + cleanup().ok(); + return Err(err); + } + + cleanup().map_err(Into::into) +}