From ac67bdbb2c0526a8b29ed11a26403013ccd2fa5e Mon Sep 17 00:00:00 2001 From: Haled Odat <8566042+HalidOdat@users.noreply.github.com> Date: Mon, 14 Aug 2023 12:35:51 +0200 Subject: [PATCH 1/4] Lazily clone test262 repository --- .gitignore | 3 + .gitmodules | 3 - boa_tester/src/main.rs | 195 ++++++++++++++++++++++++++++++++++---- boa_tester/src/results.rs | 19 ++-- test262 | 1 - 5 files changed, 189 insertions(+), 32 deletions(-) delete mode 100644 .gitmodules delete mode 160000 test262 diff --git a/.gitignore b/.gitignore index 98608e83527..a285de586d2 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,9 @@ yarn-error.log tests/js .boa_history +# test262 testing suite +test262 + # Profiling *.string_data *.string_index diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index c41542feb22..00000000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "test262"] - path = test262 - url = https://github.com/tc39/test262.git diff --git a/boa_tester/src/main.rs b/boa_tester/src/main.rs index e79ac98ed0a..23392c1ddf4 100644 --- a/boa_tester/src/main.rs +++ b/boa_tester/src/main.rs @@ -95,6 +95,7 @@ use std::{ io::Read, ops::{Add, AddAssign}, path::{Path, PathBuf}, + process::Command, }; /// Structure to allow defining ignored tests, features and files that should @@ -157,8 +158,21 @@ enum Cli { verbose: u8, /// Path to the Test262 suite. - #[arg(long, default_value = "./test262", value_hint = ValueHint::DirPath)] - test262_path: PathBuf, + #[arg( + long, + value_hint = ValueHint::DirPath, + conflicts_with = "test262_update", + conflicts_with = "test262_commit" + )] + test262_path: Option, + + /// Specify the Test262 commit when cloning the repository. + #[arg(long, conflicts_with = "test262_update")] + test262_commit: Option, + + /// Update Test262 commit of the repository. + #[arg(long)] + test262_update: bool, /// Which specific test or test suite to run. Should be a path relative to the Test262 directory: e.g. "test/language/types/number" #[arg(short, long, default_value = "test", value_hint = ValueHint::AnyPath)] @@ -204,6 +218,8 @@ enum Cli { }, } +const DEFAULT_TEST262_DIRECTORY: &str = "test262"; + /// Program entry point. fn main() -> Result<()> { color_eyre::install()?; @@ -211,6 +227,8 @@ fn main() -> Result<()> { Cli::Run { verbose, test262_path, + test262_commit, + test262_update, suite, output, optimize, @@ -218,21 +236,31 @@ fn main() -> Result<()> { ignored: ignore, edition, versioned, - } => run_test_suite( - verbose, - !disable_parallelism, - test262_path.as_path(), - suite.as_path(), - output.as_deref(), - ignore.as_path(), - edition.unwrap_or_default(), - versioned, - if optimize { - OptimizerOptions::OPTIMIZE_ALL + } => { + let test262_path = if let Some(path) = test262_path.as_deref() { + path } else { - OptimizerOptions::empty() - }, - ), + clone_test262(test262_update, test262_commit.as_deref(), verbose)?; + + Path::new(DEFAULT_TEST262_DIRECTORY) + }; + + run_test_suite( + verbose, + !disable_parallelism, + test262_path, + suite.as_path(), + output.as_deref(), + ignore.as_path(), + edition.unwrap_or_default(), + versioned, + if optimize { + OptimizerOptions::OPTIMIZE_ALL + } else { + OptimizerOptions::empty() + }, + ) + } Cli::Compare { base, new, @@ -241,12 +269,137 @@ fn main() -> Result<()> { } } +/// Returns the commit hash and commit message of the provided branch name. +fn get_last_branch_commit(branch: &str) -> Result<(String, String)> { + let result = Command::new("git") + .arg("log") + .args(["-n", "1"]) + .arg("--pretty=format:%H %s") + .arg(branch) + .current_dir(DEFAULT_TEST262_DIRECTORY) + .output()?; + + if !result.status.success() { + bail!( + "test262 getting commit hash and message failed with return code {:?}", + result.status.code() + ); + } + + let output = std::str::from_utf8(&result.stdout)?.trim(); + + let (hash, message) = output + .split_once(' ') + .expect("git log output to contain hash and message"); + + Ok((hash.into(), message.into())) +} + +fn reset_test262_commit(commit: &str, verbose: u8) -> Result<()> { + if verbose != 0 { + println!("Reset test262 to commit: {commit}..."); + } + + let result = Command::new("git") + .arg("reset") + .arg("--hard") + .arg(commit) + .current_dir(DEFAULT_TEST262_DIRECTORY) + .status()?; + + if !result.success() { + bail!( + "test262 commit {commit} checkout failed with return code: {:?}", + result.code() + ); + } + + Ok(()) +} + +fn clone_test262(update: bool, commit: Option<&str>, verbose: u8) -> Result<()> { + const TEST262_REPOSITORY: &str = "https://github.com/tc39/test262"; + + if Path::new(DEFAULT_TEST262_DIRECTORY).is_dir() { + if verbose != 0 { + println!("Fetching latest test262 commits..."); + } + let result = Command::new("git") + .arg("fetch") + .current_dir(DEFAULT_TEST262_DIRECTORY) + .status()?; + + if !result.success() { + bail!( + "Test262 fetching latest failed with return code {:?}", + result.code() + ); + } + + let (current_commit_hash, current_commit_message) = get_last_branch_commit("HEAD")?; + + if let Some(commit) = commit { + if current_commit_hash != commit { + println!("Test262 switching to commit {commit}..."); + reset_test262_commit(commit, verbose)?; + } + return Ok(()); + } + + if verbose != 0 { + println!("Checking latest Test262 with current HEAD..."); + } + let (latest_commit_hash, latest_commit_message) = get_last_branch_commit("origin/main")?; + + if current_commit_hash != latest_commit_hash { + if update { + println!("Updating Test262 repository:"); + } else { + println!("Warning Test262 repository is not in sync, use '--test262-update' to automatically update it:"); + } + + println!(" Current commit: {current_commit_hash} {current_commit_message}"); + println!(" Latest commit: {latest_commit_hash} {latest_commit_message}"); + + if update { + reset_test262_commit(&latest_commit_hash, verbose)?; + } + } + + return Ok(()); + } + + println!("Cloning test262..."); + let result = Command::new("git") + .arg("clone") + .arg(TEST262_REPOSITORY) + .arg(DEFAULT_TEST262_DIRECTORY) + .status()?; + + if !result.success() { + bail!( + "Cloning Test262 repository failed with return code {:?}", + result.code() + ); + } + + if let Some(commit) = commit { + if verbose != 0 { + println!("Reset Test262 to commit: {commit}..."); + } + + reset_test262_commit(commit, verbose)?; + } + + Ok(()) +} + /// Runs the full test suite. #[allow(clippy::too_many_arguments)] fn run_test_suite( verbose: u8, parallel: bool, - test262: &Path, + test262_path: &Path, suite: &Path, output: Option<&Path>, ignore: &Path, @@ -275,10 +428,10 @@ fn run_test_suite( if verbose != 0 { println!("Loading the test suite..."); } - let harness = read_harness(test262).wrap_err("could not read harness")?; + let harness = read_harness(test262_path).wrap_err("could not read harness")?; if suite.to_string_lossy().ends_with(".js") { - let test = read_test(&test262.join(suite)).wrap_err_with(|| { + let test = read_test(&test262_path.join(suite)).wrap_err_with(|| { let suite = suite.display(); format!("could not read the test {suite}") })?; @@ -296,7 +449,7 @@ fn run_test_suite( println!(); } else { - let suite = read_suite(&test262.join(suite), &ignored, false).wrap_err_with(|| { + let suite = read_suite(&test262_path.join(suite), &ignored, false).wrap_err_with(|| { let suite = suite.display(); format!("could not read the suite {suite}") })?; @@ -366,7 +519,7 @@ fn run_test_suite( } if let Some(output) = output { - write_json(results, output, verbose) + write_json(results, output, verbose, test262_path) .wrap_err("could not write the results to the output JSON file")?; } } diff --git a/boa_tester/src/results.rs b/boa_tester/src/results.rs index 1b78a791c28..bd3a044fe3a 100644 --- a/boa_tester/src/results.rs +++ b/boa_tester/src/results.rs @@ -6,7 +6,7 @@ use fxhash::FxHashSet; use serde::{Deserialize, Serialize}; use std::{ env, fs, - io::{self, BufReader, BufWriter}, + io::{BufReader, BufWriter}, path::Path, }; @@ -81,7 +81,12 @@ const FEATURES_FILE_NAME: &str = "features.json"; /// Writes the results of running the test suite to the given JSON output file. /// /// It will append the results to the ones already present, in an array. -pub(crate) fn write_json(results: SuiteResult, output_dir: &Path, verbose: u8) -> io::Result<()> { +pub(crate) fn write_json( + results: SuiteResult, + output_dir: &Path, + verbose: u8, + test262_path: &Path, +) -> Result<()> { let mut branch = env::var("GITHUB_REF").unwrap_or_default(); if branch.starts_with("refs/pull") { branch = "pull".to_owned(); @@ -108,7 +113,7 @@ pub(crate) fn write_json(results: SuiteResult, output_dir: &Path, verbose: u8) - let new_results = ResultInfo { commit: env::var("GITHUB_SHA").unwrap_or_default().into_boxed_str(), - test262_commit: get_test262_commit(), + test262_commit: get_test262_commit(test262_path)?, results, }; @@ -157,12 +162,12 @@ pub(crate) fn write_json(results: SuiteResult, output_dir: &Path, verbose: u8) - } /// Gets the commit OID of the test262 submodule. -fn get_test262_commit() -> Box { - let mut commit_id = fs::read_to_string(".git/modules/test262/HEAD") - .expect("did not find git submodule ref at '.git/modules/test262/HEAD'"); +fn get_test262_commit(test262_path: &Path) -> Result> { + let main_head_path = test262_path.join(".git/refs/heads/main"); + let mut commit_id = fs::read_to_string(main_head_path)?; // Remove newline. commit_id.pop(); - commit_id.into_boxed_str() + Ok(commit_id.into_boxed_str()) } /// Updates the GitHub pages repository by pulling latest changes before writing the new things. diff --git a/test262 b/test262 deleted file mode 160000 index 59bad898983..00000000000 --- a/test262 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 59bad89898333fdadf4af25519e7bdb43ec295ac From 3a0d75cb799f96bb0153995569d0ffdedf569316 Mon Sep 17 00:00:00 2001 From: Haled Odat <8566042+HalidOdat@users.noreply.github.com> Date: Thu, 21 Sep 2023 15:05:37 +0200 Subject: [PATCH 2/4] Put test262 commit in config file --- boa_tester/src/main.rs | 81 +++++++++++++++---------- test_ignore.toml => test262_config.toml | 3 + 2 files changed, 52 insertions(+), 32 deletions(-) rename test_ignore.toml => test262_config.toml (97%) diff --git a/boa_tester/src/main.rs b/boa_tester/src/main.rs index 23392c1ddf4..44c5fce96a4 100644 --- a/boa_tester/src/main.rs +++ b/boa_tester/src/main.rs @@ -91,13 +91,30 @@ use serde::{ Deserialize, Deserializer, Serialize, }; use std::{ - fs::{self, File}, - io::Read, ops::{Add, AddAssign}, path::{Path, PathBuf}, process::Command, }; +/// Structure that contains the configuration of the tester. +#[derive(Debug, Deserialize)] +struct Config { + commit: String, + ignored: Ignored, +} + +impl Config { + /// Get the `test262` repository commit. + pub(crate) fn commit(&self) -> &str { + &self.commit + } + + /// Get [`Ignored`] `test262` tests and features. + pub(crate) const fn ignored(&self) -> &Ignored { + &self.ignored + } +} + /// Structure to allow defining ignored tests, features and files that should /// be ignored even when reading. #[derive(Debug, Deserialize)] @@ -161,18 +178,13 @@ enum Cli { #[arg( long, value_hint = ValueHint::DirPath, - conflicts_with = "test262_update", conflicts_with = "test262_commit" )] test262_path: Option, - /// Specify the Test262 commit when cloning the repository. - #[arg(long, conflicts_with = "test262_update")] - test262_commit: Option, - - /// Update Test262 commit of the repository. + /// Specify the Test262 commit when cloning the repository. To checkout to the latest commit set to "latest". #[arg(long)] - test262_update: bool, + test262_commit: Option, /// Which specific test or test suite to run. Should be a path relative to the Test262 directory: e.g. "test/language/types/number" #[arg(short, long, default_value = "test", value_hint = ValueHint::AnyPath)] @@ -190,9 +202,9 @@ enum Cli { #[arg(short, long)] disable_parallelism: bool, - /// Path to a TOML file with the ignored tests, features, flags and/or files. - #[arg(short, long, default_value = "test_ignore.toml", value_hint = ValueHint::FilePath)] - ignored: PathBuf, + /// Path to a TOML file containing tester config. + #[arg(short, long, default_value = "test262_config.toml", value_hint = ValueHint::FilePath)] + config: PathBuf, /// Maximum ECMAScript edition to test for. #[arg(long)] @@ -228,30 +240,40 @@ fn main() -> Result<()> { verbose, test262_path, test262_commit, - test262_update, suite, output, optimize, disable_parallelism, - ignored: ignore, + config: config_path, edition, versioned, } => { + let config: Config = { + let input = std::fs::read_to_string(config_path)?; + toml::from_str(&input).wrap_err("could not decode tester config file")? + }; + + let test262_commit = match test262_commit.as_deref() { + Some("latest") => None, + Some(commit) => Some(commit), + None => Some(config.commit()), + }; + let test262_path = if let Some(path) = test262_path.as_deref() { path } else { - clone_test262(test262_update, test262_commit.as_deref(), verbose)?; + clone_test262(test262_commit, verbose)?; Path::new(DEFAULT_TEST262_DIRECTORY) }; run_test_suite( + &config, verbose, !disable_parallelism, test262_path, suite.as_path(), output.as_deref(), - ignore.as_path(), edition.unwrap_or_default(), versioned, if optimize { @@ -317,9 +339,11 @@ fn reset_test262_commit(commit: &str, verbose: u8) -> Result<()> { Ok(()) } -fn clone_test262(update: bool, commit: Option<&str>, verbose: u8) -> Result<()> { +fn clone_test262(commit: Option<&str>, verbose: u8) -> Result<()> { const TEST262_REPOSITORY: &str = "https://github.com/tc39/test262"; + let update = commit.is_none(); + if Path::new(DEFAULT_TEST262_DIRECTORY).is_dir() { if verbose != 0 { println!("Fetching latest test262 commits..."); @@ -355,7 +379,7 @@ fn clone_test262(update: bool, commit: Option<&str>, verbose: u8) -> Result<()> if update { println!("Updating Test262 repository:"); } else { - println!("Warning Test262 repository is not in sync, use '--test262-update' to automatically update it:"); + println!("Warning Test262 repository is not in sync, use '--test262-commit latest' to automatically update it:"); } println!(" Current commit: {current_commit_hash} {current_commit_message}"); @@ -397,12 +421,12 @@ fn clone_test262(update: bool, commit: Option<&str>, verbose: u8) -> Result<()> /// Runs the full test suite. #[allow(clippy::too_many_arguments)] fn run_test_suite( + config: &Config, verbose: u8, parallel: bool, test262_path: &Path, suite: &Path, output: Option<&Path>, - ignore: &Path, edition: SpecEdition, versioned: bool, optimizer_options: OptimizerOptions, @@ -413,18 +437,10 @@ fn run_test_suite( bail!("the output path must be a directory."); } } else { - fs::create_dir_all(path).wrap_err("could not create the output directory")?; + std::fs::create_dir_all(path).wrap_err("could not create the output directory")?; } } - let ignored = { - let mut input = String::new(); - let mut f = File::open(ignore).wrap_err("could not open ignored tests file")?; - f.read_to_string(&mut input) - .wrap_err("could not read ignored tests file")?; - toml::from_str(&input).wrap_err("could not decode ignored tests file")? - }; - if verbose != 0 { println!("Loading the test suite..."); } @@ -449,10 +465,11 @@ fn run_test_suite( println!(); } else { - let suite = read_suite(&test262_path.join(suite), &ignored, false).wrap_err_with(|| { - let suite = suite.display(); - format!("could not read the suite {suite}") - })?; + let suite = + read_suite(&test262_path.join(suite), config.ignored(), false).wrap_err_with(|| { + let suite = suite.display(); + format!("could not read the suite {suite}") + })?; if verbose != 0 { println!("Test suite loaded, starting tests..."); diff --git a/test_ignore.toml b/test262_config.toml similarity index 97% rename from test_ignore.toml rename to test262_config.toml index e3ea1c110f8..032d535c9f8 100644 --- a/test_ignore.toml +++ b/test262_config.toml @@ -1,3 +1,6 @@ +commit = "dd30d83e9e9b838615ac82c9629275b146f8648e" + +[ignored] # Not implemented yet: flags = [] From 003b2aa0a489064cdc3c9e0b71c4b0b6c9d8463e Mon Sep 17 00:00:00 2001 From: Haled Odat <8566042+HalidOdat@users.noreply.github.com> Date: Thu, 21 Sep 2023 15:35:09 +0200 Subject: [PATCH 3/4] Correct doc and add ability to specify latest commit in config --- boa_tester/src/main.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/boa_tester/src/main.rs b/boa_tester/src/main.rs index 44c5fce96a4..9da7ded3339 100644 --- a/boa_tester/src/main.rs +++ b/boa_tester/src/main.rs @@ -99,17 +99,19 @@ use std::{ /// Structure that contains the configuration of the tester. #[derive(Debug, Deserialize)] struct Config { + #[serde(default)] commit: String, + #[serde(default)] ignored: Ignored, } impl Config { - /// Get the `test262` repository commit. + /// Get the `Test262` repository commit. pub(crate) fn commit(&self) -> &str { &self.commit } - /// Get [`Ignored`] `test262` tests and features. + /// Get [`Ignored`] `Test262` tests and features. pub(crate) const fn ignored(&self) -> &Ignored { &self.ignored } @@ -182,7 +184,7 @@ enum Cli { )] test262_path: Option, - /// Specify the Test262 commit when cloning the repository. To checkout to the latest commit set to "latest". + /// Override config's Test262 commit. To checkout the latest commit set this to "latest". #[arg(long)] test262_commit: Option, @@ -253,10 +255,9 @@ fn main() -> Result<()> { toml::from_str(&input).wrap_err("could not decode tester config file")? }; - let test262_commit = match test262_commit.as_deref() { - Some("latest") => None, - Some(commit) => Some(commit), - None => Some(config.commit()), + let test262_commit = match (test262_commit.as_deref(), config.commit()) { + (Some("latest"), _) | (None, "" | "latest") => None, + (Some(commit), _) | (None, commit) => Some(commit), }; let test262_path = if let Some(path) = test262_path.as_deref() { From 55215e580706e28d4f26eddea29bd8c0d7891511 Mon Sep 17 00:00:00 2001 From: Haled Odat <8566042+HalidOdat@users.noreply.github.com> Date: Thu, 21 Sep 2023 22:21:26 +0200 Subject: [PATCH 4/4] Apply review --- boa_tester/src/main.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/boa_tester/src/main.rs b/boa_tester/src/main.rs index 9da7ded3339..ef84a240cab 100644 --- a/boa_tester/src/main.rs +++ b/boa_tester/src/main.rs @@ -255,10 +255,10 @@ fn main() -> Result<()> { toml::from_str(&input).wrap_err("could not decode tester config file")? }; - let test262_commit = match (test262_commit.as_deref(), config.commit()) { - (Some("latest"), _) | (None, "" | "latest") => None, - (Some(commit), _) | (None, commit) => Some(commit), - }; + let test262_commit = test262_commit + .as_deref() + .or_else(|| Some(config.commit())) + .filter(|s| !["", "latest"].contains(s)); let test262_path = if let Some(path) = test262_path.as_deref() { path