Merge branch 'main' into levm/fix/vmTests-fix-changes

.github/scripts/publish_levm_ef_tests_summary.sh

@@ -0,0 +1,25 @@
+curl -X POST $url \
+-H 'Content-Type: application/json; charset=utf-8' \
+--data @- <<EOF
+$(jq -n --arg text "$(cat levm_ef_tests_summary_slack.txt)" '{
+    "blocks": [
+        {
+            "type": "header",
+            "text": {
+                "type": "plain_text",
+                "text": "Daily LEVM EF Tests Run Report"
+            }
+        },
+        {
+            "type": "divider"
+        },
+        {
+            "type": "section",
+            "text": {
+                "type": "mrkdwn",
+                "text": $text
+            }             
+        }
+    ]
+}')
+EOF

.github/workflows/levm_reporter.yaml

@@ -0,0 +1,56 @@
+name: Daily LEVM Reporter
+
+on:
+  schedule:
+    # Every day at midnight
+    - cron: "0 0 * * *"
+  workflow_dispatch:
+
+env:
+  RUST_VERSION: 1.80.1
+
+jobs:
+  ef-test:
+    name: Generate Report for EF Tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v4
+
+      - name: Rustup toolchain install
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          toolchain: ${{ env.RUST_VERSION }}
+
+      - name: Caching
+        uses: Swatinem/rust-cache@v2
+
+      - name: Download EF Tests
+        run: |
+          cd crates/vm/levm
+          make download-evm-ef-tests
+
+      - name: Run tests
+        run: |
+          cd crates/vm/levm
+          make generate-evm-ef-tests-report
+
+      - name: Post results in summary
+        run: |
+          echo "# Daily LEVM EF Tests Run Report" >> $GITHUB_STEP_SUMMARY
+          cat levm_ef_tests_summary_github.txt >> $GITHUB_STEP_SUMMARY
+
+      - name: Post results to ethrex L1 slack channel
+        env:
+          url: ${{ secrets.ETHREX_L1_SLACK_WEBHOOK }}
+        run: sh .github/scripts/publish_levm_ef_tests_summary.sh
+
+      - name: Post results to ethrex L2 slack channel
+        env:
+          url: ${{ secrets.ETHREX_L2_SLACK_WEBHOOK }}
+        run: sh .github/scripts/publish_levm_ef_tests_summary.sh
+
+      - name: Post results to levm slack channel
+        env:
+          url: ${{ secrets.LEVM_SLACK_WEBHOOK }}
+        run: sh .github/scripts/publish_levm_ef_tests_summary.sh

.gitignore

@@ -48,5 +48,7 @@ tests_v3.0.0.tar.gz
 .env
 
 levm_ef_tests_report.txt
-
+levm_ef_tests_summary_slack.txt
+levm_ef_tests_summary_github.txt
+levm_ef_tests_summary.txt
 loc_report.md

cmd/ef_tests/levm/report.rs

@@ -17,6 +17,10 @@ use std::{
     time::Duration,
 };
 
+pub const LEVM_EF_TESTS_SUMMARY_SLACK_FILE_PATH: &str = "./levm_ef_tests_summary_slack.txt";
+pub const LEVM_EF_TESTS_SUMMARY_GITHUB_FILE_PATH: &str = "./levm_ef_tests_summary_slack.txt";
+pub const EF_TESTS_CACHE_FILE_PATH: &str = "./levm_ef_tests_cache.json";
+
 pub type TestVector = (usize, usize, usize);
 
 pub fn progress(reports: &[EFTestReport], time: Duration) -> String {
@@ -39,30 +43,12 @@ pub fn progress(reports: &[EFTestReport], time: Duration) -> String {
         format_duration_as_mm_ss(time)
     )
 }
-pub fn summary(reports: &[EFTestReport]) -> String {
-    let total_passed = reports.iter().filter(|report| report.passed()).count();
-    let total_run = reports.len();
-    format!(
-        "{} {}/{total_run}\n\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n",
-        "Summary:".bold(),
-        if total_passed == total_run {
-            format!("{}", total_passed).green()
-        } else if total_passed > 0 {
-            format!("{}", total_passed).yellow()
-        } else {
-            format!("{}", total_passed).red()
-        },
-        fork_summary(reports, SpecId::CANCUN),
-        fork_summary(reports, SpecId::SHANGHAI),
-        fork_summary(reports, SpecId::HOMESTEAD),
-        fork_summary(reports, SpecId::ISTANBUL),
-        fork_summary(reports, SpecId::LONDON),
-        fork_summary(reports, SpecId::BYZANTIUM),
-        fork_summary(reports, SpecId::BERLIN),
-        fork_summary(reports, SpecId::CONSTANTINOPLE),
-        fork_summary(reports, SpecId::MERGE),
-        fork_summary(reports, SpecId::FRONTIER),
-    )
+
+pub fn format_duration_as_mm_ss(duration: Duration) -> String {
+    let total_seconds = duration.as_secs();
+    let minutes = total_seconds / 60;
+    let seconds = total_seconds % 60;
+    format!("{minutes:02}:{seconds:02}")
 }
 
 pub fn write(reports: &[EFTestReport]) -> Result<PathBuf, EFTestRunnerError> {
@@ -86,8 +72,6 @@ pub fn write(reports: &[EFTestReport]) -> Result<PathBuf, EFTestRunnerError> {
     Ok(report_file_path)
 }
 
-pub const EF_TESTS_CACHE_FILE_PATH: &str = "./levm_ef_tests_cache.json";
-
 pub fn cache(reports: &[EFTestReport]) -> Result<PathBuf, EFTestRunnerError> {
     let cache_file_path = PathBuf::from(EF_TESTS_CACHE_FILE_PATH);
     let cache = serde_json::to_string_pretty(&reports).map_err(|err| {
@@ -122,11 +106,135 @@ pub fn load() -> Result<Vec<EFTestReport>, EFTestRunnerError> {
     }
 }
 
-pub fn format_duration_as_mm_ss(duration: Duration) -> String {
-    let total_seconds = duration.as_secs();
-    let minutes = total_seconds / 60;
-    let seconds = total_seconds % 60;
-    format!("{minutes:02}:{seconds:02}")
+pub fn summary_for_slack(reports: &[EFTestReport]) -> String {
+    let total_passed = reports.iter().filter(|report| report.passed()).count();
+    let total_run = reports.len();
+    let success_percentage = (total_passed as f64 / total_run as f64) * 100.0;
+    format!(
+        r#"*Summary*: {total_passed}/{total_run} ({success_percentage:.2}%)\n\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n"#,
+        fork_summary_for_slack(reports, SpecId::CANCUN),
+        fork_summary_for_slack(reports, SpecId::SHANGHAI),
+        fork_summary_for_slack(reports, SpecId::HOMESTEAD),
+        fork_summary_for_slack(reports, SpecId::ISTANBUL),
+        fork_summary_for_slack(reports, SpecId::LONDON),
+        fork_summary_for_slack(reports, SpecId::BYZANTIUM),
+        fork_summary_for_slack(reports, SpecId::BERLIN),
+        fork_summary_for_slack(reports, SpecId::CONSTANTINOPLE),
+        fork_summary_for_slack(reports, SpecId::MERGE),
+        fork_summary_for_slack(reports, SpecId::FRONTIER),
+    )
+}
+
+fn fork_summary_for_slack(reports: &[EFTestReport], fork: SpecId) -> String {
+    let fork_str: &str = fork.into();
+    let (fork_tests, fork_passed_tests, fork_success_percentage) = fork_statistics(reports, fork);
+    format!(r#"*{fork_str}:* {fork_passed_tests}/{fork_tests} ({fork_success_percentage:.2}%)"#)
+}
+
+pub fn write_summary_for_slack(reports: &[EFTestReport]) -> Result<PathBuf, EFTestRunnerError> {
+    let summary_file_path = PathBuf::from(LEVM_EF_TESTS_SUMMARY_SLACK_FILE_PATH);
+    std::fs::write(
+        LEVM_EF_TESTS_SUMMARY_SLACK_FILE_PATH,
+        summary_for_slack(reports),
+    )
+    .map_err(|err| {
+        EFTestRunnerError::Internal(InternalError::MainRunnerInternal(format!(
+            "Failed to write summary to file: {err}"
+        )))
+    })?;
+    Ok(summary_file_path)
+}
+
+pub fn summary_for_github(reports: &[EFTestReport]) -> String {
+    let total_passed = reports.iter().filter(|report| report.passed()).count();
+    let total_run = reports.len();
+    let success_percentage = (total_passed as f64 / total_run as f64) * 100.0;
+    format!(
+        r#"Summary: {total_passed}/{total_run} ({success_percentage:.2}%)\n\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n"#,
+        fork_summary_for_github(reports, SpecId::CANCUN),
+        fork_summary_for_github(reports, SpecId::SHANGHAI),
+        fork_summary_for_github(reports, SpecId::HOMESTEAD),
+        fork_summary_for_github(reports, SpecId::ISTANBUL),
+        fork_summary_for_github(reports, SpecId::LONDON),
+        fork_summary_for_github(reports, SpecId::BYZANTIUM),
+        fork_summary_for_github(reports, SpecId::BERLIN),
+        fork_summary_for_github(reports, SpecId::CONSTANTINOPLE),
+        fork_summary_for_github(reports, SpecId::MERGE),
+        fork_summary_for_github(reports, SpecId::FRONTIER),
+    )
+}
+
+fn fork_summary_for_github(reports: &[EFTestReport], fork: SpecId) -> String {
+    let fork_str: &str = fork.into();
+    let (fork_tests, fork_passed_tests, fork_success_percentage) = fork_statistics(reports, fork);
+    format!("{fork_str}: {fork_passed_tests}/{fork_tests} ({fork_success_percentage:.2}%)")
+}
+
+pub fn write_summary_for_github(reports: &[EFTestReport]) -> Result<PathBuf, EFTestRunnerError> {
+    let summary_file_path = PathBuf::from(LEVM_EF_TESTS_SUMMARY_GITHUB_FILE_PATH);
+    std::fs::write(
+        LEVM_EF_TESTS_SUMMARY_GITHUB_FILE_PATH,
+        summary_for_github(reports),
+    )
+    .map_err(|err| {
+        EFTestRunnerError::Internal(InternalError::MainRunnerInternal(format!(
+            "Failed to write summary to file: {err}"
+        )))
+    })?;
+    Ok(summary_file_path)
+}
+
+pub fn summary_for_shell(reports: &[EFTestReport]) -> String {
+    let total_passed = reports.iter().filter(|report| report.passed()).count();
+    let total_run = reports.len();
+    let success_percentage = (total_passed as f64 / total_run as f64) * 100.0;
+    format!(
+        "{} {}/{total_run} ({success_percentage:.2})\n\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n{}\n",
+        "Summary:".bold(),
+        if total_passed == total_run {
+            format!("{}", total_passed).green()
+        } else if total_passed > 0 {
+            format!("{}", total_passed).yellow()
+        } else {
+            format!("{}", total_passed).red()
+        },
+        fork_summary_shell(reports, SpecId::CANCUN),
+        fork_summary_shell(reports, SpecId::SHANGHAI),
+        fork_summary_shell(reports, SpecId::HOMESTEAD),
+        fork_summary_shell(reports, SpecId::ISTANBUL),
+        fork_summary_shell(reports, SpecId::LONDON),
+        fork_summary_shell(reports, SpecId::BYZANTIUM),
+        fork_summary_shell(reports, SpecId::BERLIN),
+        fork_summary_shell(reports, SpecId::CONSTANTINOPLE),
+        fork_summary_shell(reports, SpecId::MERGE),
+        fork_summary_shell(reports, SpecId::FRONTIER),
+    )
+}
+
+fn fork_summary_shell(reports: &[EFTestReport], fork: SpecId) -> String {
+    let fork_str: &str = fork.into();
+    let (fork_tests, fork_passed_tests, fork_success_percentage) = fork_statistics(reports, fork);
+    format!(
+        "{}: {}/{fork_tests} ({fork_success_percentage:.2}%)",
+        fork_str.bold(),
+        if fork_passed_tests == fork_tests {
+            format!("{}", fork_passed_tests).green()
+        } else if fork_passed_tests > 0 {
+            format!("{}", fork_passed_tests).yellow()
+        } else {
+            format!("{}", fork_passed_tests).red()
+        },
+    )
+}
+
+fn fork_statistics(reports: &[EFTestReport], fork: SpecId) -> (usize, usize, f64) {
+    let fork_tests = reports.iter().filter(|report| report.fork == fork).count();
+    let fork_passed_tests = reports
+        .iter()
+        .filter(|report| report.fork == fork && report.passed())
+        .count();
+    let fork_success_percentage = (fork_passed_tests as f64 / fork_tests as f64) * 100.0;
+    (fork_tests, fork_passed_tests, fork_success_percentage)
 }
 
 #[derive(Debug, Default, Clone)]
@@ -149,16 +257,16 @@ impl Display for EFTestsReport {
             },
         )?;
         writeln!(f)?;
-        writeln!(f, "{}", fork_summary(&self.0, SpecId::CANCUN))?;
-        writeln!(f, "{}", fork_summary(&self.0, SpecId::SHANGHAI))?;
-        writeln!(f, "{}", fork_summary(&self.0, SpecId::HOMESTEAD))?;
-        writeln!(f, "{}", fork_summary(&self.0, SpecId::ISTANBUL))?;
-        writeln!(f, "{}", fork_summary(&self.0, SpecId::LONDON))?;
-        writeln!(f, "{}", fork_summary(&self.0, SpecId::BYZANTIUM))?;
-        writeln!(f, "{}", fork_summary(&self.0, SpecId::BERLIN))?;
-        writeln!(f, "{}", fork_summary(&self.0, SpecId::CONSTANTINOPLE))?;
-        writeln!(f, "{}", fork_summary(&self.0, SpecId::MERGE))?;
-        writeln!(f, "{}", fork_summary(&self.0, SpecId::FRONTIER))?;
+        writeln!(f, "{}", fork_summary_shell(&self.0, SpecId::CANCUN))?;
+        writeln!(f, "{}", fork_summary_shell(&self.0, SpecId::SHANGHAI))?;
+        writeln!(f, "{}", fork_summary_shell(&self.0, SpecId::HOMESTEAD))?;
+        writeln!(f, "{}", fork_summary_shell(&self.0, SpecId::ISTANBUL))?;
+        writeln!(f, "{}", fork_summary_shell(&self.0, SpecId::LONDON))?;
+        writeln!(f, "{}", fork_summary_shell(&self.0, SpecId::BYZANTIUM))?;
+        writeln!(f, "{}", fork_summary_shell(&self.0, SpecId::BERLIN))?;
+        writeln!(f, "{}", fork_summary_shell(&self.0, SpecId::CONSTANTINOPLE))?;
+        writeln!(f, "{}", fork_summary_shell(&self.0, SpecId::MERGE))?;
+        writeln!(f, "{}", fork_summary_shell(&self.0, SpecId::FRONTIER))?;
         writeln!(f)?;
         writeln!(f, "{}", "Failed tests:".bold())?;
         writeln!(f)?;
@@ -236,26 +344,6 @@ impl Display for EFTestsReport {
     }
 }
 
-fn fork_summary(reports: &[EFTestReport], fork: SpecId) -> String {
-    let fork_str: &str = fork.into();
-    let fork_tests = reports.iter().filter(|report| report.fork == fork).count();
-    let fork_passed_tests = reports
-        .iter()
-        .filter(|report| report.fork == fork && report.passed())
-        .count();
-    format!(
-        "{}: {}/{fork_tests}",
-        fork_str.bold(),
-        if fork_passed_tests == fork_tests {
-            format!("{}", fork_passed_tests).green()
-        } else if fork_passed_tests > 0 {
-            format!("{}", fork_passed_tests).yellow()
-        } else {
-            format!("{}", fork_passed_tests).red()
-        },
-    )
-}
-
 #[derive(Debug, Default, Clone, Serialize, Deserialize)]
 pub struct EFTestReport {
     pub name: String,

cmd/ef_tests/levm/runner/mod.rs

@@ -44,15 +44,20 @@ pub struct EFTestRunnerOptions {
     pub fork: Vec<SpecId>,
     #[arg(short, long, value_name = "TESTS")]
     pub tests: Vec<String>,
+    #[arg(short, long, value_name = "SUMMARY", default_value = "false")]
+    pub summary: bool,
 }
 
 pub fn run_ef_tests(
     ef_tests: Vec<EFTest>,
-    _opts: &EFTestRunnerOptions,
+    opts: &EFTestRunnerOptions,
 ) -> Result<(), EFTestRunnerError> {
     let mut reports = report::load()?;
     if reports.is_empty() {
-        run_with_levm(&mut reports, &ef_tests)?;
+        run_with_levm(&mut reports, &ef_tests, opts)?;
+    }
+    if opts.summary {
+        return Ok(());
     }
     re_run_with_revm(&mut reports, &ef_tests)?;
     write_report(&reports)
@@ -61,6 +66,7 @@ pub fn run_ef_tests(
 fn run_with_levm(
     reports: &mut Vec<EFTestReport>,
     ef_tests: &[EFTest],
+    opts: &EFTestRunnerOptions,
 ) -> Result<(), EFTestRunnerError> {
     let levm_run_time = std::time::Instant::now();
     let mut levm_run_spinner = Spinner::new(
@@ -83,8 +89,13 @@ fn run_with_levm(
     }
     levm_run_spinner.success(&report::progress(reports, levm_run_time.elapsed()));
 
+    if opts.summary {
+        report::write_summary_for_slack(reports)?;
+        report::write_summary_for_github(reports)?;
+    }
+
     let mut summary_spinner = Spinner::new(Dots, "Loading summary...".to_owned(), Color::Cyan);
-    summary_spinner.success(&report::summary(reports));
+    summary_spinner.success(&report::summary_for_shell(reports));
     Ok(())
 }
 

crates/l2/Cargo.toml

@@ -41,3 +41,4 @@ path = "./l2.rs"
 
 [lints.clippy]
 unwrap_used = "deny"
+expect_used = "deny"

crates/l2/clippy.toml

@@ -0,0 +1 @@
+allow-unwrap-in-tests = true