Fix @serial scenarios continue running after failure on `.fail_fast…

…()` option (#252)

- make `runner::Basic::fail_fast()` available as `Cucumber::fail_fast()`

CHANGELOG.md

@@ -14,8 +14,14 @@ All user visible changes to `cucumber` crate will be documented in this file. Th
 ### Added
 
 - [Gherkin] syntax highlighting in the Book. ([#251]) 
+- `runner::Basic::fail_fast()` method as `Cucumber::fail_fast()`. ([#252])
+
+### Fixed
+
+- `@serial` `Scenario`s continue running after failure when `--fail-fast()` CLI option is specified. ([#252])
 
 [#251]: /../../pull/251
+[#252]: /../../pull/252
 
 
 

Cargo.toml

@@ -19,7 +19,7 @@ repository = "https://github.com/cucumber-rs/cucumber"
 readme = "README.md"
 categories = ["asynchronous", "development-tools::testing"]
 keywords = ["cucumber", "testing", "bdd", "atdd", "async"]
-include = ["/src/", "/tests/after_hook.rs", "/tests/fail_fast.rs", "/tests/json.rs", "/tests/junit.rs", "/tests/libtest.rs", "/tests/retry.rs", "/tests/retry_fail_on_skipped.rs", "/tests/wait.rs", "/LICENSE-*", "/README.md", "/CHANGELOG.md"]
+include = ["/src/", "/tests/after_hook.rs", "/tests/fail_fast.rs", "/tests/from_str_and_parameter.rs", "/tests/json.rs", "/tests/junit.rs", "/tests/libtest.rs", "/tests/result.rs", "/tests/retry.rs", "/tests/retry_fail_fast.rs", "/tests/retry_fail_on_skipped.rs", "/tests/wait.rs", "/LICENSE-*", "/README.md", "/CHANGELOG.md"]
 
 [package.metadata.docs.rs]
 all-features = true
@@ -86,6 +86,10 @@ harness = false
 name = "fail_fast"
 harness = false
 
+[[test]]
+name = "from_str_and_parameter"
+harness = false
+
 [[test]]
 name = "json"
 required-features = ["output-json"]
@@ -101,10 +105,18 @@ name = "libtest"
 required-features = ["libtest"]
 harness = false
 
+[[test]]
+name = "result"
+harness = false
+
 [[test]]
 name = "retry"
 harness = false
 
+[[test]]
+name = "retry_fail_fast"
+harness = false
+
 [[test]]
 name = "retry_fail_on_skipped"
 harness = false

book/src/writing/retries.md

@@ -136,6 +136,8 @@ The following [CLI option]s are related to the [scenario] retries:
 - `@retry.after(1s)` [tag] always delays 1 second before next retry attempt, even if `--retry-after` [CLI option] provides another value. Number of retries is taken from `--retry-after` [CLI option], if it's specified, otherwise defaults a single retry attempt.
 - `@retry(3).after(1s)` always retries failed scenarios at most 3 times with 1 second delay before each attempt, ignoring `--retry` and `--retry-after` [CLI option]s.
 
+> __NOTE__: When using with `--fail-fast` [CLI option] (or [`.fail_fast()` builder config][1]), [scenario]s are considered as failed only in case they exhaust all retry attempts and then still do fail.
+
 > __TIP__: It could be handy to specify `@retry` [tags][tag] only, without any explicit values, and use `--retry=n --retry-after=d --retry-tag-filter=@retry` [CLI option]s to overwrite retrying parameters without affecting any other [scenario]s.
 
 
@@ -150,3 +152,5 @@ The following [CLI option]s are related to the [scenario] retries:
 [simulation testing]: https://github.com/madsys-dev/madsim
 [step]: https://cucumber.io/docs/gherkin/reference#steps
 [tag]: https://cucumber.io/docs/cucumber/api#tags
+
+[1]: https://docs.rs/cucumber/*/cucumber/struct.Cucumber.html#method.fail_fast

src/cucumber.rs

@@ -949,6 +949,22 @@ where
         self
     }
 
+    /// Makes stop running tests on the first failure.
+    ///
+    /// __NOTE__: All the already started [`Scenario`]s at the moment of failure
+    ///           will be finished.
+    ///
+    /// __NOTE__: Retried [`Scenario`]s are considered as failed, only in case
+    ///           they exhaust all retry attempts and still do fail.
+    ///
+    /// [`Scenario`]: gherkin::Scenario
+    #[allow(clippy::missing_const_for_fn)] // false positive: drop in const
+    #[must_use]
+    pub fn fail_fast(mut self) -> Self {
+        self.runner = self.runner.fail_fast();
+        self
+    }
+
     /// Makes failed [`Scenario`]s being retried after the specified
     /// [`Duration`] passes.
     ///

src/runner/basic.rs

@@ -484,11 +484,14 @@ impl<World, Which, Before, After> Basic<World, Which, Before, After> {
         self
     }
 
-    /// Run tests until the first failure.
+    /// Makes stop running tests on the first failure.
     ///
     /// __NOTE__: All the already started [`Scenario`]s at the moment of failure
     ///           will be finished.
     ///
+    /// __NOTE__: Retried [`Scenario`]s are considered as failed, only in case
+    ///           they exhaust all retry attempts and still fail.
+    ///
     /// [`Scenario`]: gherkin::Scenario
     #[must_use]
     pub const fn fail_fast(mut self) -> Self {
@@ -2054,6 +2057,10 @@ impl Features {
     ) {
         use ScenarioType::{Concurrent, Serial};
 
+        if max_concurrent_scenarios == Some(0) {
+            return (Vec::new(), None);
+        }
+
         let mut min_dur = None;
         let mut drain =
             |storage: &mut Vec<(_, _, _, Option<RetryOptionsWithDeadline>)>,

tests/features/from_str_and_parameter/.feature

@@ -0,0 +1,6 @@
+Feature: FromStr
+  Scenario: FromStr
+    Given regex: int: 42
+    And expr: int: 42
+    And regex: quoted: 'inner'
+    And expr: quoted: 'inner'

tests/features/result/.feature

@@ -0,0 +1,19 @@
+Feature: Result
+
+  Scenario: Given ok
+    Given ok
+
+  Scenario: When ok
+    When ok
+
+  Scenario: Then ok
+    Then ok
+
+  Scenario: Given error
+    Given error
+
+  Scenario: When error
+    When error
+
+  Scenario: Then error
+    Then error

tests/features/retry_fail_fast/.feature

@@ -0,0 +1,15 @@
+Feature: retry fail fast
+
+  @serial
+  Scenario Outline: attempts
+    Given attempt <attempt>
+
+    Examples:
+      | attempt |
+      | 1       |
+      | 2       |
+      | 3       |
+      | 4       |
+      | 5       |
+      | 6       |
+      | 7       |

tests/from_str_and_parameter.rs

@@ -0,0 +1,48 @@
+use std::{convert::Infallible, str::FromStr};
+
+use cucumber::{given, Parameter, StatsWriter as _, World};
+
+#[derive(Debug, Parameter, PartialEq)]
+#[param(name = "param", regex = "'([^']*)'|(\\d+)")]
+enum Param {
+    Int(u64),
+    Quoted(String),
+}
+
+impl FromStr for Param {
+    type Err = Infallible;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Ok(s.parse::<u64>()
+            .map_or_else(|_| Self::Quoted(s.to_owned()), Param::Int))
+    }
+}
+
+#[given(regex = "^regex: int: (\\d+)$")]
+#[given(expr = "expr: int: {param}")]
+fn assert_int(_: &mut W, v: Param) {
+    assert_eq!(v, Param::Int(42));
+}
+
+#[given(regex = "^regex: quoted: '([^']*)'$")]
+#[given(expr = "expr: quoted: {param}")]
+fn assert_quoted(_: &mut W, v: Param) {
+    assert_eq!(v, Param::Quoted("inner".to_owned()));
+}
+
+#[derive(Clone, Copy, Debug, Default, World)]
+struct W;
+
+#[tokio::main]
+async fn main() {
+    let writer = W::cucumber()
+        .run("tests/features/from_str_and_parameter")
+        .await;
+
+    assert_eq!(writer.passed_steps(), 4);
+    assert_eq!(writer.skipped_steps(), 0);
+    assert_eq!(writer.failed_steps(), 0);
+    assert_eq!(writer.retried_steps(), 0);
+    assert_eq!(writer.parsing_errors(), 0);
+    assert_eq!(writer.hook_errors(), 0);
+}

tests/result.rs

@@ -0,0 +1,30 @@
+use cucumber::{given, then, when, StatsWriter as _, World};
+
+#[given("ok")]
+#[when("ok")]
+#[then("ok")]
+fn ok(_: &mut W) -> Result<(), &'static str> {
+    Ok(())
+}
+
+#[given("error")]
+#[when("error")]
+#[then("error")]
+fn error(_: &mut W) -> Result<(), &'static str> {
+    Err("error")
+}
+
+#[derive(Clone, Copy, Debug, Default, World)]
+struct W;
+
+#[tokio::main]
+async fn main() {
+    let writer = W::cucumber().run("tests/features/result").await;
+
+    assert_eq!(writer.passed_steps(), 3);
+    assert_eq!(writer.skipped_steps(), 0);
+    assert_eq!(writer.failed_steps(), 3);
+    assert_eq!(writer.retried_steps(), 0);
+    assert_eq!(writer.parsing_errors(), 0);
+    assert_eq!(writer.hook_errors(), 0);
+}

tests/retry_fail_fast.rs

@@ -0,0 +1,34 @@
+use std::sync::atomic::{AtomicUsize, Ordering};
+
+use cucumber::{given, StatsWriter as _, World};
+
+#[derive(Clone, Copy, Debug, Default, World)]
+struct W;
+
+#[given(regex = "attempt (\\d+)")]
+async fn assert(_: &mut W) {
+    static TIMES_CALLED: AtomicUsize = AtomicUsize::new(0);
+
+    match TIMES_CALLED.fetch_add(1, Ordering::SeqCst) {
+        n @ 1..=5 if n % 2 != 0 => panic!("flake"),
+        0..=5 => {}
+        _ => panic!("too much!"),
+    }
+}
+
+#[tokio::main]
+async fn main() {
+    let writer = W::cucumber()
+        .max_concurrent_scenarios(1)
+        .retries(3)
+        .fail_fast()
+        .run("tests/features/retry_fail_fast")
+        .await;
+
+    assert_eq!(writer.passed_steps(), 3);
+    assert_eq!(writer.skipped_steps(), 0);
+    assert_eq!(writer.failed_steps(), 1);
+    assert_eq!(writer.retried_steps(), 5);
+    assert_eq!(writer.parsing_errors(), 0);
+    assert_eq!(writer.hook_errors(), 0);
+}