"panicked while panicking" issues when drop handlers run concurrent code

[bkragl]

Shuttle does not properly handle concurrent code run in drop handlers after a panic. It can run into a panic while panicking, which kills the process. Here's a minimal example. It needs to hit a particular schedule which happens with probability 1/2.

use shuttle::sync::{Arc, Mutex};

struct Foo(Arc<Mutex<()>>);

impl Drop for Foo {
    fn drop(&mut self) {
        let _m = self.0.lock().unwrap();
    }
}

#[test]
fn double_panic() {
    shuttle::check_random(
        || {
            let mutex = Arc::new(Mutex::new(()));
            let _f = Foo(mutex.clone());

            shuttle::thread::spawn(move || {
                // The double panic happens in executions where this thread
                // switches to the main thread inside the `lock` function.
                let _m = mutex.lock().unwrap();
            });

            panic!("panic in test code");
        },
        100,
    );
}

Error output

test panicked in task 'main-thread'
failing schedule:
"
910103a4d0e880c2fbe3a89f0108
"
pass that string to `shuttle::replay` to replay the failure
thread 'double_panic' panicked at 'panic in test code', src/lib.rs:24:13
stack backtrace:
   0: rust_begin_unwind
             at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/panicking.rs:578:5
   1: core::panicking::panic_fmt
             at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/core/src/panicking.rs:67:14
   2: proptest_shuttle_test::double_panic::{{closure}}
             at ./src/lib.rs:24:13
   3: shuttle::runtime::runner::Runner<S>::run::{{closure}}::{{closure}}::{{closure}}
             at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/shuttle-0.6.1/src/runtime/runner.rs:69:100
   4: shuttle::thread::thread_fn
             at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/shuttle-0.6.1/src/thread.rs:102:15
   5: shuttle::runtime::execution::Execution::run::{{closure}}::{{closure}}
             at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/shuttle-0.6.1/src/runtime/execution.rs:67:25
   6: core::ops::function::FnOnce::call_once{{vtable.shim}}
             at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/core/src/ops/function.rs:250:5
   7: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/alloc/src/boxed.rs:1973:9
   8: shuttle::runtime::thread::continuation::Continuation::new::{{closure}}
             at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/shuttle-0.6.1/src/runtime/thread/continuation.rs:87:21
   9: generator::gen_impl::GeneratorImpl<A,T>::init_code::{{closure}}
             at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/generator-0.7.4/src/gen_impl.rs:336:21
  10: generator::stack::StackBox<F>::call_once
             at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/generator-0.7.4/src/stack/mod.rs:139:13
  11: generator::stack::Func::call_once
             at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/generator-0.7.4/src/stack/mod.rs:121:9
  12: generator::gen_impl::gen_init::{{closure}}
             at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/generator-0.7.4/src/gen_impl.rs:552:9
  13: core::ops::function::FnOnce::call_once
             at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/core/src/ops/function.rs:250:5
  14: std::panicking::try::do_call
             at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/panicking.rs:485:40
  15: ___rust_try
  16: std::panicking::try
             at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/panicking.rs:449:19
  17: std::panic::catch_unwind
             at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/panic.rs:140:14
  18: generator::gen_impl::catch_unwind_filter
             at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/generator-0.7.4/src/gen_impl.rs:543:5
  19: generator::gen_impl::gen_init
             at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/generator-0.7.4/src/gen_impl.rs:570:25
note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace.
thread 'double_panic' panicked at 'test panicked in task 'main-thread'
failing schedule:
"
910103a4d0e880c2fbe3a89f0108
"
pass that string to `shuttle::replay` to replay the failure', /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/shuttle-0.6.1/src/runtime/execution.rs:144:17
stack backtrace:
   0:        0x10f665546 - std::backtrace_rs::backtrace::libunwind::trace::h0a908cd09b5a35f9
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/../../backtrace/src/backtrace/libunwind.rs:93:5
   1:        0x10f665546 - std::backtrace_rs::backtrace::trace_unsynchronized::hc0e7d5d16c14a788
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5
   2:        0x10f665546 - std::sys_common::backtrace::_print_fmt::hcf9ca6805c7eb2fe
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/sys_common/backtrace.rs:65:5
   3:        0x10f665546 - <std::sys_common::backtrace::_print::DisplayBacktrace as core::fmt::Display>::fmt::hae51cb91d407e2ef
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/sys_common/backtrace.rs:44:22
   4:        0x10f68103b - core::fmt::write::h746bc0969202388b
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/core/src/fmt/mod.rs:1254:17
   5:        0x10f66250c - std::io::Write::write_fmt::h4098c2c7437a0bd7
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/io/mod.rs:1698:15
   6:        0x10f66531a - std::sys_common::backtrace::_print::he6d3aef1f6c73e2d
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/sys_common/backtrace.rs:47:5
   7:        0x10f66531a - std::sys_common::backtrace::print::h8360bf0158e89b36
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/sys_common/backtrace.rs:34:9
   8:        0x10f666f70 - std::panicking::default_hook::{{closure}}::hedf04c568eb6e0bc
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/panicking.rs:269:22
   9:        0x10f666d20 - std::panicking::default_hook::h62889b2c29e2347d
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/panicking.rs:288:9
  10:        0x10f6255c4 - <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call::he6b83307c909e624
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/alloc/src/boxed.rs:1987:9
  11:        0x10f5fe425 - shuttle::runtime::failure::init_panic_hook::{{closure}}::{{closure}}::h42de85c8fa97934f
                               at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/shuttle-0.6.1/src/runtime/failure.rs:151:13
  12:        0x10f6255c4 - <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call::he6b83307c909e624
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/alloc/src/boxed.rs:1987:9
  13:        0x10f6269e5 - generator::gen_impl::catch_unwind_filter::{{closure}}::{{closure}}::h24c0991694e38682
                               at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/generator-0.7.4/src/gen_impl.rs:539:13
  14:        0x10f66764a - <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call::h5ccd129639ee877c
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/alloc/src/boxed.rs:1987:9
  15:        0x10f66764a - std::panicking::rust_panic_with_hook::h22edd02828aff274
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/panicking.rs:695:13
  16:        0x10f6673e4 - std::panicking::begin_panic_handler::{{closure}}::h6895f44a9b7e2caa
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/panicking.rs:582:13
  17:        0x10f665989 - std::sys_common::backtrace::__rust_end_short_backtrace::h013519e9b77978c5
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/sys_common/backtrace.rs:150:18
  18:        0x10f66712d - rust_begin_unwind
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/panicking.rs:578:5
  19:        0x10f68d053 - core::panicking::panic_fmt::h2a51b60c12d11322
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/core/src/panicking.rs:67:14
  20:        0x10f5a7221 - core::panicking::panic_display::hbd949b297acfd5f4
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/core/src/panicking.rs:150:5
  21:        0x10f5a6ae1 - shuttle::runtime::execution::Execution::step::h00314f3fd4ea0f15
                               at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/shuttle-0.6.1/src/runtime/execution.rs:144:17
  22:        0x10f59d89f - shuttle::runtime::execution::Execution::run::{{closure}}::h435d08498b5500db
                               at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/shuttle-0.6.1/src/runtime/execution.rs:74:19
  23:        0x10f5a92b9 - scoped_tls::ScopedKey<T>::set::hf9c179dc9c5d2b9e
                               at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/scoped-tls-1.0.1/src/lib.rs:137:9
  24:        0x10f59d6d1 - shuttle::runtime::execution::Execution::run::hb1f06acf42b88a14
                               at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/shuttle-0.6.1/src/runtime/execution.rs:64:9
  25:        0x10f593d2a - shuttle::runtime::runner::Runner<S>::run::{{closure}}::{{closure}}::h62c8cb787820efb7
                               at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/shuttle-0.6.1/src/runtime/runner.rs:69:64
  26:        0x10f592555 - tracing::span::Span::in_scope::h9482e775058da9fb
                               at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.37/src/span.rs:1102:9
  27:        0x10f593bbe - shuttle::runtime::runner::Runner<S>::run::{{closure}}::h5b30f2386550abea
                               at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/shuttle-0.6.1/src/runtime/runner.rs:69:17
  28:        0x10f5a9197 - scoped_tls::ScopedKey<T>::set::h46c950750c8aadb1
                               at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/scoped-tls-1.0.1/src/lib.rs:137:9
  29:        0x10f5932c3 - shuttle::runtime::runner::Runner<S>::run::h6ed27f200638e525
                               at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/shuttle-0.6.1/src/runtime/runner.rs:49:9
  30:        0x10f5a1159 - shuttle::check_random::hf2b9458e9d07ee53
                               at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/shuttle-0.6.1/src/lib.rs:313:5
  31:        0x10f59223e - proptest_shuttle_test::double_panic::h236e26cd0e9ef228
                               at /Users/kraglb/Code/proptest-shuttle-test/src/lib.rs:13:5
  32:        0x10f5a6fb9 - proptest_shuttle_test::double_panic::{{closure}}::h803d69b63cde3b8b
                               at /Users/kraglb/Code/proptest-shuttle-test/src/lib.rs:12:19
  33:        0x10f594658 - core::ops::function::FnOnce::call_once::hc924793af4a1d820
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/core/src/ops/function.rs:250:5
  34:        0x10f5e34c2 - core::ops::function::FnOnce::call_once::h0fe66ffa3f9dbea8
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/core/src/ops/function.rs:250:5
  35:        0x10f5e34c2 - test::__rust_begin_short_backtrace::h460edc73ee3d7292
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/test/src/lib.rs:655:18
  36:        0x10f5b0281 - test::run_test::{{closure}}::hcb97f2f14ca1b43f
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/test/src/lib.rs:646:30
  37:        0x10f5b0281 - core::ops::function::FnOnce::call_once{{vtable.shim}}::hdfe6b8e984779cce
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/core/src/ops/function.rs:250:5
  38:        0x10f5e22c1 - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::hfa918f0892d6e1a1
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/alloc/src/boxed.rs:1973:9
  39:        0x10f5e22c1 - <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once::h71c8bbcc51698713
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/core/src/panic/unwind_safe.rs:271:9
  40:        0x10f5e22c1 - std::panicking::try::do_call::h890697a79ad71324
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/panicking.rs:485:40
  41:        0x10f5e22c1 - std::panicking::try::ha3f7932050f4d091
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/panicking.rs:449:19
  42:        0x10f5e22c1 - std::panic::catch_unwind::hc2bb78c2421caad4
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/panic.rs:140:14
  43:        0x10f5e22c1 - test::run_test_in_process::hfaa808ccf4afcea3
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/test/src/lib.rs:678:27
  44:        0x10f5e22c1 - test::run_test::run_test_inner::{{closure}}::h6df30ee50946e418
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/test/src/lib.rs:572:39
  45:        0x10f5aaba7 - test::run_test::run_test_inner::{{closure}}::h28731cdeaae649ee
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/test/src/lib.rs:599:37
  46:        0x10f5aaba7 - std::sys_common::backtrace::__rust_begin_short_backtrace::he77a8a07f5d086a1
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/sys_common/backtrace.rs:134:18
  47:        0x10f5b0151 - std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}::h7325a38e91958334
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/thread/mod.rs:526:17
  48:        0x10f5b0151 - <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once::h4f76313c8805442a
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/core/src/panic/unwind_safe.rs:271:9
  49:        0x10f5b0151 - std::panicking::try::do_call::h8ffd919c7cd22e24
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/panicking.rs:485:40
  50:        0x10f5b0151 - std::panicking::try::he40bc179d7d75f43
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/panicking.rs:449:19
  51:        0x10f5b0151 - std::panic::catch_unwind::hb878b7a33e4fb646
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/panic.rs:140:14
  52:        0x10f5b0151 - std::thread::Builder::spawn_unchecked_::{{closure}}::hf2774d6144cce2f7
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/thread/mod.rs:525:30
  53:        0x10f5b0151 - core::ops::function::FnOnce::call_once{{vtable.shim}}::hc09679af1629946e
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/core/src/ops/function.rs:250:5
  54:        0x10f66b8a9 - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::h3813d4e7f4f8581d
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/alloc/src/boxed.rs:1973:9
  55:        0x10f66b8a9 - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::ha3873a2c5aad6d58
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/alloc/src/boxed.rs:1973:9
  56:        0x10f66b8a9 - std::sys::unix::thread::Thread::new::thread_start::ha9b8d2206382b425
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/sys/unix/thread.rs:108:17
  57:     0x7ff8045c94e1 - __pthread_start
thread panicked while panicking. aborting.
error: test failed, to rerun pass `-p proptest-shuttle-test --lib`

Caused by:
  process didn't exit successfully: `/Users/kraglb/Code/proptest-shuttle-test/target/debug/deps/proptest_shuttle_test-cf7e6e3a9da694c9 double_panic --exact --nocapture` (signal: 6, SIGABRT: process abort signal)

Replaying the above schedule (910103a4d0e880c2fbe3a89f0108) also leads to a panic while panicking, because the schedule stops at the first panic, and so we hit 'schedule ended early' when we want to make further scheduling decisions after the first panic.

Replay error output

test panicked in task 'main-thread'
failing schedule:
"
910103a4d0e880c2fbe3a89f0108
"
pass that string to `shuttle::replay` to replay the failure
thread 'replay_double_panic' panicked at 'panic in test code', src/lib.rs:43:13
stack backtrace:
   0: rust_begin_unwind
             at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/panicking.rs:578:5
   1: core::panicking::panic_fmt
             at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/core/src/panicking.rs:67:14
   2: proptest_shuttle_test::replay_double_panic::{{closure}}
             at ./src/lib.rs:43:13
   3: shuttle::runtime::runner::Runner<S>::run::{{closure}}::{{closure}}::{{closure}}
             at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/shuttle-0.6.1/src/runtime/runner.rs:69:100
   4: shuttle::thread::thread_fn
             at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/shuttle-0.6.1/src/thread.rs:102:15
   5: shuttle::runtime::execution::Execution::run::{{closure}}::{{closure}}
             at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/shuttle-0.6.1/src/runtime/execution.rs:67:25
   6: core::ops::function::FnOnce::call_once{{vtable.shim}}
             at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/core/src/ops/function.rs:250:5
   7: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/alloc/src/boxed.rs:1973:9
   8: shuttle::runtime::thread::continuation::Continuation::new::{{closure}}
             at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/shuttle-0.6.1/src/runtime/thread/continuation.rs:87:21
   9: generator::gen_impl::GeneratorImpl<A,T>::init_code::{{closure}}
             at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/generator-0.7.4/src/gen_impl.rs:336:21
  10: generator::stack::StackBox<F>::call_once
             at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/generator-0.7.4/src/stack/mod.rs:139:13
  11: generator::stack::Func::call_once
             at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/generator-0.7.4/src/stack/mod.rs:121:9
  12: generator::gen_impl::gen_init::{{closure}}
             at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/generator-0.7.4/src/gen_impl.rs:552:9
  13: core::ops::function::FnOnce::call_once
             at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/core/src/ops/function.rs:250:5
  14: std::panicking::try::do_call
             at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/panicking.rs:485:40
  15: ___rust_try
  16: std::panicking::try
             at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/panicking.rs:449:19
  17: std::panic::catch_unwind
             at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/panic.rs:140:14
  18: generator::gen_impl::catch_unwind_filter
             at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/generator-0.7.4/src/gen_impl.rs:543:5
  19: generator::gen_impl::gen_init
             at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/generator-0.7.4/src/gen_impl.rs:570:25
note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace.
thread 'replay_double_panic' panicked at 'schedule ended early', /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/shuttle-0.6.1/src/scheduler/replay.rs:69:13
stack backtrace:
   0:        0x10f3e3036 - std::backtrace_rs::backtrace::libunwind::trace::h0a908cd09b5a35f9
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/../../backtrace/src/backtrace/libunwind.rs:93:5
   1:        0x10f3e3036 - std::backtrace_rs::backtrace::trace_unsynchronized::hc0e7d5d16c14a788
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5
   2:        0x10f3e3036 - std::sys_common::backtrace::_print_fmt::hcf9ca6805c7eb2fe
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/sys_common/backtrace.rs:65:5
   3:        0x10f3e3036 - <std::sys_common::backtrace::_print::DisplayBacktrace as core::fmt::Display>::fmt::hae51cb91d407e2ef
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/sys_common/backtrace.rs:44:22
   4:        0x10f3feb2b - core::fmt::write::h746bc0969202388b
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/core/src/fmt/mod.rs:1254:17
   5:        0x10f3dfffc - std::io::Write::write_fmt::h4098c2c7437a0bd7
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/io/mod.rs:1698:15
   6:        0x10f3e2e0a - std::sys_common::backtrace::_print::he6d3aef1f6c73e2d
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/sys_common/backtrace.rs:47:5
   7:        0x10f3e2e0a - std::sys_common::backtrace::print::h8360bf0158e89b36
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/sys_common/backtrace.rs:34:9
   8:        0x10f3e4a60 - std::panicking::default_hook::{{closure}}::hedf04c568eb6e0bc
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/panicking.rs:269:22
   9:        0x10f3e4810 - std::panicking::default_hook::h62889b2c29e2347d
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/panicking.rs:288:9
  10:        0x10f3a2cf4 - <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call::he6b83307c909e624
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/alloc/src/boxed.rs:1987:9
  11:        0x10f377ed5 - shuttle::runtime::failure::init_panic_hook::{{closure}}::{{closure}}::h42de85c8fa97934f
                               at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/shuttle-0.6.1/src/runtime/failure.rs:151:13
  12:        0x10f3a2cf4 - <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call::he6b83307c909e624
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/alloc/src/boxed.rs:1987:9
  13:        0x10f3a4115 - generator::gen_impl::catch_unwind_filter::{{closure}}::{{closure}}::h24c0991694e38682
                               at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/generator-0.7.4/src/gen_impl.rs:539:13
  14:        0x10f3e513a - <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call::h5ccd129639ee877c
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/alloc/src/boxed.rs:1987:9
  15:        0x10f3e513a - std::panicking::rust_panic_with_hook::h22edd02828aff274
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/panicking.rs:695:13
  16:        0x10f3e4e93 - std::panicking::begin_panic_handler::{{closure}}::h6895f44a9b7e2caa
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/panicking.rs:580:13
  17:        0x10f3e3479 - std::sys_common::backtrace::__rust_end_short_backtrace::h013519e9b77978c5
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/sys_common/backtrace.rs:150:18
  18:        0x10f3e4c1d - rust_begin_unwind
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/panicking.rs:578:5
  19:        0x10f40ab43 - core::panicking::panic_fmt::h2a51b60c12d11322
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/core/src/panicking.rs:67:14
  20:        0x10f376656 - <shuttle::scheduler::replay::ReplayScheduler as shuttle::scheduler::Scheduler>::next_task::h8ea057550dca6f79
                               at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/shuttle-0.6.1/src/scheduler/replay.rs:69:13
  21:        0x10f304bd6 - <shuttle::scheduler::metrics::MetricsScheduler<S> as shuttle::scheduler::Scheduler>::next_task::hf2b74c3aa7bbc9ff
                               at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/shuttle-0.6.1/src/scheduler/metrics.rs:92:22
  22:        0x10f371630 - shuttle::runtime::execution::ExecutionState::schedule::h89fc2a09bfd49009
                               at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/shuttle-0.6.1/src/runtime/execution.rs:567:26
  23:        0x10f3708cd - shuttle::runtime::execution::ExecutionState::maybe_yield::{{closure}}::h98f52af81cfe20db
                               at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/shuttle-0.6.1/src/runtime/execution.rs:383:26
  24:        0x10f36fd8a - shuttle::runtime::execution::ExecutionState::try_with::{{closure}}::hb9dcd22ea989e68b
                               at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/shuttle-0.6.1/src/runtime/execution.rs:258:26
  25:        0x10f37e402 - scoped_tls::ScopedKey<T>::with::hdc6b8fddccec0df1
                               at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/scoped-tls-1.0.1/src/lib.rs:171:13
  26:        0x10f36f366 - shuttle::runtime::execution::ExecutionState::try_with::h4bb2a7396c03ac15
                               at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/shuttle-0.6.1/src/runtime/execution.rs:256:13
  27:        0x10f36f06d - shuttle::runtime::execution::ExecutionState::with::ha3108570b68fa7dc
                               at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/shuttle-0.6.1/src/runtime/execution.rs:245:9
  28:        0x10f370899 - shuttle::runtime::execution::ExecutionState::maybe_yield::hdca648f3f72a96f5
                               at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/shuttle-0.6.1/src/runtime/execution.rs:377:9
  29:        0x10f363f9d - shuttle::runtime::thread::continuation::switch::h1ec241841f45c810
                               at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/shuttle-0.6.1/src/runtime/thread/continuation.rs:259:8
  30:        0x10f31d5ba - shuttle::sync::mutex::Mutex<T>::lock::hc5a83ab06bf8d63a
                               at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/shuttle-0.6.1/src/sync/mutex.rs:71:13
  31:        0x10f303a9d - <proptest_shuttle_test::Foo as core::ops::drop::Drop>::drop::hb411a29ef0c6f032
                               at /Users/kraglb/Code/proptest-shuttle-test/src/lib.rs:7:18
  32:        0x10f300f75 - core::ptr::drop_in_place<proptest_shuttle_test::Foo>::h04cb8183f3606aa5
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/core/src/ptr/mod.rs:490:1
  33:        0x10f31c3ba - proptest_shuttle_test::replay_double_panic::{{closure}}::h91ac582f84cd9cfb
                               at /Users/kraglb/Code/proptest-shuttle-test/src/lib.rs:44:9
  34:        0x10f2ffa7d - shuttle::runtime::runner::Runner<S>::run::{{closure}}::{{closure}}::{{closure}}::h9e7bb64bc90b3ad9
                               at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/shuttle-0.6.1/src/runtime/runner.rs:69:100
  35:        0x10f3129d3 - shuttle::thread::thread_fn::hcbc2b34780f1e487
                               at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/shuttle-0.6.1/src/thread.rs:102:15
  36:        0x10f30df1a - shuttle::runtime::execution::Execution::run::{{closure}}::{{closure}}::h857f7da1b457d850
                               at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/shuttle-0.6.1/src/runtime/execution.rs:67:25
  37:        0x10f3002d4 - core::ops::function::FnOnce::call_once{{vtable.shim}}::h1667835ca21b1394
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/core/src/ops/function.rs:250:5
  38:        0x10f36ea1e - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::hc62867ebb53a9c78
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/alloc/src/boxed.rs:1973:9
  39:        0x10f363431 - shuttle::runtime::thread::continuation::Continuation::new::{{closure}}::hd949b502d8cd8af8
                               at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/shuttle-0.6.1/src/runtime/thread/continuation.rs:87:21
  40:        0x10f397039 - generator::gen_impl::GeneratorImpl<A,T>::init_code::{{closure}}::h71323f9c2b036413
                               at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/generator-0.7.4/src/gen_impl.rs:336:21
  41:        0x10f38b33f - generator::stack::StackBox<F>::call_once::h856ea0bb4902f7b8
                               at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/generator-0.7.4/src/stack/mod.rs:139:13
  42:        0x10f3a2daf - generator::stack::Func::call_once::h7b0f46ad5849766c
                               at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/generator-0.7.4/src/stack/mod.rs:121:9
  43:        0x10f3a42c3 - generator::gen_impl::gen_init::{{closure}}::hb5197d56bc3a43dd
                               at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/generator-0.7.4/src/gen_impl.rs:552:9
  44:        0x10f3a1115 - core::ops::function::FnOnce::call_once::h44414163bef47564
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/core/src/ops/function.rs:250:5
  45:        0x10f3a29a8 - std::panicking::try::do_call::hf38821f2c0f1e724
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/panicking.rs:485:40
  46:        0x10f3a2d1d - ___rust_try
  47:        0x10f3a2917 - std::panicking::try::h6b11b2e34ab64c77
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/panicking.rs:449:19
  48:        0x10f3a20b1 - std::panic::catch_unwind::ha56f94586618f63b
                               at /rustc/90c541806f23a127002de5b4038be731ba1458ca/library/std/src/panic.rs:140:14
  49:        0x10f3a3fe0 - generator::gen_impl::catch_unwind_filter::h345f378589a64220
                               at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/generator-0.7.4/src/gen_impl.rs:543:5
  50:        0x10f3a414e - generator::gen_impl::gen_init::h25dfd0df328bbc96
                               at /Users/kraglb/.cargo/registry/src/index.crates.io-6f17d22bba15001f/generator-0.7.4/src/gen_impl.rs:570:25
thread panicked while panicking. aborting.
error: test failed, to rerun pass `-p proptest-shuttle-test --lib`

Caused by:
  process didn't exit successfully: `/Users/kraglb/Code/proptest-shuttle-test/target/debug/deps/proptest_shuttle_test-cf7e6e3a9da694c9 replay_double_panic --exact --nocapture` (signal: 6, SIGABRT: process abort signal)

[jamesbornholt]

I wonder if the right thing to do here is to leak all the outstanding continuations when a test panics, to avoid unwinding the other tasks. It's a little wasteful in general, but I suspect the case where the process lives much longer after the panic is very rare in our setting.