Skip test_multithreaded_compute on MoltenVK. (#4096)

Co-authored-by: Connor Fitzgerald <[email protected]>

.github/workflows/ci.yml

@@ -284,6 +284,7 @@ jobs:
           done
 
       - uses: actions/upload-artifact@v3
+        if: always() # We want artifacts even if the tests fail.
         with:
           name: comparison-images
           path: |

CHANGELOG.md

@@ -114,6 +114,10 @@ By @Valaphee in [#3402](https://github.com/gfx-rs/wgpu/pull/3402)
 
 - Ensure that limit requests and reporting is done correctly. By @OptimisticPeach in [#4107](https://github.com/gfx-rs/wgpu/pull/4107)
 
+#### Testing
+
+- Skip `test_multithreaded_compute` on MoltenVK. By @jimblandy in [#4096](https://github.com/gfx-rs/wgpu/pull/4096).
+
 ### Documentation
 
 - Add an overview of `RenderPass` and how render state works. By @kpreid in [#4055](https://github.com/gfx-rs/wgpu/pull/4055)

examples/boids/src/main.rs

@@ -345,7 +345,7 @@ fn boids() {
             .downlevel_flags(wgpu::DownlevelFlags::COMPUTE_SHADERS)
             .limits(wgpu::Limits::downlevel_defaults())
             // Lots of validation errors, maybe related to https://github.com/gfx-rs/wgpu/issues/3160
-            .molten_vk_failure(),
+            .expect_fail(wgpu_test::FailureCase::molten_vk()),
         comparisons: &[wgpu_test::ComparisonType::Mean(0.005)],
     });
 }

examples/common/src/framework.rs

@@ -625,7 +625,7 @@ pub fn test<E: Example>(mut params: FrameworkRefTest) {
 
             wgpu_test::image::compare_image_output(
                 env!("CARGO_MANIFEST_DIR").to_string() + "/../../" + params.image_path,
-                ctx.adapter_info.backend,
+                &ctx.adapter_info,
                 params.width,
                 params.height,
                 &bytes,

examples/hello-compute/src/tests.rs

@@ -1,7 +1,7 @@
 use std::sync::Arc;
 
 use super::*;
-use wgpu_test::{initialize_test, TestParameters};
+use wgpu_test::{initialize_test, FailureCase, TestParameters};
 
 wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser);
 
@@ -13,7 +13,7 @@ fn test_compute_1() {
             .downlevel_flags(wgpu::DownlevelFlags::COMPUTE_SHADERS)
             .limits(wgpu::Limits::downlevel_defaults())
             .features(wgpu::Features::TIMESTAMP_QUERY)
-            .specific_failure(None, None, Some("V3D"), true),
+            .skip(FailureCase::adapter("V3D")),
         |ctx| {
             let input = &[1, 2, 3, 4];
 
@@ -35,7 +35,7 @@ fn test_compute_2() {
             .downlevel_flags(wgpu::DownlevelFlags::COMPUTE_SHADERS)
             .limits(wgpu::Limits::downlevel_defaults())
             .features(wgpu::Features::TIMESTAMP_QUERY)
-            .specific_failure(None, None, Some("V3D"), true),
+            .skip(FailureCase::adapter("V3D")),
         |ctx| {
             let input = &[5, 23, 10, 9];
 
@@ -57,7 +57,7 @@ fn test_compute_overflow() {
             .downlevel_flags(wgpu::DownlevelFlags::COMPUTE_SHADERS)
             .limits(wgpu::Limits::downlevel_defaults())
             .features(wgpu::Features::TIMESTAMP_QUERY)
-            .specific_failure(None, None, Some("V3D"), true),
+            .skip(FailureCase::adapter("V3D")),
         |ctx| {
             let input = &[77031, 837799, 8400511, 63728127];
             pollster::block_on(assert_execute_gpu(
@@ -78,16 +78,15 @@ fn test_multithreaded_compute() {
             .downlevel_flags(wgpu::DownlevelFlags::COMPUTE_SHADERS)
             .limits(wgpu::Limits::downlevel_defaults())
             .features(wgpu::Features::TIMESTAMP_QUERY)
-            .specific_failure(None, None, Some("V3D"), true)
+            .skip(FailureCase::adapter("V3D"))
             // https://github.com/gfx-rs/wgpu/issues/3944
-            .specific_failure(
-                Some(wgpu::Backends::VULKAN),
-                None,
-                Some("swiftshader"),
-                true,
-            )
+            .skip(FailureCase::backend_adapter(
+                wgpu::Backends::VULKAN,
+                "swiftshader",
+            ))
             // https://github.com/gfx-rs/wgpu/issues/3250
-            .specific_failure(Some(wgpu::Backends::GL), None, Some("llvmpipe"), true),
+            .skip(FailureCase::backend_adapter(wgpu::Backends::GL, "llvmpipe"))
+            .skip(FailureCase::molten_vk()),
         |ctx| {
             use std::{sync::mpsc, thread, time::Duration};
 

examples/mipmap/src/main.rs

@@ -521,7 +521,7 @@ fn mipmap() {
         height: 768,
         optional_features: wgpu::Features::default(),
         base_test_parameters: wgpu_test::TestParameters::default()
-            .backend_failure(wgpu::Backends::GL),
+            .expect_fail(wgpu_test::FailureCase::backend(wgpu::Backends::GL)),
         comparisons: &[wgpu_test::ComparisonType::Mean(0.02)],
     });
 }
@@ -535,7 +535,7 @@ fn mipmap_query() {
         height: 768,
         optional_features: QUERY_FEATURES,
         base_test_parameters: wgpu_test::TestParameters::default()
-            .backend_failure(wgpu::Backends::GL),
+            .expect_fail(wgpu_test::FailureCase::backend(wgpu::Backends::GL)),
         comparisons: &[wgpu_test::ComparisonType::Mean(0.02)],
     });
 }

examples/msaa-line/src/main.rs

@@ -12,6 +12,9 @@ use std::{borrow::Cow, iter};
 use bytemuck::{Pod, Zeroable};
 use wgpu::util::DeviceExt;
 
+#[cfg(test)]
+use wgpu_test::FailureCase;
+
 #[repr(C)]
 #[derive(Clone, Copy, Pod, Zeroable)]
 struct Vertex {
@@ -326,7 +329,11 @@ fn msaa_line() {
         optional_features: wgpu::Features::TEXTURE_ADAPTER_SPECIFIC_FORMAT_FEATURES,
         base_test_parameters: wgpu_test::TestParameters::default()
             // AMD seems to render nothing on DX12 https://github.com/gfx-rs/wgpu/issues/3838
-            .specific_failure(Some(wgpu::Backends::DX12), Some(0x1002), None, false),
+            .expect_fail(FailureCase {
+                backends: Some(wgpu::Backends::DX12),
+                vendor: Some(0x1002),
+                ..FailureCase::default()
+            }),
         // There's a lot of natural variance so we check the weighted median too to differentiate
         // real failures from variance.
         comparisons: &[

examples/shadow/src/main.rs

@@ -857,9 +857,15 @@ fn shadow() {
         base_test_parameters: wgpu_test::TestParameters::default()
             .downlevel_flags(wgpu::DownlevelFlags::COMPARISON_SAMPLERS)
             // rpi4 on VK doesn't work: https://gitlab.freedesktop.org/mesa/mesa/-/issues/3916
-            .specific_failure(Some(wgpu::Backends::VULKAN), None, Some("V3D"), false)
+            .expect_fail(wgpu_test::FailureCase::backend_adapter(
+                wgpu::Backends::VULKAN,
+                "V3D",
+            ))
             // llvmpipe versions in CI are flaky: https://github.com/gfx-rs/wgpu/issues/2594
-            .specific_failure(Some(wgpu::Backends::VULKAN), None, Some("llvmpipe"), true),
+            .skip(wgpu_test::FailureCase::backend_adapter(
+                wgpu::Backends::VULKAN,
+                "llvmpipe",
+            )),
         comparisons: &[wgpu_test::ComparisonType::Mean(0.02)],
     });
 }

examples/skybox/src/main.rs

@@ -475,11 +475,8 @@ fn skybox() {
         width: 1024,
         height: 768,
         optional_features: wgpu::Features::default(),
-        base_test_parameters: wgpu_test::TestParameters::default().specific_failure(
-            Some(wgpu::Backends::GL),
-            None,
-            Some("ANGLE"),
-            false,
+        base_test_parameters: wgpu_test::TestParameters::default().expect_fail(
+            wgpu_test::FailureCase::backend_adapter(wgpu::Backends::GL, "ANGLE"),
         ),
         comparisons: &[wgpu_test::ComparisonType::Mean(0.015)],
     });

tests/src/image.rs

@@ -150,7 +150,7 @@ impl ComparisonType {
 
 pub fn compare_image_output(
     path: impl AsRef<Path> + AsRef<OsStr>,
-    backend: Backend,
+    adapter_info: &wgt::AdapterInfo,
     width: u32,
     height: u32,
     test_with_alpha: &[u8],
@@ -205,17 +205,18 @@ pub fn compare_image_output(
         }
 
         let file_stem = reference_path.file_stem().unwrap().to_string_lossy();
+        let renderer = format!(
+            "{}-{}-{}",
+            adapter_info.backend.to_str(),
+            sanitize_for_path(&adapter_info.name),
+            sanitize_for_path(&adapter_info.driver)
+        );
         // Determine the paths to write out the various intermediate files
         let actual_path = Path::new(&path).with_file_name(
-            OsString::from_str(&format!("{}-{}-actual.png", file_stem, backend.to_str(),)).unwrap(),
+            OsString::from_str(&format!("{}-{}-actual.png", file_stem, renderer)).unwrap(),
         );
         let difference_path = Path::new(&path).with_file_name(
-            OsString::from_str(&format!(
-                "{}-{}-difference.png",
-                file_stem,
-                backend.to_str(),
-            ))
-            .unwrap(),
+            OsString::from_str(&format!("{}-{}-difference.png", file_stem, renderer,)).unwrap(),
         );
 
         // Convert the error values to a false color reprensentation
@@ -246,10 +247,16 @@ pub fn compare_image_output(
 
     #[cfg(target_arch = "wasm32")]
     {
-        let _ = (path, backend, width, height, test_with_alpha, checks);
+        let _ = (path, adapter_info, width, height, test_with_alpha, checks);
     }
 }
 
+fn sanitize_for_path(s: &str) -> String {
+    s.chars()
+        .map(|ch| if ch.is_ascii_alphanumeric() { ch } else { '_' })
+        .collect()
+}
+
 fn copy_via_compute(
     device: &Device,
     encoder: &mut CommandEncoder,