[SYCL][E2E] Add functionality to split build and run of e2e tests (#1…

…6016) Adds functionality to split the compilation and execution of e2e tests across separate machines. This functionality is enabled via the `test-mode` lit parameter. By default this is set to `full` and tests are built and ran on the same system, just like before. setting `test-mode` to either `run-only` or `build-only` enables the test splitting. Two new lit features have been added: `run-mode` which is enabled when either in `run-only` or `full` testing mode, and `build-and-run-mode` which is only enabled when in `full` testing mode. When in `build-only` mode all tests that can be built on the system will be built. When running a lit test in this mode two key things change: - All `RUN:` lines will be executed unless they contain `%{run}`, `%{run-unfiltered-devices}` or `%if run-mode`. - Unsupported and requires statements are ignored. Currently the only way to mark tests as unsupported in build only mode is to include `build-and-run-mode` in a requires statement, or use `UNSUPPORTED: true`. When in `run-only` mode tests are not built, they are only executed. Deciding whether a test is supported in this mode works the same as in `full` mode. To only execute the tests we ignore all lit `RUN:` lines unless they contain `%{run}`, `%{run-unfiltered-devices}` or `%if run-mode`. Since we do not build the tests in this mode, for any test to pass we must have ran the tests in either `build-only` or `full` modes. ### Some notes/current limitations: - Currently only the spir64 triple is supported for the build only mode. - If a test is able to build, but fails during running we need to mark it as `XFAILS: run-mode` so that it does not `XPASS` when in build-only mode. - The build-only mode can be ran manually on CI with the `SYCL E2E` action, selecting the "Linux, build" runner, and adding `--param test-mode=build-only` to `LIT_OPTS`. This pr does not have the needed CI changes to be able to use the run only mode.
intel · Nov 22, 2024 · f279a8a · f279a8a
1 parent 1e1ffc0
commit f279a8a
Show file tree

Hide file tree

Showing 74 changed files with 138 additions and 62 deletions.
diff --git a/sycl/test-e2e/AmdNvidiaJIT/kernel_and_bundle.cpp b/sycl/test-e2e/AmdNvidiaJIT/kernel_and_bundle.cpp
@@ -1,5 +1,6 @@
 // UNSUPPORTED: windows
 // REQUIRES: cuda || hip
+// REQUIRES: build-and-run-mode
 
 // This test relies on debug output from a pass, make sure that the compiler
 // can generate it.

diff --git a/sycl/test-e2e/Basic/windows_version_agnostic_sycl_lib.cpp b/sycl/test-e2e/Basic/windows_version_agnostic_sycl_lib.cpp
@@ -1,4 +1,5 @@
 // REQUIRES: windows
+// REQUIRES: build-and-run-mode
 
 // RUN: %clangxx --driver-mode=cl /std:c++17 /EHsc /imsvc %sycl_include -I%opencl_include_dir %s -o %t.out /link /defaultlib:%sycl_static_libs_dir/sycl.lib
 // RUN: %{run} %t.out

diff --git a/sycl/test-e2e/Compression/no_zstd_warning.cpp b/sycl/test-e2e/Compression/no_zstd_warning.cpp
@@ -1,4 +1,5 @@
 // using --offload-compress without zstd should throw an error.
 // REQUIRES: !zstd
+// REQUIRES: build-and-run-mode
 // RUN: not %{build} %O0 -g --offload-compress %S/Inputs/single_kernel.cpp -o %t_compress.out 2>&1 | FileCheck %s
 // CHECK: '--offload-compress' option is specified but zstd is not available. The device image will not be compressed.
diff --git a/sycl/test-e2e/Config/config.cpp b/sycl/test-e2e/Config/config.cpp
@@ -8,13 +8,13 @@
 // RUN: %{build} %debug_option %O0 -o %t.out
 // RUN: echo SYCL_PRINT_EXECUTION_GRAPH=always > %t.cfg
 // RUN: %{run-unfiltered-devices} env SYCL_CONFIG_FILE_NAME=%t.cfg %t.out
-// RUN: cat *.dot > /dev/null
-// RUN: rm *.dot
+// RUN: %if run-mode %{cat *.dot > /dev/null%}
+// RUN: %if run-mode %{rm *.dot%}
 // RUN: %{run-unfiltered-devices} env SYCL_PRINT_EXECUTION_GRAPH=always %t.out
-// RUN: cat *.dot > /dev/null
-// RUN: rm *.dot
+// RUN: %if run-mode %{cat *.dot > /dev/null%}
+// RUN: %if run-mode %{rm *.dot%}
 // RUN: %{run-unfiltered-devices} %t.out
-// RUN: not cat *.dot > /dev/null
+// RUN: %if run-mode %{not cat *.dot > /dev/null%}
 
 #include <sycl/detail/core.hpp>
 

diff --git a/sycl/test-e2e/DeviceArchitecture/device_architecture_comparison_on_device_aot.cpp b/sycl/test-e2e/DeviceArchitecture/device_architecture_comparison_on_device_aot.cpp
@@ -1,6 +1,6 @@
 // REQUIRES: arch-intel_gpu_pvc, ocloc
 
-// XFAIL: arch-intel_gpu_pvc
+// XFAIL: run-mode
 // XFAIL-TRACKER: https://github.com/intel/llvm/issues/14826
 
 // RUN: %clangxx -fsycl -fsycl-targets=intel_gpu_pvc %s -o %t.out

diff --git a/sycl/test-e2e/DeviceLib/math_fp64_windows_test.cpp b/sycl/test-e2e/DeviceLib/math_fp64_windows_test.cpp
@@ -1,4 +1,5 @@
 // REQUIRES: aspect-fp64, windows
+// REQUIRES: build-and-run-mode
 
 // DEFINE: %{mathflags} = %if cl_options %{/clang:-fno-fast-math%} %else %{-fno-fast-math%}
 

diff --git a/sycl/test-e2e/DeviceLib/math_windows_test.cpp b/sycl/test-e2e/DeviceLib/math_windows_test.cpp
@@ -1,4 +1,5 @@
 // REQUIRES: windows
+// REQUIRES: build-and-run-mode
 
 // TODO: Add hypotf case back when the missing symbol is fixed.
 

diff --git a/sycl/test-e2e/ESIMD/PerformanceTests/BitonicSortK.cpp b/sycl/test-e2e/ESIMD/PerformanceTests/BitonicSortK.cpp
@@ -10,7 +10,7 @@
 
 // RUN: mkdir -p %t.dir && %{build} -o %t.dir/exec.out
 // RUN: env IGC_DumpToCustomDir=%t.dir IGC_ShaderDumpEnable=1 %{run} %t.dir/exec.out
-// RUN: python3 %S/instruction_count.py %t.dir 2914 ZTSZZN11BitonicSort5SolveEPjS0_jENKUlRN4sycl3_V17handlerEE0_clES4_E5Merge.asm
-// RUN: echo "Baseline from driver version 1.3.30872"
+// RUN: %if run-mode %{python3 %S/instruction_count.py %t.dir 2914 ZTSZZN11BitonicSort5SolveEPjS0_jENKUlRN4sycl3_V17handlerEE0_clES4_E5Merge.asm%}
+// RUN: %if run-mode %{echo "Baseline from driver version 1.3.30872"%}
 
 #include "../BitonicSortK.cpp"
diff --git a/sycl/test-e2e/ESIMD/PerformanceTests/BitonicSortKv2.cpp b/sycl/test-e2e/ESIMD/PerformanceTests/BitonicSortKv2.cpp
@@ -10,7 +10,7 @@
 
 // RUN: mkdir -p %t.dir && %{build} -o %t.dir/exec.out
 // RUN: env IGC_DumpToCustomDir=%t.dir IGC_ShaderDumpEnable=1 %{run} %t.dir/exec.out
-// RUN: python3 %S/instruction_count.py %t.dir 2969 ZTSZZN11BitonicSort5SolveEPjS0_jENKUlRN4sycl3_V17handlerEE0_clES4_E5Merge.asm
-// RUN: echo "Baseline from driver version 1.3.30872"
+// RUN: %if run-mode %{python3 %S/instruction_count.py %t.dir 2969 ZTSZZN11BitonicSort5SolveEPjS0_jENKUlRN4sycl3_V17handlerEE0_clES4_E5Merge.asm%}
+// RUN: %if run-mode %{echo "Baseline from driver version 1.3.30872"%}
 
 #include "../BitonicSortKv2.cpp"
diff --git a/sycl/test-e2e/ESIMD/PerformanceTests/Stencil.cpp b/sycl/test-e2e/ESIMD/PerformanceTests/Stencil.cpp
@@ -10,7 +10,7 @@
 
 // RUN: mkdir -p %t.dir && %{build} -o %t.dir/exec.out
 // RUN: env IGC_DumpToCustomDir=%t.dir IGC_ShaderDumpEnable=1 %{run} %t.dir/exec.out
-// RUN: python3 %S/instruction_count.py %t.dir 1699 ZTSZZ4mainENKUlRN4sycl3_V17handlerEE_clES2_E14Stencil_kernel.asm
-// RUN: echo "Baseline from driver version 1.3.29138"
+// RUN: %if run-mode %{python3 %S/instruction_count.py %t.dir 1699 ZTSZZ4mainENKUlRN4sycl3_V17handlerEE_clES2_E14Stencil_kernel.asm%}
+// RUN: %if run-mode %{echo "Baseline from driver version 1.3.29138"%}
 
-#include "../Stencil.cpp"
+#include "../Stencil.cpp"
diff --git a/sycl/test-e2e/ESIMD/PerformanceTests/invoke_simd_smoke.cpp b/sycl/test-e2e/ESIMD/PerformanceTests/invoke_simd_smoke.cpp
@@ -10,7 +10,7 @@
 
 // RUN: mkdir -p %t.dir && %{build} -fsycl -fno-sycl-device-code-split-esimd -Xclang -fsycl-allow-func-ptr -o %t.dir/exec.out
 // RUN: env  IGC_VCSaveStackCallLinkage=1 IGC_VCDirectCallsOnly=1 IGC_DumpToCustomDir=%t.dir IGC_ShaderDumpEnable=1 %{run} %t.dir/exec.out
-// RUN: python3 %S/instruction_count.py %t.dir 149 _simd16_entry_0001.asm
-// RUN: echo "Baseline from driver version 1.3.29735"
+// RUN: %if run-mode %{python3 %S/instruction_count.py %t.dir 149 _simd16_entry_0001.asm%}
+// RUN: %if run-mode %{echo "Baseline from driver version 1.3.29735"%}
 
-#include "../../InvokeSimd/invoke_simd_smoke.cpp"
+#include "../../InvokeSimd/invoke_simd_smoke.cpp"
diff --git a/sycl/test-e2e/ESIMD/PerformanceTests/matrix_transpose.cpp b/sycl/test-e2e/ESIMD/PerformanceTests/matrix_transpose.cpp
@@ -12,7 +12,7 @@
 
 // RUN: mkdir -p %t.dir && %{build} -o %t.dir/exec.out
 // RUN: env IGC_DumpToCustomDir=%t.dir IGC_ShaderDumpEnable=1 %{run} %t.dir/exec.out
-// RUN: python3 %S/instruction_count.py %t.dir %if igc-dev %{ 1059 %} %else %{ 1116 %} ZTSZZ7runTestjjjRdS_ENKUlRN4sycl3_V17handlerEE_clES3_E3K16.asm
-// RUN: echo "Baseline from driver version 1.3.30872"
+// RUN: %if run-mode %{python3 %S/instruction_count.py %t.dir %if igc-dev %{ 1059 %} %else %{ 1116 %} ZTSZZ7runTestjjjRdS_ENKUlRN4sycl3_V17handlerEE_clES3_E3K16.asm%}
+// RUN: %if run-mode %{echo "Baseline from driver version 1.3.30872"%}
 
 #include "../matrix_transpose.cpp"
diff --git a/sycl/test-e2e/ESIMD/PerformanceTests/stencil2.cpp b/sycl/test-e2e/ESIMD/PerformanceTests/stencil2.cpp
@@ -10,7 +10,7 @@
 
 // RUN: mkdir -p %t.dir && %{build} -o %t.dir/exec.out
 // RUN: env IGC_DumpToCustomDir=%t.dir IGC_ShaderDumpEnable=1 %{run} %t.dir/exec.out
-// RUN: python3 %S/instruction_count.py %t.dir 1699 ZTSZZ4mainENKUlRN4sycl3_V17handlerEE_clES2_E14Stencil_kernel.asm
-// RUN: echo "Baseline from driver version 1.3.29138"
+// RUN: %if run-mode %{python3 %S/instruction_count.py %t.dir 1699 ZTSZZ4mainENKUlRN4sycl3_V17handlerEE_clES2_E14Stencil_kernel.asm%}
+// RUN: %if run-mode %{echo "Baseline from driver version 1.3.29138"%}
 
-#include "../stencil2.cpp"
+#include "../stencil2.cpp"
diff --git a/sycl/test-e2e/ESIMD/assert.cpp b/sycl/test-e2e/ESIMD/assert.cpp
@@ -6,7 +6,7 @@
 //
 // The test still fails after GPU driver update on Linux. Temporary marking it
 // as expected to fail, whilst it is being investigated, see intel/llvm#11359
-// XFAIL: linux
+// XFAIL: linux && run-mode
 // XFAIL-TRACKER: https://github.com/intel/llvm/issues/11359
 //
 // Hanging on gen12, remove when internal tracker fixed

diff --git a/sycl/test-e2e/ESIMD/preemption.cpp b/sycl/test-e2e/ESIMD/preemption.cpp
@@ -9,7 +9,7 @@
 // UNSUPPORTED: gpu-intel-dg2 || arch-intel_gpu_pvc
 // RUN: %{build} -o %t.out
 // RUN: env IGC_DumpToCustomDir=%t.dump IGC_ShaderDumpEnable=1 %{run} %t.out
-// RUN: grep enablePreemption %t.dump/*.asm
+// RUN: %if run-mode %{grep enablePreemption %t.dump/*.asm%}
 
 // The test expects to see "enablePreemption" switch in the compilation
 // switches. It fails if does not find it.

diff --git a/sycl/test-e2e/EnqueueNativeCommand/custom-command-cuda.cpp b/sycl/test-e2e/EnqueueNativeCommand/custom-command-cuda.cpp
@@ -1,6 +1,7 @@
 // RUN: %{build} -Wno-error=deprecated-declarations -o %t.out %cuda_options
 // RUN: %{run} %t.out
 // REQUIRES: cuda, cuda_dev_kit
+// REQUIRES: build-and-run-mode
 
 #include <cuda.h>
 

diff --git a/sycl/test-e2e/EnqueueNativeCommand/custom-command-hip.cpp b/sycl/test-e2e/EnqueueNativeCommand/custom-command-hip.cpp
@@ -3,6 +3,7 @@
 // RUN: %{build} -o %t.out -I%rocm_path/include -L%rocm_path/lib -lamdhip64
 // RUN: %{run} %t.out
 // REQUIRES: hip
+// REQUIRES: build-and-run-mode
 
 #include <iostream>
 #include <sycl/backend.hpp>

diff --git a/sycl/test-e2e/EnqueueNativeCommand/custom-command-multiple-dev-cuda.cpp b/sycl/test-e2e/EnqueueNativeCommand/custom-command-multiple-dev-cuda.cpp
@@ -1,4 +1,5 @@
 // REQUIRES: cuda, cuda_dev_kit
+// REQUIRES: build-and-run-mode
 // RUN: %{build} -o %t.out %cuda_options
 // RUN: %{run} %t.out
 

diff --git a/sycl/test-e2e/HostInteropTask/interop-task-cuda-buffer-migrate.cpp b/sycl/test-e2e/HostInteropTask/interop-task-cuda-buffer-migrate.cpp
@@ -1,4 +1,5 @@
 // REQUIRES: cuda, cuda_dev_kit
+// REQUIRES: build-and-run-mode
 //
 // RUN: %{build} -o %t.out %cuda_options
 // RUN: %{run} %t.out

diff --git a/sycl/test-e2e/HostInteropTask/interop-task-cuda.cpp b/sycl/test-e2e/HostInteropTask/interop-task-cuda.cpp
@@ -1,6 +1,7 @@
 // RUN: %{build} -o %t.out %cuda_options
 // RUN: %{run} %t.out
 // REQUIRES: cuda, cuda_dev_kit
+// REQUIRES: build-and-run-mode
 
 #include <iostream>
 #include <sycl/backend.hpp>

diff --git a/sycl/test-e2e/HostInteropTask/interop-task-hip.cpp b/sycl/test-e2e/HostInteropTask/interop-task-hip.cpp
@@ -3,6 +3,7 @@
 // RUN: %{build} -Wno-error=deprecated-declarations -o %t.out -I%rocm_path/include -L%rocm_path/lib -lamdhip64
 // RUN: %{run} %t.out
 // REQUIRES: hip
+// REQUIRES: build-and-run-mode
 
 #include <iostream>
 #include <sycl/backend.hpp>

diff --git a/sycl/test-e2e/InlineAsm/asm_multiple_instructions.cpp b/sycl/test-e2e/InlineAsm/asm_multiple_instructions.cpp
@@ -6,7 +6,7 @@
 // RUN: %{run} %t.out
 // The test is failing when writing directly to output buffer.
 // If temporary variable is used (see TO_PASS mode) the test succeeded.
-// XFAIL: gpu
+// XFAIL: gpu && run-mode
 #include "include/asmhelper.h"
 #include <iostream>
 #include <vector>

diff --git a/sycl/test-e2e/InvokeSimd/Feature/ImplicitSubgroup/invoke_simd_struct.cpp b/sycl/test-e2e/InvokeSimd/Feature/ImplicitSubgroup/invoke_simd_struct.cpp
@@ -1,6 +1,6 @@
 // TODO: Passing/returning structures via invoke_simd() API is not implemented
 // in GPU driver yet. Enable the test when GPU RT supports it.
-// XFAIL: gpu
+// XFAIL: gpu && run-mode
 //
 // RUN: %{build} -DIMPL_SUBGROUP -fno-sycl-device-code-split-esimd -Xclang -fsycl-allow-func-ptr -o %t.out
 // RUN: env IGC_VCSaveStackCallLinkage=1 IGC_VCDirectCallsOnly=1 %{run} %t.out

diff --git a/sycl/test-e2e/InvokeSimd/Feature/invoke_simd_struct.cpp b/sycl/test-e2e/InvokeSimd/Feature/invoke_simd_struct.cpp
@@ -1,6 +1,6 @@
 // TODO: Passing/returning structures via invoke_simd() API is not implemented
 // in GPU driver yet. Enable the test when GPU RT supports it.
-// XFAIL: gpu
+// XFAIL: gpu, run-mode
 //
 // RUN: %{build} -fno-sycl-device-code-split-esimd -Xclang -fsycl-allow-func-ptr -o %t.out
 // RUN: env IGC_VCSaveStackCallLinkage=1 IGC_VCDirectCallsOnly=1 %{run} %t.out

diff --git a/sycl/test-e2e/InvokeSimd/Spec/ImplicitSubgroup/tuple.cpp b/sycl/test-e2e/InvokeSimd/Spec/ImplicitSubgroup/tuple.cpp
@@ -1,5 +1,5 @@
 // TODO: enable when Jira ticket resolved
-// XFAIL: gpu
+// XFAIL: *
 //
 // Check that full compilation works:
 // RUN: %clangxx -DIMPL_SUBGROUP -fsycl -fno-sycl-device-code-split-esimd -Xclang -fsycl-allow-func-ptr %S/../tuple.cpp -o %t.out

diff --git a/sycl/test-e2e/InvokeSimd/Spec/ImplicitSubgroup/tuple_return.cpp b/sycl/test-e2e/InvokeSimd/Spec/ImplicitSubgroup/tuple_return.cpp
@@ -1,5 +1,5 @@
 // TODO: enable when Jira ticket resolved
-// XFAIL: gpu
+// XFAIL: *
 //
 // Check that full compilation works:
 // RUN: %clangxx -DIMPL_SUBGROUP -fsycl -fno-sycl-device-code-split-esimd -Xclang -fsycl-allow-func-ptr %S/../tuple_return.cpp -o %t.out

diff --git a/sycl/test-e2e/InvokeSimd/Spec/ImplicitSubgroup/tuple_vadd.cpp b/sycl/test-e2e/InvokeSimd/Spec/ImplicitSubgroup/tuple_vadd.cpp
@@ -1,5 +1,5 @@
 // TODO: enable when Jira ticket resolved
-// XFAIL: gpu
+// XFAIL: *
 //
 // Check that full compilation works:
 // RUN: %clangxx -DIMPL_SUBGROUP -fsycl -fno-sycl-device-code-split-esimd -Xclang -fsycl-allow-func-ptr %S/../tuple_vadd.cpp -o %t.out

diff --git a/sycl/test-e2e/InvokeSimd/Spec/tuple.cpp b/sycl/test-e2e/InvokeSimd/Spec/tuple.cpp
@@ -1,5 +1,5 @@
 // TODO: enable when Jira ticket resolved
-// XFAIL: gpu
+// XFAIL: *
 //
 // Check that full compilation works:
 // RUN: %{build} -fno-sycl-device-code-split-esimd -Xclang -fsycl-allow-func-ptr -o %t.out

diff --git a/sycl/test-e2e/InvokeSimd/Spec/tuple_return.cpp b/sycl/test-e2e/InvokeSimd/Spec/tuple_return.cpp
@@ -1,5 +1,5 @@
 // TODO: enable when Jira ticket resolved
-// XFAIL: gpu
+// XFAIL: *
 //
 // Check that full compilation works:
 // RUN: %{build} -fno-sycl-device-code-split-esimd -Xclang -fsycl-allow-func-ptr -o %t.out

diff --git a/sycl/test-e2e/InvokeSimd/Spec/tuple_vadd.cpp b/sycl/test-e2e/InvokeSimd/Spec/tuple_vadd.cpp
@@ -1,5 +1,5 @@
 // TODO: enable when Jira ticket resolved
-// XFAIL: gpu
+// XFAIL: *
 //
 // Check that full compilation works:
 // RUN: %{build} -fno-sycl-device-code-split-esimd -Xclang -fsycl-allow-func-ptr -o %t.out

diff --git a/sycl/test-e2e/KernelAndProgram/test_cache_jit_aot.cpp b/sycl/test-e2e/KernelAndProgram/test_cache_jit_aot.cpp
@@ -2,6 +2,7 @@
 // cannot do that reliably when number of devices is unknown.
 //
 // REQUIRES: level_zero, ocloc
+// REQUIRES: build-and-run-mode
 //
 // DEFINE: %{cache_vars} = env SYCL_CACHE_PERSISTENT=1 SYCL_CACHE_TRACE=1 SYCL_CACHE_DIR=%t/cache_dir
 // DEFINE: %{build_cmd} = %{build}

diff --git a/sycl/test-e2e/KernelAndProgram/trace_kernel_program_cache.cpp b/sycl/test-e2e/KernelAndProgram/trace_kernel_program_cache.cpp
@@ -6,12 +6,12 @@
 // or SYCL_CACHE_TRACE is set to 0.
 
 // RUN: env SYCL_CACHE_IN_MEM=0 %{run} %t.out 2> %t.trace1
-// RUN: FileCheck --allow-empty --input-file=%t.trace1 --implicit-check-not "In-Memory Cache" %s
+// RUN: %if run-mode %{ FileCheck --allow-empty --input-file=%t.trace1 --implicit-check-not "In-Memory Cache" %s %}
 // RUN: env SYCL_CACHE_TRACE=0 %{run} %t.out 2> %t.trace2
-// RUN: FileCheck --allow-empty --input-file=%t.trace2 --implicit-check-not "In-Memory Cache" %s
+// RUN: %if run-mode %{ FileCheck --allow-empty --input-file=%t.trace2 --implicit-check-not "In-Memory Cache" %s %}
 
 // RUN: env SYCL_CACHE_TRACE=2 %{run} %t.out 2> %t.trace3
-// RUN: FileCheck %s --input-file=%t.trace3 --check-prefix=CHECK-CACHE-TRACE
+// RUN: %if run-mode %{ FileCheck %s --input-file=%t.trace3 --check-prefix=CHECK-CACHE-TRACE %}
 
 #include <sycl/detail/core.hpp>
 

diff --git a/sycl/test-e2e/KernelCompiler/kernel_compiler_sycl_jit.cpp b/sycl/test-e2e/KernelCompiler/kernel_compiler_sycl_jit.cpp
@@ -16,14 +16,14 @@
 // -- Test again, with caching.
 
 // DEFINE: %{cache_vars} = %{l0_leak_check} env SYCL_CACHE_PERSISTENT=1 SYCL_CACHE_TRACE=5 SYCL_CACHE_DIR=%t/cache_dir
-// RUN: rm -rf %t/cache_dir
-// RUN:  %{cache_vars} %t.out 2>&1 |  FileCheck %s --check-prefixes=CHECK-WRITTEN-TO-CACHE
-// RUN:  %{cache_vars} %t.out 2>&1 |  FileCheck %s --check-prefixes=CHECK-READ-FROM-CACHE
+// RUN: %if run-mode %{ rm -rf %t/cache_dir %}
+// RUN: %{cache_vars} %{run-unfiltered-devices} %t.out 2>&1 |  FileCheck %s --check-prefixes=CHECK-WRITTEN-TO-CACHE
+// RUN: %{cache_vars} %{run-unfiltered-devices} %t.out 2>&1 |  FileCheck %s --check-prefixes=CHECK-READ-FROM-CACHE
 
 // -- Add leak check.
-// RUN: rm -rf %t/cache_dir
-// RUN:   %{l0_leak_check} %{cache_vars} %t.out 2>&1 |  FileCheck %s --check-prefixes=CHECK-WRITTEN-TO-CACHE
-// RUN:   %{l0_leak_check} %{cache_vars} %t.out 2>&1 |  FileCheck %s --check-prefixes=CHECK-READ-FROM-CACHE
+// RUN: %if run-mode %{ rm -rf %t/cache_dir %}
+// RUN: %{l0_leak_check} %{cache_vars} %{run-unfiltered-devices} %t.out 2>&1 |  FileCheck %s --check-prefixes=CHECK-WRITTEN-TO-CACHE
+// RUN: %{l0_leak_check} %{cache_vars} %{run-unfiltered-devices} %t.out 2>&1 |  FileCheck %s --check-prefixes=CHECK-READ-FROM-CACHE
 
 // CHECK-WRITTEN-TO-CACHE: [Persistent Cache]: enabled
 // CHECK-WRITTEN-TO-CACHE-NOT: [kernel_compiler Persistent Cache]: using cached binary

diff --git a/sycl/test-e2e/Matrix/SG32/joint_matrix_colA_rowB_colC.cpp b/sycl/test-e2e/Matrix/SG32/joint_matrix_colA_rowB_colC.cpp
@@ -11,7 +11,7 @@
 // RUN: %{build} -o %t.out
 // RUN: %{run} %t.out
 
-// XFAIL:*
+// XFAIL: run-mode
 
 #include "../common.hpp"
 

diff --git a/sycl/test-e2e/Matrix/joint_matrix_colA_rowB_colC.cpp b/sycl/test-e2e/Matrix/joint_matrix_colA_rowB_colC.cpp
@@ -10,7 +10,7 @@
 // RUN: %{build} -o %t.out
 // RUN: %{run} %t.out
 
-// XFAIL:*
+// XFAIL: run-mode
 
 #include "common.hpp"
 

diff --git a/sycl/test-e2e/Matrix/joint_matrix_hip_gfx90a.cpp b/sycl/test-e2e/Matrix/joint_matrix_hip_gfx90a.cpp
@@ -10,6 +10,7 @@
 // RUN: %{run} %t.out
 
 // REQUIRES: arch-amd_gpu_gfx90a
+// REQUIRES: build-and-run-mode
 
 #include "joint_matrix_hip_apply.hpp"
 #include "joint_matrix_hip_copy.hpp"

diff --git a/sycl/test-e2e/Matrix/joint_matrix_hip_half_gfx90a.cpp b/sycl/test-e2e/Matrix/joint_matrix_hip_half_gfx90a.cpp
@@ -11,6 +11,7 @@
 
 // REQUIRES: arch-amd_gpu_gfx90a
 // REQUIRES: aspect-fp16
+// REQUIRES: build-and-run-mode
 
 #include "joint_matrix_hip_apply.hpp"
 #include "joint_matrix_hip_copy.hpp"

diff --git a/sycl/test-e2e/Matrix/joint_matrix_tensorcores_sm70.cpp b/sycl/test-e2e/Matrix/joint_matrix_tensorcores_sm70.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 // REQUIRES: cuda
+// REQUIRES: build-and-run-mode
 // RUN: %{build} -Xsycl-target-backend --cuda-gpu-arch=sm_70 -o %t.out
 // RUN: %{run} %t.out
 //

diff --git a/sycl/test-e2e/Matrix/runtime_query_hip_gfx90a.cpp b/sycl/test-e2e/Matrix/runtime_query_hip_gfx90a.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 // REQUIRES: arch-amd_gpu_gfx90a
+// REQUIRES: build-and-run-mode
 // RUN: %clangxx -fsycl -fsycl-targets=amdgcn-amd-amdhsa -Xsycl-target-backend=amdgcn-amd-amdhsa --offload-arch=gfx90a %s -o %t.out
 // RUN: %{run} %t.out
 

diff --git a/sycl/test-e2e/OptionalKernelFeatures/is_compatible/is_compatible_amdgcn.cpp b/sycl/test-e2e/OptionalKernelFeatures/is_compatible/is_compatible_amdgcn.cpp
@@ -1,4 +1,5 @@
 // REQUIRES: hip_amd, opencl, gpu, cpu
+// REQUIRES: build-and-run-mode
 
 // RUN: %clangxx -fsycl -Xsycl-target-backend=amdgcn-amd-amdhsa --offload-arch=gfx906 -fsycl-targets=amdgcn-amd-amdhsa %S/Inputs/is_compatible_with_env.cpp -o %t.out
 

diff --git a/sycl/test-e2e/OptionalKernelFeatures/is_compatible/is_compatible_nvptx64.cpp b/sycl/test-e2e/OptionalKernelFeatures/is_compatible/is_compatible_nvptx64.cpp
@@ -1,4 +1,5 @@
 // REQUIRES: cuda, opencl, gpu, cpu
+// REQUIRES: build-and-run-mode
 
 // RUN: %clangxx -fsycl -fsycl-targets=nvptx64-nvidia-cuda %S/Inputs/is_compatible_with_env.cpp -o %t.out
 

diff --git a/sycl/test-e2e/OptionalKernelFeatures/is_compatible/is_compatible_several_targets.cpp b/sycl/test-e2e/OptionalKernelFeatures/is_compatible/is_compatible_several_targets.cpp
@@ -1,4 +1,5 @@
 // REQUIRES: ocloc, level_zero, gpu, cpu
+// REQUIRES: build-and-run-mode
 
 // RUN: %clangxx -fsycl -fsycl-targets=spir64_fpga,spir64_gen -Xsycl-target-backend "-device *" %S/Inputs/is_compatible_with_env.cpp -o %t.out
 

diff --git a/sycl/test-e2e/Plugin/cuda_queue_priority.cpp b/sycl/test-e2e/Plugin/cuda_queue_priority.cpp
@@ -1,4 +1,5 @@
 // REQUIRES: gpu, cuda, cuda_dev_kit
+// REQUIRES: build-and-run-mode
 // RUN: %{build} %cuda_options -o %t.out
 // RUN: %{run} %t.out
 //

diff --git a/sycl/test-e2e/Plugin/dll-detach-order.cpp b/sycl/test-e2e/Plugin/dll-detach-order.cpp
@@ -1,4 +1,5 @@
 // REQUIRES: windows
+// REQUIRES: build-and-run-mode
 // RUN: env SYCL_UR_TRACE=-1 sycl-ls | FileCheck %s
 
 // ensure that the plugins are detached AFTER urLoaderTearDown is done executing

diff --git a/sycl/test-e2e/Plugin/level_zero_sub_sub_device.cpp b/sycl/test-e2e/Plugin/level_zero_sub_sub_device.cpp
@@ -1,6 +1,6 @@
 // REQUIRES:  arch-intel_gpu_pvc, level_zero
 
-// XFAIL: arch-intel_gpu_pvc
+// XFAIL: arch-intel_gpu_pvc && run-mode
 // XFAIL-TRACKER: https://github.com/intel/llvm/issues/15602
 
 // RUN: %{build} %level_zero_options -o %t.out

diff --git a/sycl/test-e2e/Plugin/sycl-ls-gpu-cuda.cpp b/sycl/test-e2e/Plugin/sycl-ls-gpu-cuda.cpp
@@ -1,4 +1,5 @@
 // REQUIRES: gpu, cuda
+// REQUIRES: build-and-run-mode
 
 // RUN: env ONEAPI_DEVICE_SELECTOR="cuda:*" sycl-ls --verbose >%t.cuda.out
 // RUN: FileCheck %s --check-prefixes=CHECK-BUILTIN-GPU-CUDA,CHECK-CUSTOM-GPU-CUDA --input-file %t.cuda.out

diff --git a/sycl/test-e2e/Plugin/sycl-ls-gpu-hip.cpp b/sycl/test-e2e/Plugin/sycl-ls-gpu-hip.cpp
@@ -1,4 +1,5 @@
 // REQUIRES: gpu, hip, sycl-ls
+// REQUIRES: build-and-run-mode
 
 // RUN: env ONEAPI_DEVICE_SELECTOR="hip:*" sycl-ls --verbose >%t.hip.out
 // RUN: FileCheck %s --check-prefixes=CHECK-BUILTIN-GPU-HIP,CHECK-CUSTOM-GPU-HIP --input-file %t.hip.out