diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index a4f884223..9d9e872f8 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -271,7 +271,7 @@ jobs: {runs-on: n300, enable_perf: OFF, emitc: OFF, name: "run", ttrt_flags: "--non-zero"}, {runs-on: n300, enable_perf: ON, emitc: OFF, name: "perf"}, ] - name: "run-tests (${{ matrix.build.runs-on }}, ${{ matrix.build.enable_perf }}, ${{ matrix.build.name }})" + name: "run-tests (${{ matrix.build.runs-on }}, ${{ matrix.build.enable_perf }}, ${{ matrix.build.emitc }}, ${{ matrix.build.name }})" runs-on: - in-service @@ -296,7 +296,7 @@ jobs: id: strings shell: bash env: - job-name: "run-tests (${{ matrix.build.runs-on }}, ${{ matrix.build.enable_perf }}, ${{ matrix.build.name }})" + job-name: "run-tests (${{ matrix.build.runs-on }}, ${{ matrix.build.enable_perf }}, ${{ matrix.build.emitc }}, ${{ matrix.build.name }})" run: | echo "work-dir=$(pwd)" >> "$GITHUB_OUTPUT" echo "build-output-dir=$(pwd)/build" >> "$GITHUB_OUTPUT" diff --git a/runtime/tools/python/ttrt/common/run.py b/runtime/tools/python/ttrt/common/run.py index 7a75cb5ce..d17c1b38a 100644 --- a/runtime/tools/python/ttrt/common/run.py +++ b/runtime/tools/python/ttrt/common/run.py @@ -558,33 +558,38 @@ def _execute(binaries): if event is not None: ttrt.runtime.wait(event) + # Compare to EmitC if is_emitc_testing_requested: fwd_func_name = program.program["name"] fwd_func_name_len = len(fwd_func_name) fwd_func_sym = f"_Z{fwd_func_name_len}{fwd_func_name}St6vectorIN2tt8tt_metal6TensorESaIS2_EEPNS1_2v06DeviceE" - emitc_outs = ttrt.runtime.run_so_program( - emitc_dylib_handle, - fwd_func_sym, - inputs, - device, - ) - self.logging.debug( - f"got emitc outputs for program={program_index}" - ) - - all_tensors_match = ttrt.runtime.compare_outs( - total_outputs[0], emitc_outs - ) - if not all_tensors_match: - self.logging.error( - "Failed: TTRT and EmitC outputs do not match!" + for loop in range(self["--loops"]): + emitc_outs = ttrt.runtime.run_so_program( + emitc_dylib_handle, + fwd_func_sym, + total_inputs[loop], + device, + ) + self.logging.debug( + f"got emitc outputs for program_index={program_index}, loop={loop}" ) - self.logging.error(total_outputs[0], emitc_outs) - raise Exception( - "Failed: TTRT and EmitC outputs do not match!" + + all_tensors_match = ttrt.runtime.compare_outs( + total_outputs[0], emitc_outs ) + if not all_tensors_match: + self.logging.error( + "Failed: TTRT and EmitC outputs do not match! program_index={program_index}, loop={loop}" + ) + self.logging.error( + total_outputs[loop], emitc_outs + ) + raise Exception( + "Failed: TTRT and EmitC outputs do not match! program_index={program_index}, loop={loop}" + ) + if self["--identity"]: self.logging.debug( f"checking identity with rtol={self['--rtol']} and atol={self['--atol']}"