From 3ab20b0e7fe99e997da9c1f5f642e7b36ad8ae9a Mon Sep 17 00:00:00 2001 From: edeiana Date: Wed, 9 Oct 2024 03:49:19 -0700 Subject: [PATCH] i#7031: invalidate CPU marker filter In some cases we don't want to expose the "as traced" CPU schedule in an offline trace because it might not be representative of the native execution of the traced program. To do so we implement `invalidate_cpu_filter_t`, a new filter that is used with `record_filter` as follows: ``` drrun -t drmemtrace -tool record_filter -filter_invalidate_cpu -indir path/to/input/trace -outdir path/to/output/trace ``` This filter set the value of TRACE_MARKER_TYPE_CPU_ID markers to (uintptr_t)-1, which representes an unknown CPU. We add a unit test `test_invalidate_cpu_filter()` and an end-to-end test `code_api|tool.record_filter_invalidate_cpu` which invokes the `invariant_checker` and `view` tool on the filtered trace. Fixes #7031 --- api/docs/release.dox | 4 + clients/drcachesim/CMakeLists.txt | 1 + clients/drcachesim/analyzer_multi.cpp | 3 +- clients/drcachesim/common/options.cpp | 5 + clients/drcachesim/common/options.h | 1 + .../record_filter_invalidate_cpu.templatex | 24 +++++ .../tests/record_filter_unit_tests.cpp | 67 +++++++++++++- .../tools/filter/invalidate_cpu_filter.h | 92 +++++++++++++++++++ .../drcachesim/tools/filter/record_filter.cpp | 8 +- .../tools/filter/record_filter_create.h | 4 +- .../tools/record_filter_launcher.cpp | 9 +- suite/tests/CMakeLists.txt | 8 ++ 12 files changed, 221 insertions(+), 5 deletions(-) create mode 100644 clients/drcachesim/tests/record_filter_invalidate_cpu.templatex create mode 100644 clients/drcachesim/tools/filter/invalidate_cpu_filter.h diff --git a/api/docs/release.dox b/api/docs/release.dox index f43af9d3274..a097f7d08b9 100644 --- a/api/docs/release.dox +++ b/api/docs/release.dox @@ -265,6 +265,10 @@ Further non-compatibility-affecting changes include: - Added -trace_instr_intervals_file option to the drmemtrace trace analysis tools framework. The file must be in CSV format containing a tracing interval per line where start and duration are expressed in number of instructions. + - Added invalidate_cpu_filter_t to #dynamorio::drmemtrace::record_filter_t to invalidate + the value of markers #dynamorio::drmemtrace::TRACE_MARKER_TYPE_CPU_ID. When + -filter_invalidate_cpu is used, the value of those markers is set to (uintptr_t)-1, + which means "undefined". **************************************************
diff --git a/clients/drcachesim/CMakeLists.txt b/clients/drcachesim/CMakeLists.txt index 81e54281312..a0921fa31bd 100644 --- a/clients/drcachesim/CMakeLists.txt +++ b/clients/drcachesim/CMakeLists.txt @@ -201,6 +201,7 @@ add_exported_library(drmemtrace_record_filter STATIC tools/filter/type_filter.h tools/filter/encodings2regdeps_filter.h tools/filter/func_id_filter.h + tools/filter/invalidate_cpu_filter.h tools/filter/null_filter.h) target_link_libraries(drmemtrace_record_filter drmemtrace_simulator drmemtrace_schedule_file) diff --git a/clients/drcachesim/analyzer_multi.cpp b/clients/drcachesim/analyzer_multi.cpp index c27f231831c..628e832369c 100644 --- a/clients/drcachesim/analyzer_multi.cpp +++ b/clients/drcachesim/analyzer_multi.cpp @@ -339,7 +339,8 @@ record_analyzer_multi_t::create_analysis_tool_from_options(const std::string &to op_filter_cache_size.get_value(), op_filter_trace_types.get_value(), op_filter_marker_types.get_value(), op_trim_before_timestamp.get_value(), op_trim_after_timestamp.get_value(), op_encodings2regdeps.get_value(), - op_filter_func_ids.get_value(), op_verbose.get_value()); + op_filter_func_ids.get_value(), op_invalidate_cpu.get_value(), + op_verbose.get_value()); } ERRMSG("Usage error: unsupported record analyzer type \"%s\". Only " RECORD_FILTER " is supported.\n", diff --git a/clients/drcachesim/common/options.cpp b/clients/drcachesim/common/options.cpp index 5166a8bcd63..f36c603820d 100644 --- a/clients/drcachesim/common/options.cpp +++ b/clients/drcachesim/common/options.cpp @@ -1112,6 +1112,11 @@ droption_t "for the listed function IDs and removes those belonging to " "unlisted function IDs."); +droption_t op_invalidate_cpu( + DROPTION_SCOPE_FRONTEND, "filter_invalidate_cpu", false, + "Invalidate TRACE_MARKER_TYPE_CPU_ID", + "Invalidate TRACE_MARKER_TYPE_CPU_ID by setting its value to (uintptr_t)-1."); + droption_t op_trim_before_timestamp( DROPTION_SCOPE_ALL, "trim_before_timestamp", 0, 0, (std::numeric_limits::max)(), diff --git a/clients/drcachesim/common/options.h b/clients/drcachesim/common/options.h index ef24824b4d0..761c8748fd9 100644 --- a/clients/drcachesim/common/options.h +++ b/clients/drcachesim/common/options.h @@ -230,6 +230,7 @@ extern dynamorio::droption::droption_t op_filter_trace_types; extern dynamorio::droption::droption_t op_filter_marker_types; extern dynamorio::droption::droption_t op_encodings2regdeps; extern dynamorio::droption::droption_t op_filter_func_ids; +extern dynamorio::droption::droption_t op_invalidate_cpu; extern dynamorio::droption::droption_t op_trim_before_timestamp; extern dynamorio::droption::droption_t op_trim_after_timestamp; extern dynamorio::droption::droption_t op_abort_on_invariant_error; diff --git a/clients/drcachesim/tests/record_filter_invalidate_cpu.templatex b/clients/drcachesim/tests/record_filter_invalidate_cpu.templatex new file mode 100644 index 00000000000..31eaf5eb655 --- /dev/null +++ b/clients/drcachesim/tests/record_filter_invalidate_cpu.templatex @@ -0,0 +1,24 @@ +Estimation of pi is 3.142425985001098 + +Trace invariant checks passed + +Output .* entries from .* entries. + +Output format: + +<--record#-> <--instr#->: <---tid---> + +------------------------------------------------------------ + + 1 0: +[0-9]+ + 2 0: +[0-9]+ + 3 0: +[0-9]+ + 4 0: +[0-9]+ + 5 0: +[0-9]+ + 6 0: +[0-9]+ +#ifdef X64 + 7 0: +[0-9]+ +#else + 7 0: +[0-9]+ +#endif +.* diff --git a/clients/drcachesim/tests/record_filter_unit_tests.cpp b/clients/drcachesim/tests/record_filter_unit_tests.cpp index fd6d6ccf0cf..f915477c4c6 100644 --- a/clients/drcachesim/tests/record_filter_unit_tests.cpp +++ b/clients/drcachesim/tests/record_filter_unit_tests.cpp @@ -44,6 +44,7 @@ #include "tools/filter/type_filter.h" #include "tools/filter/encodings2regdeps_filter.h" #include "tools/filter/func_id_filter.h" +#include "tools/filter/invalidate_cpu_filter.h" #include "trace_entry.h" #include "zipfile_ostream.h" @@ -600,6 +601,70 @@ test_func_id_filter() return true; } +static bool +test_invalidate_cpu_filter() +{ + constexpr addr_t PC = 0x7f6fdd3ec360; + constexpr addr_t ENCODING = 0xe78948; + std::vector entries = { + /* Trace shard header. + */ + { { TRACE_TYPE_HEADER, 0, { 0x1 } }, true, { true } }, + { { TRACE_TYPE_MARKER, TRACE_MARKER_TYPE_VERSION, { 0x2 } }, true, { true } }, + { { TRACE_TYPE_MARKER, + TRACE_MARKER_TYPE_FILETYPE, + { OFFLINE_FILE_TYPE_ARCH_X86_64 | OFFLINE_FILE_TYPE_ENCODINGS | + OFFLINE_FILE_TYPE_SYSCALL_NUMBERS | OFFLINE_FILE_TYPE_BLOCKING_SYSCALLS } }, + true, + { true } }, + { { TRACE_TYPE_THREAD, 0, { 0x4 } }, true, { true } }, + { { TRACE_TYPE_PID, 0, { 0x5 } }, true, { true } }, + { { TRACE_TYPE_MARKER, TRACE_MARKER_TYPE_CACHE_LINE_SIZE, { 0x6 } }, + true, + { true } }, + + { { TRACE_TYPE_MARKER, TRACE_MARKER_TYPE_TIMESTAMP, { 0x7 } }, true, { true } }, + { { TRACE_TYPE_MARKER, TRACE_MARKER_TYPE_CPU_ID, { 0x8 } }, true, { false } }, + // invalidate_cpu_filter overwrites the value of TRACE_MARKER_TYPE_CPU_ID with + // (uintptr_t)-1. + { { TRACE_TYPE_MARKER, TRACE_MARKER_TYPE_CPU_ID, { 0xffffffffffffffff } }, + false, + { true } }, + /* We need at least one instruction with encodings to make record_filter output + * the trace. + */ + { { TRACE_TYPE_ENCODING, 3, { ENCODING } }, true, { true } }, + { { TRACE_TYPE_INSTR, 3, { PC } }, true, { true } }, + + { { TRACE_TYPE_FOOTER, 0, { 0x0 } }, true, { true } }, + }; + + /* Construct invalidate_cpu_filter_t. + */ + std::vector> filters; + auto invalidate_cpu_filter = std::unique_ptr( + new dynamorio::drmemtrace::invalidate_cpu_filter_t()); + if (!invalidate_cpu_filter->get_error_string().empty()) { + fprintf(stderr, "Couldn't construct an invalidate_cpu_filter %s", + invalidate_cpu_filter->get_error_string().c_str()); + return false; + } + filters.push_back(std::move(invalidate_cpu_filter)); + + /* Construct record_filter_t. + */ + auto record_filter = std::unique_ptr( + new test_record_filter_t(std::move(filters), 0, /*write_archive=*/true)); + + /* Run the test. + */ + if (!process_entries_and_check_result(record_filter.get(), entries, 0)) + return false; + + fprintf(stderr, "test_invalidate_cpu_filter passed\n"); + return true; +} + static bool test_cache_and_type_filter() { @@ -1450,7 +1515,7 @@ test_main(int argc, const char *argv[]) dr_standalone_init(); if (!test_cache_and_type_filter() || !test_chunk_update() || !test_trim_filter() || !test_null_filter() || !test_wait_filter() || !test_encodings2regdeps_filter() || - !test_func_id_filter()) + !test_func_id_filter() || !test_invalidate_cpu_filter()) return 1; fprintf(stderr, "All done!\n"); dr_standalone_exit(); diff --git a/clients/drcachesim/tools/filter/invalidate_cpu_filter.h b/clients/drcachesim/tools/filter/invalidate_cpu_filter.h new file mode 100644 index 00000000000..a06934aece3 --- /dev/null +++ b/clients/drcachesim/tools/filter/invalidate_cpu_filter.h @@ -0,0 +1,92 @@ +/* ********************************************************** + * Copyright (c) 2024 Google, Inc. All rights reserved. + * **********************************************************/ + +/* + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the name of Google, Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH + * DAMAGE. + */ + +#ifndef _INVALIDATE_CPU_FILTER_H_ +#define _INVALIDATE_CPU_FILTER_H_ 1 + +#include "record_filter.h" +#include "trace_entry.h" + +#include + +#define INVALID_CPU_MARKER_VALUE ~((addr_t)0UL) + +namespace dynamorio { +namespace drmemtrace { + +/* This filter invalidates the value of TRACE_MARKER_TYPE_CPU_ID by setting its value to + * (uintptr_t)-1, which indicates that the CPU could not be determined. + */ +class invalidate_cpu_filter_t : public record_filter_t::record_filter_func_t { +public: + invalidate_cpu_filter_t() + { + } + + void * + parallel_shard_init(memtrace_stream_t *shard_stream, + bool partial_trace_filter) override + { + return nullptr; + } + + bool + parallel_shard_filter( + trace_entry_t &entry, void *shard_data, + record_filter_t::record_filter_info_t &record_filter_info) override + { + trace_type_t entry_type = static_cast(entry.type); + // Output any trace_entry_t that it's not a marker. + if (entry_type != TRACE_TYPE_MARKER) + return true; + + trace_marker_type_t marker_type = static_cast(entry.size); + // Output any trace_entry_t that it's not a CPU marker. + if (marker_type != TRACE_MARKER_TYPE_CPU_ID) + return true; + + // Invalidate CPU marker value. + entry.addr = INVALID_CPU_MARKER_VALUE; + + return true; + } + + bool + parallel_shard_exit(void *shard_data) override + { + return true; + } +}; + +} // namespace drmemtrace +} // namespace dynamorio +#endif /* _INCALIDATE_CPU_FILTER_H_ */ diff --git a/clients/drcachesim/tools/filter/record_filter.cpp b/clients/drcachesim/tools/filter/record_filter.cpp index 9d038583b61..8ce3ffab496 100644 --- a/clients/drcachesim/tools/filter/record_filter.cpp +++ b/clients/drcachesim/tools/filter/record_filter.cpp @@ -62,6 +62,7 @@ #include "type_filter.h" #include "encodings2regdeps_filter.h" #include "func_id_filter.h" +#include "invalidate_cpu_filter.h" #undef VPRINT #ifdef DEBUG @@ -119,7 +120,7 @@ record_filter_tool_create(const std::string &output_dir, uint64_t stop_timestamp const std::string &remove_marker_types, uint64_t trim_before_timestamp, uint64_t trim_after_timestamp, bool encodings2regdeps, const std::string &keep_func_ids, - unsigned int verbose) + bool invalidate_cpu, unsigned int verbose) { std::vector< std::unique_ptr> @@ -160,6 +161,11 @@ record_filter_tool_create(const std::string &output_dir, uint64_t stop_timestamp std::unique_ptr( new dynamorio::drmemtrace::func_id_filter_t(keep_func_ids_list))); } + if (invalidate_cpu) { + filter_funcs.emplace_back( + std::unique_ptr( + new dynamorio::drmemtrace::invalidate_cpu_filter_t())); + } // TODO i#5675: Add other filters. diff --git a/clients/drcachesim/tools/filter/record_filter_create.h b/clients/drcachesim/tools/filter/record_filter_create.h index 0516d7bb713..10edf952361 100644 --- a/clients/drcachesim/tools/filter/record_filter_create.h +++ b/clients/drcachesim/tools/filter/record_filter_create.h @@ -67,6 +67,8 @@ namespace drmemtrace { * @param[in] keep_func_ids A comma-separated list of integers representing the * function IDs related to #TRACE_MARKER_TYPE_FUNC_ID (and _ARG, _RETVAL, _RETADDR) * markers to preserve in the trace, while removing all other function markers. + * @param[in] invalidate_cpu Set value of TRACE_MARKER_TYPE_CPU_ID to (uintptr_t)-1, + * which represents CPU unknown. * @param[in] verbose Verbosity level for notifications. */ record_analysis_tool_t * @@ -75,7 +77,7 @@ record_filter_tool_create(const std::string &output_dir, uint64_t stop_timestamp const std::string &remove_marker_types, uint64_t trim_before_timestamp, uint64_t trim_after_timestamp, bool encodings2regdeps, const std::string &keep_func_ids, - unsigned int verbose); + bool invalidate_cpu, unsigned int verbose); } // namespace drmemtrace } // namespace dynamorio diff --git a/clients/drcachesim/tools/record_filter_launcher.cpp b/clients/drcachesim/tools/record_filter_launcher.cpp index e1a8bb7a481..2118ae27f06 100644 --- a/clients/drcachesim/tools/record_filter_launcher.cpp +++ b/clients/drcachesim/tools/record_filter_launcher.cpp @@ -138,6 +138,12 @@ droption_t "TRACE_MARKER_TYPE_FUNC_[ID | ARG | RETVAL | RETADDR] " "markers for the listed function IDs and removed those " "belonging to unlisted function IDs."); + +droption_t op_invalidate_cpu( + DROPTION_SCOPE_FRONTEND, "filter_invalidate_cpu", false, + "Invalidate TRACE_MARKER_TYPE_CPU_ID", + "Invalidate TRACE_MARKER_TYPE_CPU_ID by setting its value to (uintptr_t)-1."); + } // namespace int @@ -168,7 +174,8 @@ _tmain(int argc, const TCHAR *targv[]) op_cache_filter_size.get_value(), op_remove_trace_types.get_value(), op_remove_marker_types.get_value(), op_trim_before_timestamp.get_value(), op_trim_after_timestamp.get_value(), op_encodings2regdeps.get_value(), - op_filter_func_ids.get_value(), op_verbose.get_value())); + op_filter_func_ids.get_value(), op_invalidate_cpu.get_value(), + op_verbose.get_value())); std::vector tools; tools.push_back(record_filter.get()); diff --git a/suite/tests/CMakeLists.txt b/suite/tests/CMakeLists.txt index 577fe842833..5c9ed83743b 100644 --- a/suite/tests/CMakeLists.txt +++ b/suite/tests/CMakeLists.txt @@ -4790,6 +4790,14 @@ if (BUILD_CLIENTS) # We run basic_counts on the filtered trace to check that there are no function # related markers left. "basic_counts") + + set(testname "tool.record_filter_invalidate_cpu") + torun_record_filter("${testname}" ${kernel_xfer_app} + "record_filter_invalidate_cpu" + "${drcachesim_path}@-simulator_type@record_filter@-filter_invalidate_cpu@-indir@${testname}.${kernel_xfer_app}.*.dir/trace@-outdir@${testname}.filtered.dir" + # We run basic_counts on the filtered trace to check that there are no function + # related markers left. + "view") endif () if (X86 AND X64 AND UNIX AND NOT APPLE)