Skip to content

Commit

Permalink
i#7031: invalidate CPU marker filter
Browse files Browse the repository at this point in the history
In some cases we don't want to expose the "as traced" CPU schedule
in an offline trace because it might not be representative of the
native execution of the traced program.

To do so we implement `invalidate_cpu_filter_t`, a new filter that
is used with `record_filter` as follows:
```
drrun -t drmemtrace -tool record_filter -filter_invalidate_cpu -indir path/to/input/trace -outdir path/to/output/trace
```
This filter set the value of TRACE_MARKER_TYPE_CPU_ID markers to
(uintptr_t)-1, which representes an unknown CPU.

We add a unit test `test_invalidate_cpu_filter()` and an end-to-end
test `code_api|tool.record_filter_invalidate_cpu` which invokes the
`invariant_checker` and `view` tool on the filtered trace.

Fixes #7031
  • Loading branch information
edeiana committed Oct 9, 2024
1 parent e3fbf73 commit 3ab20b0
Show file tree
Hide file tree
Showing 12 changed files with 221 additions and 5 deletions.
4 changes: 4 additions & 0 deletions api/docs/release.dox
Original file line number Diff line number Diff line change
Expand Up @@ -265,6 +265,10 @@ Further non-compatibility-affecting changes include:
- Added -trace_instr_intervals_file option to the drmemtrace trace analysis tools
framework. The file must be in CSV format containing a <start,duration> tracing
interval per line where start and duration are expressed in number of instructions.
- Added invalidate_cpu_filter_t to #dynamorio::drmemtrace::record_filter_t to invalidate
the value of markers #dynamorio::drmemtrace::TRACE_MARKER_TYPE_CPU_ID. When
-filter_invalidate_cpu is used, the value of those markers is set to (uintptr_t)-1,
which means "undefined".

**************************************************
<hr>
Expand Down
1 change: 1 addition & 0 deletions clients/drcachesim/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,7 @@ add_exported_library(drmemtrace_record_filter STATIC
tools/filter/type_filter.h
tools/filter/encodings2regdeps_filter.h
tools/filter/func_id_filter.h
tools/filter/invalidate_cpu_filter.h
tools/filter/null_filter.h)
target_link_libraries(drmemtrace_record_filter drmemtrace_simulator
drmemtrace_schedule_file)
Expand Down
3 changes: 2 additions & 1 deletion clients/drcachesim/analyzer_multi.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -339,7 +339,8 @@ record_analyzer_multi_t::create_analysis_tool_from_options(const std::string &to
op_filter_cache_size.get_value(), op_filter_trace_types.get_value(),
op_filter_marker_types.get_value(), op_trim_before_timestamp.get_value(),
op_trim_after_timestamp.get_value(), op_encodings2regdeps.get_value(),
op_filter_func_ids.get_value(), op_verbose.get_value());
op_filter_func_ids.get_value(), op_invalidate_cpu.get_value(),
op_verbose.get_value());
}
ERRMSG("Usage error: unsupported record analyzer type \"%s\". Only " RECORD_FILTER
" is supported.\n",
Expand Down
5 changes: 5 additions & 0 deletions clients/drcachesim/common/options.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1112,6 +1112,11 @@ droption_t<std::string>
"for the listed function IDs and removes those belonging to "
"unlisted function IDs.");

droption_t<bool> op_invalidate_cpu(
DROPTION_SCOPE_FRONTEND, "filter_invalidate_cpu", false,
"Invalidate TRACE_MARKER_TYPE_CPU_ID",
"Invalidate TRACE_MARKER_TYPE_CPU_ID by setting its value to (uintptr_t)-1.");

droption_t<uint64_t> op_trim_before_timestamp(
DROPTION_SCOPE_ALL, "trim_before_timestamp", 0, 0,
(std::numeric_limits<uint64_t>::max)(),
Expand Down
1 change: 1 addition & 0 deletions clients/drcachesim/common/options.h
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,7 @@ extern dynamorio::droption::droption_t<std::string> op_filter_trace_types;
extern dynamorio::droption::droption_t<std::string> op_filter_marker_types;
extern dynamorio::droption::droption_t<bool> op_encodings2regdeps;
extern dynamorio::droption::droption_t<std::string> op_filter_func_ids;
extern dynamorio::droption::droption_t<bool> op_invalidate_cpu;
extern dynamorio::droption::droption_t<uint64_t> op_trim_before_timestamp;
extern dynamorio::droption::droption_t<uint64_t> op_trim_after_timestamp;
extern dynamorio::droption::droption_t<bool> op_abort_on_invariant_error;
Expand Down
24 changes: 24 additions & 0 deletions clients/drcachesim/tests/record_filter_invalidate_cpu.templatex
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
Estimation of pi is 3.142425985001098

Trace invariant checks passed

Output .* entries from .* entries.

Output format:

<--record#-> <--instr#->: <---tid---> <record details>

------------------------------------------------------------

1 0: +[0-9]+ <marker: version [0-9]>
2 0: +[0-9]+ <marker: filetype 0x[0-9a-f]*>
3 0: +[0-9]+ <marker: cache line size [0-9]*>
4 0: +[0-9]+ <marker: chunk instruction count [0-9]*>
5 0: +[0-9]+ <marker: page size [0-9]*>
6 0: +[0-9]+ <marker: timestamp [0-9]*>
#ifdef X64
7 0: +[0-9]+ <marker: tid [0-9]* on core 18446744073709551615>
#else
7 0: +[0-9]+ <marker: tid [0-9]* on core 4294967295>
#endif
.*
67 changes: 66 additions & 1 deletion clients/drcachesim/tests/record_filter_unit_tests.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
#include "tools/filter/type_filter.h"
#include "tools/filter/encodings2regdeps_filter.h"
#include "tools/filter/func_id_filter.h"
#include "tools/filter/invalidate_cpu_filter.h"
#include "trace_entry.h"
#include "zipfile_ostream.h"

Expand Down Expand Up @@ -600,6 +601,70 @@ test_func_id_filter()
return true;
}

static bool
test_invalidate_cpu_filter()
{
constexpr addr_t PC = 0x7f6fdd3ec360;
constexpr addr_t ENCODING = 0xe78948;
std::vector<test_case_t> entries = {
/* Trace shard header.
*/
{ { TRACE_TYPE_HEADER, 0, { 0x1 } }, true, { true } },
{ { TRACE_TYPE_MARKER, TRACE_MARKER_TYPE_VERSION, { 0x2 } }, true, { true } },
{ { TRACE_TYPE_MARKER,
TRACE_MARKER_TYPE_FILETYPE,
{ OFFLINE_FILE_TYPE_ARCH_X86_64 | OFFLINE_FILE_TYPE_ENCODINGS |
OFFLINE_FILE_TYPE_SYSCALL_NUMBERS | OFFLINE_FILE_TYPE_BLOCKING_SYSCALLS } },
true,
{ true } },
{ { TRACE_TYPE_THREAD, 0, { 0x4 } }, true, { true } },
{ { TRACE_TYPE_PID, 0, { 0x5 } }, true, { true } },
{ { TRACE_TYPE_MARKER, TRACE_MARKER_TYPE_CACHE_LINE_SIZE, { 0x6 } },
true,
{ true } },

{ { TRACE_TYPE_MARKER, TRACE_MARKER_TYPE_TIMESTAMP, { 0x7 } }, true, { true } },
{ { TRACE_TYPE_MARKER, TRACE_MARKER_TYPE_CPU_ID, { 0x8 } }, true, { false } },
// invalidate_cpu_filter overwrites the value of TRACE_MARKER_TYPE_CPU_ID with
// (uintptr_t)-1.
{ { TRACE_TYPE_MARKER, TRACE_MARKER_TYPE_CPU_ID, { 0xffffffffffffffff } },
false,
{ true } },
/* We need at least one instruction with encodings to make record_filter output
* the trace.
*/
{ { TRACE_TYPE_ENCODING, 3, { ENCODING } }, true, { true } },
{ { TRACE_TYPE_INSTR, 3, { PC } }, true, { true } },

{ { TRACE_TYPE_FOOTER, 0, { 0x0 } }, true, { true } },
};

/* Construct invalidate_cpu_filter_t.
*/
std::vector<std::unique_ptr<record_filter_func_t>> filters;
auto invalidate_cpu_filter = std::unique_ptr<record_filter_func_t>(
new dynamorio::drmemtrace::invalidate_cpu_filter_t());
if (!invalidate_cpu_filter->get_error_string().empty()) {
fprintf(stderr, "Couldn't construct an invalidate_cpu_filter %s",
invalidate_cpu_filter->get_error_string().c_str());
return false;
}
filters.push_back(std::move(invalidate_cpu_filter));

/* Construct record_filter_t.
*/
auto record_filter = std::unique_ptr<test_record_filter_t>(
new test_record_filter_t(std::move(filters), 0, /*write_archive=*/true));

/* Run the test.
*/
if (!process_entries_and_check_result(record_filter.get(), entries, 0))
return false;

fprintf(stderr, "test_invalidate_cpu_filter passed\n");
return true;
}

static bool
test_cache_and_type_filter()
{
Expand Down Expand Up @@ -1450,7 +1515,7 @@ test_main(int argc, const char *argv[])
dr_standalone_init();
if (!test_cache_and_type_filter() || !test_chunk_update() || !test_trim_filter() ||
!test_null_filter() || !test_wait_filter() || !test_encodings2regdeps_filter() ||
!test_func_id_filter())
!test_func_id_filter() || !test_invalidate_cpu_filter())
return 1;
fprintf(stderr, "All done!\n");
dr_standalone_exit();
Expand Down
92 changes: 92 additions & 0 deletions clients/drcachesim/tools/filter/invalidate_cpu_filter.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
/* **********************************************************
* Copyright (c) 2024 Google, Inc. All rights reserved.
* **********************************************************/

/*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* * Neither the name of Google, Inc. nor the names of its contributors may be
* used to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
* DAMAGE.
*/

#ifndef _INVALIDATE_CPU_FILTER_H_
#define _INVALIDATE_CPU_FILTER_H_ 1

#include "record_filter.h"
#include "trace_entry.h"

#include <cstring>

#define INVALID_CPU_MARKER_VALUE ~((addr_t)0UL)

namespace dynamorio {
namespace drmemtrace {

/* This filter invalidates the value of TRACE_MARKER_TYPE_CPU_ID by setting its value to
* (uintptr_t)-1, which indicates that the CPU could not be determined.
*/
class invalidate_cpu_filter_t : public record_filter_t::record_filter_func_t {
public:
invalidate_cpu_filter_t()
{
}

void *
parallel_shard_init(memtrace_stream_t *shard_stream,
bool partial_trace_filter) override
{
return nullptr;
}

bool
parallel_shard_filter(
trace_entry_t &entry, void *shard_data,
record_filter_t::record_filter_info_t &record_filter_info) override
{
trace_type_t entry_type = static_cast<trace_type_t>(entry.type);
// Output any trace_entry_t that it's not a marker.
if (entry_type != TRACE_TYPE_MARKER)
return true;

trace_marker_type_t marker_type = static_cast<trace_marker_type_t>(entry.size);
// Output any trace_entry_t that it's not a CPU marker.
if (marker_type != TRACE_MARKER_TYPE_CPU_ID)
return true;

// Invalidate CPU marker value.
entry.addr = INVALID_CPU_MARKER_VALUE;

return true;
}

bool
parallel_shard_exit(void *shard_data) override
{
return true;
}
};

} // namespace drmemtrace
} // namespace dynamorio
#endif /* _INCALIDATE_CPU_FILTER_H_ */
8 changes: 7 additions & 1 deletion clients/drcachesim/tools/filter/record_filter.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@
#include "type_filter.h"
#include "encodings2regdeps_filter.h"
#include "func_id_filter.h"
#include "invalidate_cpu_filter.h"

#undef VPRINT
#ifdef DEBUG
Expand Down Expand Up @@ -119,7 +120,7 @@ record_filter_tool_create(const std::string &output_dir, uint64_t stop_timestamp
const std::string &remove_marker_types,
uint64_t trim_before_timestamp, uint64_t trim_after_timestamp,
bool encodings2regdeps, const std::string &keep_func_ids,
unsigned int verbose)
bool invalidate_cpu, unsigned int verbose)
{
std::vector<
std::unique_ptr<dynamorio::drmemtrace::record_filter_t::record_filter_func_t>>
Expand Down Expand Up @@ -160,6 +161,11 @@ record_filter_tool_create(const std::string &output_dir, uint64_t stop_timestamp
std::unique_ptr<dynamorio::drmemtrace::record_filter_t::record_filter_func_t>(
new dynamorio::drmemtrace::func_id_filter_t(keep_func_ids_list)));
}
if (invalidate_cpu) {
filter_funcs.emplace_back(
std::unique_ptr<dynamorio::drmemtrace::record_filter_t::record_filter_func_t>(
new dynamorio::drmemtrace::invalidate_cpu_filter_t()));
}

// TODO i#5675: Add other filters.

Expand Down
4 changes: 3 additions & 1 deletion clients/drcachesim/tools/filter/record_filter_create.h
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,8 @@ namespace drmemtrace {
* @param[in] keep_func_ids A comma-separated list of integers representing the
* function IDs related to #TRACE_MARKER_TYPE_FUNC_ID (and _ARG, _RETVAL, _RETADDR)
* markers to preserve in the trace, while removing all other function markers.
* @param[in] invalidate_cpu Set value of TRACE_MARKER_TYPE_CPU_ID to (uintptr_t)-1,
* which represents CPU unknown.
* @param[in] verbose Verbosity level for notifications.
*/
record_analysis_tool_t *
Expand All @@ -75,7 +77,7 @@ record_filter_tool_create(const std::string &output_dir, uint64_t stop_timestamp
const std::string &remove_marker_types,
uint64_t trim_before_timestamp, uint64_t trim_after_timestamp,
bool encodings2regdeps, const std::string &keep_func_ids,
unsigned int verbose);
bool invalidate_cpu, unsigned int verbose);

} // namespace drmemtrace
} // namespace dynamorio
Expand Down
9 changes: 8 additions & 1 deletion clients/drcachesim/tools/record_filter_launcher.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,12 @@ droption_t<std::string>
"TRACE_MARKER_TYPE_FUNC_[ID | ARG | RETVAL | RETADDR] "
"markers for the listed function IDs and removed those "
"belonging to unlisted function IDs.");

droption_t<bool> op_invalidate_cpu(
DROPTION_SCOPE_FRONTEND, "filter_invalidate_cpu", false,
"Invalidate TRACE_MARKER_TYPE_CPU_ID",
"Invalidate TRACE_MARKER_TYPE_CPU_ID by setting its value to (uintptr_t)-1.");

} // namespace

int
Expand Down Expand Up @@ -168,7 +174,8 @@ _tmain(int argc, const TCHAR *targv[])
op_cache_filter_size.get_value(), op_remove_trace_types.get_value(),
op_remove_marker_types.get_value(), op_trim_before_timestamp.get_value(),
op_trim_after_timestamp.get_value(), op_encodings2regdeps.get_value(),
op_filter_func_ids.get_value(), op_verbose.get_value()));
op_filter_func_ids.get_value(), op_invalidate_cpu.get_value(),
op_verbose.get_value()));
std::vector<record_analysis_tool_t *> tools;
tools.push_back(record_filter.get());

Expand Down
8 changes: 8 additions & 0 deletions suite/tests/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -4790,6 +4790,14 @@ if (BUILD_CLIENTS)
# We run basic_counts on the filtered trace to check that there are no function
# related markers left.
"basic_counts")

set(testname "tool.record_filter_invalidate_cpu")
torun_record_filter("${testname}" ${kernel_xfer_app}
"record_filter_invalidate_cpu"
"${drcachesim_path}@-simulator_type@record_filter@-filter_invalidate_cpu@-indir@${testname}.${kernel_xfer_app}.*.dir/trace@-outdir@${testname}.filtered.dir"
# We run basic_counts on the filtered trace to check that there are no function
# related markers left.
"view")
endif ()

if (X86 AND X64 AND UNIX AND NOT APPLE)
Expand Down

0 comments on commit 3ab20b0

Please sign in to comment.