i#6662 public traces, part 2: encoding_filter

.github/workflows/ci-docs.yml

@@ -90,7 +90,7 @@ jobs:
       # We only use a non-zero build # when making multiple manual builds in one day.
       run: |
         if test -z "${{ github.event.inputs.version }}"; then
-          export VERSION_NUMBER=10.90.$((`git log -n 1 --format=%ct` / (60*60*24)))
+          export VERSION_NUMBER=10.91.$((`git log -n 1 --format=%ct` / (60*60*24)))
         else
           export VERSION_NUMBER=${{ github.event.inputs.version }}
         fi

.github/workflows/ci-package.yml

@@ -103,7 +103,7 @@ jobs:
       # We only use a non-zero build # when making multiple manual builds in one day.
       run: |
         if test -z "${{ github.event.inputs.version }}"; then
-          export VERSION_NUMBER=10.90.$((`git log -n 1 --format=%ct` / (60*60*24)))
+          export VERSION_NUMBER=10.91.$((`git log -n 1 --format=%ct` / (60*60*24)))
         else
           export VERSION_NUMBER=${{ github.event.inputs.version }}
         fi
@@ -195,7 +195,7 @@ jobs:
       # XXX: See x86 job comments on sharing the default ver# with CMakeLists.txt.
       run: |
         if test -z "${{ github.event.inputs.version }}"; then
-          export VERSION_NUMBER=10.90.$((`git log -n 1 --format=%ct` / (60*60*24)))
+          export VERSION_NUMBER=10.91.$((`git log -n 1 --format=%ct` / (60*60*24)))
         else
           export VERSION_NUMBER=${{ github.event.inputs.version }}
         fi
@@ -283,7 +283,7 @@ jobs:
       # XXX: See x86 job comments on sharing the default ver# with CMakeLists.txt.
       run: |
         if test -z "${{ github.event.inputs.version }}"; then
-          export VERSION_NUMBER=10.90.$((`git log -n 1 --format=%ct` / (60*60*24)))
+          export VERSION_NUMBER=10.91.$((`git log -n 1 --format=%ct` / (60*60*24)))
         else
           export VERSION_NUMBER=${{ github.event.inputs.version }}
         fi
@@ -371,7 +371,7 @@ jobs:
       # XXX: See x86 job comments on sharing the default ver# with CMakeLists.txt.
       run: |
         if test -z "${{ github.event.inputs.version }}"; then
-          export VERSION_NUMBER=10.90.$((`git log -n 1 --format=%ct` / (60*60*24)))
+          export VERSION_NUMBER=10.91.$((`git log -n 1 --format=%ct` / (60*60*24)))
         else
           export VERSION_NUMBER=${{ github.event.inputs.version }}
         fi
@@ -451,7 +451,7 @@ jobs:
       # XXX: See x86 job comments on sharing the default ver# with CMakeLists.txt.
       run: |
         if test -z "${{ github.event.inputs.version }}"; then
-          export VERSION_NUMBER=10.90.$((`git log -n 1 --format=%ct` / (60*60*24)))
+          export VERSION_NUMBER=10.91.$((`git log -n 1 --format=%ct` / (60*60*24)))
         else
           export VERSION_NUMBER=${{ github.event.inputs.version }}
         fi
@@ -536,7 +536,7 @@ jobs:
       # XXX: See x86 job comments on sharing the default ver# with CMakeLists.txt.
       run: |
         if test -z "${{ github.event.inputs.version }}"; then
-          export VERSION_NUMBER="10.90.$((`git log -n 1 --format=%ct` / (60*60*24)))"
+          export VERSION_NUMBER="10.91.$((`git log -n 1 --format=%ct` / (60*60*24)))"
           export PREFIX="cronbuild-"
         else
           export VERSION_NUMBER=${{ github.event.inputs.version }}

CMakeLists.txt

@@ -575,7 +575,7 @@ endif (EXISTS "${PROJECT_SOURCE_DIR}/.svn")
 
 # N.B.: When updating this, update all the default versions in ci-package.yml
 # and ci-docs.yml.  We should find a way to share (xref i#1565).
-set(VERSION_NUMBER_DEFAULT "10.90.${VERSION_NUMBER_PATCHLEVEL}")
+set(VERSION_NUMBER_DEFAULT "10.91.${VERSION_NUMBER_PATCHLEVEL}")
 # do not store the default VERSION_NUMBER in the cache to prevent a stale one
 # from preventing future version updates in a pre-existing build dir
 set(VERSION_NUMBER "" CACHE STRING "Version number: leave empty for default")

api/docs/release.dox

@@ -150,6 +150,10 @@ changes:
  - Changed the type of the AArch64 #dr_mcontext_t members svep and ffr to #dr_svep_t.
    This breaks binary compatibility with clients that were built against versions of
    DynamoRIO before this change.
+ - Changed #dynamorio::drmemtrace::record_filter_t::record_filter_func_t
+   parallel_shard_filter() interface.  Added a new parameter of type
+   #dynamorio::drmemtrace::record_filter_t::record_filter_info_t that allows
+   #dynamorio::drmemtrace::record_filter_t to share data with its filters.
 
 Further non-compatibility-affecting changes include:
  - Added DWARF-5 support to the drsyms library by linking in 4 static libraries
@@ -226,6 +230,10 @@ Further non-compatibility-affecting changes include:
    purpose of preserving register dependencies.
  - Added instr_convert_to_isa_regdeps() API that converts an #instr_t from a real ISA
    (e.g., #DR_ISA_AMD64) to the #DR_ISA_REGDEPS synthetic ISA.
+ - Added encodings2regdeps_t filter to #dynamorio::drmemtrace::record_filter_t to generate
+   #DR_ISA_REGDEPS traces.
+ - Added #dynamorio::drmemtrace::OFFLINE_FILE_TYPE_ARCH_REGDEPS file type for
+   #DR_ISA_REGDEPS traces.
 
 **************************************************
 <hr>

clients/drcachesim/CMakeLists.txt

@@ -192,12 +192,14 @@ add_exported_library(drmemtrace_simulator STATIC
   )
 
 add_exported_library(drmemtrace_record_filter STATIC
+  tools/filter/record_filter.h
   tools/filter/record_filter.cpp
   tools/filter/cache_filter.h
   tools/filter/cache_filter.cpp
   tools/filter/type_filter.h
+  tools/filter/encodings2regdeps.h
   tools/filter/null_filter.h)
-target_link_libraries(drmemtrace_record_filter drmemtrace_simulator)
+target_link_libraries(drmemtrace_record_filter drmemtrace_simulator dynamorio)
 
 add_exported_library(directory_iterator STATIC common/directory_iterator.cpp)
 add_dependencies(directory_iterator api_headers)
@@ -357,6 +359,8 @@ install_client_nonDR_header(drmemtrace simulator/cache_simulator_create.h)
 install_client_nonDR_header(drmemtrace simulator/tlb_simulator_create.h)
 install_client_nonDR_header(drmemtrace tools/view_create.h)
 install_client_nonDR_header(drmemtrace tools/func_view_create.h)
+install_client_nonDR_header(drmemtrace tools/filter/record_filter_create.h)
+install_client_nonDR_header(drmemtrace tools/filter/record_filter.h)
 # TODO i#6412: Create a separate directory for non-tracer headers so that
 # we can more cleanly separate tracer and raw2trace code.
 install_client_nonDR_header(drmemtrace tracer/raw2trace.h)
@@ -393,8 +397,12 @@ add_executable(record_filter_launcher
   tools/record_filter_launcher.cpp
   tests/test_helpers.cpp)
 target_link_libraries(record_filter_launcher drmemtrace_analyzer drmemtrace_record_filter)
+add_dependencies(record_filter_launcher api_headers)
 append_property_list(TARGET record_filter_launcher COMPILE_DEFINITIONS "NO_HELPER_MAIN")
 use_DynamoRIO_extension(record_filter_launcher droption)
+if (NOT APPLE)
+  configure_DynamoRIO_static(record_filter_launcher)
+endif()
 
 # We want to use test_helper's disable_popups() but we have _tmain and so do not want
 # the test_helper library's main symbol: so we compile ourselves and disable.

clients/drcachesim/analyzer_multi.cpp

@@ -334,7 +334,8 @@ record_analyzer_multi_t::create_analysis_tool_from_options(
             op_outdir.get_value(), op_filter_stop_timestamp.get_value(),
             op_filter_cache_size.get_value(), op_filter_trace_types.get_value(),
             op_filter_marker_types.get_value(), op_trim_before_timestamp.get_value(),
-            op_trim_after_timestamp.get_value(), op_verbose.get_value());
+            op_trim_after_timestamp.get_value(), op_encodings2regdeps.get_value(),
+            op_verbose.get_value());
     }
     ERRMSG("Usage error: unsupported record analyzer type \"%s\".  Only " RECORD_FILTER
            " is supported.\n",

clients/drcachesim/common/options.cpp

@@ -972,6 +972,16 @@ droption_t<std::string>
                            "Comma-separated integers for marker types to remove. "
                            "See trace_marker_type_t for the list of marker types.");
 
+/* XXX i#6369: we should partition our options by tool. This one should belong to the
+ * record_filter partition. For now we add the filter_ prefix to options that should be
+ * used in conjunction with record_filter.
+ */
+droption_t<bool> op_encodings2regdeps(
+    DROPTION_SCOPE_FRONTEND, "filter_encodings2regdeps", false,
+    "Enable converting the encoding of instructions to synthetic ISA DR_ISA_REGDEPS.",
+    "This option is for -simulator_type " RECORD_FILTER ". When present, it converts "
+    "the encoding of instructions from a real ISA to the DR_ISA_REGDEPS synthetic ISA.");
+
 droption_t<uint64_t> op_trim_before_timestamp(
     DROPTION_SCOPE_ALL, "trim_before_timestamp", 0, 0,
     (std::numeric_limits<uint64_t>::max)(),

clients/drcachesim/common/options.h

@@ -214,6 +214,7 @@ extern dynamorio::droption::droption_t<uint64_t> op_filter_stop_timestamp;
 extern dynamorio::droption::droption_t<int> op_filter_cache_size;
 extern dynamorio::droption::droption_t<std::string> op_filter_trace_types;
 extern dynamorio::droption::droption_t<std::string> op_filter_marker_types;
+extern dynamorio::droption::droption_t<bool> op_encodings2regdeps;
 extern dynamorio::droption::droption_t<uint64_t> op_trim_before_timestamp;
 extern dynamorio::droption::droption_t<uint64_t> op_trim_after_timestamp;
 extern dynamorio::droption::droption_t<bool> op_abort_on_invariant_error;

clients/drcachesim/common/trace_entry.h

@@ -665,6 +665,18 @@ type_is_instr(const trace_type_t type)
         type == TRACE_TYPE_INSTR_UNTAKEN_JUMP;
 }
 
+/**
+ * Returns whether \p type represents any type of instruction record whether an
+ * instruction fetch or operation hint. This is a superset of type_is_instr() and includes
+ * #TRACE_TYPE_INSTR_NO_FETCH.
+ */
+static inline bool
+is_any_instr_type(const trace_type_t type)
+{
+    return type_is_instr(type) || type == TRACE_TYPE_INSTR_MAYBE_FETCH ||
+        type == TRACE_TYPE_INSTR_NO_FETCH;
+}
+
 /** Returns whether the type represents the fetch of a branch instruction. */
 static inline bool
 type_is_instr_branch(const trace_type_t type)
@@ -889,9 +901,6 @@ typedef enum {
     OFFLINE_FILE_TYPE_ARCH_ARM32 = 0x10,       /**< Recorded on ARM (32-bit). */
     OFFLINE_FILE_TYPE_ARCH_X86_32 = 0x20,      /**< Recorded on x86 (32-bit). */
     OFFLINE_FILE_TYPE_ARCH_X86_64 = 0x40,      /**< Recorded on x86 (64-bit). */
-    OFFLINE_FILE_TYPE_ARCH_ALL = OFFLINE_FILE_TYPE_ARCH_AARCH64 |
-        OFFLINE_FILE_TYPE_ARCH_ARM32 | OFFLINE_FILE_TYPE_ARCH_X86_32 |
-        OFFLINE_FILE_TYPE_ARCH_X86_64, /**< All possible architecture types. */
     /**
      * Instruction addresses filtered online.
      * Note: this file type may transition to non-filtered. If so, the transition is
@@ -968,19 +977,36 @@ typedef enum {
      * Each trace shard represents one core and contains interleaved software threads.
      */
     OFFLINE_FILE_TYPE_CORE_SHARDED = 0x10000,
+    /**
+     * Trace filtered by the record_filter tool using -filter_encodings2regdeps.
+     * The encodings2regdeps filter replaces real ISA encodings with #DR_ISA_REGDEPS
+     * encodings. Note that these encoding changes do not update the instruction length,
+     * hence encoding size and instruction fetch size may not match.
+     */
+    OFFLINE_FILE_TYPE_ARCH_REGDEPS = 0x20000,
+    /**
+     * All possible architecture types, including synthetic ones.
+     */
+    OFFLINE_FILE_TYPE_ARCH_ALL = OFFLINE_FILE_TYPE_ARCH_AARCH64 |
+        OFFLINE_FILE_TYPE_ARCH_ARM32 | OFFLINE_FILE_TYPE_ARCH_X86_32 |
+        OFFLINE_FILE_TYPE_ARCH_X86_64 | OFFLINE_FILE_TYPE_ARCH_REGDEPS,
 } offline_file_type_t;
 
 static inline const char *
 trace_arch_string(offline_file_type_t type)
 {
-    return TESTANY(OFFLINE_FILE_TYPE_ARCH_AARCH64, type)
-        ? "aarch64"
-        : (TESTANY(OFFLINE_FILE_TYPE_ARCH_ARM32, type)
-               ? "arm"
-               : (TESTANY(OFFLINE_FILE_TYPE_ARCH_X86_32, type)
-                      ? "i386"
-                      : (TESTANY(OFFLINE_FILE_TYPE_ARCH_X86_64, type) ? "x86_64"
-                                                                      : "unspecified")));
+    if (TESTANY(OFFLINE_FILE_TYPE_ARCH_AARCH64, type))
+        return "aarch64";
+    else if (TESTANY(OFFLINE_FILE_TYPE_ARCH_ARM32, type))
+        return "arm";
+    else if (TESTANY(OFFLINE_FILE_TYPE_ARCH_X86_32, type))
+        return "i386";
+    else if (TESTANY(OFFLINE_FILE_TYPE_ARCH_X86_64, type))
+        return "x86_64";
+    else if (TESTANY(OFFLINE_FILE_TYPE_ARCH_REGDEPS, type))
+        return "regdeps";
+    else
+        return "unspecified";
 }
 
 /* We have non-client targets including this header that do not include API

clients/drcachesim/reader/reader.cpp

@@ -210,7 +210,12 @@ reader_t::process_input_entry()
                 ++cur_instr_count_;
             // Look for encoding bits that belong to this instr.
             if (last_encoding_.size > 0) {
-                if (last_encoding_.size != cur_ref_.instr.size) {
+                if (last_encoding_.size != cur_ref_.instr.size &&
+                    /* OFFLINE_FILE_TYPE_ARCH_REGDEPS traces have encodings with
+                     * size != ifetch. It's a design choice, not an error, hence
+                     * we avoid this sanity check for these traces.
+                     */
+                    !TESTANY(OFFLINE_FILE_TYPE_ARCH_REGDEPS, filetype_)) {
                     ERRMSG(
                         "Encoding size %zu != instr size %zu for PC 0x%zx at ord %" PRIu64
                         " instr %" PRIu64 " last_timestamp=0x%" PRIx64 "\n",

clients/drcachesim/tests/record_filter_encodings2regdeps.templatex

@@ -0,0 +1,11 @@
+Hello, world!
+
+Trace invariant checks passed
+
+Output .* entries from .* entries.
+
+Opcode mix tool results:
+
+    *[0-9]* : total executed instructions
+    *[0-9]* : <undecoded>
+.*

clients/drcachesim/tests/record_filter_unit_tests.cpp

@@ -42,6 +42,8 @@
 #include "tools/filter/record_filter.h"
 #include "tools/filter/trim_filter.h"
 #include "tools/filter/type_filter.h"
+#include "tools/filter/encodings2regdeps.h"
+#include "trace_entry.h"
 #include "zipfile_ostream.h"
 
 #include <inttypes.h>
@@ -287,6 +289,75 @@ process_entries_and_check_result(test_record_filter_t *record_filter,
     return true;
 }
 
+static bool
+test_encodings2regdeps_filter()
+{
+    std::vector<test_case_t> entries = {
+        // Trace shard header.
+        { { TRACE_TYPE_HEADER, 0, { 0x1 } }, true, { true } },
+        { { TRACE_TYPE_MARKER, TRACE_MARKER_TYPE_VERSION, { 0x2 } }, true, { true } },
+        // File type, modified by the record_filter encodings2regdeps.
+        { { TRACE_TYPE_MARKER,
+            TRACE_MARKER_TYPE_FILETYPE,
+            { OFFLINE_FILE_TYPE_ARCH_X86_64 | OFFLINE_FILE_TYPE_ENCODINGS |
+              OFFLINE_FILE_TYPE_SYSCALL_NUMBERS | OFFLINE_FILE_TYPE_BLOCKING_SYSCALLS } },
+          true,
+          { false } },
+        { { TRACE_TYPE_MARKER,
+            TRACE_MARKER_TYPE_FILETYPE,
+            { OFFLINE_FILE_TYPE_ARCH_REGDEPS | OFFLINE_FILE_TYPE_ENCODINGS |
+              OFFLINE_FILE_TYPE_SYSCALL_NUMBERS | OFFLINE_FILE_TYPE_BLOCKING_SYSCALLS } },
+          false,
+          { true } },
+        { { TRACE_TYPE_THREAD, 0, { 0x4 } }, true, { true } },
+        { { TRACE_TYPE_PID, 0, { 0x5 } }, true, { true } },
+        { { TRACE_TYPE_MARKER, TRACE_MARKER_TYPE_CACHE_LINE_SIZE, { 0x6 } },
+          true,
+          { true } },
+        // Unit header.
+        { { TRACE_TYPE_MARKER, TRACE_MARKER_TYPE_TIMESTAMP, { 0x7 } }, true, { true } },
+        { { TRACE_TYPE_MARKER, TRACE_MARKER_TYPE_CPU_ID, { 0x8 } }, true, { true } },
+        // Encoding, modified by the record_filter encodings2regdeps.
+        { { TRACE_TYPE_ENCODING, 4, { 0xe78948 } }, true, { false } },
+        { { TRACE_TYPE_ENCODING, 8, { 0x0006090600010011 } }, false, { true } },
+        { { TRACE_TYPE_INSTR, 3, { 0x7f6fdd3ec360 } }, true, { true } },
+        // Same instr same chunk.
+        { { TRACE_TYPE_INSTR, 3, { 0x7f6fdd3ec360 } }, true, { true } },
+        // Unit header.
+        { { TRACE_TYPE_MARKER, TRACE_MARKER_TYPE_TIMESTAMP, { 0xc } }, true, { true } },
+        { { TRACE_TYPE_MARKER, TRACE_MARKER_TYPE_CPU_ID, { 0xd } }, true, { true } },
+        // Duplicated encoding across chunk boundary.
+        { { TRACE_TYPE_ENCODING, 4, { 0xe78948 } }, true, { false } },
+        { { TRACE_TYPE_ENCODING, 8, { 0x0006090600010011 } }, false, { true } },
+        { { TRACE_TYPE_INSTR, 3, { 0x7f6fdd3ec360 } }, true, { true } },
+        // Unit header.
+        { { TRACE_TYPE_MARKER, TRACE_MARKER_TYPE_TIMESTAMP, { 0xe } }, true, { true } },
+        { { TRACE_TYPE_MARKER, TRACE_MARKER_TYPE_CPU_ID, { 0xf } }, true, { true } },
+        // Same instr in different chunk.
+        { { TRACE_TYPE_INSTR, 3, { 0x7f6fdd3ec360 } }, true, { true } },
+        // Trace shard footer.
+        { { TRACE_TYPE_FOOTER, 0, { 0x0 } }, true, { true } },
+    };
+
+    // Construct record_filter_t.
+    std::vector<std::unique_ptr<record_filter_func_t>> filters;
+    auto encodings2regdeps_filter = std::unique_ptr<record_filter_func_t>(
+        new dynamorio::drmemtrace::encodings2regdeps_t());
+    if (encodings2regdeps_filter->get_error_string() != "") {
+        fprintf(stderr, "Couldn't construct a encodings2regdeps_filter %s",
+                encodings2regdeps_filter->get_error_string().c_str());
+        return false;
+    }
+    filters.push_back(std::move(encodings2regdeps_filter));
+    auto record_filter = std::unique_ptr<test_record_filter_t>(
+        new test_record_filter_t(std::move(filters), 0));
+    if (!process_entries_and_check_result(record_filter.get(), entries, 0))
+        return false;
+
+    fprintf(stderr, "test_encodings2regdeps_filter passed\n");
+    return true;
+}
+
 static bool
 test_cache_and_type_filter()
 {
@@ -541,7 +612,9 @@ test_chunk_update()
                 return nullptr;
             }
             bool
-            parallel_shard_filter(trace_entry_t &entry, void *shard_data) override
+            parallel_shard_filter(
+                trace_entry_t &entry, void *shard_data,
+                record_filter_t::record_filter_info_t &record_filter_info) override
             {
                 bool res = true;
                 if (type_is_instr(static_cast<trace_type_t>(entry.type))) {
@@ -1105,7 +1178,7 @@ test_main(int argc, const char *argv[])
                     droption_parser_t::usage_short(DROPTION_SCOPE_ALL).c_str());
     }
     if (!test_cache_and_type_filter() || !test_chunk_update() || !test_trim_filter() ||
-        !test_null_filter() || !test_wait_filter())
+        !test_null_filter() || !test_wait_filter() || !test_encodings2regdeps_filter())
         return 1;
     fprintf(stderr, "All done!\n");
     return 0;

clients/drcachesim/tools/filter/cache_filter.cpp

@@ -87,7 +87,9 @@ cache_filter_t::parallel_shard_init(memtrace_stream_t *shard_stream,
     return per_shard;
 }
 bool
-cache_filter_t::parallel_shard_filter(trace_entry_t &entry, void *shard_data)
+cache_filter_t::parallel_shard_filter(
+    trace_entry_t &entry, void *shard_data,
+    record_filter_t::record_filter_info_t &record_filter_info)
 {
     if (entry.type == TRACE_TYPE_MARKER && entry.size == TRACE_MARKER_TYPE_FILETYPE) {
         if (filter_instrs_)

clients/drcachesim/tools/filter/cache_filter.h

@@ -55,7 +55,9 @@ class cache_filter_t : public record_filter_t::record_filter_func_t {
     parallel_shard_init(memtrace_stream_t *shard_stream,
                         bool partial_trace_filter) override;
     bool
-    parallel_shard_filter(trace_entry_t &entry, void *shard_data) override;
+    parallel_shard_filter(
+        trace_entry_t &entry, void *shard_data,
+        record_filter_t::record_filter_info_t &record_filter_info) override;
     bool
     parallel_shard_exit(void *shard_data) override;