Merge branch 'master' into i6417-add-ci-x86-32-denylist

api/docs/CMake_rundoxygen.cmake

@@ -256,6 +256,11 @@ if (embeddable)
   endforeach ()
 
   # We leverage the javascript menu from the non-embedded version.
+  # Menu hierarchy changes:
+  #   (1) Remove the outer "DynamoRIO" layer.
+  #   (2) Remove home page at the end.
+  #   (3) Insert an "API Reference" layer around API details.
+  #   (4) Move the "Extension API" menu inside "Build Your Own Tool".
   file(GLOB all_js ${CMAKE_CURRENT_BINARY_DIR}/../html/*.js)
   set(found_user_docs OFF)
   foreach (js ${all_js})
@@ -286,7 +291,31 @@ if (embeddable)
         message(FATAL_ERROR "Cannot find menu entry for page \"Extension API\"")
       endif ()
       string(REGEX MATCH "\n[^\n]+\"Extension API\"[^\n]+\n" ext_entry "${string}")
-      string(REPLACE "${ext_entry}" "\n" string "${string}")
+      # If ext_entry ends with a [, the menu items are expanded inline and the
+      # whole Extension API menu can be moved within this file.
+      string(REGEX MATCH "\\[\n" ext_multiline "${ext_entry}")
+      if (ext_multiline)
+        # Extract everything up to but not including the newly-inserted API Reference.
+        string(REGEX REPLACE
+               "(\n[^\n]+\"Extension API\",.+)(\n\\[ \"API Reference\")"
+               "\\2" string "${string}")
+        # Remember the full Extension API match.
+        set(ext_entry ${CMAKE_MATCH_1})
+        if (NOT ext_entry)
+          message(FATAL_ERROR "Cannot move menu entry for \"Extension API\"")
+        endif ()
+        # Now insert the Extension API menu right above Disassembly Library.
+        if (NOT string MATCHES "Disassembly Library")
+          message(FATAL_ERROR "Cannot find menu entry for page \"Disassembly Library\"")
+        endif ()
+        string(REGEX REPLACE "(\n[^\n]+\"Disassembly Library)" "${ext_entry}\\1"
+          string "${string}")
+        # Set the user_docs flag since we found its contents here.
+        set(found_user_docs ON)
+      else ()
+        # Remove the 1-line Extension API string from this file.
+        string(REPLACE "${ext_entry}" "\n" string "${string}")
+      endif ()
     else ()
       # Remove name so we can inline.
       string(REGEX REPLACE "var [^\n]* =\n" "" string "${string}")
@@ -319,6 +348,9 @@ if (embeddable)
     if (js MATCHES "page_user_docs.js")
       # CMake 3.6+ guarantees the glob is sorted lexicographically, so we've already
       # seen navtreedata.js.
+      if (found_user_docs)
+        message(FATAL_ERROR "Found unexpected \"page_user_docs\" menu file")
+      endif ()
       set(found_user_docs ON)
       if (ext_entry)
         if (NOT string MATCHES "Disassembly Library")

api/docs/test_suite.dox

@@ -94,7 +94,7 @@ failure email step".  Be sure to match the surrounding indentation as
 indentation matters for .yml files.
 
     - name: Setup tmate session
-      if: ${{ failure() }}
+      if: failure()
       uses: mxschmitt/action-tmate@v3
 
 Next, delete all the other workflow files in the ".github/workflows/" directory.

clients/drcachesim/scheduler/scheduler.cpp

@@ -809,10 +809,6 @@ scheduler_tmpl_t<RecordType, ReaderType>::set_initial_schedule(
     // The filetype, if present, is before the first timestamp.  If we only need the
     // filetype we avoid going as far as the timestamp.
     bool gather_filetype = options_.read_inputs_in_init;
-    // Avoid reading ahead for replay as it makes the input ords not match in tests.
-    if (options_.mapping == MAP_TO_RECORDED_OUTPUT &&
-        options_.replay_as_traced_istream != nullptr)
-        gather_filetype = false;
     if (gather_filetype || gather_timestamps) {
         sched_type_t::scheduler_status_t res =
             get_initial_input_content(gather_timestamps);
@@ -1043,7 +1039,11 @@ scheduler_tmpl_t<RecordType, ReaderType>::read_recorded_schedule()
     }
     for (int i = 0; i < static_cast<output_ordinal_t>(outputs_.size()); ++i) {
         if (outputs_[i].record.empty()) {
+            // XXX i#6630: We should auto-set the output count and avoid
+            // having extra outputs; these complicate idle computations, etc.
+            VPRINT(this, 1, "output %d empty: returning eof up front\n", i);
             set_cur_input(i, INVALID_INPUT_ORDINAL);
+            outputs_[i].at_eof = true;
         } else if (outputs_[i].record[0].type == schedule_record_t::IDLE) {
             set_cur_input(i, INVALID_INPUT_ORDINAL);
             outputs_[i].waiting = true;
@@ -1205,8 +1205,13 @@ scheduler_tmpl_t<RecordType, ReaderType>::read_traced_schedule()
                        outputs_[output_idx].record[0].key.input);
                 set_cur_input(output_idx, outputs_[output_idx].record[0].key.input);
             }
-        } else
+        } else {
+            // XXX i#6630: We should auto-set the output count and avoid
+            // having extra ouputs; these complicate idle computations, etc.
+            VPRINT(this, 1, "Output %d empty: returning eof up front\n", output_idx);
+            outputs_[output_idx].at_eof = true;
             set_cur_input(output_idx, INVALID_INPUT_ORDINAL);
+        }
     }
     return STATUS_SUCCESS;
 }

clients/drcachesim/tests/record_filter_start_idle.templatex

@@ -0,0 +1,7 @@
+Output .* entries from .* entries.
+Schedule stats tool results:
+.*
+Core #0 schedule: .*
+Core #1 schedule: .*
+Core #2 schedule: .*
+Core #3 schedule: .*

clients/drcachesim/tests/scheduler_unit_tests.cpp

@@ -3423,7 +3423,10 @@ test_replay_as_traced_from_file(const char *testdir)
     static const char *const SCHED_STRING =
         "Core #0: 1257598 \nCore #1: 1257603 \nCore #2: 1257601 \n"
         "Core #3: 1257599 => 1257604 @ <366987,87875,13331862029895453> "
-        "(<366986,87875,13331862029895453> => <1,0,0>) \n"
+        // The ordinal is really 1 ("<1,0,0>") but with the scheduler's readahead
+        // it becomes 2; easier to just check for that as trying to avoid readahead
+        // causes other problems with start-idle cores (i#6721).
+        "(<366986,87875,13331862029895453> => <2,0,0>) \n"
         "Core #4: 1257600 \nCore #5: 1257596 \nCore #6: 1257602 \n";
     static constexpr int NUM_OUTPUTS = 7; // Matches the actual trace's core footprint.
     scheduler_t scheduler;

clients/drcachesim/tools/filter/record_filter.cpp

@@ -208,6 +208,10 @@ record_filter_t::initialize_shard_output(per_shard_t *per_shard,
         // Now synchronize determining the extension.
         auto lock = std::unique_lock<std::mutex>(input_info_mutex_);
         if (!output_ext_.empty()) {
+            VPRINT(this, 2,
+                   "Shard #%d using pre-set ext=%s, ver=%" PRIu64 ", type=%" PRIu64 "\n",
+                   shard_stream->get_shard_index(), output_ext_.c_str(), version_,
+                   filetype_);
             per_shard->output_path += output_ext_;
             lock.unlock();
         } else if (!input_name.empty()) {
@@ -218,12 +222,25 @@ record_filter_t::initialize_shard_output(per_shard_t *per_shard,
             // Set the other key input data.
             version_ = shard_stream->get_version();
             filetype_ = add_to_filetype(shard_stream->get_filetype());
+            if (version_ == 0) {
+                // We give up support for version 0 to have an up-front error check
+                // rather than having some output files with bad headers (i#6721).
+                return "Version not available at shard init time";
+            }
+            VPRINT(this, 2,
+                   "Shard #%d setting ext=%s, ver=%" PRIu64 ", type=%" PRIu64 "\n",
+                   shard_stream->get_shard_index(), output_ext_.c_str(), version_,
+                   filetype_);
             per_shard->output_path += output_ext_;
             lock.unlock();
             input_info_cond_var_.notify_all();
         } else {
             // We have to wait for another shard with an input to set output_ext_.
             input_info_cond_var_.wait(lock, [this] { return !output_ext_.empty(); });
+            VPRINT(this, 2,
+                   "Shard #%d waited for ext=%s, ver=%" PRIu64 ", type=%" PRIu64 "\n",
+                   shard_stream->get_shard_index(), output_ext_.c_str(), version_,
+                   filetype_);
             per_shard->output_path += output_ext_;
             lock.unlock();
         }
@@ -649,6 +666,10 @@ record_filter_t::process_delayed_encodings(per_shard_t *per_shard, trace_entry_t
 bool
 record_filter_t::parallel_shard_memref(void *shard_data, const trace_entry_t &input_entry)
 {
+    if (!success_) {
+        // Report an error that happened during shard init.
+        return false;
+    }
     per_shard_t *per_shard = reinterpret_cast<per_shard_t *>(shard_data);
     ++per_shard->input_entry_count;
     trace_entry_t entry = input_entry;

clients/drcachesim/tools/invariant_checker.cpp

@@ -421,6 +421,9 @@ invariant_checker_t::parallel_shard_memref(void *shard_data, const memref_t &mem
         }
 #endif
         shard->expect_syscall_marker_ = false;
+        // TODO i#5949,i#6599: On WOW64 the "syscall" call is delayed by raw2trace
+        // acros the 2nd timestamp+cpuid; we disable this check until that is solved.
+#if !defined(WINDOWS) || defined(X64)
         // We expect an immediately preceding timestamp + cpuid.
         if (shard->trace_version_ >= TRACE_ENTRY_VERSION_FREQUENT_TIMESTAMPS) {
             report_if_false(
@@ -432,6 +435,7 @@ invariant_checker_t::parallel_shard_memref(void *shard_data, const memref_t &mem
                         TRACE_MARKER_TYPE_TIMESTAMP,
                 "Syscall marker not preceded by timestamp + cpuid");
         }
+#endif
     }
     if (memref.marker.type == TRACE_TYPE_MARKER &&
         memref.marker.marker_type == TRACE_MARKER_TYPE_MAYBE_BLOCKING_SYSCALL) {

suite/runsuite_wrapper.pl

@@ -222,6 +222,7 @@
                 'code_api|tool.drcacheoff.burst_traceopts' => 1, # i#4622
                 'code_api|tool.drcacheoff.burst_replaceall' => 1, # i#4622
                 'code_api|tool.drcacheoff.burst_static' => 1, # i#4486
+                'code_api|tool.drcacheoff.windows-timestamps' => 1, # i#6081
                 'code_api|api.symtest' => 1, # i#4131
                 'code_api|client.drwrap-test-detach' => 1, # i#4616
                 'code_api|client.cbr4' => 1, # i#4792
@@ -246,10 +247,6 @@
                 'code_api,thread_private,disable_traces|common.decode-stress' => 1, # i#1807
                 'code_api,thread_private,tracedump_binary|common.decode-stress' => 1, # i#1807
                 'code_api|client.file_io' => 1, # i#5802
-                'code_api|tool.drcacheoff.windows-invar' => 1, # i#6599
-                'code_api|tool.drcacheoff.invariant_checker' => 1, # i#6599
-                'code_api|tool.drcacheoff.getretaddr_record_replace_retaddr' => 1, # i#6599
-                'code_api|tool.record_filter' => 1, # i#6599
                 );
 
             %ignore_failures_64 = (

suite/tests/CMakeLists.txt

@@ -4696,6 +4696,24 @@ if (BUILD_CLIENTS)
       set(tool.record_filter_as_traced_basedir
         "${PROJECT_SOURCE_DIR}/clients/drcachesim/tests")
       set(tool.record_filter_as_traced_rawtemp ON) # no preprocessor
+
+      # Test the record_filter in as-traced mode with start-idle cores.
+      # This checked-in partial-hello,world-trace ran on 4 different cores.
+      set(trace_dir
+        "${PROJECT_SOURCE_DIR}/clients/drcachesim/tests/drmemtrace.as_traced.x64.tracedir")
+      set(sched_file "${trace_dir}/cpu_schedule.bin.zip")
+      set(outdir ${CMAKE_CURRENT_BINARY_DIR}/filter_start_idle)
+      file(MAKE_DIRECTORY ${outdir})
+      torunonly_api(tool.record_filter_start_idle "${drcachesim_path}"
+        "record_filter_start_idle"
+        "" "-simulator_type;schedule_stats;-indir;${outdir}" OFF OFF)
+      set(tool.record_filter_start_idle_runcmp "${CMAKE_CURRENT_SOURCE_DIR}/runmulti.cmake")
+      set(tool.record_filter_start_idle_precmd
+        "${drcachesim_path}@-simulator_type@record_filter@-cpu_schedule_file@${sched_file}@-core_sharded@-cores@4@-indir@${trace_dir}@-outdir@${outdir}")
+      set(tool.record_filter_start_idle_basedir
+        "${PROJECT_SOURCE_DIR}/clients/drcachesim/tests")
+      set(tool.record_filter_start_idle_rawtemp ON) # no preprocessor
+
     endif ()
 
     if (AARCH64)