add a lock checker for debugging multithread simulation

Makefile

@@ -11,16 +11,16 @@ CXXSTD = --std=c++17 -fconcepts
 
 ifeq ($(MODE), release)
     CXXFLAGS = $(CXXSTD) -O3 -DNDEBUG -I. -fPIC
+	REGRESS_LD_FLAGS =
 else ifeq ($(MODE), debug)
-    CXXFLAGS = $(CXXSTD) -O0 -g -I. -fPIC
+    CXXFLAGS = $(CXXSTD) -O0 -g -I. -fPIC -DCHECK_MULTI
+	REGRESS_LD_FLAGS =
+else ifeq ($(MODE), debug-multi)
+    CXXFLAGS = $(CXXSTD) -O0 -g -I. -fPIC -DCHECK_MULTI -DBOOST_STACKTRACE_LINK -DBOOST_STACKTRACE_USE_BACKTRACE
+	REGRESS_LD_FLAGS = -lboost_stacktrace_backtrace -ldl -lbacktrace
 else
     CXXFLAGS = $(CXXSTD) -O2 -I. -fPIC
-endif
-
-ifeq ($(MODE), debug)
-    DSLCXXFLAGS = $(CXXSTD) -O1 -g -I. -fPIC
-else
-    DSLCXXFLAGS = $(CXXSTD) -O2 -I. -fPIC
+	REGRESS_LD_FLAGS =
 endif
 
 UTIL_HEADERS  = $(wildcard util/*.hpp)
@@ -69,7 +69,7 @@ PARALLEL_REGRESSION_TESTS_EXE = $(patsubst %, regression/%, $(PARALLEL_REGRESSIO
 PARALLEL_REGRESSION_TESTS_RST = $(patsubst %, regression/%.out, $(PARALLEL_REGRESSION_TESTS))
 
 $(PARALLEL_REGRESSION_TESTS_EXE): %:%.cpp $(CACHE_OBJS) $(UTIL_OBJS) $(CRYPTO_LIB) $(CACHE_HEADERS)
-	$(CXX) $(CXXFLAGS) $< $(CACHE_OBJS) $(UTIL_OBJS) $(CRYPTO_LIB) -o $@
+	$(CXX) $(CXXFLAGS) $< $(CACHE_OBJS) $(UTIL_OBJS) $(CRYPTO_LIB) $(REGRESS_LD_FLAGS) -o $@
 
 $(PARALLEL_REGRESSION_TESTS_RST): %.out: %
 	timeout 1m $< 2>$@ 1>temp.log

cache/metadata.hpp

@@ -1,13 +1,9 @@
 #ifndef CM_CACHE_METADATA_HPP
 #define CM_CACHE_METADATA_HPP
 
-#include <cstdint>
 #include <string>
 #include "util/concept_macro.hpp"
-#include <mutex>
-#include <atomic>
-#include <thread>
-#include <cassert>
+#include "util/multithread.hpp"
 
 class CMDataBase
 {
@@ -220,31 +216,34 @@ template <typename MT> requires C_DERIVE<MT, CMMetadataCommon>
 class MetaLock : public MT {
   std::mutex mtx;
 
-#ifndef NDEBUG
+#ifdef CHECK_MULTI
   // verify no double lock or unlock
-  std::hash<std::thread::id> hasher;
   std::atomic<uint64_t> locked;
 #endif
 
 public:
   MetaLock() : MT() {}
   virtual ~MetaLock() {}
   virtual void lock() {
-#ifndef NDEBUG
-    uint64_t thread_id = hasher(std::this_thread::get_id());
+#ifdef CHECK_MULTI
+    uint64_t thread_id = global_lock_checker->thread_id();
     assert(locked.load() != thread_id || 0 ==
             "This cache line has already be locked by this thread and should not be locked by this thread again!");
 #endif
     mtx.lock();
-#ifndef NDEBUG
+#ifdef CHECK_MULTI
+    global_lock_checker->push(this);
     locked = thread_id;
 #endif
   }
+
   virtual void unlock() {
-#ifndef NDEBUG
+#ifdef CHECK_MULTI
+    uint64_t thread_id = global_lock_checker->thread_id();
     assert(locked.load() != 0 || 0 ==
            "This cache line has already be unlocked and should not be unlocked again!");
     locked = 0;
+    global_lock_checker->pop(this);
 #endif
     mtx.unlock();
   }

regression/multi-l2-msi.cpp

@@ -23,6 +23,11 @@
 //#define SAddrN 64
 //#define TestN 512
 
+PrintPool *globalPrinter;
+#ifdef CHECK_MULTI
+  LockCheck * global_lock_checker = new LockCheck;
+#endif
+
 int main(){
   auto l1d = cache_gen_multi_thread_l1<L1IW, L1WN, Data64B, MetadataBroadcastBase, ReplaceLRU_MT, MSIMultiThreadPolicy, false, false, void, true>(NCore, "l1d");
   auto core_data = get_l1_core_interface(l1d);
@@ -31,6 +36,7 @@ int main(){
 
   auto l2 = cache_gen_multi_thread_l2<L2IW, L2WN, Data64B, MetadataBroadcastBase, ReplaceLRU_MT, MSIMultiThreadPolicy, true, void, true>(1, "l2")[0];
   auto mem = new SimpleMemoryModel<Data64B, void, true, true>("mem");
+  globalPrinter = new PrintPool(256);
   SimpleTracerMT tracer(true);
 
   for(int i=0; i<NCore; i++) {

util/monitor.hpp

@@ -238,20 +238,19 @@ class SimpleTracer : public MonitorBase
 // multithread version of simple tracer
 class SimpleTracerMT : public SimpleTracer
 {
-  PrintPool pool;
   std::thread print_thread;
   std::hash<std::thread::id> hasher;
   virtual void print(std::string& msg) {
     uint16_t id = hasher(std::this_thread::get_id());
     std::string msg_ext = (boost::format("thread %04x: %s") % id % msg).str();
-    pool.add(msg_ext);
+    globalPrinter->add(msg_ext);
   }
 
 public:
-  SimpleTracerMT(bool cd = false): SimpleTracer(cd), pool(256) {
-    print_thread = std::thread(&PrintPool::print, &pool);
+  SimpleTracerMT(bool cd = false): SimpleTracer(cd){
+    print_thread = std::thread(&PrintPool::print, globalPrinter);
   }
-  virtual void stop() { pool.stop(); print_thread.join(); }
+  virtual void stop() { globalPrinter->stop(); print_thread.join(); }
 };
 
 #endif

util/multithread.hpp

@@ -2,15 +2,22 @@
 #define CM_UTIL_MULTITHREAD_HPP
 
 #include <unordered_map>
+#include <stack>
 #include <cstdint>
 #include <cassert>
 #include <atomic>
 #include <mutex>
+#include <shared_mutex>
 #include <condition_variable>
 #include <memory>
 #include <chrono>
+#include <thread>
 #include <iostream>
 
+#ifdef BOOST_STACKTRACE_LINK
+#include <boost/stacktrace.hpp>
+#endif
+
 template<typename T>
 class AtomicVar {
   std::unique_ptr<std::atomic<T> > var;
@@ -107,5 +114,78 @@ class PendingXact<false> {
   }
 };
 
+class LockCheck {
+  std::hash<std::thread::id> hasher;
+  std::mutex hasher_mtx;
+
+  #ifdef BOOST_STACKTRACE_LINK
+    std::unordered_map<uint64_t, std::stack<std::pair<void *, std::string> > > lock_map; // lock should always behave like a stack
+  #else
+    std::unordered_map<uint64_t, std::stack<void *> > lock_map;
+  #endif
+
+  std::shared_mutex lock_map_mtx;
+
+public:
+  uint64_t thread_id() {
+    std::lock_guard lock(hasher_mtx);
+    return hasher(std::this_thread::get_id());
+  }
+
+  void push(void *p) {
+    bool hit;
+    auto id = thread_id();
+    {
+      std::shared_lock lock(lock_map_mtx);
+      hit = lock_map.count(id);
+      if(hit) {
+        #ifdef BOOST_STACKTRACE_LINK
+          lock_map[id].push(std::make_pair(p, boost::stacktrace::to_string(boost::stacktrace::stacktrace())));
+        #else
+          lock_map[id].push(p);
+        #endif
+      }
+    }
+
+    if(!hit) {
+      std::lock_guard lock(lock_map_mtx);
+      #ifdef BOOST_STACKTRACE_LINK
+        lock_map[id].push(std::make_pair(p, boost::stacktrace::to_string(boost::stacktrace::stacktrace())));
+      #else
+        lock_map[id].push(p);
+      #endif
+    }
+  }
+
+  void pop(void *p) {
+    auto id = thread_id();
+    std::shared_lock lock(lock_map_mtx);
+    assert(lock_map.count(id));
+    #ifdef BOOST_STACKTRACE_LINK
+      auto [pm, trace] = lock_map[id].top();
+      assert(p == pm);
+    #else
+      assert(p == lock_map[id].top());
+    #endif
+    lock_map[id].pop();
+  }
+
+  void check() {
+    auto id = thread_id();
+    std::shared_lock lock(lock_map_mtx);
+    #ifdef BOOST_STACKTRACE_LINK
+    if(lock_map.count(id) && lock_map[id].size()) {
+      auto [p, trace] = lock_map[id].top();
+      std::cout << "metadata: " << (uint64_t)(p) << " is kept locked by thread " << id << std::endl;
+      std::cout << trace << std::endl;
+    }
+    #endif
+    assert(!lock_map.count(id) || lock_map[id].size() == 0);
+  }
+};
+
+#ifdef CHECK_MULTI
+extern LockCheck * global_lock_checker;
+#endif
 
 #endif

util/print.hpp

@@ -7,7 +7,6 @@
 #include <iostream>
 #include "util/multithread.hpp"
 
-
 // can be implemented using std::osyncstream after C++20
 class PrintPool {
   const int pool_size;
@@ -53,4 +52,11 @@ class PrintPool {
   }
 };
 
+extern PrintPool *globalPrinter;
+
+inline void global_print(std::string msg) {
+  if(globalPrinter) globalPrinter->add(msg);
+  else std::cout << msg << std::endl;
+}
+
 #endif