diff --git a/cache/cache.hpp b/cache/cache.hpp index 7044305..5d98c01 100644 --- a/cache/cache.hpp +++ b/cache/cache.hpp @@ -316,8 +316,8 @@ class CacheSkewed : public CacheBase } } - virtual void hook_read(uint64_t addr, uint32_t ai, uint32_t s, uint32_t w, bool hit, const CMMetadataBase * meta, const CMDataBase *data, uint64_t *delay) override { - if(ai < P) replacer[ai].access(s, w, true, false); + virtual void hook_read(uint64_t addr, uint32_t ai, uint32_t s, uint32_t w, bool hit, bool prefetch, const CMMetadataBase * meta, const CMDataBase *data, uint64_t *delay) override { + if(ai < P) replacer[ai].access(s, w, true, prefetch); if constexpr (EnMon || !C_VOID) monitors->hook_read(addr, ai, s, w, hit, meta, data, delay); } diff --git a/cache/coh_policy.hpp b/cache/coh_policy.hpp index 9642050..a441110 100644 --- a/cache/coh_policy.hpp +++ b/cache/coh_policy.hpp @@ -30,6 +30,7 @@ namespace coh { const uint32_t evict_act = 2; const uint32_t writeback_act = 3; const uint32_t downgrade_act = 4; + const uint32_t prefetch_act = 5; // message type constexpr inline bool is_acquire(coh_cmd_t cmd) { return cmd.msg == acquire_msg; } @@ -45,10 +46,12 @@ namespace coh { constexpr inline bool is_writeback(coh_cmd_t cmd) { return cmd.act == writeback_act; } constexpr inline bool is_downgrade(coh_cmd_t cmd) { return cmd.act == downgrade_act; } constexpr inline bool is_write(coh_cmd_t cmd) { return cmd.act == fetch_write_act || cmd.act == evict_act || cmd.act == writeback_act; } + constexpr inline bool is_prefetch(coh_cmd_t cmd) { return cmd.act == prefetch_act; } // generate command constexpr inline coh_cmd_t cmd_for_read() { return {-1, acquire_msg, fetch_read_act }; } constexpr inline coh_cmd_t cmd_for_write() { return {-1, acquire_msg, fetch_write_act}; } + constexpr inline coh_cmd_t cmd_for_prefetch() { return {-1, acquire_msg, prefetch_act }; } constexpr inline coh_cmd_t cmd_for_flush() { return {-1, flush_msg, evict_act }; } constexpr inline coh_cmd_t cmd_for_writeback() { return {-1, flush_msg, writeback_act }; } constexpr inline coh_cmd_t cmd_for_release() { return {-1, release_msg, evict_act }; } diff --git a/cache/coherence.hpp b/cache/coherence.hpp index 39fc8c1..00c7437 100644 --- a/cache/coherence.hpp +++ b/cache/coherence.hpp @@ -47,6 +47,7 @@ class OuterCohPortBase virtual void acquire_req(uint64_t addr, CMMetadataBase *meta, CMDataBase *data, coh_cmd_t cmd, uint64_t *delay) = 0; virtual void writeback_req(uint64_t addr, CMMetadataBase *meta, CMDataBase *data, coh_cmd_t cmd, uint64_t *delay) = 0; + virtual void prefetch_req(uint64_t addr, coh_cmd_t cmd, uint64_t *delay) = 0; // may not implement probe_resp() and finish_req() if the port is uncached virtual std::pair probe_resp(uint64_t addr, CMMetadataBase *meta, CMDataBase *data, coh_cmd_t cmd, uint64_t *delay) { return std::make_pair(false,false); } @@ -77,6 +78,7 @@ class InnerCohPortBase virtual void acquire_resp(uint64_t addr, CMDataBase *data_inner, CMMetadataBase *meta_inner, coh_cmd_t outer_cmd, uint64_t *delay) = 0; virtual void writeback_resp(uint64_t addr, CMDataBase *data_inner, CMMetadataBase *meta_inner, coh_cmd_t outer_cmd, uint64_t *delay) = 0; + virtual void prefetch_resp(uint64_t addr, coh_cmd_t outer_cmd, uint64_t *delay) = 0; // may not implement probe_req() and finish_resp() if the port is uncached virtual std::pair probe_req(uint64_t addr, CMMetadataBase *meta, CMDataBase *data, coh_cmd_t cmd, uint64_t *delay) { return std::make_pair(false,false); } @@ -142,6 +144,11 @@ class OuterCohPortUncached : public OuterCohPortBase Policy::meta_after_writeback(outer_cmd, meta); } + virtual void prefetch_req(uint64_t addr, coh_cmd_t outer_cmd, uint64_t *delay) override { + outer_cmd.id = coh_id; + coh->prefetch_resp(addr, outer_cmd, delay); + } + virtual void query_loc_req(uint64_t addr, std::list *locs) override { coh->query_loc_resp(addr, locs); } @@ -221,7 +228,7 @@ class InnerCohPortUncached : public InnerCohPortBase if (data_inner && data) data_inner->copy(data); Policy::meta_after_grant(cmd, meta, meta_inner); - cache->hook_read(addr, ai, s, w, hit, meta, data, delay); + cache->hook_read(addr, ai, s, w, hit, coh::is_prefetch(cmd), meta, data, delay); finish_record(addr, coh::cmd_for_finish(cmd.id), !hit, meta, ai, s); if(cmd.id == -1) finish_resp(addr, coh::cmd_for_finish(cmd.id)); } @@ -233,6 +240,10 @@ class InnerCohPortUncached : public InnerCohPortBase write_line(addr, data_inner, meta_inner, cmd, delay); } + virtual void prefetch_resp(uint64_t addr, coh_cmd_t cmd, uint64_t *delay) override { + prefetch_line(addr, cmd, delay); + } + virtual void query_loc_resp(uint64_t addr, std::list *locs) override { outer->query_loc_req(addr, locs); locs->push_front(cache->query_loc(addr)); @@ -254,24 +265,26 @@ class InnerCohPortUncached : public InnerCohPortBase cache->hook_manage(addr, ai, s, w, true, true, writeback.first, meta, data, delay); } - virtual std::tuple - access_line(uint64_t addr, coh_cmd_t cmd, uint16_t prio, uint64_t *delay) { // common function for access a line in the cache + virtual std::tuple + check_hit_or_replace(uint64_t addr, uint16_t prio, bool do_replace, bool prefetch, uint64_t *delay) { // check hit or get a replacement target uint32_t ai, s, w; - CMMetadataBase *meta; - CMDataBase *data; + CMMetadataBase *meta = nullptr; + CMDataBase *data = nullptr; bool hit; - if constexpr (EnMT) { while(true) { - hit = cache->hit(addr, &ai, &s, &w, prio, true); + hit = cache->hit(addr, &ai, &s, &w, prio, !prefetch); if(hit) { - std::tie(meta, data) = cache->access_line(ai, s, w); meta->lock(); - if(!cache->check_mt_state(ai, s, prio) || !meta->match(addr)) { // hitted acquire is intercepted by a probe and even invalidated - meta->unlock(); meta = nullptr; data = nullptr; - cache->reset_mt_state(ai, s, prio); - continue; // redo the hit check + std::tie(meta, data) = cache->access_line(ai, s, w); + if(!prefetch) { // no need to lock for prefetch + meta->lock(); + if(!cache->check_mt_state(ai, s, prio) || !meta->match(addr)) { // acquire is intercepted by a probe and even invalidated + meta->unlock(); meta = nullptr; data = nullptr; + cache->reset_mt_state(ai, s, prio); + continue; // redo the hit check + } } - } else { // miss + } else if(do_replace) { // miss if(cache->replace(addr, &ai, &s, &w, prio)) { // lock the cache set and get a replacement candidate std::tie(meta, data) = cache->access_line(ai, s, w); meta->lock(); @@ -287,9 +300,15 @@ class InnerCohPortUncached : public InnerCohPortBase } } else { hit = cache->hit(addr, &ai, &s, &w); - if(!hit) cache->replace(addr, &ai, &s, &w, prio); - std::tie(meta, data) = cache->access_line(ai, s, w); + if(!hit && do_replace) cache->replace(addr, &ai, &s, &w, prio); + if(hit || do_replace) std::tie(meta, data) = cache->access_line(ai, s, w); } + return std::make_tuple(hit, meta, data, ai, s, w); + } + + virtual std::tuple + access_line(uint64_t addr, coh_cmd_t cmd, uint16_t prio, uint64_t *delay) { // common function for access a line in the cache + auto [hit, meta, data, ai, s, w] = check_hit_or_replace(addr, prio, true, false, delay); if(hit) { auto sync = Policy::access_need_sync(cmd, meta); @@ -318,51 +337,42 @@ class InnerCohPortUncached : public InnerCohPortBase } virtual void flush_line(uint64_t addr, coh_cmd_t cmd, uint64_t *delay) { - uint32_t ai, s, w; - CMMetadataBase *meta = nullptr; - CMDataBase *data = nullptr; - bool hit; + if constexpr (!Policy::is_uncached()) + outer->writeback_req(addr, nullptr, nullptr, coh::cmd_for_flush(), delay); + else { + auto [hit, meta, data, ai, s, w] = check_hit_or_replace(addr, XactPrio::flush, false, false, delay); + auto [probe, probe_cmd] = Policy::flush_need_sync(cmd, meta); + if(!hit) return; - if constexpr (EnMT) { - while(true) { - hit = cache->hit(addr, &ai, &s, &w, XactPrio::flush, true); - if(hit) { - std::tie(meta, data) = cache->access_line(ai, s, w); meta->lock(); - if(!meta->match(addr)) { // cache line is invalidated by transactions with higher priority - meta->unlock(); meta = nullptr; data = nullptr; - cache->reset_mt_state(ai, s, XactPrio::flush); continue; // redo the hit check - } - } - break; + if(probe) { + auto [phit, pwb] = probe_req(addr, meta, data, probe_cmd, delay); // sync if necessary + if(pwb) cache->hook_write(addr, ai, s, w, true, false, meta, data, delay); // a write occurred during the probe } - } else { - hit = cache->hit(addr, &ai, &s, &w); - if(hit) std::tie(meta, data) = cache->access_line(ai, s, w); - } - auto [flush, probe, probe_cmd] = Policy::flush_need_sync(cmd, meta); - if(!flush) { - // do not handle flush at this level, and send it to the outer cache - outer->writeback_req(addr, nullptr, nullptr, coh::cmd_for_flush(), delay); - return; - } + auto writeback = Policy::writeback_need_writeback(meta); + if(writeback.first) outer->writeback_req(addr, meta, data, writeback.second, delay); // writeback if dirty - if(!hit) return; + Policy::meta_after_flush(cmd, meta, cache); + cache->hook_manage(addr, ai, s, w, hit, coh::is_evict(cmd), writeback.first, meta, data, delay); - if(probe) { - auto [phit, pwb] = probe_req(addr, meta, data, probe_cmd, delay); // sync if necessary - if(pwb) cache->hook_write(addr, ai, s, w, true, false, meta, data, delay); // a write occurred during the probe + if constexpr (EnMT) { meta->unlock(); cache->reset_mt_state(ai, s, XactPrio::flush); } } - - auto writeback = Policy::writeback_need_writeback(meta); - if(writeback.first) outer->writeback_req(addr, meta, data, writeback.second, delay); // writeback if dirty - - Policy::meta_after_flush(cmd, meta, cache); - cache->hook_manage(addr, ai, s, w, hit, coh::is_evict(cmd), writeback.first, meta, data, delay); - - if constexpr (EnMT) { meta->unlock(); cache->reset_mt_state(ai, s, XactPrio::flush); } } + virtual void prefetch_line(uint64_t addr, coh_cmd_t cmd, uint64_t *delay) { + if constexpr (!Policy::is_uncached()) + outer->prefetch_req(addr, coh::cmd_for_prefetch(), delay); + else { + auto [hit, meta, data, ai, s, w] = check_hit_or_replace(addr, XactPrio::acquire, true, true, delay); + if(!hit) { + if(meta->is_valid()) evict(meta, data, ai, s, w, delay); + outer->acquire_req(addr, meta, data, coh::cmd_for_prefetch(), delay); // fetch the missing block + cache->hook_read(addr, ai, s, w, hit, true, meta, data, delay); + finish_record(addr, coh::cmd_for_finish(-1), !hit, meta, ai, s); + finish_resp(addr, coh::cmd_for_finish(-1)); + } + } + } }; template