diff --git a/CMakeLists.txt b/CMakeLists.txt index e0383c36..232039ae 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -76,7 +76,6 @@ find_package(spdlog CONFIG REQUIRED) find_package(GTest CONFIG REQUIRED) find_package(benchmark CONFIG REQUIRED) find_package(Crc32c CONFIG REQUIRED) -find_package(prometheus-cpp CONFIG REQUIRED) find_package(httplib CONFIG REQUIRED) find_package(PkgConfig) pkg_check_modules(libunwind REQUIRED IMPORTED_TARGET GLOBAL libunwind) diff --git a/CMakePresets.json b/CMakePresets.json index 9677d51f..1d2c70eb 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -10,7 +10,8 @@ "CMAKE_TOOLCHAIN_FILE": "$env{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake", "CMAKE_EXPORT_COMPILE_COMMANDS": "ON", "BUILD_SHARED_LIBS": "ON", - "ENABLE_PROFILING": "ON" + "ENABLE_PROFILING": "ON", + "ENABLE_PERF_COUNTERS": "ON" } }, { @@ -26,8 +27,7 @@ "inherits": "base", "hidden": true, "cacheVariables": { - "CMAKE_BUILD_TYPE": "Release", - "COUNTERS_LEVEL": "none" + "CMAKE_BUILD_TYPE": "Release" } }, { @@ -68,4 +68,4 @@ } } ] -} +} \ No newline at end of file diff --git a/benchmarks/ycsb/YcsbLeanStore.hpp b/benchmarks/ycsb/YcsbLeanStore.hpp index bf7b3575..c7a830ba 100644 --- a/benchmarks/ycsb/YcsbLeanStore.hpp +++ b/benchmarks/ycsb/YcsbLeanStore.hpp @@ -1,5 +1,7 @@ #include "Ycsb.hpp" +#include "leanstore-c/PerfCounters.h" #include "leanstore-c/StoreOption.h" +#include "leanstore-c/leanstore-c.h" #include "leanstore/KVInterface.hpp" #include "leanstore/LeanStore.hpp" #include "leanstore/btree/BasicKV.hpp" @@ -17,14 +19,20 @@ #include #include +#include #include #include #include +#include #include #include #include +#include #include +#include +#include + namespace leanstore::ycsb { constexpr std::string kTableName = "ycsb_leanstore"; @@ -44,8 +52,6 @@ class YcsbLeanStore : public YcsbExecutor { option->mEnableEagerGc = true; option->mWorkerThreads = FLAGS_ycsb_threads; option->mBufferPoolSize = FLAGS_ycsb_mem_kb * 1024; - option->mEnableMetrics = true; - option->mMetricsPort = 8080; auto res = LeanStore::Open(option); if (!res) { @@ -55,6 +61,9 @@ class YcsbLeanStore : public YcsbExecutor { } mStore = std::move(res.value()); + + // start metrics http exposer for cpu/mem profiling + StartMetricsHttpExposer(8080); } ~YcsbLeanStore() override { @@ -164,6 +173,12 @@ class YcsbLeanStore : public YcsbExecutor { a = 0; } + std::vector workerPerfCounters; + for (auto i = 0u; i < mStore->mStoreOption->mWorkerThreads; i++) { + mStore->ExecSync( + i, [&]() { workerPerfCounters.push_back(cr::WorkerContext::My().GetPerfCounters()); }); + } + for (uint64_t workerId = 0; workerId < mStore->mStoreOption->mWorkerThreads; workerId++) { mStore->ExecAsync(workerId, [&]() { uint8_t key[FLAGS_ycsb_key_size]; @@ -239,11 +254,45 @@ class YcsbLeanStore : public YcsbExecutor { a = 0; } + std::thread perfContextReporter([&]() { + auto reportPeriod = 1; + const char* counterFilePath = "/tmp/leanstore/worker-counters.txt"; + std::ofstream ost; + + while (keepRunning) { + sleep(reportPeriod); + uint64_t txWithRemoteDependencies = 0; + uint64_t lcbExecuted = 0; + uint64_t lcbTotalLatNs [[maybe_unused]] = 0; + uint64_t gcExecuted = 0; + uint64_t gcTotalLatNs [[maybe_unused]] = 0; + uint64_t txCommitWait = 0; + + // collect counters + for (auto* perfCounters : workerPerfCounters) { + txWithRemoteDependencies += atomic_exchange(&perfCounters->mTxWithRemoteDependencies, 0); + txCommitWait += atomic_exchange(&perfCounters->mTxCommitWait, 0); + + lcbExecuted += atomic_exchange(&perfCounters->mLcbExecuted, 0); + lcbTotalLatNs += atomic_exchange(&perfCounters->mLcbTotalLatNs, 0); + + gcExecuted += atomic_exchange(&perfCounters->mGcExecuted, 0); + gcTotalLatNs += atomic_exchange(&perfCounters->mGcTotalLatNs, 0); + } + ost.open(counterFilePath, std::ios_base::app); + ost << std::format("TxWithDep: {}, txCommitWait: {}, LcbExec: {}, GcExec: {}", + txWithRemoteDependencies, txCommitWait, lcbExecuted, gcExecuted) + << std::endl; + ost.close(); + } + }); + printTpsSummary(1, FLAGS_ycsb_run_for_seconds, mStore->mStoreOption->mWorkerThreads, threadCommitted, threadAborted); // Shutdown threads keepRunning = false; + perfContextReporter.join(); mStore->WaitAll(); } }; diff --git a/benchmarks/ycsb/YcsbWiredTiger.hpp b/benchmarks/ycsb/YcsbWiredTiger.hpp index f22ac32a..52535d1f 100644 --- a/benchmarks/ycsb/YcsbWiredTiger.hpp +++ b/benchmarks/ycsb/YcsbWiredTiger.hpp @@ -1,6 +1,7 @@ #pragma once #include "Ycsb.hpp" +#include "leanstore-c/leanstore-c.h" #include "leanstore/utils/Defer.hpp" #include "leanstore/utils/Log.hpp" #include "leanstore/utils/Parallelize.hpp" @@ -26,6 +27,7 @@ class YcsbWiredTiger : public YcsbExecutor { public: YcsbWiredTiger() : mConn(nullptr) { + StartMetricsHttpExposer(8080); } ~YcsbWiredTiger() override { @@ -195,7 +197,8 @@ class YcsbWiredTiger : public YcsbExecutor { } std::string configString( - "create, direct_io=[data, log, checkpoint], log=(enabled=false), statistics_log=(wait=1), " + "create, direct_io=[data, log, checkpoint], " + "log=(enabled=true,archive=true), statistics_log=(wait=1), " "statistics=(all, clear), session_max=2000, eviction=(threads_max=4), cache_size=" + std::to_string(FLAGS_ycsb_mem_kb / 1024) + "M"); int ret = wiredtiger_open(dataDir.c_str(), nullptr, configString.c_str(), &mConn); diff --git a/docker/monitor-compose/docker-compose.yml b/docker/monitor-compose/docker-compose.yml deleted file mode 100644 index 277fca40..00000000 --- a/docker/monitor-compose/docker-compose.yml +++ /dev/null @@ -1,26 +0,0 @@ -# docker-compose.yml -version: '3' -services: - prometheus: - network_mode: "host" - image: prom/prometheus:latest - command: - - --storage.tsdb.retention.time=7d - - --config.file=/etc/prometheus/prometheus.yml - ports: - - "9090:9090" - volumes: - - ./prometheus.yml:/etc/prometheus/prometheus.yml - grafana: - network_mode: "host" - image: grafana/grafana:latest - ports: - - "3000:3000" - volumes: - - ./grafana/dashboards:/etc/grafana/provisioning/dashboards - - ./grafana/datasources:/etc/grafana/provisioning/datasources - node_exporter: - network_mode: "host" - image: prom/node-exporter:latest - ports: - - "9100:9100" diff --git a/docker/monitor-compose/grafana/dashboards/dashboards.yaml b/docker/monitor-compose/grafana/dashboards/dashboards.yaml deleted file mode 100644 index f4b30782..00000000 --- a/docker/monitor-compose/grafana/dashboards/dashboards.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: 1 - -providers: - - name: 'default' - orgId: 1 - folder: '' - type: file - disableDeletion: false - editable: true - options: - path: /etc/grafana/provisioning/dashboards diff --git a/docker/monitor-compose/grafana/dashboards/my_dashboard.json b/docker/monitor-compose/grafana/dashboards/my_dashboard.json deleted file mode 100644 index 035b8b96..00000000 --- a/docker/monitor-compose/grafana/dashboards/my_dashboard.json +++ /dev/null @@ -1,84 +0,0 @@ -{ - "title": "Node Monitoring", - "uid": "prod-node-monitoring", - "timezone": "browser", - "panels": [ - { - "title": "CPU Usage", - "type": "graph", - "datasource": "Prometheus", - "targets": [ - { - "expr": "100 - (avg(irate(node_cpu_seconds_total{mode='idle'}[5m])) by (instance) * 100)", - "legendFormat": "{{ instance }}" - } - ], - "timeFrom": "1h", - "gridPos": { - "h": 8, - "w": 12 - } - }, - { - "title": "Memory Usage", - "type": "graph", - "datasource": "Prometheus", - "targets": [ - { - "expr": "node_memory_MemTotal_bytes - node_memory_MemFree_bytes", - "legendFormat": "{{ instance }}" - } - ], - "timeFrom": "1h", - "gridPos": { - "h": 8, - "w": 12 - } - }, - { - "title": "Disk Usage", - "type": "graph", - "datasource": "Prometheus", - "targets": [ - { - "expr": "node_filesystem_size_bytes{fstype!~'tmpfs|squashfs'} - node_filesystem_free_bytes{fstype!~'tmpfs|squashfs'}", - "legendFormat": "{{ instance }}" - } - ], - "timeFrom": "1h", - "gridPos": { - "h": 8, - "w": 12 - } - } - ], - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - } -} diff --git a/docker/monitor-compose/grafana/datasources/datasources.yaml b/docker/monitor-compose/grafana/datasources/datasources.yaml deleted file mode 100644 index f9cc5269..00000000 --- a/docker/monitor-compose/grafana/datasources/datasources.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# datasources.yaml -apiVersion: 1 -datasources: - - name: Prometheus - type: prometheus - url: http://localhost:9090 - access: proxy diff --git a/docker/monitor-compose/prometheus.yml b/docker/monitor-compose/prometheus.yml deleted file mode 100644 index 79ee6dc6..00000000 --- a/docker/monitor-compose/prometheus.yml +++ /dev/null @@ -1,11 +0,0 @@ -# prometheus.yml -scrape_configs: - - job_name: 'node' - scrape_interval: 5s - static_configs: - - targets: ['localhost:9100'] - - job_name: 'leanstore' - scrape_interval: 5s - metrics_path: /metrics - static_configs: - - targets: ['localhost:8080'] diff --git a/include/leanstore-c/PerfCounters.h b/include/leanstore-c/PerfCounters.h new file mode 100644 index 00000000..8cab1aee --- /dev/null +++ b/include/leanstore-c/PerfCounters.h @@ -0,0 +1,83 @@ +#ifndef LEANSTORE_PERF_COUNTERS_H +#define LEANSTORE_PERF_COUNTERS_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +//! The counter type. +typedef atomic_ullong CounterType; + +//! The performance counters for each worker. +typedef struct PerfCounters { + + // --------------------------------------------------------------------------- + // Transaction related counters + // --------------------------------------------------------------------------- + + //! The number of transactions committed. + CounterType mTxCommitted; + + //! The number of transactions commit wait. + CounterType mTxCommitWait; + + //! The number of transactions aborted. + CounterType mTxAborted; + + ///! The number of transactions with remote dependencies. + CounterType mTxWithRemoteDependencies; + + //! The number of transactions without remote dependencies. + CounterType mTxWithoutRemoteDependencies; + + //! The number of short running transactions. + CounterType mTxShortRunning; + + //! The number of long running transactions. + CounterType mTxLongRunning; + + // --------------------------------------------------------------------------- + // MVCC concurrency control related counters + // --------------------------------------------------------------------------- + + //! The number of LCB query executed. + CounterType mLcbExecuted; + + //! The total latency of LCB query in nanoseconds. + CounterType mLcbTotalLatNs; + + // --------------------------------------------------------------------------- + // MVCC garbage collection related counters + // --------------------------------------------------------------------------- + + //! The number of MVCC garbage collection executed. + CounterType mGcExecuted; + + //! The total latency of MVCC garbage collection in nanoseconds. + CounterType mGcTotalLatNs; + + // --------------------------------------------------------------------------- + // Contention split related counters + // --------------------------------------------------------------------------- + + //! The number of contention split succeed. + CounterType mContentionSplitSucceed; + + //! The number of contention split failed. + CounterType mContentionSplitFailed; + + //! The number of normal split succeed. + CounterType mSplitSucceed; + + //! The number of normal split failed. + CounterType mSplitFailed; + +} PerfCounters; + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/include/leanstore-c/StoreOption.h b/include/leanstore-c/StoreOption.h index d625983c..d91e62dd 100644 --- a/include/leanstore-c/StoreOption.h +++ b/include/leanstore-c/StoreOption.h @@ -8,191 +8,183 @@ extern "C" { #endif - //! The log level - typedef enum LogLevel { - kDebug = 0, - kInfo, - kWarn, - kError, - } LogLevel; +//! The log level +typedef enum LogLevel { + kDebug = 0, + kInfo, + kWarn, + kError, +} LogLevel; - //! The options for creating a new store. - typedef struct StoreOption { - // --------------------------------------------------------------------------- - // Store related options - // --------------------------------------------------------------------------- +//! The options for creating a new store. +typedef struct StoreOption { + // --------------------------------------------------------------------------- + // Store related options + // --------------------------------------------------------------------------- - //! Whether to create store from scratch. - bool mCreateFromScratch; + //! Whether to create store from scratch. + bool mCreateFromScratch; - //! The directory for all the database files. - const char* mStoreDir; + //! The directory for all the database files. + const char* mStoreDir; - // --------------------------------------------------------------------------- - // log related options - // --------------------------------------------------------------------------- + // --------------------------------------------------------------------------- + // log related options + // --------------------------------------------------------------------------- - //! The log level - LogLevel mLogLevel; + //! The log level + LogLevel mLogLevel; - // --------------------------------------------------------------------------- - // Worker thread related options - // --------------------------------------------------------------------------- + // --------------------------------------------------------------------------- + // Worker thread related options + // --------------------------------------------------------------------------- - //! The number of worker threads. - uint64_t mWorkerThreads; + //! The number of worker threads. + uint64_t mWorkerThreads; - //! The WAL buffer size for each worker (bytes). - uint64_t mWalBufferSize; + //! The WAL buffer size for each worker (bytes). + uint64_t mWalBufferSize; - // --------------------------------------------------------------------------- - // Buffer pool related options - // --------------------------------------------------------------------------- + // --------------------------------------------------------------------------- + // Buffer pool related options + // --------------------------------------------------------------------------- - //! The page size (bytes). For buffer manager. - uint64_t mPageSize; + //! The page size (bytes). For buffer manager. + uint64_t mPageSize; - uint64_t mBufferFrameSize; + uint64_t mBufferFrameSize; - //! The number of partitions. For buffer manager. - uint32_t mNumPartitions; + //! The number of partitions. For buffer manager. + uint32_t mNumPartitions; - //! The buffer pool size (bytes). For buffer manager. - uint64_t mBufferPoolSize; + //! The buffer pool size (bytes). For buffer manager. + uint64_t mBufferPoolSize; - //! The free percentage of the buffer pool. In the range of [0, 100]. - uint32_t mFreePct; + //! The free percentage of the buffer pool. In the range of [0, 100]. + uint32_t mFreePct; - //! The number of page evictor threads. - uint32_t mNumBufferProviders; + //! The number of page evictor threads. + uint32_t mNumBufferProviders; - //! The async buffer - uint32_t mBufferWriteBatchSize; + //! The async buffer + uint32_t mBufferWriteBatchSize; - //! Whether to perform crc check for buffer frames. - bool mEnableBufferCrcCheck; + //! Whether to perform crc check for buffer frames. + bool mEnableBufferCrcCheck; - //! BufferFrame recycle batch size. Everytime a batch of buffer frames is - //! randomly picked and verified by page evictors, some of them are COOLed, - //! some of them are EVICted. - uint64_t mBufferFrameRecycleBatchSize; + //! BufferFrame recycle batch size. Everytime a batch of buffer frames is randomly picked and + //! verified by page evictors, some of them are COOLed, some of them are EVICted. + uint64_t mBufferFrameRecycleBatchSize; - //! Whether to reclaim unused free page ids - bool mEnableReclaimPageIds; + //! Whether to reclaim unused free page ids + bool mEnableReclaimPageIds; - // --------------------------------------------------------------------------- - // Logging and recovery related options - // --------------------------------------------------------------------------- + // --------------------------------------------------------------------------- + // Logging and recovery related options + // --------------------------------------------------------------------------- - //! Whether to enable write-ahead log. - bool mEnableWal; + //! Whether to enable write-ahead log. + bool mEnableWal; - //! Whether to execute fsync after each WAL write. - bool mEnableWalFsync; + //! Whether to execute fsync after each WAL write. + bool mEnableWalFsync; - // --------------------------------------------------------------------------- - // Generic BTree related options - // --------------------------------------------------------------------------- + // --------------------------------------------------------------------------- + // Generic BTree related options + // --------------------------------------------------------------------------- - //! Whether to enable bulk insert. - bool mEnableBulkInsert; + //! Whether to enable bulk insert. + bool mEnableBulkInsert; - //! Whether to enable X-Merge - bool mEnableXMerge; + //! Whether to enable X-Merge + bool mEnableXMerge; - //! The number of children to merge in X-Merge - uint64_t mXMergeK; + //! The number of children to merge in X-Merge + uint64_t mXMergeK; - //! The target percentage of the number of children to merge in X-Merge - double mXMergeTargetPct; + //! The target percentage of the number of children to merge in X-Merge + double mXMergeTargetPct; - //! Whether to enable contention split. - bool mEnableContentionSplit; + //! Whether to enable contention split. + bool mEnableContentionSplit; - //! Contention split probability, as exponent of 2 - uint64_t mContentionSplitProbility; + //! Contention split probability, as exponent of 2 + uint64_t mContentionSplitProbility; - //! Contention stats sample probability, as exponent of 2 - uint64_t mContentionSplitSampleProbability; + //! Contention stats sample probability, as exponent of 2 + uint64_t mContentionSplitSampleProbability; - //! Contention percentage to trigger the split, in the range of [0, 100]. - uint64_t mContentionSplitThresholdPct; + //! Contention percentage to trigger the split, in the range of [0, 100]. + uint64_t mContentionSplitThresholdPct; - //! Whether to enable btree hints optimization. Available options: - //! 0: disabled - //! 1: serial - //! 2: AVX512 - int64_t mBTreeHints; + //! Whether to enable btree hints optimization. Available options: + //! 0: disabled + //! 1: serial + //! 2: AVX512 + int64_t mBTreeHints; - //! Whether to enable heads optimization in LowerBound search. - bool mEnableHeadOptimization; + //! Whether to enable heads optimization in LowerBound search. + bool mEnableHeadOptimization; - //! Whether to enable optimistic scan. Jump to next leaf directly if the - //! pointer in the parent has not changed - bool mEnableOptimisticScan; + //! Whether to enable optimistic scan. Jump to next leaf directly if the pointer in the parent has + //! not changed + bool mEnableOptimisticScan; - // --------------------------------------------------------------------------- - // Transaction related options - // --------------------------------------------------------------------------- + // --------------------------------------------------------------------------- + // Transaction related options + // --------------------------------------------------------------------------- - //! Whether to enable long running transaction. - bool mEnableLongRunningTx; + //! Whether to enable long running transaction. + bool mEnableLongRunningTx; - //! Whether to enable fat tuple. - bool mEnableFatTuple; + //! Whether to enable fat tuple. + bool mEnableFatTuple; - //! Whether to enable garbage collection. - bool mEnableGc; + //! Whether to enable garbage collection. + bool mEnableGc; - //! Whether to enable eager garbage collection. To enable eager garbage - //! collection, the garbage collection must be enabled first. Once enabled, - //! the garbage collection will be triggered after each transaction commit and - //! abort. - bool mEnableEagerGc; + //! Whether to enable eager garbage collection. To enable eager garbage collection, the garbage + //! collection must be enabled first. Once enabled, the garbage collection will be triggered after + //! each transaction commit and abort. + bool mEnableEagerGc; - // --------------------------------------------------------------------------- - // Metrics related options - // --------------------------------------------------------------------------- + // --------------------------------------------------------------------------- + // Metrics related options + // --------------------------------------------------------------------------- - //! Whether to enable metrics. - bool mEnableMetrics; + //! Whether to enable cpu counters. + bool mEnableCpuCounters; - //! The metrics port. - int32_t mMetricsPort; + //! Whether to enable time measure. + bool mEnableTimeMeasure; - //! Whether to enable cpu counters. - bool mEnableCpuCounters; + //! Whether to enable perf events. + bool mEnablePerfEvents; - //! Whether to enable time measure. - bool mEnableTimeMeasure; +} StoreOption; - //! Whether to enable perf events. - bool mEnablePerfEvents; +//! Create a new store option. +//! @param storeDir the directory for all the database files. The string content is deep copied to +//! the created store option. +StoreOption* CreateStoreOption(const char* storeDir); - } StoreOption; +//! Create a new store option from an existing store option. +//! @param storeDir the existing store option. +StoreOption* CreateStoreOptionFrom(const StoreOption* storeDir); - //! Create a new store option. - //! @param storeDir the directory for all the database files. The string content is deep copied to - //! the created store option. - StoreOption* CreateStoreOption(const char* storeDir); +//! Destroy a store option. +//! @param option the store option to destroy. +void DestroyStoreOption(const StoreOption* option); - //! Create a new store option from an existing store option. - //! @param storeDir the existing store option. - StoreOption* CreateStoreOptionFrom(const StoreOption* storeDir); +//! The options for creating a new BTree. +typedef struct BTreeConfig { + //! Whether to enable write-ahead log. + bool mEnableWal; - //! Destroy a store option. - //! @param option the store option to destroy. - void DestroyStoreOption(const StoreOption* option); + //! Whether to enable bulk insert. + bool mUseBulkInsert; - //! The options for creating a new BTree. - typedef struct BTreeConfig { - //! Whether to enable write-ahead log. - bool mEnableWal; - - //! Whether to enable bulk insert. - bool mUseBulkInsert; - - } BTreeConfig; +} BTreeConfig; #ifdef __cplusplus } diff --git a/include/leanstore/Exceptions.hpp b/include/leanstore/Exceptions.hpp index 1dac15ff..1a012ebd 100644 --- a/include/leanstore/Exceptions.hpp +++ b/include/leanstore/Exceptions.hpp @@ -1,13 +1,10 @@ #pragma once +#include +#include #include #include -#include -#include -#include - -//-------------------------------------------------------------------------------------- #define Generic_Exception(name) \ struct name : public std::exception { \ const std::string msg; \ @@ -21,8 +18,8 @@ virtual const char* what() const noexcept override { \ return msg.c_str(); \ } \ - }; \ -//-------------------------------------------------------------------------------------- + }; + namespace leanstore { namespace ex { Generic_Exception(GenericException); @@ -31,29 +28,17 @@ Generic_Exception(UnReachable); Generic_Exception(TODO); } // namespace ex } // namespace leanstore -// ------------------------------------------------------------------------------------- + #define UNREACHABLE() \ throw leanstore::ex::UnReachable(std::string(__FILE__) + ":" + \ std::string(std::to_string(__LINE__))); -// ------------------------------------------------------------------------------------- + #define always_check(e) \ (__builtin_expect(!(e), 0) ? throw leanstore::ex::EnsureFailed( \ std::string(__func__) + " in " + std::string(__FILE__) + "@" + \ std::to_string(__LINE__) + " msg: " + std::string(#e)) \ : (void)0) -#ifdef PARANOID -#define PARANOID(e) always_check(e); -#define PARANOID_BLOCK() if constexpr (true) -#else -#ifdef DEBUG -#define PARANOID_BLOCK() if constexpr (true) -#else -#define PARANOID_BLOCK() if constexpr (false) -#endif -#define PARANOID(e) assert(e) -#endif - #ifdef DEBUG #define ENSURE(e) assert(e); #else @@ -63,15 +48,6 @@ Generic_Exception(TODO); #define TODOException() \ throw leanstore::ex::TODO(std::string(__FILE__) + ":" + std::string(std::to_string(__LINE__))); -#define explainIfNot(e) \ - if (!(e)) { \ - raise(SIGTRAP); \ - }; -#define RAISE_WHEN(e) \ - if (e) { \ - raise(SIGTRAP); \ - }; -// ------------------------------------------------------------------------------------- #ifdef MACRO_CHECK_DEBUG #define DEBUG_BLOCK() if (true) #define RELEASE_BLOCK() if (true) @@ -90,24 +66,12 @@ Generic_Exception(TODO); #endif #endif #endif -// ------------------------------------------------------------------------------------- -#ifdef MACRO_COUNTERS_ALL -#define COUNTERS_BLOCK() if constexpr (true) -#else -#define COUNTERS_BLOCK() if constexpr (false) -#endif -// ------------------------------------------------------------------------------------- + template -inline void DO_NOT_OPTIMIZE(const T& value) { +inline void DoNotOptimize(const T& value) { #if defined(__clang__) asm volatile("" : : "g"(value) : "memory"); #else asm volatile("" : : "i,r,m"(value) : "memory"); #endif } - -#define POSIX_CHECK(expr) \ - if (!(expr)) { \ - perror(#expr); \ - raise(SIGTRAP); \ - } diff --git a/include/leanstore/LeanStore.hpp b/include/leanstore/LeanStore.hpp index 385350cc..57d942de 100644 --- a/include/leanstore/LeanStore.hpp +++ b/include/leanstore/LeanStore.hpp @@ -12,14 +12,6 @@ #include #include -//! forward declarations -namespace leanstore::telemetry { - -class MetricsManager; -class MetricsHttpExposer; - -} // namespace leanstore::telemetry - //! forward declarations namespace leanstore::storage::btree { @@ -81,12 +73,6 @@ class LeanStore { //! transactions. Start from a positive number, 0 indicates invalid timestamp std::atomic mSysTso = 1; - //! The metrics manager - std::unique_ptr mMetricsManager; - - //! The http metrics exposer - std::unique_ptr mMetricsExposer; - #ifdef DEBUG utils::DebugFlagsRegistry mDebugFlagsRegistry; #endif diff --git a/include/leanstore/PerfEvent.hpp b/include/leanstore/PerfEvent.hpp deleted file mode 100644 index 210f68a7..00000000 --- a/include/leanstore/PerfEvent.hpp +++ /dev/null @@ -1,328 +0,0 @@ -/* - -Copyright (c) 2018 Viktor Leis - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - */ - -#pragma once - -#if defined(__linux__) - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -struct PerfEvent { - struct event { - struct read_format { - uint64_t value; - uint64_t time_enabled; - uint64_t time_running; - uint64_t id; - }; - - perf_event_attr pe; - int fd; - read_format prev; - read_format data; - - double readCounter() { - double multiplexingCorrection = static_cast(data.time_enabled - prev.time_enabled) / - (data.time_running - prev.time_running); - return (data.value - prev.value) * multiplexingCorrection; - } - }; - - std::vector events; - std::vector names; - std::chrono::time_point startTime; - std::chrono::time_point stopTime; - std::map params; - const bool inherit; - bool printHeader; - - bool isIntel() { -#ifdef __x86_64__ - // EAX=0: Highest Function Parameter and Manufacturer ID - // ecx is often an input as well as an output - unsigned eax = 0, ebx, ecx = 0, edx; - asm volatile("cpuid" : "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx) : "0"(0), "2"(ecx)); - return ecx == 0x6c65746e; -#else - return false; -#endif - } - PerfEvent(bool inherit = true) : inherit(inherit), printHeader(true) { - registerCounter("cycle", PERF_TYPE_HARDWARE, PERF_COUNT_HW_CPU_CYCLES); - registerCounter("cycle-kernel", PERF_TYPE_HARDWARE, PERF_COUNT_HW_CPU_CYCLES, 1); - registerCounter("instr", PERF_TYPE_HARDWARE, PERF_COUNT_HW_INSTRUCTIONS); - registerCounter("L1-miss", PERF_TYPE_HW_CACHE, - PERF_COUNT_HW_CACHE_L1D | (PERF_COUNT_HW_CACHE_OP_READ << 8) | - (PERF_COUNT_HW_CACHE_RESULT_MISS << 16)); - if (isIntel()) - registerCounter("LLC-miss", PERF_TYPE_HARDWARE, PERF_COUNT_HW_CACHE_MISSES); - registerCounter("br-miss", PERF_TYPE_HARDWARE, PERF_COUNT_HW_BRANCH_MISSES); - registerCounter("task", PERF_TYPE_SOFTWARE, PERF_COUNT_SW_TASK_CLOCK); - // additional counters can be found in linux/perf_event.h - - for (unsigned i = 0; i < events.size(); i++) { - auto& event = events[i]; - event.fd = syscall(__NR_perf_event_open, &event.pe, 0, -1, -1, 0); - if (event.fd < 0) { - std::cerr << "Error opening counter " << names[i] << std::endl; - events.resize(0); - names.resize(0); - return; - } - } - } - - void registerCounter(const std::string& name, uint64_t type, uint64_t eventID, - const bool excludeUser = false) { - names.push_back(name); - events.push_back(event()); - auto& event = events.back(); - auto& pe = event.pe; - memset(&pe, 0, sizeof(struct perf_event_attr)); - pe.type = type; - pe.size = sizeof(struct perf_event_attr); - pe.config = eventID; - pe.disabled = true; - pe.inherit = inherit; - pe.inherit_stat = 0; - pe.exclude_user = excludeUser; - pe.exclude_kernel = false; - pe.exclude_hv = false; - pe.read_format = PERF_FORMAT_TOTAL_TIME_ENABLED | PERF_FORMAT_TOTAL_TIME_RUNNING; - } - - void startCounters() { - for (unsigned i = 0; i < events.size(); i++) { - auto& event = events[i]; - ioctl(event.fd, PERF_EVENT_IOC_RESET, 0); - ioctl(event.fd, PERF_EVENT_IOC_ENABLE, 0); - if (read(event.fd, &event.prev, sizeof(uint64_t) * 3) != sizeof(uint64_t) * 3) - std::cerr << "Error reading counter " << names[i] << std::endl; - } - startTime = std::chrono::steady_clock::now(); - } - - ~PerfEvent() { - for (auto& event : events) { - close(event.fd); - } - } - - void stopCounters() { - stopTime = std::chrono::steady_clock::now(); - for (unsigned i = 0; i < events.size(); i++) { - auto& event = events[i]; - if (read(event.fd, &event.data, sizeof(uint64_t) * 3) != sizeof(uint64_t) * 3) - std::cerr << "Error reading counter " << names[i] << std::endl; - ioctl(event.fd, PERF_EVENT_IOC_DISABLE, 0); - } - } - - double getDuration() { - return std::chrono::duration(stopTime - startTime).count(); - } - - double getIPC() { - return getCounter("instr") / getCounter("cycle"); - } - - double getCPUs() { - return getCounter("task") / (getDuration() * 1e9); - } - - double getGHz() { - return getCounter("cycle") / getCounter("task"); - } - - double getCounter(const std::string& name) { - for (unsigned i = 0; i < events.size(); i++) - if (names[i] == name) - return events[i].readCounter(); - return -1; - } - - static void printCounter(std::ostream& headerOut, std::ostream& dataOut, std::string name, - std::string counterValue, bool addComma = true) { - auto width = std::max(name.length(), counterValue.length()); - headerOut << std::setw(width) << name << (addComma ? "," : "") << " "; - dataOut << std::setw(width) << counterValue << (addComma ? "," : "") << " "; - } - - template - static void printCounter(std::ostream& headerOut, std::ostream& dataOut, std::string name, - T counterValue, bool addComma = true) { - std::stringstream stream; - stream << std::fixed << std::setprecision(2) << counterValue; - PerfEvent::printCounter(headerOut, dataOut, name, stream.str(), addComma); - } - - void printReport(std::ostream& out, uint64_t normalizationConstant) { - std::stringstream header; - std::stringstream data; - printReport(header, data, normalizationConstant); - out << header.str() << std::endl; - out << data.str() << std::endl; - } - - void printReport(std::ostream& headerOut, std::ostream& dataOut, uint64_t normalizationConstant) { - if (!events.size()) - return; - - // print all metrics - for (unsigned i = 0; i < events.size(); i++) { - printCounter(headerOut, dataOut, names[i], events[i].readCounter() / normalizationConstant); - } - - printCounter(headerOut, dataOut, "scale", normalizationConstant); - - // derived metrics - printCounter(headerOut, dataOut, "IPC", getIPC()); - printCounter(headerOut, dataOut, "CPU", getCPUs()); - printCounter(headerOut, dataOut, "GHz", getGHz(), false); - } - // ------------------------------------------------------------------------------------- - void printCSVHeaders(std::ostream& headerOut) { - if (!events.size()) - return; - - // print all metrics - for (unsigned i = 0; i < events.size(); i++) { - headerOut << "," << names[i]; - } - - // derived metrics - headerOut << "," << "IPC"; - headerOut << "," << "CPU"; - headerOut << "," << "GHz"; - } - // ------------------------------------------------------------------------------------- - void printCSVData(std::ostream& dataOut, uint64_t normalizationConstant) { - if (!events.size()) - return; - // print all metrics - for (unsigned i = 0; i < events.size(); i++) { - dataOut << "," << events[i].readCounter() / normalizationConstant; - } - // derived metrics - dataOut << "," << getIPC(); - dataOut << "," << getCPUs(); - dataOut << "," << getGHz(); - } - // ------------------------------------------------------------------------------------- - std::vector getEventsName() { - std::vector extendedNames; - for (auto& name : names) { - extendedNames.push_back(name); - } - extendedNames.push_back("IPC"); - extendedNames.push_back("CPU"); - extendedNames.push_back("GHz"); - return extendedNames; - } - std::unordered_map getCountersMap() { - std::unordered_map res; - // print all metrics - for (unsigned i = 0; i < events.size(); i++) { - res[names[i]] = events[i].readCounter(); - } - // derived metrics - res["IPC"] = getIPC(); - res["CPU"] = getCPUs(); - res["GHz"] = getGHz(); - return res; - } - // ------------------------------------------------------------------------------------- - void setParam(const std::string& name, const std::string& value) { - params[name] = value; - } - - void setParam(const std::string& name, const char* value) { - params[name] = value; - } - - template - void setParam(const std::string& name, T value) { - setParam(name, std::to_string(value)); - } - - void printParams(std::ostream& header, std::ostream& data) { - for (auto& p : params) { - printCounter(header, data, p.first, p.second); - } - } -}; - -struct PerfEventBlock { - PerfEvent& e; - uint64_t scale; - bool print_in_destructor = true; - PerfEventBlock(PerfEvent& e, uint64_t scale = 1) : e(e), scale(scale) { - e.startCounters(); - } - - void printCounters() { - std::stringstream header; - std::stringstream data; - e.printParams(header, data); - PerfEvent::printCounter(header, data, "time", e.getDuration()); - e.printReport(header, data, scale); - if (e.printHeader) { - std::cout << header.str() << std::endl; - e.printHeader = false; - } - std::cout << data.str() << std::endl; - } - - ~PerfEventBlock() { - if (print_in_destructor) { - e.stopCounters(); - printCounters(); - } - } -}; - -#else -#include -struct PerfEvent { - void startCounters() { - } - void stopCounters() { - } - void printReport(std::ostream&, uint64_t) { - } -}; -#endif diff --git a/include/leanstore/btree/core/BTreeGeneric.hpp b/include/leanstore/btree/core/BTreeGeneric.hpp index 79e90f4d..0250b8a1 100644 --- a/include/leanstore/btree/core/BTreeGeneric.hpp +++ b/include/leanstore/btree/core/BTreeGeneric.hpp @@ -6,7 +6,6 @@ #include "leanstore/buffer-manager/BufferManager.hpp" #include "leanstore/buffer-manager/GuardedBufferFrame.hpp" #include "leanstore/buffer-manager/TreeRegistry.hpp" -#include "leanstore/profiling/counters/WorkerCounters.hpp" #include "leanstore/sync/HybridLatch.hpp" #include "leanstore/utils/Log.hpp" @@ -312,10 +311,6 @@ inline void BTreeGeneric::FindLeafCanJump(Slice key, GuardedBufferFramemIsLeaf) { - COUNTERS_BLOCK() { - WorkerCounters::MyCounters().dt_inner_page[mTreeId]++; - } - auto& childSwip = guardedTarget->LookupInner(key); LS_DCHECK(!childSwip.IsEmpty()); guardedParent = std::move(guardedTarget); @@ -334,10 +329,6 @@ inline void BTreeGeneric::FindLeafCanJump(Slice key, GuardedBufferFrame inline ParentSwipHandler BTreeGeneric::FindParent(BTreeGeneric& btree, BufferFrame& bfToFind) { - COUNTERS_BLOCK() { - WorkerCounters::MyCounters().dt_find_parent[btree.mTreeId]++; - } - // Check whether search on the wrong tree or the root node is evicted GuardedBufferFrame guardedParent(btree.mStore->mBufferManager.get(), btree.mMetaNodeSwip); @@ -349,9 +340,6 @@ inline ParentSwipHandler BTreeGeneric::FindParent(BTreeGeneric& btree, BufferFra auto* childSwip = &guardedParent->mRightMostChildSwip; if (&childSwip->AsBufferFrameMasked() == &bfToFind) { guardedParent.JumpIfModifiedByOthers(); - COUNTERS_BLOCK() { - WorkerCounters::MyCounters().dt_find_parent_root[btree.mTreeId]++; - } return {.mParentGuard = std::move(guardedParent.mGuard), .mParentBf = &btree.mMetaNodeSwip.AsBufferFrame(), .mChildSwip = *childSwip}; @@ -415,9 +403,6 @@ inline ParentSwipHandler BTreeGeneric::FindParent(BTreeGeneric& btree, BufferFra .mParentBf = guardedChild.mBf, .mChildSwip = *childSwip, .mPosInParent = posInParent}; - COUNTERS_BLOCK() { - WorkerCounters::MyCounters().dt_find_parent_slow[btree.mTreeId]++; - } return parentHandler; } diff --git a/include/leanstore/btree/core/PessimisticExclusiveIterator.hpp b/include/leanstore/btree/core/PessimisticExclusiveIterator.hpp index 6b11373b..e451ddf7 100644 --- a/include/leanstore/btree/core/PessimisticExclusiveIterator.hpp +++ b/include/leanstore/btree/core/PessimisticExclusiveIterator.hpp @@ -2,6 +2,7 @@ #include "leanstore/KVInterface.hpp" #include "leanstore/btree/core/PessimisticIterator.hpp" +#include "leanstore/utils/CounterUtil.hpp" #include "leanstore/utils/Log.hpp" #include "leanstore/utils/RandomGenerator.hpp" #include "leanstore/utils/UserThread.hpp" @@ -24,7 +25,7 @@ class PessimisticExclusiveIterator : public PessimisticIterator { } virtual OpCode SeekToInsertWithHint(Slice key, bool higher = true) { - ENSURE(mSlotId != -1); + LS_DCHECK(Valid()); mSlotId = mGuardedLeaf->LinearSearchWithBias(key, mSlotId, higher); if (mSlotId == -1) { return SeekToInsert(key); @@ -73,12 +74,11 @@ class PessimisticExclusiveIterator : public PessimisticIterator { SetToInvalid(); mBTree.TrySplitMayJump(sysTxId, *bf); - COUNTERS_BLOCK() { - WorkerCounters::MyCounters().dt_split[mBTree.mTreeId]++; - } + COUNTER_INC(&leanstore::cr::tlsPerfCounters.mSplitSucceed); JUMPMU_BREAK; } JUMPMU_CATCH() { + COUNTER_INC(&leanstore::cr::tlsPerfCounters.mSplitFailed); } } } @@ -166,23 +166,16 @@ class PessimisticExclusiveIterator : public PessimisticIterator { TXID sysTxId = mBTree.mStore->AllocSysTxTs(); mBTree.TrySplitMayJump(sysTxId, *mGuardedLeaf.mBf, splitSlot); + COUNTER_INC(&leanstore::cr::tlsPerfCounters.mContentionSplitSucceed); LS_DLOG("[Contention Split] succeed, pageId={}, contention pct={}, split " "slot={}", mGuardedLeaf.mBf->mHeader.mPageId, contentionPct, splitSlot); - - COUNTERS_BLOCK() { - WorkerCounters::MyCounters().contention_split_succ_counter[mBTree.mTreeId]++; - WorkerCounters::MyCounters().dt_split[mBTree.mTreeId]++; - } } JUMPMU_CATCH() { + COUNTER_INC(&leanstore::cr::tlsPerfCounters.mContentionSplitFailed); Log::Info("[Contention Split] contention split failed, pageId={}, contention " "pct={}, split slot={}", mGuardedLeaf.mBf->mHeader.mPageId, contentionPct, splitSlot); - - COUNTERS_BLOCK() { - WorkerCounters::MyCounters().contention_split_fail_counter[mBTree.mTreeId]++; - } } } } diff --git a/include/leanstore/btree/core/PessimisticIterator.hpp b/include/leanstore/btree/core/PessimisticIterator.hpp index 0e230529..ef1d0dee 100644 --- a/include/leanstore/btree/core/PessimisticIterator.hpp +++ b/include/leanstore/btree/core/PessimisticIterator.hpp @@ -322,9 +322,6 @@ class PessimisticIterator : public Iterator { }; inline void PessimisticIterator::Next() { - COUNTERS_BLOCK() { - WorkerCounters::MyCounters().dt_next_tuple[mBTree.mTreeId]++; - } if (!Valid()) { return; } @@ -384,9 +381,6 @@ inline void PessimisticIterator::Next() { JUMPMU_CONTINUE; } ENSURE(mSlotId < mGuardedLeaf->mNumSlots); - COUNTERS_BLOCK() { - WorkerCounters::MyCounters().dt_next_tuple_opt[mBTree.mTreeId]++; - } JUMPMU_RETURN; } } @@ -409,9 +403,6 @@ inline void PessimisticIterator::Next() { JUMPMU_CATCH() { } }); - COUNTERS_BLOCK() { - WorkerCounters::MyCounters().dt_empty_leaf[mBTree.mTreeId]++; - } continue; } mSlotId = mGuardedLeaf->LowerBound(assembedFence()); @@ -423,10 +414,6 @@ inline void PessimisticIterator::Next() { } inline void PessimisticIterator::Prev() { - COUNTERS_BLOCK() { - WorkerCounters::MyCounters().dt_prev_tuple[mBTree.mTreeId]++; - } - while (true) { ENSURE(mGuardedLeaf.mGuard.mState != GuardState::kOptimisticShared); // If we are not at the beginning of the leaf, return the previous key @@ -491,9 +478,6 @@ inline void PessimisticIterator::Prev() { if (mGuardedLeaf->mNumSlots == 0) { JUMPMU_CONTINUE; } - COUNTERS_BLOCK() { - WorkerCounters::MyCounters().dt_prev_tuple_opt[mBTree.mTreeId]++; - } JUMPMU_RETURN; } } @@ -508,9 +492,6 @@ inline void PessimisticIterator::Prev() { }); if (mGuardedLeaf->mNumSlots == 0) { - COUNTERS_BLOCK() { - WorkerCounters::MyCounters().dt_empty_leaf[mBTree.mTreeId]++; - } continue; } bool isEqual = false; @@ -533,14 +514,6 @@ inline void PessimisticIterator::seekToTargetPage( Log::Fatal("Unsupported latch mode: {}", uint64_t(mMode)); } - COUNTERS_BLOCK() { - if (mMode == LatchMode::kPessimisticExclusive) { - WorkerCounters::MyCounters().dt_goto_page_exec[mBTree.mTreeId]++; - } else { - WorkerCounters::MyCounters().dt_goto_page_shared[mBTree.mTreeId]++; - } - } - while (true) { mLeafPosInParent = -1; JUMPMU_TRY() { @@ -554,10 +527,6 @@ inline void PessimisticIterator::seekToTargetPage( mBTree.mStore->mBufferManager.get(), mGuardedParent, mGuardedParent->mRightMostChildSwip); for (uint16_t level = 0; !mGuardedLeaf->mIsLeaf; level++) { - COUNTERS_BLOCK() { - WorkerCounters::MyCounters().dt_inner_page[mBTree.mTreeId]++; - } - mLeafPosInParent = childPosGetter(mGuardedLeaf); auto* childSwip = mGuardedLeaf->ChildSwipIncludingRightMost(mLeafPosInParent); diff --git a/include/leanstore/buffer-manager/BufferManager.hpp b/include/leanstore/buffer-manager/BufferManager.hpp index bc010071..66f181ca 100644 --- a/include/leanstore/buffer-manager/BufferManager.hpp +++ b/include/leanstore/buffer-manager/BufferManager.hpp @@ -6,7 +6,6 @@ #include "leanstore/buffer-manager/PageEvictor.hpp" #include "leanstore/buffer-manager/Partition.hpp" #include "leanstore/buffer-manager/Swip.hpp" -#include "leanstore/profiling/tables/BMTable.hpp" #include "leanstore/utils/RandomGenerator.hpp" #include "leanstore/utils/Result.hpp" @@ -19,10 +18,6 @@ namespace leanstore { class LeanStore; } // namespace leanstore -namespace leanstore::profiling { -class BMTable; -} // namespace leanstore::profiling - namespace leanstore::storage { template class GuardedBufferFrame; @@ -171,7 +166,6 @@ class BufferManager { } friend class leanstore::LeanStore; - friend class leanstore::profiling::BMTable; }; } // namespace leanstore::storage diff --git a/include/leanstore/buffer-manager/FreeList.hpp b/include/leanstore/buffer-manager/FreeList.hpp index 2d3d3d5d..bbdbc86f 100644 --- a/include/leanstore/buffer-manager/FreeList.hpp +++ b/include/leanstore/buffer-manager/FreeList.hpp @@ -1,7 +1,7 @@ #pragma once -#include "leanstore/Exceptions.hpp" #include "leanstore/buffer-manager/BufferFrame.hpp" +#include "leanstore/utils/Log.hpp" #include @@ -22,8 +22,8 @@ class FreeList { }; inline void FreeList::PushFront(BufferFrame& bf) { - PARANOID(bf.mHeader.mState == State::kFree); - assert(!bf.mHeader.mLatch.IsLockedExclusively()); + LS_DCHECK(bf.mHeader.mState == State::kFree); + LS_DCHECK(!bf.mHeader.mLatch.IsLockedExclusively()); JumpScoped> guard(mMutex); bf.mHeader.mNextFreeBf = mHead; @@ -46,7 +46,7 @@ inline BufferFrame& FreeList::PopFrontMayJump() { } else { mHead = mHead->mHeader.mNextFreeBf; mSize--; - PARANOID(freeBf->mHeader.mState == State::kFree); + LS_DCHECK(freeBf->mHeader.mState == State::kFree); } return *freeBf; } diff --git a/include/leanstore/concurrency/ConcurrencyControl.hpp b/include/leanstore/concurrency/ConcurrencyControl.hpp index edda4119..41b3acf2 100644 --- a/include/leanstore/concurrency/ConcurrencyControl.hpp +++ b/include/leanstore/concurrency/ConcurrencyControl.hpp @@ -3,10 +3,8 @@ #include "leanstore/LeanStore.hpp" #include "leanstore/Units.hpp" #include "leanstore/concurrency/HistoryStorage.hpp" -#include "leanstore/profiling/counters/CRCounters.hpp" #include "leanstore/sync/HybridLatch.hpp" #include "leanstore/utils/Log.hpp" -#include "leanstore/utils/Misc.hpp" #include #include @@ -194,7 +192,6 @@ class ConcurrencyControl { //! @return: true if the version is found, false otherwise. inline bool GetVersion(WORKERID newerWorkerId, TXID newerTxId, COMMANDID newerCommandId, std::function getCallback) { - utils::Timer timer(CRCounters::MyCounters().cc_ms_history_tree_retrieve); auto isRemoveCommand = newerCommandId & kRemoveCommandMark; return Other(newerWorkerId) .mHistoryStorage.GetVersion(newerTxId, newerCommandId, isRemoveCommand, getCallback); diff --git a/include/leanstore/concurrency/Logging.hpp b/include/leanstore/concurrency/Logging.hpp index 5f64d8a6..d000da62 100644 --- a/include/leanstore/concurrency/Logging.hpp +++ b/include/leanstore/concurrency/Logging.hpp @@ -1,8 +1,10 @@ #pragma once +#include "leanstore-c/PerfCounters.h" #include "leanstore/Units.hpp" #include "leanstore/concurrency/Transaction.hpp" #include "leanstore/sync/OptimisticGuarded.hpp" +#include "leanstore/utils/CounterUtil.hpp" #include #include @@ -107,8 +109,10 @@ class Logging { mSignaledCommitTs.store(signaledCommitTs, std::memory_order_release); } - bool SafeToCommit(const TXID commitTs) { - return commitTs <= mSignaledCommitTs.load(); + void WaitToCommit(const TXID commitTs) { + COUNTER_INC(&tlsPerfCounters.mTxCommitWait); + while (!(commitTs <= mSignaledCommitTs.load())) { + } } void ReserveContiguousBuffer(uint32_t requestedSize); diff --git a/include/leanstore/concurrency/WorkerContext.hpp b/include/leanstore/concurrency/WorkerContext.hpp index 097f6be6..8dc00d6a 100644 --- a/include/leanstore/concurrency/WorkerContext.hpp +++ b/include/leanstore/concurrency/WorkerContext.hpp @@ -1,5 +1,6 @@ #pragma once +#include "leanstore-c/PerfCounters.h" #include "leanstore/Units.hpp" #include "leanstore/concurrency/ConcurrencyControl.hpp" #include "leanstore/concurrency/Logging.hpp" @@ -47,9 +48,11 @@ class WorkerContext { //! All the workers. std::vector& mAllWorkers; + //! Construct a WorkerContext. WorkerContext(uint64_t workerId, std::vector& allWorkers, leanstore::LeanStore* store); + //! Destruct a WorkerContext. ~WorkerContext(); //! Whether a user transaction is started. @@ -67,6 +70,9 @@ class WorkerContext { //! Aborts a user transaction. void AbortTx(); + //! Get the PerfCounters of the current worker. + PerfCounters* GetPerfCounters(); + public: //! Thread-local storage for WorkerContext. static thread_local std::unique_ptr sTlsWorkerCtx; diff --git a/include/leanstore/concurrency/WorkerThread.hpp b/include/leanstore/concurrency/WorkerThread.hpp index 34ed6f59..ac135fde 100644 --- a/include/leanstore/concurrency/WorkerThread.hpp +++ b/include/leanstore/concurrency/WorkerThread.hpp @@ -2,9 +2,6 @@ #include "leanstore/LeanStore.hpp" #include "leanstore/Units.hpp" -#include "leanstore/profiling/counters/CPUCounters.hpp" -#include "leanstore/profiling/counters/CRCounters.hpp" -#include "leanstore/profiling/counters/WorkerCounters.hpp" #include "leanstore/utils/UserThread.hpp" #include @@ -78,13 +75,6 @@ class WorkerThread : public utils::UserThread { }; inline void WorkerThread::runImpl() { - if (utils::tlsStore->mStoreOption->mEnableCpuCounters) { - CPUCounters::registerThread(mThreadName, false); - } - - WorkerCounters::MyCounters().mWorkerId = mWorkerId; - CRCounters::MyCounters().mWorkerId = mWorkerId; - while (mKeepRunning) { // wait until there is a job std::unique_lock guard(mMutex); diff --git a/include/leanstore/profiling/counters/CPUCounters.hpp b/include/leanstore/profiling/counters/CPUCounters.hpp deleted file mode 100644 index dc214307..00000000 --- a/include/leanstore/profiling/counters/CPUCounters.hpp +++ /dev/null @@ -1,31 +0,0 @@ -#pragma once - -#include -#include -#include - -class PerfEvent; - -namespace leanstore { - -// NOLINTBEGIN - -struct CPUCounters { - std::unique_ptr e; - - std::string name; - - static uint64_t id; - - static std::unordered_map threads; - - static std::mutex mutex; - - static uint64_t registerThread(std::string name, bool perf_inherit = false); - - static void removeThread(uint64_t id); -}; - -// NOLINTEND - -} // namespace leanstore diff --git a/include/leanstore/profiling/counters/CRCounters.hpp b/include/leanstore/profiling/counters/CRCounters.hpp deleted file mode 100644 index 0c340de3..00000000 --- a/include/leanstore/profiling/counters/CRCounters.hpp +++ /dev/null @@ -1,72 +0,0 @@ -#pragma once - -#include "leanstore/utils/EnumerableThreadLocal.hpp" - -#include - -namespace leanstore { - -struct CRCounters { - std::atomic mWorkerId = -1; - std::atomic written_log_bytes = 0; - std::atomic wal_reserve_blocked = 0; - std::atomic wal_reserve_immediate = 0; - - std::atomic gct_total_ms = 0; - std::atomic gct_phase_1_ms = 0; - std::atomic gct_phase_2_ms = 0; - std::atomic gct_write_ms = 0; - std::atomic gct_write_bytes = 0; - - std::atomic gct_rounds = 0; - std::atomic gct_committed_tx = 0; - std::atomic rfa_committed_tx = 0; - - std::atomic cc_prepare_igc = 0; - std::atomic cc_cross_workers_visibility_check = 0; - std::atomic cc_versions_space_removed = {0}; - std::atomic cc_snapshot_restart = 0; - - // Time - std::atomic cc_ms_snapshotting = 0; // Everything related to commit log - std::atomic cc_ms_gc = 0; - std::atomic cc_ms_gc_graveyard = 0; - std::atomic cc_ms_gc_history_tree = 0; - std::atomic cc_ms_gc_cm = 0; - std::atomic cc_ms_committing = 0; - std::atomic cc_ms_history_tree_insert = 0; - std::atomic cc_ms_history_tree_retrieve = 0; - std::atomic cc_ms_refresh_global_state = 0; - - std::atomic cc_ms_oltp_tx = 0; - std::atomic cc_ms_olap_tx = 0; - std::atomic cc_ms_fat_tuple = 0; - std::atomic cc_ms_fat_tuple_conversion = 0; - - std::atomic cc_ms_start_tx = 0; - std::atomic cc_ms_commit_tx = 0; - std::atomic cc_ms_abort_tx = 0; - - // Latency - // ATTENTION: buffer overflow if more than max_dt_id in system are - // registered - static constexpr uint64_t latency_tx_capacity = 1024; - std::atomic cc_ms_precommit_latency[latency_tx_capacity] = {0}; - std::atomic cc_ms_commit_latency[latency_tx_capacity] = {0}; - std::atomic cc_flushes_counter[latency_tx_capacity] = {0}; - std::atomic cc_latency_cursor = {0}; - std::atomic cc_rfa_ms_precommit_latency[latency_tx_capacity] = {0}; - std::atomic cc_rfa_ms_commit_latency[latency_tx_capacity] = {0}; - std::atomic cc_rfa_latency_cursor = {0}; - - CRCounters() { - } - - static utils::EnumerableThreadLocal sCounters; - - static CRCounters& MyCounters() { - return *sCounters.Local(); - } -}; - -} // namespace leanstore diff --git a/include/leanstore/profiling/counters/PPCounters.hpp b/include/leanstore/profiling/counters/PPCounters.hpp deleted file mode 100644 index 0a0b30dc..00000000 --- a/include/leanstore/profiling/counters/PPCounters.hpp +++ /dev/null @@ -1,59 +0,0 @@ -#pragma once - -#include "leanstore/utils/EnumerableThreadLocal.hpp" - -#include - -namespace leanstore { - -// NOLINTBEGIN - -class PPCounters { -public: - PPCounters() = default; - -public: - // ATTENTION: These counters should be only used by page evictor threads or - // slow path worker code - std::atomic mPhase1MS = 0; - - std::atomic mPhase2MS = 0; - - std::atomic mPhase3MS = 0; - - // Phase 1 detailed - std::atomic mFindParentMS = 0; - - std::atomic mIterateChildrenMS = 0; - - // Phase 3 detailed - std::atomic async_wb_ms = 0; - - std::atomic submit_ms = 0; - - std::atomic phase_1_counter = 0; - - std::atomic phase_2_counter = 0; - - std::atomic phase_3_counter = 0; - - std::atomic evicted_pages = 0; - - std::atomic pp_thread_rounds = 0; - - std::atomic touched_bfs_counter = 0; - - std::atomic flushed_pages_counter = 0; - - std::atomic unswizzled_pages_counter = 0; - - static utils::EnumerableThreadLocal sCounters; - - static PPCounters& MyCounters() { - return *sCounters.Local(); - } -}; - -// NOLINTEND - -} // namespace leanstore diff --git a/include/leanstore/profiling/counters/WorkerCounters.hpp b/include/leanstore/profiling/counters/WorkerCounters.hpp deleted file mode 100644 index 51ebf681..00000000 --- a/include/leanstore/profiling/counters/WorkerCounters.hpp +++ /dev/null @@ -1,131 +0,0 @@ -#pragma once - -#include "leanstore/utils/EnumerableThreadLocal.hpp" - -#include - -namespace leanstore { - -struct WorkerCounters { - static constexpr uint64_t max_researchy_counter = 10; - - // ATTENTION: buffer overflow if more than max_dt_id in system are registered - static constexpr uint64_t max_dt_id = 1000; - - std::atomic t_id = 9999; // used by tpcc - std::atomic variable_for_workload = 0; // Used by tpcc - - std::atomic mWorkerId = -1; - - std::atomic hot_hit_counter = 0; // TODO: give it a try ? - std::atomic cold_hit_counter = 0; - std::atomic read_operations_counter = 0; - std::atomic allocate_operations_counter = 0; - std::atomic mNumContentions = 0; - std::atomic tx = 0; - std::atomic long_running_tx = 0; - std::atomic olap_scanned_tuples = 0; - std::atomic tx_abort = 0; - std::atomic olap_tx_abort = 0; - std::atomic tmp = 0; - - // Space and contention management - std::atomic contention_split_succ_counter[max_dt_id] = {0}; - std::atomic contention_split_fail_counter[max_dt_id] = {0}; - std::atomic dt_split[max_dt_id] = {0}; - std::atomic dt_merge_succ[max_dt_id] = {0}; - std::atomic dt_merge_parent_succ[max_dt_id] = {0}; - std::atomic dt_merge_fail[max_dt_id] = {0}; - std::atomic dt_merge_parent_fail[max_dt_id] = {0}; - std::atomic xmerge_partial_counter[max_dt_id] = {0}; - std::atomic xmerge_full_counter[max_dt_id] = {0}; - - std::atomic dt_page_reads[max_dt_id] = {0}; - std::atomic dt_page_writes[max_dt_id] = {0}; - - // without structural change - std::atomic dt_restarts_update_same_size[max_dt_id] = {0}; - - // includes insert, remove, update with different size - std::atomic dt_restarts_structural_change[max_dt_id] = {0}; - std::atomic dt_restarts_read[max_dt_id] = {0}; - - // temporary counter used to track some value for an idea in My mind - std::atomic dt_researchy[max_dt_id][max_researchy_counter] = {}; - - std::atomic dt_find_parent[max_dt_id] = {0}; - std::atomic dt_find_parent_root[max_dt_id] = {0}; - std::atomic dt_find_parent_fast[max_dt_id] = {0}; - std::atomic dt_find_parent_slow[max_dt_id] = {0}; - - std::atomic dt_empty_leaf[max_dt_id] = {0}; - std::atomic dt_goto_page_exec[max_dt_id] = {0}; - std::atomic dt_goto_page_shared[max_dt_id] = {0}; - std::atomic dt_next_tuple[max_dt_id] = {0}; - std::atomic dt_next_tuple_opt[max_dt_id] = {0}; - std::atomic dt_prev_tuple[max_dt_id] = {0}; - std::atomic dt_prev_tuple_opt[max_dt_id] = {0}; - std::atomic dt_inner_page[max_dt_id] = {0}; - std::atomic dt_scan_asc[max_dt_id] = {0}; - std::atomic dt_scan_desc[max_dt_id] = {0}; - std::atomic dt_scan_callback[max_dt_id] = {0}; - // ------------------------------------------------------------------------------------- - std::atomic dt_range_removed[max_dt_id] = {0}; - std::atomic dt_append[max_dt_id] = {0}; - std::atomic dt_append_opt[max_dt_id] = {0}; - // ------------------------------------------------------------------------------------- - std::atomic cc_read_versions_visited[max_dt_id] = {0}; - std::atomic cc_read_versions_visited_not_found[max_dt_id] = {0}; - std::atomic cc_read_chains_not_found[max_dt_id] = {0}; - std::atomic cc_read_chains[max_dt_id] = {0}; - // ------------------------------------------------------------------------------------- - std::atomic cc_update_versions_visited[max_dt_id] = {0}; - std::atomic cc_update_versions_removed[max_dt_id] = {0}; - std::atomic cc_update_versions_kept[max_dt_id] = {0}; - std::atomic cc_update_versions_kept_max[max_dt_id] = {0}; - std::atomic cc_update_versions_skipped[max_dt_id] = {0}; - std::atomic cc_update_versions_recycled[max_dt_id] = {0}; - std::atomic cc_update_versions_created[max_dt_id] = {0}; - std::atomic cc_update_chains[max_dt_id] = {0}; - std::atomic cc_update_chains_hwm[max_dt_id] = {0}; - std::atomic cc_update_chains_pgc[max_dt_id] = {0}; - std::atomic cc_update_chains_pgc_skipped[max_dt_id] = {0}; - std::atomic cc_update_chains_pgc_workers_visited[max_dt_id] = {0}; - std::atomic cc_update_chains_pgc_heavy[max_dt_id] = {0}; - std::atomic cc_update_chains_pgc_heavy_removed[max_dt_id] = {0}; - std::atomic cc_update_chains_pgc_light[max_dt_id] = {0}; - std::atomic cc_update_chains_pgc_light_removed[max_dt_id] = {0}; - // ------------------------------------------------------------------------------------- - std::atomic cc_versions_space_inserted[max_dt_id] = {0}; - std::atomic cc_versions_space_inserted_opt[max_dt_id] = {0}; - // ------------------------------------------------------------------------------------- - std::atomic cc_todo_removed[max_dt_id] = {0}; - std::atomic cc_todo_moved_gy[max_dt_id] = {0}; - std::atomic cc_todo_oltp_executed[max_dt_id] = {0}; - std::atomic cc_gc_long_tx_executed[max_dt_id] = {0}; - // ------------------------------------------------------------------------------------- - std::atomic cc_fat_tuple_triggered[max_dt_id] = {0}; - std::atomic cc_fat_tuple_convert[max_dt_id] = {0}; - std::atomic cc_fat_tuple_decompose[max_dt_id] = {0}; - - // WAL - std::atomic wal_write_bytes = 0; - std::atomic wal_read_bytes = 0; - std::atomic wal_buffer_hit = 0; - std::atomic wal_buffer_miss = 0; - - // ------------------------------------------------------------------------------------- - WorkerCounters() { - t_id = sNumWorkers++; - } - - static std::atomic sNumWorkers; - - static utils::EnumerableThreadLocal sCounters; - - static WorkerCounters& MyCounters() { - return *sCounters.Local(); - } -}; - -} // namespace leanstore diff --git a/include/leanstore/profiling/tables/BMTable.hpp b/include/leanstore/profiling/tables/BMTable.hpp deleted file mode 100644 index 34658f4c..00000000 --- a/include/leanstore/profiling/tables/BMTable.hpp +++ /dev/null @@ -1,41 +0,0 @@ -#pragma once - -#include "ProfilingTable.hpp" - -namespace leanstore { - -namespace storage { - -class BufferManager; - -} // namespace storage -namespace profiling { - -class BMTable : public ProfilingTable { -private: - leanstore::storage::BufferManager& bm; - - int64_t local_phase_1_ms = 0; - - int64_t local_phase_2_ms = 0; - - int64_t local_phase_3_ms = 0; - - int64_t local_poll_ms = 0; - - int64_t total; - - uint64_t local_total_free; - - int64_t local_total_cool; - -public: - BMTable(leanstore::storage::BufferManager& bm); - - virtual std::string getName() override; - virtual void open() override; - virtual void next() override; -}; - -} // namespace profiling -} // namespace leanstore diff --git a/include/leanstore/profiling/tables/CPUTable.hpp b/include/leanstore/profiling/tables/CPUTable.hpp deleted file mode 100644 index e1be2ff0..00000000 --- a/include/leanstore/profiling/tables/CPUTable.hpp +++ /dev/null @@ -1,15 +0,0 @@ -#pragma once - -#include "ProfilingTable.hpp" - -namespace leanstore::profiling { - -class CPUTable : public ProfilingTable { -public: - std::unordered_map workers_agg_events, pp_agg_events, ww_agg_events; - virtual std::string getName() override; - virtual void open() override; - virtual void next() override; -}; - -} // namespace leanstore::profiling diff --git a/include/leanstore/profiling/tables/CRTable.hpp b/include/leanstore/profiling/tables/CRTable.hpp deleted file mode 100644 index b1934ffe..00000000 --- a/include/leanstore/profiling/tables/CRTable.hpp +++ /dev/null @@ -1,19 +0,0 @@ -#pragma once -#include "ProfilingTable.hpp" -// ------------------------------------------------------------------------------------- -// ------------------------------------------------------------------------------------- -// ------------------------------------------------------------------------------------- -namespace leanstore { -namespace profiling { -class CRTable : public ProfilingTable { -private: - uint64_t wal_hits, wal_miss; - double p1, p2, total, write, wal_total, wal_hit_pct, wal_miss_pct; - -public: - virtual std::string getName() override; - virtual void open() override; - virtual void next() override; -}; -} // namespace profiling -} // namespace leanstore diff --git a/include/leanstore/profiling/tables/ConfigsTable.hpp b/include/leanstore/profiling/tables/ConfigsTable.hpp deleted file mode 100644 index 4837b6ce..00000000 --- a/include/leanstore/profiling/tables/ConfigsTable.hpp +++ /dev/null @@ -1,17 +0,0 @@ -#pragma once -#include "ProfilingTable.hpp" -// ------------------------------------------------------------------------------------- -// ------------------------------------------------------------------------------------- -// ------------------------------------------------------------------------------------- -namespace leanstore { -namespace profiling { -class ConfigsTable : public ProfilingTable { -public: - virtual std::string getName() override; - virtual void open() override; - virtual void next() override; - uint64_t hash(); - void add(std::string name, std::string value); -}; -} // namespace profiling -} // namespace leanstore diff --git a/include/leanstore/profiling/tables/DTTable.hpp b/include/leanstore/profiling/tables/DTTable.hpp deleted file mode 100644 index 9c5299b9..00000000 --- a/include/leanstore/profiling/tables/DTTable.hpp +++ /dev/null @@ -1,22 +0,0 @@ -#pragma once - -#include "ProfilingTable.hpp" -#include "leanstore/buffer-manager/BufferManager.hpp" - -namespace leanstore::profiling { - -class DTTable : public ProfilingTable { -private: - std::string dt_name; - TREEID mTreeId; - leanstore::storage::BufferManager& bm; - -public: - DTTable(leanstore::storage::BufferManager& bm); - - virtual std::string getName() override; - virtual void open() override; - virtual void next() override; -}; - -} // namespace leanstore::profiling diff --git a/include/leanstore/profiling/tables/LatencyTable.hpp b/include/leanstore/profiling/tables/LatencyTable.hpp deleted file mode 100644 index 47ef70d7..00000000 --- a/include/leanstore/profiling/tables/LatencyTable.hpp +++ /dev/null @@ -1,14 +0,0 @@ -#pragma once - -#include "ProfilingTable.hpp" - -namespace leanstore::profiling { - -class LatencyTable : public ProfilingTable { -public: - virtual std::string getName() override; - virtual void open() override; - virtual void next() override; -}; - -} // namespace leanstore::profiling diff --git a/include/leanstore/profiling/tables/ProfilingTable.hpp b/include/leanstore/profiling/tables/ProfilingTable.hpp deleted file mode 100644 index f98185ea..00000000 --- a/include/leanstore/profiling/tables/ProfilingTable.hpp +++ /dev/null @@ -1,114 +0,0 @@ -#pragma once - -#include "leanstore/Exceptions.hpp" -#include "leanstore/Units.hpp" - -#include -#include -#include -#include -#include -#include - -namespace leanstore { -namespace profiling { - -struct Column { - std::function generator; - - std::vector values; - - Column(std::function&& g) : generator(g) { - } - - std::string to_string(int8_t x) { - return std::to_string(x); - } - std::string to_string(int16_t x) { - return std::to_string(x); - } - std::string to_string(int32_t x) { - return std::to_string(x); - } - std::string to_string(int64_t x) { - return std::to_string(x); - } - std::string to_string(uint8_t x) { - return std::to_string(x); - } - std::string to_string(uint16_t x) { - return std::to_string(x); - } - std::string to_string(uint32_t x) { - return std::to_string(x); - } - std::string to_string(uint64_t x) { - return std::to_string(x); - } - std::string to_string(double x) { - std::stringstream stream; - stream << std::fixed << std::setprecision(1) << x; - return stream.str(); - } - std::string to_string(float x) { - std::stringstream stream; - stream << std::fixed << std::setprecision(1) << x; - return stream.str(); - } - std::string to_string(std::string x) { - return x; - } - - template - Column& operator<<(T x) { - values.push_back(to_string(x)); - return *this; - } -}; - -using ColumnGenerator = std::function; -using ColumnValues = std::vector; - -class ProfilingTable { -protected: - std::unordered_map columns; - -public: - // Open -> getColumns() -> next -> getColumns() - void clear() { - for (auto& column : columns) { - column.second.values.clear(); - } - } - uint64_t size() { - return columns.begin()->second.values.size(); - } - std::string get(std::string key, std::string column) { - auto& c = columns.at("key"); - for (uint64_t r_i = 0; r_i < size(); r_i++) { - if (c.values[r_i] == key) { - return columns.at(column).values[r_i]; - } - } - ENSURE(false); - } - double getDouble(std::string key, std::string column) { - return std::stod(get(key, column)); - } - virtual std::string getName() { - return "null"; - } - virtual void open() { - } - virtual void next() { - } - virtual std::unordered_map& getColumns() { - return columns; - } - Column& operator[](std::string name) { - return columns.at(name); - } -}; - -} // namespace profiling -} // namespace leanstore diff --git a/include/leanstore/telemetry/MetricOnlyTimer.hpp b/include/leanstore/telemetry/MetricOnlyTimer.hpp deleted file mode 100644 index 24516b36..00000000 --- a/include/leanstore/telemetry/MetricOnlyTimer.hpp +++ /dev/null @@ -1,37 +0,0 @@ -#pragma once - -#include "leanstore/Exceptions.hpp" - -#include - -namespace leanstore::telemetry { - -class MetricOnlyTimer { -public: - typedef std::chrono::steady_clock::time_point TimePoint; - -public: - MetricOnlyTimer() { - COUNTERS_BLOCK() { - mStartedAt = std::chrono::steady_clock::now(); - } - } - - ~MetricOnlyTimer() = default; - - double ElaspedUs() { - double elaspedUs{0}; - COUNTERS_BLOCK() { - auto stoppedAt = std::chrono::steady_clock::now(); - auto elaspedNs = - std::chrono::duration_cast(stoppedAt - mStartedAt).count(); - elaspedUs = elaspedNs / 1000.0; - } - return elaspedUs; - } - -private: - std::chrono::steady_clock::time_point mStartedAt; -}; - -} // namespace leanstore::telemetry diff --git a/include/leanstore/utils/CounterUtil.hpp b/include/leanstore/utils/CounterUtil.hpp new file mode 100644 index 00000000..35c31e12 --- /dev/null +++ b/include/leanstore/utils/CounterUtil.hpp @@ -0,0 +1,75 @@ +#pragma once + +#include "leanstore-c/PerfCounters.h" + +#ifdef ENABLE_PERF_COUNTERS +#include +#endif + +namespace leanstore { + +namespace cr { +extern thread_local PerfCounters tlsPerfCounters; +} // namespace cr + +//! ScopedTimer for perf counters +class ScopedTimer { +private: +#ifdef ENABLE_PERF_COUNTERS + //! Counter to cumulate the time elasped + CounterType* mCounterToCum; + + //! Start timepoint + std::chrono::steady_clock::time_point mStartedAt; +#endif + +public: + ScopedTimer(CounterType* counter [[maybe_unused]]) { +#ifdef ENABLE_PERF_COUNTERS + mCounterToCum = counter; + mStartedAt = std::chrono::steady_clock::now(); +#endif + } + + ~ScopedTimer() { +#ifdef ENABLE_PERF_COUNTERS + auto stoppedAt = std::chrono::steady_clock::now(); + auto elaspedNs = + std::chrono::duration_cast(stoppedAt - mStartedAt).count(); + *mCounterToCum += elaspedNs; +#endif + } +}; + +#ifdef ENABLE_PERF_COUNTERS + +//------------------------------------------------------------------------------ +// Macros when counters are enabled +//------------------------------------------------------------------------------ + +#define SCOPED_TIME_INTERNAL_INTERNAL(LINE) scopedTimerAtLine##LINE +#define SCOPED_TIME_INTERNAL(LINE) SCOPED_TIME_INTERNAL_INTERNAL(LINE) + +//! Macro to create a ScopedTimer +#define COUNTER_TIMER_SCOPED(counter) \ + leanstore::ScopedTimer SCOPED_TIME_INTERNAL(__LINE__){counter}; + +//! Macro to inc a counter +#define COUNTER_INC(counter) atomic_fetch_add(counter, 1); + +//! Macro to declare a block of code that will be executed only if counters are enabled +#define COUNTERS_BLOCK() if constexpr (true) + +#else + +//------------------------------------------------------------------------------ +// Macros when counters are disabled +//------------------------------------------------------------------------------ + +#define COUNTER_TIMER_SCOPED(counter) +#define COUNTER_INC(counter) +#define COUNTERS_BLOCK() if constexpr (false) + +#endif + +} // namespace leanstore \ No newline at end of file diff --git a/include/leanstore/utils/EnumerableThreadLocal.hpp b/include/leanstore/utils/EnumerableThreadLocal.hpp deleted file mode 100644 index 20e731ea..00000000 --- a/include/leanstore/utils/EnumerableThreadLocal.hpp +++ /dev/null @@ -1,85 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include - -namespace leanstore { -namespace utils { - -template -class EnumerableThreadLocal { -public: - inline static std::shared_mutex sMutex; - - inline static std::vector> sAllThreadLocals; - - inline static thread_local T* sThreadLocal; - -public: - inline static T* Local() { - if (sThreadLocal == nullptr) { - std::unique_lock guard(sMutex); - sAllThreadLocals.push_back(std::make_unique()); - sThreadLocal = sAllThreadLocals[sAllThreadLocals.size() - 1].get(); - } - return sThreadLocal; - } -}; - -template -inline static SumT Sum(EnumerableThreadLocal& counters, CounterT CountersT::*c) { - std::shared_lock guard(EnumerableThreadLocal::sMutex); - SumT result = 0; - for (auto& counter : counters.sAllThreadLocals) { - result += ((*counter.get()).*c).exchange(0); - } - return result; -} - -template -inline static SumT Sum(EnumerableThreadLocal& counters, CounterT CountersT::*c, - uint64_t row) { - std::shared_lock guard(EnumerableThreadLocal::sMutex); - SumT result = 0; - for (auto& counter : counters.sAllThreadLocals) { - result += ((*counter.get()).*c)[row].exchange(0); - } - return result; -} - -template -inline static SumT Sum(EnumerableThreadLocal& counters, CounterT CountersT::*c, - uint64_t row, uint64_t col) { - std::shared_lock guard(EnumerableThreadLocal::sMutex); - SumT result = 0; - for (auto& counter : counters.sAllThreadLocals) { - result += ((*counter.get()).*c)[row][col].exchange(0); - } - return result; -} - -template -inline static SumT Max(EnumerableThreadLocal& counters, CounterT CountersT::*c, - uint64_t row) { - std::shared_lock guard(EnumerableThreadLocal::sMutex); - SumT result = 0; - for (auto& counter : counters.sAllThreadLocals) { - result = std::max(result, ((*counter.get()).*c)[row].exchange(0)); - } - return result; -} - -template -inline static void ForEach(EnumerableThreadLocal& tlsObjContainer, - std::function tlsObjCallBack) { - std::shared_lock guard(EnumerableThreadLocal::sMutex); - for (auto& tlsObj : tlsObjContainer.sAllThreadLocals) { - tlsObjCallBack(tlsObj.get()); - } -} - -} // namespace utils -} // namespace leanstore \ No newline at end of file diff --git a/include/leanstore/utils/Misc.hpp b/include/leanstore/utils/Misc.hpp index 226b8a6d..d9268b1f 100644 --- a/include/leanstore/utils/Misc.hpp +++ b/include/leanstore/utils/Misc.hpp @@ -3,7 +3,6 @@ #include "leanstore/utils/JumpMU.hpp" #include "leanstore/utils/Log.hpp" -#include #include #include @@ -165,14 +164,4 @@ class AlignedBuffer { } }; -struct Timer { - std::atomic& mTimeCounterUS; - - std::chrono::high_resolution_clock::time_point mStartTimePoint; - - Timer(std::atomic& timeCounterUS); - - ~Timer(); -}; - } // namespace leanstore::utils diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 83c6e1b3..7219e88d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -22,8 +22,6 @@ set(leanstore_deps spdlog::spdlog_header_only Crc32c::crc32c httplib::httplib - prometheus-cpp::core - prometheus-cpp::pull PkgConfig::libunwind ) target_link_libraries(leanstore PUBLIC ${leanstore_deps}) @@ -66,14 +64,10 @@ if(SANI) target_link_libraries(leanstore asan) endif(SANI) -option(PARANOID "Enable sanity checks in release mode" OFF) -if(PARANOID) - target_compile_definitions(leanstore PUBLIC PARANOID) -endif(PARANOID) -option(COUNTERS_LEVEL "Which counters to leave in leanstore build" "all") -if(COUNTERS_LEVEL STREQUAL "all") - target_compile_definitions(leanstore PUBLIC MACRO_COUNTERS_ALL) +option(ENABLE_PERF_COUNTERS "Whether to enable perf counters" OFF) +if(ENABLE_PERF_COUNTERS) + target_compile_definitions(leanstore PUBLIC ENABLE_PERF_COUNTERS) endif() set(CHECKS_LEVEL "default" CACHE STRING "Which checks to leave in leanstore build") diff --git a/src/LeanStore.cpp b/src/LeanStore.cpp index 72a87403..89488059 100644 --- a/src/LeanStore.cpp +++ b/src/LeanStore.cpp @@ -6,15 +6,12 @@ #include "leanstore/btree/core/BTreeGeneric.hpp" #include "leanstore/buffer-manager/BufferManager.hpp" #include "leanstore/concurrency/CRManager.hpp" -#include "leanstore/profiling/tables/BMTable.hpp" #include "leanstore/utils/Defer.hpp" #include "leanstore/utils/Error.hpp" #include "leanstore/utils/Log.hpp" #include "leanstore/utils/Misc.hpp" #include "leanstore/utils/Result.hpp" #include "leanstore/utils/UserThread.hpp" -#include "telemetry/MetricsHttpExposer.hpp" -#include "telemetry/MetricsManager.hpp" #include #include @@ -58,25 +55,12 @@ Result> LeanStore::Open(StoreOption* option) { return std::make_unique(option); } -LeanStore::LeanStore(StoreOption* option) - : mStoreOption(option), - mMetricsManager(nullptr), - mMetricsExposer(nullptr) { +LeanStore::LeanStore(StoreOption* option) : mStoreOption(option) { utils::tlsStore = this; Log::Info("LeanStore starting ..."); SCOPED_DEFER(Log::Info("LeanStore started")); - // Expose the metrics - if (mStoreOption->mEnableMetrics) { - mMetricsManager = std::make_unique(); - - //! Expose the metrics via HTTP - mMetricsExposer = std::make_unique(this); - mMetricsExposer->SetCollectable(mMetricsManager->GetRegistry()); - mMetricsExposer->Start(); - } - initPageAndWalFd(); // create global btree catalog @@ -157,11 +141,6 @@ void LeanStore::initPageAndWalFd() { LeanStore::~LeanStore() { Log::Info("LeanStore stopping ..."); SCOPED_DEFER({ - // stop metrics manager in the last - if (mStoreOption->mEnableMetrics) { - mMetricsManager = nullptr; - mMetricsExposer = nullptr; - } DestroyStoreOption(mStoreOption); Log::Info("LeanStore stopped"); }); diff --git a/src/btree/BasicKV.cpp b/src/btree/BasicKV.cpp index 51d57e9d..60473956 100644 --- a/src/btree/BasicKV.cpp +++ b/src/btree/BasicKV.cpp @@ -45,7 +45,6 @@ OpCode BasicKV::lookupOptimistic(Slice key, ValCallback valCallback) { JUMPMU_RETURN OpCode::kNotFound; } JUMPMU_CATCH() { - WorkerCounters::MyCounters().dt_restarts_read[mTreeId]++; return OpCode::kOther; } } @@ -64,7 +63,6 @@ OpCode BasicKV::lookupPessimistic(Slice key, ValCallback valCallback) { JUMPMU_RETURN OpCode::kNotFound; } JUMPMU_CATCH() { - WorkerCounters::MyCounters().dt_restarts_read[mTreeId]++; } } } @@ -108,10 +106,6 @@ bool BasicKV::IsRangeEmpty(Slice startKey, Slice endKey) { } OpCode BasicKV::ScanAsc(Slice startKey, ScanCallback callback) { - COUNTERS_BLOCK() { - WorkerCounters::MyCounters().dt_scan_asc[mTreeId]++; - } - JUMPMU_TRY() { auto iter = GetIterator(); if (iter.SeekToFirstGreaterEqual(startKey); !iter.Valid()) { @@ -135,9 +129,6 @@ OpCode BasicKV::ScanAsc(Slice startKey, ScanCallback callback) { } OpCode BasicKV::ScanDesc(Slice scanKey, ScanCallback callback) { - COUNTERS_BLOCK() { - WorkerCounters::MyCounters().dt_scan_desc[mTreeId]++; - } JUMPMU_TRY() { auto iter = GetIterator(); if (iter.SeekToLastLessEqual(scanKey); !iter.Valid()) { @@ -220,7 +211,6 @@ OpCode BasicKV::PrefixLookup(Slice key, PrefixLookupCallback callback) { JUMPMU_RETURN ret; } JUMPMU_CATCH() { - WorkerCounters::MyCounters().dt_restarts_read[mTreeId]++; } } @@ -262,7 +252,6 @@ OpCode BasicKV::PrefixLookupForPrev(Slice key, PrefixLookupCallback callback) { JUMPMU_RETURN ret; } JUMPMU_CATCH() { - WorkerCounters::MyCounters().dt_restarts_read[mTreeId]++; } } @@ -364,9 +353,6 @@ OpCode BasicKV::RangeRemove(Slice startKey, Slice endKey, bool pageWise) { xIter.AssembleKey(); auto currentKey = xIter.Key(); if (currentKey >= startKey && currentKey <= endKey) { - COUNTERS_BLOCK() { - WorkerCounters::MyCounters().dt_range_removed[mTreeId]++; - } auto ret = xIter.RemoveCurrent(); ENSURE(ret == OpCode::kOK); if (xIter.mSlotId == xIter.mGuardedLeaf->mNumSlots) { @@ -400,9 +386,6 @@ OpCode BasicKV::RangeRemove(Slice startKey, Slice endKey, bool pageWise) { if (pageStartKey >= startKey && pageEndKey <= endKey) { // Purge the whole page - COUNTERS_BLOCK() { - WorkerCounters::MyCounters().dt_range_removed[mTreeId] += guardedLeaf->mNumSlots; - } guardedLeaf->Reset(); didPurgeFullPage = true; } diff --git a/src/btree/ChainedTuple.cpp b/src/btree/ChainedTuple.cpp index a5a069c0..50f959f2 100644 --- a/src/btree/ChainedTuple.cpp +++ b/src/btree/ChainedTuple.cpp @@ -117,9 +117,6 @@ void ChainedTuple::Update(PessimisticExclusiveIterator& xIter, Slice key, SCOPED_DEFER({ WriteUnlock(); xIter.UpdateContentionStats(); - COUNTERS_BLOCK() { - WorkerCounters::MyCounters().cc_update_versions_created[treeId]++; - } }); if (!xIter.mBTree.mConfig.mEnableWal) { diff --git a/src/btree/TransactionKV.cpp b/src/btree/TransactionKV.cpp index 5e0d294b..7c6c259c 100644 --- a/src/btree/TransactionKV.cpp +++ b/src/btree/TransactionKV.cpp @@ -11,14 +11,12 @@ #include "leanstore/btree/core/PessimisticSharedIterator.hpp" #include "leanstore/concurrency/WorkerContext.hpp" #include "leanstore/sync/HybridGuard.hpp" -#include "leanstore/telemetry/MetricOnlyTimer.hpp" #include "leanstore/utils/Defer.hpp" #include "leanstore/utils/Error.hpp" #include "leanstore/utils/Log.hpp" #include "leanstore/utils/Misc.hpp" #include "leanstore/utils/Result.hpp" #include "leanstore/utils/UserThread.hpp" -#include "telemetry/MetricsManager.hpp" #include #include @@ -57,7 +55,6 @@ OpCode TransactionKV::lookupOptimistic(Slice key, ValCallback valCallback) { auto slotId = guardedLeaf->LowerBound(key); if (slotId != -1) { auto [ret, versionsRead] = getVisibleTuple(guardedLeaf->Value(slotId), valCallback); - METRIC_COUNTER_INC(mStore->mMetricsManager, tx_version_read_total, versionsRead); guardedLeaf.JumpIfModifiedByOthers(); JUMPMU_RETURN ret; } @@ -66,7 +63,6 @@ OpCode TransactionKV::lookupOptimistic(Slice key, ValCallback valCallback) { JUMPMU_RETURN OpCode::kNotFound; } JUMPMU_CATCH() { - WorkerCounters::MyCounters().dt_restarts_read[mTreeId]++; } // lock optimistically failed, return kOther to retry @@ -74,11 +70,6 @@ OpCode TransactionKV::lookupOptimistic(Slice key, ValCallback valCallback) { } OpCode TransactionKV::Lookup(Slice key, ValCallback valCallback) { - telemetry::MetricOnlyTimer timer; - SCOPED_DEFER({ - METRIC_COUNTER_INC(mStore->mMetricsManager, tx_kv_lookup_total, 1); - METRIC_HIST_OBSERVE(mStore->mMetricsManager, tx_kv_lookup_us, timer.ElaspedUs()); - }); LS_DCHECK(cr::WorkerContext::My().IsTxStarted(), "WorkerContext is not in a transaction, workerId={}, startTs={}", cr::WorkerContext::My().mWorkerId, cr::WorkerContext::My().mActiveTx.mStartTs); @@ -88,7 +79,6 @@ OpCode TransactionKV::Lookup(Slice key, ValCallback valCallback) { return OpCode::kNotFound; } auto [ret, versionsRead] = getVisibleTuple(gIter.Val(), valCallback); - METRIC_COUNTER_INC(mStore->mMetricsManager, tx_version_read_total, versionsRead); return ret; }; @@ -111,7 +101,6 @@ OpCode TransactionKV::Lookup(Slice key, ValCallback valCallback) { } auto [ret, versionsRead] = getVisibleTuple(iter.Val(), valCallback); - METRIC_COUNTER_INC(mStore->mMetricsManager, tx_version_read_total, versionsRead); if (cr::ActiveTx().IsLongRunning() && ret == OpCode::kNotFound) { ret = lookupInGraveyard(); } @@ -120,11 +109,6 @@ OpCode TransactionKV::Lookup(Slice key, ValCallback valCallback) { OpCode TransactionKV::UpdatePartial(Slice key, MutValCallback updateCallBack, UpdateDesc& updateDesc) { - telemetry::MetricOnlyTimer timer; - SCOPED_DEFER({ - METRIC_COUNTER_INC(mStore->mMetricsManager, tx_kv_update_total, 1); - METRIC_HIST_OBSERVE(mStore->mMetricsManager, tx_kv_update_us, timer.ElaspedUs()); - }); LS_DCHECK(cr::WorkerContext::My().IsTxStarted()); JUMPMU_TRY() { auto xIter = GetExclusiveIterator(); @@ -608,7 +592,6 @@ void TransactionKV::undoLastRemove(const WalTxRemove* walRemove) { bool TransactionKV::UpdateInFatTuple(PessimisticExclusiveIterator& xIter, Slice key, MutValCallback updateCallBack, UpdateDesc& updateDesc) { - utils::Timer timer(CRCounters::MyCounters().cc_ms_fat_tuple); while (true) { auto* fatTuple = reinterpret_cast(xIter.MutableVal().Data()); LS_DCHECK(fatTuple->IsWriteLocked(), "Tuple should be write locked"); @@ -823,9 +806,6 @@ void TransactionKV::GarbageCollect(const uint8_t* versionData, WORKERID versionW "versionWorkerId={}, versionTxId={}, removedKey={}", ToString(ret), versionWorkerId, versionTxId, removedKey.ToString()); xIter.TryMergeIfNeeded(); - COUNTERS_BLOCK() { - WorkerCounters::MyCounters().cc_todo_moved_gy[mTreeId]++; - } } else { Log::Fatal("Meet a remove version upper than mCc.mLocalWmkOfShortTx, " "should not happen, mCc.mLocalWmkOfShortTx={}, " @@ -889,14 +869,6 @@ void TransactionKV::unlock(const uint8_t* walEntryPtr) { // TODO: index range lock for serializability template OpCode TransactionKV::scan4ShortRunningTx(Slice key, ScanCallback callback) { - COUNTERS_BLOCK() { - if constexpr (asc) { - WorkerCounters::MyCounters().dt_scan_asc[mTreeId]++; - } else { - WorkerCounters::MyCounters().dt_scan_desc[mTreeId]++; - } - } - bool keepScanning = true; JUMPMU_TRY() { auto iter = GetIterator(); @@ -909,20 +881,8 @@ OpCode TransactionKV::scan4ShortRunningTx(Slice key, ScanCallback callback) { while (iter.Valid()) { iter.AssembleKey(); Slice scannedKey = iter.Key(); - auto [opCode, versionsRead] = getVisibleTuple(iter.Val(), [&](Slice scannedVal) { - COUNTERS_BLOCK() { - WorkerCounters::MyCounters().dt_scan_callback[mTreeId] += cr::ActiveTx().IsLongRunning(); - } - keepScanning = callback(scannedKey, scannedVal); - }); - COUNTERS_BLOCK() { - WorkerCounters::MyCounters().cc_read_chains[mTreeId]++; - WorkerCounters::MyCounters().cc_read_versions_visited[mTreeId] += versionsRead; - if (opCode != OpCode::kOK) { - WorkerCounters::MyCounters().cc_read_chains_not_found[mTreeId]++; - WorkerCounters::MyCounters().cc_read_versions_visited_not_found[mTreeId] += versionsRead; - } - } + getVisibleTuple(iter.Val(), + [&](Slice scannedVal) { keepScanning = callback(scannedKey, scannedVal); }); if (!keepScanning) { JUMPMU_RETURN OpCode::kOK; } @@ -944,14 +904,6 @@ OpCode TransactionKV::scan4ShortRunningTx(Slice key, ScanCallback callback) { // TODO: support scanning desc template OpCode TransactionKV::scan4LongRunningTx(Slice key, ScanCallback callback) { - COUNTERS_BLOCK() { - if constexpr (asc) { - WorkerCounters::MyCounters().dt_scan_asc[mTreeId]++; - } else { - WorkerCounters::MyCounters().dt_scan_desc[mTreeId]++; - } - } - bool keepScanning = true; JUMPMU_TRY() { auto iter = GetIterator(); @@ -994,12 +946,7 @@ OpCode TransactionKV::scan4LongRunningTx(Slice key, ScanCallback callback) { gRange(); auto takeFromOltp = [&]() { - getVisibleTuple(iter.Val(), [&](Slice value) { - COUNTERS_BLOCK() { - WorkerCounters::MyCounters().dt_scan_callback[mTreeId] += cr::ActiveTx().IsLongRunning(); - } - keepScanning = callback(iter.Key(), value); - }); + getVisibleTuple(iter.Val(), [&](Slice value) { keepScanning = callback(iter.Key(), value); }); if (!keepScanning) { return false; } @@ -1030,13 +977,7 @@ OpCode TransactionKV::scan4LongRunningTx(Slice key, ScanCallback callback) { } else if (gRet == OpCode::kOK && oRet != OpCode::kOK) { gIter.AssembleKey(); Slice gKey = gIter.Key(); - getVisibleTuple(gIter.Val(), [&](Slice value) { - COUNTERS_BLOCK() { - WorkerCounters::MyCounters().dt_scan_callback[mTreeId] += - cr::ActiveTx().IsLongRunning(); - } - keepScanning = callback(gKey, value); - }); + getVisibleTuple(gIter.Val(), [&](Slice value) { keepScanning = callback(gKey, value); }); if (!keepScanning) { JUMPMU_RETURN OpCode::kOK; } @@ -1052,13 +993,7 @@ OpCode TransactionKV::scan4LongRunningTx(Slice key, ScanCallback callback) { JUMPMU_RETURN OpCode::kOK; } } else { - getVisibleTuple(gIter.Val(), [&](Slice value) { - COUNTERS_BLOCK() { - WorkerCounters::MyCounters().dt_scan_callback[mTreeId] += - cr::ActiveTx().IsLongRunning(); - } - keepScanning = callback(gKey, value); - }); + getVisibleTuple(gIter.Val(), [&](Slice value) { keepScanning = callback(gKey, value); }); if (!keepScanning) { JUMPMU_RETURN OpCode::kOK; } diff --git a/src/btree/Tuple.cpp b/src/btree/Tuple.cpp index 78a19193..16a1adf5 100644 --- a/src/btree/Tuple.cpp +++ b/src/btree/Tuple.cpp @@ -41,8 +41,6 @@ static uint64_t MaxFatTupleLength() { } bool Tuple::ToFat(PessimisticExclusiveIterator& xIter) { - utils::Timer timer(CRCounters::MyCounters().cc_ms_fat_tuple_conversion); - // Process the chain tuple MutableSlice mutRawVal = xIter.MutableVal(); auto& chainedTuple = *ChainedTuple::From(mutRawVal.Data()); @@ -180,8 +178,6 @@ void FatTuple::GarbageCollection() { if (mNumDeltas == 0) { return; } - utils::Timer timer(CRCounters::MyCounters().cc_ms_gc); - auto appendDelta = [](FatTuple& fatTuple, uint8_t* delta, uint16_t deltaSize) { assert(fatTuple.mPayloadCapacity >= (fatTuple.mPayloadSize + deltaSize + sizeof(uint16_t))); const uint16_t i = fatTuple.mNumDeltas++; @@ -432,12 +428,6 @@ void FatTuple::ConvertToChained(TREEID treeId) { } new (this) ChainedTuple(*this); - - COUNTERS_BLOCK() { - WorkerCounters::MyCounters().cc_fat_tuple_decompose[treeId]++; - } } -// TODO: Implement inserts after remove cases - } // namespace leanstore::storage::btree diff --git a/src/btree/core/BTreeGeneric.cpp b/src/btree/core/BTreeGeneric.cpp index 4f589c18..5c6b6b97 100644 --- a/src/btree/core/BTreeGeneric.cpp +++ b/src/btree/core/BTreeGeneric.cpp @@ -9,7 +9,6 @@ #include "leanstore/buffer-manager/BufferFrame.hpp" #include "leanstore/buffer-manager/BufferManager.hpp" #include "leanstore/buffer-manager/GuardedBufferFrame.hpp" -#include "leanstore/profiling/counters/WorkerCounters.hpp" #include "leanstore/utils/Defer.hpp" #include "leanstore/utils/Log.hpp" #include "leanstore/utils/Misc.hpp" @@ -324,9 +323,6 @@ bool BTreeGeneric::TryMergeMayJump(TXID sysTxId, BufferFrame& toMerge, bool swiz TryMergeMayJump(sysTxId, *guardedParent.mBf, true); } JUMPMU_CATCH() { - COUNTERS_BLOCK() { - WorkerCounters::MyCounters().dt_merge_fail[mTreeId]++; - } } } }); @@ -339,14 +335,6 @@ bool BTreeGeneric::TryMergeMayJump(TXID sysTxId, BufferFrame& toMerge, bool swiz succeed = mergeAndReclaimRight(); } - COUNTERS_BLOCK() { - if (succeed) { - WorkerCounters::MyCounters().dt_merge_succ[mTreeId]++; - } else { - WorkerCounters::MyCounters().dt_merge_fail[mTreeId]++; - } - } - return succeed; } @@ -442,7 +430,6 @@ int16_t BTreeGeneric::mergeLeftIntoRight(ExclusiveGuardedBufferFrame& BTreeGeneric::XMergeReturnCode BTreeGeneric::XMerge(GuardedBufferFrame& guardedParent, GuardedBufferFrame& guardedChild, ParentSwipHandler& parentHandler) { - WorkerCounters::MyCounters().dt_researchy[0][1]++; if (guardedChild->FillFactorAfterCompaction() >= 0.9) { return XMergeReturnCode::kNothing; } @@ -519,11 +506,9 @@ BTreeGeneric::XMergeReturnCode BTreeGeneric::XMerge(GuardedBufferFrame 0 || pos2 < sHintCount; - } else { - WorkerCounters::MyCounters().dt_researchy[0][2]++; - WorkerCounters::MyCounters().dt_researchy[0][3] += pos > 0 || pos2 < sHintCount; - } } else { } } diff --git a/src/buffer-manager/AsyncWriteBuffer.cpp b/src/buffer-manager/AsyncWriteBuffer.cpp index b0d7a523..b325de9c 100644 --- a/src/buffer-manager/AsyncWriteBuffer.cpp +++ b/src/buffer-manager/AsyncWriteBuffer.cpp @@ -1,8 +1,6 @@ #include "leanstore/buffer-manager/AsyncWriteBuffer.hpp" -#include "leanstore/Exceptions.hpp" #include "leanstore/buffer-manager/BufferFrame.hpp" -#include "leanstore/profiling/counters/WorkerCounters.hpp" #include "leanstore/utils/Log.hpp" #include "leanstore/utils/Result.hpp" @@ -25,9 +23,6 @@ bool AsyncWriteBuffer::IsFull() { void AsyncWriteBuffer::Add(const BufferFrame& bf) { LS_DCHECK(uint64_t(&bf) % 512 == 0, "BufferFrame is not aligned to 512 bytes"); - COUNTERS_BLOCK() { - WorkerCounters::MyCounters().dt_page_writes[bf.mPage.mBTreeId]++; - } // record the written buffer frame and page id for later use auto pageId = bf.mHeader.mPageId; diff --git a/src/buffer-manager/BufferManager.cpp b/src/buffer-manager/BufferManager.cpp index 957c2aa1..75d82ac9 100644 --- a/src/buffer-manager/BufferManager.cpp +++ b/src/buffer-manager/BufferManager.cpp @@ -1,6 +1,5 @@ #include "leanstore/buffer-manager/BufferManager.hpp" -#include "leanstore/Exceptions.hpp" #include "leanstore/LeanStore.hpp" #include "leanstore/Units.hpp" #include "leanstore/buffer-manager/BufferFrame.hpp" @@ -8,7 +7,6 @@ #include "leanstore/concurrency/CRManager.hpp" #include "leanstore/concurrency/GroupCommitter.hpp" #include "leanstore/concurrency/Recovery.hpp" -#include "leanstore/profiling/counters/WorkerCounters.hpp" #include "leanstore/sync/HybridLatch.hpp" #include "leanstore/sync/ScopedHybridGuard.hpp" #include "leanstore/utils/AsyncIo.hpp" @@ -221,10 +219,6 @@ BufferFrame& BufferManager::AllocNewPageMayJump(TREEID treeId) { new (&freeBf) BufferFrame(); freeBf.Init(partition.NextPageId()); - COUNTERS_BLOCK() { - WorkerCounters::MyCounters().allocate_operations_counter++; - } - freeBf.mPage.mBTreeId = treeId; freeBf.mPage.mPsn++; // mark the page as dirty LS_DLOG("Alloc new page, pageId={}, btreeId={}", freeBf.mHeader.mPageId, freeBf.mPage.mBTreeId); @@ -302,9 +296,6 @@ BufferFrame* BufferManager::ResolveSwipMayJump(HybridGuard& nodeGuard, Swip& swi // 3. Read page at pageId to the target buffer frame ReadPageSync(pageId, &bf.mPage); LS_DLOG("Read page from disk, pageId={}, btreeId={}", pageId, bf.mPage.mBTreeId); - COUNTERS_BLOCK() { - WorkerCounters::MyCounters().dt_page_reads[bf.mPage.mBTreeId]++; - } // 4. Intialize the buffer frame header LS_DCHECK(!bf.mHeader.mIsBeingWrittenBack); @@ -433,10 +424,6 @@ void BufferManager::ReadPageSync(PID pageId, void* pageBuffer) { bytesLeft -= bytesRead; } - - COUNTERS_BLOCK() { - WorkerCounters::MyCounters().read_operations_counter++; - } } BufferFrame& BufferManager::ReadPageSync(PID pageId) { diff --git a/src/buffer-manager/PageEvictor.cpp b/src/buffer-manager/PageEvictor.cpp index 405a1ca5..9a7cf212 100644 --- a/src/buffer-manager/PageEvictor.cpp +++ b/src/buffer-manager/PageEvictor.cpp @@ -2,8 +2,6 @@ #include "leanstore/buffer-manager/BufferManager.hpp" #include "leanstore/buffer-manager/TreeRegistry.hpp" -#include "leanstore/profiling/counters/CPUCounters.hpp" -#include "leanstore/profiling/counters/PPCounters.hpp" #include "leanstore/utils/Defer.hpp" #include "leanstore/utils/Log.hpp" @@ -14,8 +12,6 @@ namespace leanstore::storage { using Time = decltype(std::chrono::high_resolution_clock::now()); void PageEvictor::runImpl() { - CPUCounters::registerThread(mThreadName); - while (mKeepRunning) { auto& targetPartition = mStore->mBufferManager->RandomPartition(); if (!targetPartition.NeedMoreFreeBfs()) { @@ -30,10 +26,6 @@ void PageEvictor::runImpl() { // Phase 3 FlushAndRecycleBufferFrames(targetPartition); - - COUNTERS_BLOCK() { - PPCounters::MyCounters().pp_thread_rounds++; - } } } @@ -42,15 +34,6 @@ void PageEvictor::PickBufferFramesToCool(Partition& targetPartition) { SCOPED_DEFER(LS_DLOG("Phase1: PickBufferFramesToCool ended, mEvictCandidateBfs.size={}", mEvictCandidateBfs.size())); - COUNTERS_BLOCK() { - auto phase1Begin = std::chrono::high_resolution_clock::now(); - SCOPED_DEFER({ - auto phase1End = std::chrono::high_resolution_clock::now(); - PPCounters::MyCounters().mPhase1MS += - (std::chrono::duration_cast(phase1End - phase1Begin).count()); - }); - } - // [corner cases]: prevent starving when free list is empty and cooling to // the required level can not be achieved uint64_t failedAttempts = 0; @@ -59,9 +42,6 @@ void PageEvictor::PickBufferFramesToCool(Partition& targetPartition) { while (mCoolCandidateBfs.size() > 0) { auto* coolCandidate = mCoolCandidateBfs.back(); mCoolCandidateBfs.pop_back(); - COUNTERS_BLOCK() { - PPCounters::MyCounters().phase_1_counter++; - } JUMPMU_TRY() { BMOptimisticGuard readGuard(coolCandidate->mHeader.mLatch); if (coolCandidate->ShouldRemainInMem()) { @@ -92,10 +72,6 @@ void PageEvictor::PickBufferFramesToCool(Partition& targetPartition) { } readGuard.JumpIfModifiedByOthers(); - COUNTERS_BLOCK() { - PPCounters::MyCounters().touched_bfs_counter++; - } - // Iterate all the child pages to check whether all the children are // evicted, otherwise pick the fist met unevicted child as the next // cool page candidate. @@ -103,9 +79,6 @@ void PageEvictor::PickBufferFramesToCool(Partition& targetPartition) { bool pickedAChild(false); [[maybe_unused]] Time iterateChildrenBegin; [[maybe_unused]] Time iterateChildrenEnd; - COUNTERS_BLOCK() { - iterateChildrenBegin = std::chrono::high_resolution_clock::now(); - } mStore->mTreeRegistry->IterateChildSwips( coolCandidate->mPage.mBTreeId, *coolCandidate, [&](Swip& childSwip) { @@ -126,13 +99,6 @@ void PageEvictor::PickBufferFramesToCool(Partition& targetPartition) { return true; }); - COUNTERS_BLOCK() { - iterateChildrenEnd = std::chrono::high_resolution_clock::now(); - PPCounters::MyCounters().mIterateChildrenMS += - (std::chrono::duration_cast(iterateChildrenEnd - - iterateChildrenBegin) - .count()); - } if (!allChildrenEvicted || pickedAChild) { LS_DLOG("Cool candidate discarded, not all the children are " "evicted, pageId={}, allChildrenEvicted={}, pickedAChild={}", @@ -143,22 +109,12 @@ void PageEvictor::PickBufferFramesToCool(Partition& targetPartition) { [[maybe_unused]] Time findParentBegin; [[maybe_unused]] Time findParentEnd; - COUNTERS_BLOCK() { - findParentBegin = std::chrono::high_resolution_clock::now(); - } TREEID btreeId = coolCandidate->mPage.mBTreeId; readGuard.JumpIfModifiedByOthers(); auto parentHandler = mStore->mTreeRegistry->FindParent(btreeId, *coolCandidate); LS_DCHECK(parentHandler.mParentGuard.mState == GuardState::kOptimisticShared); LS_DCHECK(parentHandler.mParentGuard.mLatch != reinterpret_cast(0x99)); - COUNTERS_BLOCK() { - findParentEnd = std::chrono::high_resolution_clock::now(); - PPCounters::MyCounters().mFindParentMS += - (std::chrono::duration_cast(findParentEnd - - findParentBegin) - .count()); - } readGuard.JumpIfModifiedByOthers(); auto checkResult = mStore->mTreeRegistry->CheckSpaceUtilization( coolCandidate->mPage.mBTreeId, *coolCandidate); @@ -194,9 +150,6 @@ void PageEvictor::PickBufferFramesToCool(Partition& targetPartition) { coolCandidate->mHeader.mPageId); } - COUNTERS_BLOCK() { - PPCounters::MyCounters().unswizzled_pages_counter++; - } failedAttempts = 0; } JUMPMU_CATCH() { @@ -213,7 +166,7 @@ void PageEvictor::randomBufferFramesToCoolOrEvict() { mCoolCandidateBfs.clear(); for (auto i = 0u; i < mStore->mStoreOption->mBufferFrameRecycleBatchSize; i++) { auto* randomBf = &mStore->mBufferManager->RandomBufferFrame(); - DO_NOT_OPTIMIZE(randomBf->mHeader.mState); + DoNotOptimize(randomBf->mHeader.mState); mCoolCandidateBfs.push_back(randomBf); } } @@ -328,7 +281,6 @@ void PageEvictor::FlushAndRecycleBufferFrames(Partition& targetPartition) { // For recovery, so much has to be done here... writtenBf.mHeader.mFlushedPsn = flushedPsn; writtenBf.mHeader.mIsBeingWrittenBack = false; - PPCounters::MyCounters().flushed_pages_counter++; } JUMPMU_CATCH() { writtenBf.mHeader.mCrc = 0; @@ -380,10 +332,6 @@ void PageEvictor::evictFlushedBf(BufferFrame& cooledBf, BMOptimisticGuard& optim if (mFreeBfList.Size() <= std::min(mStore->mStoreOption->mWorkerThreads, 128)) { mFreeBfList.PopTo(targetPartition); } - - COUNTERS_BLOCK() { - PPCounters::MyCounters().evicted_pages++; - } }; } // namespace leanstore::storage \ No newline at end of file diff --git a/src/concurrency/ConcurrencyControl.cpp b/src/concurrency/ConcurrencyControl.cpp index f186918b..2e974984 100644 --- a/src/concurrency/ConcurrencyControl.cpp +++ b/src/concurrency/ConcurrencyControl.cpp @@ -1,21 +1,20 @@ #include "leanstore/concurrency/ConcurrencyControl.hpp" +#include "leanstore-c/PerfCounters.h" #include "leanstore/Exceptions.hpp" #include "leanstore/Units.hpp" #include "leanstore/buffer-manager/TreeRegistry.hpp" #include "leanstore/concurrency/CRManager.hpp" #include "leanstore/concurrency/WorkerContext.hpp" -#include "leanstore/profiling/counters/WorkerCounters.hpp" #include "leanstore/sync/HybridLatch.hpp" #include "leanstore/sync/ScopedHybridGuard.hpp" +#include "leanstore/utils/CounterUtil.hpp" #include "leanstore/utils/Defer.hpp" #include "leanstore/utils/JumpMU.hpp" #include "leanstore/utils/Log.hpp" -#include "leanstore/utils/Misc.hpp" #include "leanstore/utils/RandomGenerator.hpp" #include -#include #include #include @@ -27,7 +26,6 @@ namespace leanstore::cr { void CommitTree::AppendCommitLog(TXID startTs, TXID commitTs) { LS_DCHECK(mCommitLog.size() < mCapacity); - utils::Timer timer(CRCounters::MyCounters().cc_ms_committing); storage::ScopedHybridGuard xGuard(mLatch, storage::LatchMode::kPessimisticExclusive); mCommitLog.push_back({commitTs, startTs}); LS_DLOG("Commit log appended, workerId={}, startTs={}, commitTs={}", @@ -35,7 +33,6 @@ void CommitTree::AppendCommitLog(TXID startTs, TXID commitTs) { } void CommitTree::CompactCommitLog() { - utils::Timer timer(CRCounters::MyCounters().cc_ms_gc_cm); if (mCommitLog.size() < mCapacity) { return; } @@ -81,6 +78,9 @@ void CommitTree::CompactCommitLog() { } TXID CommitTree::Lcb(TXID startTs) { + COUNTER_INC(&tlsPerfCounters.mLcbExecuted); + COUNTER_TIMER_SCOPED(&tlsPerfCounters.mLcbTotalLatNs); + while (true) { JUMPMU_TRY() { storage::ScopedHybridGuard oGuard(mLatch, storage::LatchMode::kOptimisticOrJump); @@ -113,7 +113,6 @@ std::optional> CommitTree::lcbNoLatch(TXID startTs) { COMMANDID ConcurrencyControl::PutVersion(TREEID treeId, bool isRemoveCommand, uint64_t versionSize, std::function putCallBack) { - utils::Timer timer(CRCounters::MyCounters().cc_ms_history_tree_insert); auto& curWorker = WorkerContext::My(); auto commandId = curWorker.mCommandId++; if (isRemoveCommand) { @@ -154,7 +153,6 @@ bool ConcurrencyControl::VisibleForMe(WORKERID workerId, TXID txId) { } // Now we need to query LCB on the target worker and update the local cache. - utils::Timer timer(CRCounters::MyCounters().cc_ms_snapshotting); TXID largestVisibleTxId = Other(workerId).mCommitTree.Lcb(ActiveTx().mStartTs); if (largestVisibleTxId) { mLcbCacheKey[workerId] = ActiveTx().mStartTs; @@ -180,17 +178,18 @@ bool ConcurrencyControl::VisibleForAll(TXID txId) { // TODO: smooth purge, we should not let the system hang on this, as a quick // fix, it should be enough if we purge in small batches void ConcurrencyControl::GarbageCollection() { - utils::Timer timer(CRCounters::MyCounters().cc_ms_gc); if (!mStore->mStoreOption->mEnableGc) { return; } + COUNTER_INC(&tlsPerfCounters.mGcExecuted); + COUNTER_TIMER_SCOPED(&tlsPerfCounters.mGcTotalLatNs); + updateGlobalTxWatermarks(); updateLocalWatermarks(); // remove versions that are nolonger needed by any transaction if (mCleanedWmkOfShortTx <= mLocalWmkOfAllTx) { - utils::Timer timer(CRCounters::MyCounters().cc_ms_gc_history_tree); LS_DLOG("Garbage collect history tree, workerId={}, fromTxId={}, toTxId(mLocalWmkOfAllTx)={}", WorkerContext::My().mWorkerId, 0, mLocalWmkOfAllTx); mHistoryStorage.PurgeVersions( @@ -199,9 +198,6 @@ void ConcurrencyControl::GarbageCollection() { uint64_t versionSize [[maybe_unused]], const bool calledBefore) { mStore->mTreeRegistry->GarbageCollect(treeId, versionData, WorkerContext::My().mWorkerId, versionTxId, calledBefore); - COUNTERS_BLOCK() { - WorkerCounters::MyCounters().cc_gc_long_tx_executed[treeId]++; - } }, 0); mCleanedWmkOfShortTx = mLocalWmkOfAllTx + 1; @@ -214,7 +210,6 @@ void ConcurrencyControl::GarbageCollection() { // move tombstones to graveyard if (mStore->mStoreOption->mEnableLongRunningTx && mLocalWmkOfAllTx < mLocalWmkOfShortTx && mCleanedWmkOfShortTx <= mLocalWmkOfShortTx) { - utils::Timer timer(CRCounters::MyCounters().cc_ms_gc_graveyard); LS_DLOG("Garbage collect graveyard, workerId={}, fromTxId={}, " "toTxId(mLocalWmkOfShortTx)={}", WorkerContext::My().mWorkerId, mCleanedWmkOfShortTx, mLocalWmkOfShortTx); @@ -224,9 +219,6 @@ void ConcurrencyControl::GarbageCollection() { const bool calledBefore) { mStore->mTreeRegistry->GarbageCollect(treeId, versionData, WorkerContext::My().mWorkerId, versionTxId, calledBefore); - COUNTERS_BLOCK() { - WorkerCounters::MyCounters().cc_todo_oltp_executed[treeId]++; - } }); mCleanedWmkOfShortTx = mLocalWmkOfShortTx + 1; } else { @@ -255,7 +247,6 @@ void ConcurrencyControl::updateGlobalTxWatermarks() { return; } - utils::Timer timer(CRCounters::MyCounters().cc_ms_refresh_global_state); auto meetGcProbability = mStore->mStoreOption->mEnableEagerGc || utils::RandomGenerator::RandU64(0, WorkerContext::My().mAllWorkers.size()) == 0; diff --git a/src/concurrency/GroupCommitter.cpp b/src/concurrency/GroupCommitter.cpp index 5b0cb503..729671c8 100644 --- a/src/concurrency/GroupCommitter.cpp +++ b/src/concurrency/GroupCommitter.cpp @@ -2,9 +2,6 @@ #include "leanstore/concurrency/CRManager.hpp" #include "leanstore/concurrency/WorkerContext.hpp" -#include "leanstore/profiling/counters/CPUCounters.hpp" -#include "leanstore/telemetry/MetricOnlyTimer.hpp" -#include "telemetry/MetricsManager.hpp" #include #include @@ -19,8 +16,6 @@ namespace leanstore::cr { constexpr size_t kAligment = 4096; void GroupCommitter::runImpl() { - CPUCounters::registerThread(mThreadName, false); - TXID minFlushedSysTx = std::numeric_limits::max(); TXID minFlushedUsrTx = std::numeric_limits::max(); std::vector numRfaTxs(mWorkerCtxs.size(), 0); @@ -43,11 +38,6 @@ void GroupCommitter::runImpl() { void GroupCommitter::collectWalRecords(TXID& minFlushedSysTx, TXID& minFlushedUsrTx, std::vector& numRfaTxs, std::vector& walFlushReqCopies) { - leanstore::telemetry::MetricOnlyTimer timer; - SCOPED_DEFER({ - METRIC_HIST_OBSERVE(mStore->mMetricsManager, group_committer_prep_iocbs_us, timer.ElaspedUs()); - }); - minFlushedSysTx = std::numeric_limits::max(); minFlushedUsrTx = std::numeric_limits::max(); @@ -94,11 +84,6 @@ void GroupCommitter::collectWalRecords(TXID& minFlushedSysTx, TXID& minFlushedUs } void GroupCommitter::flushWalRecords() { - leanstore::telemetry::MetricOnlyTimer timer; - SCOPED_DEFER({ - METRIC_HIST_OBSERVE(mStore->mMetricsManager, group_committer_write_iocbs_us, timer.ElaspedUs()); - }); - // submit all log writes using a single system call. if (auto res = mAIo.SubmitAll(); !res) { Log::Error("Failed to submit all IO, error={}", res.error().ToString()); @@ -122,11 +107,6 @@ void GroupCommitter::flushWalRecords() { void GroupCommitter::determineCommitableTx(TXID minFlushedSysTx, TXID minFlushedUsrTx, const std::vector& numRfaTxs, const std::vector& walFlushReqCopies) { - leanstore::telemetry::MetricOnlyTimer timer; - SCOPED_DEFER({ - METRIC_HIST_OBSERVE(mStore->mMetricsManager, group_committer_commit_txs_us, timer.ElaspedUs()); - }); - for (WORKERID workerId = 0; workerId < mWorkerCtxs.size(); workerId++) { auto& logging = mWorkerCtxs[workerId]->mLogging; const auto& reqCopy = walFlushReqCopies[workerId]; @@ -200,8 +180,6 @@ void GroupCommitter::append(uint8_t* buf, uint64_t lower, uint64_t upper) { mAIo.PrepareWrite(mWalFd, bufAligned, countAligned, offsetAligned); mWalSize += upper - lower; - - METRIC_COUNTER_INC(mStore->mMetricsManager, group_committer_disk_write_total, countAligned); }; } // namespace leanstore::cr \ No newline at end of file diff --git a/src/concurrency/HistoryStorage.cpp b/src/concurrency/HistoryStorage.cpp index 6c93a5d4..5fbe09ed 100644 --- a/src/concurrency/HistoryStorage.cpp +++ b/src/concurrency/HistoryStorage.cpp @@ -4,7 +4,6 @@ #include "leanstore/btree/BasicKV.hpp" #include "leanstore/btree/core/BTreeNode.hpp" #include "leanstore/btree/core/PessimisticExclusiveIterator.hpp" -#include "leanstore/profiling/counters/CRCounters.hpp" #include "leanstore/sync/HybridLatch.hpp" #include "leanstore/sync/ScopedHybridGuard.hpp" #include "leanstore/utils/JumpMU.hpp" @@ -52,9 +51,6 @@ void HistoryStorage::PutVersion(TXID txId, COMMANDID commandId, TREEID treeId, b auto& versionMeta = *new (xIter.MutableVal().Data()) VersionMeta(); versionMeta.mTreeId = treeId; insertCallBack(versionMeta.mPayload); - COUNTERS_BLOCK() { - WorkerCounters::MyCounters().cc_versions_space_inserted_opt[treeId]++; - } xIter.mGuardedLeaf.unlock(); JUMPMU_RETURN; } @@ -89,9 +85,6 @@ void HistoryStorage::PutVersion(TXID txId, COMMANDID commandId, TREEID treeId, b session->mLastTxId = txId; } - COUNTERS_BLOCK() { - WorkerCounters::MyCounters().cc_versions_space_inserted[treeId]++; - } JUMPMU_RETURN; } JUMPMU_CATCH() { @@ -294,9 +287,6 @@ void HistoryStorage::PurgeVersions(TXID fromTxId, TXID toTxId, UNREACHABLE(); } } - COUNTERS_BLOCK() { - CRCounters::MyCounters().cc_versions_space_removed += versionsRemoved; - } } void HistoryStorage::VisitRemovedVersions(TXID fromTxId, TXID toTxId, diff --git a/src/concurrency/Logging.cpp b/src/concurrency/Logging.cpp index 40a43b81..67e70fa0 100644 --- a/src/concurrency/Logging.cpp +++ b/src/concurrency/Logging.cpp @@ -3,7 +3,6 @@ #include "leanstore/Exceptions.hpp" #include "leanstore/concurrency/WalEntry.hpp" #include "leanstore/concurrency/WorkerContext.hpp" -#include "leanstore/profiling/counters/WorkerCounters.hpp" #include "leanstore/utils/Log.hpp" #include "utils/ToJson.hpp" @@ -96,9 +95,6 @@ void Logging::SubmitWALEntryComplex(uint64_t totalSize) { mWalBuffered += totalSize; publishWalFlushReq(); - COUNTERS_BLOCK() { - WorkerCounters::MyCounters().wal_write_bytes += totalSize; - } LS_DLOG("SubmitWal, workerId={}, startTs={}, walJson={}", WorkerContext::My().mWorkerId, WorkerContext::My().mActiveTx.mStartTs, utils::ToJsonString(mActiveWALEntryComplex)); } diff --git a/src/concurrency/WorkerContext.cpp b/src/concurrency/WorkerContext.cpp index 731b5487..9f286c94 100644 --- a/src/concurrency/WorkerContext.cpp +++ b/src/concurrency/WorkerContext.cpp @@ -1,5 +1,6 @@ #include "leanstore/concurrency/WorkerContext.hpp" +#include "leanstore-c/PerfCounters.h" #include "leanstore/LeanStore.hpp" #include "leanstore/buffer-manager/TreeRegistry.hpp" #include "leanstore/concurrency/CRManager.hpp" @@ -7,11 +8,9 @@ #include "leanstore/concurrency/Logging.hpp" #include "leanstore/concurrency/Transaction.hpp" #include "leanstore/concurrency/WalEntry.hpp" -#include "leanstore/profiling/counters/CRCounters.hpp" -#include "leanstore/telemetry/MetricOnlyTimer.hpp" +#include "leanstore/utils/CounterUtil.hpp" #include "leanstore/utils/Defer.hpp" #include "leanstore/utils/Log.hpp" -#include "telemetry/MetricsManager.hpp" #include #include @@ -21,6 +20,7 @@ namespace leanstore::cr { thread_local std::unique_ptr WorkerContext::sTlsWorkerCtx = nullptr; thread_local WorkerContext* WorkerContext::sTlsWorkerCtxRaw = nullptr; +thread_local PerfCounters tlsPerfCounters; WorkerContext::WorkerContext(uint64_t workerId, std::vector& allWorkers, leanstore::LeanStore* store) @@ -29,7 +29,6 @@ WorkerContext::WorkerContext(uint64_t workerId, std::vector& all mActiveTxId(0), mWorkerId(workerId), mAllWorkers(allWorkers) { - CRCounters::MyCounters().mWorkerId = workerId; // init wal buffer mLogging.mWalBufferSize = mStore->mStoreOption->mWalBufferSize; @@ -95,7 +94,15 @@ void WorkerContext::StartTx(TxMode mode, IsolationLevel level, bool isReadOnly) } void WorkerContext::CommitTx() { - SCOPED_DEFER(mActiveTx.mState = TxState::kCommitted); + SCOPED_DEFER({ + COUNTER_INC(&tlsPerfCounters.mTxCommitted); + if (mActiveTx.mHasRemoteDependency) { + COUNTER_INC(&tlsPerfCounters.mTxWithRemoteDependencies); + } else { + COUNTER_INC(&tlsPerfCounters.mTxWithoutRemoteDependencies); + } + mActiveTx.mState = TxState::kCommitted; + }); if (!mActiveTx.mIsDurable) { return; @@ -142,27 +149,27 @@ void WorkerContext::CommitTx() { "hasRemoteDep={}", mWorkerId, mActiveTx.mStartTs, mActiveTx.mCommitTs, mActiveTx.mMaxObservedSysTxId, mActiveTx.mHasRemoteDependency); - telemetry::MetricOnlyTimer timer; - while (!mLogging.SafeToCommit(mActiveTx.mCommitTs)) { - } - METRIC_HIST_OBSERVE(mStore->mMetricsManager, tx_commit_wal_wait_us, timer.ElaspedUs()); + mLogging.WaitToCommit(mActiveTx.mCommitTs); } -//! TODO(jian.z): revert changes made in-place on the btree -//! process of a transaction abort: +//! TODO(jian.z): revert changes made in-place on the btree process of a transaction abort: //! //! 1. Read previous wal entries //! 2. Undo the changes via btree operations //! 3. Write compensation wal entries during the undo process -//! 4. Purge versions in history tree, clean garbages made by the aborted -//! transaction +//! 4. Purge versions in history tree, clean garbages made by the aborted transaction //! //! It may share the same code with the recovery process? void WorkerContext::AbortTx() { SCOPED_DEFER({ mActiveTx.mState = TxState::kAborted; - METRIC_COUNTER_INC(mStore->mMetricsManager, tx_abort_total, 1); + COUNTER_INC(&tlsPerfCounters.mTxAborted); + if (mActiveTx.mHasRemoteDependency) { + COUNTER_INC(&tlsPerfCounters.mTxWithRemoteDependencies); + } else { + COUNTER_INC(&tlsPerfCounters.mTxWithoutRemoteDependencies); + } mActiveTxId.store(0, std::memory_order_release); Log::Info("Transaction aborted, workerId={}, startTs={}, commitTs={}, maxObservedSysTx={}", mWorkerId, mActiveTx.mStartTs, mActiveTx.mCommitTs, mActiveTx.mMaxObservedSysTxId); @@ -198,4 +205,8 @@ void WorkerContext::AbortTx() { } } +PerfCounters* WorkerContext::GetPerfCounters() { + return &tlsPerfCounters; +} + } // namespace leanstore::cr diff --git a/src/leanstore-c/StoreOption.cpp b/src/leanstore-c/StoreOption.cpp index f28fb9a3..edb92755 100644 --- a/src/leanstore-c/StoreOption.cpp +++ b/src/leanstore-c/StoreOption.cpp @@ -51,8 +51,6 @@ static const StoreOption kDefaultStoreOption = { .mEnableEagerGc = false, // Metrics related options - .mEnableMetrics = false, - .mMetricsPort = 8080, .mEnableCpuCounters = true, .mEnableTimeMeasure = false, .mEnablePerfEvents = false, diff --git a/src/leanstore-c/leanstore-c.cpp b/src/leanstore-c/leanstore-c.cpp index d6234b67..6d179b2c 100644 --- a/src/leanstore-c/leanstore-c.cpp +++ b/src/leanstore-c/leanstore-c.cpp @@ -13,7 +13,6 @@ #include #include #include -#include #include #include @@ -281,7 +280,7 @@ static std::mutex sGlobalMetricsHttpExposerMutex; void StartMetricsHttpExposer(int32_t port) { std::unique_lock guard{sGlobalMetricsHttpExposerMutex}; - sGlobalMetricsHttpExposer = new leanstore::telemetry::MetricsHttpExposer(nullptr, port); + sGlobalMetricsHttpExposer = new leanstore::telemetry::MetricsHttpExposer(port); sGlobalMetricsHttpExposer->Start(); } diff --git a/src/profiling/counters/CPUCounters.cpp b/src/profiling/counters/CPUCounters.cpp deleted file mode 100644 index f0423f89..00000000 --- a/src/profiling/counters/CPUCounters.cpp +++ /dev/null @@ -1,28 +0,0 @@ -#include "leanstore/profiling/counters/CPUCounters.hpp" - -#include "leanstore/LeanStore.hpp" -#include "leanstore/PerfEvent.hpp" -#include "leanstore/utils/UserThread.hpp" - -namespace leanstore { - -std::mutex CPUCounters::mutex; -uint64_t CPUCounters::id = 0; -std::unordered_map CPUCounters::threads; - -uint64_t CPUCounters::registerThread(std::string name, bool perfInherit) { - if (!utils::tlsStore->mStoreOption->mEnablePerfEvents) { - return 0; - } - - std::unique_lock guard(mutex); - threads[id] = {.e = std::make_unique(perfInherit), .name = name}; - return id++; -} - -void CPUCounters::removeThread(uint64_t id) { - std::unique_lock guard(mutex); - threads.erase(id); -} - -} // namespace leanstore diff --git a/src/profiling/counters/CRCounters.cpp b/src/profiling/counters/CRCounters.cpp deleted file mode 100644 index 4a08c672..00000000 --- a/src/profiling/counters/CRCounters.cpp +++ /dev/null @@ -1,7 +0,0 @@ -#include "leanstore/profiling/counters/CRCounters.hpp" - -namespace leanstore { - -utils::EnumerableThreadLocal CRCounters::sCounters; - -} // namespace leanstore diff --git a/src/profiling/counters/PPCounters.cpp b/src/profiling/counters/PPCounters.cpp deleted file mode 100644 index 146bd1e2..00000000 --- a/src/profiling/counters/PPCounters.cpp +++ /dev/null @@ -1,9 +0,0 @@ -#include "leanstore/profiling/counters/PPCounters.hpp" - -#include "leanstore/utils/EnumerableThreadLocal.hpp" - -namespace leanstore { - -utils::EnumerableThreadLocal PPCounters::sCounters; - -} // namespace leanstore \ No newline at end of file diff --git a/src/profiling/counters/WorkerCounters.cpp b/src/profiling/counters/WorkerCounters.cpp deleted file mode 100644 index 09608754..00000000 --- a/src/profiling/counters/WorkerCounters.cpp +++ /dev/null @@ -1,11 +0,0 @@ -#include "leanstore/profiling/counters/WorkerCounters.hpp" - -#include "leanstore/utils/EnumerableThreadLocal.hpp" - -namespace leanstore { - -std::atomic WorkerCounters::sNumWorkers = 0; - -utils::EnumerableThreadLocal WorkerCounters::sCounters; - -} // namespace leanstore diff --git a/src/profiling/tables/BMTable.cpp b/src/profiling/tables/BMTable.cpp deleted file mode 100644 index b8b93b18..00000000 --- a/src/profiling/tables/BMTable.cpp +++ /dev/null @@ -1,104 +0,0 @@ -#include "leanstore/profiling/tables/BMTable.hpp" - -#include "leanstore/buffer-manager/BufferManager.hpp" -#include "leanstore/profiling/counters/PPCounters.hpp" -#include "leanstore/profiling/counters/WorkerCounters.hpp" -#include "leanstore/utils/EnumerableThreadLocal.hpp" -#include "leanstore/utils/UserThread.hpp" - -namespace leanstore { -namespace profiling { - -BMTable::BMTable(storage::BufferManager& bm) : ProfilingTable(), bm(bm) { -} - -std::string BMTable::getName() { - return "bm"; -} - -void BMTable::open() { - columns.emplace("key", [](Column& col) { col << 0; }); - - columns.emplace("space_usage_gib", [&](Column& col) { - const double gib = bm.ConsumedPages() * 1.0 * utils::tlsStore->mStoreOption->mPageSize / - 1024.0 / 1024.0 / 1024.0; - col << gib; - }); - - columns.emplace("space_usage_kib", [&](Column& col) { - const double kib = bm.ConsumedPages() * 1.0 * utils::tlsStore->mStoreOption->mPageSize / 1024.0; - col << kib; - }); - - columns.emplace("consumed_pages", [&](Column& col) { col << bm.ConsumedPages(); }); - columns.emplace("p1_pct", [&](Column& col) { col << (local_phase_1_ms * 100.0 / total); }); - columns.emplace("p2_pct", [&](Column& col) { col << (local_phase_2_ms * 100.0 / total); }); - columns.emplace("p3_pct", [&](Column& col) { col << (local_phase_3_ms * 100.0 / total); }); - columns.emplace("poll_pct", [&](Column& col) { col << ((local_poll_ms * 100.0 / total)); }); - columns.emplace("find_parent_pct", [&](Column& col) { - auto res = Sum(PPCounters::sCounters, &PPCounters::mFindParentMS); - col << (res * 100.0 / total); - }); - columns.emplace("iterate_children_pct", [&](Column& col) { - auto res = Sum(PPCounters::sCounters, &PPCounters::mIterateChildrenMS); - col << (res * 100.0 / total); - }); - columns.emplace( - "pc1", [&](Column& col) { col << Sum(PPCounters::sCounters, &PPCounters::phase_1_counter); }); - columns.emplace( - "pc2", [&](Column& col) { col << Sum(PPCounters::sCounters, &PPCounters::phase_2_counter); }); - columns.emplace( - "pc3", [&](Column& col) { col << Sum(PPCounters::sCounters, &PPCounters::phase_3_counter); }); - columns.emplace("free_pct", [&](Column& col) { col << (local_total_free * 100.0 / bm.mNumBfs); }); - columns.emplace("evicted_mib", [&](Column& col) { - auto res = Sum(PPCounters::sCounters, &PPCounters::evicted_pages); - col << (res * utils::tlsStore->mStoreOption->mPageSize / 1024.0 / 1024.0); - }); - columns.emplace("rounds", [&](Column& col) { - col << (Sum(PPCounters::sCounters, &PPCounters::pp_thread_rounds)); - }); - columns.emplace("touches", [&](Column& col) { - col << (Sum(PPCounters::sCounters, &PPCounters::touched_bfs_counter)); - }); - columns.emplace("unswizzled", [&](Column& col) { - col << (Sum(PPCounters::sCounters, &PPCounters::unswizzled_pages_counter)); - }); - columns.emplace("submit_ms", [&](Column& col) { - auto res = Sum(PPCounters::sCounters, &PPCounters::submit_ms); - col << (res * 100.0 / total); - }); - columns.emplace("async_mb_ws", [&](Column& col) { - col << (Sum(PPCounters::sCounters, &PPCounters::async_wb_ms)); - }); - columns.emplace("w_mib", [&](Column& col) { - auto res = Sum(PPCounters::sCounters, &PPCounters::flushed_pages_counter); - col << (res * utils::tlsStore->mStoreOption->mPageSize / 1024.0 / 1024.0); - }); - - columns.emplace("allocate_ops", [&](Column& col) { - col << (Sum(WorkerCounters::sCounters, &WorkerCounters::allocate_operations_counter)); - }); - columns.emplace("r_mib", [&](Column& col) { - auto res = Sum(WorkerCounters::sCounters, &WorkerCounters::read_operations_counter); - col << (res * utils::tlsStore->mStoreOption->mPageSize / 1024.0 / 1024.0); - }); -} - -void BMTable::next() { - clear(); - local_phase_1_ms = Sum(PPCounters::sCounters, &PPCounters::mPhase1MS); - local_phase_2_ms = Sum(PPCounters::sCounters, &PPCounters::mPhase2MS); - local_phase_3_ms = Sum(PPCounters::sCounters, &PPCounters::mPhase3MS); - - local_total_free = 0; - for (uint64_t i = 0; i < bm.mNumPartitions; i++) { - local_total_free += bm.GetPartition(i).mFreeBfList.mSize.load(); - } - total = local_phase_1_ms + local_phase_2_ms + local_phase_3_ms; - for (auto& c : columns) { - c.second.generator(c.second); - } -} - -} // namespace profiling -} // namespace leanstore diff --git a/src/profiling/tables/CPUTable.cpp b/src/profiling/tables/CPUTable.cpp deleted file mode 100644 index 89d6e8d3..00000000 --- a/src/profiling/tables/CPUTable.cpp +++ /dev/null @@ -1,67 +0,0 @@ -#include "leanstore/profiling/tables/CPUTable.hpp" - -#include "leanstore/PerfEvent.hpp" -#include "leanstore/profiling/counters/CPUCounters.hpp" - -#include - -namespace leanstore { -namespace profiling { -// ------------------------------------------------------------------------------------- -std::string CPUTable::getName() { - return "cpu"; -} -// ------------------------------------------------------------------------------------- -void CPUTable::open() { - PerfEvent e; - for (const auto& event_name : e.getEventsName()) { - workers_agg_events[event_name] = 0; - pp_agg_events[event_name] = 0; - ww_agg_events[event_name] = 0; - columns.emplace(event_name, [](Column&) {}); - } - columns.emplace("key", [](Column&) {}); -} -// ------------------------------------------------------------------------------------- -void CPUTable::next() { - clear(); - // ------------------------------------------------------------------------------------- - for (auto& c : workers_agg_events) { - c.second = 0; - } - for (auto& c : pp_agg_events) { - c.second = 0; - } - for (auto& c : ww_agg_events) { - c.second = 0; - } - // ------------------------------------------------------------------------------------- - { - std::unique_lock guard(CPUCounters::mutex); - for (auto& thread : CPUCounters::threads) { - thread.second.e->stopCounters(); - auto events_map = thread.second.e->getCountersMap(); - columns.at("key") << thread.second.name; - for (auto& event : events_map) { - double event_value; - if (std::isnan(event.second)) { - event_value = 0; - } else { - event_value = event.second; - } - if (thread.second.name.rfind("worker", 0) == 0) { - workers_agg_events[event.first] += event_value; - } else if (thread.second.name.rfind("pp", 0) == 0) { - pp_agg_events[event.first] += event_value; - } else if (thread.second.name.rfind("ww") == 0) { - ww_agg_events[event.first] += event_value; - } - columns.at(event.first) << event.second; - } - thread.second.e->startCounters(); - } - } -} -// ------------------------------------------------------------------------------------- -} // namespace profiling -} // namespace leanstore diff --git a/src/profiling/tables/CRTable.cpp b/src/profiling/tables/CRTable.cpp deleted file mode 100644 index 0cc0a095..00000000 --- a/src/profiling/tables/CRTable.cpp +++ /dev/null @@ -1,146 +0,0 @@ -#include "leanstore/profiling/tables/CRTable.hpp" - -#include "leanstore/profiling/counters/CRCounters.hpp" -#include "leanstore/profiling/counters/WorkerCounters.hpp" - -namespace leanstore { -namespace profiling { - -std::string CRTable::getName() { - return "cr"; -} - -void CRTable::open() { - columns.emplace("key", [&](Column& out) { out << 0; }); - columns.emplace("wal_reserve_blocked", [&](Column& col) { - col << (Sum(CRCounters::sCounters, &CRCounters::wal_reserve_blocked)); - }); - columns.emplace("wal_reserve_immediate", [&](Column& col) { - col << (Sum(CRCounters::sCounters, &CRCounters::wal_reserve_immediate)); - }); - columns.emplace("gct_phase_1_pct", [&](Column& col) { col << 100.0 * p1 / total; }); - columns.emplace("gct_phase_2_pct", [&](Column& col) { col << 100.0 * p2 / total; }); - columns.emplace("gct_write_pct", [&](Column& col) { col << 100.0 * write / total; }); - columns.emplace("gct_committed_tx", [&](Column& col) { - col << Sum(CRCounters::sCounters, &CRCounters::gct_committed_tx); - }); - columns.emplace("gct_rounds", - [&](Column& col) { col << Sum(CRCounters::sCounters, &CRCounters::gct_rounds); }); - columns.emplace("tx", - [](Column& col) { col << Sum(WorkerCounters::sCounters, &WorkerCounters::tx); }); - columns.emplace("tx_abort", [](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::tx_abort); - }); - columns.emplace("long_running_tx", [](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::long_running_tx); - }); - columns.emplace("olap_scanned_tuples", [](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::olap_scanned_tuples); - }); - columns.emplace("olap_tx_abort", [](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::olap_tx_abort); - }); - columns.emplace("rfa_committed_tx", [&](Column& col) { - col << Sum(CRCounters::sCounters, &CRCounters::rfa_committed_tx); - }); - columns.emplace("cc_snapshot_restart", [](Column& col) { - col << Sum(CRCounters::sCounters, &CRCounters::cc_snapshot_restart); - }); - - columns.emplace("wal_read_gib", [&](Column& col) { - col << (Sum(WorkerCounters::sCounters, &WorkerCounters::wal_read_bytes) * 1.0) / 1024.0 / - 1024.0 / 1024.0; - }); - columns.emplace("gct_write_gib", [&](Column& col) { - col << (Sum(CRCounters::sCounters, &CRCounters::gct_write_bytes) * 1.0) / 1024.0 / 1024.0 / - 1024.0; - }); - columns.emplace("wal_write_gib", [&](Column& col) { - col << (Sum(WorkerCounters::sCounters, &WorkerCounters::wal_write_bytes) * 1.0) / 1024.0 / - 1024.0 / 1024.0; - }); - columns.emplace("wal_miss_pct", [&](Column& col) { col << wal_miss_pct; }); - columns.emplace("wal_hit_pct", [&](Column& col) { col << wal_hit_pct; }); - columns.emplace("wal_miss", [&](Column& col) { col << wal_miss; }); - columns.emplace("wal_hit", [&](Column& col) { col << wal_hits; }); - columns.emplace("wal_total", [&](Column& col) { col << wal_total; }); - - columns.emplace("cc_prepare_igc", [&](Column& col) { - col << Sum(CRCounters::sCounters, &CRCounters::cc_prepare_igc); - }); - columns.emplace("cc_cross_workers_visibility_check", [&](Column& col) { - col << Sum(CRCounters::sCounters, &CRCounters::cc_cross_workers_visibility_check); - }); - columns.emplace("cc_versions_space_removed", [&](Column& col) { - col << Sum(CRCounters::sCounters, &CRCounters::cc_versions_space_removed); - }); - - columns.emplace("cc_ms_oltp_tx", [&](Column& col) { - col << Sum(CRCounters::sCounters, &CRCounters::cc_ms_oltp_tx); - }); - columns.emplace("cc_ms_olap_tx", [&](Column& col) { - col << Sum(CRCounters::sCounters, &CRCounters::cc_ms_olap_tx); - }); - columns.emplace("cc_ms_gc", - [&](Column& col) { col << Sum(CRCounters::sCounters, &CRCounters::cc_ms_gc); }); - columns.emplace("cc_ms_gc_cm", [&](Column& col) { - col << Sum(CRCounters::sCounters, &CRCounters::cc_ms_gc_cm); - }); - columns.emplace("cc_ms_gc_graveyard", [&](Column& col) { - col << Sum(CRCounters::sCounters, &CRCounters::cc_ms_gc_graveyard); - }); - columns.emplace("cc_ms_gc_history_tree", [&](Column& col) { - col << Sum(CRCounters::sCounters, &CRCounters::cc_ms_gc_history_tree); - }); - columns.emplace("cc_ms_fat_tuple", [&](Column& col) { - col << Sum(CRCounters::sCounters, &CRCounters::cc_ms_fat_tuple); - }); - columns.emplace("cc_ms_fat_tuple_conversion", [&](Column& col) { - col << Sum(CRCounters::sCounters, &CRCounters::cc_ms_fat_tuple_conversion); - }); - columns.emplace("cc_ms_snapshotting", [&](Column& col) { - col << Sum(CRCounters::sCounters, &CRCounters::cc_ms_snapshotting); - }); - columns.emplace("cc_ms_committing", [&](Column& col) { - col << Sum(CRCounters::sCounters, &CRCounters::cc_ms_committing); - }); - columns.emplace("cc_ms_history_tree_insert", [&](Column& col) { - col << Sum(CRCounters::sCounters, &CRCounters::cc_ms_history_tree_insert); - }); - columns.emplace("cc_ms_history_tree_retrieve", [&](Column& col) { - col << Sum(CRCounters::sCounters, &CRCounters::cc_ms_history_tree_retrieve); - }); - columns.emplace("cc_ms_refresh_global_state", [&](Column& col) { - col << Sum(CRCounters::sCounters, &CRCounters::cc_ms_refresh_global_state); - }); - - columns.emplace("cc_ms_start_tx", [&](Column& col) { - col << Sum(CRCounters::sCounters, &CRCounters::cc_ms_start_tx); - }); - columns.emplace("cc_ms_commit_tx", [&](Column& col) { - col << Sum(CRCounters::sCounters, &CRCounters::cc_ms_commit_tx); - }); - columns.emplace("cc_ms_abort_tx", [&](Column& col) { - col << Sum(CRCounters::sCounters, &CRCounters::cc_ms_abort_tx); - }); -} - -void CRTable::next() { - wal_hits = Sum(WorkerCounters::sCounters, &WorkerCounters::wal_buffer_hit); - wal_miss = Sum(WorkerCounters::sCounters, &WorkerCounters::wal_buffer_miss); - wal_total = wal_hits + wal_miss; - wal_hit_pct = wal_hits * 1.0 / wal_total; - wal_miss_pct = wal_miss * 1.0 / wal_total; - - p1 = Sum(CRCounters::sCounters, &CRCounters::gct_phase_1_ms); - p2 = Sum(CRCounters::sCounters, &CRCounters::gct_phase_2_ms); - write = Sum(CRCounters::sCounters, &CRCounters::gct_write_ms); - total = p1 + p2 + write; - clear(); - for (auto& c : columns) { - c.second.generator(c.second); - } -} - -} // namespace profiling -} // namespace leanstore diff --git a/src/profiling/tables/ConfigsTable.cpp b/src/profiling/tables/ConfigsTable.cpp deleted file mode 100644 index 7b6bce46..00000000 --- a/src/profiling/tables/ConfigsTable.cpp +++ /dev/null @@ -1,103 +0,0 @@ -#include "leanstore/profiling/tables/ConfigsTable.hpp" - -#include "leanstore/LeanStore.hpp" -#include "leanstore/utils/UserThread.hpp" - -namespace leanstore { -namespace profiling { - -std::string ConfigsTable::getName() { - return "configs"; -} - -void ConfigsTable::add(std::string name, std::string value) { - columns.emplace(name, [&, value](Column& col) { col << value; }); -} - -void ConfigsTable::open() { - columns.emplace("c_worker_threads", - [&](Column& col) { col << utils::tlsStore->mStoreOption->mWorkerThreads; }); - - columns.emplace("c_free_pct", - [&](Column& col) { col << utils::tlsStore->mStoreOption->mFreePct; }); - - columns.emplace("c_buffer_frame_providers", - [&](Column& col) { col << utils::tlsStore->mStoreOption->mNumBufferProviders; }); - - columns.emplace("c_num_partitions", - [&](Column& col) { col << utils::tlsStore->mStoreOption->mNumPartitions; }); - - columns.emplace("c_buffer_pool_size", - [&](Column& col) { col << utils::tlsStore->mStoreOption->mBufferPoolSize; }); - - columns.emplace("c_bulk_insert", - [&](Column& col) { col << utils::tlsStore->mStoreOption->mEnableBulkInsert; }); - - columns.emplace("c_contention_split", [&](Column& col) { - col << utils::tlsStore->mStoreOption->mEnableContentionSplit; - }); - - columns.emplace("c_contention_split_sample_probability", [&](Column& col) { - col << utils::tlsStore->mStoreOption->mContentionSplitSampleProbability; - }); - - columns.emplace("c_cm_period", [&](Column& col) { - col << utils::tlsStore->mStoreOption->mContentionSplitProbility; - }); - - columns.emplace("c_contention_split_threshold_pct", [&](Column& col) { - col << utils::tlsStore->mStoreOption->mContentionSplitThresholdPct; - }); - - columns.emplace("c_xmerge", - [&](Column& col) { col << utils::tlsStore->mStoreOption->mEnableXMerge; }); - - columns.emplace("c_xmerge_k", - [&](Column& col) { col << utils::tlsStore->mStoreOption->mXMergeK; }); - - columns.emplace("c_xmerge_target_pct", - [&](Column& col) { col << utils::tlsStore->mStoreOption->mXMergeTargetPct; }); - - columns.emplace("c_btree_heads", [&](Column& col) { - col << utils::tlsStore->mStoreOption->mEnableHeadOptimization; - }); - - columns.emplace("c_btree_hints", - [&](Column& col) { col << utils::tlsStore->mStoreOption->mBTreeHints; }); - - columns.emplace("c_wal", [&](Column& col) { col << utils::tlsStore->mStoreOption->mEnableWal; }); - - columns.emplace("c_wal_io_hack", [&](Column& col) { col << 1; }); - - columns.emplace("c_wal_fsync", - [&](Column& col) { col << utils::tlsStore->mStoreOption->mEnableWalFsync; }); - - columns.emplace("c_enable_garbage_collection", - [&](Column& col) { col << utils::tlsStore->mStoreOption->mEnableGc; }); - - columns.emplace("c_vi_fat_tuple", - [&](Column& col) { col << utils::tlsStore->mStoreOption->mEnableFatTuple; }); - - columns.emplace("c_enable_long_running_transaction", - [&](Column& col) { col << utils::tlsStore->mStoreOption->mEnableLongRunningTx; }); - - for (auto& c : columns) { - c.second.generator(c.second); - } -} - -uint64_t ConfigsTable::hash() { - std::stringstream configConcatenation; - for (const auto& c : columns) { - configConcatenation << c.second.values[0]; - } - return std::hash{}(configConcatenation.str()); -} - -void ConfigsTable::next() { - // one time is enough - return; -} - -} // namespace profiling -} // namespace leanstore diff --git a/src/profiling/tables/DTTable.cpp b/src/profiling/tables/DTTable.cpp deleted file mode 100644 index 11683a40..00000000 --- a/src/profiling/tables/DTTable.cpp +++ /dev/null @@ -1,233 +0,0 @@ -#include "leanstore/profiling/tables/DTTable.hpp" - -#include "leanstore/buffer-manager/TreeRegistry.hpp" -#include "leanstore/profiling/counters/WorkerCounters.hpp" - -namespace leanstore::profiling { - -DTTable::DTTable(leanstore::storage::BufferManager& bm) : bm(bm) { -} - -std::string DTTable::getName() { - return "dt"; -} - -void DTTable::open() { - columns.emplace("key", [&](Column& col) { col << mTreeId; }); - columns.emplace("dt_name", [&](Column& col) { col << dt_name; }); - columns.emplace("dt_page_reads", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::dt_page_reads, mTreeId); - }); - columns.emplace("dt_page_writes", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::dt_page_writes, mTreeId); - }); - columns.emplace("dt_restarts_update_same_size", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::dt_restarts_update_same_size, mTreeId); - }); - columns.emplace("dt_restarts_structural_change", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::dt_restarts_structural_change, mTreeId); - }); - columns.emplace("dt_restarts_read", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::dt_restarts_read, mTreeId); - }); - - columns.emplace("dt_empty_leaf", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::dt_empty_leaf, mTreeId); - }); - columns.emplace("dt_goto_page_exec", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::dt_goto_page_exec, mTreeId); - }); - columns.emplace("dt_goto_page_shared", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::dt_goto_page_shared, mTreeId); - }); - columns.emplace("dt_next_tuple", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::dt_next_tuple, mTreeId); - }); - columns.emplace("dt_next_tuple_opt", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::dt_next_tuple_opt, mTreeId); - }); - columns.emplace("dt_prev_tuple", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::dt_prev_tuple, mTreeId); - }); - columns.emplace("dt_prev_tuple_opt", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::dt_prev_tuple_opt, mTreeId); - }); - columns.emplace("dt_inner_page", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::dt_inner_page, mTreeId); - }); - columns.emplace("dt_scan_asc", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::dt_scan_asc, mTreeId); - }); - columns.emplace("dt_scan_desc", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::dt_scan_desc, mTreeId); - }); - columns.emplace("dt_scan_callback", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::dt_scan_callback, mTreeId); - }); - - columns.emplace("dt_append", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::dt_append, mTreeId); - }); - columns.emplace("dt_append_opt", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::dt_append_opt, mTreeId); - }); - columns.emplace("dt_range_removed", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::dt_range_removed, mTreeId); - }); - - for (uint64_t r_i = 0; r_i < WorkerCounters::max_researchy_counter; r_i++) { - columns.emplace("dt_researchy_" + std::to_string(r_i), [&, r_i](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::dt_researchy, mTreeId, r_i); - }); - } - - columns.emplace("contention_split_succ_counter", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::contention_split_succ_counter, mTreeId); - }); - columns.emplace("contention_split_fail_counter", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::contention_split_fail_counter, mTreeId); - }); - columns.emplace("dt_split", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::dt_split, mTreeId); - }); - columns.emplace("dt_merge_succ", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::dt_merge_succ, mTreeId); - }); - columns.emplace("dt_merge_fail", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::dt_merge_fail, mTreeId); - }); - columns.emplace("dt_merge_parent_succ", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::dt_merge_parent_succ, mTreeId); - }); - columns.emplace("dt_merge_parent_fail", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::dt_merge_parent_fail, mTreeId); - }); - columns.emplace("xmerge_partial_counter", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::xmerge_partial_counter, mTreeId); - }); - columns.emplace("xmerge_full_counter", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::xmerge_full_counter, mTreeId); - }); - - columns.emplace("dt_find_parent", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::dt_find_parent, mTreeId); - }); - columns.emplace("dt_find_parent_root", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::dt_find_parent_root, mTreeId); - }); - columns.emplace("dt_find_parent_fast", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::dt_find_parent_fast, mTreeId); - }); - columns.emplace("dt_find_parent_slow", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::dt_find_parent_slow, mTreeId); - }); - - columns.emplace("cc_read_versions_visited", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::cc_read_versions_visited, mTreeId); - }); - columns.emplace("cc_read_versions_visited_not_found", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::cc_read_versions_visited_not_found, - mTreeId); - }); - columns.emplace("cc_read_chains", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::cc_read_chains, mTreeId); - }); - columns.emplace("cc_read_chains_not_found", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::cc_read_chains_not_found, mTreeId); - }); - - columns.emplace("cc_update_versions_visited", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::cc_update_versions_visited, mTreeId); - }); - columns.emplace("cc_update_versions_removed", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::cc_update_versions_removed, mTreeId); - }); - columns.emplace("cc_update_versions_skipped", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::cc_update_versions_skipped, mTreeId); - }); - columns.emplace("cc_update_versions_kept", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::cc_update_versions_kept, mTreeId); - }); - columns.emplace("cc_update_versions_kept_max", [&](Column& col) { - col << Max(WorkerCounters::sCounters, &WorkerCounters::cc_update_versions_kept_max, mTreeId); - }); - columns.emplace("cc_update_versions_recycled", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::cc_update_versions_recycled, mTreeId); - }); - columns.emplace("cc_update_versions_created", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::cc_update_versions_created, mTreeId); - }); - columns.emplace("cc_update_chains", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::cc_update_chains, mTreeId); - }); - columns.emplace("cc_update_chains_hwm", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::cc_update_chains_hwm, mTreeId); - }); - columns.emplace("cc_update_chains_pgc", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::cc_update_chains_pgc, mTreeId); - }); - columns.emplace("cc_update_chains_pgc_skipped", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::cc_update_chains_pgc_skipped, mTreeId); - }); - columns.emplace("cc_update_chains_pgc_workers_visited", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::cc_update_chains_pgc_workers_visited, - mTreeId); - }); - columns.emplace("cc_update_chains_pgc_heavy_removed", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::cc_update_chains_pgc_heavy_removed, - mTreeId); - }); - columns.emplace("cc_update_chains_pgc_heavy", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::cc_update_chains_pgc_heavy, mTreeId); - }); - columns.emplace("cc_update_chains_pgc_light_removed", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::cc_update_chains_pgc_light_removed, - mTreeId); - }); - columns.emplace("cc_update_chains_pgc_light", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::cc_update_chains_pgc_light, mTreeId); - }); - - columns.emplace("cc_todo_removed", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::cc_todo_removed, mTreeId); - }); - columns.emplace("cc_todo_moved_gy", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::cc_todo_moved_gy, mTreeId); - }); - columns.emplace("cc_todo_oltp_executed", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::cc_todo_oltp_executed, mTreeId); - }); - columns.emplace("cc_gc_long_tx_executed", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::cc_gc_long_tx_executed, mTreeId); - }); - - columns.emplace("cc_fat_tuple_triggered", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::cc_fat_tuple_triggered, mTreeId); - }); - columns.emplace("cc_fat_tuple_convert", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::cc_fat_tuple_convert, mTreeId); - }); - columns.emplace("cc_fat_tuple_decompose", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::cc_fat_tuple_decompose, mTreeId); - }); - - columns.emplace("cc_versions_space_inserted", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::cc_versions_space_inserted, mTreeId); - }); - columns.emplace("cc_versions_space_inserted_opt", [&](Column& col) { - col << Sum(WorkerCounters::sCounters, &WorkerCounters::cc_versions_space_inserted_opt, mTreeId); - }); -} - -void DTTable::next() { - clear(); - for (const auto& entry : bm.mStore->mTreeRegistry->mTrees) { - mTreeId = entry.first; - dt_name = std::get<1>(entry.second); - for (auto& c : columns) { - c.second.generator(c.second); - } - } -} - -} // namespace leanstore::profiling diff --git a/src/profiling/tables/LatencyTable.cpp b/src/profiling/tables/LatencyTable.cpp deleted file mode 100644 index 2e66355f..00000000 --- a/src/profiling/tables/LatencyTable.cpp +++ /dev/null @@ -1,44 +0,0 @@ -#include "leanstore/profiling/tables/LatencyTable.hpp" - -#include "leanstore/profiling/counters/CRCounters.hpp" -#include "leanstore/utils/EnumerableThreadLocal.hpp" - -namespace leanstore { -namespace profiling { - -std::string LatencyTable::getName() { - return "latency"; -} - -void LatencyTable::open() { - columns.emplace("key", [](Column&) {}); - columns.emplace("tx_i", [](Column&) {}); - columns.emplace("cc_ms_precommit_latency", [](Column&) {}); - columns.emplace("cc_ms_commit_latency", [](Column&) {}); - columns.emplace("cc_flushes_counter", [](Column&) {}); - columns.emplace("cc_rfa_ms_precommit_latency", [](Column&) {}); - columns.emplace("cc_rfa_ms_commit_latency", [](Column&) {}); -} - -void LatencyTable::next() { - clear(); - - // utils::EnumerableThreadLocal sCounters; - utils::ForEach(CRCounters::sCounters, [&](CRCounters* tls) { - if (tls->mWorkerId.load() != -1) { - return; - } - for (uint64_t i = 0; i < CRCounters::latency_tx_capacity; i++) { - columns.at("key") << tls->mWorkerId.load(); - columns.at("i") << i; - columns.at("cc_ms_precommit_latency") << tls->cc_ms_precommit_latency[i].load(); - columns.at("cc_ms_commit_latency") << tls->cc_ms_commit_latency[i].load(); - columns.at("cc_flushes_counter") << tls->cc_flushes_counter[i].load(); - columns.at("cc_rfa_ms_precommit_latency") << tls->cc_rfa_ms_precommit_latency[i].load(); - columns.at("cc_rfa_ms_commit_latency") << tls->cc_rfa_ms_commit_latency[i].load(); - } - }); -} - -} // namespace profiling -} // namespace leanstore diff --git a/src/telemetry/MetricsHttpExposer.cpp b/src/telemetry/MetricsHttpExposer.cpp index 45fee57c..2f779e70 100644 --- a/src/telemetry/MetricsHttpExposer.cpp +++ b/src/telemetry/MetricsHttpExposer.cpp @@ -1,20 +1,11 @@ #include "telemetry/MetricsHttpExposer.hpp" -#include "leanstore/LeanStore.hpp" - namespace leanstore::telemetry { -MetricsHttpExposer::MetricsHttpExposer(LeanStore* store) - : leanstore::telemetry::MetricsHttpExposer(store, store->mStoreOption->mMetricsPort) { -} - -MetricsHttpExposer::MetricsHttpExposer(LeanStore* store, int32_t port) - : UserThread(store, "MetricsExposer"), +MetricsHttpExposer::MetricsHttpExposer(int32_t port) + : UserThread(nullptr, "MetricsExposer"), mPort(port) { mServer.new_task_queue = [] { return new httplib::ThreadPool(1); }; - mServer.Get("/metrics", [&](const httplib::Request& req, httplib::Response& res) { - handleMetrics(req, res); - }); mServer.Get("/heap", [&](const httplib::Request& req, httplib::Response& res) { handleHeap(req, res); }); diff --git a/src/telemetry/MetricsHttpExposer.hpp b/src/telemetry/MetricsHttpExposer.hpp index c869753e..61722a05 100644 --- a/src/telemetry/MetricsHttpExposer.hpp +++ b/src/telemetry/MetricsHttpExposer.hpp @@ -7,11 +7,8 @@ #include #include #endif -#include -#include -#include -#include +#include #include #include @@ -21,20 +18,20 @@ namespace leanstore::telemetry { const std::string kContentType("text/plain; version=0.0.4; charset=utf-8"); class MetricsHttpExposer : public utils::UserThread { -public: - MetricsHttpExposer(LeanStore* store); +private: + //! The http server + httplib::Server mServer; - MetricsHttpExposer(LeanStore* store, int32_t port); + //! The port to expose metrics + int32_t mPort; + +public: + MetricsHttpExposer(int32_t port); ~MetricsHttpExposer() override { mServer.stop(); } - void SetCollectable(std::shared_ptr collectable) { - auto guard = std::unique_lock(mCollectableMutex); - mCollectable = collectable; - } - protected: void runImpl() override { while (mKeepRunning) { @@ -43,23 +40,6 @@ class MetricsHttpExposer : public utils::UserThread { } private: - void handleMetrics(const httplib::Request&, httplib::Response& res) { - auto guard = std::unique_lock(mCollectableMutex); - if (mCollectable != nullptr) { - auto metrics = mCollectable->Collect(); - guard.unlock(); - const prometheus::TextSerializer serializer; - res.set_content(serializer.Serialize(metrics), kContentType); - return; - } - - // empty - guard.unlock(); - const prometheus::TextSerializer serializer; - std::vector empty; - res.set_content(serializer.Serialize(empty), kContentType); - } - void handleHeap(const httplib::Request& req [[maybe_unused]], httplib::Response& res) { #ifdef ENABLE_PROFILING // get the profiling time in seconds from the query @@ -125,18 +105,6 @@ class MetricsHttpExposer : public utils::UserThread { buffer << stream.rdbuf(); res.set_content(buffer.str(), kContentType); } - - //! The http server - httplib::Server mServer; - - //! The port to expose metrics - int32_t mPort; - - //! The mutex to protect mCollectable - std::mutex mCollectableMutex; - - //! The Collectable to expose metrics - std::shared_ptr mCollectable; }; } // namespace leanstore::telemetry \ No newline at end of file diff --git a/src/telemetry/MetricsManager.hpp b/src/telemetry/MetricsManager.hpp deleted file mode 100644 index 43d5d536..00000000 --- a/src/telemetry/MetricsManager.hpp +++ /dev/null @@ -1,170 +0,0 @@ -#pragma once - -#include "leanstore/Exceptions.hpp" - -#include -#include -#include -#include -#include - -#include -#include - -#include - -namespace leanstore::telemetry { - -const std::vector kBoundariesUs{ - 0.001, // 1ns - 0.004, // 4ns - 0.016, // 16ns - 0.064, // 64ns - 0.256, // 256ns - 1, // 1us - 4, // 4us - 16, // 16us - 64, // 64us - 256, // 256us - 1000, // 1ms - 4000, // 4ms - 16000, // 16ms - 64000, // 64ms - 256000, // 256ms - 1000000, // 1s - 4000000, // 4s - 16000000, // 16s -}; - -// ----------------------------------------------------------------------------- -// Macros exported -// ----------------------------------------------------------------------------- - -//! All the counter metrics. -#define METRIC_COUNTER_LIST(ACTION, ...) \ - ACTION(tx_kv_update_total, "number of tx kv updates", __VA_ARGS__) \ - ACTION(tx_kv_lookup_total, "number of tx kv lookups", __VA_ARGS__) \ - ACTION(tx_abort_total, "number of tx aborts", __VA_ARGS__) \ - ACTION(tx_version_read_total, "mvcc versions read", __VA_ARGS__) \ - ACTION(group_committer_disk_write_total, "", __VA_ARGS__) - -//! All the histogram metrics are listed here. -#define METRIC_HIST_LIST(ACTION, ...) \ - ACTION(tx_commit_wal_wait_us, kBoundariesUs, "", __VA_ARGS__) \ - ACTION(tx_kv_lookup_us, kBoundariesUs, "", __VA_ARGS__) \ - ACTION(tx_kv_update_us, kBoundariesUs, "", __VA_ARGS__) \ - ACTION(group_committer_prep_iocbs_us, kBoundariesUs, "", __VA_ARGS__) \ - ACTION(group_committer_write_iocbs_us, kBoundariesUs, "", __VA_ARGS__) \ - ACTION(group_committer_commit_txs_us, kBoundariesUs, "", __VA_ARGS__) - -//! Macro to update a counter metric -#define METRIC_COUNTER_INC(metricsMgr, metricName, value) metricsMgr->inc_##metricName(value); - -//! Macro to update a histogram metric -#define METRIC_HIST_OBSERVE(metricsMgr, metricName, value) metricsMgr->observe_##metricName(value); - -// ----------------------------------------------------------------------------- -// Macros not exported for counters -// ----------------------------------------------------------------------------- - -//! Macro to declare a counter metric -#define DECLARE_METRIC_COUNTER(metricName, help, ...) prometheus::Counter* m_##metricName; - -//! Macro to initialize a counter metric -#define INIT_METRIC_COUNTER(metricName, help, ...) \ - m_##metricName = &createCounterFamily(#metricName, help)->Add({}); - -//! Macro to define a function to increase a counter metric -#define DEFINE_METRIC_FUNC_COUNTER_INC(metricName, help, ...) \ - void inc_##metricName(double val = 1) { \ - COUNTERS_BLOCK() { \ - m_##metricName->Increment(val); \ - } \ - } - -// ----------------------------------------------------------------------------- -// Macros not exported for histograms -// ----------------------------------------------------------------------------- - -//! Macro to declare a histogram metric -#define DECLARE_METRIC_HIST(metricName, boundaries, help, ...) \ - prometheus::Histogram* m_##metricName; - -//! Macro to initialize a histogram metric -#define INIT_METRIC_HIST(metricName, boundaries, help, ...) \ - m_##metricName = &createHistogramFamily(#metricName, help)->Add({}, boundaries); - -//! Macro to define a function to observe a value to a histogram -#define DEFINE_METRIC_FUNC_HIST_OBSERVE(metricName, boundaries, help, ...) \ - void observe_##metricName(double val) { \ - COUNTERS_BLOCK() { \ - m_##metricName->Observe(val); \ - } \ - } - -// ----------------------------------------------------------------------------- -// Macros not exported -// ----------------------------------------------------------------------------- - -//! Macro to define a function to get the current value of a counter metric -#define DEFINE_METRIC_FUNC_GET(metricName, help, ...) \ - double get_##metricName() { \ - return m_##metricName->Value(); \ - } - -//! The MetricsManager class is used to manage all the metrics for one -//! LeanStore. It's expected to be a singleton inside a LeanStore instance. -class MetricsManager { -public: - //! The constructor of the MetricsManager. - MetricsManager() { - mRegistry = std::make_shared(); - - METRIC_COUNTER_LIST(INIT_METRIC_COUNTER); - METRIC_HIST_LIST(INIT_METRIC_HIST); - } - - METRIC_COUNTER_LIST(DEFINE_METRIC_FUNC_COUNTER_INC); - METRIC_HIST_LIST(DEFINE_METRIC_FUNC_HIST_OBSERVE); - - //! Get the metrics registry - std::shared_ptr GetRegistry() { - return mRegistry; - } - -private: - //! Create a counter family - prometheus::Family* createCounterFamily(const std::string& metricName, - const std::string& help) { - return &prometheus::BuildCounter().Name(metricName).Help(help).Register(*mRegistry); - } - - //! Create a histogram family - prometheus::Family* createHistogramFamily(const std::string& metricName, - const std::string& help) { - return &prometheus::BuildHistogram().Name(metricName).Help(help).Register(*mRegistry); - } - - //! Create linear buckets - static prometheus::Histogram::BucketBoundaries createLinearBuckets(double start, double end, - double step) { - auto bucketBoundaries = prometheus::Histogram::BucketBoundaries{}; - for (auto i = start; i < end; i += step) { - bucketBoundaries.push_back(i); - } - return bucketBoundaries; - } - - //! The registry for all the metrics - std::shared_ptr mRegistry; - - METRIC_COUNTER_LIST(DECLARE_METRIC_COUNTER); - METRIC_HIST_LIST(DECLARE_METRIC_HIST); -}; - -#undef DECLARE_METRIC_COUNTER -#undef INIT_METRIC_COUNTER -#undef DEFINE_METRIC_FUNC_COUNTER_INC -#undef DEFINE_METRIC_FUNC_GET - -} // namespace leanstore::telemetry diff --git a/src/utils/Misc.cpp b/src/utils/Misc.cpp index d8fcab25..b6fa1b77 100644 --- a/src/utils/Misc.cpp +++ b/src/utils/Misc.cpp @@ -1,8 +1,5 @@ #include "leanstore/utils/Misc.hpp" -#include "leanstore/LeanStore.hpp" -#include "leanstore/utils/UserThread.hpp" - #include namespace leanstore::utils { @@ -11,20 +8,4 @@ uint32_t CRC(const uint8_t* src, uint64_t size) { return crc32c::Crc32c(src, size); } -Timer::Timer(std::atomic& timeCounterUS) : mTimeCounterUS(timeCounterUS) { - if (tlsStore->mStoreOption->mEnableTimeMeasure) { - mStartTimePoint = std::chrono::high_resolution_clock::now(); - } -} - -Timer::~Timer() { - if (tlsStore->mStoreOption->mEnableTimeMeasure) { - auto endTimePoint = std::chrono::high_resolution_clock::now(); - const uint64_t duration = - std::chrono::duration_cast(endTimePoint - mStartTimePoint) - .count(); - mTimeCounterUS += duration; - } -} - } // namespace leanstore::utils diff --git a/tests/AbortTest.cpp b/tests/AbortTest.cpp index aa9324bd..7be2df6a 100644 --- a/tests/AbortTest.cpp +++ b/tests/AbortTest.cpp @@ -25,7 +25,7 @@ class AbortTest : public ::testing::Test { void SetUp() override { auto* curTest = ::testing::UnitTest::GetInstance()->current_test_info(); auto curTestName = std::string(curTest->test_case_name()) + "_" + std::string(curTest->name()); - std::string storeDir = "/tmp/" + curTestName; + std::string storeDir = "/tmp/leanstore/" + curTestName; uint32_t sessionLimit = 2; mStore = StoreFactory::NewLeanStoreMVCC(storeDir, sessionLimit); ASSERT_NE(mStore, nullptr); diff --git a/tests/AnomaliesTest.cpp b/tests/AnomaliesTest.cpp index 7a97c590..05ac4412 100644 --- a/tests/AnomaliesTest.cpp +++ b/tests/AnomaliesTest.cpp @@ -24,7 +24,7 @@ class AnomaliesTest : public ::testing::Test { void SetUp() override { auto* curTest = ::testing::UnitTest::GetInstance()->current_test_info(); auto curTestName = std::string(curTest->test_case_name()) + "_" + std::string(curTest->name()); - std::string storeDir = "/tmp/" + curTestName; + std::string storeDir = "/tmp/leanstore/" + curTestName; uint32_t sessionLimit = 4; mStore = StoreFactory::NewLeanStoreMVCC(storeDir, sessionLimit); ASSERT_NE(mStore, nullptr); diff --git a/tests/telemetry/MetricsManagerTest.cpp b/tests/telemetry/MetricsManagerTest.cpp deleted file mode 100644 index b9f05282..00000000 --- a/tests/telemetry/MetricsManagerTest.cpp +++ /dev/null @@ -1,38 +0,0 @@ -#include "telemetry/MetricsManager.hpp" - -#include "leanstore-c/StoreOption.h" -#include "leanstore/LeanStore.hpp" - -#include -#include - -#include - -namespace leanstore::telemetry::test { - -class MetricsManagerTest : public ::testing::Test { -protected: - void SetUp() override { - } - - void TearDown() override { - } -}; - -TEST_F(MetricsManagerTest, Basic) { - auto* option = CreateStoreOption("/tmp/leanstore/MetricsManagerTest"); - option->mEnableMetrics = true; - auto res = leanstore::LeanStore::Open(option); - ASSERT_TRUE(res); - - auto store = std::move(res.value()); - METRIC_COUNTER_INC(store->mMetricsManager, tx_abort_total, 100); - - httplib::Client cli("0.0.0.0", store->mStoreOption->mMetricsPort); - auto result = cli.Get("/metrics"); - ASSERT_TRUE(result) << "Error: " << result.error(); - ASSERT_EQ(result->status, httplib::StatusCode::OK_200) << "HTTP status: " << result->status; - ASSERT_TRUE(result->body.contains("tx_abort_total")); -} - -} // namespace leanstore::telemetry::test diff --git a/vcpkg.json b/vcpkg.json index c8ae5181..55af7f15 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -10,13 +10,6 @@ "rapidjson", "gperftools", "tabulate", - { - "name": "prometheus-cpp", - "features": [ - "pull" - ], - "version>=": "1.1.0" - }, { "name": "cpp-httplib", "version>=": "0.14.3"