From 152f1f634e5f555c6e375dc622ff8eaacff19739 Mon Sep 17 00:00:00 2001 From: Thomas Droxler Date: Tue, 27 Aug 2024 13:16:18 +0200 Subject: [PATCH] Add more metrics * Latest synced height per chain * Number of fungible tokens * Number of non-fungible tokens * Number of events --- .../grafana/explorer-backend-overview.json | 857 +++++++++++++----- .../org/alephium/explorer/ExplorerState.scala | 9 +- .../org/alephium/explorer/SyncServices.scala | 4 +- .../alephium/explorer/cache/BlockCache.scala | 27 +- .../alephium/explorer/cache/MetricCache.scala | 113 +++ .../persistence/queries/EventQueries.scala | 11 + .../persistence/queries/TokenQueries.scala | 20 + .../service/BlockFlowSyncService.scala | 23 +- .../explorer/cache/TestMetricCache.scala | 35 + .../service/BlockFlowSyncServiceSpec.scala | 8 +- 10 files changed, 853 insertions(+), 254 deletions(-) create mode 100644 app/src/main/scala/org/alephium/explorer/cache/MetricCache.scala create mode 100644 app/src/test/scala/org/alephium/explorer/cache/TestMetricCache.scala diff --git a/app/src/main/resources/grafana/explorer-backend-overview.json b/app/src/main/resources/grafana/explorer-backend-overview.json index 66f45698a..b19567a8b 100644 --- a/app/src/main/resources/grafana/explorer-backend-overview.json +++ b/app/src/main/resources/grafana/explorer-backend-overview.json @@ -3,7 +3,10 @@ "list": [ { "builtIn": 1, - "datasource": "-- Grafana --", + "datasource": { + "type": "datasource", + "uid": "grafana" + }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -13,250 +16,515 @@ ] }, "editable": true, - "gnetId": null, + "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 1, + "id": 15, "links": [], "panels": [ { - "cacheTimeout": null, - "datasource": "Prometheus", - "description": "Database Current Connections", + "datasource": { + "type": "prometheus", + "uid": "FWwc47WVk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, "gridPos": { - "h": 5, - "w": 6, + "h": 6, + "w": 4, "x": 0, "y": 0 }, - "id": 16, - "links": [], + "id": 23, "options": { - "fieldOptions": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": [ - "mean" + "lastNotNull" ], - "defaults": { - "mappings": [ + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "FWwc47WVk" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "alephimum_explorer_backend_fungible_count", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Number of fungible tokens", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "FWwc47WVk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "id": 0, - "op": "=", - "text": "N/A", - "type": 1, - "value": "null" + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 } - ], - "nullValueMode": "connected", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 24, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "FWwc47WVk" }, - "overrides": [], + "disableTextWrap": false, + "editorMode": "builder", + "expr": "alephimum_explorer_backend_nft_count", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Number of NFT", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "FWwc47WVk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 8, + "y": 0 + }, + "id": 25, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "FWwc47WVk" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "alephimum_explorer_backend_event_count", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Number of events", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "FWwc47WVk" + }, + "description": "Database Idle Connections", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 14, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", "values": false }, - "orientation": "horizontal", "showThresholdLabels": false, - "showThresholdMarkers": true + "showThresholdMarkers": true, + "sizing": "auto" }, - "pluginVersion": "6.7.4", + "pluginVersion": "10.4.7", "targets": [ { - "expr": "hikaricp_connections", + "datasource": { + "type": "prometheus", + "uid": "FWwc47WVk" + }, + "expr": "hikaricp_idle_connections", "instant": true, "interval": "", "legendFormat": "", "refId": "A" } ], - "timeFrom": null, - "timeShift": null, - "title": "DB Current Connections", + "title": "DB Idle Connections", "type": "gauge" }, { - "datasource": "Prometheus", - "description": "Database Active Connections", + "datasource": { + "type": "prometheus", + "uid": "FWwc47WVk" + }, + "description": "Threads Awaiting Connection", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, "gridPos": { "h": 5, "w": 6, - "x": 6, + "x": 18, "y": 0 }, - "id": 10, + "id": 12, "options": { - "fieldOptions": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { "calcs": [ "mean" ], - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [], + "fields": "", "values": false }, - "orientation": "auto", "showThresholdLabels": false, - "showThresholdMarkers": true + "showThresholdMarkers": true, + "sizing": "auto" }, - "pluginVersion": "6.7.4", + "pluginVersion": "10.4.7", "targets": [ { - "expr": "hikaricp_active_connections", + "datasource": { + "type": "prometheus", + "uid": "FWwc47WVk" + }, + "expr": "hikaricp_pending_threads", "instant": true, "interval": "", "legendFormat": "", "refId": "A" } ], - "timeFrom": null, - "timeShift": null, - "title": "DB Active Connections", + "title": "DB Pending Threads", "type": "gauge" }, { - "datasource": "Prometheus", - "description": "Database Idle Connections", + "datasource": { + "type": "prometheus", + "uid": "FWwc47WVk" + }, + "description": "Database Current Connections", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, "gridPos": { "h": 5, "w": 6, "x": 12, - "y": 0 + "y": 5 }, - "id": 14, + "id": 16, "options": { - "fieldOptions": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "horizontal", + "reduceOptions": { "calcs": [ "mean" ], - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [], + "fields": "", "values": false }, - "orientation": "auto", "showThresholdLabels": false, - "showThresholdMarkers": true + "showThresholdMarkers": true, + "sizing": "auto" }, - "pluginVersion": "6.7.4", + "pluginVersion": "10.4.7", "targets": [ { - "expr": "hikaricp_idle_connections", + "datasource": { + "type": "prometheus", + "uid": "FWwc47WVk" + }, + "expr": "hikaricp_connections", "instant": true, "interval": "", "legendFormat": "", "refId": "A" } ], - "timeFrom": null, - "timeShift": null, - "title": "DB Idle Connections", + "title": "DB Current Connections", "type": "gauge" }, { - "datasource": "Prometheus", - "description": "Threads Awaiting Connection", + "datasource": { + "type": "prometheus", + "uid": "FWwc47WVk" + }, + "description": "Database Active Connections", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, "gridPos": { "h": 5, "w": 6, "x": 18, - "y": 0 + "y": 5 }, - "id": 12, + "id": 10, "options": { - "fieldOptions": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { "calcs": [ "mean" ], - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [], + "fields": "", "values": false }, - "orientation": "auto", "showThresholdLabels": false, - "showThresholdMarkers": true + "showThresholdMarkers": true, + "sizing": "auto" }, - "pluginVersion": "6.7.4", + "pluginVersion": "10.4.7", "targets": [ { - "expr": "hikaricp_pending_threads", + "datasource": { + "type": "prometheus", + "uid": "FWwc47WVk" + }, + "expr": "hikaricp_active_connections", "instant": true, "interval": "", "legendFormat": "", "refId": "A" } ], - "timeFrom": null, - "timeShift": null, - "title": "DB Pending Threads", + "title": "DB Active Connections", "type": "gauge" }, { "aliasColors": {}, "bars": false, - "cacheTimeout": null, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "FWwc47WVk" + }, "description": "Number of requests per second, grouped by endpoint", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 5 + "y": 6 }, "hiddenSeries": false, "id": 2, @@ -271,13 +539,12 @@ }, "lines": true, "linewidth": 1, - "links": [], "nullPointMode": "null as zero", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, - "pluginVersion": "6.7.4", + "pluginVersion": "10.4.7", "pointradius": 2, "points": false, "renderer": "flot", @@ -287,6 +554,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "FWwc47WVk" + }, "expr": "sum(irate(explorer_backend_request_total{app=\"explorer-backend\"}[1m])) by (path)", "format": "time_series", "interval": "", @@ -295,6 +566,10 @@ "refId": "A" }, { + "datasource": { + "type": "prometheus", + "uid": "FWwc47WVk" + }, "expr": "sum(irate(explorer_backend_request_total{app=\"explorer-backend\"}[1m]))", "interval": "", "legendFormat": "total", @@ -302,9 +577,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "Number of Requests Per Second", "tooltip": { "shared": true, @@ -313,9 +586,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -323,24 +594,17 @@ { "decimals": 0, "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true }, { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { @@ -348,15 +612,24 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "FWwc47WVk" + }, "description": "Failed requests count in last 1 minute", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 5 + "y": 10 }, "hiddenSeries": false, "id": 8, @@ -373,9 +646,10 @@ "linewidth": 1, "nullPointMode": "null as zero", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "10.4.7", "pointradius": 2, "points": false, "renderer": "flot", @@ -385,6 +659,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "FWwc47WVk" + }, "expr": "sum(increase(explorer_backend_request_total{app=\"explorer-backend\",status=\"5xx\"}[1m])) by (path)", "interval": "", "legendFormat": "{{path}}", @@ -392,9 +670,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "Failed Requests Count", "tooltip": { "shared": true, @@ -403,9 +679,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -413,24 +687,17 @@ { "decimals": 0, "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true }, { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { @@ -438,14 +705,23 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "FWwc47WVk" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 13 + "y": 14 }, "hiddenSeries": false, "id": 4, @@ -463,9 +739,10 @@ "linewidth": 1, "nullPointMode": "null as zero", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "10.4.7", "pointradius": 2, "points": false, "renderer": "flot", @@ -475,24 +752,40 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "FWwc47WVk" + }, "expr": "histogram_quantile(0.99, sum(irate(explorer_backend_request_duration_seconds_bucket{app=\"explorer-backend\"}[1m])) by (le))", "interval": "", "legendFormat": "P99", "refId": "A" }, { + "datasource": { + "type": "prometheus", + "uid": "FWwc47WVk" + }, "expr": "histogram_quantile(0.95, sum(irate(explorer_backend_request_duration_seconds_bucket{app=\"explorer-backend\"}[1m])) by (le))", "interval": "", "legendFormat": "P95", "refId": "B" }, { + "datasource": { + "type": "prometheus", + "uid": "FWwc47WVk" + }, "expr": "histogram_quantile(0.50, sum(irate(explorer_backend_request_duration_seconds_bucket{app=\"explorer-backend\"}[1m])) by (le))", "interval": "", "legendFormat": "P50", "refId": "C" }, { + "datasource": { + "type": "prometheus", + "uid": "FWwc47WVk" + }, "expr": "sum(irate(explorer_backend_request_duration_seconds_sum{app=\"explorer-backend\"}[1m])) / sum(irate(explorer_backend_request_duration_seconds_count{app=\"explorer-backend\"}[1m]))", "interval": "", "legendFormat": "Average", @@ -500,9 +793,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "Request Duration", "tooltip": { "shared": true, @@ -511,33 +802,24 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true }, { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { @@ -545,15 +827,24 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "FWwc47WVk" + }, "description": "", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 13 + "y": 18 }, "hiddenSeries": false, "id": 6, @@ -570,9 +861,10 @@ "linewidth": 1, "nullPointMode": "null as zero", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "10.4.7", "pointradius": 2, "points": false, "renderer": "flot", @@ -582,6 +874,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "FWwc47WVk" + }, "expr": "(sum(irate(explorer_backend_request_duration_seconds_sum{app=\"explorer-backend\"}[1m])) by (path)) / (sum(irate(explorer_backend_request_duration_seconds_count{app=\"explorer-backend\"}[1m])) by (path))", "interval": "", "legendFormat": "{{path}}", @@ -589,9 +885,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "Average Request Duration By Endpoint", "tooltip": { "shared": true, @@ -600,33 +894,24 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true }, { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { @@ -634,15 +919,24 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "FWwc47WVk" + }, "description": "Postgres transactions per second(committed + rollback)", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 21 + "y": 22 }, "hiddenSeries": false, "id": 18, @@ -659,9 +953,10 @@ "linewidth": 1, "nullPointMode": "null as zero", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "10.4.7", "pointradius": 2, "points": false, "renderer": "flot", @@ -671,6 +966,10 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "FWwc47WVk" + }, "expr": "irate(postgres_transactions_total{app=\"explorer-backend\"}[1m])", "interval": "", "legendFormat": "Transactions Per Second", @@ -678,9 +977,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "Postgres Transactions Per Second", "tooltip": { "shared": true, @@ -689,9 +986,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -699,24 +994,17 @@ { "decimals": 0, "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true }, { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { @@ -724,15 +1012,24 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "FWwc47WVk" + }, "description": "Postgres disk blocks read per second", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, - "y": 21 + "y": 26 }, "hiddenSeries": false, "id": 20, @@ -749,9 +1046,10 @@ "linewidth": 1, "nullPointMode": "null as zero", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "10.4.7", "pointradius": 2, "points": false, "renderer": "flot", @@ -761,12 +1059,20 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "FWwc47WVk" + }, "expr": "irate(postgres_blocks_hits_total{app=\"explorer-backend\"}[1m])", "interval": "", "legendFormat": "Blocks Read From Cache", "refId": "A" }, { + "datasource": { + "type": "prometheus", + "uid": "FWwc47WVk" + }, "expr": "irate(postgres_blocks_reads_total{app=\"explorer-backend\"}[1m])", "interval": "", "legendFormat": "Blocks Read From Disk", @@ -774,9 +1080,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "Postgres Disk Blocks Read Per Second", "tooltip": { "shared": true, @@ -785,9 +1089,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -795,24 +1097,17 @@ { "decimals": 0, "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true }, { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { @@ -820,15 +1115,24 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "Prometheus", + "datasource": { + "type": "prometheus", + "uid": "FWwc47WVk" + }, "description": "Postgres rows read/write per second", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 29 + "y": 30 }, "hiddenSeries": false, "id": 22, @@ -845,9 +1149,10 @@ "linewidth": 1, "nullPointMode": "null as zero", "options": { - "dataLinks": [] + "alertThreshold": true }, "percentage": false, + "pluginVersion": "10.4.7", "pointradius": 2, "points": false, "renderer": "flot", @@ -857,24 +1162,40 @@ "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "FWwc47WVk" + }, "expr": "irate(postgres_rows_fetched_total{app=\"explorer-backend\"}[1m])", "interval": "", "legendFormat": "Rows Fetched", "refId": "A" }, { + "datasource": { + "type": "prometheus", + "uid": "FWwc47WVk" + }, "expr": "irate(postgres_rows_inserted_total{app=\"explorer-backend\"}[1m])", "interval": "", "legendFormat": "Rows Inserted", "refId": "B" }, { + "datasource": { + "type": "prometheus", + "uid": "FWwc47WVk" + }, "expr": "irate(postgres_rows_updated_total{app=\"explorer-backend\"}[1m])", "interval": "", "legendFormat": "Rows Updated", "refId": "C" }, { + "datasource": { + "type": "prometheus", + "uid": "FWwc47WVk" + }, "expr": "irate(postgres_rows_deleted_total{app=\"explorer-backend\"}[1m])", "interval": "", "legendFormat": "Rows Deleted", @@ -882,9 +1203,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "Postgres Rows Read/Write Per Second", "tooltip": { "shared": true, @@ -893,9 +1212,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -903,29 +1220,83 @@ { "decimals": 3, "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true }, { "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } + },{ + "datasource": "Prometheus", + "description": "Current Block Height Per Chain", + "fieldConfig": { + "defaults": { + "custom": { + "align": null, + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + }, + { + "color": "green", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 78 + }, + "id": 73, + "options": { + "displayMode": "lcd", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "7.2.1", + "targets": [ + { + "expr": "sum(alephimum_explorer_backend_latest_blocks_synced) by (chain_from, chain_to)", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Chain ({{chain_from}}, {{chain_to}})", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Current Block Height Per Chain", + "type": "bargauge" } ], - "schemaVersion": 22, - "style": "dark", + "schemaVersion": 39, "tags": [], "templating": { "list": [] @@ -951,8 +1322,6 @@ "timezone": "", "title": "Explorer Backend Overview", "uid": "oknDQWN7k", - "variables": { - "list": [] - }, - "version": 17 + "version": 2, + "weekStart": "" } diff --git a/app/src/main/scala/org/alephium/explorer/ExplorerState.scala b/app/src/main/scala/org/alephium/explorer/ExplorerState.scala index 6567da68e..8a1c6ebf8 100644 --- a/app/src/main/scala/org/alephium/explorer/ExplorerState.scala +++ b/app/src/main/scala/org/alephium/explorer/ExplorerState.scala @@ -23,7 +23,7 @@ import com.typesafe.scalalogging.StrictLogging import slick.basic.DatabaseConfig import slick.jdbc.PostgresProfile -import org.alephium.explorer.cache.{BlockCache, TransactionCache} +import org.alephium.explorer.cache.{BlockCache, MetricCache, TransactionCache} import org.alephium.explorer.config.{BootMode, ExplorerConfig} import org.alephium.explorer.persistence.Database import org.alephium.explorer.service._ @@ -64,6 +64,12 @@ sealed trait ExplorerState extends Service with StrictLogging { directCliqueAccess = config.directCliqueAccess ) + implicit lazy val metricCache: MetricCache = + new MetricCache( + database, + config.cacheRowCountReloadPeriod + ) + override def startSelfOnce(): Future[Unit] = { Future.unit } @@ -77,6 +83,7 @@ sealed trait ExplorerState extends Service with StrictLogging { override def subServices: ArraySeq[Service] = { val writeOnlyServices = ArraySeq( + metricCache, transactionCache, blockFlowClient, database diff --git a/app/src/main/scala/org/alephium/explorer/SyncServices.scala b/app/src/main/scala/org/alephium/explorer/SyncServices.scala index 0203cba90..c32e45cf8 100644 --- a/app/src/main/scala/org/alephium/explorer/SyncServices.scala +++ b/app/src/main/scala/org/alephium/explorer/SyncServices.scala @@ -30,7 +30,7 @@ import sttp.model.Uri import org.alephium.api.model.{ChainParams, PeerAddress} import org.alephium.explorer.RichAVector._ -import org.alephium.explorer.cache.BlockCache +import org.alephium.explorer.cache.{BlockCache, MetricCache} import org.alephium.explorer.config.{BootMode, ExplorerConfig} import org.alephium.explorer.error.ExplorerError._ import org.alephium.explorer.service._ @@ -47,6 +47,7 @@ object SyncServices extends StrictLogging { dc: DatabaseConfig[PostgresProfile], blockFlowClient: BlockFlowClient, blockCache: BlockCache, + metricCache: MetricCache, groupSetting: GroupSetting ): Future[Unit] = config.bootMode match { @@ -86,6 +87,7 @@ object SyncServices extends StrictLogging { dc: DatabaseConfig[PostgresProfile], blockFlowClient: BlockFlowClient, blockCache: BlockCache, + metricCache: MetricCache, groupSetting: GroupSetting ): Future[Unit] = Future.fromTry { diff --git a/app/src/main/scala/org/alephium/explorer/cache/BlockCache.scala b/app/src/main/scala/org/alephium/explorer/cache/BlockCache.scala index 77d96f727..237a2e281 100644 --- a/app/src/main/scala/org/alephium/explorer/cache/BlockCache.scala +++ b/app/src/main/scala/org/alephium/explorer/cache/BlockCache.scala @@ -22,6 +22,7 @@ import scala.concurrent.duration._ import scala.jdk.FutureConverters._ import com.github.benmanes.caffeine.cache.{AsyncCacheLoader, Caffeine} +import io.prometheus.metrics.core.metrics.Gauge import slick.basic.DatabaseConfig import slick.jdbc.PostgresProfile import slick.jdbc.PostgresProfile.api._ @@ -60,7 +61,7 @@ object BlockCache { ec: ExecutionContext, dc: DatabaseConfig[PostgresProfile] ): BlockCache = { - val groupConfig: GroupConfig = groupSetting.groupConfig + implicit val groupConfig: GroupConfig = groupSetting.groupConfig /* * `Option.get` is used to avoid unnecessary memory allocations. @@ -125,6 +126,16 @@ object BlockCache { ) } + val latestBlocksSynced: Gauge = Gauge + .builder() + .name( + "alephimum_explorer_backend_latest_blocks_synced" + ) + .help( + "Latest blocks synced per chainindex" + ) + .labelNames("chain_from", "chain_to") + .register() } /** Cache used by Block queries. @@ -135,7 +146,13 @@ class BlockCache( blockTimes: CaffeineAsyncCache[ChainIndex, Duration], rowCount: AsyncReloadingCache[Int], latestBlocks: CaffeineAsyncCache[ChainIndex, LatestBlock] -) { +)(implicit groupConfig: GroupConfig) { + + private val latestBlocksSyncedLabeled = + groupConfig.cliqueChainIndexes.map(chainIndex => + BlockCache.latestBlocksSynced + .labelValues(chainIndex.from.value.toString, chainIndex.to.value.toString) + ) /** Operations on `blockTimes` cache */ def getAllBlockTimes(chainIndexes: Iterable[ChainIndex])(implicit @@ -150,8 +167,12 @@ class BlockCache( ): Future[ArraySeq[(ChainIndex, LatestBlock)]] = latestBlocks.getAll(groupSetting.chainIndexes) - def putLatestBlock(chainIndex: ChainIndex, block: LatestBlock): Unit = + def putLatestBlock(chainIndex: ChainIndex, block: LatestBlock): Unit = { latestBlocks.put(chainIndex, block) + latestBlocksSyncedLabeled + .get(chainIndex.flattenIndex) + .foreach(_.set(block.height.value.toDouble)) + } def getMainChainBlockCount(): Int = rowCount.get() diff --git a/app/src/main/scala/org/alephium/explorer/cache/MetricCache.scala b/app/src/main/scala/org/alephium/explorer/cache/MetricCache.scala new file mode 100644 index 000000000..a5b923884 --- /dev/null +++ b/app/src/main/scala/org/alephium/explorer/cache/MetricCache.scala @@ -0,0 +1,113 @@ +// Copyright 2018 The Alephium Authors +// This file is part of the alephium project. +// +// The library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the library. If not, see . + +package org.alephium.explorer.cache + +import scala.collection.immutable.ArraySeq +import scala.concurrent.{ExecutionContext,Future} +import scala.concurrent.duration._ + +import io.prometheus.metrics.core.metrics.Gauge + +import org.alephium.explorer.persistence.Database +import org.alephium.explorer.persistence.DBRunner._ +import org.alephium.explorer.persistence.queries.{EventQueries,TokenQueries} +import org.alephium.util.Service + +class MetricCache(database:Database, reloadPeriod: FiniteDuration)(implicit val executionContext:ExecutionContext) extends Service { + + private val fungibleCount: AsyncReloadingCache[Int] ={ + AsyncReloadingCache(0, reloadPeriod) { _ => + run(TokenQueries.countFungibles())(database.databaseConfig).map { count => + MetricCache.fungibleCountGauge.set(count.toDouble) + count + } + } + } + + private val nftCount: AsyncReloadingCache[Int] ={ + AsyncReloadingCache(0, reloadPeriod) { _ => + run(TokenQueries.countNFT())(database.databaseConfig).map { count => + MetricCache.nftCountGauge.set(count.toDouble) + count + } + } + } + + private val eventCount: AsyncReloadingCache[Int] ={ + AsyncReloadingCache(0, reloadPeriod) { _ => + run(EventQueries.countEvents())(database.databaseConfig).map { count => + MetricCache.eventCountGauge.set(count.toDouble) + count + } + } + } + + def reloadTokenCountIfOverdue():Unit = { + val _= fungibleCount.get() + val _= nftCount.get() + val _= eventCount.get() + } + + def reloadEventCountIfOverdue():Unit = { + val _= fungibleCount.get() + val _= nftCount.get() + val _= eventCount.get() + } + + override def startSelfOnce(): Future[Unit] = { + for { + _<- fungibleCount.expireAndReloadFuture().map(_ => ()) + _<- nftCount.expireAndReloadFuture().map(_ => ()) + _<- eventCount.expireAndReloadFuture().map(_ => ()) + } yield () + } + + override def stopSelfOnce(): Future[Unit] = { + Future.unit + } + + override def subServices: ArraySeq[Service] = ArraySeq(database) +} +object MetricCache { + val fungibleCountGauge: Gauge = Gauge + .builder() + .name( + "alephimum_explorer_backend_fungible_count" + ).help( + "Number of fungible tokens in the system" + ) + .register() + + val nftCountGauge: Gauge = Gauge + .builder() + .name( + "alephimum_explorer_backend_nft_count" + ).help( + "Number of NFT in the system" + ) + .register() + + val eventCountGauge: Gauge = Gauge + .builder() + .name( + "alephimum_explorer_backend_event_count" + ).help( + "Number of events in the system" + ) + .register() + +} diff --git a/app/src/main/scala/org/alephium/explorer/persistence/queries/EventQueries.scala b/app/src/main/scala/org/alephium/explorer/persistence/queries/EventQueries.scala index 46946a596..277947126 100644 --- a/app/src/main/scala/org/alephium/explorer/persistence/queries/EventQueries.scala +++ b/app/src/main/scala/org/alephium/explorer/persistence/queries/EventQueries.scala @@ -16,6 +16,8 @@ package org.alephium.explorer.persistence.queries +import scala.concurrent.ExecutionContext + import slick.jdbc.{PositionedParameters, SetParameter, SQLActionBuilder} import slick.jdbc.PostgresProfile.api._ @@ -98,4 +100,13 @@ object EventQueries { .paginate(pagination) .asASE[EventEntity](eventGetResult) } + + def countEvents()(implicit + ec: ExecutionContext + ): DBActionR[Int] = { + sql""" + SELECT count(*) + FROM events + """.asAS[Int].exactlyOne + } } diff --git a/app/src/main/scala/org/alephium/explorer/persistence/queries/TokenQueries.scala b/app/src/main/scala/org/alephium/explorer/persistence/queries/TokenQueries.scala index 251318447..b5b632f03 100644 --- a/app/src/main/scala/org/alephium/explorer/persistence/queries/TokenQueries.scala +++ b/app/src/main/scala/org/alephium/explorer/persistence/queries/TokenQueries.scala @@ -356,4 +356,24 @@ object TokenQueries extends StrictLogging { WHERE token = $token """ } + + def countFungibles()(implicit + ec: ExecutionContext + ): DBActionR[Int] = { + sql""" + SELECT count(*) + FROM token_info + WHERE interface_id = '0001' + """.asAS[Int].exactlyOne + } + + def countNFT()(implicit + ec: ExecutionContext + ): DBActionR[Int] = { + sql""" + SELECT count(*) + FROM token_info + WHERE interface_id = '0003' OR interface_id = '000301' + """.asAS[Int].exactlyOne + } } diff --git a/app/src/main/scala/org/alephium/explorer/service/BlockFlowSyncService.scala b/app/src/main/scala/org/alephium/explorer/service/BlockFlowSyncService.scala index e0dacdc97..18e214757 100644 --- a/app/src/main/scala/org/alephium/explorer/service/BlockFlowSyncService.scala +++ b/app/src/main/scala/org/alephium/explorer/service/BlockFlowSyncService.scala @@ -30,7 +30,7 @@ import sttp.model.Uri import org.alephium.explorer.{foldFutures, GroupSetting} import org.alephium.explorer.api.model.Height -import org.alephium.explorer.cache.BlockCache +import org.alephium.explorer.cache.{BlockCache, MetricCache} import org.alephium.explorer.error.ExplorerError.BlocksInDifferentChains import org.alephium.explorer.persistence.DBRunner._ import org.alephium.explorer.persistence.dao.BlockDao @@ -73,6 +73,7 @@ case object BlockFlowSyncService extends StrictLogging { dc: DatabaseConfig[PostgresProfile], blockFlowClient: BlockFlowClient, cache: BlockCache, + metricCache: MetricCache, groupSetting: GroupSetting, scheduler: Scheduler ): Future[Unit] = @@ -88,6 +89,7 @@ case object BlockFlowSyncService extends StrictLogging { dc: DatabaseConfig[PostgresProfile], blockFlowClient: BlockFlowClient, cache: BlockCache, + metricCache: MetricCache, groupSetting: GroupSetting ): Future[Unit] = { if (initialBackStepDone.get()) { @@ -105,6 +107,7 @@ case object BlockFlowSyncService extends StrictLogging { dc: DatabaseConfig[PostgresProfile], blockFlowClient: BlockFlowClient, cache: BlockCache, + metricCache: MetricCache, groupSetting: GroupSetting ): Future[Unit] = { logger.debug("Start syncing") @@ -144,6 +147,7 @@ case object BlockFlowSyncService extends StrictLogging { dc: DatabaseConfig[PostgresProfile], blockFlowClient: BlockFlowClient, cache: BlockCache, + metricCache: MetricCache, groupSetting: GroupSetting ): Future[Int] = { blockFlowClient.fetchBlocks(from, to, uri).flatMap { multiChain => @@ -355,18 +359,33 @@ case object BlockFlowSyncService extends StrictLogging { } } + private def reloadMetricCache(blocks: ArraySeq[BlockEntityWithEvents])(implicit + metricCache: MetricCache + ): Unit = { + if (blocks.map(_.block).exists(_.outputs.exists(_.tokens.map(_.nonEmpty).getOrElse(false)))) { + metricCache.reloadTokenCountIfOverdue() + } + if (blocks.flatMap(_.events).nonEmpty) { + metricCache.reloadEventCountIfOverdue() + } + } + private def insertBlocks(blocksWithEvents: ArraySeq[BlockEntityWithEvents])(implicit ec: ExecutionContext, dc: DatabaseConfig[PostgresProfile], blockFlowClient: BlockFlowClient, cache: BlockCache, + metricCache: MetricCache, groupSetting: GroupSetting ): Future[Int] = { if (blocksWithEvents.nonEmpty) { for { _ <- foldFutures(blocksWithEvents)(insertWithEvents) _ <- BlockDao.updateLatestBlock(blocksWithEvents.last.block) - } yield blocksWithEvents.size + } yield { + reloadMetricCache(blocksWithEvents) + blocksWithEvents.size + } } else { Future.successful(0) } diff --git a/app/src/test/scala/org/alephium/explorer/cache/TestMetricCache.scala b/app/src/test/scala/org/alephium/explorer/cache/TestMetricCache.scala new file mode 100644 index 000000000..00383e2e8 --- /dev/null +++ b/app/src/test/scala/org/alephium/explorer/cache/TestMetricCache.scala @@ -0,0 +1,35 @@ +// Copyright 2018 The Alephium Authors +// This file is part of the alephium project. +// +// The library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the library. If not, see . + +package org.alephium.explorer.cache + +import scala.concurrent.ExecutionContext +import scala.concurrent.duration._ + +import org.alephium.explorer.persistence.Database + +object TestMetricCache { + + def apply(database: Database)(implicit + ec: ExecutionContext + ): MetricCache = + new MetricCache( + database= database, + reloadPeriod = 1.second + ) + +} + diff --git a/app/src/test/scala/org/alephium/explorer/service/BlockFlowSyncServiceSpec.scala b/app/src/test/scala/org/alephium/explorer/service/BlockFlowSyncServiceSpec.scala index 5bc20755b..a79365e49 100644 --- a/app/src/test/scala/org/alephium/explorer/service/BlockFlowSyncServiceSpec.scala +++ b/app/src/test/scala/org/alephium/explorer/service/BlockFlowSyncServiceSpec.scala @@ -32,9 +32,10 @@ import org.alephium.explorer.GenCoreUtil.timestampMaxValue import org.alephium.explorer.GenDBModel._ import org.alephium.explorer.Generators._ import org.alephium.explorer.api.model._ -import org.alephium.explorer.cache.{BlockCache, TestBlockCache} +import org.alephium.explorer.cache._ +import org.alephium.explorer.config.BootMode import org.alephium.explorer.error.ExplorerError -import org.alephium.explorer.persistence.DatabaseFixtureForAll +import org.alephium.explorer.persistence.{Database, DatabaseFixtureForAll} import org.alephium.explorer.persistence.dao.BlockDao import org.alephium.explorer.persistence.model._ import org.alephium.explorer.util.Scheduler @@ -211,7 +212,8 @@ class BlockFlowSyncServiceSpec extends AlephiumFutureSpec with DatabaseFixtureFo def blockFlow: ArraySeq[ArraySeq[BlockEntryTest]] = blockEntitiesToBlockEntries(blockFlowEntity) - implicit val blockCache: BlockCache = TestBlockCache() + implicit val blockCache: BlockCache = TestBlockCache() + implicit val metricCache: MetricCache = TestMetricCache(new Database(BootMode.ReadWrite)) def blockEntities = ArraySeq.from(blockFlowEntity.flatten)