diff --git a/gradle/modules.properties b/gradle/modules.properties index bff4c00c..8290bd36 100644 --- a/gradle/modules.properties +++ b/gradle/modules.properties @@ -12,6 +12,8 @@ com.lmax.disruptor=com.lmax:disruptor io.helidon.webserver=io.helidon.webserver:helidon-webserver io.helidon.webserver.grpc=io.helidon.webserver:helidon-webserver-grpc io.helidon.webserver.testing.junit5=io.helidon.webserver.testing.junit5:helidon-webserver-testing-junit5 + +io.helidon.logging=io.helidon.logging:helidon-logging-jul org.antlr.antlr4.runtime=org.antlr:antlr4-runtime com.google.common=com.google.guava:guava diff --git a/server/build.gradle.kts b/server/build.gradle.kts index 75b890ef..322e4b97 100644 --- a/server/build.gradle.kts +++ b/server/build.gradle.kts @@ -29,9 +29,11 @@ mainModuleInfo { annotationProcessor("com.google.auto.service.processor") runtimeOnly("com.swirlds.config.impl") runtimeOnly("org.apache.logging.log4j.slf4j2.impl") + runtimeOnly("io.helidon.logging") } testModuleInfo { + annotationProcessor("dagger.compiler") requires("org.junit.jupiter.api") requires("org.mockito") requires("org.mockito.junit.jupiter") diff --git a/server/docker/Dockerfile b/server/docker/Dockerfile index 693784aa..50b1c5ad 100644 --- a/server/docker/Dockerfile +++ b/server/docker/Dockerfile @@ -24,5 +24,8 @@ COPY --from=distributions server-${VERSION}.tar . # Extract the TAR file RUN tar -xvf server-${VERSION}.tar +# Copy the logging properties file +COPY logging.properties logging.properties + # RUN the bin script for starting the server ENTRYPOINT ["/bin/bash", "-c", "/app/server-${VERSION}/bin/server"] diff --git a/server/docker/logging.properties b/server/docker/logging.properties new file mode 100644 index 00000000..91d8f441 --- /dev/null +++ b/server/docker/logging.properties @@ -0,0 +1,32 @@ +# Log Level Values +# +# SEVERE: indicates a critical error or failure +# WARNING: warns of potential issues or errors +# INFO: reports normal operational information +# CONFIG: provides configuration-related information +# FINE: provides detailed debugging information +# FINER: provides finer-grained debugging information +# FINEST: provides the most detailed debugging information + +# Set the default logging level +.level=INFO + +# Helidon loggers +io.helidon.webserver.level=SEVERE +io.helidon.config.level=SEVERE +io.helidon.security.level=INFO +io.helidon.common.level=INFO + +# Configure the app log level +#com.hedera.block.level=FINE +#com.hedera.block.server.level=FINE + +# Configure specific loggers +#com.hedera.block.server.mediator.LiveStreamMediatorImpl.level=FINE +#com.hedera.block.server.persistence.storage.write.BlockAsDirWriter.level=FINE +#com.hedera.block.server.producer.ProducerBlockItemObserver.level=FINE + +# Console handler configuration +handlers = java.util.logging.ConsoleHandler +java.util.logging.ConsoleHandler.level = FINE +java.util.logging.ConsoleHandler.formatter = java.util.logging.SimpleFormatter diff --git a/server/docker/metrics/dashboards/block-node-server.json b/server/docker/metrics/dashboards/block-node-server.json index f32013ac..e28ff6b8 100644 --- a/server/docker/metrics/dashboards/block-node-server.json +++ b/server/docker/metrics/dashboards/block-node-server.json @@ -21,13 +21,189 @@ "links": [], "panels": [ { - "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "id": 16, + "title": "Errors", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 15, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "hedera_block_node_live_block_stream_mediator_error_total", + "instant": false, + "legendFormat": "Block Item Errors", + "range": true, + "refId": "A" + } + ], + "title": "Mediator Errors", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 18, + "x": 6, + "y": 1 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate( hedera_block_node_live_block_stream_mediator_error_total [$__rate_interval])", + "instant": false, + "legendFormat": "Mediator Errors", + "range": true, + "refId": "A" + } + ], + "title": "Rate of Mediator Errors", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, "id": 9, "panels": [], "title": "Live Stream", @@ -52,20 +228,21 @@ "value": null } ] - } + }, + "unit": "short" }, "overrides": [] }, "gridPos": { - "h": 7, + "h": 8, "w": 6, "x": 0, - "y": 1 + "y": 10 }, - "id": 3, + "id": 13, "options": { "colorMode": "value", - "graphMode": "area", + "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", @@ -80,7 +257,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.1.3", + "pluginVersion": "11.1.4", "targets": [ { "datasource": { @@ -88,14 +265,14 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "hedera_block_node_live_block_items_total", + "expr": "hedera_block_node_live_block_items_received_total", "instant": false, "legendFormat": "BlockItems", "range": true, "refId": "A" } ], - "title": "Live Block Item Counter", + "title": "Live Block Items Received from Producer", "type": "stat" }, { @@ -103,6 +280,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -137,7 +315,7 @@ "mode": "none" }, "thresholdsStyle": { - "mode": "area" + "mode": "off" } }, "mappings": [], @@ -145,7 +323,7 @@ "mode": "absolute", "steps": [ { - "color": "dark-red", + "color": "red", "value": null }, { @@ -163,12 +341,12 @@ "overrides": [] }, "gridPos": { - "h": 7, + "h": 8, "w": 18, "x": 6, - "y": 1 + "y": 10 }, - "id": 7, + "id": 12, "options": { "legend": { "calcs": [], @@ -181,7 +359,6 @@ "sort": "none" } }, - "pluginVersion": "11.1.3", "targets": [ { "datasource": { @@ -189,14 +366,14 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "rate( hedera_block_node_live_block_items_total [$__rate_interval])", + "expr": "rate( hedera_block_node_live_block_items_received_total [$__rate_interval])", "instant": false, "legendFormat": "BlockItems", "range": true, "refId": "A" } ], - "title": "Live Block Item Counter", + "title": "Rate of Live Block Items Received from Producer", "type": "timeseries" }, { @@ -218,7 +395,8 @@ "value": null } ] - } + }, + "unit": "short" }, "overrides": [] }, @@ -226,12 +404,12 @@ "h": 7, "w": 6, "x": 0, - "y": 8 + "y": 18 }, - "id": 6, + "id": 3, "options": { "colorMode": "value", - "graphMode": "area", + "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", @@ -246,7 +424,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.1.3", + "pluginVersion": "11.1.4", "targets": [ { "datasource": { @@ -254,14 +432,14 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "hedera_block_node_blocks_persisted_total", + "expr": "hedera_block_node_live_block_items_total", "instant": false, - "legendFormat": "Block Persistence Counter", + "legendFormat": "BlockItems", "range": true, "refId": "A" } ], - "title": "Block Persistence Counter", + "title": "Live Block Item Counter", "type": "stat" }, { @@ -311,20 +489,20 @@ "mode": "absolute", "steps": [ { - "color": "red", + "color": "dark-red", "value": null }, { "color": "#EAB839", - "value": 2 + "value": 5 }, { "color": "green", - "value": 4 + "value": 30 } ] }, - "unit": "wps" + "unit": "reqps" }, "overrides": [] }, @@ -332,9 +510,9 @@ "h": 7, "w": 18, "x": 6, - "y": 8 + "y": 18 }, - "id": 8, + "id": 7, "options": { "legend": { "calcs": [], @@ -355,14 +533,14 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "rate(hedera_block_node_blocks_persisted_total[$__rate_interval])", + "expr": "rate( hedera_block_node_live_block_items_total [$__rate_interval])", "instant": false, - "legendFormat": "Block Persistence Counter", + "legendFormat": "BlockItems", "range": true, "refId": "A" } ], - "title": "Block Persistence Counter", + "title": "Rate of Live Block Items Received by Mediator", "type": "timeseries" }, { @@ -375,41 +553,33 @@ "color": { "mode": "thresholds" }, - "displayName": "Subscribers", "mappings": [], - "max": 20, - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null - }, - { - "color": "yellow", - "value": 10 - }, - { - "color": "red", - "value": 15 } ] - } + }, + "unit": "short" }, "overrides": [] }, "gridPos": { - "h": 9, - "w": 9, + "h": 7, + "w": 6, "x": 0, - "y": 15 + "y": 25 }, - "id": 4, + "id": 11, "options": { - "minVizHeight": 75, - "minVizWidth": 75, + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -417,11 +587,11 @@ "fields": "", "values": false }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "11.1.3", + "pluginVersion": "11.1.4", "targets": [ { "datasource": { @@ -429,15 +599,15 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "hedera_block_node_subscribers", + "expr": "hedera_block_node_live_block_items_consumed_total", "instant": false, - "legendFormat": "__auto", + "legendFormat": "BlockItems Consumed", "range": true, "refId": "A" } ], - "title": "Subscribers", - "type": "gauge" + "title": "Live Block Items Consumed Counter", + "type": "stat" }, { "datasource": { @@ -486,11 +656,15 @@ "mode": "absolute", "steps": [ { - "color": "green", + "color": "red", "value": null }, { - "color": "red", + "color": "#EAB839", + "value": 5 + }, + { + "color": "green", "value": 80 } ] @@ -500,12 +674,12 @@ "overrides": [] }, "gridPos": { - "h": 9, - "w": 15, - "x": 9, - "y": 15 + "h": 7, + "w": 18, + "x": 6, + "y": 25 }, - "id": 5, + "id": 10, "options": { "legend": { "calcs": [], @@ -525,14 +699,613 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "rate(hedera_block_node_single_blocks_retrieved_total[$__rate_interval])", + "expr": "rate( hedera_block_node_live_block_items_consumed_total [$__rate_interval])", "instant": false, - "legendFormat": "RPS of Single Blocks Retrieval", + "legendFormat": "BlockItems", + "range": true, + "refId": "A" + } + ], + "title": "Rate of Block Items Sent to Consumer(s)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "displayName": "Subscribers", + "mappings": [], + "max": 20, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 10 + }, + { + "color": "red", + "value": 15 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 32 + }, + "id": 4, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "hedera_block_node_subscribers", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Subscribers", + "type": "gauge" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 18, + "panels": [], + "title": "Persistence", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 41 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "hedera_block_node_blocks_persisted_total", + "instant": false, + "legendFormat": "Block Persistence Counter", + "range": true, + "refId": "A" + } + ], + "title": "Block Persistence Counter", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "#EAB839", + "value": 2 + }, + { + "color": "green", + "value": 4 + } + ] + }, + "unit": "wps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 18, + "x": 6, + "y": 41 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(hedera_block_node_blocks_persisted_total[$__rate_interval])", + "instant": false, + "legendFormat": "Block Persistence Counter", + "range": true, + "refId": "A" + } + ], + "title": "Rate of Blocks Written to the File System", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 17, + "panels": [], + "title": "Single Block", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 49 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "hedera_block_node_single_blocks_retrieved_total", + "instant": false, + "legendFormat": "Single Blocks Retrieved", + "range": true, + "refId": "A" + } + ], + "title": "Single Blocks Retrieved", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 18, + "x": 6, + "y": 49 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(hedera_block_node_single_blocks_retrieved_total[$__rate_interval])", + "instant": false, + "legendFormat": "RPS of Single Blocks Retrieval", + "range": true, + "refId": "A" + } + ], + "title": "Rate of Single Blocks Retrieved", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 57 + }, + "id": 21, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "hedera_block_node_single_blocks_not_found_total", + "instant": false, + "legendFormat": "Single Blocks Not Found", + "range": true, + "refId": "A" + } + ], + "title": "Single Blocks Not Found", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 18, + "x": 6, + "y": 57 + }, + "id": 20, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate( hedera_block_node_single_blocks_not_found_total [$__rate_interval])", + "instant": false, + "legendFormat": "Single Blocks Not Found", "range": true, "refId": "A" } ], - "title": "Rate of Single Blocks Retrieval", + "title": "Rate of Single Blocks Not Found", "type": "timeseries" } ], diff --git a/server/docker/update-env.sh b/server/docker/update-env.sh index a840ba65..eda32dd2 100755 --- a/server/docker/update-env.sh +++ b/server/docker/update-env.sh @@ -14,7 +14,7 @@ echo "REGISTRY_PREFIX=" >> .env echo "BLOCKNODE_STORAGE_ROOT_PATH=/app/storage" >> .env if [ $# -eq 2 ]; then - echo "SERVER_OPTS='-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005'" >> .env + echo "SERVER_OPTS='-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005'" >> .env fi echo "DEBUG $2" diff --git a/server/src/main/java/com/hedera/block/server/BlockStreamService.java b/server/src/main/java/com/hedera/block/server/BlockStreamService.java index 8f16dd8b..009bfc60 100644 --- a/server/src/main/java/com/hedera/block/server/BlockStreamService.java +++ b/server/src/main/java/com/hedera/block/server/BlockStreamService.java @@ -22,6 +22,8 @@ import static com.hedera.block.server.Constants.SINGLE_BLOCK_METHOD_NAME; import static com.hedera.block.server.Translator.fromPbj; import static com.hedera.block.server.Translator.toPbj; +import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.SingleBlocksNotFound; +import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.SingleBlocksRetrieved; import static java.lang.System.Logger; import static java.lang.System.Logger.Level.DEBUG; import static java.lang.System.Logger.Level.ERROR; @@ -65,6 +67,7 @@ public class BlockStreamService implements GrpcService { private final ServiceStatus serviceStatus; private final BlockReader blockReader; private final BlockNodeContext blockNodeContext; + private final MetricsService metricsService; /** * Constructor for the BlockStreamService class. It initializes the BlockStreamService with the @@ -89,6 +92,7 @@ public class BlockStreamService implements GrpcService { this.blockReader = blockReader; this.serviceStatus = serviceStatus; this.blockNodeContext = blockNodeContext; + this.metricsService = blockNodeContext.metricsService(); } /** @@ -136,7 +140,7 @@ StreamObserver protocPublishB LOGGER.log(DEBUG, "Executing bidirectional publishBlockStream gRPC method"); return new ProducerBlockItemObserver( - streamMediator, publishStreamResponseObserver, serviceStatus); + streamMediator, publishStreamResponseObserver, blockNodeContext, serviceStatus); } void protocSubscribeBlockStream( @@ -201,11 +205,11 @@ private void singleBlock( singleBlockResponseStreamObserver.onNext( fromPbjSingleBlockSuccessResponse(blockOpt.get())); - final MetricsService metricsService = blockNodeContext.metricsService(); - metricsService.singleBlocksRetrieved().increment(); + metricsService.get(SingleBlocksRetrieved).increment(); } else { LOGGER.log(DEBUG, "Block number {0} not found", blockNumber); singleBlockResponseStreamObserver.onNext(buildSingleBlockNotFoundResponse()); + metricsService.get(SingleBlocksNotFound).increment(); } } catch (IOException e) { LOGGER.log(ERROR, "Error reading block number: {0}", blockNumber); diff --git a/server/src/main/java/com/hedera/block/server/Constants.java b/server/src/main/java/com/hedera/block/server/Constants.java index 8fbd6900..9f12b594 100644 --- a/server/src/main/java/com/hedera/block/server/Constants.java +++ b/server/src/main/java/com/hedera/block/server/Constants.java @@ -25,6 +25,12 @@ private Constants() {} /** Constant mapped to the application.properties file in resources with default values */ @NonNull public static final String APPLICATION_PROPERTIES = "app.properties"; + /** + * Constant mapped to the Helidon logging.properties file in the docker directory with default + * values. + */ + @NonNull public static final String LOGGING_PROPERTIES = "logging.properties"; + /** Constant mapped to the name of the service in the .proto file */ @NonNull public static final String SERVICE_NAME = "BlockStreamService"; diff --git a/server/src/main/java/com/hedera/block/server/Server.java b/server/src/main/java/com/hedera/block/server/Server.java index 48b39490..1365bac7 100644 --- a/server/src/main/java/com/hedera/block/server/Server.java +++ b/server/src/main/java/com/hedera/block/server/Server.java @@ -16,6 +16,9 @@ package com.hedera.block.server; +import static com.hedera.block.server.Constants.APPLICATION_PROPERTIES; +import static com.hedera.block.server.Constants.LOGGING_PROPERTIES; +import static io.helidon.config.ConfigSources.file; import static java.lang.System.Logger; import static java.lang.System.Logger.Level.INFO; @@ -24,8 +27,10 @@ import com.swirlds.config.extensions.sources.ClasspathFileConfigSource; import com.swirlds.config.extensions.sources.SystemEnvironmentConfigSource; import com.swirlds.config.extensions.sources.SystemPropertiesConfigSource; +import io.helidon.config.Config; import java.io.IOException; import java.nio.file.Path; +import java.nio.file.Paths; /** Main class for the block node server */ public class Server { @@ -43,14 +48,20 @@ private Server() {} public static void main(final String[] args) throws IOException { LOGGER.log(INFO, "Starting BlockNode Server"); + // Set the global configuration + final Config config = + Config.builder() + .sources(file(Paths.get("/app", LOGGING_PROPERTIES)).optional()) + .build(); + + Config.global(config); + // Init BlockNode Configuration Configuration configuration = ConfigurationBuilder.create() .withSource(SystemEnvironmentConfigSource.getInstance()) .withSource(SystemPropertiesConfigSource.getInstance()) - .withSource( - new ClasspathFileConfigSource( - Path.of(Constants.APPLICATION_PROPERTIES))) + .withSources(new ClasspathFileConfigSource(Path.of(APPLICATION_PROPERTIES))) .autoDiscoverExtensions() .build(); diff --git a/server/src/main/java/com/hedera/block/server/ServiceStatus.java b/server/src/main/java/com/hedera/block/server/ServiceStatus.java index 1dc330d6..457e2c9d 100644 --- a/server/src/main/java/com/hedera/block/server/ServiceStatus.java +++ b/server/src/main/java/com/hedera/block/server/ServiceStatus.java @@ -35,9 +35,9 @@ public interface ServiceStatus { /** * Sets the running status of the service. * - * @param running true if the service is running, false otherwise + * @param className the name of the class stopping the service */ - void setRunning(final boolean running); + void stopRunning(final String className); /** * Sets the web server instance. diff --git a/server/src/main/java/com/hedera/block/server/ServiceStatusImpl.java b/server/src/main/java/com/hedera/block/server/ServiceStatusImpl.java index dd3243a1..53377428 100644 --- a/server/src/main/java/com/hedera/block/server/ServiceStatusImpl.java +++ b/server/src/main/java/com/hedera/block/server/ServiceStatusImpl.java @@ -16,6 +16,8 @@ package com.hedera.block.server; +import static java.lang.System.Logger.Level.DEBUG; + import edu.umd.cs.findbugs.annotations.NonNull; import io.helidon.webserver.WebServer; import java.util.concurrent.atomic.AtomicBoolean; @@ -29,6 +31,8 @@ @Singleton public class ServiceStatusImpl implements ServiceStatus { + private final System.Logger LOGGER = System.getLogger(getClass().getName()); + private final AtomicBoolean isRunning = new AtomicBoolean(true); private WebServer webServer; @@ -48,10 +52,11 @@ public boolean isRunning() { /** * Sets the running status of the service. * - * @param running true if the service is running, false otherwise + * @param className the name of the class stopping the service */ - public void setRunning(final boolean running) { - isRunning.set(running); + public void stopRunning(final String className) { + LOGGER.log(DEBUG, String.format("%s set the status to stopped", className)); + isRunning.set(false); } /** diff --git a/server/src/main/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserver.java b/server/src/main/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserver.java index 3860bcde..25a028c8 100644 --- a/server/src/main/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserver.java +++ b/server/src/main/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserver.java @@ -17,6 +17,7 @@ package com.hedera.block.server.consumer; import static com.hedera.block.server.Translator.fromPbj; +import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.LiveBlockItemsConsumed; import static java.lang.System.Logger; import static java.lang.System.Logger.Level.DEBUG; import static java.lang.System.Logger.Level.ERROR; @@ -24,6 +25,7 @@ import com.hedera.block.server.config.BlockNodeContext; import com.hedera.block.server.data.ObjectEvent; import com.hedera.block.server.mediator.SubscriptionHandler; +import com.hedera.block.server.metrics.MetricsService; import com.hedera.hapi.block.SubscribeStreamResponse; import com.hedera.hapi.block.stream.BlockItem; import com.hedera.pbj.runtime.OneOf; @@ -45,6 +47,7 @@ public class ConsumerStreamResponseObserver private final Logger LOGGER = System.getLogger(getClass().getName()); + private final MetricsService metricsService; private final StreamObserver subscribeStreamResponseObserver; private final SubscriptionHandler> subscriptionHandler; @@ -98,6 +101,7 @@ public ConsumerStreamResponseObserver( .getConfigData(ConsumerConfig.class) .timeoutThresholdMillis(); this.subscriptionHandler = subscriptionHandler; + this.metricsService = context.metricsService(); // The ServerCallStreamObserver can be configured with Runnable handlers to // be executed when a downstream consumer closes the connection. The handlers @@ -212,6 +216,9 @@ public void send(@NonNull final SubscribeStreamResponse subscribeStreamResponse) if (streamStarted) { LOGGER.log(DEBUG, "Sending BlockItem downstream: {0}", blockItem); + + // Increment counter + metricsService.get(LiveBlockItemsConsumed).increment(); subscribeStreamResponseObserver.onNext(fromPbj(subscribeStreamResponse)); } } diff --git a/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediatorImpl.java b/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediatorImpl.java index 7feee6c3..7691b62b 100644 --- a/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediatorImpl.java +++ b/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediatorImpl.java @@ -16,6 +16,9 @@ package com.hedera.block.server.mediator; +import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.LiveBlockItems; +import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.LiveBlockStreamMediatorError; +import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Gauge.Subscribers; import static java.lang.System.Logger; import static java.lang.System.Logger.Level.DEBUG; import static java.lang.System.Logger.Level.ERROR; @@ -34,7 +37,6 @@ import com.lmax.disruptor.RingBuffer; import com.lmax.disruptor.dsl.Disruptor; import com.lmax.disruptor.util.DaemonThreadFactory; -import com.swirlds.metrics.api.LongGauge; import edu.umd.cs.findbugs.annotations.NonNull; import java.io.IOException; import java.util.Map; @@ -62,7 +64,6 @@ class LiveStreamMediatorImpl private final BlockWriter blockWriter; private final ServiceStatus serviceStatus; - private final BlockNodeContext blockNodeContext; private final MetricsService metricsService; /** @@ -97,7 +98,6 @@ class LiveStreamMediatorImpl this.ringBuffer = disruptor.start(); this.executor = Executors.newCachedThreadPool(DaemonThreadFactory.INSTANCE); this.serviceStatus = serviceStatus; - this.blockNodeContext = blockNodeContext; this.metricsService = blockNodeContext.metricsService(); } @@ -121,21 +121,25 @@ public void publish(@NonNull final BlockItem blockItem) throws IOException { ringBuffer.publishEvent((event, sequence) -> event.set(subscribeStreamResponse)); // Increment the block item counter - metricsService.liveBlockItems().increment(); + metricsService.get(LiveBlockItems).increment(); try { // Persist the BlockItem blockWriter.write(blockItem); } catch (IOException e) { + + // Increment the error counter + metricsService.get(LiveBlockStreamMediatorError).increment(); + // Disable BlockItem publication for upstream producers - serviceStatus.setRunning(false); + serviceStatus.stopRunning(this.getClass().getName()); LOGGER.log( ERROR, "An exception occurred while attempting to persist the BlockItem: " + blockItem, e); - LOGGER.log(DEBUG, "Send a response to end the stream"); + LOGGER.log(ERROR, "Send a response to end the stream"); // Publish the block for all subscribers to receive final SubscribeStreamResponse endStreamResponse = buildEndStreamResponse(); @@ -143,7 +147,7 @@ public void publish(@NonNull final BlockItem blockItem) throws IOException { // Unsubscribe all downstream consumers for (final var subscriber : subscribers.keySet()) { - LOGGER.log(DEBUG, "Unsubscribing: {0}", subscriber); + LOGGER.log(ERROR, String.format("Unsubscribing: %s", subscriber)); unsubscribe(subscriber); } @@ -169,7 +173,8 @@ public void subscribe( // Keep track of the subscriber subscribers.put(handler, batchEventProcessor); - updateSubscriberMetrics(); + // update the subscriber metrics + metricsService.get(Subscribers).set(subscribers.size()); } @Override @@ -190,7 +195,8 @@ public void unsubscribe( ringBuffer.removeGatingSequence(batchEventProcessor.getSequence()); } - updateSubscriberMetrics(); + // update the subscriber metrics + metricsService.get(Subscribers).set(subscribers.size()); } @Override @@ -208,10 +214,4 @@ private static SubscribeStreamResponse buildEndStreamResponse() { .status(SubscribeStreamResponseCode.READ_STREAM_SUCCESS) .build(); } - - private void updateSubscriberMetrics() { - @NonNull final MetricsService metricsService = blockNodeContext.metricsService(); - @NonNull final LongGauge longGauge = metricsService.subscribers(); - longGauge.set(subscribers.size()); - } } diff --git a/server/src/main/java/com/hedera/block/server/metrics/BlockNodeMetricTypes.java b/server/src/main/java/com/hedera/block/server/metrics/BlockNodeMetricTypes.java new file mode 100644 index 00000000..c5d968e4 --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/metrics/BlockNodeMetricTypes.java @@ -0,0 +1,124 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.metrics; + +import edu.umd.cs.findbugs.annotations.NonNull; + +/** + * The BlockNodeMetricNames class contains the names of the metrics used by the BlockNode. + * + *

These names are used to register the metrics with the metrics service. + */ +public final class BlockNodeMetricTypes { + private BlockNodeMetricTypes() {} + + /** + * Add new counting metrics to this enum to automatically register them with the metrics + * service. + * + *

Each enum value should have a unique grafana label and meaningful description. These + * counters can capture data on standard operations or errors. + */ + public enum Counter implements MetricMetadata { + // Standard counters + /** The number of live block items received from a producer. */ + LiveBlockItemsReceived("live_block_items_received", "Live Block Items Received"), + + /** The number of live block items received before publishing to the RingBuffer. */ + LiveBlockItems("live_block_items", "Live BlockItems"), + + /** + * The number of blocks persisted to storage. + * + *

Block items are not counted here, only the blocks. + */ + BlocksPersisted("blocks_persisted", "Blocks Persisted"), + + /** The number of live block items consumed from the by each consumer observer. */ + LiveBlockItemsConsumed("live_block_items_consumed", "Live Block Items Consumed"), + + /** The number of single blocks retrieved from the singleBlock rpc service. */ + SingleBlocksRetrieved("single_blocks_retrieved", "Single Blocks Retrieved"), + + /** The number of single blocks not found via the singleBlock rpc service. */ + SingleBlocksNotFound("single_blocks_not_found", "Single Blocks Not Found"), + + // Error counters + + /** The number of errors encountered by the live block stream mediator. */ + LiveBlockStreamMediatorError( + "live_block_stream_mediator_error", "Live Block Stream Mediator Error"); + + private final String grafanaLabel; + private final String description; + + Counter(String grafanaLabel, String description) { + this.grafanaLabel = grafanaLabel; + this.description = description; + } + + @Override + @NonNull + public String grafanaLabel() { + return grafanaLabel; + } + + @Override + @NonNull + public String description() { + return description; + } + } + + /** + * Add new gauge metrics to this enum to automatically register them with the metrics service. + * + *

Each enum value should have a unique grafana label and meaningful description. These + * gauges can capture data on standard operations or errors. + */ + public enum Gauge implements MetricMetadata { + + /** The number of subscribers receiving the live block stream. */ + Subscribers("subscribers", "Subscribers"); + + private final String grafanaLabel; + private final String description; + + Gauge(String grafanaLabel, String description) { + this.grafanaLabel = grafanaLabel; + this.description = description; + } + + @Override + @NonNull + public String grafanaLabel() { + return grafanaLabel; + } + + @Override + @NonNull + public String description() { + return description; + } + } + + private interface MetricMetadata { + String grafanaLabel(); + + String description(); + } +} diff --git a/server/src/main/java/com/hedera/block/server/metrics/MetricsService.java b/server/src/main/java/com/hedera/block/server/metrics/MetricsService.java index 6269a74e..205c17e1 100644 --- a/server/src/main/java/com/hedera/block/server/metrics/MetricsService.java +++ b/server/src/main/java/com/hedera/block/server/metrics/MetricsService.java @@ -23,34 +23,18 @@ /** Use member variables of this class to update metric data for the Hedera Block Node. */ public interface MetricsService { /** - * Update the counter of live block items transiting via the live stream. + * Use this method to get a specific counter for the given metric type. * - * @return use this metric to increase the counter of block items received + * @param key to get a specific counter + * @return the counter */ - @NonNull - Counter liveBlockItems(); + Counter get(@NonNull BlockNodeMetricTypes.Counter key); /** - * Update the counter of blocks persisted to storage. + * Use this method to get a specific gauge for the given metric type. * - * @return use this counter to increase the amount of blocks persisted to disk + * @param key to get a specific gauge + * @return the gauge */ - @NonNull - Counter blocksPersisted(); - - /** - * Update the counter of single blocks retrieved from storage. - * - * @return use this metric to increase the counter of single blocks retrieved - */ - @NonNull - Counter singleBlocksRetrieved(); - - /** - * Update the gauge of subscribers currently consuming to the live stream. - * - * @return Use this to increase or decrease the amount of current subscribers to the live stream - */ - @NonNull - LongGauge subscribers(); + LongGauge get(@NonNull BlockNodeMetricTypes.Gauge key); } diff --git a/server/src/main/java/com/hedera/block/server/metrics/MetricsServiceImpl.java b/server/src/main/java/com/hedera/block/server/metrics/MetricsServiceImpl.java index a05052df..77631833 100644 --- a/server/src/main/java/com/hedera/block/server/metrics/MetricsServiceImpl.java +++ b/server/src/main/java/com/hedera/block/server/metrics/MetricsServiceImpl.java @@ -20,6 +20,7 @@ import com.swirlds.metrics.api.LongGauge; import com.swirlds.metrics.api.Metrics; import edu.umd.cs.findbugs.annotations.NonNull; +import java.util.EnumMap; import javax.inject.Inject; /** @@ -32,69 +33,58 @@ public class MetricsServiceImpl implements MetricsService { private static final String CATEGORY = "hedera_block_node"; - // Live BlockItem Counter - private static final Counter.Config LIVE_BLOCK_ITEM_COUNTER = - new Counter.Config(CATEGORY, "live_block_items").withDescription("Live BlockItems"); + private final EnumMap counters = + new EnumMap<>(BlockNodeMetricTypes.Counter.class); + private final EnumMap gauges = + new EnumMap<>(BlockNodeMetricTypes.Gauge.class); - // Block Persistence Counter - private static final Counter.Config BLOCK_PERSISTENCE_COUNTER = - new Counter.Config(CATEGORY, "blocks_persisted").withDescription("Blocks Persisted"); - - // Subscriber Gauge - private static final LongGauge.Config SUBSCRIBER_GAUGE = - new LongGauge.Config(CATEGORY, "subscribers").withDescription("Subscribers"); - - // Single Block Retrieved Counter - private static final Counter.Config SINGLE_BLOCK_RETRIEVED_COUNTER = - new Counter.Config(CATEGORY, "single_blocks_retrieved") - .withDescription("Single Blocks Retrieved"); - - private final Counter liveBlockItems; - - private final Counter blocksPersisted; - - private final Counter singleBlocksRetrieved; - - private final LongGauge subscribers; - - /** Update the counter of live block items transiting via the live stream. */ - @Override - @NonNull - public final Counter liveBlockItems() { - return liveBlockItems; - } - - /** Update the counter of blocks persisted to storage. */ - @Override - @NonNull - public final Counter blocksPersisted() { - return blocksPersisted; + /** + * Create singleton instance of metrics service to be used throughout the application. + * + * @param metrics the metrics instance + */ + @Inject + public MetricsServiceImpl(@NonNull final Metrics metrics) { + // Initialize the counters + for (BlockNodeMetricTypes.Counter counter : BlockNodeMetricTypes.Counter.values()) { + counters.put( + counter, + metrics.getOrCreate( + new Counter.Config(CATEGORY, counter.grafanaLabel()) + .withDescription(counter.description()))); + } + + // Initialize the gauges + for (BlockNodeMetricTypes.Gauge gauge : BlockNodeMetricTypes.Gauge.values()) { + gauges.put( + gauge, + metrics.getOrCreate( + new LongGauge.Config(CATEGORY, gauge.grafanaLabel()) + .withDescription(gauge.description()))); + } } - /** Update the counter of single blocks retrieved from storage. */ - @Override + /** + * Use this method to get a specific counter for the given metric type. + * + * @param key to get a specific counter + * @return the counter + */ @NonNull - public final Counter singleBlocksRetrieved() { - return singleBlocksRetrieved; - } - - /** Update the gauge of subscribers currently consuming to the live stream. */ @Override - @NonNull - public final LongGauge subscribers() { - return subscribers; + public Counter get(@NonNull BlockNodeMetricTypes.Counter key) { + return counters.get(key); } /** - * Create singleton instance of metrics service to be used throughout the application. + * Use this method to get a specific gauge for the given metric type. * - * @param metrics the metrics instance + * @param key to get a specific gauge + * @return the gauge */ - @Inject - public MetricsServiceImpl(@NonNull final Metrics metrics) { - this.liveBlockItems = metrics.getOrCreate(LIVE_BLOCK_ITEM_COUNTER); - this.blocksPersisted = metrics.getOrCreate(BLOCK_PERSISTENCE_COUNTER); - this.singleBlocksRetrieved = metrics.getOrCreate(SINGLE_BLOCK_RETRIEVED_COUNTER); - this.subscribers = metrics.getOrCreate(SUBSCRIBER_GAUGE); + @NonNull + @Override + public LongGauge get(@NonNull BlockNodeMetricTypes.Gauge key) { + return gauges.get(key); } } diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriter.java b/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriter.java index 91d7580a..e5c96e75 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriter.java +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriter.java @@ -17,6 +17,7 @@ package com.hedera.block.server.persistence.storage.write; import static com.hedera.block.server.Constants.BLOCK_FILE_EXTENSION; +import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.BlocksPersisted; import static java.lang.System.Logger; import static java.lang.System.Logger.Level.DEBUG; import static java.lang.System.Logger.Level.ERROR; @@ -55,7 +56,7 @@ class BlockAsDirWriter implements BlockWriter { private Path currentBlockDir; private final FileAttribute> filePerms; private final BlockRemover blockRemover; - private final BlockNodeContext blockNodeContext; + private final MetricsService metricsService; /** * Use the corresponding builder to construct a new BlockAsDirWriter with the given parameters. @@ -86,7 +87,7 @@ class BlockAsDirWriter implements BlockWriter { // Initialize the block node root directory if it does not exist FileUtils.createPathIfNotExists(blockNodeRootPath, INFO, filePerms); - this.blockNodeContext = blockNodeContext; + this.metricsService = blockNodeContext.metricsService(); } /** @@ -162,8 +163,7 @@ private void resetState(@NonNull final BlockItem blockItem) throws IOException { blockNodeFileNameIndex = 0; // Increment the block counter - final MetricsService metricsService = blockNodeContext.metricsService(); - metricsService.blocksPersisted().increment(); + metricsService.get(BlocksPersisted).increment(); } private void repairPermissions(@NonNull final Path path) throws IOException { diff --git a/server/src/main/java/com/hedera/block/server/producer/ProducerBlockItemObserver.java b/server/src/main/java/com/hedera/block/server/producer/ProducerBlockItemObserver.java index d8bb5fb3..18151583 100644 --- a/server/src/main/java/com/hedera/block/server/producer/ProducerBlockItemObserver.java +++ b/server/src/main/java/com/hedera/block/server/producer/ProducerBlockItemObserver.java @@ -18,13 +18,16 @@ import static com.hedera.block.server.Translator.fromPbj; import static com.hedera.block.server.Translator.toPbj; +import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.LiveBlockItemsReceived; import static com.hedera.block.server.producer.Util.getFakeHash; import static java.lang.System.Logger; import static java.lang.System.Logger.Level.DEBUG; import static java.lang.System.Logger.Level.ERROR; import com.hedera.block.server.ServiceStatus; +import com.hedera.block.server.config.BlockNodeContext; import com.hedera.block.server.mediator.Publisher; +import com.hedera.block.server.metrics.MetricsService; import com.hedera.hapi.block.Acknowledgement; import com.hedera.hapi.block.EndOfStream; import com.hedera.hapi.block.ItemAcknowledgement; @@ -53,6 +56,7 @@ public class ProducerBlockItemObserver publishStreamResponseObserver; private final Publisher publisher; private final ServiceStatus serviceStatus; + private final MetricsService metricsService; /** * Constructor for the ProducerBlockStreamObserver class. It is responsible for calling the @@ -60,9 +64,11 @@ public class ProducerBlockItemObserver * to the upstream producer via the responseStreamObserver. * * @param publisher the block item publisher to used to pass block items to consumers as they - * arrive from the upstream producer + * arrive from the upstream producer. * @param publishStreamResponseObserver the response stream observer to send responses back to - * the upstream producer for each block item processed + * the upstream producer for each block item processed. + * @param blockNodeContext the block node context used to access context objects for the Block + * Node (e.g. - the metrics service). * @param serviceStatus the service status used to determine if the downstream service is * accepting block items. In the event of an unrecoverable exception, it will be used to * stop the web server. @@ -72,10 +78,12 @@ public ProducerBlockItemObserver( @NonNull final StreamObserver publishStreamResponseObserver, + @NonNull final BlockNodeContext blockNodeContext, @NonNull final ServiceStatus serviceStatus) { this.publisher = publisher; this.publishStreamResponseObserver = publishStreamResponseObserver; + this.metricsService = blockNodeContext.metricsService(); this.serviceStatus = serviceStatus; } @@ -92,8 +100,12 @@ public void onNext( try { + LOGGER.log(DEBUG, "Received PublishStreamRequest from producer"); final BlockItem blockItem = toPbj(BlockItem.PROTOBUF, publishStreamRequest.getBlockItem().toByteArray()); + LOGGER.log(DEBUG, "Received block item: " + blockItem); + + metricsService.get(LiveBlockItemsReceived).increment(); // Publish the block to all the subscribers unless // there's an issue with the StreamMediator. @@ -113,10 +125,12 @@ public void onNext( } } else { + LOGGER.log(ERROR, "StreamMediator is not accepting BlockItems"); + // Close the upstream connection to the producer(s) final var errorResponse = buildErrorStreamResponse(); publishStreamResponseObserver.onNext(errorResponse); - LOGGER.log(DEBUG, "StreamMediator is not accepting BlockItems"); + LOGGER.log(ERROR, "Error PublishStreamResponse sent to upstream producer"); } } catch (IOException io) { final var errorResponse = buildErrorStreamResponse(); diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index 4a938f30..63352467 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -26,6 +26,7 @@ requires dagger; requires io.grpc.stub; requires io.helidon.common; + requires io.helidon.config; requires io.helidon.webserver.grpc; requires io.helidon.webserver; requires javax.inject; diff --git a/server/src/test/java/com/hedera/block/server/BlockStreamServiceIntegrationTest.java b/server/src/test/java/com/hedera/block/server/BlockStreamServiceIntegrationTest.java index 6f668682..ef4f985a 100644 --- a/server/src/test/java/com/hedera/block/server/BlockStreamServiceIntegrationTest.java +++ b/server/src/test/java/com/hedera/block/server/BlockStreamServiceIntegrationTest.java @@ -17,6 +17,7 @@ package com.hedera.block.server; import static com.hedera.block.server.Translator.fromPbj; +import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.LiveBlockItems; import static com.hedera.block.server.producer.Util.getFakeHash; import static com.hedera.block.server.util.PersistTestUtils.generateBlockItems; import static java.lang.System.Logger; @@ -230,7 +231,7 @@ public void testSubscribeBlockStream() throws IOException { streamObserver.onNext(fromPbj(publishStreamRequest)); // Verify the counter was incremented - assertEquals(1, blockNodeContext.metricsService().liveBlockItems().get()); + assertEquals(1, blockNodeContext.metricsService().get(LiveBlockItems).get()); verify(blockWriter, timeout(testTimeout).times(1)).write(blockItems.getFirst()); diff --git a/server/src/test/java/com/hedera/block/server/mediator/LiveStreamMediatorImplTest.java b/server/src/test/java/com/hedera/block/server/mediator/LiveStreamMediatorImplTest.java index 06b63ea7..63e1813f 100644 --- a/server/src/test/java/com/hedera/block/server/mediator/LiveStreamMediatorImplTest.java +++ b/server/src/test/java/com/hedera/block/server/mediator/LiveStreamMediatorImplTest.java @@ -17,6 +17,7 @@ package com.hedera.block.server.mediator; import static com.hedera.block.server.Translator.fromPbj; +import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.LiveBlockItems; import static com.hedera.block.server.util.PersistTestUtils.generateBlockItems; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -131,7 +132,7 @@ public void testUnsubscribeEach() throws InterruptedException, IOException { "Expected the mediator to have unsubscribed observer3"); // Confirm the counter was never incremented - assertEquals(0, blockNodeContext.metricsService().liveBlockItems().get()); + assertEquals(0, blockNodeContext.metricsService().get(LiveBlockItems).get()); } @Test @@ -148,7 +149,7 @@ blockWriter, blockNodeContext, new ServiceStatusImpl()) streamMediator.publish(blockItem); // Verify the counter was incremented - assertEquals(1, blockNodeContext.metricsService().liveBlockItems().get()); + assertEquals(1, blockNodeContext.metricsService().get(LiveBlockItems).get()); // Confirm the BlockStorage write method was // called despite the absence of subscribers @@ -201,7 +202,7 @@ blockWriter, blockNodeContext, new ServiceStatusImpl()) // Acting as a producer, notify the mediator of a new block streamMediator.publish(blockItem); - assertEquals(1, blockNodeContext.metricsService().liveBlockItems().get()); + assertEquals(1, blockNodeContext.metricsService().get(LiveBlockItems).get()); // Confirm each subscriber was notified of the new block verify(streamObserver1, timeout(testTimeout).times(1)) @@ -248,7 +249,7 @@ blockWriter, blockNodeContext, new ServiceStatusImpl()) streamMediator.unsubscribe(concreteObserver3); // Confirm the counter was never incremented - assertEquals(0, blockNodeContext.metricsService().liveBlockItems().get()); + assertEquals(0, blockNodeContext.metricsService().get(LiveBlockItems).get()); } @Test @@ -277,7 +278,7 @@ blockWriter, blockNodeContext, new ServiceStatusImpl()) testConsumerBlockItemObserver.getOnCancel().run(); // Verify the block item incremented the counter - assertEquals(1, blockNodeContext.metricsService().liveBlockItems().get()); + assertEquals(1, blockNodeContext.metricsService().get(LiveBlockItems).get()); // Verify the event made it to the consumer verify(serverCallStreamObserver, timeout(testTimeout).times(1)).setOnCancelHandler(any()); @@ -313,7 +314,7 @@ blockWriter, blockNodeContext, new ServiceStatusImpl()) testConsumerBlockItemObserver.getOnClose().run(); // Verify the block item incremented the counter - assertEquals(1, blockNodeContext.metricsService().liveBlockItems().get()); + assertEquals(1, blockNodeContext.metricsService().get(LiveBlockItems).get()); // Verify the event made it to the consumer verify(serverCallStreamObserver, timeout(testTimeout).times(1)).setOnCancelHandler(any()); @@ -348,7 +349,7 @@ blockWriter, blockNodeContext, new ServiceStatusImpl()) streamMediator.publish(secondBlockItem); // Confirm the counter was incremented only once - assertEquals(1, blockNodeContext.metricsService().liveBlockItems().get()); + assertEquals(1, blockNodeContext.metricsService().get(LiveBlockItems).get()); // Confirm the BlockPersistenceHandler write method was only called // once despite the second block being published. diff --git a/server/src/test/java/com/hedera/block/server/metrics/MetricsServiceTest.java b/server/src/test/java/com/hedera/block/server/metrics/MetricsServiceTest.java index d198d293..83e1dab4 100644 --- a/server/src/test/java/com/hedera/block/server/metrics/MetricsServiceTest.java +++ b/server/src/test/java/com/hedera/block/server/metrics/MetricsServiceTest.java @@ -16,69 +16,122 @@ package com.hedera.block.server.metrics; -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.Mockito.*; - -import com.swirlds.metrics.api.Counter; -import com.swirlds.metrics.api.LongGauge; -import com.swirlds.metrics.api.Metrics; +import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.BlocksPersisted; +import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.LiveBlockItems; +import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.LiveBlockItemsConsumed; +import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.SingleBlocksRetrieved; +import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Gauge.Subscribers; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.hedera.block.server.config.BlockNodeContext; +import com.hedera.block.server.util.TestConfigUtil; +import com.swirlds.config.api.Configuration; +import dagger.BindsInstance; +import dagger.Component; +import java.io.IOException; +import javax.inject.Singleton; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -class MetricsServiceTest { +public class MetricsServiceTest { - @Test - void MetricsService_initializesLiveBlockItemsCounter() { - Metrics metrics = mock(Metrics.class); - Counter liveBlockItems = mock(Counter.class); - when(metrics.getOrCreate(any(Counter.Config.class))).thenReturn(liveBlockItems); + @Singleton + @Component(modules = {MetricsInjectionModule.class}) + public interface MetricsServiceTestComponent { + + MetricsService getMetricsService(); - MetricsService service = new MetricsServiceImpl(metrics); + @Component.Factory + interface Factory { + MetricsServiceTestComponent create(@BindsInstance Configuration configuration); + } + } - assertEquals(liveBlockItems, service.liveBlockItems()); + private MetricsService metricsService; - service.liveBlockItems().increment(); - verify(liveBlockItems, times(1)).increment(); + @BeforeEach + public void setUp() throws IOException { + final BlockNodeContext context = TestConfigUtil.getTestBlockNodeContext(); + final Configuration configuration = context.configuration(); + final MetricsServiceTestComponent testComponent = + DaggerMetricsServiceTest_MetricsServiceTestComponent.factory() + .create(configuration); + this.metricsService = testComponent.getMetricsService(); } @Test - void MetricsService_initializesBlocksPersistedCounter() { - Metrics metrics = mock(Metrics.class); - Counter blocksPersisted = mock(Counter.class); - when(metrics.getOrCreate(any(Counter.Config.class))).thenReturn(blocksPersisted); - - MetricsService service = new MetricsServiceImpl(metrics); + void MetricsService_verifyLiveBlockItemsCounter() { - assertEquals(blocksPersisted, service.blocksPersisted()); + for (int i = 0; i < 10; i++) { + metricsService.get(LiveBlockItems).increment(); + } - service.blocksPersisted().increment(); - verify(blocksPersisted, times(1)).increment(); + assertEquals(LiveBlockItems.grafanaLabel(), metricsService.get(LiveBlockItems).getName()); + assertEquals( + LiveBlockItems.description(), metricsService.get(LiveBlockItems).getDescription()); + assertEquals(10, metricsService.get(LiveBlockItems).get()); } @Test - void MetricsService_initializesSingleBlocksRetrievedCounter() { - Metrics metrics = mock(Metrics.class); - Counter singleBlocksRetrieved = mock(Counter.class); - when(metrics.getOrCreate(any(Counter.Config.class))).thenReturn(singleBlocksRetrieved); + void MetricsService_verifyBlocksPersistedCounter() { - MetricsService service = new MetricsServiceImpl(metrics); + for (int i = 0; i < 10; i++) { + metricsService.get(BlocksPersisted).increment(); + } - assertEquals(singleBlocksRetrieved, service.singleBlocksRetrieved()); + assertEquals(BlocksPersisted.grafanaLabel(), metricsService.get(BlocksPersisted).getName()); + assertEquals( + BlocksPersisted.description(), + metricsService.get(BlocksPersisted).getDescription()); + assertEquals(10, metricsService.get(BlocksPersisted).get()); + } - service.singleBlocksRetrieved().increment(); - verify(singleBlocksRetrieved, times(1)).increment(); + @Test + void MetricsService_verifySingleBlocksRetrievedCounter() { + + for (int i = 0; i < 10; i++) { + metricsService.get(SingleBlocksRetrieved).increment(); + } + + assertEquals( + SingleBlocksRetrieved.grafanaLabel(), + metricsService.get(SingleBlocksRetrieved).getName()); + assertEquals( + SingleBlocksRetrieved.description(), + metricsService.get(SingleBlocksRetrieved).getDescription()); + assertEquals(10, metricsService.get(SingleBlocksRetrieved).get()); } @Test - void MetricsService_initializesSubscribersGauge() { - Metrics metrics = mock(Metrics.class); - LongGauge subscribers = mock(LongGauge.class); - when(metrics.getOrCreate(any(LongGauge.Config.class))).thenReturn(subscribers); + void MetricsService_verifyLiveBlockItemsConsumedCounter() { + + for (int i = 0; i < 10; i++) { + metricsService.get(LiveBlockItemsConsumed).increment(); + } + + assertEquals( + LiveBlockItemsConsumed.grafanaLabel(), + metricsService.get(LiveBlockItemsConsumed).getName()); + assertEquals( + LiveBlockItemsConsumed.description(), + metricsService.get(LiveBlockItemsConsumed).getDescription()); + assertEquals(10, metricsService.get(LiveBlockItemsConsumed).get()); + } + + @Test + void MetricsService_verifySubscribersGauge() { + + assertEquals(Subscribers.grafanaLabel(), metricsService.get(Subscribers).getName()); + assertEquals(Subscribers.description(), metricsService.get(Subscribers).getDescription()); - MetricsService service = new MetricsServiceImpl(metrics); + // Set the subscribers to various values and verify + metricsService.get(Subscribers).set(10); + assertEquals(10, metricsService.get(Subscribers).get()); - assertEquals(subscribers, service.subscribers()); + metricsService.get(Subscribers).set(3); + assertEquals(3, metricsService.get(Subscribers).get()); - service.subscribers().set(5); - verify(subscribers, times(1)).set(5); + metricsService.get(Subscribers).set(0); + assertEquals(0, metricsService.get(Subscribers).get()); } } diff --git a/server/src/test/java/com/hedera/block/server/producer/ProducerBlockItemObserverTest.java b/server/src/test/java/com/hedera/block/server/producer/ProducerBlockItemObserverTest.java index 9c6e7bca..242ef05e 100644 --- a/server/src/test/java/com/hedera/block/server/producer/ProducerBlockItemObserverTest.java +++ b/server/src/test/java/com/hedera/block/server/producer/ProducerBlockItemObserverTest.java @@ -17,6 +17,7 @@ package com.hedera.block.server.producer; import static com.hedera.block.server.Translator.fromPbj; +import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.LiveBlockItems; import static com.hedera.block.server.producer.Util.getFakeHash; import static com.hedera.block.server.util.PersistTestUtils.generateBlockItems; import static com.hedera.block.server.util.PersistTestUtils.reverseByteArray; @@ -88,10 +89,14 @@ public class ProducerBlockItemObserverTest { @Test public void testProducerOnNext() throws IOException, NoSuchAlgorithmException { + final BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); final List blockItems = generateBlockItems(1); final ProducerBlockItemObserver producerBlockItemObserver = new ProducerBlockItemObserver( - streamMediator, publishStreamResponseObserver, serviceStatus); + streamMediator, + publishStreamResponseObserver, + blockNodeContext, + serviceStatus); when(serviceStatus.isRunning()).thenReturn(true); @@ -172,14 +177,17 @@ blockWriter, blockNodeContext, new ServiceStatusImpl()) final ProducerBlockItemObserver producerBlockItemObserver = new ProducerBlockItemObserver( - streamMediator, publishStreamResponseObserver, serviceStatus); + streamMediator, + publishStreamResponseObserver, + blockNodeContext, + serviceStatus); final PublishStreamRequest publishStreamRequest = PublishStreamRequest.newBuilder().blockItem(blockItem).build(); producerBlockItemObserver.onNext(fromPbj(publishStreamRequest)); // Confirm the block item counter was incremented - assertEquals(1, blockNodeContext.metricsService().liveBlockItems().get()); + assertEquals(1, blockNodeContext.metricsService().get(LiveBlockItems).get()); // Confirm each subscriber was notified of the new block verify(streamObserver1, timeout(testTimeout).times(1)) @@ -195,10 +203,15 @@ blockWriter, blockNodeContext, new ServiceStatusImpl()) } @Test - public void testOnError() { + public void testOnError() throws IOException { + + final BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); final ProducerBlockItemObserver producerBlockItemObserver = new ProducerBlockItemObserver( - streamMediator, publishStreamResponseObserver, serviceStatus); + streamMediator, + publishStreamResponseObserver, + blockNodeContext, + serviceStatus); final Throwable t = new Throwable("Test error"); producerBlockItemObserver.onError(t); @@ -206,13 +219,17 @@ public void testOnError() { } @Test - public void testItemAckBuilderExceptionTest() { + public void testItemAckBuilderExceptionTest() throws IOException { when(serviceStatus.isRunning()).thenReturn(true); + final BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); final ProducerBlockItemObserver testProducerBlockItemObserver = new TestProducerBlockItemObserver( - streamMediator, publishStreamResponseObserver, serviceStatus); + streamMediator, + publishStreamResponseObserver, + blockNodeContext, + serviceStatus); final List blockItems = generateBlockItems(1); final BlockItem blockHeader = blockItems.getFirst(); @@ -231,10 +248,16 @@ public void testItemAckBuilderExceptionTest() { } @Test - public void testBlockItemThrowsParseException() throws InvalidProtocolBufferException { + public void testBlockItemThrowsParseException() + throws IOException, InvalidProtocolBufferException { + + final BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); final ProducerBlockItemObserver producerBlockItemObserver = new ProducerBlockItemObserver( - streamMediator, publishStreamResponseObserver, serviceStatus); + streamMediator, + publishStreamResponseObserver, + blockNodeContext, + serviceStatus); // Create a pbj block item final List blockItems = generateBlockItems(1); @@ -275,11 +298,13 @@ public void testBlockItemThrowsParseException() throws InvalidProtocolBufferExce private static class TestProducerBlockItemObserver extends ProducerBlockItemObserver { public TestProducerBlockItemObserver( - StreamMediator> streamMediator, - StreamObserver + final StreamMediator> + streamMediator, + final StreamObserver publishStreamResponseObserver, - ServiceStatus serviceStatus) { - super(streamMediator, publishStreamResponseObserver, serviceStatus); + final BlockNodeContext blockNodeContext, + final ServiceStatus serviceStatus) { + super(streamMediator, publishStreamResponseObserver, blockNodeContext, serviceStatus); } @NonNull diff --git a/settings.gradle.kts b/settings.gradle.kts index b195e4fa..4770f346 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -44,6 +44,7 @@ dependencyResolutionManagement { // Compile time dependencies version("io.helidon.webserver.http2", "4.1.0") version("io.helidon.webserver.grpc", "4.1.0") + version("io.helidon.logging", "4.1.0") version("com.lmax.disruptor", "4.0.0") version("com.github.spotbugs.annotations", "4.7.3") version("com.swirlds.metrics.api", swirldsVersion)