diff --git a/server/docker/metrics/dashboards/block-node-server.json b/server/docker/metrics/dashboards/block-node-server.json index e28ff6b8..831b956f 100644 --- a/server/docker/metrics/dashboards/block-node-server.json +++ b/server/docker/metrics/dashboards/block-node-server.json @@ -21,6 +21,7 @@ "links": [], "panels": [ { + "collapsed": false, "gridPos": { "h": 1, "w": 24, @@ -28,6 +29,7 @@ "y": 0 }, "id": 16, + "panels": [], "title": "Errors", "type": "row" }, @@ -60,7 +62,7 @@ "overrides": [] }, "gridPos": { - "h": 8, + "h": 6, "w": 6, "x": 0, "y": 1 @@ -156,12 +158,13 @@ "value": 1 } ] - } + }, + "unit": "reqps" }, "overrides": [] }, "gridPos": { - "h": 8, + "h": 6, "w": 18, "x": 6, "y": 1 @@ -196,13 +199,175 @@ "title": "Rate of Mediator Errors", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 7 + }, + "id": 28, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "hedera_block_node_stream_persistence_handler_error_total", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Stream Persistence Errors", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 18, + "x": 6, + "y": 7 + }, + "id": 29, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(hedera_block_node_stream_persistence_handler_error_total [$__rate_interval])", + "instant": false, + "legendFormat": "Stream Persistence Errors", + "range": true, + "refId": "A" + } + ], + "title": "Rate of Stream Persistence Errors", + "type": "timeseries" + }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 9 + "y": 14 }, "id": 9, "panels": [], @@ -234,10 +399,10 @@ "overrides": [] }, "gridPos": { - "h": 8, + "h": 5, "w": 6, "x": 0, - "y": 10 + "y": 15 }, "id": 13, "options": { @@ -341,10 +506,10 @@ "overrides": [] }, "gridPos": { - "h": 8, + "h": 5, "w": 18, "x": 6, - "y": 10 + "y": 15 }, "id": 12, "options": { @@ -401,10 +566,10 @@ "overrides": [] }, "gridPos": { - "h": 7, + "h": 5, "w": 6, "x": 0, - "y": 18 + "y": 20 }, "id": 3, "options": { @@ -507,10 +672,10 @@ "overrides": [] }, "gridPos": { - "h": 7, + "h": 5, "w": 18, "x": 6, - "y": 18 + "y": 20 }, "id": 7, "options": { @@ -568,7 +733,7 @@ "overrides": [] }, "gridPos": { - "h": 7, + "h": 5, "w": 6, "x": 0, "y": 25 @@ -674,7 +839,7 @@ "overrides": [] }, "gridPos": { - "h": 7, + "h": 5, "w": 18, "x": 6, "y": 25 @@ -719,7 +884,80 @@ "color": { "mode": "thresholds" }, - "displayName": "Subscribers", + "displayName": "Producers", + "mappings": [], + "max": 20, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 5 + }, + { + "color": "red", + "value": 10 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 30 + }, + "id": 25, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "hedera_block_node_producers", + "instant": false, + "legendFormat": "Producers", + "range": true, + "refId": "A" + } + ], + "title": "Producers", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "displayName": "Consumers", "mappings": [], "max": 20, "min": 0, @@ -744,10 +982,10 @@ "overrides": [] }, "gridPos": { - "h": 8, + "h": 7, "w": 6, - "x": 0, - "y": 32 + "x": 6, + "y": 30 }, "id": 4, "options": { @@ -773,14 +1011,14 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "hedera_block_node_subscribers", + "expr": "hedera_block_node_consumers", "instant": false, - "legendFormat": "__auto", + "legendFormat": "Consumers", "range": true, "refId": "A" } ], - "title": "Subscribers", + "title": "Consumers", "type": "gauge" }, { @@ -789,7 +1027,7 @@ "h": 1, "w": 24, "x": 0, - "y": 40 + "y": 37 }, "id": 18, "panels": [], @@ -821,10 +1059,10 @@ "overrides": [] }, "gridPos": { - "h": 7, + "h": 6, "w": 6, "x": 0, - "y": 41 + "y": 38 }, "id": 6, "options": { @@ -927,10 +1165,10 @@ "overrides": [] }, "gridPos": { - "h": 7, + "h": 6, "w": 18, "x": 6, - "y": 41 + "y": 38 }, "id": 8, "options": { @@ -969,11 +1207,11 @@ "h": 1, "w": 24, "x": 0, - "y": 48 + "y": 44 }, - "id": 17, + "id": 23, "panels": [], - "title": "Single Block", + "title": "Live Stream Responses", "type": "row" }, { @@ -993,10 +1231,6 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] }, @@ -1005,12 +1239,12 @@ "overrides": [] }, "gridPos": { - "h": 8, + "h": 6, "w": 6, "x": 0, - "y": 49 + "y": 45 }, - "id": 19, + "id": 24, "options": { "colorMode": "value", "graphMode": "none", @@ -1036,14 +1270,14 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "hedera_block_node_single_blocks_retrieved_total", + "expr": "hedera_block_node_successful_pub_stream_resp_total", "instant": false, - "legendFormat": "Single Blocks Retrieved", + "legendFormat": "PublishStreamResponses", "range": true, "refId": "A" } ], - "title": "Single Blocks Retrieved", + "title": "PublishStreamResponses Generated", "type": "stat" }, { @@ -1107,12 +1341,12 @@ "overrides": [] }, "gridPos": { - "h": 8, + "h": 6, "w": 18, "x": 6, - "y": 49 + "y": 45 }, - "id": 5, + "id": 22, "options": { "legend": { "calcs": [], @@ -1132,14 +1366,14 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "rate(hedera_block_node_single_blocks_retrieved_total[$__rate_interval])", + "expr": "rate( hedera_block_node_successful_pub_stream_resp_total [$__rate_interval])", "instant": false, - "legendFormat": "RPS of Single Blocks Retrieval", + "legendFormat": "PublishStreamResponses", "range": true, "refId": "A" } ], - "title": "Rate of Single Blocks Retrieved", + "title": "Rate of PublishStreamResponses Generated", "type": "timeseries" }, { @@ -1159,22 +1393,357 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] - }, - "unit": "short" + } }, "overrides": [] }, "gridPos": { - "h": 8, + "h": 6, "w": 6, "x": 0, - "y": 57 + "y": 51 + }, + "id": 27, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "hedera_block_node_successful_pub_stream_resp_sent_total", + "instant": false, + "legendFormat": "PublishStreamResponses", + "range": true, + "refId": "A" + } + ], + "title": "PublishStreamResponses Sent", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 18, + "x": 6, + "y": 51 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(hedera_block_node_successful_pub_stream_resp_sent_total[$__rate_interval])", + "instant": false, + "legendFormat": "PublishStreamResponses", + "range": true, + "refId": "A" + } + ], + "title": "Rate of PublishStreamResponses Sent to Producers", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 57 + }, + "id": 17, + "panels": [], + "title": "Single Block", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 58 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "hedera_block_node_single_blocks_retrieved_total", + "instant": false, + "legendFormat": "Single Blocks Retrieved", + "range": true, + "refId": "A" + } + ], + "title": "Single Blocks Retrieved", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 18, + "x": 6, + "y": 58 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(hedera_block_node_single_blocks_retrieved_total[$__rate_interval])", + "instant": false, + "legendFormat": "RPS of Single Blocks Retrieval", + "range": true, + "refId": "A" + } + ], + "title": "Rate of Single Blocks Retrieved", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 64 }, "id": 21, "options": { @@ -1259,8 +1828,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1273,10 +1841,10 @@ "overrides": [] }, "gridPos": { - "h": 8, + "h": 6, "w": 18, "x": 6, - "y": 57 + "y": 64 }, "id": 20, "options": { @@ -1325,4 +1893,4 @@ "uid": "edu86nutnxts0c", "version": 1, "weekStart": "" -} +} \ No newline at end of file diff --git a/server/src/main/java/com/hedera/block/server/BlockNodeApp.java b/server/src/main/java/com/hedera/block/server/BlockNodeApp.java index 53b0d7e0..77d49a36 100644 --- a/server/src/main/java/com/hedera/block/server/BlockNodeApp.java +++ b/server/src/main/java/com/hedera/block/server/BlockNodeApp.java @@ -20,6 +20,7 @@ import static java.lang.System.Logger.Level.INFO; import com.hedera.block.server.health.HealthService; +import com.hedera.block.server.service.ServiceStatus; import edu.umd.cs.findbugs.annotations.NonNull; import io.helidon.webserver.WebServer; import io.helidon.webserver.WebServerConfig; diff --git a/server/src/main/java/com/hedera/block/server/BlockNodeAppInjectionComponent.java b/server/src/main/java/com/hedera/block/server/BlockNodeAppInjectionComponent.java index 114bedc0..8d469746 100644 --- a/server/src/main/java/com/hedera/block/server/BlockNodeAppInjectionComponent.java +++ b/server/src/main/java/com/hedera/block/server/BlockNodeAppInjectionComponent.java @@ -20,7 +20,9 @@ import com.hedera.block.server.health.HealthInjectionModule; import com.hedera.block.server.mediator.MediatorInjectionModule; import com.hedera.block.server.metrics.MetricsInjectionModule; +import com.hedera.block.server.notifier.NotifierInjectionModule; import com.hedera.block.server.persistence.PersistenceInjectionModule; +import com.hedera.block.server.service.ServiceInjectionModule; import com.swirlds.config.api.Configuration; import dagger.BindsInstance; import dagger.Component; @@ -30,6 +32,8 @@ @Singleton @Component( modules = { + NotifierInjectionModule.class, + ServiceInjectionModule.class, BlockNodeAppInjectionModule.class, HealthInjectionModule.class, PersistenceInjectionModule.class, diff --git a/server/src/main/java/com/hedera/block/server/BlockNodeAppInjectionModule.java b/server/src/main/java/com/hedera/block/server/BlockNodeAppInjectionModule.java index 023293f7..f23cab5a 100644 --- a/server/src/main/java/com/hedera/block/server/BlockNodeAppInjectionModule.java +++ b/server/src/main/java/com/hedera/block/server/BlockNodeAppInjectionModule.java @@ -33,16 +33,6 @@ @Module public interface BlockNodeAppInjectionModule { - /** - * Binds the service status to the service status implementation. - * - * @param serviceStatus needs a service status implementation - * @return the service status implementation - */ - @Singleton - @Binds - ServiceStatus bindServiceStatus(ServiceStatusImpl serviceStatus); - /** * Provides a block node context singleton. * diff --git a/server/src/main/java/com/hedera/block/server/BlockStreamService.java b/server/src/main/java/com/hedera/block/server/BlockStreamService.java index 009bfc60..4639157b 100644 --- a/server/src/main/java/com/hedera/block/server/BlockStreamService.java +++ b/server/src/main/java/com/hedera/block/server/BlockStreamService.java @@ -32,11 +32,14 @@ import com.google.protobuf.InvalidProtocolBufferException; import com.hedera.block.server.config.BlockNodeContext; import com.hedera.block.server.consumer.ConsumerStreamResponseObserver; -import com.hedera.block.server.data.ObjectEvent; -import com.hedera.block.server.mediator.StreamMediator; +import com.hedera.block.server.events.BlockNodeEventHandler; +import com.hedera.block.server.events.ObjectEvent; +import com.hedera.block.server.mediator.LiveStreamMediator; import com.hedera.block.server.metrics.MetricsService; +import com.hedera.block.server.notifier.Notifier; import com.hedera.block.server.persistence.storage.read.BlockReader; import com.hedera.block.server.producer.ProducerBlockItemObserver; +import com.hedera.block.server.service.ServiceStatus; import com.hedera.hapi.block.SingleBlockRequest; import com.hedera.hapi.block.SingleBlockResponse; import com.hedera.hapi.block.SingleBlockResponseCode; @@ -44,7 +47,6 @@ import com.hedera.hapi.block.SubscribeStreamResponseCode; import com.hedera.hapi.block.protoc.BlockService; import com.hedera.hapi.block.stream.Block; -import com.hedera.hapi.block.stream.BlockItem; import com.hedera.pbj.runtime.ParseException; import edu.umd.cs.findbugs.annotations.NonNull; import io.grpc.stub.StreamObserver; @@ -63,12 +65,15 @@ public class BlockStreamService implements GrpcService { private final Logger LOGGER = System.getLogger(getClass().getName()); - private final StreamMediator> streamMediator; + private final LiveStreamMediator streamMediator; private final ServiceStatus serviceStatus; private final BlockReader blockReader; + private final BlockNodeContext blockNodeContext; private final MetricsService metricsService; + private final Notifier notifier; + /** * Constructor for the BlockStreamService class. It initializes the BlockStreamService with the * given parameters. @@ -82,17 +87,22 @@ public class BlockStreamService implements GrpcService { */ @Inject BlockStreamService( - @NonNull - final StreamMediator> - streamMediator, + @NonNull final LiveStreamMediator streamMediator, @NonNull final BlockReader blockReader, @NonNull final ServiceStatus serviceStatus, + @NonNull + final BlockNodeEventHandler> + streamPersistenceHandler, + @NonNull final Notifier notifier, @NonNull final BlockNodeContext blockNodeContext) { - this.streamMediator = streamMediator; this.blockReader = blockReader; this.serviceStatus = serviceStatus; + this.notifier = notifier; this.blockNodeContext = blockNodeContext; this.metricsService = blockNodeContext.metricsService(); + + streamMediator.subscribe(streamPersistenceHandler); + this.streamMediator = streamMediator; } /** @@ -139,8 +149,23 @@ StreamObserver protocPublishB publishStreamResponseObserver) { LOGGER.log(DEBUG, "Executing bidirectional publishBlockStream gRPC method"); - return new ProducerBlockItemObserver( - streamMediator, publishStreamResponseObserver, blockNodeContext, serviceStatus); + // Unsubscribe any expired notifiers + notifier.unsubscribeAllExpired(); + + final var producerBlockItemObserver = + new ProducerBlockItemObserver( + Clock.systemDefaultZone(), + streamMediator, + notifier, + publishStreamResponseObserver, + blockNodeContext, + serviceStatus); + + // Register the producer observer with the notifier to publish responses back to the + // producer + notifier.subscribe(producerBlockItemObserver); + + return producerBlockItemObserver; } void protocSubscribeBlockStream( @@ -152,16 +177,18 @@ void protocSubscribeBlockStream( subscribeStreamResponseObserver) { LOGGER.log(DEBUG, "Executing Server Streaming subscribeBlockStream gRPC method"); - // Return a custom StreamObserver to handle streaming blocks from the producer. if (serviceStatus.isRunning()) { - final var streamObserver = + // Unsubscribe any expired notifiers + streamMediator.unsubscribeAllExpired(); + + final var consumerStreamResponseObserver = new ConsumerStreamResponseObserver( - blockNodeContext, Clock.systemDefaultZone(), streamMediator, - subscribeStreamResponseObserver); + subscribeStreamResponseObserver, + blockNodeContext); - streamMediator.subscribe(streamObserver); + streamMediator.subscribe(consumerStreamResponseObserver); } else { LOGGER.log( ERROR, diff --git a/server/src/main/java/com/hedera/block/server/Server.java b/server/src/main/java/com/hedera/block/server/Server.java index 1365bac7..c5ca20ba 100644 --- a/server/src/main/java/com/hedera/block/server/Server.java +++ b/server/src/main/java/com/hedera/block/server/Server.java @@ -57,7 +57,7 @@ public static void main(final String[] args) throws IOException { Config.global(config); // Init BlockNode Configuration - Configuration configuration = + final Configuration configuration = ConfigurationBuilder.create() .withSource(SystemEnvironmentConfigSource.getInstance()) .withSource(SystemPropertiesConfigSource.getInstance()) diff --git a/server/src/main/java/com/hedera/block/server/config/BlockNodeConfigExtension.java b/server/src/main/java/com/hedera/block/server/config/BlockNodeConfigExtension.java index 62065e39..870dea54 100644 --- a/server/src/main/java/com/hedera/block/server/config/BlockNodeConfigExtension.java +++ b/server/src/main/java/com/hedera/block/server/config/BlockNodeConfigExtension.java @@ -18,7 +18,10 @@ import com.google.auto.service.AutoService; import com.hedera.block.server.consumer.ConsumerConfig; +import com.hedera.block.server.mediator.MediatorConfig; +import com.hedera.block.server.notifier.NotifierConfig; import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; +import com.hedera.block.server.service.ServiceConfig; import com.swirlds.common.metrics.config.MetricsConfig; import com.swirlds.common.metrics.platform.prometheus.PrometheusConfig; import com.swirlds.config.api.ConfigurationExtension; @@ -43,6 +46,9 @@ public BlockNodeConfigExtension() { @Override public Set> getConfigDataTypes() { return Set.of( + ServiceConfig.class, + MediatorConfig.class, + NotifierConfig.class, MetricsConfig.class, PrometheusConfig.class, ConsumerConfig.class, diff --git a/server/src/main/java/com/hedera/block/server/config/ConfigInjectionModule.java b/server/src/main/java/com/hedera/block/server/config/ConfigInjectionModule.java index 94365187..2de4a652 100644 --- a/server/src/main/java/com/hedera/block/server/config/ConfigInjectionModule.java +++ b/server/src/main/java/com/hedera/block/server/config/ConfigInjectionModule.java @@ -17,6 +17,8 @@ package com.hedera.block.server.config; import com.hedera.block.server.consumer.ConsumerConfig; +import com.hedera.block.server.mediator.MediatorConfig; +import com.hedera.block.server.notifier.NotifierConfig; import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; import com.swirlds.common.metrics.config.MetricsConfig; import com.swirlds.common.metrics.platform.prometheus.PrometheusConfig; @@ -79,4 +81,28 @@ static PrometheusConfig providePrometheusConfig(Configuration configuration) { static ConsumerConfig provideConsumerConfig(Configuration configuration) { return configuration.getConfigData(ConsumerConfig.class); } + + /** + * Provides a mediator configuration singleton using the configuration. + * + * @param configuration is the configuration singleton + * @return a mediator configuration singleton + */ + @Singleton + @Provides + static MediatorConfig provideMediatorConfig(Configuration configuration) { + return configuration.getConfigData(MediatorConfig.class); + } + + /** + * Provides a notifier configuration singleton using the configuration. + * + * @param configuration is the configuration singleton + * @return a notifier configuration singleton + */ + @Singleton + @Provides + static NotifierConfig provideNotifierConfig(Configuration configuration) { + return configuration.getConfigData(NotifierConfig.class); + } } diff --git a/server/src/main/java/com/hedera/block/server/consumer/ConsumerConfig.java b/server/src/main/java/com/hedera/block/server/consumer/ConsumerConfig.java index fc4d0124..12baba62 100644 --- a/server/src/main/java/com/hedera/block/server/consumer/ConsumerConfig.java +++ b/server/src/main/java/com/hedera/block/server/consumer/ConsumerConfig.java @@ -26,4 +26,21 @@ * timed out and will be disconnected */ @ConfigData("consumer") -public record ConsumerConfig(@ConfigProperty(defaultValue = "1500") long timeoutThresholdMillis) {} +public record ConsumerConfig(@ConfigProperty(defaultValue = "1500") long timeoutThresholdMillis) { + private static final System.Logger LOGGER = System.getLogger(ConsumerConfig.class.getName()); + + /** + * Validate the configuration. + * + * @throws IllegalArgumentException if the configuration is invalid + */ + public ConsumerConfig { + if (timeoutThresholdMillis <= 0) { + throw new IllegalArgumentException("Timeout threshold must be greater than 0"); + } + + LOGGER.log( + System.Logger.Level.INFO, + "Consumer configuration timeoutThresholdMillis: " + timeoutThresholdMillis); + } +} diff --git a/server/src/main/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserver.java b/server/src/main/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserver.java index 25a028c8..07f9822c 100644 --- a/server/src/main/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserver.java +++ b/server/src/main/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserver.java @@ -23,13 +23,14 @@ import static java.lang.System.Logger.Level.ERROR; import com.hedera.block.server.config.BlockNodeContext; -import com.hedera.block.server.data.ObjectEvent; +import com.hedera.block.server.events.BlockNodeEventHandler; +import com.hedera.block.server.events.LivenessCalculator; +import com.hedera.block.server.events.ObjectEvent; import com.hedera.block.server.mediator.SubscriptionHandler; import com.hedera.block.server.metrics.MetricsService; import com.hedera.hapi.block.SubscribeStreamResponse; import com.hedera.hapi.block.stream.BlockItem; import com.hedera.pbj.runtime.OneOf; -import com.lmax.disruptor.EventHandler; import edu.umd.cs.findbugs.annotations.NonNull; import io.grpc.stub.ServerCallStreamObserver; import io.grpc.stub.StreamObserver; @@ -39,22 +40,18 @@ /** * The ConsumerBlockItemObserver class is the primary integration point between the LMAX Disruptor * and an instance of a downstream consumer (represented by subscribeStreamResponseObserver provided - * by Helidon). The ConsumerBlockItemObserver implements the EventHandler interface so the Disruptor - * can invoke the onEvent() method when a new SubscribeStreamResponse is available. + * by Helidon). The ConsumerBlockItemObserver implements the BlockNodeEventHandler interface so the + * Disruptor can invoke the onEvent() method when a new SubscribeStreamResponse is available. */ public class ConsumerStreamResponseObserver - implements EventHandler> { + implements BlockNodeEventHandler> { private final Logger LOGGER = System.getLogger(getClass().getName()); private final MetricsService metricsService; private final StreamObserver subscribeStreamResponseObserver; - private final SubscriptionHandler> subscriptionHandler; - - private final long timeoutThresholdMillis; - private final InstantSource producerLivenessClock; - private long producerLivenessMillis; + private final SubscriptionHandler subscriptionHandler; private final AtomicBoolean isResponsePermitted = new AtomicBoolean(true); private final ResponseSender statusResponseSender = new StatusResponseSender(); @@ -63,6 +60,8 @@ public class ConsumerStreamResponseObserver private static final String PROTOCOL_VIOLATION_MESSAGE = "Protocol Violation. %s is OneOf type %s but %s is null.\n%s"; + private final LivenessCalculator livenessCalculator; + /** * The onCancel handler to execute when the consumer cancels the stream. This handler is * protected to facilitate testing. @@ -80,28 +79,31 @@ public class ConsumerStreamResponseObserver * SubscribeStreamResponse events from the Disruptor and passing them to the downstream consumer * via the subscribeStreamResponseObserver. * - * @param context contains the context with metrics and configuration for the application * @param producerLivenessClock the clock to use to determine the producer liveness * @param subscriptionHandler the subscription handler to use to manage the subscription * lifecycle * @param subscribeStreamResponseObserver the observer to use to send responses to the consumer + * @param blockNodeContext contains the context with metrics and configuration for the + * application */ public ConsumerStreamResponseObserver( - @NonNull final BlockNodeContext context, @NonNull final InstantSource producerLivenessClock, - @NonNull - final SubscriptionHandler> - subscriptionHandler, + @NonNull final SubscriptionHandler subscriptionHandler, @NonNull final StreamObserver - subscribeStreamResponseObserver) { + subscribeStreamResponseObserver, + @NonNull final BlockNodeContext blockNodeContext) { + + this.livenessCalculator = + new LivenessCalculator( + producerLivenessClock, + blockNodeContext + .configuration() + .getConfigData(ConsumerConfig.class) + .timeoutThresholdMillis()); - this.timeoutThresholdMillis = - context.configuration() - .getConfigData(ConsumerConfig.class) - .timeoutThresholdMillis(); this.subscriptionHandler = subscriptionHandler; - this.metricsService = context.metricsService(); + this.metricsService = blockNodeContext.metricsService(); // The ServerCallStreamObserver can be configured with Runnable handlers to // be executed when a downstream consumer closes the connection. The handlers @@ -117,7 +119,7 @@ public ConsumerStreamResponseObserver( // Do not allow additional responses to be sent. isResponsePermitted.set(false); subscriptionHandler.unsubscribe(this); - LOGGER.log(DEBUG, "Consumer cancelled stream. Observer unsubscribed."); + LOGGER.log(DEBUG, "Consumer cancelled the stream. Observer unsubscribed."); }; serverCallStreamObserver.setOnCancelHandler(onCancel); @@ -127,14 +129,12 @@ public ConsumerStreamResponseObserver( // Do not allow additional responses to be sent. isResponsePermitted.set(false); subscriptionHandler.unsubscribe(this); - LOGGER.log(DEBUG, "Consumer completed stream. Observer unsubscribed."); + LOGGER.log(DEBUG, "Consumer completed stream. Observer unsubscribed."); }; serverCallStreamObserver.setOnCloseHandler(onClose); } this.subscribeStreamResponseObserver = subscribeStreamResponseObserver; - this.producerLivenessClock = producerLivenessClock; - this.producerLivenessMillis = producerLivenessClock.millis(); } /** @@ -157,15 +157,14 @@ public void onEvent( // Only send the response if the consumer has not cancelled // or closed the stream. if (isResponsePermitted.get()) { - final long currentMillis = producerLivenessClock.millis(); - if (currentMillis - producerLivenessMillis > timeoutThresholdMillis) { + if (isTimeoutExpired()) { subscriptionHandler.unsubscribe(this); LOGGER.log( DEBUG, "Producer liveness timeout. Unsubscribed ConsumerBlockItemObserver."); } else { // Refresh the producer liveness and pass the BlockItem to the downstream observer. - producerLivenessMillis = currentMillis; + livenessCalculator.refresh(); final SubscribeStreamResponse subscribeStreamResponse = event.get(); final ResponseSender responseSender = getResponseSender(subscribeStreamResponse); @@ -174,6 +173,11 @@ public void onEvent( } } + @Override + public boolean isTimeoutExpired() { + return livenessCalculator.isTimeoutExpired(); + } + @NonNull private ResponseSender getResponseSender( @NonNull final SubscribeStreamResponse subscribeStreamResponse) { @@ -215,7 +219,7 @@ public void send(@NonNull final SubscribeStreamResponse subscribeStreamResponse) } if (streamStarted) { - LOGGER.log(DEBUG, "Sending BlockItem downstream: {0}", blockItem); + LOGGER.log(DEBUG, "Sending BlockItem downstream: " + blockItem); // Increment counter metricsService.get(LiveBlockItemsConsumed).increment(); @@ -225,12 +229,13 @@ public void send(@NonNull final SubscribeStreamResponse subscribeStreamResponse) } } + // TODO: Implement another StatusResponseSender that will unsubscribe the observer once the + // status code is fixed. private final class StatusResponseSender implements ResponseSender { public void send(@NonNull final SubscribeStreamResponse subscribeStreamResponse) { LOGGER.log( DEBUG, - "Sending SubscribeStreamResponse downstream: {0} ", - subscribeStreamResponse); + "Sending SubscribeStreamResponse downstream: " + subscribeStreamResponse); subscribeStreamResponseObserver.onNext(fromPbj(subscribeStreamResponse)); } } diff --git a/server/src/main/java/com/hedera/block/server/events/BlockNodeEventHandler.java b/server/src/main/java/com/hedera/block/server/events/BlockNodeEventHandler.java new file mode 100644 index 00000000..8c21af2a --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/events/BlockNodeEventHandler.java @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.events; + +import com.lmax.disruptor.EventHandler; + +/** + * Use this interface to combine the contract for handling block node events + * + * @param the type of the event value + */ +public interface BlockNodeEventHandler extends EventHandler { + + /** + * Use this method to check if the underlying event handler is timed out. + * + * @return true if the timeout has expired, false otherwise + */ + default boolean isTimeoutExpired() { + return false; + } +} diff --git a/server/src/main/java/com/hedera/block/server/events/LivenessCalculator.java b/server/src/main/java/com/hedera/block/server/events/LivenessCalculator.java new file mode 100644 index 00000000..c62187be --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/events/LivenessCalculator.java @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.events; + +import edu.umd.cs.findbugs.annotations.NonNull; +import java.time.InstantSource; + +/** + * Use this class to calculate and refresh the current liveness of a component based on a timeout + * threshold. + */ +public final class LivenessCalculator { + private final long timeoutThresholdMillis; + private final InstantSource clock; + private long lastMillis; + + /** + * Once constructed, the liveness calculator will use the given clock and timeout threshold to + * determine expiry. + * + * @param clock the clock to use for time calculations + * @param timeoutThresholdMillis the timeout threshold in milliseconds + */ + public LivenessCalculator( + @NonNull final InstantSource clock, final long timeoutThresholdMillis) { + + this.clock = clock; + this.timeoutThresholdMillis = timeoutThresholdMillis; + this.lastMillis = clock.millis(); + } + + /** + * Returns true if the timeout has expired based on the configuration window and the current + * time. + * + * @return true if the timeout has expired + */ + public boolean isTimeoutExpired() { + return clock.millis() - lastMillis > timeoutThresholdMillis; + } + + /** + * Use refresh to reset the liveness calculator to the beginning of the configured threshold + * window. + */ + public void refresh() { + lastMillis = clock.millis(); + } +} diff --git a/server/src/main/java/com/hedera/block/server/data/ObjectEvent.java b/server/src/main/java/com/hedera/block/server/events/ObjectEvent.java similarity index 97% rename from server/src/main/java/com/hedera/block/server/data/ObjectEvent.java rename to server/src/main/java/com/hedera/block/server/events/ObjectEvent.java index d9256d92..40c59c6a 100644 --- a/server/src/main/java/com/hedera/block/server/data/ObjectEvent.java +++ b/server/src/main/java/com/hedera/block/server/events/ObjectEvent.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.hedera.block.server.data; +package com.hedera.block.server.events; import edu.umd.cs.findbugs.annotations.NonNull; diff --git a/server/src/main/java/com/hedera/block/server/exception/BlockStreamProtocolException.java b/server/src/main/java/com/hedera/block/server/exception/BlockStreamProtocolException.java new file mode 100644 index 00000000..e54140f1 --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/exception/BlockStreamProtocolException.java @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.exception; + +/** + * Use this checked exception to represent a Block Node protocol exception encountered while + * processing block items. + */ +public class BlockStreamProtocolException extends Exception { + + /** + * Constructs a new exception with the specified detail message. + * + * @param message the detail message + */ + public BlockStreamProtocolException(String message) { + super(message); + } +} diff --git a/server/src/main/java/com/hedera/block/server/health/HealthServiceImpl.java b/server/src/main/java/com/hedera/block/server/health/HealthServiceImpl.java index 8c094482..f790c449 100644 --- a/server/src/main/java/com/hedera/block/server/health/HealthServiceImpl.java +++ b/server/src/main/java/com/hedera/block/server/health/HealthServiceImpl.java @@ -16,7 +16,7 @@ package com.hedera.block.server.health; -import com.hedera.block.server.ServiceStatus; +import com.hedera.block.server.service.ServiceStatus; import edu.umd.cs.findbugs.annotations.NonNull; import io.helidon.webserver.http.HttpRules; import io.helidon.webserver.http.ServerRequest; diff --git a/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediator.java b/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediator.java new file mode 100644 index 00000000..716afab3 --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediator.java @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.mediator; + +import com.hedera.block.server.notifier.Notifiable; +import com.hedera.hapi.block.SubscribeStreamResponse; +import com.hedera.hapi.block.stream.BlockItem; + +/** + * Use this interface to combine the contract for mediating the live stream of blocks from the + * Hedera network with the contract to be notified of critical system events. + */ +public interface LiveStreamMediator + extends StreamMediator, Notifiable {} diff --git a/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediatorBuilder.java b/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediatorBuilder.java index 68ae63cc..2816f1a7 100644 --- a/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediatorBuilder.java +++ b/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediatorBuilder.java @@ -16,14 +16,13 @@ package com.hedera.block.server.mediator; -import com.hedera.block.server.ServiceStatus; import com.hedera.block.server.config.BlockNodeContext; -import com.hedera.block.server.data.ObjectEvent; +import com.hedera.block.server.events.BlockNodeEventHandler; +import com.hedera.block.server.events.ObjectEvent; import com.hedera.block.server.persistence.storage.write.BlockWriter; +import com.hedera.block.server.service.ServiceStatus; import com.hedera.hapi.block.SubscribeStreamResponse; -import com.hedera.hapi.block.stream.BlockItem; import com.lmax.disruptor.BatchEventProcessor; -import com.lmax.disruptor.EventHandler; import edu.umd.cs.findbugs.annotations.NonNull; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -38,12 +37,11 @@ */ public class LiveStreamMediatorBuilder { - private final BlockWriter blockWriter; private final BlockNodeContext blockNodeContext; private final ServiceStatus serviceStatus; private Map< - EventHandler>, + BlockNodeEventHandler>, BatchEventProcessor>> subscribers; @@ -51,11 +49,9 @@ public class LiveStreamMediatorBuilder { private static final int SUBSCRIBER_INIT_CAPACITY = 32; private LiveStreamMediatorBuilder( - @NonNull final BlockWriter blockWriter, @NonNull final BlockNodeContext blockNodeContext, @NonNull final ServiceStatus serviceStatus) { this.subscribers = new ConcurrentHashMap<>(SUBSCRIBER_INIT_CAPACITY); - this.blockWriter = blockWriter; this.blockNodeContext = blockNodeContext; this.serviceStatus = serviceStatus; } @@ -63,7 +59,6 @@ private LiveStreamMediatorBuilder( /** * Create a new instance of the builder using the minimum required parameters. * - * @param blockWriter is required for the stream mediator to persist block items to storage. * @param blockNodeContext is required to provide metrics reporting mechanisms to the stream * mediator. * @param serviceStatus is required to provide the stream mediator with access to check the @@ -72,10 +67,9 @@ private LiveStreamMediatorBuilder( */ @NonNull public static LiveStreamMediatorBuilder newBuilder( - @NonNull final BlockWriter blockWriter, @NonNull final BlockNodeContext blockNodeContext, @NonNull final ServiceStatus serviceStatus) { - return new LiveStreamMediatorBuilder(blockWriter, blockNodeContext, serviceStatus); + return new LiveStreamMediatorBuilder(blockNodeContext, serviceStatus); } /** @@ -91,7 +85,7 @@ public static LiveStreamMediatorBuilder newBuilder( public LiveStreamMediatorBuilder subscribers( @NonNull final Map< - EventHandler>, + BlockNodeEventHandler>, BatchEventProcessor>> subscribers) { this.subscribers = subscribers; @@ -105,8 +99,7 @@ public LiveStreamMediatorBuilder subscribers( * @return the stream mediator to handle live stream events between a producer and N consumers. */ @NonNull - public StreamMediator> build() { - return new LiveStreamMediatorImpl( - subscribers, blockWriter, serviceStatus, blockNodeContext); + public LiveStreamMediator build() { + return new LiveStreamMediatorImpl(subscribers, serviceStatus, blockNodeContext); } } diff --git a/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediatorImpl.java b/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediatorImpl.java index 7691b62b..0ba032c9 100644 --- a/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediatorImpl.java +++ b/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediatorImpl.java @@ -18,85 +18,69 @@ import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.LiveBlockItems; import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.LiveBlockStreamMediatorError; -import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Gauge.Subscribers; +import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Gauge.Consumers; import static java.lang.System.Logger; import static java.lang.System.Logger.Level.DEBUG; import static java.lang.System.Logger.Level.ERROR; -import com.hedera.block.server.ServiceStatus; import com.hedera.block.server.config.BlockNodeContext; -import com.hedera.block.server.data.ObjectEvent; +import com.hedera.block.server.events.BlockNodeEventHandler; +import com.hedera.block.server.events.ObjectEvent; import com.hedera.block.server.metrics.MetricsService; -import com.hedera.block.server.persistence.storage.write.BlockWriter; +import com.hedera.block.server.service.ServiceStatus; import com.hedera.hapi.block.SubscribeStreamResponse; import com.hedera.hapi.block.SubscribeStreamResponseCode; import com.hedera.hapi.block.stream.BlockItem; import com.lmax.disruptor.BatchEventProcessor; -import com.lmax.disruptor.BatchEventProcessorBuilder; -import com.lmax.disruptor.EventHandler; -import com.lmax.disruptor.RingBuffer; -import com.lmax.disruptor.dsl.Disruptor; -import com.lmax.disruptor.util.DaemonThreadFactory; import edu.umd.cs.findbugs.annotations.NonNull; -import java.io.IOException; import java.util.Map; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; /** - * LiveStreamMediatorImpl is an implementation of the StreamMediator interface. It is responsible - * for managing the subscribe and unsubscribe operations of downstream consumers. It also proxies - * block items to the subscribers as they arrive via a RingBuffer and persists the block items to a - * store. + * Use LiveStreamMediatorImpl to mediate the live stream of blocks from a producer to multiple + * consumers. + * + *

As an implementation of the StreamMediator interface, it proxies block items to the + * subscribers as they arrive via a RingBuffer maintained in the base class and persists the block + * items to a store. */ -class LiveStreamMediatorImpl - implements StreamMediator> { +class LiveStreamMediatorImpl extends SubscriptionHandlerBase + implements LiveStreamMediator { private final Logger LOGGER = System.getLogger(getClass().getName()); - private final RingBuffer> ringBuffer; - private final ExecutorService executor; - - private final Map< - EventHandler>, - BatchEventProcessor>> - subscribers; - - private final BlockWriter blockWriter; private final ServiceStatus serviceStatus; private final MetricsService metricsService; /** - * Constructs a new LiveStreamMediatorImpl instance with the given subscribers, block writer, - * and service status. This constructor is primarily used for testing purposes. Users of this - * constructor should take care to supply a thread-safe map implementation for the subscribers - * to handle the dynamic addition and removal of subscribers at runtime. + * Constructs a new LiveStreamMediatorImpl instance with the given subscribers, and service + * status. This constructor is primarily used for testing purposes. Users of this constructor + * should take care to supply a thread-safe map implementation for the subscribers to handle the + * dynamic addition and removal of subscribers at runtime. * * @param subscribers the map of subscribers to batch event processors. It's recommended the map * implementation is thread-safe - * @param blockWriter the block writer to persist block items * @param serviceStatus the service status to stop the service and web server if an exception * occurs while persisting a block item, stop the web server for maintenance, etc + * @param blockNodeContext contains the context with metrics and configuration for the + * application */ LiveStreamMediatorImpl( @NonNull final Map< - EventHandler>, + BlockNodeEventHandler>, BatchEventProcessor>> subscribers, - @NonNull final BlockWriter blockWriter, @NonNull final ServiceStatus serviceStatus, @NonNull final BlockNodeContext blockNodeContext) { - this.subscribers = subscribers; - this.blockWriter = blockWriter; + super( + subscribers, + blockNodeContext.metricsService().get(Consumers), + blockNodeContext + .configuration() + .getConfigData(MediatorConfig.class) + .ringBufferSize()); - // Initialize and start the disruptor - final Disruptor> disruptor = - // TODO: replace ring buffer size with a configurable value, create a MediatorConfig - new Disruptor<>(ObjectEvent::new, 2048, DaemonThreadFactory.INSTANCE); - this.ringBuffer = disruptor.start(); - this.executor = Executors.newCachedThreadPool(DaemonThreadFactory.INSTANCE); this.serviceStatus = serviceStatus; this.metricsService = blockNodeContext.metricsService(); } @@ -107,15 +91,14 @@ class LiveStreamMediatorImpl * unsubscribed. * * @param blockItem the block item from the upstream producer to publish to downstream consumers - * @throws IOException is thrown if an exception occurs while persisting the block item */ @Override - public void publish(@NonNull final BlockItem blockItem) throws IOException { + public void publish(@NonNull final BlockItem blockItem) { if (serviceStatus.isRunning()) { // Publish the block for all subscribers to receive - LOGGER.log(DEBUG, "Publishing BlockItem: {0}", blockItem); + LOGGER.log(DEBUG, "Publishing BlockItem: " + blockItem); final var subscribeStreamResponse = SubscribeStreamResponse.newBuilder().blockItem(blockItem).build(); ringBuffer.publishEvent((event, sequence) -> event.set(subscribeStreamResponse)); @@ -123,86 +106,26 @@ public void publish(@NonNull final BlockItem blockItem) throws IOException { // Increment the block item counter metricsService.get(LiveBlockItems).increment(); - try { - // Persist the BlockItem - blockWriter.write(blockItem); - } catch (IOException e) { - - // Increment the error counter - metricsService.get(LiveBlockStreamMediatorError).increment(); - - // Disable BlockItem publication for upstream producers - serviceStatus.stopRunning(this.getClass().getName()); - LOGGER.log( - ERROR, - "An exception occurred while attempting to persist the BlockItem: " - + blockItem, - e); - - LOGGER.log(ERROR, "Send a response to end the stream"); - - // Publish the block for all subscribers to receive - final SubscribeStreamResponse endStreamResponse = buildEndStreamResponse(); - ringBuffer.publishEvent((event, sequence) -> event.set(endStreamResponse)); - - // Unsubscribe all downstream consumers - for (final var subscriber : subscribers.keySet()) { - LOGGER.log(ERROR, String.format("Unsubscribing: %s", subscriber)); - unsubscribe(subscriber); - } - - throw e; - } } else { LOGGER.log(ERROR, "StreamMediator is not accepting BlockItems"); } } @Override - public void subscribe( - @NonNull final EventHandler> handler) { - - // Initialize the batch event processor and set it on the ring buffer - final var batchEventProcessor = - new BatchEventProcessorBuilder() - .build(ringBuffer, ringBuffer.newBarrier(), handler); - - ringBuffer.addGatingSequences(batchEventProcessor.getSequence()); - executor.execute(batchEventProcessor); - - // Keep track of the subscriber - subscribers.put(handler, batchEventProcessor); - - // update the subscriber metrics - metricsService.get(Subscribers).set(subscribers.size()); - } - - @Override - public void unsubscribe( - @NonNull final EventHandler> handler) { + public void notifyUnrecoverableError() { - // Remove the subscriber - final var batchEventProcessor = subscribers.remove(handler); - if (batchEventProcessor == null) { - LOGGER.log(ERROR, "Subscriber not found: {0}", handler); + // Disable BlockItem publication for upstream producers + serviceStatus.stopRunning(this.getClass().getName()); + LOGGER.log(ERROR, "An exception occurred. Stopping the service."); - } else { + // Increment the error counter + metricsService.get(LiveBlockStreamMediatorError).increment(); - // Stop the processor - batchEventProcessor.halt(); + LOGGER.log(ERROR, "Sending an error response to end the stream for all consumers."); - // Remove the gating sequence from the ring buffer - ringBuffer.removeGatingSequence(batchEventProcessor.getSequence()); - } - - // update the subscriber metrics - metricsService.get(Subscribers).set(subscribers.size()); - } - - @Override - public boolean isSubscribed( - @NonNull EventHandler> handler) { - return subscribers.containsKey(handler); + // Publish an end of stream response to all downstream consumers + final SubscribeStreamResponse endStreamResponse = buildEndStreamResponse(); + ringBuffer.publishEvent((event, sequence) -> event.set(endStreamResponse)); } @NonNull diff --git a/server/src/main/java/com/hedera/block/server/mediator/MediatorConfig.java b/server/src/main/java/com/hedera/block/server/mediator/MediatorConfig.java new file mode 100644 index 00000000..646e5ec1 --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/mediator/MediatorConfig.java @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.mediator; + +import static java.lang.System.Logger.Level.INFO; + +import com.swirlds.config.api.ConfigData; +import com.swirlds.config.api.ConfigProperty; + +/** + * Constructor to initialize the Mediator configuration. + * + *

MediatorConfig will set the ring buffer size for the mediator. + * + * @param ringBufferSize the size of the ring buffer used by the mediator + */ +// TODO: defaultValue here should be 67108864 +@ConfigData("mediator") +public record MediatorConfig(@ConfigProperty(defaultValue = "1024") int ringBufferSize) { + private static final System.Logger LOGGER = System.getLogger(MediatorConfig.class.getName()); + + /** + * Validate the configuration. + * + * @throws IllegalArgumentException if the configuration is invalid + */ + public MediatorConfig { + if (ringBufferSize <= 0) { + throw new IllegalArgumentException("Ring buffer size must be greater than 0"); + } + + if ((ringBufferSize & (ringBufferSize - 1)) != 0) { + throw new IllegalArgumentException("Ring buffer size must be a power of 2"); + } + + LOGGER.log(INFO, "Mediator configuration mediator.ringBufferSize: " + ringBufferSize); + } +} diff --git a/server/src/main/java/com/hedera/block/server/mediator/MediatorInjectionModule.java b/server/src/main/java/com/hedera/block/server/mediator/MediatorInjectionModule.java index f50e107b..7a8c2290 100644 --- a/server/src/main/java/com/hedera/block/server/mediator/MediatorInjectionModule.java +++ b/server/src/main/java/com/hedera/block/server/mediator/MediatorInjectionModule.java @@ -16,12 +16,11 @@ package com.hedera.block.server.mediator; -import com.hedera.block.server.ServiceStatus; import com.hedera.block.server.config.BlockNodeContext; -import com.hedera.block.server.data.ObjectEvent; -import com.hedera.block.server.persistence.storage.write.BlockWriter; +import com.hedera.block.server.notifier.Notifiable; +import com.hedera.block.server.service.ServiceStatus; import com.hedera.hapi.block.SubscribeStreamResponse; -import com.hedera.hapi.block.stream.BlockItem; +import dagger.Binds; import dagger.Module; import dagger.Provides; import edu.umd.cs.findbugs.annotations.NonNull; @@ -34,18 +33,35 @@ public interface MediatorInjectionModule { /** * Provides the stream mediator. * - * @param blockWriter the block writer * @param blockNodeContext the block node context * @param serviceStatus the service status * @return the stream mediator */ @Provides @Singleton - static StreamMediator> providesStreamMediator( - @NonNull BlockWriter blockWriter, - @NonNull BlockNodeContext blockNodeContext, - @NonNull ServiceStatus serviceStatus) { - return LiveStreamMediatorBuilder.newBuilder(blockWriter, blockNodeContext, serviceStatus) - .build(); + static LiveStreamMediator providesLiveStreamMediator( + @NonNull BlockNodeContext blockNodeContext, @NonNull ServiceStatus serviceStatus) { + return LiveStreamMediatorBuilder.newBuilder(blockNodeContext, serviceStatus).build(); } + + /** + * Binds the subscription handler to the live stream mediator. + * + * @param liveStreamMediator the live stream mediator + * @return the subscription handler + */ + @Binds + @Singleton + SubscriptionHandler bindSubscriptionHandler( + @NonNull final LiveStreamMediator liveStreamMediator); + + /** + * Binds the mediator to the notifiable interface. + * + * @param liveStreamMediator the live stream mediator + * @return the notifiable interface + */ + @Binds + @Singleton + Notifiable bindMediator(@NonNull final LiveStreamMediator liveStreamMediator); } diff --git a/server/src/main/java/com/hedera/block/server/mediator/Publisher.java b/server/src/main/java/com/hedera/block/server/mediator/Publisher.java index a34455bc..b31e3ba6 100644 --- a/server/src/main/java/com/hedera/block/server/mediator/Publisher.java +++ b/server/src/main/java/com/hedera/block/server/mediator/Publisher.java @@ -17,7 +17,6 @@ package com.hedera.block.server.mediator; import edu.umd.cs.findbugs.annotations.NonNull; -import java.io.IOException; /** * The Publisher interface defines the contract for publishing data emitted by the producer to @@ -31,8 +30,6 @@ public interface Publisher { * Publishes the given data to the downstream subscribers. * * @param data the data emitted by an upstream producer to publish to downstream subscribers. - * @throws IOException thrown if an I/O error occurs while publishing the item to the - * subscribers. */ - void publish(@NonNull final U data) throws IOException; + void publish(@NonNull final U data); } diff --git a/server/src/main/java/com/hedera/block/server/mediator/SubscriptionHandler.java b/server/src/main/java/com/hedera/block/server/mediator/SubscriptionHandler.java index b3ec6639..391b1a90 100644 --- a/server/src/main/java/com/hedera/block/server/mediator/SubscriptionHandler.java +++ b/server/src/main/java/com/hedera/block/server/mediator/SubscriptionHandler.java @@ -16,7 +16,8 @@ package com.hedera.block.server.mediator; -import com.lmax.disruptor.EventHandler; +import com.hedera.block.server.events.BlockNodeEventHandler; +import com.hedera.block.server.events.ObjectEvent; import edu.umd.cs.findbugs.annotations.NonNull; /** @@ -32,14 +33,14 @@ public interface SubscriptionHandler { * * @param handler the handler to subscribe */ - void subscribe(@NonNull final EventHandler handler); + void subscribe(@NonNull final BlockNodeEventHandler> handler); /** * Unsubscribes the given handler from the stream of events. * * @param handler the handler to unsubscribe */ - void unsubscribe(@NonNull final EventHandler handler); + void unsubscribe(@NonNull final BlockNodeEventHandler> handler); /** * Checks if the given handler is subscribed to the stream of events. @@ -47,5 +48,8 @@ public interface SubscriptionHandler { * @param handler the handler to check * @return true if the handler is subscribed, false otherwise */ - boolean isSubscribed(@NonNull final EventHandler handler); + boolean isSubscribed(@NonNull final BlockNodeEventHandler> handler); + + /** Unsubscribes all the expired handlers from the stream of events. */ + void unsubscribeAllExpired(); } diff --git a/server/src/main/java/com/hedera/block/server/mediator/SubscriptionHandlerBase.java b/server/src/main/java/com/hedera/block/server/mediator/SubscriptionHandlerBase.java new file mode 100644 index 00000000..bd058f28 --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/mediator/SubscriptionHandlerBase.java @@ -0,0 +1,146 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.mediator; + +import com.hedera.block.server.events.BlockNodeEventHandler; +import com.hedera.block.server.events.ObjectEvent; +import com.lmax.disruptor.BatchEventProcessor; +import com.lmax.disruptor.BatchEventProcessorBuilder; +import com.lmax.disruptor.RingBuffer; +import com.lmax.disruptor.dsl.Disruptor; +import com.lmax.disruptor.util.DaemonThreadFactory; +import com.swirlds.metrics.api.LongGauge; +import edu.umd.cs.findbugs.annotations.NonNull; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * Inherit from this class to leverage RingBuffer subscription handling. + * + *

Subclasses may use the ringBuffer to publish events to the subscribers. This base class + * contains the logic to manage subscriptions to the ring buffer. + * + * @param the type of the subscription events + */ +public abstract class SubscriptionHandlerBase implements SubscriptionHandler { + + private final Map>, BatchEventProcessor>> + subscribers; + + /** The ring buffer to publish events to the subscribers. */ + protected final RingBuffer> ringBuffer; + + private final LongGauge subscriptionGauge; + private final ExecutorService executor; + + /** + * Constructs an abstract SubscriptionHandler instance with the given subscribers, block writer, + * and service status. Users of this constructor should take care to supply a thread-safe map + * implementation for the subscribers to handle the dynamic addition and removal of subscribers + * at runtime. + * + * @param subscribers the map of subscribers to batch event processors. It's recommended the map + * implementation is thread-safe + * @param subscriptionGauge the gauge to keep track of the number of subscribers + * @param ringBufferSize the size of the ring buffer + */ + protected SubscriptionHandlerBase( + @NonNull + final Map< + BlockNodeEventHandler>, + BatchEventProcessor>> + subscribers, + @NonNull final LongGauge subscriptionGauge, + final int ringBufferSize) { + + this.subscribers = subscribers; + this.subscriptionGauge = subscriptionGauge; + + // Initialize and start the disruptor + final Disruptor> disruptor = + new Disruptor<>(ObjectEvent::new, ringBufferSize, DaemonThreadFactory.INSTANCE); + this.ringBuffer = disruptor.start(); + this.executor = Executors.newCachedThreadPool(DaemonThreadFactory.INSTANCE); + } + + /** + * Subscribes the given handler to the stream of events. + * + * @param handler the handler to subscribe + */ + @Override + public void subscribe(@NonNull final BlockNodeEventHandler> handler) { + + if (!subscribers.containsKey(handler)) { + // Initialize the batch event processor and set it on the ring buffer + final var batchEventProcessor = + new BatchEventProcessorBuilder() + .build(ringBuffer, ringBuffer.newBarrier(), handler); + + ringBuffer.addGatingSequences(batchEventProcessor.getSequence()); + executor.execute(batchEventProcessor); + + // Keep track of the subscriber + subscribers.put(handler, batchEventProcessor); + + // Update the subscriber metrics. + subscriptionGauge.set(subscribers.size()); + } + } + + /** + * Unsubscribes the given handler from the stream of events. + * + * @param handler the handler to unsubscribe + */ + @Override + public void unsubscribe(@NonNull final BlockNodeEventHandler> handler) { + + // Remove the subscriber + final var batchEventProcessor = subscribers.remove(handler); + if (batchEventProcessor != null) { + // Stop the processor + batchEventProcessor.halt(); + + // Remove the gating sequence from the ring buffer + ringBuffer.removeGatingSequence(batchEventProcessor.getSequence()); + } + + // Update the subscriber metrics. + subscriptionGauge.set(subscribers.size()); + } + + /** + * Checks if the given handler is subscribed to the stream of events. + * + * @param handler the handler to check + * @return true if the handler is subscribed, false otherwise + */ + @Override + public boolean isSubscribed(@NonNull BlockNodeEventHandler> handler) { + return subscribers.containsKey(handler); + } + + /** Unsubscribes all the expired handlers from the stream of events. */ + @Override + public void unsubscribeAllExpired() { + subscribers.keySet().stream() + .filter(BlockNodeEventHandler::isTimeoutExpired) + .forEach(this::unsubscribe); + } +} diff --git a/server/src/main/java/com/hedera/block/server/metrics/BlockNodeMetricTypes.java b/server/src/main/java/com/hedera/block/server/metrics/BlockNodeMetricTypes.java index c5d968e4..a7aafc0a 100644 --- a/server/src/main/java/com/hedera/block/server/metrics/BlockNodeMetricTypes.java +++ b/server/src/main/java/com/hedera/block/server/metrics/BlockNodeMetricTypes.java @@ -41,11 +41,15 @@ public enum Counter implements MetricMetadata { /** The number of live block items received before publishing to the RingBuffer. */ LiveBlockItems("live_block_items", "Live BlockItems"), - /** - * The number of blocks persisted to storage. - * - *

Block items are not counted here, only the blocks. - */ + /** The number of PublishStreamResponses generated and published to the subscribers. */ + SuccessfulPubStreamResp( + "successful_pub_stream_resp", "Successful Publish Stream Responses"), + + /** The number of PublishStreamResponses sent to the producers. */ + SuccessfulPubStreamRespSent( + "successful_pub_stream_resp_sent", "Successful Publish Stream Responses Sent"), + + /** The number of blocks persisted to storage. */ BlocksPersisted("blocks_persisted", "Blocks Persisted"), /** The number of live block items consumed from the by each consumer observer. */ @@ -61,7 +65,11 @@ public enum Counter implements MetricMetadata { /** The number of errors encountered by the live block stream mediator. */ LiveBlockStreamMediatorError( - "live_block_stream_mediator_error", "Live Block Stream Mediator Error"); + "live_block_stream_mediator_error", "Live Block Stream Mediator Error"), + + /** The number of errors encountered by the stream persistence handler. */ + StreamPersistenceHandlerError( + "stream_persistence_handler_error", "Stream Persistence Handler Error"); private final String grafanaLabel; private final String description; @@ -93,7 +101,10 @@ public String description() { public enum Gauge implements MetricMetadata { /** The number of subscribers receiving the live block stream. */ - Subscribers("subscribers", "Subscribers"); + Consumers("consumers", "Consumers"), + + /** The number of producers publishing block items. */ + Producers("producers", "Producers"); private final String grafanaLabel; private final String description; diff --git a/server/src/main/java/com/hedera/block/server/notifier/Notifiable.java b/server/src/main/java/com/hedera/block/server/notifier/Notifiable.java new file mode 100644 index 00000000..3d5c869e --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/notifier/Notifiable.java @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.notifier; + +/** Use this interface contract to notify the implementation of critical system events. */ +public interface Notifiable { + + /** + * This method is called to notify of an unrecoverable error and the system will be shut down. + */ + void notifyUnrecoverableError(); +} diff --git a/server/src/main/java/com/hedera/block/server/notifier/Notifier.java b/server/src/main/java/com/hedera/block/server/notifier/Notifier.java new file mode 100644 index 00000000..8d4cc508 --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/notifier/Notifier.java @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.notifier; + +import com.hedera.block.server.mediator.StreamMediator; +import com.hedera.hapi.block.PublishStreamResponse; +import com.hedera.hapi.block.stream.BlockItem; + +/** + * Use this interface to combine the contract for streaming block items with the contract to be + * notified of critical system events. + */ +public interface Notifier extends StreamMediator, Notifiable {} diff --git a/server/src/main/java/com/hedera/block/server/notifier/NotifierConfig.java b/server/src/main/java/com/hedera/block/server/notifier/NotifierConfig.java new file mode 100644 index 00000000..1b72085a --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/notifier/NotifierConfig.java @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.notifier; + +import static java.lang.System.Logger.Level.INFO; + +import com.swirlds.config.api.ConfigData; +import com.swirlds.config.api.ConfigProperty; + +/** + * Constructor to initialize the Notifier configuration. + * + *

NotifierConfig will set the ring buffer size for the notifier. + * + * @param ringBufferSize the size of the ring buffer used by the notifier + */ +// TODO: defaultValue here should be 2048 +@ConfigData("notifier") +public record NotifierConfig(@ConfigProperty(defaultValue = "1024") int ringBufferSize) { + private static final System.Logger LOGGER = System.getLogger(NotifierConfig.class.getName()); + + /** + * Validate the configuration. + * + * @throws IllegalArgumentException if the configuration is invalid + */ + public NotifierConfig { + if (ringBufferSize <= 0) { + throw new IllegalArgumentException("Ring buffer size must be greater than 0"); + } + + if ((ringBufferSize & (ringBufferSize - 1)) != 0) { + throw new IllegalArgumentException("Ring buffer size must be a power of 2"); + } + + LOGGER.log(INFO, "Notifier configuration notifier.ringBufferSize: " + ringBufferSize); + } +} diff --git a/server/src/main/java/com/hedera/block/server/notifier/NotifierImpl.java b/server/src/main/java/com/hedera/block/server/notifier/NotifierImpl.java new file mode 100644 index 00000000..a5338088 --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/notifier/NotifierImpl.java @@ -0,0 +1,167 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.notifier; + +import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.SuccessfulPubStreamResp; +import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Gauge.Producers; +import static com.hedera.block.server.producer.Util.getFakeHash; +import static java.lang.System.Logger.Level.ERROR; + +import com.hedera.block.server.config.BlockNodeContext; +import com.hedera.block.server.mediator.SubscriptionHandlerBase; +import com.hedera.block.server.metrics.MetricsService; +import com.hedera.block.server.service.ServiceStatus; +import com.hedera.hapi.block.Acknowledgement; +import com.hedera.hapi.block.EndOfStream; +import com.hedera.hapi.block.ItemAcknowledgement; +import com.hedera.hapi.block.PublishStreamResponse; +import com.hedera.hapi.block.PublishStreamResponseCode; +import com.hedera.hapi.block.stream.BlockItem; +import com.hedera.pbj.runtime.io.buffer.Bytes; +import edu.umd.cs.findbugs.annotations.NonNull; +import java.security.NoSuchAlgorithmException; +import java.util.concurrent.ConcurrentHashMap; +import javax.inject.Inject; +import javax.inject.Singleton; + +/** + * Use NotifierImpl to mediate the stream of responses from the persistence layer back to multiple + * producers. + * + *

As an implementation of the StreamMediator interface, it proxies block item persistence + * responses back to the producers as they arrive via a RingBuffer maintained in the base class and + * persists the block items to a store. It also notifies the mediator of critical system events and + * will stop the server in the event of an unrecoverable error. + */ +@Singleton +public class NotifierImpl extends SubscriptionHandlerBase + implements Notifier { + + private final System.Logger LOGGER = System.getLogger(getClass().getName()); + + /** The initial capacity of producers in the subscriber map. */ + private static final int SUBSCRIBER_INIT_CAPACITY = 5; + + private final Notifiable mediator; + private final MetricsService metricsService; + private final ServiceStatus serviceStatus; + + /** + * Constructs a new NotifierImpl instance with the given mediator, block node context, and + * service status. + * + * @param mediator the mediator to notify of critical system events + * @param blockNodeContext the block node context + * @param serviceStatus the service status to stop the service and web server if an exception + * occurs + */ + @Inject + public NotifierImpl( + @NonNull final Notifiable mediator, + @NonNull final BlockNodeContext blockNodeContext, + @NonNull final ServiceStatus serviceStatus) { + + super( + new ConcurrentHashMap<>(SUBSCRIBER_INIT_CAPACITY), + blockNodeContext.metricsService().get(Producers), + blockNodeContext + .configuration() + .getConfigData(NotifierConfig.class) + .ringBufferSize()); + + this.mediator = mediator; + this.metricsService = blockNodeContext.metricsService(); + this.serviceStatus = serviceStatus; + } + + @Override + public void notifyUnrecoverableError() { + + mediator.notifyUnrecoverableError(); + + // Publish an end of stream response to the producers. + final PublishStreamResponse errorStreamResponse = buildErrorStreamResponse(); + ringBuffer.publishEvent((event, sequence) -> event.set(errorStreamResponse)); + + // Stop the server + serviceStatus.stopWebServer(getClass().getName()); + } + + /** + * Publishes the given block item to all subscribed producers. + * + * @param blockItem the block item from the persistence layer to publish a response to upstream + * producers + */ + @Override + public void publish(@NonNull BlockItem blockItem) { + + try { + if (serviceStatus.isRunning()) { + // Publish the block item to the subscribers + final var publishStreamResponse = + PublishStreamResponse.newBuilder() + .acknowledgement(buildAck(blockItem)) + .build(); + ringBuffer.publishEvent((event, sequence) -> event.set(publishStreamResponse)); + + metricsService.get(SuccessfulPubStreamResp).increment(); + } else { + LOGGER.log(ERROR, "Notifier is not running."); + } + + } catch (NoSuchAlgorithmException e) { + + // Stop the server + serviceStatus.stopRunning(getClass().getName()); + + final var errorResponse = buildErrorStreamResponse(); + LOGGER.log(ERROR, "Error calculating hash: ", e); + + // Send an error response to all the producers. + ringBuffer.publishEvent((event, sequence) -> event.set(errorResponse)); + } + } + + @NonNull + static PublishStreamResponse buildErrorStreamResponse() { + // TODO: Replace this with a real error enum. + final EndOfStream endOfStream = + EndOfStream.newBuilder() + .status(PublishStreamResponseCode.STREAM_ITEMS_UNKNOWN) + .build(); + return PublishStreamResponse.newBuilder().status(endOfStream).build(); + } + + /** + * Protected method meant for testing. Builds an Acknowledgement for the block item. + * + * @param blockItem the block item to build the Acknowledgement for + * @return the Acknowledgement for the block item + * @throws NoSuchAlgorithmException if the hash algorithm is not supported + */ + @NonNull + Acknowledgement buildAck(@NonNull final BlockItem blockItem) throws NoSuchAlgorithmException { + final ItemAcknowledgement itemAck = + ItemAcknowledgement.newBuilder() + // TODO: Replace this with a real hash generator + .itemHash(Bytes.wrap(getFakeHash(blockItem))) + .build(); + + return Acknowledgement.newBuilder().itemAck(itemAck).build(); + } +} diff --git a/server/src/main/java/com/hedera/block/server/notifier/NotifierInjectionModule.java b/server/src/main/java/com/hedera/block/server/notifier/NotifierInjectionModule.java new file mode 100644 index 00000000..f9c950e6 --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/notifier/NotifierInjectionModule.java @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.notifier; + +import dagger.Binds; +import dagger.Module; +import javax.inject.Singleton; + +/** A Dagger module for providing dependencies for Notifier Module. */ +@Module +public interface NotifierInjectionModule { + + /** + * Provides the notifier. + * + * @param notifier requires a notifier implementation + * @return the notifier + */ + @Binds + @Singleton + Notifier bindNotifier(NotifierImpl notifier); +} diff --git a/server/src/main/java/com/hedera/block/server/persistence/PersistenceInjectionModule.java b/server/src/main/java/com/hedera/block/server/persistence/PersistenceInjectionModule.java index 2cddc12b..1efa810f 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/PersistenceInjectionModule.java +++ b/server/src/main/java/com/hedera/block/server/persistence/PersistenceInjectionModule.java @@ -17,13 +17,17 @@ package com.hedera.block.server.persistence; import com.hedera.block.server.config.BlockNodeContext; +import com.hedera.block.server.events.BlockNodeEventHandler; +import com.hedera.block.server.events.ObjectEvent; import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; import com.hedera.block.server.persistence.storage.read.BlockAsDirReaderBuilder; import com.hedera.block.server.persistence.storage.read.BlockReader; import com.hedera.block.server.persistence.storage.write.BlockAsDirWriterBuilder; import com.hedera.block.server.persistence.storage.write.BlockWriter; +import com.hedera.hapi.block.SubscribeStreamResponse; import com.hedera.hapi.block.stream.Block; import com.hedera.hapi.block.stream.BlockItem; +import dagger.Binds; import dagger.Module; import dagger.Provides; import java.io.IOException; @@ -60,4 +64,15 @@ static BlockWriter providesBlockWriter(BlockNodeContext blockNodeCont static BlockReader providesBlockReader(PersistenceStorageConfig config) { return BlockAsDirReaderBuilder.newBuilder(config).build(); } + + /** + * Binds the block node event handler to the stream persistence handler. + * + * @param streamPersistenceHandler the stream persistence handler + * @return the block node event handler + */ + @Binds + @Singleton + BlockNodeEventHandler> bindBlockNodeEventHandler( + StreamPersistenceHandlerImpl streamPersistenceHandler); } diff --git a/server/src/main/java/com/hedera/block/server/persistence/StreamPersistenceHandlerImpl.java b/server/src/main/java/com/hedera/block/server/persistence/StreamPersistenceHandlerImpl.java new file mode 100644 index 00000000..a34638b9 --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/persistence/StreamPersistenceHandlerImpl.java @@ -0,0 +1,158 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence; + +import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.StreamPersistenceHandlerError; +import static java.lang.System.Logger.Level.DEBUG; +import static java.lang.System.Logger.Level.ERROR; + +import com.hedera.block.server.config.BlockNodeContext; +import com.hedera.block.server.events.BlockNodeEventHandler; +import com.hedera.block.server.events.ObjectEvent; +import com.hedera.block.server.exception.BlockStreamProtocolException; +import com.hedera.block.server.mediator.SubscriptionHandler; +import com.hedera.block.server.metrics.MetricsService; +import com.hedera.block.server.notifier.Notifier; +import com.hedera.block.server.persistence.storage.write.BlockWriter; +import com.hedera.block.server.service.ServiceStatus; +import com.hedera.hapi.block.SubscribeStreamResponse; +import com.hedera.hapi.block.stream.BlockItem; +import com.hedera.pbj.runtime.OneOf; +import edu.umd.cs.findbugs.annotations.NonNull; +import java.io.IOException; +import java.util.Optional; +import javax.inject.Inject; +import javax.inject.Singleton; + +/** + * Use the StreamPersistenceHandlerImpl to persist live block items passed asynchronously through + * the LMAX Disruptor + * + *

This implementation is the primary integration point between the LMAX Disruptor and the file + * system. The stream persistence handler implements the EventHandler interface so the Disruptor can + * invoke the onEvent() method when a new SubscribeStreamResponse is available. + */ +@Singleton +public class StreamPersistenceHandlerImpl + implements BlockNodeEventHandler> { + + private final System.Logger LOGGER = System.getLogger(getClass().getName()); + + private final SubscriptionHandler subscriptionHandler; + private final BlockWriter blockWriter; + private final Notifier notifier; + private final MetricsService metricsService; + private final ServiceStatus serviceStatus; + + private static final String PROTOCOL_VIOLATION_MESSAGE = + "Protocol Violation. %s is OneOf type %s but %s is null.\n%s"; + + /** + * Constructs a new StreamPersistenceHandlerImpl instance with the given subscription handler, + * notifier, block writer, + * + * @param subscriptionHandler is used to unsubscribe from the mediator if an error occurs. + * @param notifier is used to pass successful response messages back to producers and to trigger + * error handling in the event of unrecoverable errors. + * @param blockWriter is used to persist the block items. + * @param blockNodeContext contains the context with metrics and configuration for the + * application. + * @param serviceStatus is used to stop the service and web server if an exception occurs while + * persisting a block item, stop the web server for maintenance, etc. + */ + @Inject + public StreamPersistenceHandlerImpl( + @NonNull final SubscriptionHandler subscriptionHandler, + @NonNull final Notifier notifier, + @NonNull final BlockWriter blockWriter, + @NonNull final BlockNodeContext blockNodeContext, + @NonNull final ServiceStatus serviceStatus) { + this.subscriptionHandler = subscriptionHandler; + this.blockWriter = blockWriter; + this.notifier = notifier; + this.metricsService = blockNodeContext.metricsService(); + this.serviceStatus = serviceStatus; + } + + /** + * The onEvent method is invoked by the Disruptor when a new SubscribeStreamResponse is + * available. The method processes the response and persists the block item to the file system. + * + * @param event the ObjectEvent containing the SubscribeStreamResponse + * @param l the sequence number of the event + * @param b true if the event is the last in the sequence + */ + @Override + public void onEvent(ObjectEvent event, long l, boolean b) { + try { + if (serviceStatus.isRunning()) { + + final SubscribeStreamResponse subscribeStreamResponse = event.get(); + final OneOf oneOfTypeOneOf = + subscribeStreamResponse.response(); + switch (oneOfTypeOneOf.kind()) { + case BLOCK_ITEM -> { + final BlockItem blockItem = subscribeStreamResponse.blockItem(); + if (blockItem == null) { + final String message = + PROTOCOL_VIOLATION_MESSAGE.formatted( + "SubscribeStreamResponse", + "BLOCK_ITEM", + "block_item", + subscribeStreamResponse); + LOGGER.log(ERROR, message); + metricsService.get(StreamPersistenceHandlerError).increment(); + throw new BlockStreamProtocolException(message); + } else { + // Persist the BlockItem + Optional result = blockWriter.write(blockItem); + if (result.isPresent()) { + // Publish the block item back upstream to the notifier + // to send responses to producers. + notifier.publish(blockItem); + } + } + } + case STATUS -> LOGGER.log( + DEBUG, "Unexpected received a status message rather than a block item"); + default -> { + final String message = "Unknown response type: " + oneOfTypeOneOf.kind(); + LOGGER.log(ERROR, message); + metricsService.get(StreamPersistenceHandlerError).increment(); + throw new BlockStreamProtocolException(message); + } + } + } else { + LOGGER.log( + ERROR, "Service is not running. Block item will not be processed further."); + } + + } catch (BlockStreamProtocolException | IOException e) { + + metricsService.get(StreamPersistenceHandlerError).increment(); + + // Trigger the server to stop accepting new requests + serviceStatus.stopRunning(getClass().getName()); + + // Unsubscribe from the mediator to avoid additional onEvent calls. + subscriptionHandler.unsubscribe(this); + + // Broadcast the problem to the notifier + notifier.notifyUnrecoverableError(); + } + } +} diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/PersistenceStorageConfig.java b/server/src/main/java/com/hedera/block/server/persistence/storage/PersistenceStorageConfig.java index f2f0d1fe..49054927 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/storage/PersistenceStorageConfig.java +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/PersistenceStorageConfig.java @@ -17,6 +17,7 @@ package com.hedera.block.server.persistence.storage; import static java.lang.System.Logger.Level.ERROR; +import static java.lang.System.Logger.Level.INFO; import com.swirlds.config.api.ConfigData; import com.swirlds.config.api.ConfigProperty; @@ -33,6 +34,8 @@ */ @ConfigData("persistence.storage") public record PersistenceStorageConfig(@ConfigProperty(defaultValue = "") String rootPath) { + private static final System.Logger LOGGER = + System.getLogger(PersistenceStorageConfig.class.getName()); /** * Constructor to set the default root path if not provided, it will be set to the data @@ -57,6 +60,7 @@ public record PersistenceStorageConfig(@ConfigProperty(defaultValue = "") String } } + LOGGER.log(INFO, "Persistence Storage configuration persistence.storage.rootPath: " + path); rootPath = path.toString(); } } diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriter.java b/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriter.java index e5c96e75..7bee8240 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriter.java +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriter.java @@ -36,6 +36,7 @@ import java.nio.file.Path; import java.nio.file.attribute.FileAttribute; import java.nio.file.attribute.PosixFilePermission; +import java.util.Optional; import java.util.Set; /** @@ -97,7 +98,7 @@ class BlockAsDirWriter implements BlockWriter { * @throws IOException if an error occurs while writing the block item */ @Override - public void write(@NonNull final BlockItem blockItem) throws IOException { + public Optional write(@NonNull final BlockItem blockItem) throws IOException { if (blockItem.hasBlockHeader()) { resetState(blockItem); @@ -126,6 +127,13 @@ public void write(@NonNull final BlockItem blockItem) throws IOException { } } } + + if (blockItem.hasBlockProof()) { + metricsService.get(BlocksPersisted).increment(); + return Optional.of(blockItem); + } + + return Optional.empty(); } /** @@ -161,9 +169,6 @@ private void resetState(@NonNull final BlockItem blockItem) throws IOException { // Reset blockNodeFileNameIndex = 0; - - // Increment the block counter - metricsService.get(BlocksPersisted).increment(); } private void repairPermissions(@NonNull final Path path) throws IOException { @@ -174,13 +179,8 @@ private void repairPermissions(@NonNull final Path path) throws IOException { "Block node root directory is not writable. Attempting to change the" + " permissions."); - try { - // Attempt to restore the permissions on the block node root directory - Files.setPosixFilePermissions(path, filePerms.value()); - } catch (IOException e) { - LOGGER.log(ERROR, "Error setting permissions on the path: " + path, e); - throw e; - } + // Attempt to restore the permissions on the block node root directory + Files.setPosixFilePermissions(path, filePerms.value()); } } diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockWriter.java b/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockWriter.java index 84baa9e1..af663033 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockWriter.java +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockWriter.java @@ -18,6 +18,7 @@ import edu.umd.cs.findbugs.annotations.NonNull; import java.io.IOException; +import java.util.Optional; /** * BlockWriter defines the contract for writing block items to storage. @@ -30,7 +31,9 @@ public interface BlockWriter { * Write the block item to storage. * * @param blockItem the block item to write to storage. + * @return an optional containing the block item written to storage if the block item was a + * block proof signaling the end of the block, an empty optional otherwise. * @throws IOException when failing to write the block item to storage. */ - void write(@NonNull final V blockItem) throws IOException; + Optional write(@NonNull final V blockItem) throws IOException; } diff --git a/server/src/main/java/com/hedera/block/server/producer/ProducerBlockItemObserver.java b/server/src/main/java/com/hedera/block/server/producer/ProducerBlockItemObserver.java index 18151583..db65ce6c 100644 --- a/server/src/main/java/com/hedera/block/server/producer/ProducerBlockItemObserver.java +++ b/server/src/main/java/com/hedera/block/server/producer/ProducerBlockItemObserver.java @@ -19,27 +19,30 @@ import static com.hedera.block.server.Translator.fromPbj; import static com.hedera.block.server.Translator.toPbj; import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.LiveBlockItemsReceived; -import static com.hedera.block.server.producer.Util.getFakeHash; +import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.SuccessfulPubStreamRespSent; import static java.lang.System.Logger; import static java.lang.System.Logger.Level.DEBUG; import static java.lang.System.Logger.Level.ERROR; -import com.hedera.block.server.ServiceStatus; import com.hedera.block.server.config.BlockNodeContext; +import com.hedera.block.server.consumer.ConsumerConfig; +import com.hedera.block.server.events.BlockNodeEventHandler; +import com.hedera.block.server.events.LivenessCalculator; +import com.hedera.block.server.events.ObjectEvent; import com.hedera.block.server.mediator.Publisher; +import com.hedera.block.server.mediator.SubscriptionHandler; import com.hedera.block.server.metrics.MetricsService; -import com.hedera.hapi.block.Acknowledgement; +import com.hedera.block.server.service.ServiceStatus; import com.hedera.hapi.block.EndOfStream; -import com.hedera.hapi.block.ItemAcknowledgement; import com.hedera.hapi.block.PublishStreamResponse; import com.hedera.hapi.block.PublishStreamResponseCode; import com.hedera.hapi.block.stream.BlockItem; import com.hedera.pbj.runtime.ParseException; -import com.hedera.pbj.runtime.io.buffer.Bytes; import edu.umd.cs.findbugs.annotations.NonNull; +import io.grpc.stub.ServerCallStreamObserver; import io.grpc.stub.StreamObserver; -import java.io.IOException; -import java.security.NoSuchAlgorithmException; +import java.time.InstantSource; +import java.util.concurrent.atomic.AtomicBoolean; /** * The ProducerBlockStreamObserver class plugs into Helidon's server-initiated bidirectional gRPC @@ -48,43 +51,99 @@ * server). */ public class ProducerBlockItemObserver - implements StreamObserver { + implements StreamObserver, + BlockNodeEventHandler> { private final Logger LOGGER = System.getLogger(getClass().getName()); private final StreamObserver publishStreamResponseObserver; + private final SubscriptionHandler subscriptionHandler; private final Publisher publisher; private final ServiceStatus serviceStatus; private final MetricsService metricsService; + private final AtomicBoolean isResponsePermitted = new AtomicBoolean(true); + + private final LivenessCalculator livenessCalculator; + + /** + * The onCancel handler to execute when the producer cancels the stream. This handler is + * protected to facilitate testing. + */ + protected Runnable onCancel; + + /** + * The onClose handler to execute when the producer closes the stream. This handler is protected + * to facilitate testing. + */ + protected Runnable onClose; + /** * Constructor for the ProducerBlockStreamObserver class. It is responsible for calling the * mediator with blocks as they arrive from the upstream producer. It also sends responses back * to the upstream producer via the responseStreamObserver. * + * @param producerLivenessClock the clock used to calculate the producer liveness. * @param publisher the block item publisher to used to pass block items to consumers as they * arrive from the upstream producer. + * @param subscriptionHandler the subscription handler used to * @param publishStreamResponseObserver the response stream observer to send responses back to * the upstream producer for each block item processed. * @param blockNodeContext the block node context used to access context objects for the Block * Node (e.g. - the metrics service). - * @param serviceStatus the service status used to determine if the downstream service is - * accepting block items. In the event of an unrecoverable exception, it will be used to - * stop the web server. + * @param serviceStatus the service status used to stop the server in the event of an + * unrecoverable error. */ public ProducerBlockItemObserver( + @NonNull final InstantSource producerLivenessClock, @NonNull final Publisher publisher, + @NonNull final SubscriptionHandler subscriptionHandler, @NonNull final StreamObserver publishStreamResponseObserver, @NonNull final BlockNodeContext blockNodeContext, @NonNull final ServiceStatus serviceStatus) { + this.livenessCalculator = + new LivenessCalculator( + producerLivenessClock, + blockNodeContext + .configuration() + .getConfigData(ConsumerConfig.class) + .timeoutThresholdMillis()); + this.publisher = publisher; this.publishStreamResponseObserver = publishStreamResponseObserver; + this.subscriptionHandler = subscriptionHandler; this.metricsService = blockNodeContext.metricsService(); this.serviceStatus = serviceStatus; + + if (publishStreamResponseObserver + instanceof + ServerCallStreamObserver + serverCallStreamObserver) { + + onCancel = + () -> { + // The producer has cancelled the stream. + // Do not allow additional responses to be sent. + isResponsePermitted.set(false); + subscriptionHandler.unsubscribe(this); + LOGGER.log(DEBUG, "Producer cancelled the stream. Observer unsubscribed."); + }; + serverCallStreamObserver.setOnCancelHandler(onCancel); + + onClose = + () -> { + // The producer has closed the stream. + // Do not allow additional responses to be sent. + isResponsePermitted.set(false); + subscriptionHandler.unsubscribe(this); + LOGGER.log(DEBUG, "Producer completed the stream. Observer unsubscribed."); + }; + serverCallStreamObserver.setOnCloseHandler(onClose); + } } /** @@ -99,7 +158,6 @@ public void onNext( @NonNull final com.hedera.hapi.block.protoc.PublishStreamRequest publishStreamRequest) { try { - LOGGER.log(DEBUG, "Received PublishStreamRequest from producer"); final BlockItem blockItem = toPbj(BlockItem.PROTOBUF, publishStreamRequest.getBlockItem().toByteArray()); @@ -110,34 +168,20 @@ public void onNext( // Publish the block to all the subscribers unless // there's an issue with the StreamMediator. if (serviceStatus.isRunning()) { + // Refresh the producer liveness + livenessCalculator.refresh(); // Publish the block to the mediator publisher.publish(blockItem); - try { - // Send a successful response - publishStreamResponseObserver.onNext(buildSuccessStreamResponse(blockItem)); - - } catch (IOException | NoSuchAlgorithmException e) { - final var errorResponse = buildErrorStreamResponse(); - publishStreamResponseObserver.onNext(errorResponse); - LOGGER.log(ERROR, "Error calculating hash: ", e); - } - } else { - LOGGER.log(ERROR, "StreamMediator is not accepting BlockItems"); + LOGGER.log(ERROR, getClass().getName() + " is not accepting BlockItems"); // Close the upstream connection to the producer(s) final var errorResponse = buildErrorStreamResponse(); publishStreamResponseObserver.onNext(errorResponse); LOGGER.log(ERROR, "Error PublishStreamResponse sent to upstream producer"); } - } catch (IOException io) { - final var errorResponse = buildErrorStreamResponse(); - publishStreamResponseObserver.onNext(errorResponse); - LOGGER.log(ERROR, "Exception thrown publishing BlockItem: ", io); - LOGGER.log(ERROR, "Shutting down the web server"); - serviceStatus.stopWebServer(); } catch (ParseException e) { final var errorResponse = buildErrorStreamResponse(); publishStreamResponseObserver.onNext(errorResponse); @@ -146,15 +190,28 @@ public void onNext( "Error parsing inbound block item from a producer: " + publishStreamRequest.getBlockItem(), e); - serviceStatus.stopWebServer(); + + // Stop the server + serviceStatus.stopWebServer(getClass().getName()); } } - @NonNull - private com.hedera.hapi.block.protoc.PublishStreamResponse buildSuccessStreamResponse( - @NonNull final BlockItem blockItem) throws IOException, NoSuchAlgorithmException { - final Acknowledgement ack = buildAck(blockItem); - return fromPbj(PublishStreamResponse.newBuilder().acknowledgement(ack).build()); + @Override + public void onEvent( + ObjectEvent event, long sequence, boolean endOfBatch) { + + if (isResponsePermitted.get()) { + if (isTimeoutExpired()) { + subscriptionHandler.unsubscribe(this); + LOGGER.log( + DEBUG, + "Producer liveness timeout. Unsubscribed ProducerBlockItemObserver."); + } else { + LOGGER.log(DEBUG, "Publishing response to upstream producer: " + this); + publishStreamResponseObserver.onNext(fromPbj(event.get())); + metricsService.get(SuccessfulPubStreamRespSent).increment(); + } + } } @NonNull @@ -167,25 +224,6 @@ private static com.hedera.hapi.block.protoc.PublishStreamResponse buildErrorStre return fromPbj(PublishStreamResponse.newBuilder().status(endOfStream).build()); } - /** - * Protected method meant for testing. Builds an Acknowledgement for the block item. - * - * @param blockItem the block item to build the Acknowledgement for - * @return the Acknowledgement for the block item - * @throws NoSuchAlgorithmException if the hash algorithm is not supported - */ - @NonNull - protected Acknowledgement buildAck(@NonNull final BlockItem blockItem) - throws NoSuchAlgorithmException { - final ItemAcknowledgement itemAck = - ItemAcknowledgement.newBuilder() - // TODO: Replace this with a real hash generator - .itemHash(Bytes.wrap(getFakeHash(blockItem))) - .build(); - - return Acknowledgement.newBuilder().itemAck(itemAck).build(); - } - /** * Helidon triggers this method when an error occurs on the bidirectional stream to the upstream * producer. @@ -207,4 +245,9 @@ public void onCompleted() { LOGGER.log(DEBUG, "ProducerBlockStreamObserver completed"); publishStreamResponseObserver.onCompleted(); } + + @Override + public boolean isTimeoutExpired() { + return livenessCalculator.isTimeoutExpired(); + } } diff --git a/server/src/main/java/com/hedera/block/server/service/ServiceConfig.java b/server/src/main/java/com/hedera/block/server/service/ServiceConfig.java new file mode 100644 index 00000000..2c36f445 --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/service/ServiceConfig.java @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.service; + +import com.swirlds.config.api.ConfigData; +import com.swirlds.config.api.ConfigProperty; + +/** + * Use this configuration across the service package. + * + *

ServiceConfig will set the default shutdown delay for the service. + * + * @param delayMillis the delay in milliseconds for the service + */ +@ConfigData("service") +public record ServiceConfig(@ConfigProperty(defaultValue = "500") int delayMillis) { + private static final System.Logger LOGGER = System.getLogger(ServiceConfig.class.getName()); + + /** + * Validate the configuration. + * + * @throws IllegalArgumentException if the configuration is invalid + */ + public ServiceConfig { + if (delayMillis <= 0) { + throw new IllegalArgumentException("Delay milliseconds must be greater than 0"); + } + + LOGGER.log( + System.Logger.Level.INFO, + "Service configuration service.delayMillis: " + delayMillis); + } +} diff --git a/server/src/main/java/com/hedera/block/server/service/ServiceInjectionModule.java b/server/src/main/java/com/hedera/block/server/service/ServiceInjectionModule.java new file mode 100644 index 00000000..3d432ce6 --- /dev/null +++ b/server/src/main/java/com/hedera/block/server/service/ServiceInjectionModule.java @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.service; + +import dagger.Binds; +import dagger.Module; +import javax.inject.Singleton; + +/** A Dagger module for providing dependencies for Service Module. */ +@Module +public interface ServiceInjectionModule { + + /** + * Binds the service status to the service status implementation. + * + * @param serviceStatus needs a service status implementation + * @return the service status implementation + */ + @Singleton + @Binds + ServiceStatus bindServiceStatus(ServiceStatusImpl serviceStatus); +} diff --git a/server/src/main/java/com/hedera/block/server/ServiceStatus.java b/server/src/main/java/com/hedera/block/server/service/ServiceStatus.java similarity index 90% rename from server/src/main/java/com/hedera/block/server/ServiceStatus.java rename to server/src/main/java/com/hedera/block/server/service/ServiceStatus.java index 457e2c9d..744d357e 100644 --- a/server/src/main/java/com/hedera/block/server/ServiceStatus.java +++ b/server/src/main/java/com/hedera/block/server/service/ServiceStatus.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.hedera.block.server; +package com.hedera.block.server.service; import edu.umd.cs.findbugs.annotations.NonNull; import io.helidon.webserver.WebServer; @@ -49,6 +49,8 @@ public interface ServiceStatus { /** * Stops the service and web server. This method is called to shut down the service and the web * server in the event of an error or when the service needs to restart. + * + * @param className the name of the class stopping the service */ - void stopWebServer(); + void stopWebServer(final String className); } diff --git a/server/src/main/java/com/hedera/block/server/ServiceStatusImpl.java b/server/src/main/java/com/hedera/block/server/service/ServiceStatusImpl.java similarity index 67% rename from server/src/main/java/com/hedera/block/server/ServiceStatusImpl.java rename to server/src/main/java/com/hedera/block/server/service/ServiceStatusImpl.java index 53377428..a6d1b8c5 100644 --- a/server/src/main/java/com/hedera/block/server/ServiceStatusImpl.java +++ b/server/src/main/java/com/hedera/block/server/service/ServiceStatusImpl.java @@ -14,10 +14,12 @@ * limitations under the License. */ -package com.hedera.block.server; +package com.hedera.block.server.service; import static java.lang.System.Logger.Level.DEBUG; +import static java.lang.System.Logger.Level.ERROR; +import com.hedera.block.server.config.BlockNodeContext; import edu.umd.cs.findbugs.annotations.NonNull; import io.helidon.webserver.WebServer; import java.util.concurrent.atomic.AtomicBoolean; @@ -36,9 +38,19 @@ public class ServiceStatusImpl implements ServiceStatus { private final AtomicBoolean isRunning = new AtomicBoolean(true); private WebServer webServer; - /** Constructor for the ServiceStatusImpl class. */ + private final int delayMillis; + + /** + * Use the ServiceStatusImpl to check the status of the block node server and to shut it down if + * necessary. + * + * @param blockNodeContext the block node context + */ @Inject - public ServiceStatusImpl() {} + public ServiceStatusImpl(@NonNull final BlockNodeContext blockNodeContext) { + this.delayMillis = + blockNodeContext.configuration().getConfigData(ServiceConfig.class).delayMillis(); + } /** * Checks if the service is running. @@ -71,13 +83,25 @@ public void setWebServer(@NonNull final WebServer webServer) { /** * Stops the service and web server. This method is called to shut down the service and the web * server in the event of an unrecoverable exception or during expected maintenance. + * + * @param className the name of the class stopping the service */ - public void stopWebServer() { + public void stopWebServer(@NonNull final String className) { + + LOGGER.log(DEBUG, String.format("%s is stopping the server", className)); // Flag the service to stop // accepting new connections isRunning.set(false); + try { + // Delay briefly while outbound termination messages + // are sent to the consumers and producers, etc. + Thread.sleep(delayMillis); + } catch (InterruptedException e) { + LOGGER.log(ERROR, "An exception was thrown waiting to shut down the server: ", e); + } + // Stop the web server webServer.stop(); } diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index 63352467..ccd73557 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -4,6 +4,7 @@ module com.hedera.block.server { exports com.hedera.block.server; exports com.hedera.block.server.consumer; + exports com.hedera.block.server.exception; exports com.hedera.block.server.persistence.storage; exports com.hedera.block.server.persistence.storage.write; exports com.hedera.block.server.persistence.storage.read; @@ -11,9 +12,11 @@ exports com.hedera.block.server.config; exports com.hedera.block.server.mediator; exports com.hedera.block.server.metrics; - exports com.hedera.block.server.data; + exports com.hedera.block.server.events; exports com.hedera.block.server.health; exports com.hedera.block.server.persistence; + exports com.hedera.block.server.notifier; + exports com.hedera.block.server.service; requires com.hedera.block.stream; requires com.google.protobuf; diff --git a/server/src/main/resources/app.properties b/server/src/main/resources/app.properties index 63486a1d..aa769cb0 100644 --- a/server/src/main/resources/app.properties +++ b/server/src/main/resources/app.properties @@ -1 +1,8 @@ prometheus.endpointPortNumber=9999 + +# Ring buffer sizes for the mediator and notifier +mediator.ringBufferSize=67108864 +notifier.ringBufferSize=2048 + +# Timeout for consumers to wait for block item before timing out. Default is 1500 milliseconds. +#consumer.timeoutThresholdMillis=1500 diff --git a/server/src/test/java/com/hedera/block/server/BlockNodeAppInjectionModuleTest.java b/server/src/test/java/com/hedera/block/server/BlockNodeAppInjectionModuleTest.java index cfafca95..625c5959 100644 --- a/server/src/test/java/com/hedera/block/server/BlockNodeAppInjectionModuleTest.java +++ b/server/src/test/java/com/hedera/block/server/BlockNodeAppInjectionModuleTest.java @@ -17,10 +17,11 @@ package com.hedera.block.server; import com.hedera.block.server.config.BlockNodeContext; -import com.hedera.block.server.data.ObjectEvent; +import com.hedera.block.server.events.ObjectEvent; import com.hedera.block.server.mediator.StreamMediator; import com.hedera.block.server.metrics.MetricsService; import com.hedera.block.server.persistence.storage.read.BlockReader; +import com.hedera.block.server.service.ServiceStatus; import com.hedera.block.server.util.TestConfigUtil; import com.hedera.hapi.block.SubscribeStreamResponse; import com.hedera.hapi.block.stream.Block; diff --git a/server/src/test/java/com/hedera/block/server/BlockNodeAppTest.java b/server/src/test/java/com/hedera/block/server/BlockNodeAppTest.java index 92f7f480..f12a1bc2 100644 --- a/server/src/test/java/com/hedera/block/server/BlockNodeAppTest.java +++ b/server/src/test/java/com/hedera/block/server/BlockNodeAppTest.java @@ -21,6 +21,7 @@ import static org.mockito.Mockito.when; import com.hedera.block.server.health.HealthService; +import com.hedera.block.server.service.ServiceStatus; import io.helidon.webserver.WebServer; import io.helidon.webserver.WebServerConfig; import io.helidon.webserver.grpc.GrpcRouting; diff --git a/server/src/test/java/com/hedera/block/server/BlockStreamServiceIntegrationTest.java b/server/src/test/java/com/hedera/block/server/BlockStreamServiceIntegrationTest.java index ef4f985a..e0f3dc99 100644 --- a/server/src/test/java/com/hedera/block/server/BlockStreamServiceIntegrationTest.java +++ b/server/src/test/java/com/hedera/block/server/BlockStreamServiceIntegrationTest.java @@ -23,24 +23,27 @@ import static java.lang.System.Logger; import static java.lang.System.Logger.Level.INFO; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doCallRealMethod; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.timeout; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import com.hedera.block.server.config.BlockNodeContext; -import com.hedera.block.server.data.ObjectEvent; +import com.hedera.block.server.events.BlockNodeEventHandler; +import com.hedera.block.server.events.ObjectEvent; +import com.hedera.block.server.mediator.LiveStreamMediator; import com.hedera.block.server.mediator.LiveStreamMediatorBuilder; -import com.hedera.block.server.mediator.StreamMediator; -import com.hedera.block.server.persistence.storage.FileUtils; -import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; -import com.hedera.block.server.persistence.storage.read.BlockAsDirReaderBuilder; +import com.hedera.block.server.notifier.Notifier; +import com.hedera.block.server.notifier.NotifierImpl; +import com.hedera.block.server.persistence.StreamPersistenceHandlerImpl; import com.hedera.block.server.persistence.storage.read.BlockReader; -import com.hedera.block.server.persistence.storage.remove.BlockAsDirRemover; -import com.hedera.block.server.persistence.storage.remove.BlockRemover; import com.hedera.block.server.persistence.storage.write.BlockAsDirWriterBuilder; import com.hedera.block.server.persistence.storage.write.BlockWriter; +import com.hedera.block.server.service.ServiceStatus; +import com.hedera.block.server.service.ServiceStatusImpl; import com.hedera.block.server.util.TestConfigUtil; import com.hedera.block.server.util.TestUtils; import com.hedera.hapi.block.Acknowledgement; @@ -56,24 +59,20 @@ import com.hedera.hapi.block.SubscribeStreamResponseCode; import com.hedera.hapi.block.stream.Block; import com.hedera.hapi.block.stream.BlockItem; -import com.hedera.pbj.runtime.ParseException; import com.hedera.pbj.runtime.io.buffer.Bytes; import com.lmax.disruptor.BatchEventProcessor; -import com.lmax.disruptor.EventHandler; import edu.umd.cs.findbugs.annotations.NonNull; import io.grpc.stub.StreamObserver; import io.helidon.webserver.WebServer; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.attribute.FileAttribute; -import java.nio.file.attribute.PosixFilePermission; import java.security.NoSuchAlgorithmException; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -87,11 +86,19 @@ public class BlockStreamServiceIntegrationTest { private final Logger LOGGER = System.getLogger(getClass().getName()); - @Mock private StreamMediator> streamMediator; + @Mock private Notifier notifier; @Mock private StreamObserver - publishStreamResponseObserver; + publishStreamResponseObserver1; + + @Mock + private StreamObserver + publishStreamResponseObserver2; + + @Mock + private StreamObserver + publishStreamResponseObserver3; @Mock private StreamObserver @@ -124,7 +131,6 @@ public class BlockStreamServiceIntegrationTest { subscribeStreamObserver6; @Mock private WebServer webServer; - @Mock private ServiceStatus serviceStatus; @Mock private BlockReader blockReader; @Mock private BlockWriter blockWriter; @@ -133,19 +139,18 @@ public class BlockStreamServiceIntegrationTest { private Path testPath; private BlockNodeContext blockNodeContext; - private PersistenceStorageConfig testConfig; - private static final int testTimeout = 1000; + private static final int testTimeout = 2000; @BeforeEach public void setUp() throws IOException { testPath = Files.createTempDirectory(TEMP_DIR); LOGGER.log(INFO, "Created temp directory: " + testPath.toString()); - blockNodeContext = - TestConfigUtil.getTestBlockNodeContext( - Map.of("persistence.storage.rootPath", testPath.toString())); - testConfig = blockNodeContext.configuration().getConfigData(PersistenceStorageConfig.class); + Map properties = new HashMap<>(); + properties.put("persistence.storage.rootPath", testPath.toString()); + + blockNodeContext = TestConfigUtil.getTestBlockNodeContext(properties); } @AfterEach @@ -157,58 +162,123 @@ public void tearDown() { public void testPublishBlockStreamRegistrationAndExecution() throws IOException, NoSuchAlgorithmException { + final ServiceStatus serviceStatus = new ServiceStatusImpl(blockNodeContext); + final var streamMediator = + LiveStreamMediatorBuilder.newBuilder(blockNodeContext, serviceStatus).build(); + final var notifier = new NotifierImpl(streamMediator, blockNodeContext, serviceStatus); + final var blockNodeEventHandler = + new StreamPersistenceHandlerImpl( + streamMediator, notifier, blockWriter, blockNodeContext, serviceStatus); final BlockStreamService blockStreamService = new BlockStreamService( - streamMediator, blockReader, serviceStatus, blockNodeContext); + streamMediator, + blockReader, + serviceStatus, + blockNodeEventHandler, + notifier, + blockNodeContext); + + // Register 3 producers + final StreamObserver + publishStreamObserver = + blockStreamService.protocPublishBlockStream(publishStreamResponseObserver1); + blockStreamService.protocPublishBlockStream(publishStreamResponseObserver2); + blockStreamService.protocPublishBlockStream(publishStreamResponseObserver3); - // Enable the serviceStatus - when(serviceStatus.isRunning()).thenReturn(true); + // Register 3 consumers + blockStreamService.protocSubscribeBlockStream( + subscribeStreamRequest, subscribeStreamObserver1); + blockStreamService.protocSubscribeBlockStream( + subscribeStreamRequest, subscribeStreamObserver2); + blockStreamService.protocSubscribeBlockStream( + subscribeStreamRequest, subscribeStreamObserver3); - final StreamObserver streamObserver = - blockStreamService.protocPublishBlockStream(publishStreamResponseObserver); + List blockItems = generateBlockItems(1); + for (int i = 0; i < blockItems.size(); i++) { + if (i == 9) { + when(blockWriter.write(blockItems.get(i))) + .thenReturn(Optional.of(blockItems.get(i))); + } else { + when(blockWriter.write(blockItems.get(i))).thenReturn(Optional.empty()); + } + } - final BlockItem blockItem = generateBlockItems(1).getFirst(); - final PublishStreamRequest publishStreamRequest = - PublishStreamRequest.newBuilder().blockItem(blockItem).build(); + for (BlockItem blockItem : blockItems) { + final PublishStreamRequest publishStreamRequest = + PublishStreamRequest.newBuilder().blockItem(blockItem).build(); - // Calling onNext() as Helidon will - streamObserver.onNext(fromPbj(publishStreamRequest)); + // Calling onNext() as Helidon does with each block item for + // the first producer. + publishStreamObserver.onNext(fromPbj(publishStreamRequest)); + } + + // Verify all 10 BlockItems were sent to each of the 3 consumers + verify(subscribeStreamObserver1, timeout(testTimeout).times(1)) + .onNext(fromPbj(buildSubscribeStreamResponse(blockItems.getFirst()))); + verify(subscribeStreamObserver1, timeout(testTimeout).times(8)) + .onNext(fromPbj(buildSubscribeStreamResponse(blockItems.get(1)))); + verify(subscribeStreamObserver1, timeout(testTimeout).times(1)) + .onNext(fromPbj(buildSubscribeStreamResponse(blockItems.get(9)))); + + verify(subscribeStreamObserver2, timeout(testTimeout).times(1)) + .onNext(fromPbj(buildSubscribeStreamResponse(blockItems.getFirst()))); + verify(subscribeStreamObserver2, timeout(testTimeout).times(8)) + .onNext(fromPbj(buildSubscribeStreamResponse(blockItems.get(1)))); + verify(subscribeStreamObserver2, timeout(testTimeout).times(1)) + .onNext(fromPbj(buildSubscribeStreamResponse(blockItems.get(9)))); + + verify(subscribeStreamObserver3, timeout(testTimeout).times(1)) + .onNext(fromPbj(buildSubscribeStreamResponse(blockItems.getFirst()))); + verify(subscribeStreamObserver3, timeout(testTimeout).times(8)) + .onNext(fromPbj(buildSubscribeStreamResponse(blockItems.get(1)))); + verify(subscribeStreamObserver3, timeout(testTimeout).times(1)) + .onNext(fromPbj(buildSubscribeStreamResponse(blockItems.get(9)))); - final Acknowledgement itemAck = buildAck(blockItem); + // Only 1 response is expected per block sent + final Acknowledgement itemAck = buildAck(blockItems.get(9)); final PublishStreamResponse publishStreamResponse = PublishStreamResponse.newBuilder().acknowledgement(itemAck).build(); - // Verify the BlockItem message is sent to the mediator - verify(streamMediator, timeout(testTimeout).times(1)).publish(blockItem); + // Verify all 3 producers received the response + verify(publishStreamResponseObserver1, timeout(testTimeout).times(1)) + .onNext(fromPbj(publishStreamResponse)); + + verify(publishStreamResponseObserver2, timeout(testTimeout).times(1)) + .onNext(fromPbj(publishStreamResponse)); - // Verify our custom StreamObserver implementation builds and sends - // a response back to the producer - verify(publishStreamResponseObserver, timeout(testTimeout).times(1)) + verify(publishStreamResponseObserver3, timeout(testTimeout).times(1)) .onNext(fromPbj(publishStreamResponse)); // Close the stream as Helidon does - streamObserver.onCompleted(); + publishStreamObserver.onCompleted(); // verify the onCompleted() method is invoked on the wrapped StreamObserver - verify(publishStreamResponseObserver, timeout(testTimeout).times(1)).onCompleted(); + verify(publishStreamResponseObserver1, timeout(testTimeout).times(1)).onCompleted(); } @Test public void testSubscribeBlockStream() throws IOException { - final ServiceStatus serviceStatus = new ServiceStatusImpl(); + final ServiceStatus serviceStatus = new ServiceStatusImpl(blockNodeContext); serviceStatus.setWebServer(webServer); final BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); final var streamMediator = - LiveStreamMediatorBuilder.newBuilder(blockWriter, blockNodeContext, serviceStatus) - .build(); + LiveStreamMediatorBuilder.newBuilder(blockNodeContext, serviceStatus).build(); // Build the BlockStreamService + final var blockNodeEventHandler = + new StreamPersistenceHandlerImpl( + streamMediator, notifier, blockWriter, blockNodeContext, serviceStatus); final BlockStreamService blockStreamService = new BlockStreamService( - streamMediator, blockReader, serviceStatus, blockNodeContext); + streamMediator, + blockReader, + serviceStatus, + blockNodeEventHandler, + notifier, + blockNodeContext); // Subscribe the consumers blockStreamService.protocSubscribeBlockStream( @@ -220,7 +290,7 @@ public void testSubscribeBlockStream() throws IOException { // Subscribe the producer final StreamObserver streamObserver = - blockStreamService.protocPublishBlockStream(publishStreamResponseObserver); + blockStreamService.protocPublishBlockStream(publishStreamResponseObserver1); // Build the BlockItem final List blockItems = generateBlockItems(1); @@ -250,16 +320,13 @@ public void testSubscribeBlockStream() throws IOException { public void testFullHappyPath() throws IOException { int numberOfBlocks = 100; - final BlockStreamService blockStreamService = buildBlockStreamService(); - - // Enable the serviceStatus - when(serviceStatus.isRunning()).thenReturn(true); + final BlockWriter blockWriter = + BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); + final BlockStreamService blockStreamService = buildBlockStreamService(blockWriter); // Pass a StreamObserver to the producer as Helidon does final StreamObserver streamObserver = - blockStreamService.protocPublishBlockStream(publishStreamResponseObserver); - - final List blockItems = generateBlockItems(numberOfBlocks); + blockStreamService.protocPublishBlockStream(publishStreamResponseObserver1); blockStreamService.protocSubscribeBlockStream( subscribeStreamRequest, subscribeStreamObserver1); @@ -268,6 +335,7 @@ public void testFullHappyPath() throws IOException { blockStreamService.protocSubscribeBlockStream( subscribeStreamRequest, subscribeStreamObserver3); + final List blockItems = generateBlockItems(numberOfBlocks); for (BlockItem blockItem : blockItems) { final PublishStreamRequest publishStreamRequest = PublishStreamRequest.newBuilder().blockItem(blockItem).build(); @@ -282,21 +350,20 @@ public void testFullHappyPath() throws IOException { numberOfBlocks, 0, numberOfBlocks, subscribeStreamObserver3, blockItems); streamObserver.onCompleted(); + + verify(publishStreamResponseObserver1, timeout(testTimeout).times(100)).onNext(any()); } @Test - public void testFullWithSubscribersAddedDynamically() throws IOException { + public void testFullWithSubscribersAddedDynamically() { int numberOfBlocks = 100; - final BlockStreamService blockStreamService = buildBlockStreamService(); - - // Enable the serviceStatus - when(serviceStatus.isRunning()).thenReturn(true); + final BlockStreamService blockStreamService = buildBlockStreamService(blockWriter); // Pass a StreamObserver to the producer as Helidon does final StreamObserver streamObserver = - blockStreamService.protocPublishBlockStream(publishStreamResponseObserver); + blockStreamService.protocPublishBlockStream(publishStreamResponseObserver1); final List blockItems = generateBlockItems(numberOfBlocks); @@ -358,24 +425,32 @@ public void testFullWithSubscribersAddedDynamically() throws IOException { } @Test - public void testSubAndUnsubWhileStreaming() throws IOException { + public void testSubAndUnsubWhileStreaming() throws InterruptedException { int numberOfBlocks = 100; final LinkedHashMap< - EventHandler>, + BlockNodeEventHandler>, BatchEventProcessor>> - subscribers = new LinkedHashMap<>(); - final var streamMediator = buildStreamMediator(subscribers, FileUtils.defaultPerms); - final var blockStreamService = - buildBlockStreamService(streamMediator, blockReader, serviceStatus); + consumers = new LinkedHashMap<>(); - // Enable the serviceStatus - when(serviceStatus.isRunning()).thenReturn(true); + final ServiceStatus serviceStatus = new ServiceStatusImpl(blockNodeContext); + final var streamMediator = buildStreamMediator(consumers, serviceStatus); + final var blockNodeEventHandler = + new StreamPersistenceHandlerImpl( + streamMediator, notifier, blockWriter, blockNodeContext, serviceStatus); + final var blockStreamService = + new BlockStreamService( + streamMediator, + blockReader, + serviceStatus, + blockNodeEventHandler, + notifier, + blockNodeContext); // Pass a StreamObserver to the producer as Helidon does final StreamObserver streamObserver = - blockStreamService.protocPublishBlockStream(publishStreamResponseObserver); + blockStreamService.protocPublishBlockStream(publishStreamResponseObserver1); final List blockItems = generateBlockItems(numberOfBlocks); @@ -387,17 +462,29 @@ public void testSubAndUnsubWhileStreaming() throws IOException { subscribeStreamRequest, subscribeStreamObserver3); for (int i = 0; i < blockItems.size(); i++) { - final PublishStreamRequest publishStreamRequest = - PublishStreamRequest.newBuilder().blockItem(blockItems.get(i)).build(); - // Remove a subscriber + // Transmit the BlockItem + streamObserver.onNext( + fromPbj( + PublishStreamRequest.newBuilder() + .blockItem(blockItems.get(i)) + .build())); + + // Remove 1st subscriber if (i == 10) { - final var k = subscribers.firstEntry().getKey(); + // Pause here to ensure the last sent block item is received. + // This makes the test deterministic. + Thread.sleep(50); + final var k = consumers.firstEntry().getKey(); streamMediator.unsubscribe(k); } + // Remove 2nd subscriber if (i == 60) { - final var k = subscribers.firstEntry().getKey(); + // Pause here to ensure the last sent block item is received. + // This makes the test deterministic. + Thread.sleep(50); + final var k = consumers.firstEntry().getKey(); streamMediator.unsubscribe(k); } @@ -407,11 +494,12 @@ public void testSubAndUnsubWhileStreaming() throws IOException { subscribeStreamRequest, subscribeStreamObserver4); } - // Transmit the BlockItem - streamObserver.onNext(fromPbj(publishStreamRequest)); - + // Remove 3rd subscriber if (i == 70) { - final var k = subscribers.firstEntry().getKey(); + // Pause here to ensure the last sent block item is received. + // This makes the test deterministic. + Thread.sleep(50); + final var k = consumers.firstEntry().getKey(); streamMediator.unsubscribe(k); } @@ -449,23 +537,39 @@ public void testSubAndUnsubWhileStreaming() throws IOException { } @Test - public void testMediatorExceptionHandlingWhenPersistenceFailure() - throws IOException, ParseException { + public void testMediatorExceptionHandlingWhenPersistenceFailure() throws IOException { final ConcurrentHashMap< - EventHandler>, + BlockNodeEventHandler>, BatchEventProcessor>> - subscribers = new ConcurrentHashMap<>(); + consumers = new ConcurrentHashMap<>(); - // Initialize the underlying BlockReader and BlockWriter with ineffective - // permissions to repair the file system. The BlockWriter will not be able - // to write the BlockItem or fix the permissions causing the BlockReader to - // throw an IOException. - final ServiceStatus serviceStatus = new ServiceStatusImpl(); + // Use a spy to use the real object but also verify the behavior. + final ServiceStatus serviceStatus = spy(new ServiceStatusImpl(blockNodeContext)); + doCallRealMethod().when(serviceStatus).setWebServer(webServer); + doCallRealMethod().when(serviceStatus).isRunning(); + doCallRealMethod().when(serviceStatus).stopWebServer(any()); serviceStatus.setWebServer(webServer); - final var streamMediator = buildStreamMediator(subscribers, TestUtils.getNoPerms()); + final List blockItems = generateBlockItems(1); + + // Use a spy to make sure the write() method throws an IOException + final BlockWriter blockWriter = + spy(BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build()); + doThrow(IOException.class).when(blockWriter).write(blockItems.getFirst()); + + final var streamMediator = buildStreamMediator(consumers, serviceStatus); + final var notifier = new NotifierImpl(streamMediator, blockNodeContext, serviceStatus); + final var blockNodeEventHandler = + new StreamPersistenceHandlerImpl( + streamMediator, notifier, blockWriter, blockNodeContext, serviceStatus); final var blockStreamService = - buildBlockStreamService(streamMediator, blockReader, serviceStatus); + new BlockStreamService( + streamMediator, + blockReader, + serviceStatus, + blockNodeEventHandler, + notifier, + blockNodeContext); // Subscribe the consumers blockStreamService.protocSubscribeBlockStream( @@ -477,26 +581,27 @@ public void testMediatorExceptionHandlingWhenPersistenceFailure() // Initialize the producer final StreamObserver streamObserver = - blockStreamService.protocPublishBlockStream(publishStreamResponseObserver); - - // Change the permissions on the file system to trigger an - // IOException when the BlockPersistenceHandler tries to write - // the first BlockItem to the file system. - removeRootPathWritePerms(testConfig); + blockStreamService.protocPublishBlockStream(publishStreamResponseObserver1); // Transmit a BlockItem - final List blockItems = generateBlockItems(1); final PublishStreamRequest publishStreamRequest = PublishStreamRequest.newBuilder().blockItem(blockItems.getFirst()).build(); streamObserver.onNext(fromPbj(publishStreamRequest)); + // Use verify to make sure the serviceStatus.stopRunning() method is called + // before the next block is transmitted. + verify(serviceStatus, timeout(testTimeout).times(2)).stopRunning(any()); + // Simulate another producer attempting to connect to the Block Node after the exception. // Later, verify they received a response indicating the stream is closed. final StreamObserver expectedNoOpStreamObserver = - blockStreamService.protocPublishBlockStream(publishStreamResponseObserver); + blockStreamService.protocPublishBlockStream(publishStreamResponseObserver2); expectedNoOpStreamObserver.onNext(fromPbj(publishStreamRequest)); + verify(publishStreamResponseObserver2, timeout(testTimeout).times(1)) + .onNext(fromPbj(buildEndOfStreamResponse())); + // Build a request to invoke the singleBlock service final com.hedera.hapi.block.protoc.SingleBlockRequest singleBlockRequest = com.hedera.hapi.block.protoc.SingleBlockRequest.newBuilder() @@ -513,7 +618,7 @@ public void testMediatorExceptionHandlingWhenPersistenceFailure() blockStreamService.protocSubscribeBlockStream( fromPbj(subscribeStreamRequest), subscribeStreamObserver4); - // The BlockItem passed through since it was published + // The BlockItem expected to pass through since it was published // before the IOException was thrown. final SubscribeStreamResponse subscribeStreamResponse = SubscribeStreamResponse.newBuilder().blockItem(blockItems.getFirst()).build(); @@ -537,28 +642,16 @@ public void testMediatorExceptionHandlingWhenPersistenceFailure() verify(subscribeStreamObserver3, timeout(testTimeout).times(1)) .onNext(fromPbj(endStreamResponse)); - // Verify all the consumers were unsubscribed - for (final var s : subscribers.keySet()) { - assertFalse(streamMediator.isSubscribed(s)); - } + assertEquals(3, consumers.size()); // Verify the publishBlockStream service returned the expected // error code indicating the service is not available. - final EndOfStream endOfStream = - EndOfStream.newBuilder() - .status(PublishStreamResponseCode.STREAM_ITEMS_UNKNOWN) - .build(); - final var endOfStreamResponse = - PublishStreamResponse.newBuilder().status(endOfStream).build(); - verify(publishStreamResponseObserver, timeout(testTimeout).times(2)) - .onNext(fromPbj(endOfStreamResponse)); - verify(webServer, timeout(testTimeout).times(1)).stop(); - - // Now verify the block was removed from the file system. - final BlockReader blockReader = - BlockAsDirReaderBuilder.newBuilder(testConfig).build(); - final Optional blockOpt = blockReader.read(1); - assertTrue(blockOpt.isEmpty()); + verify(publishStreamResponseObserver1, timeout(testTimeout).times(1)) + .onNext(fromPbj(buildEndOfStreamResponse())); + + // Adding extra time to allow the service to stop given + // the built-in delay. + verify(webServer, timeout(2000).times(1)).stop(); // Verify the singleBlock service returned the expected // error code indicating the service is not available. @@ -579,12 +672,6 @@ public void testMediatorExceptionHandlingWhenPersistenceFailure() .onNext(fromPbj(expectedSubscriberStreamNotAvailable)); } - private void removeRootPathWritePerms(final PersistenceStorageConfig config) - throws IOException { - final Path blockNodeRootPath = Path.of(config.rootPath()); - Files.setPosixFilePermissions(blockNodeRootPath, TestUtils.getNoWrite().value()); - } - private static void verifySubscribeStreamResponse( int numberOfBlocks, int blockItemsToWait, @@ -625,50 +712,46 @@ private static SubscribeStreamResponse buildSubscribeStreamResponse(BlockItem bl return SubscribeStreamResponse.newBuilder().blockItem(blockItem).build(); } - private BlockStreamService buildBlockStreamService() throws IOException { - final var streamMediator = - buildStreamMediator(new ConcurrentHashMap<>(32), FileUtils.defaultPerms); + private static PublishStreamResponse buildEndOfStreamResponse() { + final EndOfStream endOfStream = + EndOfStream.newBuilder() + .status(PublishStreamResponseCode.STREAM_ITEMS_UNKNOWN) + .build(); + return PublishStreamResponse.newBuilder().status(endOfStream).build(); + } - return buildBlockStreamService(streamMediator, blockReader, serviceStatus); + private BlockStreamService buildBlockStreamService(final BlockWriter blockWriter) { + + final ServiceStatus serviceStatus = new ServiceStatusImpl(blockNodeContext); + final var streamMediator = buildStreamMediator(new ConcurrentHashMap<>(32), serviceStatus); + final var notifier = new NotifierImpl(streamMediator, blockNodeContext, serviceStatus); + final var blockNodeEventHandler = + new StreamPersistenceHandlerImpl( + streamMediator, notifier, blockWriter, blockNodeContext, serviceStatus); + + return new BlockStreamService( + streamMediator, + blockReader, + serviceStatus, + blockNodeEventHandler, + notifier, + blockNodeContext); } - private StreamMediator> buildStreamMediator( + private LiveStreamMediator buildStreamMediator( final Map< - EventHandler>, + BlockNodeEventHandler>, BatchEventProcessor>> subscribers, - final FileAttribute> filePerms) - throws IOException { - - // Initialize with concrete a concrete BlockReader, BlockWriter and Mediator - final BlockRemover blockRemover = - new BlockAsDirRemover(Path.of(testConfig.rootPath()), FileUtils.defaultPerms); - - final BlockWriter blockWriter = - BlockAsDirWriterBuilder.newBuilder(blockNodeContext) - .blockRemover(blockRemover) - .filePerms(filePerms) - .build(); + final ServiceStatus serviceStatus) { - final ServiceStatus serviceStatus = new ServiceStatusImpl(); serviceStatus.setWebServer(webServer); - return LiveStreamMediatorBuilder.newBuilder(blockWriter, blockNodeContext, serviceStatus) + return LiveStreamMediatorBuilder.newBuilder(blockNodeContext, serviceStatus) .subscribers(subscribers) .build(); } - private BlockStreamService buildBlockStreamService( - final StreamMediator> streamMediator, - final BlockReader blockReader, - final ServiceStatus serviceStatus) - throws IOException { - - final BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); - - return new BlockStreamService(streamMediator, blockReader, serviceStatus, blockNodeContext); - } - public static Acknowledgement buildAck(@NonNull final BlockItem blockItem) throws NoSuchAlgorithmException { ItemAcknowledgement itemAck = diff --git a/server/src/test/java/com/hedera/block/server/BlockStreamServiceTest.java b/server/src/test/java/com/hedera/block/server/BlockStreamServiceTest.java index 5b11f516..0e624ad3 100644 --- a/server/src/test/java/com/hedera/block/server/BlockStreamServiceTest.java +++ b/server/src/test/java/com/hedera/block/server/BlockStreamServiceTest.java @@ -39,20 +39,20 @@ import static org.mockito.Mockito.when; import com.google.protobuf.Descriptors; -import com.google.protobuf.InvalidProtocolBufferException; import com.hedera.block.server.config.BlockNodeContext; -import com.hedera.block.server.data.ObjectEvent; -import com.hedera.block.server.mediator.StreamMediator; +import com.hedera.block.server.mediator.LiveStreamMediator; +import com.hedera.block.server.notifier.Notifier; +import com.hedera.block.server.persistence.StreamPersistenceHandlerImpl; import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; import com.hedera.block.server.persistence.storage.read.BlockAsDirReaderBuilder; import com.hedera.block.server.persistence.storage.read.BlockReader; import com.hedera.block.server.persistence.storage.write.BlockAsDirWriterBuilder; import com.hedera.block.server.persistence.storage.write.BlockWriter; +import com.hedera.block.server.service.ServiceStatus; import com.hedera.block.server.util.TestConfigUtil; import com.hedera.block.server.util.TestUtils; import com.hedera.hapi.block.SingleBlockResponse; import com.hedera.hapi.block.SingleBlockResponseCode; -import com.hedera.hapi.block.SubscribeStreamResponse; import com.hedera.hapi.block.stream.Block; import com.hedera.hapi.block.stream.BlockItem; import com.hedera.pbj.runtime.ParseException; @@ -62,7 +62,6 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.security.NoSuchAlgorithmException; import java.util.List; import java.util.Map; import java.util.Optional; @@ -76,12 +75,16 @@ @ExtendWith(MockitoExtension.class) public class BlockStreamServiceTest { + @Mock private Notifier notifier; + @Mock private StreamObserver responseObserver; - @Mock private StreamMediator> streamMediator; + @Mock private LiveStreamMediator streamMediator; @Mock private BlockReader blockReader; + @Mock private BlockWriter blockWriter; + @Mock private ServiceStatus serviceStatus; private final Logger LOGGER = System.getLogger(getClass().getName()); @@ -111,11 +114,19 @@ public void tearDown() { } @Test - public void testServiceName() throws IOException, NoSuchAlgorithmException { + public void testServiceName() { + final var blockNodeEventHandler = + new StreamPersistenceHandlerImpl( + streamMediator, notifier, blockWriter, blockNodeContext, serviceStatus); final BlockStreamService blockStreamService = new BlockStreamService( - streamMediator, blockReader, serviceStatus, blockNodeContext); + streamMediator, + blockReader, + serviceStatus, + blockNodeEventHandler, + notifier, + blockNodeContext); // Verify the service name assertEquals(Constants.SERVICE_NAME, blockStreamService.serviceName()); @@ -125,10 +136,19 @@ public void testServiceName() throws IOException, NoSuchAlgorithmException { } @Test - public void testProto() throws IOException, NoSuchAlgorithmException { + public void testProto() { + + final var blockNodeEventHandler = + new StreamPersistenceHandlerImpl( + streamMediator, notifier, blockWriter, blockNodeContext, serviceStatus); final BlockStreamService blockStreamService = new BlockStreamService( - streamMediator, blockReader, serviceStatus, blockNodeContext); + streamMediator, + blockReader, + serviceStatus, + blockNodeEventHandler, + notifier, + blockNodeContext); Descriptors.FileDescriptor fileDescriptor = blockStreamService.proto(); // Verify the current rpc methods @@ -141,10 +161,18 @@ public void testProto() throws IOException, NoSuchAlgorithmException { @Test void testSingleBlockHappyPath() throws IOException, ParseException { + final var blockNodeEventHandler = + new StreamPersistenceHandlerImpl( + streamMediator, notifier, blockWriter, blockNodeContext, serviceStatus); final BlockReader blockReader = BlockAsDirReaderBuilder.newBuilder(config).build(); final BlockStreamService blockStreamService = new BlockStreamService( - streamMediator, blockReader, serviceStatus, blockNodeContext); + streamMediator, + blockReader, + serviceStatus, + blockNodeEventHandler, + notifier, + blockNodeContext); // Enable the serviceStatus when(serviceStatus.isRunning()).thenReturn(true); @@ -196,9 +224,17 @@ void testSingleBlockNotFoundPath() throws IOException, ParseException { .build(); // Call the service + final var blockNodeEventHandler = + new StreamPersistenceHandlerImpl( + streamMediator, notifier, blockWriter, blockNodeContext, serviceStatus); final BlockStreamService blockStreamService = new BlockStreamService( - streamMediator, blockReader, serviceStatus, blockNodeContext); + streamMediator, + blockReader, + serviceStatus, + blockNodeEventHandler, + notifier, + blockNodeContext); // Enable the serviceStatus when(serviceStatus.isRunning()).thenReturn(true); @@ -208,11 +244,19 @@ void testSingleBlockNotFoundPath() throws IOException, ParseException { } @Test - void testSingleBlockServiceNotAvailable() throws InvalidProtocolBufferException { + void testSingleBlockServiceNotAvailable() { + final var blockNodeEventHandler = + new StreamPersistenceHandlerImpl( + streamMediator, notifier, blockWriter, blockNodeContext, serviceStatus); final BlockStreamService blockStreamService = new BlockStreamService( - streamMediator, blockReader, serviceStatus, blockNodeContext); + streamMediator, + blockReader, + serviceStatus, + blockNodeEventHandler, + notifier, + blockNodeContext); // Set the service status to not running when(serviceStatus.isRunning()).thenReturn(false); @@ -231,9 +275,17 @@ void testSingleBlockServiceNotAvailable() throws InvalidProtocolBufferException @Test public void testSingleBlockIOExceptionPath() throws IOException, ParseException { + final var blockNodeEventHandler = + new StreamPersistenceHandlerImpl( + streamMediator, notifier, blockWriter, blockNodeContext, serviceStatus); final BlockStreamService blockStreamService = new BlockStreamService( - streamMediator, blockReader, serviceStatus, blockNodeContext); + streamMediator, + blockReader, + serviceStatus, + blockNodeEventHandler, + notifier, + blockNodeContext); when(serviceStatus.isRunning()).thenReturn(true); when(blockReader.read(1)).thenThrow(new IOException("Test exception")); @@ -252,9 +304,17 @@ public void testSingleBlockIOExceptionPath() throws IOException, ParseException @Test public void testSingleBlockParseExceptionPath() throws IOException, ParseException { + final var blockNodeEventHandler = + new StreamPersistenceHandlerImpl( + streamMediator, notifier, blockWriter, blockNodeContext, serviceStatus); final BlockStreamService blockStreamService = new BlockStreamService( - streamMediator, blockReader, serviceStatus, blockNodeContext); + streamMediator, + blockReader, + serviceStatus, + blockNodeEventHandler, + notifier, + blockNodeContext); when(serviceStatus.isRunning()).thenReturn(true); when(blockReader.read(1)).thenThrow(new ParseException("Test exception")); @@ -274,9 +334,7 @@ public void testSingleBlockParseExceptionPath() throws IOException, ParseExcepti @Test public void testUpdateInvokesRoutingWithLambdas() { - final BlockStreamService blockStreamService = - new BlockStreamService( - streamMediator, blockReader, serviceStatus, blockNodeContext); + final BlockStreamService blockStreamService = getBlockStreamService(); GrpcService.Routing routing = mock(GrpcService.Routing.class); blockStreamService.update(routing); @@ -291,13 +349,36 @@ public void testUpdateInvokesRoutingWithLambdas() { .unary(eq(SINGLE_BLOCK_METHOD_NAME), any(ServerCalls.UnaryMethod.class)); } + private BlockStreamService getBlockStreamService() { + final var blockNodeEventHandler = + new StreamPersistenceHandlerImpl( + streamMediator, notifier, blockWriter, blockNodeContext, serviceStatus); + final BlockStreamService blockStreamService = + new BlockStreamService( + streamMediator, + blockReader, + serviceStatus, + blockNodeEventHandler, + notifier, + blockNodeContext); + return blockStreamService; + } + @Test public void testProtocParseExceptionHandling() { // TODO: We might be able to remove this test once we can remove the Translator class + final var blockNodeEventHandler = + new StreamPersistenceHandlerImpl( + streamMediator, notifier, blockWriter, blockNodeContext, serviceStatus); final BlockStreamService blockStreamService = new BlockStreamService( - streamMediator, blockReader, serviceStatus, blockNodeContext); + streamMediator, + blockReader, + serviceStatus, + blockNodeEventHandler, + notifier, + blockNodeContext); // Build a request to invoke the service final com.hedera.hapi.block.protoc.SingleBlockRequest singleBlockRequest = diff --git a/server/src/test/java/com/hedera/block/server/config/ConfigInjectionModuleTest.java b/server/src/test/java/com/hedera/block/server/config/ConfigInjectionModuleTest.java index 6c9d7505..898fe0b7 100644 --- a/server/src/test/java/com/hedera/block/server/config/ConfigInjectionModuleTest.java +++ b/server/src/test/java/com/hedera/block/server/config/ConfigInjectionModuleTest.java @@ -19,6 +19,8 @@ import static org.junit.jupiter.api.Assertions.*; import com.hedera.block.server.consumer.ConsumerConfig; +import com.hedera.block.server.mediator.MediatorConfig; +import com.hedera.block.server.notifier.NotifierConfig; import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; import com.hedera.block.server.util.TestConfigUtil; import com.swirlds.common.metrics.config.MetricsConfig; @@ -91,4 +93,30 @@ void testProvideConsumerConfig() throws IOException { assertNotNull(providedConfig); assertSame(consumerConfig, providedConfig); } + + @Test + void testProvideMediatorConfig() throws IOException { + BlockNodeContext context = TestConfigUtil.getTestBlockNodeContext(); + Configuration configuration = context.configuration(); + MediatorConfig mediatorConfig = configuration.getConfigData(MediatorConfig.class); + + MediatorConfig providedConfig = ConfigInjectionModule.provideMediatorConfig(configuration); + + // Verify the config + assertNotNull(providedConfig); + assertSame(mediatorConfig, providedConfig); + } + + @Test + void testProvideNotifierConfig() throws IOException { + BlockNodeContext context = TestConfigUtil.getTestBlockNodeContext(); + Configuration configuration = context.configuration(); + NotifierConfig notifierConfig = configuration.getConfigData(NotifierConfig.class); + + NotifierConfig providedConfig = ConfigInjectionModule.provideNotifierConfig(configuration); + + // Verify the config + assertNotNull(providedConfig); + assertSame(notifierConfig, providedConfig); + } } diff --git a/server/src/test/java/com/hedera/block/server/consumer/ConsumerConfigTest.java b/server/src/test/java/com/hedera/block/server/consumer/ConsumerConfigTest.java new file mode 100644 index 00000000..6b68fdfd --- /dev/null +++ b/server/src/test/java/com/hedera/block/server/consumer/ConsumerConfigTest.java @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.consumer; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.junit.jupiter.api.Test; + +public class ConsumerConfigTest { + + @Test + public void testConsumerConfig_happyPath() { + ConsumerConfig consumerConfig = new ConsumerConfig(3000); + assert (consumerConfig.timeoutThresholdMillis() == 3000); + } + + @Test + public void testConsumerConfig_negativeTimeoutThresholdMillis() { + IllegalArgumentException exception = + assertThrows(IllegalArgumentException.class, () -> new ConsumerConfig(-1)); + assertEquals("Timeout threshold must be greater than 0", exception.getMessage()); + } +} diff --git a/server/src/test/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserverTest.java b/server/src/test/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserverTest.java index 444e8180..6eb7053f 100644 --- a/server/src/test/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserverTest.java +++ b/server/src/test/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserverTest.java @@ -26,7 +26,7 @@ import static org.mockito.Mockito.when; import com.hedera.block.server.config.BlockNodeContext; -import com.hedera.block.server.data.ObjectEvent; +import com.hedera.block.server.events.ObjectEvent; import com.hedera.block.server.mediator.StreamMediator; import com.hedera.block.server.util.TestConfigUtil; import com.hedera.hapi.block.SubscribeStreamResponse; @@ -35,6 +35,7 @@ import com.hedera.hapi.block.stream.input.EventHeader; import com.hedera.hapi.block.stream.output.BlockHeader; import com.hedera.hapi.platform.event.EventCore; +import edu.umd.cs.findbugs.annotations.NonNull; import io.grpc.stub.ServerCallStreamObserver; import io.grpc.stub.StreamObserver; import java.io.IOException; @@ -54,7 +55,7 @@ public class ConsumerStreamResponseObserverTest { private static final int testTimeout = 1000; - @Mock private StreamMediator> streamMediator; + @Mock private StreamMediator streamMediator; @Mock private StreamObserver @@ -85,7 +86,7 @@ public void testProducerTimeoutWithinWindow() { final var consumerBlockItemObserver = new ConsumerStreamResponseObserver( - testContext, testClock, streamMediator, responseStreamObserver); + testClock, streamMediator, responseStreamObserver, testContext); final BlockHeader blockHeader = BlockHeader.newBuilder().number(1).build(); final BlockItem blockItem = BlockItem.newBuilder().blockHeader(blockHeader).build(); @@ -113,7 +114,7 @@ public void testProducerTimeoutOutsideWindow() throws InterruptedException { final var consumerBlockItemObserver = new ConsumerStreamResponseObserver( - testContext, testClock, streamMediator, responseStreamObserver); + testClock, streamMediator, responseStreamObserver, testContext); consumerBlockItemObserver.onEvent(objectEvent, 0, true); verify(streamMediator).unsubscribe(consumerBlockItemObserver); @@ -127,7 +128,7 @@ public void testHandlersSetOnObserver() throws InterruptedException { when(testClock.millis()).thenReturn(TEST_TIME, TEST_TIME + TIMEOUT_THRESHOLD_MILLIS); new ConsumerStreamResponseObserver( - testContext, testClock, streamMediator, serverCallStreamObserver); + testClock, streamMediator, serverCallStreamObserver, testContext); verify(serverCallStreamObserver, timeout(testTimeout).times(1)).setOnCloseHandler(any()); verify(serverCallStreamObserver, timeout(testTimeout).times(1)).setOnCancelHandler(any()); @@ -138,7 +139,7 @@ public void testResponseNotPermittedAfterCancel() { final TestConsumerStreamResponseObserver consumerStreamResponseObserver = new TestConsumerStreamResponseObserver( - testContext, testClock, streamMediator, serverCallStreamObserver); + testClock, streamMediator, serverCallStreamObserver, testContext); final List blockItems = generateBlockItems(1); final SubscribeStreamResponse subscribeStreamResponse = @@ -164,7 +165,7 @@ public void testResponseNotPermittedAfterClose() { final TestConsumerStreamResponseObserver consumerStreamResponseObserver = new TestConsumerStreamResponseObserver( - testContext, testClock, streamMediator, serverCallStreamObserver); + testClock, streamMediator, serverCallStreamObserver, testContext); final List blockItems = generateBlockItems(1); final SubscribeStreamResponse subscribeStreamResponse = @@ -180,7 +181,7 @@ public void testResponseNotPermittedAfterClose() { // Attempt to send another BlockItem consumerStreamResponseObserver.onEvent(objectEvent, 0, true); - // Confirm that canceling the observer allowed only 1 response to be sent. + // Confirm that closing the observer allowed only 1 response to be sent. verify(serverCallStreamObserver, timeout(testTimeout).times(1)) .onNext(fromPbj(subscribeStreamResponse)); } @@ -194,7 +195,7 @@ public void testConsumerNotToSendBeforeBlockHeader() { final var consumerBlockItemObserver = new ConsumerStreamResponseObserver( - testContext, testClock, streamMediator, responseStreamObserver); + testClock, streamMediator, responseStreamObserver, testContext); // Send non-header BlockItems to validate that the observer does not send them for (int i = 1; i <= 10; i++) { @@ -242,7 +243,7 @@ public void testSubscriberStreamResponseIsBlockItemWhenBlockItemIsNull() { final var consumerBlockItemObserver = new ConsumerStreamResponseObserver( - testContext, testClock, streamMediator, responseStreamObserver); + testClock, streamMediator, responseStreamObserver, testContext); assertThrows( IllegalArgumentException.class, () -> consumerBlockItemObserver.onEvent(objectEvent, 0, true)); @@ -257,7 +258,7 @@ public void testSubscribeStreamResponseTypeNotSupported() { final var consumerBlockItemObserver = new ConsumerStreamResponseObserver( - testContext, testClock, streamMediator, responseStreamObserver); + testClock, streamMediator, responseStreamObserver, testContext); assertThrows( IllegalArgumentException.class, @@ -267,16 +268,19 @@ public void testSubscribeStreamResponseTypeNotSupported() { private static class TestConsumerStreamResponseObserver extends ConsumerStreamResponseObserver { public TestConsumerStreamResponseObserver( - BlockNodeContext context, - InstantSource producerLivenessClock, - StreamMediator> subscriptionHandler, - StreamObserver - subscribeStreamResponseObserver) { + @NonNull final InstantSource producerLivenessClock, + @NonNull + final StreamMediator + subscriptionHandler, + @NonNull + final StreamObserver + subscribeStreamResponseObserver, + @NonNull final BlockNodeContext blockNodeContext) { super( - context, producerLivenessClock, subscriptionHandler, - subscribeStreamResponseObserver); + subscribeStreamResponseObserver, + blockNodeContext); } public void cancel() { diff --git a/server/src/test/java/com/hedera/block/server/health/HealthServiceTest.java b/server/src/test/java/com/hedera/block/server/health/HealthServiceTest.java index 55f91197..d87f257a 100644 --- a/server/src/test/java/com/hedera/block/server/health/HealthServiceTest.java +++ b/server/src/test/java/com/hedera/block/server/health/HealthServiceTest.java @@ -19,7 +19,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.*; -import com.hedera.block.server.ServiceStatus; +import com.hedera.block.server.service.ServiceStatus; import io.helidon.webserver.http.HttpRules; import io.helidon.webserver.http.ServerRequest; import io.helidon.webserver.http.ServerResponse; diff --git a/server/src/test/java/com/hedera/block/server/mediator/LiveStreamMediatorImplTest.java b/server/src/test/java/com/hedera/block/server/mediator/LiveStreamMediatorImplTest.java index 63e1813f..2b97dc0b 100644 --- a/server/src/test/java/com/hedera/block/server/mediator/LiveStreamMediatorImplTest.java +++ b/server/src/test/java/com/hedera/block/server/mediator/LiveStreamMediatorImplTest.java @@ -18,34 +18,43 @@ import static com.hedera.block.server.Translator.fromPbj; import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.LiveBlockItems; +import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.LiveBlockStreamMediatorError; import static com.hedera.block.server.util.PersistTestUtils.generateBlockItems; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.timeout; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import com.hedera.block.server.ServiceStatusImpl; import com.hedera.block.server.config.BlockNodeContext; import com.hedera.block.server.consumer.ConsumerStreamResponseObserver; -import com.hedera.block.server.data.ObjectEvent; +import com.hedera.block.server.events.BlockNodeEventHandler; +import com.hedera.block.server.events.ObjectEvent; +import com.hedera.block.server.metrics.BlockNodeMetricTypes; +import com.hedera.block.server.notifier.Notifier; +import com.hedera.block.server.notifier.NotifierImpl; +import com.hedera.block.server.persistence.StreamPersistenceHandlerImpl; import com.hedera.block.server.persistence.storage.write.BlockWriter; +import com.hedera.block.server.service.ServiceStatus; +import com.hedera.block.server.service.ServiceStatusImpl; import com.hedera.block.server.util.TestConfigUtil; import com.hedera.hapi.block.SubscribeStreamResponse; +import com.hedera.hapi.block.SubscribeStreamResponseCode; import com.hedera.hapi.block.stream.BlockItem; import com.hedera.hapi.block.stream.output.BlockHeader; -import com.lmax.disruptor.EventHandler; +import com.swirlds.metrics.api.LongGauge; import edu.umd.cs.findbugs.annotations.NonNull; import io.grpc.stub.ServerCallStreamObserver; import io.grpc.stub.StreamObserver; import java.io.IOException; import java.time.InstantSource; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -54,11 +63,12 @@ @ExtendWith(MockitoExtension.class) public class LiveStreamMediatorImplTest { - @Mock private EventHandler> observer1; - @Mock private EventHandler> observer2; - @Mock private EventHandler> observer3; + @Mock private BlockNodeEventHandler> observer1; + @Mock private BlockNodeEventHandler> observer2; + @Mock private BlockNodeEventHandler> observer3; @Mock private BlockWriter blockWriter; + @Mock private Notifier notifier; @Mock private StreamObserver streamObserver1; @@ -83,11 +93,13 @@ public class LiveStreamMediatorImplTest { private final BlockNodeContext testContext; public LiveStreamMediatorImplTest() throws IOException { - this.testContext = - TestConfigUtil.getTestBlockNodeContext( - Map.of( - TestConfigUtil.CONSUMER_TIMEOUT_THRESHOLD_KEY, - String.valueOf(TIMEOUT_THRESHOLD_MILLIS))); + Map properties = new HashMap<>(); + properties.put( + TestConfigUtil.CONSUMER_TIMEOUT_THRESHOLD_KEY, + String.valueOf(TIMEOUT_THRESHOLD_MILLIS)); + properties.put(TestConfigUtil.MEDIATOR_RING_BUFFER_SIZE_KEY, String.valueOf(1024)); + + this.testContext = TestConfigUtil.getTestBlockNodeContext(properties); } @Test @@ -96,7 +108,7 @@ public void testUnsubscribeEach() throws InterruptedException, IOException { final BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); final var streamMediatorBuilder = LiveStreamMediatorBuilder.newBuilder( - blockWriter, blockNodeContext, new ServiceStatusImpl()); + blockNodeContext, new ServiceStatusImpl(blockNodeContext)); final var streamMediator = streamMediatorBuilder.build(); // Set up the subscribers @@ -114,7 +126,7 @@ public void testUnsubscribeEach() throws InterruptedException, IOException { streamMediator.isSubscribed(observer3), "Expected the mediator to have observer3 subscribed"); - Thread.sleep(testTimeout); + Thread.sleep(100); streamMediator.unsubscribe(observer1); assertFalse( @@ -139,12 +151,18 @@ public void testUnsubscribeEach() throws InterruptedException, IOException { public void testMediatorPersistenceWithoutSubscribers() throws IOException { final BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); + final ServiceStatus serviceStatus = new ServiceStatusImpl(blockNodeContext); final var streamMediator = - LiveStreamMediatorBuilder.newBuilder( - blockWriter, blockNodeContext, new ServiceStatusImpl()) - .build(); + LiveStreamMediatorBuilder.newBuilder(blockNodeContext, serviceStatus).build(); final BlockItem blockItem = BlockItem.newBuilder().build(); + // register the stream validator + when(blockWriter.write(blockItem)).thenReturn(Optional.empty()); + final var streamValidator = + new StreamPersistenceHandlerImpl( + streamMediator, notifier, blockWriter, blockNodeContext, serviceStatus); + streamMediator.subscribe(streamValidator); + // Acting as a producer, notify the mediator of a new block streamMediator.publish(blockItem); @@ -160,24 +178,23 @@ blockWriter, blockNodeContext, new ServiceStatusImpl()) public void testMediatorPublishEventToSubscribers() throws IOException { final BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); + final ServiceStatus serviceStatus = new ServiceStatusImpl(blockNodeContext); final var streamMediator = - LiveStreamMediatorBuilder.newBuilder( - blockWriter, blockNodeContext, new ServiceStatusImpl()) - .build(); + LiveStreamMediatorBuilder.newBuilder(blockNodeContext, serviceStatus).build(); when(testClock.millis()).thenReturn(TEST_TIME, TEST_TIME + TIMEOUT_THRESHOLD_MILLIS); final var concreteObserver1 = new ConsumerStreamResponseObserver( - testContext, testClock, streamMediator, streamObserver1); + testClock, streamMediator, streamObserver1, testContext); final var concreteObserver2 = new ConsumerStreamResponseObserver( - testContext, testClock, streamMediator, streamObserver2); + testClock, streamMediator, streamObserver2, testContext); final var concreteObserver3 = new ConsumerStreamResponseObserver( - testContext, testClock, streamMediator, streamObserver3); + testClock, streamMediator, streamObserver3, testContext); // Set up the subscribers streamMediator.subscribe(concreteObserver1); @@ -199,6 +216,13 @@ blockWriter, blockNodeContext, new ServiceStatusImpl()) final SubscribeStreamResponse subscribeStreamResponse = SubscribeStreamResponse.newBuilder().blockItem(blockItem).build(); + // register the stream validator + when(blockWriter.write(blockItem)).thenReturn(Optional.empty()); + final var streamValidator = + new StreamPersistenceHandlerImpl( + streamMediator, notifier, blockWriter, blockNodeContext, serviceStatus); + streamMediator.subscribe(streamValidator); + // Acting as a producer, notify the mediator of a new block streamMediator.publish(blockItem); @@ -213,31 +237,30 @@ blockWriter, blockNodeContext, new ServiceStatusImpl()) .onNext(fromPbj(subscribeStreamResponse)); // Confirm the BlockStorage write method was called - verify(blockWriter).write(blockItem); + verify(blockWriter, timeout(testTimeout).times(1)).write(blockItem); } @Test public void testSubAndUnsubHandling() throws IOException { final BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); + final ServiceStatus serviceStatus = new ServiceStatusImpl(blockNodeContext); final var streamMediator = - LiveStreamMediatorBuilder.newBuilder( - blockWriter, blockNodeContext, new ServiceStatusImpl()) - .build(); + LiveStreamMediatorBuilder.newBuilder(blockNodeContext, serviceStatus).build(); when(testClock.millis()).thenReturn(TEST_TIME, TEST_TIME + TIMEOUT_THRESHOLD_MILLIS); final var concreteObserver1 = new ConsumerStreamResponseObserver( - testContext, testClock, streamMediator, streamObserver1); + testClock, streamMediator, streamObserver1, testContext); final var concreteObserver2 = new ConsumerStreamResponseObserver( - testContext, testClock, streamMediator, streamObserver2); + testClock, streamMediator, streamObserver2, testContext); final var concreteObserver3 = new ConsumerStreamResponseObserver( - testContext, testClock, streamMediator, streamObserver3); + testClock, streamMediator, streamObserver3, testContext); // Set up the subscribers streamMediator.subscribe(concreteObserver1); @@ -252,26 +275,64 @@ blockWriter, blockNodeContext, new ServiceStatusImpl()) assertEquals(0, blockNodeContext.metricsService().get(LiveBlockItems).get()); } + @Test + public void testSubscribeWhenHandlerAlreadySubscribed() throws IOException { + final BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); + final LongGauge consumersGauge = + blockNodeContext.metricsService().get(BlockNodeMetricTypes.Gauge.Consumers); + final ServiceStatus serviceStatus = new ServiceStatusImpl(blockNodeContext); + final var streamMediator = + LiveStreamMediatorBuilder.newBuilder(blockNodeContext, serviceStatus).build(); + + final var concreteObserver1 = + new ConsumerStreamResponseObserver( + testClock, streamMediator, streamObserver1, testContext); + + streamMediator.subscribe(concreteObserver1); + assertTrue(streamMediator.isSubscribed(concreteObserver1)); + assertEquals(1L, consumersGauge.get()); + + // Attempt to "re-subscribe" the observer + // Should not increment the counter or change the implementation + streamMediator.subscribe(concreteObserver1); + assertTrue(streamMediator.isSubscribed(concreteObserver1)); + assertEquals(1L, consumersGauge.get()); + + streamMediator.unsubscribe(concreteObserver1); + assertFalse(streamMediator.isSubscribed(concreteObserver1)); + + // Confirm the counter was decremented + assertEquals(0L, consumersGauge.get()); + } + @Test public void testOnCancelSubscriptionHandling() throws IOException { final BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); + final ServiceStatus serviceStatus = new ServiceStatusImpl(blockNodeContext); final var streamMediator = - LiveStreamMediatorBuilder.newBuilder( - blockWriter, blockNodeContext, new ServiceStatusImpl()) - .build(); + LiveStreamMediatorBuilder.newBuilder(blockNodeContext, serviceStatus).build(); when(testClock.millis()).thenReturn(TEST_TIME, TEST_TIME + TIMEOUT_THRESHOLD_MILLIS); + final List blockItems = generateBlockItems(1); + + // register the stream validator + when(blockWriter.write(blockItems.getFirst())).thenReturn(Optional.empty()); + final var streamValidator = + new StreamPersistenceHandlerImpl( + streamMediator, notifier, blockWriter, blockNodeContext, serviceStatus); + streamMediator.subscribe(streamValidator); + + // register the test observer final var testConsumerBlockItemObserver = new TestConsumerStreamResponseObserver( - testContext, testClock, streamMediator, serverCallStreamObserver); + testClock, streamMediator, serverCallStreamObserver, testContext); streamMediator.subscribe(testConsumerBlockItemObserver); assertTrue(streamMediator.isSubscribed(testConsumerBlockItemObserver)); // Simulate the producer notifying the mediator of a new block - final List blockItems = generateBlockItems(1); streamMediator.publish(blockItems.getFirst()); // Simulate the consumer cancelling the stream @@ -285,29 +346,42 @@ blockWriter, blockNodeContext, new ServiceStatusImpl()) // Confirm the mediator unsubscribed the consumer assertFalse(streamMediator.isSubscribed(testConsumerBlockItemObserver)); + + // Confirm the BlockStorage write method was called + verify(blockWriter, timeout(testTimeout).times(1)).write(blockItems.getFirst()); + + // Confirm the stream validator is still subscribed + assertTrue(streamMediator.isSubscribed(streamValidator)); } @Test public void testOnCloseSubscriptionHandling() throws IOException { final BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); + final ServiceStatus serviceStatus = new ServiceStatusImpl(blockNodeContext); final var streamMediator = - LiveStreamMediatorBuilder.newBuilder( - blockWriter, blockNodeContext, new ServiceStatusImpl()) - .build(); + LiveStreamMediatorBuilder.newBuilder(blockNodeContext, serviceStatus).build(); // testClock configured to be outside the timeout window when(testClock.millis()).thenReturn(TEST_TIME, TEST_TIME + TIMEOUT_THRESHOLD_MILLIS + 1); + final List blockItems = generateBlockItems(1); + + // register the stream validator + when(blockWriter.write(blockItems.getFirst())).thenReturn(Optional.empty()); + final var streamValidator = + new StreamPersistenceHandlerImpl( + streamMediator, notifier, blockWriter, blockNodeContext, serviceStatus); + streamMediator.subscribe(streamValidator); + final var testConsumerBlockItemObserver = new TestConsumerStreamResponseObserver( - testContext, testClock, streamMediator, serverCallStreamObserver); + testClock, streamMediator, serverCallStreamObserver, testContext); streamMediator.subscribe(testConsumerBlockItemObserver); assertTrue(streamMediator.isSubscribed(testConsumerBlockItemObserver)); // Simulate the producer notifying the mediator of a new block - final List blockItems = generateBlockItems(1); streamMediator.publish(blockItems.getFirst()); // Simulate the consumer completing the stream @@ -321,15 +395,46 @@ blockWriter, blockNodeContext, new ServiceStatusImpl()) // Confirm the mediator unsubscribed the consumer assertFalse(streamMediator.isSubscribed(testConsumerBlockItemObserver)); + + // Confirm the BlockStorage write method was called + verify(blockWriter, timeout(testTimeout).times(1)).write(blockItems.getFirst()); + + // Confirm the stream validator is still subscribed + assertTrue(streamMediator.isSubscribed(streamValidator)); } @Test - public void testMediatorBlocksPublishAfterException() throws IOException { + public void testMediatorBlocksPublishAfterException() throws IOException, InterruptedException { + final BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); + final ServiceStatus serviceStatus = new ServiceStatusImpl(blockNodeContext); final var streamMediator = - LiveStreamMediatorBuilder.newBuilder( - blockWriter, blockNodeContext, new ServiceStatusImpl()) - .build(); + LiveStreamMediatorBuilder.newBuilder(blockNodeContext, serviceStatus).build(); + + final var concreteObserver1 = + new ConsumerStreamResponseObserver( + testClock, streamMediator, streamObserver1, testContext); + + final var concreteObserver2 = + new ConsumerStreamResponseObserver( + testClock, streamMediator, streamObserver2, testContext); + + final var concreteObserver3 = + new ConsumerStreamResponseObserver( + testClock, streamMediator, streamObserver3, testContext); + + // Set up the subscribers + streamMediator.subscribe(concreteObserver1); + streamMediator.subscribe(concreteObserver2); + streamMediator.subscribe(concreteObserver3); + + final Notifier notifier = new NotifierImpl(streamMediator, blockNodeContext, serviceStatus); + final var streamValidator = + new StreamPersistenceHandlerImpl( + streamMediator, notifier, blockWriter, blockNodeContext, serviceStatus); + + // Set up the stream verifier + streamMediator.subscribe(streamValidator); final List blockItems = generateBlockItems(1); final BlockItem firstBlockItem = blockItems.getFirst(); @@ -340,34 +445,59 @@ blockWriter, blockNodeContext, new ServiceStatusImpl()) // future. In that case, we need to make sure a second producer // is not able to publish a block after the first producer fails. doThrow(new IOException()).when(blockWriter).write(firstBlockItem); - try { - streamMediator.publish(firstBlockItem); - fail("Expected an IOException to be thrown"); - } catch (IOException e) { - final BlockItem secondBlockItem = blockItems.get(1); - streamMediator.publish(secondBlockItem); + streamMediator.publish(firstBlockItem); - // Confirm the counter was incremented only once - assertEquals(1, blockNodeContext.metricsService().get(LiveBlockItems).get()); + Thread.sleep(testTimeout); - // Confirm the BlockPersistenceHandler write method was only called - // once despite the second block being published. - verify(blockWriter, timeout(testTimeout).times(1)).write(firstBlockItem); - } + // Confirm the counter was incremented only once + assertEquals(1, blockNodeContext.metricsService().get(LiveBlockItems).get()); + + // Confirm the error counter was incremented + assertEquals(1, blockNodeContext.metricsService().get(LiveBlockStreamMediatorError).get()); + + // Send another block item after the exception + streamMediator.publish(blockItems.get(1)); + + final var subscribeStreamResponse = + SubscribeStreamResponse.newBuilder().blockItem(firstBlockItem).build(); + verify(streamObserver1, timeout(testTimeout).times(1)) + .onNext(fromPbj(subscribeStreamResponse)); + verify(streamObserver2, timeout(testTimeout).times(1)) + .onNext(fromPbj(subscribeStreamResponse)); + verify(streamObserver3, timeout(testTimeout).times(1)) + .onNext(fromPbj(subscribeStreamResponse)); + + // TODO: Replace READ_STREAM_SUCCESS (2) with a generic error code? + final SubscribeStreamResponse endOfStreamResponse = + SubscribeStreamResponse.newBuilder() + .status(SubscribeStreamResponseCode.READ_STREAM_SUCCESS) + .build(); + verify(streamObserver1, timeout(testTimeout).times(1)).onNext(fromPbj(endOfStreamResponse)); + verify(streamObserver2, timeout(testTimeout).times(1)).onNext(fromPbj(endOfStreamResponse)); + verify(streamObserver3, timeout(testTimeout).times(1)).onNext(fromPbj(endOfStreamResponse)); + + // verify write method only called once despite the second block being published. + verify(blockWriter, timeout(testTimeout).times(1)).write(firstBlockItem); } @Test public void testUnsubscribeWhenNotSubscribed() throws IOException { final BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); + final ServiceStatus serviceStatus = new ServiceStatusImpl(blockNodeContext); final var streamMediator = - LiveStreamMediatorBuilder.newBuilder( - blockWriter, blockNodeContext, new ServiceStatusImpl()) - .build(); + LiveStreamMediatorBuilder.newBuilder(blockNodeContext, serviceStatus).build(); + + // register the stream validator + final var streamValidator = + new StreamPersistenceHandlerImpl( + streamMediator, notifier, blockWriter, blockNodeContext, serviceStatus); + streamMediator.subscribe(streamValidator); + final var testConsumerBlockItemObserver = new TestConsumerStreamResponseObserver( - testContext, testClock, streamMediator, serverCallStreamObserver); + testClock, streamMediator, serverCallStreamObserver, testContext); // Confirm the observer is not subscribed assertFalse(streamMediator.isSubscribed(testConsumerBlockItemObserver)); @@ -377,17 +507,20 @@ blockWriter, blockNodeContext, new ServiceStatusImpl()) // Confirm the observer is still not subscribed assertFalse(streamMediator.isSubscribed(testConsumerBlockItemObserver)); + + // Confirm the stream validator is still subscribed + assertTrue(streamMediator.isSubscribed(streamValidator)); } private static class TestConsumerStreamResponseObserver extends ConsumerStreamResponseObserver { public TestConsumerStreamResponseObserver( - BlockNodeContext context, - final InstantSource producerLivenessClock, - final StreamMediator> - streamMediator, - final StreamObserver - responseStreamObserver) { - super(context, producerLivenessClock, streamMediator, responseStreamObserver); + @NonNull final InstantSource producerLivenessClock, + @NonNull final StreamMediator streamMediator, + @NonNull + final StreamObserver + responseStreamObserver, + @NonNull final BlockNodeContext blockNodeContext) { + super(producerLivenessClock, streamMediator, responseStreamObserver, blockNodeContext); } @NonNull diff --git a/server/src/test/java/com/hedera/block/server/mediator/MediatorConfigTest.java b/server/src/test/java/com/hedera/block/server/mediator/MediatorConfigTest.java new file mode 100644 index 00000000..8586c6d5 --- /dev/null +++ b/server/src/test/java/com/hedera/block/server/mediator/MediatorConfigTest.java @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.mediator; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.stream.IntStream; +import org.junit.jupiter.api.Test; + +public class MediatorConfigTest { + + @Test + public void testMediatorConfig_happyPath() { + MediatorConfig mediatorConfig = new MediatorConfig(2048); + assertEquals(2048, mediatorConfig.ringBufferSize()); + } + + @Test + public void testMediatorConfig_negativeRingBufferSize() { + IllegalArgumentException exception = + assertThrows(IllegalArgumentException.class, () -> new MediatorConfig(-1)); + assertEquals("Ring buffer size must be greater than 0", exception.getMessage()); + } + + @Test + public void testMediatorConfig_powerOf2Values() { + + int[] powerOf2Values = IntStream.iterate(2, n -> n * 2).limit(30).toArray(); + + // Test the power of 2 values + for (int powerOf2Value : powerOf2Values) { + MediatorConfig mediatorConfig = new MediatorConfig(powerOf2Value); + assertEquals(powerOf2Value, mediatorConfig.ringBufferSize()); + } + + // Test the non-power of 2 values + for (int powerOf2Value : powerOf2Values) { + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> new MediatorConfig(powerOf2Value + 1)); + assertEquals("Ring buffer size must be a power of 2", exception.getMessage()); + } + } +} diff --git a/server/src/test/java/com/hedera/block/server/mediator/MediatorInjectionModuleTest.java b/server/src/test/java/com/hedera/block/server/mediator/MediatorInjectionModuleTest.java index 2a05d499..5ba33db4 100644 --- a/server/src/test/java/com/hedera/block/server/mediator/MediatorInjectionModuleTest.java +++ b/server/src/test/java/com/hedera/block/server/mediator/MediatorInjectionModuleTest.java @@ -18,12 +18,12 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; -import com.hedera.block.server.ServiceStatus; import com.hedera.block.server.config.BlockNodeContext; -import com.hedera.block.server.data.ObjectEvent; -import com.hedera.block.server.persistence.storage.write.BlockWriter; +import com.hedera.block.server.service.ServiceStatus; +import com.hedera.block.server.util.TestConfigUtil; import com.hedera.hapi.block.SubscribeStreamResponse; import com.hedera.hapi.block.stream.BlockItem; +import java.io.IOException; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -33,10 +33,6 @@ @ExtendWith(MockitoExtension.class) class MediatorInjectionModuleTest { - @Mock private BlockWriter blockWriter; - - @Mock private BlockNodeContext blockNodeContext; - @Mock private ServiceStatus serviceStatus; @BeforeEach @@ -45,11 +41,13 @@ void setup() { } @Test - void testProvidesStreamMediator() { + void testProvidesStreamMediator() throws IOException { + + BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); + // Call the method under test - StreamMediator> streamMediator = - MediatorInjectionModule.providesStreamMediator( - blockWriter, blockNodeContext, serviceStatus); + StreamMediator streamMediator = + MediatorInjectionModule.providesLiveStreamMediator(blockNodeContext, serviceStatus); // Verify that the streamMediator is correctly instantiated assertNotNull(streamMediator); diff --git a/server/src/test/java/com/hedera/block/server/metrics/MetricsServiceTest.java b/server/src/test/java/com/hedera/block/server/metrics/MetricsServiceTest.java index 83e1dab4..35fe4cb7 100644 --- a/server/src/test/java/com/hedera/block/server/metrics/MetricsServiceTest.java +++ b/server/src/test/java/com/hedera/block/server/metrics/MetricsServiceTest.java @@ -20,7 +20,7 @@ import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.LiveBlockItems; import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.LiveBlockItemsConsumed; import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.SingleBlocksRetrieved; -import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Gauge.Subscribers; +import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Gauge.Consumers; import static org.junit.jupiter.api.Assertions.assertEquals; import com.hedera.block.server.config.BlockNodeContext; @@ -121,17 +121,17 @@ void MetricsService_verifyLiveBlockItemsConsumedCounter() { @Test void MetricsService_verifySubscribersGauge() { - assertEquals(Subscribers.grafanaLabel(), metricsService.get(Subscribers).getName()); - assertEquals(Subscribers.description(), metricsService.get(Subscribers).getDescription()); + assertEquals(Consumers.grafanaLabel(), metricsService.get(Consumers).getName()); + assertEquals(Consumers.description(), metricsService.get(Consumers).getDescription()); // Set the subscribers to various values and verify - metricsService.get(Subscribers).set(10); - assertEquals(10, metricsService.get(Subscribers).get()); + metricsService.get(Consumers).set(10); + assertEquals(10, metricsService.get(Consumers).get()); - metricsService.get(Subscribers).set(3); - assertEquals(3, metricsService.get(Subscribers).get()); + metricsService.get(Consumers).set(3); + assertEquals(3, metricsService.get(Consumers).get()); - metricsService.get(Subscribers).set(0); - assertEquals(0, metricsService.get(Subscribers).get()); + metricsService.get(Consumers).set(0); + assertEquals(0, metricsService.get(Consumers).get()); } } diff --git a/server/src/test/java/com/hedera/block/server/notifier/NotifierConfigTest.java b/server/src/test/java/com/hedera/block/server/notifier/NotifierConfigTest.java new file mode 100644 index 00000000..329da42e --- /dev/null +++ b/server/src/test/java/com/hedera/block/server/notifier/NotifierConfigTest.java @@ -0,0 +1,59 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.notifier; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.stream.IntStream; +import org.junit.jupiter.api.Test; + +public class NotifierConfigTest { + + @Test + public void testNotifierConfig_happyPath() { + NotifierConfig notifierConfig = new NotifierConfig(2048); + assertEquals(2048, notifierConfig.ringBufferSize()); + } + + @Test + public void testNotifierConfig_negativeRingBufferSize() { + IllegalArgumentException exception = + assertThrows(IllegalArgumentException.class, () -> new NotifierConfig(-1)); + assertEquals("Ring buffer size must be greater than 0", exception.getMessage()); + } + + @Test + public void testMediatorConfig_powerOf2Values() { + + int[] powerOf2Values = IntStream.iterate(2, n -> n * 2).limit(30).toArray(); + + for (int powerOf2Value : powerOf2Values) { + NotifierConfig notifierConfig = new NotifierConfig(powerOf2Value); + assertEquals(powerOf2Value, notifierConfig.ringBufferSize()); + } + + // Test the non-power of 2 values + for (int powerOf2Value : powerOf2Values) { + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> new NotifierConfig(powerOf2Value + 1)); + assertEquals("Ring buffer size must be a power of 2", exception.getMessage()); + } + } +} diff --git a/server/src/test/java/com/hedera/block/server/notifier/NotifierImplTest.java b/server/src/test/java/com/hedera/block/server/notifier/NotifierImplTest.java new file mode 100644 index 00000000..4124f9b0 --- /dev/null +++ b/server/src/test/java/com/hedera/block/server/notifier/NotifierImplTest.java @@ -0,0 +1,373 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.notifier; + +import static com.hedera.block.server.BlockStreamServiceIntegrationTest.buildAck; +import static com.hedera.block.server.Translator.fromPbj; +import static com.hedera.block.server.notifier.NotifierImpl.buildErrorStreamResponse; +import static com.hedera.block.server.util.PersistTestUtils.generateBlockItems; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.timeout; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.hedera.block.server.config.BlockNodeContext; +import com.hedera.block.server.mediator.Publisher; +import com.hedera.block.server.mediator.SubscriptionHandler; +import com.hedera.block.server.producer.ProducerBlockItemObserver; +import com.hedera.block.server.service.ServiceStatus; +import com.hedera.block.server.service.ServiceStatusImpl; +import com.hedera.block.server.util.TestConfigUtil; +import com.hedera.hapi.block.Acknowledgement; +import com.hedera.hapi.block.PublishStreamResponse; +import com.hedera.hapi.block.stream.BlockItem; +import edu.umd.cs.findbugs.annotations.NonNull; +import io.grpc.stub.StreamObserver; +import java.io.IOException; +import java.security.NoSuchAlgorithmException; +import java.time.InstantSource; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +public class NotifierImplTest { + + @Mock private Notifiable mediator; + @Mock private Publisher publisher; + @Mock private ServiceStatus serviceStatus; + @Mock private SubscriptionHandler subscriptionHandler; + + @Mock + private StreamObserver streamObserver1; + + @Mock + private StreamObserver streamObserver2; + + @Mock + private StreamObserver streamObserver3; + + @Mock private InstantSource testClock; + + private final long TIMEOUT_THRESHOLD_MILLIS = 100L; + private final long TEST_TIME = 1_719_427_664_950L; + + private static final int testTimeout = 1000; + + private final BlockNodeContext testContext; + + public NotifierImplTest() throws IOException { + Map properties = new HashMap<>(); + properties.put(TestConfigUtil.MEDIATOR_RING_BUFFER_SIZE_KEY, String.valueOf(1024)); + this.testContext = TestConfigUtil.getTestBlockNodeContext(properties); + } + + @Test + public void testRegistration() throws NoSuchAlgorithmException { + + final ServiceStatus serviceStatus = new ServiceStatusImpl(testContext); + final var notifier = new NotifierImpl(mediator, testContext, serviceStatus); + + when(testClock.millis()).thenReturn(TEST_TIME, TEST_TIME + TIMEOUT_THRESHOLD_MILLIS); + + final var concreteObserver1 = + new ProducerBlockItemObserver( + testClock, + publisher, + subscriptionHandler, + streamObserver1, + testContext, + serviceStatus); + + final var concreteObserver2 = + new ProducerBlockItemObserver( + testClock, + publisher, + subscriptionHandler, + streamObserver2, + testContext, + serviceStatus); + + final var concreteObserver3 = + new ProducerBlockItemObserver( + testClock, + publisher, + subscriptionHandler, + streamObserver3, + testContext, + serviceStatus); + + notifier.subscribe(concreteObserver1); + notifier.subscribe(concreteObserver2); + notifier.subscribe(concreteObserver3); + + assertTrue( + notifier.isSubscribed(concreteObserver1), + "Expected the notifier to have observer1 subscribed"); + assertTrue( + notifier.isSubscribed(concreteObserver2), + "Expected the notifier to have observer2 subscribed"); + assertTrue( + notifier.isSubscribed(concreteObserver3), + "Expected the notifier to have observer3 subscribed"); + + List blockItems = generateBlockItems(1); + notifier.publish(blockItems.getFirst()); + + // Verify the response was received by all observers + final var publishStreamResponse = + PublishStreamResponse.newBuilder() + .acknowledgement(buildAck(blockItems.getFirst())) + .build(); + verify(streamObserver1, timeout(testTimeout).times(1)) + .onNext(fromPbj(publishStreamResponse)); + verify(streamObserver2, timeout(testTimeout).times(1)) + .onNext(fromPbj(publishStreamResponse)); + verify(streamObserver3, timeout(testTimeout).times(1)) + .onNext(fromPbj(publishStreamResponse)); + + // Unsubscribe the observers + notifier.unsubscribe(concreteObserver1); + assertFalse( + notifier.isSubscribed(concreteObserver1), + "Expected the notifier to have unsubscribed observer1"); + + notifier.unsubscribe(concreteObserver2); + assertFalse( + notifier.isSubscribed(concreteObserver2), + "Expected the notifier to have unsubscribed observer2"); + + notifier.unsubscribe(concreteObserver3); + assertFalse( + notifier.isSubscribed(concreteObserver3), + "Expected the notifier to have unsubscribed observer3"); + } + + @Test + public void testTimeoutExpiredHandling() throws InterruptedException { + + when(serviceStatus.isRunning()).thenReturn(true); + + final var notifier = new NotifierImpl(mediator, testContext, serviceStatus); + + // Set the clocks to be expired + final InstantSource testClock1 = mock(InstantSource.class); + when(testClock1.millis()).thenReturn(TEST_TIME, TEST_TIME + 1501L); + + final InstantSource testClock2 = mock(InstantSource.class); + when(testClock2.millis()).thenReturn(TEST_TIME, TEST_TIME + 1501L); + + final InstantSource testClock3 = mock(InstantSource.class); + when(testClock3.millis()).thenReturn(TEST_TIME, TEST_TIME + 1501L); + + final var concreteObserver1 = + new ProducerBlockItemObserver( + testClock1, + publisher, + notifier, + streamObserver1, + testContext, + serviceStatus); + + final var concreteObserver2 = + new ProducerBlockItemObserver( + testClock2, + publisher, + notifier, + streamObserver2, + testContext, + serviceStatus); + + final var concreteObserver3 = + new ProducerBlockItemObserver( + testClock3, + publisher, + notifier, + streamObserver3, + testContext, + serviceStatus); + + notifier.subscribe(concreteObserver1); + notifier.subscribe(concreteObserver2); + notifier.subscribe(concreteObserver3); + + assertTrue( + notifier.isSubscribed(concreteObserver1), + "Expected the notifier to have observer1 subscribed"); + assertTrue( + notifier.isSubscribed(concreteObserver2), + "Expected the notifier to have observer2 subscribed"); + assertTrue( + notifier.isSubscribed(concreteObserver3), + "Expected the notifier to have observer3 subscribed"); + + List blockItems = generateBlockItems(1); + notifier.publish(blockItems.getFirst()); + + Thread.sleep(testTimeout); + + assertFalse( + notifier.isSubscribed(concreteObserver1), + "Expected the notifier to have observer1 unsubscribed"); + assertFalse( + notifier.isSubscribed(concreteObserver2), + "Expected the notifier to have observer2 unsubscribed"); + assertFalse( + notifier.isSubscribed(concreteObserver3), + "Expected the notifier to have observer3 unsubscribed"); + } + + @Test + public void testPublishThrowsNoSuchAlgorithmException() { + + when(serviceStatus.isRunning()).thenReturn(true); + final var notifier = new TestNotifier(mediator, testContext, serviceStatus); + final var concreteObserver1 = + new ProducerBlockItemObserver( + testClock, + publisher, + subscriptionHandler, + streamObserver1, + testContext, + serviceStatus); + + final var concreteObserver2 = + new ProducerBlockItemObserver( + testClock, + publisher, + subscriptionHandler, + streamObserver2, + testContext, + serviceStatus); + + final var concreteObserver3 = + new ProducerBlockItemObserver( + testClock, + publisher, + subscriptionHandler, + streamObserver3, + testContext, + serviceStatus); + + notifier.subscribe(concreteObserver1); + notifier.subscribe(concreteObserver2); + notifier.subscribe(concreteObserver3); + + assertTrue( + notifier.isSubscribed(concreteObserver1), + "Expected the notifier to have observer1 subscribed"); + assertTrue( + notifier.isSubscribed(concreteObserver2), + "Expected the notifier to have observer2 subscribed"); + assertTrue( + notifier.isSubscribed(concreteObserver3), + "Expected the notifier to have observer3 subscribed"); + + List blockItems = generateBlockItems(1); + notifier.publish(blockItems.getFirst()); + + final PublishStreamResponse errorResponse = buildErrorStreamResponse(); + verify(streamObserver1, timeout(testTimeout).times(1)).onNext(fromPbj(errorResponse)); + verify(streamObserver2, timeout(testTimeout).times(1)).onNext(fromPbj(errorResponse)); + verify(streamObserver3, timeout(testTimeout).times(1)).onNext(fromPbj(errorResponse)); + } + + @Test + public void testServiceStatusNotRunning() throws NoSuchAlgorithmException { + + // Set the serviceStatus to not running + when(serviceStatus.isRunning()).thenReturn(false); + final var notifier = new TestNotifier(mediator, testContext, serviceStatus); + final var concreteObserver1 = + new ProducerBlockItemObserver( + testClock, + publisher, + subscriptionHandler, + streamObserver1, + testContext, + serviceStatus); + + final var concreteObserver2 = + new ProducerBlockItemObserver( + testClock, + publisher, + subscriptionHandler, + streamObserver2, + testContext, + serviceStatus); + + final var concreteObserver3 = + new ProducerBlockItemObserver( + testClock, + publisher, + subscriptionHandler, + streamObserver3, + testContext, + serviceStatus); + + notifier.subscribe(concreteObserver1); + notifier.subscribe(concreteObserver2); + notifier.subscribe(concreteObserver3); + + assertTrue( + notifier.isSubscribed(concreteObserver1), + "Expected the notifier to have observer1 subscribed"); + assertTrue( + notifier.isSubscribed(concreteObserver2), + "Expected the notifier to have observer2 subscribed"); + assertTrue( + notifier.isSubscribed(concreteObserver3), + "Expected the notifier to have observer3 subscribed"); + + final List blockItems = generateBlockItems(1); + notifier.publish(blockItems.getFirst()); + + // Verify once the serviceStatus is not running that we do not publish the responses + final var publishStreamResponse = + PublishStreamResponse.newBuilder() + .acknowledgement(buildAck(blockItems.getFirst())) + .build(); + verify(streamObserver1, timeout(testTimeout).times(0)) + .onNext(fromPbj(publishStreamResponse)); + verify(streamObserver2, timeout(testTimeout).times(0)) + .onNext(fromPbj(publishStreamResponse)); + verify(streamObserver3, timeout(testTimeout).times(0)) + .onNext(fromPbj(publishStreamResponse)); + } + + private static final class TestNotifier extends NotifierImpl { + public TestNotifier( + @NonNull final Notifiable mediator, + @NonNull final BlockNodeContext blockNodeContext, + @NonNull final ServiceStatus serviceStatus) { + super(mediator, blockNodeContext, serviceStatus); + } + + @Override + @NonNull + Acknowledgement buildAck(@NonNull final BlockItem blockItem) + throws NoSuchAlgorithmException { + throw new NoSuchAlgorithmException("Test exception"); + } + } +} diff --git a/server/src/test/java/com/hedera/block/server/persistence/PersistenceInjectionModuleTest.java b/server/src/test/java/com/hedera/block/server/persistence/PersistenceInjectionModuleTest.java index 70ed73e6..61763d49 100644 --- a/server/src/test/java/com/hedera/block/server/persistence/PersistenceInjectionModuleTest.java +++ b/server/src/test/java/com/hedera/block/server/persistence/PersistenceInjectionModuleTest.java @@ -21,10 +21,16 @@ import static org.mockito.Mockito.when; import com.hedera.block.server.config.BlockNodeContext; +import com.hedera.block.server.events.BlockNodeEventHandler; +import com.hedera.block.server.events.ObjectEvent; +import com.hedera.block.server.mediator.SubscriptionHandler; +import com.hedera.block.server.notifier.Notifier; import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; import com.hedera.block.server.persistence.storage.read.BlockReader; import com.hedera.block.server.persistence.storage.write.BlockWriter; +import com.hedera.block.server.service.ServiceStatus; import com.hedera.block.server.util.TestConfigUtil; +import com.hedera.hapi.block.SubscribeStreamResponse; import com.hedera.hapi.block.stream.Block; import com.hedera.hapi.block.stream.BlockItem; import com.swirlds.config.api.Configuration; @@ -39,8 +45,11 @@ class PersistenceInjectionModuleTest { @Mock private BlockNodeContext blockNodeContext; - @Mock private PersistenceStorageConfig persistenceStorageConfig; + @Mock private SubscriptionHandler subscriptionHandler; + @Mock private Notifier notifier; + @Mock private BlockWriter blockWriter; + @Mock private ServiceStatus serviceStatus; @BeforeEach void setup() throws IOException { @@ -87,4 +96,21 @@ void testProvidesBlockReader() { assertNotNull(blockReader); } + + @Test + void testProvidesStreamValidatorBuilder() throws IOException { + + BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); + + // Call the method under test + BlockNodeEventHandler> streamVerifier = + new StreamPersistenceHandlerImpl( + subscriptionHandler, + notifier, + blockWriter, + blockNodeContext, + serviceStatus); + + assertNotNull(streamVerifier); + } } diff --git a/server/src/test/java/com/hedera/block/server/persistence/StreamPersistenceHandlerImplTest.java b/server/src/test/java/com/hedera/block/server/persistence/StreamPersistenceHandlerImplTest.java new file mode 100644 index 00000000..87c23d5f --- /dev/null +++ b/server/src/test/java/com/hedera/block/server/persistence/StreamPersistenceHandlerImplTest.java @@ -0,0 +1,178 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.persistence; + +import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.StreamPersistenceHandlerError; +import static com.hedera.block.server.util.PersistTestUtils.generateBlockItems; +import static com.hedera.hapi.block.SubscribeStreamResponseCode.READ_STREAM_SUCCESS; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.timeout; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.hedera.block.server.config.BlockNodeContext; +import com.hedera.block.server.events.ObjectEvent; +import com.hedera.block.server.mediator.SubscriptionHandler; +import com.hedera.block.server.metrics.MetricsService; +import com.hedera.block.server.notifier.Notifier; +import com.hedera.block.server.persistence.storage.write.BlockWriter; +import com.hedera.block.server.service.ServiceStatus; +import com.hedera.block.server.util.TestConfigUtil; +import com.hedera.hapi.block.SubscribeStreamResponse; +import com.hedera.hapi.block.stream.BlockItem; +import com.hedera.pbj.runtime.OneOf; +import java.io.IOException; +import java.util.List; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +public class StreamPersistenceHandlerImplTest { + + @Mock private SubscriptionHandler subscriptionHandler; + + @Mock private BlockWriter blockWriter; + + @Mock private Notifier notifier; + + @Mock private BlockNodeContext blockNodeContext; + + @Mock private ServiceStatus serviceStatus; + + @Mock private MetricsService metricsService; + + private static final int testTimeout = 0; + + @Test + public void testOnEventWhenServiceIsNotRunning() { + + when(blockNodeContext.metricsService()).thenReturn(metricsService); + when(serviceStatus.isRunning()).thenReturn(false); + + final var streamPersistenceHandler = + new StreamPersistenceHandlerImpl( + subscriptionHandler, + notifier, + blockWriter, + blockNodeContext, + serviceStatus); + + final List blockItems = generateBlockItems(1); + final var subscribeStreamResponse = + SubscribeStreamResponse.newBuilder().blockItem(blockItems.getFirst()).build(); + final ObjectEvent event = new ObjectEvent<>(); + event.set(subscribeStreamResponse); + + streamPersistenceHandler.onEvent(event, 0, false); + + // Indirectly confirm the branch we're in by verifying + // these methods were not called. + verify(notifier, never()).publish(blockItems.getFirst()); + verify(metricsService, never()).get(StreamPersistenceHandlerError); + } + + @Test + public void testBlockItemIsNull() throws IOException { + + final BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); + when(serviceStatus.isRunning()).thenReturn(true); + + final var streamPersistenceHandler = + new StreamPersistenceHandlerImpl( + subscriptionHandler, + notifier, + blockWriter, + blockNodeContext, + serviceStatus); + + final List blockItems = generateBlockItems(1); + final var subscribeStreamResponse = + spy(SubscribeStreamResponse.newBuilder().blockItem(blockItems.getFirst()).build()); + + // Force the block item to be null + when(subscribeStreamResponse.blockItem()).thenReturn(null); + final ObjectEvent event = new ObjectEvent<>(); + event.set(subscribeStreamResponse); + + streamPersistenceHandler.onEvent(event, 0, false); + + verify(serviceStatus, timeout(testTimeout).times(1)).stopRunning(any()); + verify(subscriptionHandler, timeout(testTimeout).times(1)).unsubscribe(any()); + verify(notifier, timeout(testTimeout).times(1)).notifyUnrecoverableError(); + } + + @Test + public void testSubscribeStreamResponseTypeUnknown() throws IOException { + final BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); + when(serviceStatus.isRunning()).thenReturn(true); + + final var streamPersistenceHandler = + new StreamPersistenceHandlerImpl( + subscriptionHandler, + notifier, + blockWriter, + blockNodeContext, + serviceStatus); + + final List blockItems = generateBlockItems(1); + final var subscribeStreamResponse = + spy(SubscribeStreamResponse.newBuilder().blockItem(blockItems.getFirst()).build()); + + // Force the block item to be UNSET + final OneOf illegalOneOf = + new OneOf<>(SubscribeStreamResponse.ResponseOneOfType.UNSET, null); + when(subscribeStreamResponse.response()).thenReturn(illegalOneOf); + + final ObjectEvent event = new ObjectEvent<>(); + event.set(subscribeStreamResponse); + + streamPersistenceHandler.onEvent(event, 0, false); + + verify(serviceStatus, timeout(testTimeout).times(1)).stopRunning(any()); + verify(subscriptionHandler, timeout(testTimeout).times(1)).unsubscribe(any()); + verify(notifier, timeout(testTimeout).times(1)).notifyUnrecoverableError(); + } + + @Test + public void testSubscribeStreamResponseTypeStatus() { + when(blockNodeContext.metricsService()).thenReturn(metricsService); + when(serviceStatus.isRunning()).thenReturn(true); + + final var streamPersistenceHandler = + new StreamPersistenceHandlerImpl( + subscriptionHandler, + notifier, + blockWriter, + blockNodeContext, + serviceStatus); + + final SubscribeStreamResponse subscribeStreamResponse = + spy(SubscribeStreamResponse.newBuilder().status(READ_STREAM_SUCCESS).build()); + final ObjectEvent event = new ObjectEvent<>(); + event.set(subscribeStreamResponse); + + streamPersistenceHandler.onEvent(event, 0, false); + + verify(serviceStatus, never()).stopRunning(any()); + verify(subscriptionHandler, never()).unsubscribe(any()); + verify(notifier, never()).notifyUnrecoverableError(); + } +} diff --git a/server/src/test/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriterTest.java b/server/src/test/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriterTest.java index d1196503..ace12e14 100644 --- a/server/src/test/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriterTest.java +++ b/server/src/test/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriterTest.java @@ -17,11 +17,20 @@ package com.hedera.block.server.persistence.storage.write; import static com.hedera.block.server.persistence.storage.read.BlockAsDirReaderTest.removeBlockReadPerms; +import static com.hedera.block.server.util.PersistTestUtils.generateBlockItems; import static java.lang.System.Logger; import static java.lang.System.Logger.Level.ERROR; import static java.lang.System.Logger.Level.INFO; -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.Mockito.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.doCallRealMethod; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.same; +import static org.mockito.Mockito.spy; import com.hedera.block.server.config.BlockNodeContext; import com.hedera.block.server.persistence.storage.FileUtils; @@ -30,7 +39,6 @@ import com.hedera.block.server.persistence.storage.read.BlockReader; import com.hedera.block.server.persistence.storage.remove.BlockAsDirRemover; import com.hedera.block.server.persistence.storage.remove.BlockRemover; -import com.hedera.block.server.util.PersistTestUtils; import com.hedera.block.server.util.TestConfigUtil; import com.hedera.block.server.util.TestUtils; import com.hedera.hapi.block.stream.Block; @@ -83,12 +91,24 @@ public void tearDown() { public void testWriterAndReaderHappyPath() throws IOException, ParseException { // Write a block - final List blockItems = PersistTestUtils.generateBlockItems(1); + final List blockItems = generateBlockItems(1); final BlockWriter blockWriter = - BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); - for (BlockItem blockItem : blockItems) { - blockWriter.write(blockItem); + BlockAsDirWriterBuilder.newBuilder(blockNodeContext) + .filePerms(FileUtils.defaultPerms) + .build(); + for (int i = 0; i < 10; i++) { + if (i == 9) { + Optional result = blockWriter.write(blockItems.get(i)); + if (result.isPresent()) { + assertEquals(blockItems.get(i), result.get()); + } else { + fail("The optional should contain the last block proof block item"); + } + } else { + Optional result = blockWriter.write(blockItems.get(i)); + assertTrue(result.isEmpty()); + } } // Confirm the block @@ -108,6 +128,8 @@ public void testWriterAndReaderHappyPath() throws IOException, ParseException { hasBlockProof = true; } else if (blockItem.hasEventHeader()) { hasStartEvent = true; + } else { + fail("Unknown block item type"); } } @@ -119,7 +141,7 @@ public void testWriterAndReaderHappyPath() throws IOException, ParseException { @Test public void testRemoveBlockWritePerms() throws IOException, ParseException { - final List blockItems = PersistTestUtils.generateBlockItems(1); + final List blockItems = generateBlockItems(1); final BlockWriter blockWriter = BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); @@ -129,7 +151,8 @@ public void testRemoveBlockWritePerms() throws IOException, ParseException { // The first BlockItem contains a header which will create a new block directory. // The BlockWriter will attempt to repair the permissions and should succeed. - blockWriter.write(blockItems.getFirst()); + Optional result = blockWriter.write(blockItems.getFirst()); + assertFalse(result.isPresent()); // Confirm we're able to read 1 block item BlockReader blockReader = BlockAsDirReaderBuilder.newBuilder(testConfig).build(); @@ -141,7 +164,8 @@ public void testRemoveBlockWritePerms() throws IOException, ParseException { // Remove all permissions on the block directory and // attempt to write the next block item removeBlockAllPerms(1, testConfig); - blockWriter.write(blockItems.get(1)); + result = blockWriter.write(blockItems.get(1)); + assertFalse(result.isPresent()); // There should now be 2 blockItems in the block blockOpt = blockReader.read(1); @@ -151,7 +175,8 @@ public void testRemoveBlockWritePerms() throws IOException, ParseException { // Remove read permission on the block directory removeBlockReadPerms(1, testConfig); - blockWriter.write(blockItems.get(2)); + result = blockWriter.write(blockItems.get(2)); + assertFalse(result.isPresent()); // There should now be 3 blockItems in the block blockOpt = blockReader.read(1); @@ -163,7 +188,7 @@ public void testRemoveBlockWritePerms() throws IOException, ParseException { @Test public void testUnrecoverableIOExceptionOnWrite() throws IOException { - final List blockItems = PersistTestUtils.generateBlockItems(1); + final List blockItems = generateBlockItems(1); final BlockRemover blockRemover = new BlockAsDirRemover(Path.of(testConfig.rootPath()), FileUtils.defaultPerms); @@ -179,14 +204,15 @@ public void testUnrecoverableIOExceptionOnWrite() throws IOException { @Test public void testRemoveRootDirReadPerm() throws IOException, ParseException { - final List blockItems = PersistTestUtils.generateBlockItems(1); + final List blockItems = generateBlockItems(1); final BlockWriter blockWriter = BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); // Write the first block item to create the block // directory - blockWriter.write(blockItems.getFirst()); + Optional result = blockWriter.write(blockItems.getFirst()); + assertFalse(result.isPresent()); // Remove root dir read permissions and // block dir read permissions @@ -196,7 +222,17 @@ public void testRemoveRootDirReadPerm() throws IOException, ParseException { // Attempt to write the remaining block // items for (int i = 1; i < 10; i++) { - blockWriter.write(blockItems.get(i)); + if (i == 9) { + result = blockWriter.write(blockItems.get(i)); + if (result.isPresent()) { + assertEquals(blockItems.get(i), result.get()); + } else { + fail("The optional should contain the last block proof block item"); + } + } else { + result = blockWriter.write(blockItems.get(i)); + assertTrue(result.isEmpty()); + } } BlockReader blockReader = BlockAsDirReaderBuilder.newBuilder(testConfig).build(); @@ -207,13 +243,13 @@ public void testRemoveRootDirReadPerm() throws IOException, ParseException { @Test public void testPartialBlockRemoval() throws IOException, ParseException { - final List blockItems = PersistTestUtils.generateBlockItems(3); + final List blockItems = generateBlockItems(3); final BlockRemover blockRemover = new BlockAsDirRemover(Path.of(testConfig.rootPath()), FileUtils.defaultPerms); // Use a spy of TestBlockAsDirWriter to proxy block items to the real writer // for the first 22 block items. Then simulate an IOException on the 23rd block item - // thrown from a protected write method in the real class. This should trigger the + // thrown from a protected write method in the real class. This should trigger the // blockRemover instance to remove the partially written block. final TestBlockAsDirWriter blockWriter = spy( @@ -232,7 +268,16 @@ public void testPartialBlockRemoval() throws IOException, ParseException { // Now make the calls for (int i = 0; i < 23; i++) { - blockWriter.write(blockItems.get(i)); + Optional result = blockWriter.write(blockItems.get(i)); + if (i == 9 || i == 19) { + // The last block item in each block is the block proof + // and should be returned by the write method + assertTrue(result.isPresent()); + assertEquals(blockItems.get(i), result.get()); + } else { + // The write method should return an empty optional + assertTrue(result.isEmpty()); + } } // Verify the IOException was thrown on the 23rd block item diff --git a/server/src/test/java/com/hedera/block/server/producer/ProducerBlockItemObserverTest.java b/server/src/test/java/com/hedera/block/server/producer/ProducerBlockItemObserverTest.java index 242ef05e..571e1e2c 100644 --- a/server/src/test/java/com/hedera/block/server/producer/ProducerBlockItemObserverTest.java +++ b/server/src/test/java/com/hedera/block/server/producer/ProducerBlockItemObserverTest.java @@ -17,28 +17,21 @@ package com.hedera.block.server.producer; import static com.hedera.block.server.Translator.fromPbj; -import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.LiveBlockItems; import static com.hedera.block.server.producer.Util.getFakeHash; import static com.hedera.block.server.util.PersistTestUtils.generateBlockItems; import static com.hedera.block.server.util.PersistTestUtils.reverseByteArray; -import static com.hedera.block.server.util.TestConfigUtil.CONSUMER_TIMEOUT_THRESHOLD_KEY; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.timeout; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import com.google.protobuf.InvalidProtocolBufferException; -import com.hedera.block.server.ServiceStatus; -import com.hedera.block.server.ServiceStatusImpl; import com.hedera.block.server.config.BlockNodeContext; -import com.hedera.block.server.consumer.ConsumerConfig; -import com.hedera.block.server.consumer.ConsumerStreamResponseObserver; -import com.hedera.block.server.data.ObjectEvent; -import com.hedera.block.server.mediator.LiveStreamMediatorBuilder; -import com.hedera.block.server.mediator.StreamMediator; -import com.hedera.block.server.persistence.storage.write.BlockWriter; +import com.hedera.block.server.events.ObjectEvent; +import com.hedera.block.server.mediator.Publisher; +import com.hedera.block.server.mediator.SubscriptionHandler; +import com.hedera.block.server.service.ServiceStatus; +import com.hedera.block.server.service.ServiceStatusImpl; import com.hedera.block.server.util.TestConfigUtil; import com.hedera.hapi.block.Acknowledgement; import com.hedera.hapi.block.EndOfStream; @@ -46,17 +39,17 @@ import com.hedera.hapi.block.PublishStreamRequest; import com.hedera.hapi.block.PublishStreamResponse; import com.hedera.hapi.block.PublishStreamResponseCode; -import com.hedera.hapi.block.SubscribeStreamResponse; import com.hedera.hapi.block.stream.BlockItem; -import com.hedera.hapi.block.stream.output.BlockHeader; import com.hedera.pbj.runtime.io.buffer.Bytes; import edu.umd.cs.findbugs.annotations.NonNull; +import io.grpc.stub.ServerCallStreamObserver; import io.grpc.stub.StreamObserver; import java.io.IOException; import java.security.NoSuchAlgorithmException; import java.time.InstantSource; import java.util.List; import java.util.Map; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -65,141 +58,33 @@ @ExtendWith(MockitoExtension.class) public class ProducerBlockItemObserverTest { - @Mock private StreamMediator> streamMediator; + @Mock private InstantSource testClock; + @Mock private Publisher publisher; + @Mock private SubscriptionHandler subscriptionHandler; @Mock private StreamObserver publishStreamResponseObserver; - @Mock private BlockWriter blockWriter; - - @Mock - private StreamObserver streamObserver1; - @Mock - private StreamObserver streamObserver2; - - @Mock - private StreamObserver streamObserver3; + private ServerCallStreamObserver + serverCallStreamObserver; @Mock private ServiceStatus serviceStatus; - @Mock private InstantSource testClock; - private static final int testTimeout = 1000; - - @Test - public void testProducerOnNext() throws IOException, NoSuchAlgorithmException { - - final BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); - final List blockItems = generateBlockItems(1); - final ProducerBlockItemObserver producerBlockItemObserver = - new ProducerBlockItemObserver( - streamMediator, - publishStreamResponseObserver, - blockNodeContext, - serviceStatus); - - when(serviceStatus.isRunning()).thenReturn(true); - - final BlockItem blockHeader = blockItems.getFirst(); - final PublishStreamRequest publishStreamRequest = - PublishStreamRequest.newBuilder().blockItem(blockHeader).build(); - producerBlockItemObserver.onNext(fromPbj(publishStreamRequest)); - - verify(streamMediator, timeout(testTimeout).times(1)).publish(blockHeader); - - final Acknowledgement ack = buildAck(blockHeader); - final PublishStreamResponse publishStreamResponse = - PublishStreamResponse.newBuilder().acknowledgement(ack).build(); - - verify(publishStreamResponseObserver, timeout(testTimeout).times(1)) - .onNext(fromPbj(publishStreamResponse)); - - // Helidon will call onCompleted after onNext - producerBlockItemObserver.onCompleted(); - - verify(publishStreamResponseObserver, timeout(testTimeout).times(1)).onCompleted(); - } + @Mock private ObjectEvent objectEvent; - @Test - public void testProducerWithManyConsumers() throws IOException { + private final long TIMEOUT_THRESHOLD_MILLIS = 50L; + private static final int testTimeout = 1000; - final BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); - final var streamMediator = - LiveStreamMediatorBuilder.newBuilder( - blockWriter, blockNodeContext, new ServiceStatusImpl()) - .build(); + BlockNodeContext testContext; - // Mock a clock with 2 different return values in response to anticipated - // millis() calls. Here the second call will always be inside the timeout window. - final BlockNodeContext testContext = + @BeforeEach + public void setUp() throws IOException { + this.testContext = TestConfigUtil.getTestBlockNodeContext( - Map.of(CONSUMER_TIMEOUT_THRESHOLD_KEY, "100")); - final ConsumerConfig consumerConfig = - testContext.configuration().getConfigData(ConsumerConfig.class); - - long TEST_TIME = 1_719_427_664_950L; - when(testClock.millis()) - .thenReturn(TEST_TIME, TEST_TIME + consumerConfig.timeoutThresholdMillis()); - - final var concreteObserver1 = - new ConsumerStreamResponseObserver( - testContext, testClock, streamMediator, streamObserver1); - - final var concreteObserver2 = - new ConsumerStreamResponseObserver( - testContext, testClock, streamMediator, streamObserver2); - - final var concreteObserver3 = - new ConsumerStreamResponseObserver( - testContext, testClock, streamMediator, streamObserver3); - - // Set up the subscribers - streamMediator.subscribe(concreteObserver1); - streamMediator.subscribe(concreteObserver2); - streamMediator.subscribe(concreteObserver3); - - assertTrue( - streamMediator.isSubscribed(concreteObserver1), - "Expected the mediator to have observer1 subscribed"); - assertTrue( - streamMediator.isSubscribed(concreteObserver2), - "Expected the mediator to have observer2 subscribed"); - assertTrue( - streamMediator.isSubscribed(concreteObserver3), - "Expected the mediator to have observer3 subscribed"); - - final BlockHeader blockHeader = BlockHeader.newBuilder().number(1).build(); - final BlockItem blockItem = BlockItem.newBuilder().blockHeader(blockHeader).build(); - final SubscribeStreamResponse subscribeStreamResponse = - SubscribeStreamResponse.newBuilder().blockItem(blockItem).build(); - - when(serviceStatus.isRunning()).thenReturn(true); - - final ProducerBlockItemObserver producerBlockItemObserver = - new ProducerBlockItemObserver( - streamMediator, - publishStreamResponseObserver, - blockNodeContext, - serviceStatus); - - final PublishStreamRequest publishStreamRequest = - PublishStreamRequest.newBuilder().blockItem(blockItem).build(); - producerBlockItemObserver.onNext(fromPbj(publishStreamRequest)); - - // Confirm the block item counter was incremented - assertEquals(1, blockNodeContext.metricsService().get(LiveBlockItems).get()); - - // Confirm each subscriber was notified of the new block - verify(streamObserver1, timeout(testTimeout).times(1)) - .onNext(fromPbj(subscribeStreamResponse)); - verify(streamObserver2, timeout(testTimeout).times(1)) - .onNext(fromPbj(subscribeStreamResponse)); - verify(streamObserver3, timeout(testTimeout).times(1)) - .onNext(fromPbj(subscribeStreamResponse)); - - // Confirm the BlockStorage write method was - // called despite the absence of subscribers - verify(blockWriter).write(blockItem); + Map.of( + TestConfigUtil.CONSUMER_TIMEOUT_THRESHOLD_KEY, + String.valueOf(TIMEOUT_THRESHOLD_MILLIS))); } @Test @@ -208,7 +93,9 @@ public void testOnError() throws IOException { final BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); final ProducerBlockItemObserver producerBlockItemObserver = new ProducerBlockItemObserver( - streamMediator, + testClock, + publisher, + subscriptionHandler, publishStreamResponseObserver, blockNodeContext, serviceStatus); @@ -219,42 +106,14 @@ public void testOnError() throws IOException { } @Test - public void testItemAckBuilderExceptionTest() throws IOException { - - when(serviceStatus.isRunning()).thenReturn(true); - - final BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); - final ProducerBlockItemObserver testProducerBlockItemObserver = - new TestProducerBlockItemObserver( - streamMediator, - publishStreamResponseObserver, - blockNodeContext, - serviceStatus); - - final List blockItems = generateBlockItems(1); - final BlockItem blockHeader = blockItems.getFirst(); - final PublishStreamRequest publishStreamRequest = - PublishStreamRequest.newBuilder().blockItem(blockHeader).build(); - testProducerBlockItemObserver.onNext(fromPbj(publishStreamRequest)); - - final EndOfStream endOfStream = - EndOfStream.newBuilder() - .status(PublishStreamResponseCode.STREAM_ITEMS_UNKNOWN) - .build(); - final PublishStreamResponse errorResponse = - PublishStreamResponse.newBuilder().status(endOfStream).build(); - verify(publishStreamResponseObserver, timeout(testTimeout).times(1)) - .onNext(fromPbj(errorResponse)); - } - - @Test - public void testBlockItemThrowsParseException() - throws IOException, InvalidProtocolBufferException { + public void testBlockItemThrowsParseException() throws IOException { final BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); final ProducerBlockItemObserver producerBlockItemObserver = new ProducerBlockItemObserver( - streamMediator, + testClock, + publisher, + subscriptionHandler, publishStreamResponseObserver, blockNodeContext, serviceStatus); @@ -293,36 +152,139 @@ public void testBlockItemThrowsParseException() verify(publishStreamResponseObserver, timeout(testTimeout).times(1)) .onNext(fromPbj(PublishStreamResponse.newBuilder().status(endOfStream).build())); - verify(serviceStatus, timeout(testTimeout).times(1)).stopWebServer(); + verify(serviceStatus, timeout(testTimeout).times(1)).stopWebServer(any()); } - private static class TestProducerBlockItemObserver extends ProducerBlockItemObserver { - public TestProducerBlockItemObserver( - final StreamMediator> - streamMediator, - final StreamObserver - publishStreamResponseObserver, - final BlockNodeContext blockNodeContext, - final ServiceStatus serviceStatus) { - super(streamMediator, publishStreamResponseObserver, blockNodeContext, serviceStatus); - } + @Test + public void testResponseNotPermittedAfterCancel() throws NoSuchAlgorithmException { - @NonNull - @Override - protected Acknowledgement buildAck(@NonNull final BlockItem blockItem) - throws NoSuchAlgorithmException { - throw new NoSuchAlgorithmException("test no such algorithm exception"); - } + final TestProducerBlockItemObserver producerStreamResponseObserver = + new TestProducerBlockItemObserver( + testClock, + publisher, + subscriptionHandler, + serverCallStreamObserver, + testContext, + serviceStatus); + + final List blockItems = generateBlockItems(1); + final ItemAcknowledgement itemAck = + ItemAcknowledgement.newBuilder() + .itemHash(Bytes.wrap(getFakeHash(blockItems.getLast()))) + .build(); + final PublishStreamResponse publishStreamResponse = + PublishStreamResponse.newBuilder() + .acknowledgement(Acknowledgement.newBuilder().itemAck(itemAck).build()) + .build(); + when(objectEvent.get()).thenReturn(publishStreamResponse); + + // Confirm that the observer is called with the first BlockItem + producerStreamResponseObserver.onEvent(objectEvent, 0, true); + + // Cancel the observer + producerStreamResponseObserver.cancel(); + + // Attempt to send another BlockItem + producerStreamResponseObserver.onEvent(objectEvent, 0, true); + + // Confirm that canceling the observer allowed only 1 response to be sent. + verify(serverCallStreamObserver, timeout(testTimeout).times(1)) + .onNext(fromPbj(publishStreamResponse)); } - @NonNull - private static Acknowledgement buildAck(@NonNull final BlockItem blockItem) - throws NoSuchAlgorithmException { - ItemAcknowledgement itemAck = + @Test + public void testResponseNotPermittedAfterClose() throws NoSuchAlgorithmException { + + final TestProducerBlockItemObserver producerBlockItemObserver = + new TestProducerBlockItemObserver( + testClock, + publisher, + subscriptionHandler, + serverCallStreamObserver, + testContext, + serviceStatus); + + final List blockItems = generateBlockItems(1); + final ItemAcknowledgement itemAck = ItemAcknowledgement.newBuilder() - .itemHash(Bytes.wrap(getFakeHash(blockItem))) + .itemHash(Bytes.wrap(getFakeHash(blockItems.getLast()))) + .build(); + final PublishStreamResponse publishStreamResponse = + PublishStreamResponse.newBuilder() + .acknowledgement(Acknowledgement.newBuilder().itemAck(itemAck).build()) .build(); + when(objectEvent.get()).thenReturn(publishStreamResponse); + + // Confirm that the observer is called with the first BlockItem + producerBlockItemObserver.onEvent(objectEvent, 0, true); + + // Cancel the observer + producerBlockItemObserver.close(); + + // Attempt to send another BlockItem + producerBlockItemObserver.onEvent(objectEvent, 0, true); + + // Confirm that closing the observer allowed only 1 response to be sent. + verify(serverCallStreamObserver, timeout(testTimeout).times(1)) + .onNext(fromPbj(publishStreamResponse)); + } + + @Test + public void testOnlyErrorStreamResponseAllowedAfterStatusChange() { + + final ServiceStatus serviceStatus = new ServiceStatusImpl(testContext); + + final ProducerBlockItemObserver producerBlockItemObserver = + new ProducerBlockItemObserver( + testClock, + publisher, + subscriptionHandler, + serverCallStreamObserver, + testContext, + serviceStatus); - return Acknowledgement.newBuilder().itemAck(itemAck).build(); + final List blockItems = generateBlockItems(1); + final PublishStreamRequest publishStreamRequest = + PublishStreamRequest.newBuilder().blockItem(blockItems.getFirst()).build(); + + // Confirm that the observer is called with the first BlockItem + producerBlockItemObserver.onNext(fromPbj(publishStreamRequest)); + + // Change the status of the service + serviceStatus.stopRunning(getClass().getName()); + + // Confirm that the observer is called with the first BlockItem + producerBlockItemObserver.onNext(fromPbj(publishStreamRequest)); + + // Confirm that closing the observer allowed only 1 response to be sent. + verify(serverCallStreamObserver, timeout(testTimeout).times(1)).onNext(any()); + } + + private static class TestProducerBlockItemObserver extends ProducerBlockItemObserver { + public TestProducerBlockItemObserver( + @NonNull final InstantSource clock, + @NonNull final Publisher publisher, + @NonNull final SubscriptionHandler subscriptionHandler, + @NonNull + final StreamObserver + publishStreamResponseObserver, + @NonNull final BlockNodeContext blockNodeContext, + @NonNull final ServiceStatus serviceStatus) { + super( + clock, + publisher, + subscriptionHandler, + publishStreamResponseObserver, + blockNodeContext, + serviceStatus); + } + + public void cancel() { + onCancel.run(); + } + + public void close() { + onClose.run(); + } } } diff --git a/server/src/test/java/com/hedera/block/server/service/ServiceConfigTest.java b/server/src/test/java/com/hedera/block/server/service/ServiceConfigTest.java new file mode 100644 index 00000000..c5c26e93 --- /dev/null +++ b/server/src/test/java/com/hedera/block/server/service/ServiceConfigTest.java @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.service; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.junit.jupiter.api.Test; + +public class ServiceConfigTest { + @Test + public void testServiceConfig_happyPath() { + ServiceConfig serviceConfig = new ServiceConfig(2000); + assertEquals(2000, serviceConfig.delayMillis()); + } + + @Test + public void testServiceConfig_negativeDelayMillis() { + IllegalArgumentException exception = + assertThrows(IllegalArgumentException.class, () -> new ServiceConfig(-1)); + assertEquals("Delay milliseconds must be greater than 0", exception.getMessage()); + } +} diff --git a/server/src/test/java/com/hedera/block/server/util/TestConfigUtil.java b/server/src/test/java/com/hedera/block/server/util/TestConfigUtil.java index 4653035a..d4779376 100644 --- a/server/src/test/java/com/hedera/block/server/util/TestConfigUtil.java +++ b/server/src/test/java/com/hedera/block/server/util/TestConfigUtil.java @@ -34,6 +34,7 @@ public class TestConfigUtil { public static final String CONSUMER_TIMEOUT_THRESHOLD_KEY = "consumer.timeoutThresholdMillis"; + public static final String MEDIATOR_RING_BUFFER_SIZE_KEY = "mediator.ringBufferSize"; private static final String TEST_APP_PROPERTIES_FILE = "app.properties"; diff --git a/server/src/test/resources/app.properties b/server/src/test/resources/app.properties index ff6694ed..f1ea3712 100644 --- a/server/src/test/resources/app.properties +++ b/server/src/test/resources/app.properties @@ -1,2 +1,4 @@ # Test Properties File prometheus.endpointEnabled=false +mediator.ringBufferSize=1024 +notifier.ringBufferSize=1024 diff --git a/server/src/test/resources/block_service.proto b/server/src/test/resources/block_service.proto index 460f63a3..6bd87c17 100644 --- a/server/src/test/resources/block_service.proto +++ b/server/src/test/resources/block_service.proto @@ -150,8 +150,8 @@ message BlockItem { // TransactionOutput transaction_output = 6; // StateChanges state_changes = 7; // FilteredItemHash filtered_item_hash = 8; -// BlockProof block_proof = 9; - BlockProof block_proof = 3; + BlockProof block_proof = 9; +// BlockProof block_proof = 3; // RecordFileItem record_file = 10; } }