feat: 00071 hedera protobufs

.github/workflows/pr-checks.yaml

@@ -71,15 +71,15 @@ jobs:
 
       - name: Build
         id: gradle-build
-        run: ${GRADLE_EXEC} build
+        run: ${GRADLE_EXEC} build -x :stream:test  # TODO: Remove -x :stream:test when PBJ bug is fixed
 
       - name: Style Check
         id: spotless-check
-        run: ${GRADLE_EXEC} spotlessCheck
+        run: ${GRADLE_EXEC} spotlessCheck -x :stream:spotlessCheck # Do not run spotless on the generated PBJ files
 
       - name: Unit Tests
         id: unit-tests
-        run: ${GRADLE_EXEC} check
+        run: ${GRADLE_EXEC} check -x :stream:test  # TODO: Remove -x :stream:test when PBJ bug is fixed
 
       - name: Upload coverage report
         uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673 # v4.5.0

.github/workflows/release-push-image.yaml

@@ -62,8 +62,7 @@ jobs:
           java-version: 21
 
       - name: Build
-        run: ./gradlew clean build
-
+        run: ./gradlew clean build -x :stream:test # TODO: Remove -x :stream:test when PBJ bug is fixed
       - name: Login to GitHub Container Registry
         uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0
         with:

buildSrc/build.gradle.kts

@@ -20,15 +20,15 @@ repositories { gradlePluginPortal() }
 
 dependencies {
     implementation("com.adarshr:gradle-test-logger-plugin:4.0.0")
-    implementation("com.autonomousapps:dependency-analysis-gradle-plugin:1.31.0")
+    implementation("com.autonomousapps:dependency-analysis-gradle-plugin:1.32.0")
     implementation("com.diffplug.spotless:spotless-plugin-gradle:6.25.0")
     implementation("com.google.protobuf:protobuf-gradle-plugin:0.9.4")
     implementation("com.gradle.publish:plugin-publish-plugin:1.2.1")
     implementation("gradle.plugin.lazy.zoo.gradle:git-data-plugin:1.2.2")
     implementation("io.github.gradle-nexus:publish-plugin:1.3.0")
     implementation("me.champeau.jmh:jmh-gradle-plugin:0.7.2")
     implementation("net.swiftzer.semver:semver:1.3.0")
-    implementation("org.gradlex:java-module-dependencies:1.6.5")
     implementation("org.gradlex:extra-java-module-info:1.8")
     implementation("org.gradlex:jvm-dependency-conflict-resolution:2.0")
+    implementation("org.gradlex:java-module-dependencies:1.6.5")
 }

buildSrc/src/main/kotlin/Utils.kt

@@ -31,7 +31,7 @@ class Utils {
                 lines = gradlePropFile.readLines(Charsets.UTF_8)
             }
 
-            var versionStr = "version=${newVersion.toString()}"
+            var versionStr = "version=$newVersion"
             val finalLines: List<String>
 
 

buildSrc/src/main/kotlin/com.hedera.block.jpms-modules.gradle.kts

@@ -31,16 +31,11 @@ jvmDependencyConflicts.patch {
             "com.google.code.findbugs:jsr305",
             "com.google.errorprone:error_prone_annotations",
             "com.google.guava:listenablefuture",
-            "com.google.j2objc:j2objc-annotations",
             "org.checkerframework:checker-compat-qual",
             "org.checkerframework:checker-qual",
             "org.codehaus.mojo:animal-sniffer-annotations"
         )
 
-    module("com.google.guava:guava") {
-        annotationLibraries.forEach { removeDependency(it) }
-        removeDependency("com.google.guava:failureaccess")
-    }
     module("io.grpc:grpc-api") { annotationLibraries.forEach { removeDependency(it) } }
     module("io.grpc:grpc-core") { annotationLibraries.forEach { removeDependency(it) } }
     module("io.grpc:grpc-context") { annotationLibraries.forEach { removeDependency(it) } }
@@ -50,61 +45,95 @@ jvmDependencyConflicts.patch {
         removeDependency(/* dependency = */ "com.google.protobuf:protobuf-javalite")
         addApiDependency("com.google.protobuf:protobuf-java")
     }
+    module("io.grpc:grpc-services") { annotationLibraries.forEach { removeDependency(it) } }
     module("io.grpc:grpc-stub") { annotationLibraries.forEach { removeDependency(it) } }
+    module("io.grpc:grpc-testing") { annotationLibraries.forEach { removeDependency(it) } }
     module("io.grpc:grpc-util") { annotationLibraries.forEach { removeDependency(it) } }
-    // Added for metrics and logging, but also several platform classes
+    module("com.google.guava:guava") {
+        (annotationLibraries -
+                "com.google.code.findbugs:jsr305" -
+                "com.google.errorprone:error_prone_annotations" -
+                "org.checkerframework:checker-qual")
+            .forEach { removeDependency(it) }
+    }
+    module("com.google.protobuf:protobuf-java-util") {
+        annotationLibraries.forEach { removeDependency(it) }
+    }
     module("io.prometheus:simpleclient") {
         removeDependency("io.prometheus:simpleclient_tracer_otel")
         removeDependency("io.prometheus:simpleclient_tracer_otel_agent")
     }
+    module("org.jetbrains.kotlin:kotlin-stdlib") {
+        removeDependency("org.jetbrains.kotlin:kotlin-stdlib-common")
+    }
+    module("junit:junit") { removeDependency("org.hamcrest:hamcrest-core") }
+    module("org.hyperledger.besu:secp256k1") { addApiDependency("net.java.dev.jna:jna") }
 }
 
 // Fix or enhance the 'module-info.class' of third-party Modules. This is about the
 // 'module-info.class' inside the Jar files. In our full Java Modules setup every
 // Jar needs to have this file. If it is missing, it is added by what is configured here.
 extraJavaModuleInfo {
-    failOnAutomaticModules = false // Only allow Jars with 'module-info' on all module paths
+    failOnAutomaticModules = true // Only allow Jars with 'module-info' on all module paths
 
-    module("com.google.api.grpc:proto-google-common-protos", "com.google.api.grpc.common")
-    module("com.google.guava:guava", "com.google.common") {
+    module("io.grpc:grpc-api", "io.grpc") {
         exportAllPackages()
         requireAllDefinedDependencies()
         requires("java.logging")
     }
-    module("com.google.protobuf:protobuf-java", "com.google.protobuf") {
+
+    module("io.grpc:grpc-core", "io.grpc.internal")
+    module("io.grpc:grpc-context", "io.grpc.context")
+    module("io.grpc:grpc-stub", "io.grpc.stub") {
         exportAllPackages()
         requireAllDefinedDependencies()
         requires("java.logging")
     }
-    module("io.grpc:grpc-api", "io.grpc") {
+    module("io.grpc:grpc-testing", "io.grpc.testing")
+    module("io.grpc:grpc-services", "io.grpc.services")
+    module("io.grpc:grpc-util", "io.grpc.util")
+    module("io.grpc:grpc-protobuf", "io.grpc.protobuf")
+    module("io.grpc:grpc-protobuf-lite", "io.grpc.protobuf.lite")
+    module("com.github.spotbugs:spotbugs-annotations", "com.github.spotbugs.annotations")
+    module("com.google.code.findbugs:jsr305", "java.annotation") {
+        exportAllPackages()
+        mergeJar("javax.annotation:javax.annotation-api")
+    }
+    module("com.google.errorprone:error_prone_annotations", "com.google.errorprone.annotations")
+    module("com.google.j2objc:j2objc-annotations", "com.google.j2objc.annotations")
+    module("com.google.protobuf:protobuf-java", "com.google.protobuf") {
         exportAllPackages()
         requireAllDefinedDependencies()
         requires("java.logging")
     }
-    module("io.grpc:grpc-core", "io.grpc.internal")
-    module("io.grpc:grpc-context", "io.grpc.context")
-    module("io.grpc:grpc-protobuf", "io.grpc.protobuf")
-    module("io.grpc:grpc-protobuf-lite", "io.grpc.protobuf.lite")
-    module("io.grpc:grpc-stub", "io.grpc.stub") {
+    module("com.google.guava:guava", "com.google.common") {
         exportAllPackages()
         requireAllDefinedDependencies()
         requires("java.logging")
     }
-    module("io.grpc:grpc-util", "io.grpc.util")
+    module("com.google.guava:failureaccess", "com.google.common.util.concurrent.internal")
+    module("com.google.api.grpc:proto-google-common-protos", "com.google.api.grpc.common")
+
     module("io.perfmark:perfmark-api", "io.perfmark")
+    module("javax.inject:javax.inject", "javax.inject")
 
-    module("junit:junit", "junit")
-    module("org.mockito:mockito-core", "org.mockito")
-    module("org.mockito:mockito-junit-jupiter", "org.mockito.junit.jupiter")
+    module("commons-codec:commons-codec", "org.apache.commons.codec")
+    module("org.apache.commons:commons-math3", "org.apache.commons.math3")
+    module("org.apache.commons:commons-collections4", "org.apache.commons.collections4")
+    module("com.esaulpaugh:headlong", "headlong")
 
-    // spotbugs
-    module("com.github.spotbugs:spotbugs-annotations", "com.github.spotbugs.annotations")
-    module("com.google.code.findbugs:jsr305", "java.annotation") { exportAllPackages() }
+    module("org.checkerframework:checker-qual", "org.checkerframework.checker.qual")
+    module("net.i2p.crypto:eddsa", "net.i2p.crypto.eddsa")
+    module("org.antlr:antlr4-runtime", "org.antlr.antlr4.runtime")
 
     // needed for metrics and logging, but also several platform classes
     module("com.goterl:resource-loader", "resource.loader")
     module("com.goterl:lazysodium-java", "lazysodium.java")
     module("org.hyperledger.besu:secp256k1", "org.hyperledger.besu.nativelib.secp256k1")
+    module("net.java.dev.jna:jna", "com.sun.jna") {
+        exportAllPackages()
+        requires("java.logging")
+    }
     module("io.prometheus:simpleclient", "io.prometheus.simpleclient")
     module("io.prometheus:simpleclient_common", "io.prometheus.simpleclient_common")
     module("io.prometheus:simpleclient_httpserver", "io.prometheus.simpleclient.httpserver") {
@@ -117,4 +146,22 @@ extraJavaModuleInfo {
     module("com.google.auto.service:auto-service-annotations", "com.google.auto.service")
     module("com.google.auto.service:auto-service", "com.google.auto.service.processor")
     module("com.google.auto:auto-common", "com.google.auto.common")
+
+    // JMH only
+    module("net.sf.jopt-simple:jopt-simple", "jopt.simple")
+    module("org.openjdk.jmh:jmh-core", "jmh.core")
+    module("org.openjdk.jmh:jmh-generator-asm", "jmh.generator.asm")
+    module("org.openjdk.jmh:jmh-generator-bytecode", "jmh.generator.bytecode")
+    module("org.openjdk.jmh:jmh-generator-reflection", "jmh.generator.reflection")
+
+    module("com.google.protobuf:protobuf-java-util", "com.google.protobuf.util")
+    module("junit:junit", "junit")
+    module("org.hamcrest:hamcrest", "org.hamcrest")
+    module("org.json:json", "org.json")
+    module("org.mockito:mockito-core", "org.mockito")
+    module("org.objenesis:objenesis", "org.objenesis")
+    module("org.rnorth.duct-tape:duct-tape", "org.rnorth.ducttape")
+    module("org.testcontainers:junit-jupiter", "org.testcontainers.junit.jupiter")
+    module("org.testcontainers:testcontainers", "org.testcontainers")
+    module("org.mockito:mockito-junit-jupiter", "org.mockito.junit.jupiter")
 }

buildSrc/src/main/kotlin/com.hedera.block.protobuf.gradle.kts

@@ -0,0 +1,65 @@
+/*
+ * Copyright (C) 2023-2024 Hedera Hashgraph, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import com.google.protobuf.gradle.id
+import com.hedera.block.tasks.GitClone
+
+plugins {
+    id("java-library")
+    id("com.hedera.block.conventions")
+    id("com.google.protobuf")
+}
+
+tasks.register<GitClone>("cloneHederaProtobufs") {
+    url = "https://github.com/hashgraph/hedera-protobufs.git"
+    offline = gradle.startParameter.isOffline
+    localCloneDirectory = layout.buildDirectory.dir("hedera-protobufs")
+}
+
+// Configure Protobuf Plugin to download protoc executable rather than using local installed version
+protobuf {
+    val libs = the<VersionCatalogsExtension>().named("libs")
+    protoc { artifact = "com.google.protobuf:protoc:" + libs.findVersion("google.proto").get() }
+    plugins {
+        // Add GRPC plugin as we need to generate GRPC services
+        id("grpc") {
+            artifact =
+                "io.grpc:protoc-gen-grpc-java:" + libs.findVersion("grpc.protobuf.grpc").get()
+        }
+    }
+    generateProtoTasks { ofSourceSet("main").forEach { it.plugins { id("grpc") } } }
+}
+
+tasks.javadoc {
+    options {
+        this as StandardJavadocDocletOptions
+        // There are violations in the generated protobuf code
+        addStringOption("Xdoclint:-reference,-html", "-quiet")
+    }
+}
+
+// Give JUnit more ram and make it execute tests in parallel
+tasks.test {
+    // We are running a lot of tests 10s of thousands, so they need to run in parallel. Make each
+    // class run in parallel.
+    systemProperties["junit.jupiter.execution.parallel.enabled"] = true
+    systemProperties["junit.jupiter.execution.parallel.mode.default"] = "concurrent"
+    // limit amount of threads, so we do not use all CPU
+    systemProperties["junit.jupiter.execution.parallel.config.dynamic.factor"] = "0.9"
+    // us parallel GC to keep up with high temporary garbage creation,
+    // and allow GC to use 40% of CPU if needed
+    jvmArgs("-XX:+UseParallelGC", "-XX:GCTimeRatio=90")
+}

buildSrc/src/main/kotlin/com/hedera/block/tasks/GitClone.kt

@@ -0,0 +1,91 @@
+/*
+ * Copyright (C) 2022-2024 Hedera Hashgraph, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.hedera.block.tasks
+
+import javax.inject.Inject
+import org.gradle.api.DefaultTask
+import org.gradle.api.file.DirectoryProperty
+import org.gradle.api.provider.Property
+import org.gradle.api.tasks.Input
+import org.gradle.api.tasks.Optional
+import org.gradle.api.tasks.OutputDirectory
+import org.gradle.api.tasks.TaskAction
+import org.gradle.process.ExecOperations
+
+abstract class GitClone : DefaultTask() {
+
+    @get:Input abstract val url: Property<String>
+
+    @get:Input @get:Optional abstract val tag: Property<String>
+
+    @get:Input @get:Optional abstract val branch: Property<String>
+
+    @get:Input abstract val offline: Property<Boolean>
+
+    @get:OutputDirectory abstract val localCloneDirectory: DirectoryProperty
+
+    @get:Inject protected abstract val exec: ExecOperations
+
+    init {
+        // If a 'branch' is configured, the task is never up-to-date as it may change
+        outputs.upToDateWhen { !branch.isPresent }
+    }
+
+    @TaskAction
+    fun cloneOrUpdate() {
+        if (!tag.isPresent && !branch.isPresent || tag.isPresent && branch.isPresent) {
+            throw RuntimeException("Define either 'tag' or 'branch'")
+        }
+
+        val localClone = localCloneDirectory.get()
+        if (!offline.get()) {
+            exec.exec {
+                if (!localClone.dir(".git").asFile.exists()) {
+                    workingDir = localClone.asFile.parentFile
+                    commandLine(
+                        "git",
+                        "clone",
+                        "https://github.com/hashgraph/hedera-protobufs.git",
+                        "-q"
+                    )
+                } else {
+                    workingDir = localClone.asFile
+                    commandLine("git", "fetch", "-q")
+                }
+            }
+        }
+        if (tag.isPresent) {
+            exec.exec {
+                workingDir = localClone.asFile
+                commandLine("git", "checkout", tag.get(), "-q")
+            }
+            exec.exec {
+                workingDir = localClone.asFile
+                commandLine("git", "reset", "--hard", tag.get(), "-q")
+            }
+        } else {
+            exec.exec {
+                workingDir = localClone.asFile
+                commandLine("git", "checkout", branch.get(), "-q")
+            }
+            exec.exec {
+                workingDir = localClone.asFile
+                commandLine("git", "reset", "--hard", "origin/${branch.get()}", "-q")
+            }
+        }
+    }
+}

codecov.yml

@@ -20,3 +20,4 @@ coverage:
 
 ignore:
   - "server/src/main/java/com/hedera/block/server/Server.java"
+  - "server/src/main/java/com/hedera/block/server/Translator.java"

gradle/modules.properties

@@ -12,5 +12,24 @@ com.lmax.disruptor=com.lmax:disruptor
 io.helidon.webserver=io.helidon.webserver:helidon-webserver
 io.helidon.webserver.grpc=io.helidon.webserver:helidon-webserver-grpc
 io.helidon.webserver.testing.junit5=io.helidon.webserver.testing.junit5:helidon-webserver-testing-junit5
-io.grpc=io.grpc:grpc-stub
-grpc.protobuf=io.grpc:grpc-protobuf
+google.proto=com.google.protobuf:protoc
+
+org.antlr.antlr4.runtime=org.antlr:antlr4-runtime
+
+com.google.common=com.google.guava:guava
+io.grpc.protobuf=io.grpc:grpc-protobuf
+io.grpc.stub=io.grpc:grpc-stub
+io.grpc=io.grpc:grpc-api
+com.hedera.pbj.runtime=com.hedera.pbj:pbj-runtime
+
+com.google.protobuf=com.google.protobuf:protobuf-java
+com.google.protobuf.util=com.google.protobuf:protobuf-java-util
+com.apache.commons.codec=commons-codec:commons-codec
+org.apache.commons.collections4=org.apache.commons:commons-collections4
+org.apache.commons.io=commons-io:commons-io
+org.apache.commons.lang3=org.apache.commons:commons-lang3
+org.apache.commons.compress=org.apache.commons:commons-compress
+
+org.slf4j=org.apache.logging.log4j:log4j-slf4j2-impl
+java.annotation=javax.annotation:javax.annotation-api
+org.apache.logging.log4j.slf4j2.impl=org.apache.logging.log4j:log4j-slf4j2-impl