Makr kafka dependency for runner-google-cloud-dataflow-java (#30300)

apache · Feb 13, 2024 · 491174e · 491174e
1 parent 9452a29
commit 491174e
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 2 deletions.
diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle
@@ -89,7 +89,8 @@ dependencies {
   implementation project(path: ":sdks:java:core", configuration: "shadow")
   implementation project(":sdks:java:extensions:avro")
   implementation project(":sdks:java:extensions:google-cloud-platform-core")
-  implementation project(":sdks:java:io:kafka")
+  // io-kafka is only used in PTransform override so it is optional
+  provided project(":sdks:java:io:kafka")
   implementation project(":sdks:java:io:google-cloud-platform")
   implementation project(":runners:core-construction-java")
   implementation library.java.avro

diff --git a/...le-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java b/...le-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java
@@ -565,7 +565,13 @@ private List<PTransformOverride> getOverrides(boolean streaming) {
                 new StreamingPubsubIOWriteOverrideFactory(this)));
       }
 
-      overridesBuilder.add(KafkaIO.Read.KAFKA_READ_OVERRIDE);
+      try {
+        overridesBuilder.add(KafkaIO.Read.KAFKA_READ_OVERRIDE);
+      } catch (NoClassDefFoundError e) {
+        // Do nothing. io-kafka is an optional dependency of runners-google-cloud-dataflow-java
+        // and only needed when KafkaIO is used in the pipeline.
+      }
+
       overridesBuilder.add(SubscribeTransform.V1_READ_OVERRIDE);
 
       if (!hasExperiment(options, "enable_file_dynamic_sharding")) {