[GOBBLIN-1914] Configurable Gobblin Application Master class for Yarn

apache · Sep 20, 2023 · 648e2ef · 648e2ef
1 parent a289613
commit 648e2ef
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 6 deletions.
diff --git a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnAppLauncher.java b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnAppLauncher.java
@@ -801,23 +801,25 @@ private void addJobConfPackage(String jobConfPackagePath, Path destDir, Map<Stri
 
   @VisibleForTesting
   protected String buildApplicationMasterCommand(String applicationId, int memoryMbs) {
-    String appMasterClassName = GobblinApplicationMaster.class.getSimpleName();
+    String appMasterClass = ConfigUtils.getString(
+       config, GobblinYarnConfigurationKeys.APP_MASTER_CLASS, GobblinYarnConfigurationKeys.DEFAULT_APP_MASTER_CLASS);
+    String logFileName = GobblinApplicationMaster.class.getSimpleName();
     return new StringBuilder()
         .append(ApplicationConstants.Environment.JAVA_HOME.$()).append("/bin/java")
         .append(" -Xmx").append((int) (memoryMbs * this.jvmMemoryXmxRatio) - this.jvmMemoryOverheadMbs).append("M")
         .append(" -D").append(GobblinYarnConfigurationKeys.JVM_USER_TIMEZONE_CONFIG).append("=").append(this.containerTimezone)
         .append(" -D").append(GobblinYarnConfigurationKeys.GOBBLIN_YARN_CONTAINER_LOG_DIR_NAME).append("=").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR)
-        .append(" -D").append(GobblinYarnConfigurationKeys.GOBBLIN_YARN_CONTAINER_LOG_FILE_NAME).append("=").append(appMasterClassName).append(".").append(ApplicationConstants.STDOUT)
+        .append(" -D").append(GobblinYarnConfigurationKeys.GOBBLIN_YARN_CONTAINER_LOG_FILE_NAME).append("=").append(logFileName).append(".").append(ApplicationConstants.STDOUT)
         .append(" ").append(JvmUtils.formatJvmArguments(this.appMasterJvmArgs))
-        .append(" ").append(GobblinApplicationMaster.class.getName())
+        .append(" ").append(appMasterClass)
         .append(" --").append(GobblinClusterConfigurationKeys.APPLICATION_NAME_OPTION_NAME)
         .append(" ").append(this.applicationName)
         .append(" --").append(GobblinClusterConfigurationKeys.APPLICATION_ID_OPTION_NAME)
         .append(" ").append(applicationId)
         .append(" 1>").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append(File.separator).append(
-            appMasterClassName).append(".").append(ApplicationConstants.STDOUT)
+            logFileName).append(".").append(ApplicationConstants.STDOUT)
         .append(" 2>").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append(File.separator).append(
-            appMasterClassName).append(".").append(ApplicationConstants.STDERR)
+            logFileName).append(".").append(ApplicationConstants.STDERR)
         .toString();
   }
 
@@ -1071,4 +1073,4 @@ public void run() {
 
     gobblinYarnAppLauncher.launch();
   }
-}
+}
diff --git a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java
@@ -30,6 +30,9 @@ public class GobblinYarnConfigurationKeys {
   public static final String GOBBLIN_YARN_PREFIX = "gobblin.yarn.";
 
   // General Gobblin Yarn application configuration properties.
+
+  public static final String APP_MASTER_CLASS = GOBBLIN_YARN_PREFIX + "app.master.class";
+  public static final String DEFAULT_APP_MASTER_CLASS = GobblinApplicationMaster.class.getName();
   public static final String APPLICATION_NAME_KEY = GOBBLIN_YARN_PREFIX + "app.name";
   public static final String APP_QUEUE_KEY = GOBBLIN_YARN_PREFIX + "app.queue";
   public static final String APP_REPORT_INTERVAL_MINUTES_KEY = GOBBLIN_YARN_PREFIX + "app.report.interval.minutes";