Configurable logs

apache · Sep 21, 2023 · 3c25ab9 · 3c25ab9
1 parent 471a96e
commit 3c25ab9
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 5 deletions.
diff --git a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnAppLauncher.java b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnAppLauncher.java
@@ -342,7 +342,8 @@ public void initializeYarnClients(Config config) {
    * @throws IOException if there's something wrong launching the application
    * @throws YarnException if there's something wrong launching the application
    */
-  public void launch() throws IOException, YarnException, InterruptedException {
+  public void launch()
+      throws IOException, YarnException, InterruptedException, ClassNotFoundException {
     this.eventBus.register(this);
 
     if (this.isHelixClusterManaged) {
@@ -603,7 +604,8 @@ Optional<ApplicationId> getReconnectableApplicationId() throws YarnException, IO
    * @throws YarnException if there's anything wrong setting up and submitting the Yarn application
    */
   @VisibleForTesting
-  ApplicationId setupAndSubmitApplication() throws IOException, YarnException, InterruptedException {
+  ApplicationId setupAndSubmitApplication()
+      throws IOException, YarnException, InterruptedException, ClassNotFoundException {
     LOGGER.info("creating new yarn application");
     YarnClientApplication gobblinYarnApp = this.yarnClient.createApplication();
     ApplicationSubmissionContext appSubmissionContext = gobblinYarnApp.getApplicationSubmissionContext();
@@ -800,10 +802,12 @@ private void addJobConfPackage(String jobConfPackagePath, Path destDir, Map<Stri
   }
 
   @VisibleForTesting
-  protected String buildApplicationMasterCommand(String applicationId, int memoryMbs) {
+  protected String buildApplicationMasterCommand(String applicationId, int memoryMbs)
+      throws ClassNotFoundException {
     String appMasterClass = ConfigUtils.getString(
        config, GobblinYarnConfigurationKeys.APP_MASTER_CLASS, GobblinYarnConfigurationKeys.DEFAULT_APP_MASTER_CLASS);
-    String logFileName = GobblinApplicationMaster.class.getSimpleName();
+    String logFileName = ConfigUtils.getString(
+        config, GobblinYarnConfigurationKeys.APP_MASTER_LOG_FILE_NAME, Class.forName(appMasterClass).getSimpleName());
     return new StringBuilder()
         .append(ApplicationConstants.Environment.JAVA_HOME.$()).append("/bin/java")
         .append(" -Xmx").append((int) (memoryMbs * this.jvmMemoryXmxRatio) - this.jvmMemoryOverheadMbs).append("M")

diff --git a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnConfigurationKeys.java
@@ -32,6 +32,7 @@ public class GobblinYarnConfigurationKeys {
   // General Gobblin Yarn application configuration properties.
   public static final String APP_MASTER_CLASS = GOBBLIN_YARN_PREFIX + "app.master.class";
   public static final String DEFAULT_APP_MASTER_CLASS = GobblinApplicationMaster.class.getName();
+  public static final String APP_MASTER_LOG_FILE_NAME = GOBBLIN_YARN_PREFIX + "app.master.log.file.name";
   public static final String APPLICATION_NAME_KEY = GOBBLIN_YARN_PREFIX + "app.name";
   public static final String APP_QUEUE_KEY = GOBBLIN_YARN_PREFIX + "app.queue";
   public static final String APP_REPORT_INTERVAL_MINUTES_KEY = GOBBLIN_YARN_PREFIX + "app.report.interval.minutes";

diff --git a/gobblin-yarn/src/test/java/org/apache/gobblin/yarn/GobblinYarnAppLauncherTest.java b/gobblin-yarn/src/test/java/org/apache/gobblin/yarn/GobblinYarnAppLauncherTest.java
@@ -224,11 +224,15 @@ public void setUp() throws Exception {
   }
 
   @Test
-  public void testBuildApplicationMasterCommand() {
+  public void testBuildApplicationMasterCommand()
+      throws ClassNotFoundException {
     String command = this.gobblinYarnAppLauncher.buildApplicationMasterCommand("application_1234_3456", 64);
 
     // 41 is from 64 * 0.8 - 10
     Assert.assertTrue(command.contains("-Xmx41"));
+    Assert.assertTrue(command.contains("org.apache.gobblin.yarn.GobblinApplicationMaster"));
+    Assert.assertTrue(command.contains("GobblinApplicationMaster.stdout"));
+    Assert.assertTrue(command.contains("GobblinApplicationMaster.stderr"));
   }
 
   @Test