From 9bd42245d35c32b64ae7b1280766ebd70178770c Mon Sep 17 00:00:00 2001 From: Akshay Rai Date: Mon, 19 Apr 2021 13:27:19 -0700 Subject: [PATCH 1/5] Isolate engines and allow configuration of different jdks on the client --- .../build.gradle | 22 ++++++--- .../linkedin/transport/plugin/Defaults.java | 14 +++--- .../transport/plugin/TransportPlugin.java | 49 ++++++++++++++++--- .../plugin/TransportPluginConfig.java | 6 +++ 4 files changed, 71 insertions(+), 20 deletions(-) diff --git a/transportable-udfs-examples/transportable-udfs-example-udfs/build.gradle b/transportable-udfs-examples/transportable-udfs-example-udfs/build.gradle index 40d7f387..d1480f66 100644 --- a/transportable-udfs-examples/transportable-udfs-example-udfs/build.gradle +++ b/transportable-udfs-examples/transportable-udfs-example-udfs/build.gradle @@ -7,18 +7,26 @@ buildscript { apply plugin: 'java' apply plugin: 'com.linkedin.transport.plugin' +println 'Running build for Hive, Presto & Spark' +transport.engines = ['hive', 'presto', 'spark'] +java { + toolchain.languageVersion.set(JavaLanguageVersion.of(8)) +} + +// If the license plugin is applied, disable license checks for the autogenerated source sets +tasks.whenTaskAdded { task -> + if (task.name.startsWith("licensePresto") + || task.name.startsWith("licenseHive") + || task.name.startsWith("licenseSpark")) { + task.enabled = false + } +} + dependencies { // TODO: Reference all external dependencies from a single gradle file compile('com.google.guava:guava:24.1-jre') compile('org.apache.commons:commons-io:1.3.2') } -// If the license plugin is applied, disable license checks for the autogenerated source sets -plugins.withId('com.github.hierynomus.license') { - licenseHive.enabled = false - licensePresto.enabled = false - licenseSpark.enabled = false -} - // TODO: Add a debugPlatform flag to allow debugging specific test methods in IntelliJ // for a particular platform other than default diff --git a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/Defaults.java b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/Defaults.java index 64a967f4..7013bc36 100644 --- a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/Defaults.java +++ b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/Defaults.java @@ -56,7 +56,7 @@ private static Properties loadDefaultVersions() { getDependencyConfiguration(RUNTIME_ONLY, "com.linkedin.transport:transportable-udfs-test-generic", "transport") ); - static final List DEFAULT_PLATFORMS = ImmutableList.of( + static final Platform PRESTO_PLATFORM = new Platform( "presto", Language.JAVA, @@ -73,7 +73,9 @@ private static Properties loadDefaultVersions() { // converters drop dependencies with classifiers, so we apply this dependency explicitly getDependencyConfiguration(RUNTIME_ONLY, "io.prestosql:presto-main", "presto", "tests") ), - ImmutableList.of(new ThinJarPackaging(), new DistributionPackaging())), + ImmutableList.of(new ThinJarPackaging(), new DistributionPackaging())); + + static final Platform HIVE_PLATFORM = new Platform( "hive", Language.JAVA, @@ -86,7 +88,9 @@ private static Properties loadDefaultVersions() { getDependencyConfiguration(RUNTIME_ONLY, "com.linkedin.transport:transportable-udfs-test-hive", "transport") ), - ImmutableList.of(new ShadedJarPackaging(ImmutableList.of("org.apache.hadoop", "org.apache.hive"), null))), + ImmutableList.of(new ShadedJarPackaging(ImmutableList.of("org.apache.hadoop", "org.apache.hive"), null))); + + static final Platform SPARK_PLATFORM = new Platform( "spark", Language.SCALA, @@ -102,9 +106,7 @@ private static Properties loadDefaultVersions() { ), ImmutableList.of(new ShadedJarPackaging( ImmutableList.of("org.apache.hadoop", "org.apache.spark"), - ImmutableList.of("com.linkedin.transport.spark.**"))) - ) - ); + ImmutableList.of("com.linkedin.transport.spark.**")))); private static DependencyConfiguration getDependencyConfiguration(ConfigurationType configurationType, String module, String platform) { diff --git a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPlugin.java b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPlugin.java index 2f8e2984..6206bed6 100644 --- a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPlugin.java +++ b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPlugin.java @@ -11,8 +11,11 @@ import java.io.File; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Arrays; import java.util.List; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; +import org.gradle.api.InvalidUserDataException; import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.Task; @@ -23,6 +26,7 @@ import org.gradle.api.plugins.JavaPluginConvention; import org.gradle.api.plugins.scala.ScalaPlugin; import org.gradle.api.tasks.SourceSet; +import org.gradle.api.tasks.SourceSetContainer; import org.gradle.api.tasks.TaskProvider; import org.gradle.api.tasks.testing.Test; import org.gradle.language.base.plugins.LifecycleBasePlugin; @@ -47,26 +51,31 @@ */ public class TransportPlugin implements Plugin { + private static final String HIVE_ENGINE = "hive"; + private static final String SPARK_ENGINE = "spark"; + private static final String PRESTO_ENGINE = "presto"; + public void apply(Project project) { TransportPluginConfig extension = project.getExtensions().create("transport", TransportPluginConfig.class, project); + AtomicReference mainSourceSet = new AtomicReference<>(); + AtomicReference testSourceSet = new AtomicReference<>(); project.getPlugins().withType(JavaPlugin.class, (javaPlugin) -> { project.getPlugins().apply(ScalaPlugin.class); project.getPlugins().apply(DistributionPlugin.class); project.getConfigurations().create(ShadowBasePlugin.getCONFIGURATION_NAME()); - JavaPluginConvention javaConvention = project.getConvention().getPlugin(JavaPluginConvention.class); - SourceSet mainSourceSet = javaConvention.getSourceSets().getByName(extension.mainSourceSetName); - SourceSet testSourceSet = javaConvention.getSourceSets().getByName(extension.testSourceSetName); + SourceSetContainer sourceSets = project.getConvention().getPlugin(JavaPluginConvention.class).getSourceSets(); + mainSourceSet.set(sourceSets.getByName(extension.mainSourceSetName)); + testSourceSet.set(sourceSets.getByName(extension.testSourceSetName)); - configureBaseSourceSets(project, mainSourceSet, testSourceSet); - Defaults.DEFAULT_PLATFORMS.forEach( - platform -> configurePlatform(project, platform, mainSourceSet, testSourceSet, extension.outputDirFile)); + configureBaseSourceSets(project, mainSourceSet.get(), testSourceSet.get()); }); + // Disable Jacoco for platform test tasks as it is known to cause issues with Presto and Hive tests project.getPlugins().withType(JacocoPlugin.class, (jacocoPlugin) -> { - Defaults.DEFAULT_PLATFORMS.forEach(platform -> { + Arrays.asList(Defaults.PRESTO_PLATFORM, Defaults.HIVE_PLATFORM).forEach(platform -> { project.getTasksByName(testTaskName(platform), true).forEach(task -> { JacocoTaskExtension jacocoExtension = task.getExtensions().findByType(JacocoTaskExtension.class); if (jacocoExtension != null) { @@ -75,6 +84,32 @@ public void apply(Project project) { }); }); }); + + // Process this after the client configures the 'engine' plugin parameter + project.afterEvaluate(p -> { + if (extension.engines == null || extension.engines.isEmpty()) { + throw new InvalidUserDataException("Please specify engines"); + } + + // Isolate engine/platform dependencies + extension.engines.forEach(engine -> { + if (engine.equalsIgnoreCase(HIVE_ENGINE)) { + project.getPlugins().withType(JavaPlugin.class, (javaPlugin) -> { + configurePlatform(project, Defaults.HIVE_PLATFORM, mainSourceSet.get(), testSourceSet.get(), extension.outputDirFile); + }); + + } else if (engine.equalsIgnoreCase(SPARK_ENGINE)) { + project.getPlugins().withType(JavaPlugin.class, (javaPlugin) -> { + configurePlatform(project, Defaults.SPARK_PLATFORM, mainSourceSet.get(), testSourceSet.get(), extension.outputDirFile); + }); + + } else if (engine.equalsIgnoreCase(PRESTO_ENGINE)) { + project.getPlugins().withType(JavaPlugin.class, (javaPlugin) -> { + configurePlatform(project, Defaults.PRESTO_PLATFORM, mainSourceSet.get(), testSourceSet.get(), extension.outputDirFile); + }); + } + }); + }); } /** diff --git a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPluginConfig.java b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPluginConfig.java index deff599a..9b915fb8 100644 --- a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPluginConfig.java +++ b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPluginConfig.java @@ -6,6 +6,7 @@ package com.linkedin.transport.plugin; import java.io.File; +import java.util.List; import org.gradle.api.Project; @@ -33,6 +34,11 @@ public class TransportPluginConfig { * The output code-gen directory, relative the to the project directory. */ public File outputDirFile; + /** + * The execution engines for which the udf artifacts need to be built. + * (The client configures this while applying the transport plugin) + */ + public List engines; /** * Create a config object from the gradle {@link Project}. From dfc4722d98619f5a233f3945814b908691fbaf2f Mon Sep 17 00:00:00 2001 From: Akshay Rai Date: Thu, 22 Apr 2021 19:22:15 -0700 Subject: [PATCH 2/5] Addressed comments from Shardul --- .../build.gradle | 14 +-- .../linkedin/transport/plugin/Defaults.java | 18 ++-- .../linkedin/transport/plugin/Platform.java | 9 +- .../transport/plugin/TransportPlugin.java | 86 +++++++++---------- .../plugin/TransportPluginConfig.java | 6 -- transportable-udfs-presto/build.gradle | 23 +---- .../build.gradle | 4 + 7 files changed, 70 insertions(+), 90 deletions(-) diff --git a/transportable-udfs-examples/transportable-udfs-example-udfs/build.gradle b/transportable-udfs-examples/transportable-udfs-example-udfs/build.gradle index d1480f66..04628b2d 100644 --- a/transportable-udfs-examples/transportable-udfs-example-udfs/build.gradle +++ b/transportable-udfs-examples/transportable-udfs-example-udfs/build.gradle @@ -7,10 +7,10 @@ buildscript { apply plugin: 'java' apply plugin: 'com.linkedin.transport.plugin' -println 'Running build for Hive, Presto & Spark' -transport.engines = ['hive', 'presto', 'spark'] -java { - toolchain.languageVersion.set(JavaLanguageVersion.of(8)) +dependencies { + // TODO: Reference all external dependencies from a single gradle file + compile('com.google.guava:guava:24.1-jre') + compile('org.apache.commons:commons-io:1.3.2') } // If the license plugin is applied, disable license checks for the autogenerated source sets @@ -22,11 +22,5 @@ tasks.whenTaskAdded { task -> } } -dependencies { - // TODO: Reference all external dependencies from a single gradle file - compile('com.google.guava:guava:24.1-jre') - compile('org.apache.commons:commons-io:1.3.2') -} - // TODO: Add a debugPlatform flag to allow debugging specific test methods in IntelliJ // for a particular platform other than default diff --git a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/Defaults.java b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/Defaults.java index 7013bc36..9347a819 100644 --- a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/Defaults.java +++ b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/Defaults.java @@ -16,6 +16,7 @@ import java.io.InputStream; import java.util.List; import java.util.Properties; +import org.gradle.jvm.toolchain.JavaLanguageVersion; import static com.linkedin.transport.plugin.ConfigurationType.*; @@ -56,11 +57,12 @@ private static Properties loadDefaultVersions() { getDependencyConfiguration(RUNTIME_ONLY, "com.linkedin.transport:transportable-udfs-test-generic", "transport") ); - static final Platform PRESTO_PLATFORM = + static final List DEFAULT_PLATFORMS = ImmutableList.of( new Platform( "presto", Language.JAVA, PrestoWrapperGenerator.class, + JavaLanguageVersion.of(11), ImmutableList.of( getDependencyConfiguration(IMPLEMENTATION, "com.linkedin.transport:transportable-udfs-presto", "transport"), @@ -73,13 +75,12 @@ private static Properties loadDefaultVersions() { // converters drop dependencies with classifiers, so we apply this dependency explicitly getDependencyConfiguration(RUNTIME_ONLY, "io.prestosql:presto-main", "presto", "tests") ), - ImmutableList.of(new ThinJarPackaging(), new DistributionPackaging())); - - static final Platform HIVE_PLATFORM = + ImmutableList.of(new ThinJarPackaging(), new DistributionPackaging())), new Platform( "hive", Language.JAVA, HiveWrapperGenerator.class, + JavaLanguageVersion.of(8), ImmutableList.of( getDependencyConfiguration(IMPLEMENTATION, "com.linkedin.transport:transportable-udfs-hive", "transport"), getDependencyConfiguration(COMPILE_ONLY, "org.apache.hive:hive-exec", "hive") @@ -88,13 +89,12 @@ private static Properties loadDefaultVersions() { getDependencyConfiguration(RUNTIME_ONLY, "com.linkedin.transport:transportable-udfs-test-hive", "transport") ), - ImmutableList.of(new ShadedJarPackaging(ImmutableList.of("org.apache.hadoop", "org.apache.hive"), null))); - - static final Platform SPARK_PLATFORM = + ImmutableList.of(new ShadedJarPackaging(ImmutableList.of("org.apache.hadoop", "org.apache.hive"), null))), new Platform( "spark", Language.SCALA, SparkWrapperGenerator.class, + JavaLanguageVersion.of(8), ImmutableList.of( getDependencyConfiguration(IMPLEMENTATION, "com.linkedin.transport:transportable-udfs-spark", "transport"), @@ -106,7 +106,9 @@ private static Properties loadDefaultVersions() { ), ImmutableList.of(new ShadedJarPackaging( ImmutableList.of("org.apache.hadoop", "org.apache.spark"), - ImmutableList.of("com.linkedin.transport.spark.**")))); + ImmutableList.of("com.linkedin.transport.spark.**"))) + ) + ); private static DependencyConfiguration getDependencyConfiguration(ConfigurationType configurationType, String module, String platform) { diff --git a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/Platform.java b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/Platform.java index b3d87679..7acf6847 100644 --- a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/Platform.java +++ b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/Platform.java @@ -8,6 +8,7 @@ import com.linkedin.transport.codegen.WrapperGenerator; import com.linkedin.transport.plugin.packaging.Packaging; import java.util.List; +import org.gradle.jvm.toolchain.JavaLanguageVersion; /** @@ -21,12 +22,14 @@ public class Platform { private final List _defaultWrapperDependencyConfigurations; private final List _defaultTestDependencyConfigurations; private final List _packaging; + private final JavaLanguageVersion _javaLanguageVersion; public Platform(String name, Language language, Class wrapperGeneratorClass, - List defaultWrapperDependencyConfigurations, + JavaLanguageVersion javaLanguageVersion, List defaultWrapperDependencyConfigurations, List defaultTestDependencyConfigurations, List packaging) { _name = name; _language = language; + _javaLanguageVersion = javaLanguageVersion; _wrapperGeneratorClass = wrapperGeneratorClass; _defaultWrapperDependencyConfigurations = defaultWrapperDependencyConfigurations; _defaultTestDependencyConfigurations = defaultTestDependencyConfigurations; @@ -56,4 +59,8 @@ public List getDefaultTestDependencyConfigurations() { public List getPackaging() { return _packaging; } + + public JavaLanguageVersion getJavaLanguageVersion() { + return _javaLanguageVersion; + } } diff --git a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPlugin.java b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPlugin.java index 6206bed6..3034f6c8 100644 --- a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPlugin.java +++ b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPlugin.java @@ -11,11 +11,8 @@ import java.io.File; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Arrays; import java.util.List; -import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; -import org.gradle.api.InvalidUserDataException; import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.Task; @@ -26,9 +23,10 @@ import org.gradle.api.plugins.JavaPluginConvention; import org.gradle.api.plugins.scala.ScalaPlugin; import org.gradle.api.tasks.SourceSet; -import org.gradle.api.tasks.SourceSetContainer; import org.gradle.api.tasks.TaskProvider; +import org.gradle.api.tasks.compile.JavaCompile; import org.gradle.api.tasks.testing.Test; +import org.gradle.jvm.toolchain.JavaToolchainService; import org.gradle.language.base.plugins.LifecycleBasePlugin; import org.gradle.testing.jacoco.plugins.JacocoPlugin; import org.gradle.testing.jacoco.plugins.JacocoTaskExtension; @@ -51,31 +49,27 @@ */ public class TransportPlugin implements Plugin { - private static final String HIVE_ENGINE = "hive"; - private static final String SPARK_ENGINE = "spark"; - private static final String PRESTO_ENGINE = "presto"; - public void apply(Project project) { TransportPluginConfig extension = project.getExtensions().create("transport", TransportPluginConfig.class, project); - AtomicReference mainSourceSet = new AtomicReference<>(); - AtomicReference testSourceSet = new AtomicReference<>(); project.getPlugins().withType(JavaPlugin.class, (javaPlugin) -> { project.getPlugins().apply(ScalaPlugin.class); project.getPlugins().apply(DistributionPlugin.class); project.getConfigurations().create(ShadowBasePlugin.getCONFIGURATION_NAME()); - SourceSetContainer sourceSets = project.getConvention().getPlugin(JavaPluginConvention.class).getSourceSets(); - mainSourceSet.set(sourceSets.getByName(extension.mainSourceSetName)); - testSourceSet.set(sourceSets.getByName(extension.testSourceSetName)); + JavaPluginConvention javaConvention = project.getConvention().getPlugin(JavaPluginConvention.class); + SourceSet mainSourceSet = javaConvention.getSourceSets().getByName(extension.mainSourceSetName); + SourceSet testSourceSet = javaConvention.getSourceSets().getByName(extension.testSourceSetName); - configureBaseSourceSets(project, mainSourceSet.get(), testSourceSet.get()); + configureBaseSourceSets(project, mainSourceSet, testSourceSet); + Defaults.DEFAULT_PLATFORMS.forEach( + platform -> configurePlatform(project, platform, mainSourceSet, testSourceSet, extension.outputDirFile)); }); // Disable Jacoco for platform test tasks as it is known to cause issues with Presto and Hive tests project.getPlugins().withType(JacocoPlugin.class, (jacocoPlugin) -> { - Arrays.asList(Defaults.PRESTO_PLATFORM, Defaults.HIVE_PLATFORM).forEach(platform -> { + Defaults.DEFAULT_PLATFORMS.forEach(platform -> { project.getTasksByName(testTaskName(platform), true).forEach(task -> { JacocoTaskExtension jacocoExtension = task.getExtensions().findByType(JacocoTaskExtension.class); if (jacocoExtension != null) { @@ -84,32 +78,6 @@ public void apply(Project project) { }); }); }); - - // Process this after the client configures the 'engine' plugin parameter - project.afterEvaluate(p -> { - if (extension.engines == null || extension.engines.isEmpty()) { - throw new InvalidUserDataException("Please specify engines"); - } - - // Isolate engine/platform dependencies - extension.engines.forEach(engine -> { - if (engine.equalsIgnoreCase(HIVE_ENGINE)) { - project.getPlugins().withType(JavaPlugin.class, (javaPlugin) -> { - configurePlatform(project, Defaults.HIVE_PLATFORM, mainSourceSet.get(), testSourceSet.get(), extension.outputDirFile); - }); - - } else if (engine.equalsIgnoreCase(SPARK_ENGINE)) { - project.getPlugins().withType(JavaPlugin.class, (javaPlugin) -> { - configurePlatform(project, Defaults.SPARK_PLATFORM, mainSourceSet.get(), testSourceSet.get(), extension.outputDirFile); - }); - - } else if (engine.equalsIgnoreCase(PRESTO_ENGINE)) { - project.getPlugins().withType(JavaPlugin.class, (javaPlugin) -> { - configurePlatform(project, Defaults.PRESTO_PLATFORM, mainSourceSet.get(), testSourceSet.get(), extension.outputDirFile); - }); - } - }); - }); } /** @@ -154,6 +122,7 @@ private SourceSet configureSourceSet(Project project, Platform platform, SourceS Path wrapperResourceOutputDir = platformBaseDir.resolve("resources"); return javaConvention.getSourceSets().create(platform.getName(), sourceSet -> { + /* Creates a SourceSet and set the source directories for a given platform. E.g. For the Presto platform, @@ -227,9 +196,32 @@ private TaskProvider configureGenerateWrappersTask(Project task.dependsOn(project.getTasks().named(inputSourceSet.getClassesTaskName())); }); - project.getTasks() - .named(outputSourceSet.getCompileTaskName(platform.getLanguage().toString())) - .configure(task -> task.dependsOn(generateWrappersTask)); + + switch (platform.getLanguage()) { + case JAVA: + project.getTasks() + .named(outputSourceSet.getCompileTaskName(platform.getLanguage().toString()), JavaCompile.class) + .configure(task -> { + task.dependsOn(generateWrappersTask); + // configure compile task to run with platform specific jdk + JavaToolchainService javaToolchains = project.getExtensions().getByType(JavaToolchainService.class); + task.getJavaCompiler().set(javaToolchains.compilerFor(toolChainSpec -> { + toolChainSpec.getLanguageVersion().set(platform.getJavaLanguageVersion()); + })); + }); + break; + case SCALA: + project.getTasks() + .named(outputSourceSet.getCompileTaskName(platform.getLanguage().toString())) + .configure(task -> { + task.dependsOn(generateWrappersTask); + // todo: configure task to run with platform specific jdk (currently picks from local env) + // Toolchain support is only available in the Java plugins and for the tasks they define. + // Support for the Scala plugin is not released yet. + // Ref: https://docs.gradle.org/7.0/userguide/toolchains.html#sec:consuming + }); + break; + } return generateWrappersTask; } @@ -292,6 +284,12 @@ task prestoTest(type: Test, dependsOn: test) { task.setClasspath(testClasspath); task.useTestNG(); task.mustRunAfter(project.getTasks().named("test")); + + // configure test task to run with platform specific jdk + JavaToolchainService javaToolchains = project.getExtensions().getByType(JavaToolchainService.class); + task.getJavaLauncher().set(javaToolchains.launcherFor(toolChainSpec -> { + toolChainSpec.getLanguageVersion().set(platform.getJavaLanguageVersion()); + })); }); } diff --git a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPluginConfig.java b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPluginConfig.java index 9b915fb8..deff599a 100644 --- a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPluginConfig.java +++ b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPluginConfig.java @@ -6,7 +6,6 @@ package com.linkedin.transport.plugin; import java.io.File; -import java.util.List; import org.gradle.api.Project; @@ -34,11 +33,6 @@ public class TransportPluginConfig { * The output code-gen directory, relative the to the project directory. */ public File outputDirFile; - /** - * The execution engines for which the udf artifacts need to be built. - * (The client configures this while applying the transport plugin) - */ - public List engines; /** * Create a config object from the gradle {@link Project}. diff --git a/transportable-udfs-presto/build.gradle b/transportable-udfs-presto/build.gradle index 4141cb79..4f4213c1 100644 --- a/transportable-udfs-presto/build.gradle +++ b/transportable-udfs-presto/build.gradle @@ -1,26 +1,7 @@ apply plugin: 'java' -buildscript { - repositories { - mavenCentral() - } - dependencies { - classpath group:'io.prestosql', name: 'presto-main', version: project.ext.'presto-version' - } -} - -import com.google.common.base.StandardSystemProperty -import io.prestosql.server.JavaVersion -task verifyPrestoJvmRequirements(type:Exec) { - String javaVersion = StandardSystemProperty.JAVA_VERSION.value() - if (javaVersion == null) { - throw new GradleException("Java version not defined") - } - JavaVersion version = JavaVersion.parse(javaVersion) - if (!(version.getMajor() == 8 && version.getUpdate().isPresent() && version.getUpdate().getAsInt() >= 151) - || (version.getMajor() >= 9)) { - throw new GradleException(String.format("Presto requires Java 8u151+ (found %s)", version)) - } +java { + toolchain.languageVersion.set(JavaLanguageVersion.of(11)) } dependencies { diff --git a/transportable-udfs-test/transportable-udfs-test-presto/build.gradle b/transportable-udfs-test/transportable-udfs-test-presto/build.gradle index 0e7a6615..982751a3 100644 --- a/transportable-udfs-test/transportable-udfs-test-presto/build.gradle +++ b/transportable-udfs-test/transportable-udfs-test-presto/build.gradle @@ -1,5 +1,9 @@ apply plugin: 'java' +java { + toolchain.languageVersion.set(JavaLanguageVersion.of(11)) +} + dependencies { compile project(":transportable-udfs-api") compile project(":transportable-udfs-test:transportable-udfs-test-api") From ff32bfab8438f7d351cb1286db97ce95dc541bab Mon Sep 17 00:00:00 2001 From: Akshay Rai Date: Fri, 23 Apr 2021 17:46:30 -0700 Subject: [PATCH 3/5] fixed checkstyles --- .../main/java/com/linkedin/transport/plugin/TransportPlugin.java | 1 + 1 file changed, 1 insertion(+) diff --git a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPlugin.java b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPlugin.java index 3034f6c8..3386e91d 100644 --- a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPlugin.java +++ b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPlugin.java @@ -211,6 +211,7 @@ private TaskProvider configureGenerateWrappersTask(Project }); break; case SCALA: + default: project.getTasks() .named(outputSourceSet.getCompileTaskName(platform.getLanguage().toString())) .configure(task -> { From 0719536751d45d8d9f147500bd5d5744f4bad62b Mon Sep 17 00:00:00 2001 From: Akshay Rai Date: Sun, 25 Apr 2021 18:18:07 -0700 Subject: [PATCH 4/5] address comments - cleaned up references --- .../build.gradle | 10 ++-- .../transport/plugin/TransportPlugin.java | 46 ++++++++----------- 2 files changed, 24 insertions(+), 32 deletions(-) diff --git a/transportable-udfs-examples/transportable-udfs-example-udfs/build.gradle b/transportable-udfs-examples/transportable-udfs-example-udfs/build.gradle index 04628b2d..40d7f387 100644 --- a/transportable-udfs-examples/transportable-udfs-example-udfs/build.gradle +++ b/transportable-udfs-examples/transportable-udfs-example-udfs/build.gradle @@ -14,12 +14,10 @@ dependencies { } // If the license plugin is applied, disable license checks for the autogenerated source sets -tasks.whenTaskAdded { task -> - if (task.name.startsWith("licensePresto") - || task.name.startsWith("licenseHive") - || task.name.startsWith("licenseSpark")) { - task.enabled = false - } +plugins.withId('com.github.hierynomus.license') { + licenseHive.enabled = false + licensePresto.enabled = false + licenseSpark.enabled = false } // TODO: Add a debugPlatform flag to allow debugging specific test methods in IntelliJ diff --git a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPlugin.java b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPlugin.java index 3386e91d..8700e336 100644 --- a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPlugin.java +++ b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPlugin.java @@ -32,6 +32,7 @@ import org.gradle.testing.jacoco.plugins.JacocoTaskExtension; import static com.linkedin.transport.plugin.ConfigurationType.*; +import static com.linkedin.transport.plugin.Language.*; import static com.linkedin.transport.plugin.SourceSetUtils.*; @@ -196,34 +197,27 @@ private TaskProvider configureGenerateWrappersTask(Project task.dependsOn(project.getTasks().named(inputSourceSet.getClassesTaskName())); }); - - switch (platform.getLanguage()) { - case JAVA: - project.getTasks() - .named(outputSourceSet.getCompileTaskName(platform.getLanguage().toString()), JavaCompile.class) - .configure(task -> { - task.dependsOn(generateWrappersTask); - // configure compile task to run with platform specific jdk - JavaToolchainService javaToolchains = project.getExtensions().getByType(JavaToolchainService.class); - task.getJavaCompiler().set(javaToolchains.compilerFor(toolChainSpec -> { - toolChainSpec.getLanguageVersion().set(platform.getJavaLanguageVersion()); - })); - }); - break; - case SCALA: - default: - project.getTasks() - .named(outputSourceSet.getCompileTaskName(platform.getLanguage().toString())) - .configure(task -> { - task.dependsOn(generateWrappersTask); - // todo: configure task to run with platform specific jdk (currently picks from local env) - // Toolchain support is only available in the Java plugins and for the tasks they define. - // Support for the Scala plugin is not released yet. - // Ref: https://docs.gradle.org/7.0/userguide/toolchains.html#sec:consuming - }); - break; + // todo: configure task to run with platform specific jdk (currently picks from local env) + // Toolchain support is only available in the Java plugins and for the tasks they define. + // Support for the Scala plugin is not released yet. + // Ref: https://docs.gradle.org/7.0/userguide/toolchains.html#sec:consuming + if (platform.getLanguage() == JAVA) { + project.getTasks() + .named(outputSourceSet.getCompileTaskName(platform.getLanguage().toString()), JavaCompile.class, task -> { + // configure compile task to run with platform specific jdk + JavaToolchainService javaToolchains = project.getExtensions().getByType(JavaToolchainService.class); + task.getJavaCompiler().set(javaToolchains.compilerFor(toolChainSpec -> { + toolChainSpec.getLanguageVersion().set(platform.getJavaLanguageVersion()); + })); + }); } + project.getTasks() + .named(outputSourceSet.getCompileTaskName(platform.getLanguage().toString())) + .configure(task -> { + task.dependsOn(generateWrappersTask); + }); + return generateWrappersTask; } From a4904a985dd6b5edb28d57662d3a0bd1a45633d7 Mon Sep 17 00:00:00 2001 From: Akshay Rai Date: Sun, 25 Apr 2021 18:35:03 -0700 Subject: [PATCH 5/5] checkin missing files --- .../linkedin/transport/plugin/TransportPlugin.java | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPlugin.java b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPlugin.java index 8700e336..47188a92 100644 --- a/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPlugin.java +++ b/transportable-udfs-plugin/src/main/java/com/linkedin/transport/plugin/TransportPlugin.java @@ -67,7 +67,6 @@ public void apply(Project project) { Defaults.DEFAULT_PLATFORMS.forEach( platform -> configurePlatform(project, platform, mainSourceSet, testSourceSet, extension.outputDirFile)); }); - // Disable Jacoco for platform test tasks as it is known to cause issues with Presto and Hive tests project.getPlugins().withType(JacocoPlugin.class, (jacocoPlugin) -> { Defaults.DEFAULT_PLATFORMS.forEach(platform -> { @@ -123,7 +122,6 @@ private SourceSet configureSourceSet(Project project, Platform platform, SourceS Path wrapperResourceOutputDir = platformBaseDir.resolve("resources"); return javaConvention.getSourceSets().create(platform.getName(), sourceSet -> { - /* Creates a SourceSet and set the source directories for a given platform. E.g. For the Presto platform, @@ -197,14 +195,11 @@ private TaskProvider configureGenerateWrappersTask(Project task.dependsOn(project.getTasks().named(inputSourceSet.getClassesTaskName())); }); - // todo: configure task to run with platform specific jdk (currently picks from local env) - // Toolchain support is only available in the Java plugins and for the tasks they define. - // Support for the Scala plugin is not released yet. - // Ref: https://docs.gradle.org/7.0/userguide/toolchains.html#sec:consuming + // Configure Java compile tasks to run with platform specific jdk + // TODO: set platform specific jdks/toolchain for scala tasks when support is available if (platform.getLanguage() == JAVA) { project.getTasks() .named(outputSourceSet.getCompileTaskName(platform.getLanguage().toString()), JavaCompile.class, task -> { - // configure compile task to run with platform specific jdk JavaToolchainService javaToolchains = project.getExtensions().getByType(JavaToolchainService.class); task.getJavaCompiler().set(javaToolchains.compilerFor(toolChainSpec -> { toolChainSpec.getLanguageVersion().set(platform.getJavaLanguageVersion()); @@ -214,9 +209,7 @@ private TaskProvider configureGenerateWrappersTask(Project project.getTasks() .named(outputSourceSet.getCompileTaskName(platform.getLanguage().toString())) - .configure(task -> { - task.dependsOn(generateWrappersTask); - }); + .configure(task -> task.dependsOn(generateWrappersTask)); return generateWrappersTask; }