build.gradle

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

import groovy.transform.Memoized
import java.util.regex.Matcher
import java.util.regex.Pattern

buildscript {
  repositories {
    gradlePluginPortal()
  }
  dependencies {
    classpath 'gradle.plugin.com.github.jengelman.gradle.plugins:shadow:7.0.0'
    classpath 'com.palantir.baseline:gradle-baseline-java:4.0.0'
    classpath 'com.palantir.gradle.gitversion:gradle-git-version:0.12.3'
    classpath 'com.diffplug.spotless:spotless-plugin-gradle:5.14.1'
    classpath 'gradle.plugin.org.inferred:gradle-processors:3.3.0'
    classpath 'me.champeau.jmh:jmh-gradle-plugin:0.6.6'
    classpath "com.github.alisiikh:gradle-scalastyle-plugin:3.4.1"
  }
}

plugins {
  id 'nebula.dependency-recommender' version '9.0.2'
}

try {
  // apply this plugin in a try-catch block so that we can handle cases without .git directory
  apply plugin: 'com.palantir.git-version'
} catch (Exception e) {
  project.logger.error(e.getMessage())
}

if (JavaVersion.current() == JavaVersion.VERSION_1_8) {
  project.ext.jdkVersion = '8'
} else if (JavaVersion.current() == JavaVersion.VERSION_11) {
  project.ext.jdkVersion = '11'
} else {
  throw new GradleException("This build must be run with JDK 8 or 11")
}

dependencyRecommendations {
  propertiesFile file: file('versions.props')
}

allprojects {
  group = "org.apache.iceberg"
  version = getProjectVersion()
  repositories {
    mavenCentral()
    mavenLocal()
  }
}

subprojects {
  apply plugin: 'nebula.dependency-recommender'
  apply plugin: 'java-library'

  configurations {
    testImplementation.extendsFrom compileOnly

    compileClasspath {
      // do not exclude Guava so the bundle project can reference classes.
      if (project.name != 'iceberg-bundled-guava') {
        exclude group: 'com.google.guava', module: 'guava'
      }
      // contains a copy of Guava
      exclude group: 'org.apache.spark', module: 'spark-network-common_2.12'
    }

    all {
      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
      exclude group: 'org.mortbay.jetty'
      exclude group: 'com.sun.jersey'
      exclude group: 'com.sun.jersey.contribs'
      exclude group: 'org.pentaho', module: 'pentaho-aggdesigner-algorithm'
    }

    testArtifacts
  }

  compileJava {
    options.encoding = "UTF-8"
  }

  compileTestJava {
    options.encoding = "UTF-8"
  }

  javadoc {
    options.encoding = 'UTF-8'
  }

  sourceCompatibility = '1.8'
  targetCompatibility = '1.8'

  dependencies {
    implementation 'org.slf4j:slf4j-api'
    implementation 'com.github.stephenc.findbugs:findbugs-annotations'

    testImplementation 'org.junit.vintage:junit-vintage-engine'
    testImplementation 'org.junit.jupiter:junit-jupiter-engine'
    testImplementation 'org.junit.jupiter:junit-jupiter'
    testImplementation 'org.slf4j:slf4j-simple'
    testImplementation 'org.mockito:mockito-core'
    testImplementation 'org.assertj:assertj-core'
  }

  test {
    def logDir = "${rootDir}/build/testlogs"
    def logFile = "${logDir}/${project.name}.log"
    mkdir("${logDir}")
    delete("${logFile}")
    def buildLog = new File(logFile)
    addTestOutputListener(new TestOutputListener() {
      def lastDescriptor
      @Override
      void onOutput(TestDescriptor testDescriptor, TestOutputEvent testOutputEvent) {
        if (lastDescriptor != testDescriptor) {
          buildLog << "--------\n- Test log for: "<< testDescriptor << "\n--------\n"
          lastDescriptor = testDescriptor
        }
        buildLog << testOutputEvent.destination << " " << testOutputEvent.message
      }
    })

    testLogging {
      events "failed"
      exceptionFormat "full"
    }
  }
}

project(':iceberg-bundled-guava') {
  apply plugin: 'com.github.johnrengelman.shadow'

  tasks.jar.dependsOn tasks.shadowJar

  dependencies {
    compileOnly('com.google.guava:guava') {
      exclude group: 'com.google.code.findbugs'
      // may be LGPL - use ALv2 findbugs-annotations instead
      exclude group: 'com.google.errorprone'
      exclude group: 'com.google.j2objc'
    }
  }

  shadowJar {
    classifier null
    configurations = [project.configurations.compileClasspath]
    zip64 true

    // include the LICENSE and NOTICE files for the shaded Jar
    from(projectDir) {
      include 'LICENSE'
      include 'NOTICE'
    }

    dependencies {
      exclude(dependency('com.github.stephenc.findbugs:findbugs-annotations'))
      exclude(dependency('org.slf4j:slf4j-api'))
      exclude(dependency('org.checkerframework:checker-qual'))
    }

    relocate 'com.google.common', 'org.apache.iceberg.relocated.com.google.common'

    minimize()
  }

  jar {
    archiveClassifier = 'empty'
  }
}

project(':iceberg-api') {
  dependencies {
    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
    compileOnly "com.google.errorprone:error_prone_annotations"
    testImplementation "org.apache.avro:avro"
  }
}

project(':iceberg-common') {
  dependencies {
    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
  }
}

project(':iceberg-core') {
  dependencies {
    api project(':iceberg-api')
    implementation project(':iceberg-common')
    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')

    implementation("org.apache.avro:avro") {
      exclude group: 'org.tukaani' // xz compression is not supported
    }

    implementation "com.fasterxml.jackson.core:jackson-databind"
    implementation "com.fasterxml.jackson.core:jackson-core"
    implementation "com.github.ben-manes.caffeine:caffeine"
    implementation "org.roaringbitmap:RoaringBitmap"
    compileOnly("org.apache.hadoop:hadoop-client") {
      exclude group: 'org.apache.avro', module: 'avro'
      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
    }

    testImplementation "org.xerial:sqlite-jdbc"
    testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
  }
}

project(':iceberg-data') {
  dependencies {
    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
    api project(':iceberg-api')
    implementation project(':iceberg-core')
    compileOnly project(':iceberg-parquet')
    compileOnly project(':iceberg-orc')
    compileOnly("org.apache.hadoop:hadoop-common") {
      exclude group: 'commons-beanutils'
      exclude group: 'org.apache.avro', module: 'avro'
      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
    }

    implementation("org.apache.orc:orc-core::nohive") {
      exclude group: 'org.apache.hadoop'
      exclude group: 'commons-lang'
      // These artifacts are shaded and included in the orc-core fat jar
      exclude group: 'com.google.protobuf', module: 'protobuf-java'
      exclude group: 'org.apache.hive', module: 'hive-storage-api'
    }

    implementation("org.apache.parquet:parquet-avro") {
      exclude group: 'org.apache.avro', module: 'avro'
      // already shaded by Parquet
      exclude group: 'it.unimi.dsi'
      exclude group: 'org.codehaus.jackson'
    }

    compileOnly "org.apache.avro:avro"

    testImplementation("org.apache.hadoop:hadoop-client") {
      exclude group: 'org.apache.avro', module: 'avro'
      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
    }

    testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
    testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
  }

  test {
    // Only for TestSplitScan as of Gradle 5.0+
    maxHeapSize '1500m'
  }
}

project(':iceberg-aliyun') {
  dependencies {
    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
    api project(':iceberg-api')
    implementation project(':iceberg-core')
    implementation project(':iceberg-common')

    compileOnly 'com.aliyun.oss:aliyun-sdk-oss'
    compileOnly 'javax.xml.bind:jaxb-api'
    compileOnly 'javax.activation:activation'
    compileOnly 'org.glassfish.jaxb:jaxb-runtime'
    compileOnly("org.apache.hadoop:hadoop-common") {
      exclude group: 'org.apache.avro', module: 'avro'
      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
      exclude group: 'javax.servlet', module: 'servlet-api'
      exclude group: 'com.google.code.gson', module: 'gson'
    }

    testImplementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-xml'
    testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
    testImplementation 'org.springframework:spring-web'
    testImplementation('org.springframework.boot:spring-boot-starter-jetty') {
      exclude module: 'logback-classic'
      exclude group: 'org.eclipse.jetty.websocket', module: 'javax-websocket-server-impl'
      exclude group: 'org.eclipse.jetty.websocket', module: 'websocket-server'
    }
    testImplementation('org.springframework.boot:spring-boot-starter-web') {
      exclude module: 'logback-classic'
      exclude module: 'spring-boot-starter-logging'
    }
  }
}

project(':iceberg-aws') {
  dependencies {
    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
    api project(':iceberg-api')
    implementation project(':iceberg-common')
    implementation project(':iceberg-core')

    compileOnly 'software.amazon.awssdk:url-connection-client'
    compileOnly 'software.amazon.awssdk:s3'
    compileOnly 'software.amazon.awssdk:kms'
    compileOnly 'software.amazon.awssdk:glue'
    compileOnly 'software.amazon.awssdk:sts'
    compileOnly 'software.amazon.awssdk:dynamodb'

    compileOnly("org.apache.hadoop:hadoop-common") {
      exclude group: 'org.apache.avro', module: 'avro'
      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
      exclude group: 'javax.servlet', module: 'servlet-api'
      exclude group: 'com.google.code.gson', module: 'gson'
    }

    testImplementation 'software.amazon.awssdk:iam'
    testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
    testImplementation("com.adobe.testing:s3mock-junit4") {
      exclude module: "spring-boot-starter-logging"
      exclude module: "logback-classic"
      exclude group: 'junit'
    }
  }

  sourceSets {
    integration {
      java.srcDir "$projectDir/src/integration/java"
      resources.srcDir "$projectDir/src/integration/resources"
      compileClasspath += main.output + test.output
      runtimeClasspath += main.output + test.output
    }
  }

  configurations {
    integrationImplementation.extendsFrom testImplementation
    integrationRuntime.extendsFrom testRuntimeOnly
  }

  task integrationTest(type: Test) {
    testClassesDirs = sourceSets.integration.output.classesDirs
    classpath = sourceSets.integration.runtimeClasspath
  }
}

project(':iceberg-gcp') {
  dependencies {
    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
    api project(':iceberg-api')
    implementation project(':iceberg-common')
    implementation project(':iceberg-core')

    implementation platform('com.google.cloud:libraries-bom')
    implementation 'com.google.cloud:google-cloud-storage'

    testImplementation 'com.google.cloud:google-cloud-nio'

    testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')

    testImplementation("org.apache.hadoop:hadoop-common") {
      exclude group: 'org.apache.avro', module: 'avro'
      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
      exclude group: 'javax.servlet', module: 'servlet-api'
      exclude group: 'com.google.code.gson', module: 'gson'
    }
  }
}

project(':iceberg-hive-metastore') {
  dependencies {
    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
    implementation project(':iceberg-core')
    api project(':iceberg-api')
    implementation project(':iceberg-common')

    implementation "com.github.ben-manes.caffeine:caffeine"

    compileOnly "org.apache.avro:avro"

    compileOnly("org.apache.hive:hive-metastore") {
      exclude group: 'org.apache.avro', module: 'avro'
      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
      exclude group: 'org.pentaho' // missing dependency
      exclude group: 'org.apache.hbase'
      exclude group: 'org.apache.logging.log4j'
      exclude group: 'co.cask.tephra'
      exclude group: 'com.google.code.findbugs', module: 'jsr305'
      exclude group: 'org.eclipse.jetty.aggregate', module: 'jetty-all'
      exclude group: 'org.eclipse.jetty.orbit', module: 'javax.servlet'
      exclude group: 'org.apache.parquet', module: 'parquet-hadoop-bundle'
      exclude group: 'com.tdunning', module: 'json'
      exclude group: 'javax.transaction', module: 'transaction-api'
      exclude group: 'com.zaxxer', module: 'HikariCP'
    }

    // By default, hive-exec is a fat/uber jar and it exports a guava library
    // that's really old. We use the core classifier to be able to override our guava
    // version. Luckily, hive-exec seems to work okay so far with this version of guava
    // See: https://github.com/apache/hive/blob/master/ql/pom.xml#L911 for more context.
    testImplementation("org.apache.hive:hive-exec::core") {
      exclude group: 'org.apache.avro', module: 'avro'
      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
      exclude group: 'org.pentaho' // missing dependency
      exclude group: 'org.apache.hive', module: 'hive-llap-tez'
      exclude group: 'org.apache.logging.log4j'
      exclude group: 'com.google.protobuf', module: 'protobuf-java'
      exclude group: 'org.apache.calcite'
      exclude group: 'org.apache.calcite.avatica'
      exclude group: 'com.google.code.findbugs', module: 'jsr305'
    }

    testImplementation("org.apache.hive:hive-metastore") {
      exclude group: 'org.apache.avro', module: 'avro'
      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
      exclude group: 'org.pentaho' // missing dependency
      exclude group: 'org.apache.hbase'
      exclude group: 'org.apache.logging.log4j'
      exclude group: 'co.cask.tephra'
      exclude group: 'com.google.code.findbugs', module: 'jsr305'
      exclude group: 'org.eclipse.jetty.aggregate', module: 'jetty-all'
      exclude group: 'org.eclipse.jetty.orbit', module: 'javax.servlet'
      exclude group: 'org.apache.parquet', module: 'parquet-hadoop-bundle'
      exclude group: 'com.tdunning', module: 'json'
      exclude group: 'javax.transaction', module: 'transaction-api'
      exclude group: 'com.zaxxer', module: 'HikariCP'
    }

    compileOnly("org.apache.hadoop:hadoop-client") {
      exclude group: 'org.apache.avro', module: 'avro'
      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
    }

    testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
  }
}

project(':iceberg-orc') {
  dependencies {
    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
    api project(':iceberg-api')
    implementation project(':iceberg-core')
    implementation("org.apache.avro:avro") {
      exclude group: 'org.tukaani' // xz compression is not supported
    }

    implementation("org.apache.orc:orc-core::nohive") {
      exclude group: 'org.apache.hadoop'
      exclude group: 'commons-lang'
      // These artifacts are shaded and included in the orc-core fat jar
      exclude group: 'com.google.protobuf', module: 'protobuf-java'
      exclude group: 'org.apache.hive', module: 'hive-storage-api'
    }

    compileOnly("org.apache.hadoop:hadoop-common") {
      exclude group: 'commons-beanutils'
      exclude group: 'org.apache.avro', module: 'avro'
      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
    }
    compileOnly("org.apache.hadoop:hadoop-client") {
      exclude group: 'org.apache.avro', module: 'avro'
    }

    testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
    testImplementation project(':iceberg-common')
  }
}

project(':iceberg-parquet') {
  dependencies {
    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
    api project(':iceberg-api')
    implementation project(':iceberg-core')
    implementation project(':iceberg-common')

    implementation("org.apache.parquet:parquet-avro") {
      exclude group: 'org.apache.avro', module: 'avro'
      // already shaded by Parquet
      exclude group: 'it.unimi.dsi'
      exclude group: 'org.codehaus.jackson'
    }

    compileOnly "org.apache.avro:avro"
    compileOnly("org.apache.hadoop:hadoop-client") {
      exclude group: 'org.apache.avro', module: 'avro'
    }

    testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
  }
}

project(':iceberg-arrow') {
  dependencies {
    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
    api project(':iceberg-api')
    implementation project(':iceberg-core')
    implementation project(':iceberg-parquet')

    implementation("org.apache.arrow:arrow-vector") {
      exclude group: 'io.netty', module: 'netty-buffer'
      exclude group: 'io.netty', module: 'netty-common'
      exclude group: 'com.google.code.findbugs', module: 'jsr305'
    }
    implementation("org.apache.arrow:arrow-memory-netty") {
      exclude group: 'com.google.code.findbugs', module: 'jsr305'
      exclude group: 'io.netty', module: 'netty-common'
      exclude group: 'io.netty', module: 'netty-buffer'
    }

    runtimeOnly("io.netty:netty-buffer")

    implementation("org.apache.parquet:parquet-avro") {
      exclude group: 'org.apache.avro', module: 'avro'
      // already shaded by Parquet
      exclude group: 'it.unimi.dsi'
      exclude group: 'org.codehaus.jackson'
    }

    testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
    // To run ArrowReaderTest test cases, :netty-common is needed.
    // We import :netty-common through :arrow-memory-netty
    // so that the same version as used by the :arrow-memory-netty module is picked.
    testImplementation("org.apache.arrow:arrow-memory-netty")
    testImplementation("org.apache.hadoop:hadoop-common")
    testImplementation("org.apache.hadoop:hadoop-mapreduce-client-core")
  }
}

project(':iceberg-pig') {
  dependencies {
    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
    api project(':iceberg-api')
    implementation project(':iceberg-common')
    implementation project(':iceberg-core')
    implementation project(':iceberg-parquet')

    implementation("org.apache.parquet:parquet-avro") {
      exclude group: 'org.apache.avro', module: 'avro'
      // already shaded by Parquet
      exclude group: 'it.unimi.dsi'
      exclude group: 'org.codehaus.jackson'
    }

    compileOnly("org.apache.pig:pig") {
      exclude group: "junit", module: "junit"
    }
    compileOnly("org.apache.hadoop:hadoop-mapreduce-client-core")
    compileOnly("org.apache.hadoop:hadoop-client") {
      exclude group: 'org.apache.avro', module: 'avro'
    }

    testImplementation("org.apache.hadoop:hadoop-minicluster") {
      exclude group: 'org.apache.avro', module: 'avro'
    }
  }
}

project(':iceberg-nessie') {
  test {
    useJUnitPlatform()
  }

  dependencies {
    api project(':iceberg-api')
    implementation project(':iceberg-common')
    implementation project(':iceberg-core')
    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
    implementation "org.projectnessie:nessie-client"

    testImplementation "org.projectnessie:nessie-jaxrs-testextension"
    // Need to "pull in" el-api explicitly :(
    testImplementation "jakarta.el:jakarta.el-api"

    compileOnly "org.apache.hadoop:hadoop-common"
    testImplementation "org.apache.avro:avro"

    testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
  }
}

project(':iceberg-dell') {
  dependencies {
    implementation project(':iceberg-core')
    implementation project(':iceberg-common')
    implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
    compileOnly 'com.emc.ecs:object-client-bundle'

    testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
    testImplementation("org.apache.hadoop:hadoop-common") {
      exclude group: 'org.apache.avro', module: 'avro'
      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
    }
    testImplementation "javax.xml.bind:jaxb-api"
    testImplementation "javax.activation:activation"
    testImplementation "org.glassfish.jaxb:jaxb-runtime"
  }
}

@Memoized
boolean versionFileExists() {
  return file('version.txt').exists()
}

@Memoized
String getVersionFromFile() {
  return file('version.txt').text.trim()
}

String getProjectVersion() {
  if (versionFileExists()) {
    return getVersionFromFile()
  }

  try {
    // we're fetching the version from the latest tag (prefixed with 'release-base-'),
    // which can look like this: '0.13.0-2-g805400f0.dirty' but we're only interested in the MAJOR.MINOR.PATCH part
    String version = gitVersion(prefix: 'release-base-')
    Pattern pattern = Pattern.compile("^([0-9]+)\\.([0-9]+)\\.([0-9]+)(.*)?\$")
    Matcher matcher = pattern.matcher(version)
    if (matcher.matches()) {
      // bump the MINOR version and always set the PATCH version to 0
      return matcher.group(1) + "." + (Integer.valueOf(matcher.group(2)) + 1) + ".0-SNAPSHOT"
    }
    return version
  } catch (Exception e) {
    throw new Exception("Neither version.txt nor git version exists: " + e.getMessage(), e)
  }
}

String getJavadocVersion() {
  if (versionFileExists()) {
    return getVersionFromFile()
  }

  try {
    // use the branch name in place of version in Javadoc
    return versionDetails().branchName
  } catch (NullPointerException e) {
    throw new Exception("Neither version.txt nor git version exists")
  }
}

apply from: 'jmh.gradle'
apply from: 'baseline.gradle'
apply from: 'deploy.gradle'
apply from: 'tasks.gradle'