build.gradle

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

apply plugin: 'eclipse'
apply plugin: 'idea'
apply plugin: 'java'

defaultTasks 'distTar'

task wrapper(type: Wrapper) {
    description = "Updates gradlew and supporting files."
    gradleVersion = '2.3'
}

version = "$SAMZA_VERSION"

repositories {
    mavenLocal()
    mavenCentral()
    maven { url "https://repository.apache.org/content/groups/public" }
}


idea {
    module {
        sourceDirs += file('src/main/java')
        testSourceDirs += file('src/test/java')
    }
}

// a configuration for dependencies that need exploding into package
configurations {
    explode
}

dependencies {
    compile(group: 'org.codehaus.jackson', name: 'jackson-jaxrs', version: '1.9.13')
    compile(group: 'org.slf4j', name: 'slf4j-api', version: "$SLF4J_VERSION")
    compile(group: 'org.slf4j', name: 'slf4j-log4j12', version: "$SLF4J_VERSION")
    compile(group: 'org.schwering', name: 'irclib', version: '1.10')
    compile(group: 'org.apache.samza', name: 'samza-api', version: "$SAMZA_VERSION")
    compile(group: 'org.apache.samza', name: 'samza-kv_2.11', version: "$SAMZA_VERSION")

    compile(group: 'org.apache.samza', name: 'samza-test_2.11', version: "$SAMZA_VERSION")
    compile(group: 'org.apache.samza', name: 'samza-kafka_2.11', version: "$SAMZA_VERSION")
    compile(group: 'org.apache.samza', name: 'samza-kv-rocksdb_2.11', version: "$SAMZA_VERSION")
    compile(group: 'org.apache.samza', name: 'samza-azure', version: "$SAMZA_VERSION")
    testCompile(group: 'junit', name: 'junit', version: "4.12")
    explode (group: 'org.apache.samza', name: 'samza-shell',  ext: 'tgz', classifier: 'dist', version: "$SAMZA_VERSION")
    runtime(group: 'org.apache.samza', name: 'samza-core_2.11', version: "$SAMZA_VERSION")
    runtime(group: 'org.apache.samza', name: 'samza-log4j', version: "$SAMZA_VERSION")

    runtime(group: 'org.apache.samza', name: 'samza-shell', version: "$SAMZA_VERSION")
    runtime(group: 'org.apache.samza', name: 'samza-yarn_2.11', version: "$SAMZA_VERSION")
    runtime(group: 'org.apache.kafka', name: 'kafka_2.11', version: "$KAFKA_VERSION")
    runtime(group: 'org.apache.hadoop', name: 'hadoop-hdfs', version: "$HADOOP_VERSION")
}

// make the samza distribution .tgz file
task distTar(dependsOn: build, type: Tar) {
    destinationDir(new File(project.buildDir, "/distributions"))
    compression(Compression.GZIP)
    classifier('dist')
    extension('tar.gz')
    into("config") {
        from("src/main/config") {
            include "wikipedia-feed.properties"
            include "wikipedia-parser.properties"
            include "wikipedia-stats.properties"
            include "wikipedia-application.properties"

            // expand the Maven tokens with Gradle equivalents.  Also change 'target' (Maven) to 'build/distributions' (Gradle)
            filter { String line ->
                line.replaceAll('[\$][{]basedir[}]', project.projectDir.toString()).replaceAll('[\$][{]project.artifactId[}]', project.name.toString()).replaceAll('/target/', '/build/distributions/').replaceAll('[\$][{]pom.version[}]', version)
            }
        }
    }

    into("bin") {
        from {
            configurations.explode.collect { tarTree(it) }
        }
    }

    into("lib") {
        from configurations.runtime
        from configurations.runtime.artifacts.files
        from("src/main/resources/") {
            include "log4j.xml"
        }
    }
}

// install everything
task installGrid(type: Exec) {
    workingDir(project.projectDir)
    commandLine("bin/grid", "install", "all")
    outputs.upToDateWhen {
        ["kafka", "zookeeper", "yarn"].every {
            (new File(project.projectDir, "deploy/" + it)).exists()
        }
    }
}

// update the Samza job
task deployHelloSamza(dependsOn: [distTar, installGrid], type: Sync) {
    into(new File(project.projectDir, "/deploy/samza"))
    from(tarTree(distTar.archivePath))
}


// run everything
task startGrid(type: Exec) {
    workingDir(project.projectDir)
    commandLine("bin/grid", "start", "all")
    outputs.upToDateWhen {
        // use running zookeeper as proxy
        File zookeeperPidFile = new File("/tmp/zookeeper/zookeeper_server.pid")
        zookeeperPidFile.exists() &&
            "kill -0 ${zookeeperPidFile.text}".execute().waitFor() == 0
    }
}

// stop everything
task stopGrid(type: Exec) {
    workingDir(project.projectDir)
    commandLine("bin/grid", "stop", "all")
}

//
// Samza helpers
//

// helper task to run Samza jobs
class SamzaTask extends DefaultTask {
    String configFile;

    @TaskAction
    def startSamza() {
        project.exec {
            workingDir(project.projectDir)
            commandLine("deploy/samza/bin/run-job.sh",
                    "--config-factory=org.apache.samza.config.factories.PropertiesConfigFactory",
                    "--config-path=file://${project.projectDir}/deploy/samza/config/${configFile}")
        }
    }
}

// helper to run the Samza job to retrieve Wikipedia data
task runWikiFeed(dependsOn: [startGrid, deployHelloSamza], type: SamzaTask) {
    configFile("wikipedia-feed.properties")
}

// helper to run the Samza job process Wikipedia data
task runWikiParser(dependsOn: [startGrid, deployHelloSamza], type: SamzaTask) {
    configFile("wikipedia-parser.properties")
}

// helper to run the Samza job to summarize stats on Wikipedia edits
task runWikiStats(dependsOn: [startGrid, deployHelloSamza], type: SamzaTask) {
    configFile("wikipedia-stats.properties")
}


//
// Kafka helpers
//

// show all Kafka topics
task listKafkaTopics(type: Exec) {
    workingDir(project.projectDir)
    commandLine("deploy/kafka/bin/kafka-topics.sh",
            "--zookeeper", "localhost:2181",
            "--list")
}

// helper task to monitor a Kafka topic
class KafkaDumpTask extends DefaultTask {
    String topic;

    @TaskAction
    def dumpTopic() {
        project.exec {
            workingDir(project.projectDir)
            commandLine("deploy/kafka/bin/kafka-console-consumer.sh",
                    "--zookeeper", "localhost:2181",
                    "--topic", "${topic}")
        }
    }
}

// helper to dump the wikipedia-raw topic
task dumpWikiRaw(dependsOn: startGrid, type: KafkaDumpTask) {
    topic("wikipedia-raw")
}

// helper to dump the wikipedia-edits topic
task dumpWikiEdits(dependsOn: startGrid, type: KafkaDumpTask) {
    topic("wikipedia-edits")
}

// helper to dump the wikipedia-stats topic
task dumpWikiStats(dependsOn: startGrid, type: KafkaDumpTask) {
    topic("wikipedia-stats")
}