Skip to content

Commit

Permalink
Implementing remote execution break mode. WIP
Browse files Browse the repository at this point in the history
  • Loading branch information
@skrugly
skrugly committed Aug 26, 2020
1 parent a39bbc4 commit 35335ac
Show file tree
Hide file tree
Showing 3 changed files with 140 additions and 24 deletions.
2 changes: 1 addition & 1 deletion build.gradle
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
group 'com.instamotor'
version '1.3.5'
version '1.4.0-RC-1'

buildscript {
ext.kotlin_version = '1.3.72'
Expand Down
141 changes: 125 additions & 16 deletions plugin/src/main/kotlin/Mirakle.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import org.gradle.api.Plugin
import org.gradle.api.Task
import org.gradle.api.execution.TaskExecutionListener
import org.gradle.api.internal.AbstractTask
import org.gradle.api.internal.tasks.DefaultTaskDependency
import org.gradle.api.invocation.Gradle
import org.gradle.api.logging.LogLevel
import org.gradle.api.logging.configuration.ConsoleOutput
Expand All @@ -28,9 +29,10 @@ import java.io.OutputStreamWriter
import java.nio.file.Files
import java.util.*
import java.util.concurrent.atomic.AtomicBoolean
import java.util.regex.Pattern
import javax.inject.Inject

class Mirakle : Plugin<Gradle> {
open class Mirakle : Plugin<Gradle> {
override fun apply(gradle: Gradle) {
gradle.rootProject { it.extensions.create("mirakle", MirakleExtension::class.java) }

Expand All @@ -45,17 +47,24 @@ class Mirakle : Plugin<Gradle> {
gradle.assertNonSupportedFeatures()

val originalStartParams = gradle.startParameter.newInstance()
val breakMode = originalStartParams.projectProperties[BREAK_MODE]?.toBoolean() ?: false

gradle.startParameter.apply {
setTaskNames(listOf("mirakle"))
setExcludedTaskNames(emptyList())
useEmptySettings()
buildFile = File(originalStartParams.currentDir, "mirakle.gradle").takeIf(File::exists)
?: //a way to make Gradle not evaluate project's default build.gradle file on local machine
File(originalStartParams.currentDir, "mirakle_build_file_stub").also { stub ->
stub.createNewFile()
gradle.rootProject { it.afterEvaluate { stub.delete() } }
}
if (breakMode) {
setTaskNames(listOf("mirakle") + taskNames)
} else {
setTaskNames(listOf("mirakle"))
setExcludedTaskNames(emptyList())
}
if (!breakMode) {
useEmptySettings()
buildFile = File(originalStartParams.currentDir, "mirakle.gradle").takeIf(File::exists)
?: //a way to make Gradle not evaluate project's default build.gradle file on local machine
File(originalStartParams.currentDir, "mirakle_build_file_stub").also { stub ->
stub.createNewFile()
gradle.rootProject { it.afterEvaluate { stub.delete() } }
}
}
}

gradle.rootProject { project ->
Expand Down Expand Up @@ -93,7 +102,6 @@ class Mirakle : Plugin<Gradle> {
"-P$BUILD_ON_REMOTE=true",
"-p ${config.remoteFolder}/\"${project.name}\""
)
args(startParamsToArgs(originalStartParams))

isIgnoreExitValue = true

Expand Down Expand Up @@ -172,15 +180,13 @@ class Mirakle : Plugin<Gradle> {
.forProjectDirectory(gradle.rootProject.projectDir)
.connect()

try {
connection.use { connection ->
connection.newBuild()
.withArguments(startParamsToArgs(originalStartParams).plus("-P$FALLBACK=true"))
.setStandardInput(upload.standardInput ?: System.`in`)
.setStandardOutput(upload.standardOutput ?: System.out)
.setStandardError(upload.errorOutput ?: System.err)
.run()
} finally {
connection.close()
}
}
}
Expand All @@ -198,6 +204,98 @@ class Mirakle : Plugin<Gradle> {
}
}

if (breakMode && config.breakOnTasks.isNotEmpty()) {
if (config.downloadInParallel) {
throw MirakleException("Mirakle break mode doesn't work with download in parallel yet.")
}

gradle.taskGraph.whenReady { taskGraph ->
val breakTaskPatterns = config.breakOnTasks.map(Pattern::compile)

val graphWithoutMirakle = taskGraph.allTasks.filterNot(Task::isMirakleTask)

val breakOnTasks = graphWithoutMirakle.filter { task ->
breakTaskPatterns.any { it.matcher(task.name).find() }
}

when {
breakOnTasks.isEmpty() -> {
execute.args(startParamsToArgs(originalStartParams))
graphWithoutMirakle.forEach {
it.enabled = false
}
}
breakOnTasks.size == 1 -> {
val breakOnTask = breakOnTasks.first()

val tasksForRemoteExecution = graphWithoutMirakle
.takeWhile { it != breakOnTask }
.onEach { it.enabled = false }

originalStartParams.newInstance().apply {
setTaskNames(tasksForRemoteExecution.map(Task::getPath))
}.let { execute.args(startParamsToArgs(it)) }

breakOnTask.apply {
enabled = true
mustRunAfter(download)
onlyIf { execute.didWork && execute.execResult!!.exitValue == 0 }

if (inputs.files.files.isNotEmpty()) {
val inputsNotInProjectDir = inputs.files.files.filter {
!it.path.startsWith(project.rootDir.path)
}

if (inputsNotInProjectDir.isNotEmpty()) {
println("${breakOnTask.toString().capitalize()} declares input not from project dir. That is not supported by Mirakle yet.")
throw MirakleException()
}

// Need to include all of the parent directories down to the desired directory
val includeRules = inputs.files.files.map { file ->
val pathToInclude = file.path.substringAfter(project.rootDir.path).drop(1)
val split = pathToInclude.split("/")

(1 until split.size).mapIndexed { _, i ->
split.take(i).joinToString(separator = "/")
}.map {
"--include=$it"
}.let {
val isFile = split.last().contains('.')
if (isFile) {
it.plus("--include=$pathToInclude")
} else {
it.plus("--include=$pathToInclude/***")
}
}
}.flatten().toSet()

//val excludeAllRule = "--exclude=/**" // TODO does it make sense to exclude all except the inputs?

download.setArgs(includeRules /*+ excludeAllRule */+ download.args!!.toList())
}
}

execute.doFirst {
println("Mirakle will break remote execution on $breakOnTask")
}

mirakle.dependsOn(breakOnTask)
gradle.logTasks(graphWithoutMirakle - tasksForRemoteExecution)
}
else -> {
println("Task execution graph contains more than 1 task to break on. That is not supported by Mirakle yet.")
throw MirakleException()
}
}
}
} else {
gradle.startParameter.apply {
setTaskNames(taskNames - originalStartParams.taskNames)
}
execute.args(startParamsToArgs(originalStartParams))
}

gradle.supportAndroidStudioAdvancedProfiling(config, upload, execute, download)

gradle.logTasks(upload, execute, download)
Expand All @@ -207,6 +305,13 @@ class Mirakle : Plugin<Gradle> {
}
}

class MirakleBreakMode : Mirakle() {
override fun apply(gradle: Gradle) {
gradle.startParameter.projectProperties = gradle.startParameter.projectProperties.plus(BREAK_MODE to "true")
super.apply(gradle)
}
}

open class MirakleExtension {
var host: String? = null

Expand Down Expand Up @@ -245,6 +350,8 @@ open class MirakleExtension {

var downloadInParallel = false
var downloadInterval = 2000L

var breakOnTasks = emptySet<String>()
}

fun startParamsToArgs(params: StartParameter) = with(params) {
Expand Down Expand Up @@ -346,17 +453,19 @@ fun getMainframerConfigOrNull(projectDir: File, mirakleConfig: MirakleExtension)
rsyncFromRemoteArgs += "--exclude-from=${remoteIgnore.path}"
}

// Mainframer doesn't have fallback and parallel download features implemented yet
// Use Mirakle's parameters instead
// Mainframer doesn't have these config params, use Mirakle's parameters instead
fallback = mirakleConfig.fallback
downloadInParallel = mirakleConfig.downloadInParallel
downloadInterval = mirakleConfig.downloadInterval
sshArgs = mirakleConfig.sshArgs
breakOnTasks = mirakleConfig.breakOnTasks
}
}
}

const val BUILD_ON_REMOTE = "mirakle.build.on.remote"
const val FALLBACK = "mirakle.build.fallback"
const val BREAK_MODE = "mirakle.break.mode"

//TODO test
fun Gradle.supportAndroidStudioAdvancedProfiling(config: MirakleExtension, upload: Exec, execute: Exec, download: Exec) {
Expand Down
21 changes: 14 additions & 7 deletions plugin/src/main/kotlin/utils.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import groovy.lang.Closure
import org.apache.tools.ant.taskdefs.condition.Os
import org.gradle.BuildAdapter
import org.gradle.BuildResult
import org.gradle.api.GradleException
import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.execution.TaskExecutionListener
Expand All @@ -10,6 +11,8 @@ import org.gradle.api.invocation.Gradle
import org.gradle.api.tasks.TaskState
import org.gradle.internal.service.ServiceRegistry

fun Gradle.logTasks(tasks: List<Task>) = logTasks(*tasks.toTypedArray())

fun Gradle.logTasks(vararg task: Task) {
task.forEach { targetTask ->
addListener(object : TaskExecutionListener {
Expand All @@ -25,7 +28,8 @@ fun Gradle.logTasks(vararg task: Task) {
if (task == targetTask && task.didWork) {
val finishTime = System.currentTimeMillis()
buildFinished {
println("Task ${task.name} took: ${prettyTime(finishTime - startTime)}")
val taskName = if (task.isMirakleTask()) task.name else task.path.drop(1)
println("Task $taskName took : ${prettyTime(finishTime - startTime)}")
}
}
}
Expand Down Expand Up @@ -60,7 +64,7 @@ fun prettyTime(timeInMs: Long): String {
return result.toString()
}

class MirakleException(message: String) : RuntimeException(message)
class MirakleException(message: String? = null) : GradleException(message)

inline fun <reified T : Task> Project.task(name: String, noinline configuration: T.() -> Unit) =
tasks.create(name, T::class.java, configuration)
Expand All @@ -84,11 +88,14 @@ class KotlinClosure1<in T : Any, V : Any>(
fun doCall(it: T): V? = it.function()
}

val Task.services: ServiceRegistry get() {
val field = AbstractTask::class.java.getDeclaredField("services")
field.isAccessible = true
return field.get(this) as ServiceRegistry
}
val Task.services: ServiceRegistry
get() {
val field = AbstractTask::class.java.getDeclaredField("services")
field.isAccessible = true
return field.get(this) as ServiceRegistry
}

fun Task.isMirakleTask() = name == "mirakle" || name == "uploadToRemote" || name == "executeOnRemote" || name == "downloadFromRemote" || name == "downloadInParallel" || name == "fallback"

/*
* On Windows rsync is used under Cygwin environment
Expand Down

0 comments on commit 35335ac

Please sign in to comment.