Skip to content

Commit

Permalink
Extract some of the build logic to build a docker image for RFS and m…
Browse files Browse the repository at this point in the history
…ove it to DocumentFromSnapshotMigration

Signed-off-by: Greg Schohn <[email protected]>
  • Loading branch information
gregschohn committed Jun 19, 2024
1 parent 8733c01 commit dbb93a4
Show file tree
Hide file tree
Showing 11 changed files with 200 additions and 84 deletions.
112 changes: 112 additions & 0 deletions DocumentsFromSnapshotMigration/build-preloaded-source-image.gradle
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
import com.bmuschko.gradle.docker.tasks.container.*
import com.github.dockerjava.core.DefaultDockerClientConfig
import com.github.dockerjava.core.DockerClientImpl
import com.github.dockerjava.httpclient5.ApacheDockerHttpClient

def createDockerClient() {
def config = DefaultDockerClientConfig.createDefaultConfigBuilder().build()
def httpClient = new ApacheDockerHttpClient.Builder()
.dockerHost(config.getDockerHost())
.sslConfig(config.getSSLConfig())
.maxConnections(100)
.build()

return DockerClientImpl.getInstance(config, httpClient)
}

def uniqueId = UUID.randomUUID();
def myNetworkName = "rfs-preload-source-${uniqueId}";
def createNetworkTask = task createNetwork(type: Exec) {
commandLine 'docker', 'network', 'create', myNetworkName
doLast {
println 'Network created'
}
}
task createInitialElasticsearchContainer(type: DockerCreateContainer) {
dependsOn createNetwork, buildDockerImage_emptyElasticsearchSource_7_17
targetImageId 'migrations/emptyElasticsearchSource_7_17:latest'
containerName = "elasticsearch-${uniqueId}"
hostConfig.network = myNetworkName
hostConfig.dns = ['elasticsearch']
networkAliases = ['elasticsearch']
hostName = 'elasticsearch'
}

def startSourceTask = task startInitialElasticsearchContainer(type: DockerStartContainer) {
dependsOn createInitialElasticsearchContainer
targetContainerId createInitialElasticsearchContainer.getContainerId()
}

task createClientContainer(type: DockerCreateContainer) {
dependsOn startInitialElasticsearchContainer, buildDockerImage_trafficGenerator
targetImageId 'migrations/osb_traffic_generator:latest'
containerName = "traffic-generator-container-${uniqueId}"
hostConfig.network = myNetworkName
cmd = ['default_osb_test_workloads', "http://elasticsearch:9200"]
}

def startClientTask = task startClientContainer(type: DockerStartContainer) {
dependsOn createClientContainer
targetContainerId createClientContainer.getContainerId()
}

task waitClientContainer(type: DockerWaitContainer) {
dependsOn startClientContainer
targetContainerId createClientContainer.getContainerId()
}

// Task to commit the source container to create a new image that will include the loaded data
def sourceContainerCommitTask = task commitSourceContainer() {
dependsOn waitClientContainer

doLast {
def client = createDockerClient()
def containerId = createInitialElasticsearchContainer.getContainerId().get()

client.commitCmd(containerId)
.withRepository("elasticsearch_rfs_source")
.withTag("latest")
.exec()
}
}

task removeClientContainer(type: DockerRemoveContainer) {
dependsOn waitClientContainer
targetContainerId createClientContainer.getContainerId()
}
startClientTask.finalizedBy(removeClientContainer)

// Task to stop and remove the primary container
task stopInitialElasticsearchContainer(type: DockerStopContainer) {
dependsOn commitSourceContainer
targetContainerId createInitialElasticsearchContainer.getContainerId()
}

task removeInitialElasticsearchContainer(type: DockerRemoveContainer) {
dependsOn stopInitialElasticsearchContainer
targetContainerId createInitialElasticsearchContainer.getContainerId()
}
startSourceTask.finalizedBy(removeInitialElasticsearchContainer)

def deleteNetworkTask = task deleteNetwork(type: Exec) {
mustRunAfter removeInitialElasticsearchContainer, removeClientContainer
commandLine 'docker', 'network', 'rm', myNetworkName
doLast {
println 'Custom network removed'
}
ignoreExitValue = true
}
createNetworkTask.finalizedBy(deleteNetworkTask)

// Orchestration task
task orchestrateContainers {
dependsOn removeInitialElasticsearchContainer
doLast {
println 'Primary container modified, committed as a new image, and cleaned up.'
println 'Client container executed, waited for completion, and cleaned up.'
}
}

task buildDockerImage_elasticsearchRFSSource {
dependsOn sourceContainerCommitTask
}
88 changes: 88 additions & 0 deletions DocumentsFromSnapshotMigration/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -3,18 +3,34 @@ plugins {
id 'java'
id 'jacoco'
id 'io.freefair.lombok' version '8.6'
id "com.avast.gradle.docker-compose" version "0.17.4"
id 'com.bmuschko.docker-remote-api'
}

import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage
import groovy.transform.Canonical
import org.opensearch.migrations.common.CommonUtils

java.sourceCompatibility = JavaVersion.VERSION_11
java.targetCompatibility = JavaVersion.VERSION_11

@Canonical
class DockerServiceProps {
String projectName = ""
String dockerImageName = ""
String inputDir = ""
Map<String, String> buildArgs = [:]
List<String> taskDependencies = []
}

repositories {
mavenCentral()
}

dependencies {
implementation platform('io.projectreactor:reactor-bom:2023.0.5')
testImplementation platform('io.projectreactor:reactor-bom:2023.0.5')

implementation project(":commonDependencyVersionConstraints")

implementation project(":RFS")
Expand All @@ -40,13 +56,20 @@ dependencies {
testImplementation group: 'org.opensearch', name: 'opensearch-testcontainers'
testImplementation group: 'org.testcontainers', name: 'testcontainers'

testRuntimeOnly group: 'org.junit.jupiter', name: 'junit-jupiter-engine'

testImplementation platform('io.projectreactor:reactor-bom:2023.0.5')
}

application {
mainClassName = 'com.rfs.RfsMigrateDocuments'
}

// Cleanup additional docker build directory
clean.doFirst {
delete project.file("./docker/build")
}

// Utility task to allow copying required libraries into a 'dependencies' folder for security scanning
tasks.register('copyDependencies', Sync) {
duplicatesStrategy = DuplicatesStrategy.EXCLUDE
Expand All @@ -55,6 +78,71 @@ tasks.register('copyDependencies', Sync) {
into "${buildDir}/dependencies"
}

task copyDockerRuntimeJars (type: Sync) {
duplicatesStrategy = DuplicatesStrategy.EXCLUDE
description = 'Copy runtime JARs and app jar to docker build directory'

// Define the destination directory
def buildDir = project.file("./docker/build/runtimeJars")
into buildDir

// Add all the required runtime JARs to be copied
from configurations.runtimeClasspath
from tasks.named('jar')
include '*.jar'
}

DockerServiceProps[] dockerServices = [
new DockerServiceProps([projectName:"reindexFromSnapshot",
dockerImageName:"reindex_from_snapshot",
inputDir:"./docker",
taskDependencies:["copyDockerRuntimeJars"]]),
new DockerServiceProps([projectName:"emptyElasticsearchSource_7_10",
dockerImageName:"empty_elasticsearch_source_7_10",
inputDir:"./docker/TestSource_ES_7_10"]),
new DockerServiceProps([projectName:"emptyElasticsearchSource_7_17",
dockerImageName:"empty_elasticsearch_source_7_17",
inputDir:"./docker/TestSource_ES_7_17"]),
new DockerServiceProps([projectName:"trafficGenerator",
dockerImageName:"osb_traffic_generator",
inputDir:"./docker/TrafficGenerator",
taskDependencies:[":TrafficCapture:dockerSolution:buildDockerImage_elasticsearchTestConsole"]]),
] as DockerServiceProps[]

for (dockerService in dockerServices) {
task "buildDockerImage_${dockerService.projectName}" (type: DockerBuildImage) {
def hash = CommonUtils.calculateDockerHash(project.fileTree("docker/${dockerService.projectName}"))
for (dep in dockerService.taskDependencies) {
dependsOn dep
}
inputDir = project.file(dockerService.inputDir)
buildArgs = dockerService.buildArgs
images.add("migrations/${dockerService.dockerImageName}:${hash}")
images.add("migrations/${dockerService.dockerImageName}:${version}")
images.add("migrations/${dockerService.dockerImageName}:latest")
}
}

apply from: 'build-preloaded-source-image.gradle'

dockerCompose {
useComposeFiles = ['docker/docker-compose.yml']
projectName = 'rfs-compose'
}

// ../gradlew buildDockerImages
task buildDockerImages {
for (dockerService in dockerServices) {
dependsOn "buildDockerImage_${dockerService.projectName}"
}
}

tasks.named("buildDockerImage_elasticsearchRFSSource") {
dependsOn(':TrafficCapture:dockerSolution:buildDockerImage_elasticsearchTestConsole')
}
tasks.getByName('composeUp')
.dependsOn(tasks.getByName('buildDockerImages'))

jacocoTestReport {
reports {
xml.required = true
Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
84 changes: 0 additions & 84 deletions RFS/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -2,28 +2,14 @@ plugins {
id 'application'
id 'java'
id 'jacoco'
id "com.avast.gradle.docker-compose" version "0.17.4"
id 'com.bmuschko.docker-remote-api'
id 'io.freefair.lombok' version '8.6'
id 'java-test-fixtures'
id 'com.dorongold.task-tree' version '4.0.0'
}

import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage
import groovy.transform.Canonical
import org.opensearch.migrations.common.CommonUtils

java.sourceCompatibility = JavaVersion.VERSION_11
java.targetCompatibility = JavaVersion.VERSION_11

@Canonical
class DockerServiceProps {
String projectName = ""
String dockerImageName = ""
String inputDir = ""
Map<String, String> buildArgs = [:]
List<String> taskDependencies = []
}

repositories {
mavenCentral()
Expand Down Expand Up @@ -104,11 +90,6 @@ task migrateDocuments (type: JavaExec) {
mainClass = 'com.rfs.RfsMigrateDocuments'
}

// Cleanup additional docker build directory
clean.doFirst {
delete project.file("./docker/build")
}

// Utility task to allow copying required libraries into a 'dependencies' folder for security scanning
tasks.register('copyDependencies', Sync) {
duplicatesStrategy = DuplicatesStrategy.EXCLUDE
Expand All @@ -126,71 +107,6 @@ jacocoTestReport {
}
}

task copyDockerRuntimeJars (type: Sync) {
duplicatesStrategy = DuplicatesStrategy.EXCLUDE
description = 'Copy runtime JARs and app jar to docker build directory'

// Define the destination directory
def buildDir = project.file("./docker/build/runtimeJars")
into buildDir

// Add all the required runtime JARs to be copied
from configurations.runtimeClasspath
from tasks.named('jar')
include '*.jar'
}

DockerServiceProps[] dockerServices = [
new DockerServiceProps([projectName:"reindexFromSnapshot",
dockerImageName:"reindex_from_snapshot",
inputDir:"./docker",
taskDependencies:["copyDockerRuntimeJars"]]),
new DockerServiceProps([projectName:"emptyElasticsearchSource_7_10",
dockerImageName:"empty_elasticsearch_source_7_10",
inputDir:"./docker/TestSource_ES_7_10"]),
new DockerServiceProps([projectName:"emptyElasticsearchSource_7_17",
dockerImageName:"empty_elasticsearch_source_7_17",
inputDir:"./docker/TestSource_ES_7_17"]),
new DockerServiceProps([projectName:"trafficGenerator",
dockerImageName:"osb_traffic_generator",
inputDir:"./docker/TrafficGenerator",
taskDependencies:[":TrafficCapture:dockerSolution:buildDockerImage_elasticsearchTestConsole"]]),
] as DockerServiceProps[]

for (dockerService in dockerServices) {
task "buildDockerImage_${dockerService.projectName}" (type: DockerBuildImage) {
def hash = CommonUtils.calculateDockerHash(project.fileTree("docker/${dockerService.projectName}"))
for (dep in dockerService.taskDependencies) {
dependsOn dep
}
inputDir = project.file(dockerService.inputDir)
buildArgs = dockerService.buildArgs
images.add("migrations/${dockerService.dockerImageName}:${hash}")
images.add("migrations/${dockerService.dockerImageName}:${version}")
images.add("migrations/${dockerService.dockerImageName}:latest")
}
}

apply from: 'build-preloaded-source-image.gradle'

dockerCompose {
useComposeFiles = ['docker/docker-compose.yml']
projectName = 'rfs-compose'
}

// ../gradlew buildDockerImages
task buildDockerImages {
for (dockerService in dockerServices) {
dependsOn "buildDockerImage_${dockerService.projectName}"
}
}

tasks.named("buildDockerImage_elasticsearchRFSSource") {
dependsOn(':TrafficCapture:dockerSolution:buildDockerImage_elasticsearchTestConsole')
}
tasks.getByName('composeUp')
.dependsOn(tasks.getByName('buildDockerImages'))

test {
useJUnitPlatform {
excludeTags 'longTest'
Expand Down

0 comments on commit dbb93a4

Please sign in to comment.