From a9b6709f8453f953b271d167cd15768ddc345e39 Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Wed, 19 Jun 2024 13:40:05 -0400 Subject: [PATCH] Extract some of the build logic to build a docker image for RFS and move it to DocumentFromSnapshotMigration Signed-off-by: Greg Schohn --- .../build-preloaded-source-image.gradle | 112 ++++++++++++++++++ DocumentsFromSnapshotMigration/build.gradle | 88 ++++++++++++++ .../docker/Dockerfile | 0 .../docker/TestSource_ES_7_10/Dockerfile | 0 .../TestSource_ES_7_10/container-start.sh | 0 .../docker/TestSource_ES_7_17/Dockerfile | 0 .../TestSource_ES_7_17/container-start.sh | 0 .../docker/TrafficGenerator/Dockerfile | 0 .../TrafficGenerator/generateDataset.sh | 0 .../docker/docker-compose.yml | 0 RFS/build.gradle | 84 ------------- build.gradle | 2 +- 12 files changed, 201 insertions(+), 85 deletions(-) create mode 100644 DocumentsFromSnapshotMigration/build-preloaded-source-image.gradle rename {RFS => DocumentsFromSnapshotMigration}/docker/Dockerfile (100%) rename {RFS => DocumentsFromSnapshotMigration}/docker/TestSource_ES_7_10/Dockerfile (100%) rename {RFS => DocumentsFromSnapshotMigration}/docker/TestSource_ES_7_10/container-start.sh (100%) rename {RFS => DocumentsFromSnapshotMigration}/docker/TestSource_ES_7_17/Dockerfile (100%) rename {RFS => DocumentsFromSnapshotMigration}/docker/TestSource_ES_7_17/container-start.sh (100%) rename {RFS => DocumentsFromSnapshotMigration}/docker/TrafficGenerator/Dockerfile (100%) rename {RFS => DocumentsFromSnapshotMigration}/docker/TrafficGenerator/generateDataset.sh (100%) rename {RFS => DocumentsFromSnapshotMigration}/docker/docker-compose.yml (100%) diff --git a/DocumentsFromSnapshotMigration/build-preloaded-source-image.gradle b/DocumentsFromSnapshotMigration/build-preloaded-source-image.gradle new file mode 100644 index 000000000..b0b25b3fa --- /dev/null +++ b/DocumentsFromSnapshotMigration/build-preloaded-source-image.gradle @@ -0,0 +1,112 @@ +import com.bmuschko.gradle.docker.tasks.container.* +import com.github.dockerjava.core.DefaultDockerClientConfig +import com.github.dockerjava.core.DockerClientImpl +import com.github.dockerjava.httpclient5.ApacheDockerHttpClient + +def createDockerClient() { + def config = DefaultDockerClientConfig.createDefaultConfigBuilder().build() + def httpClient = new ApacheDockerHttpClient.Builder() + .dockerHost(config.getDockerHost()) + .sslConfig(config.getSSLConfig()) + .maxConnections(100) + .build() + + return DockerClientImpl.getInstance(config, httpClient) +} + +def uniqueId = UUID.randomUUID(); +def myNetworkName = "rfs-preload-source-${uniqueId}"; +def createNetworkTask = task createNetwork(type: Exec) { + commandLine 'docker', 'network', 'create', myNetworkName + doLast { + println 'Network created' + } +} +task createInitialElasticsearchContainer(type: DockerCreateContainer) { + dependsOn createNetwork, buildDockerImage_emptyElasticsearchSource_7_17 + targetImageId 'migrations/emptyElasticsearchSource_7_17:latest' + containerName = "elasticsearch-${uniqueId}" + hostConfig.network = myNetworkName + hostConfig.dns = ['elasticsearch'] + networkAliases = ['elasticsearch'] + hostName = 'elasticsearch' +} + +def startSourceTask = task startInitialElasticsearchContainer(type: DockerStartContainer) { + dependsOn createInitialElasticsearchContainer + targetContainerId createInitialElasticsearchContainer.getContainerId() +} + +task createClientContainer(type: DockerCreateContainer) { + dependsOn startInitialElasticsearchContainer, buildDockerImage_trafficGenerator + targetImageId 'migrations/osb_traffic_generator:latest' + containerName = "traffic-generator-container-${uniqueId}" + hostConfig.network = myNetworkName + cmd = ['default_osb_test_workloads', "http://elasticsearch:9200"] +} + +def startClientTask = task startClientContainer(type: DockerStartContainer) { + dependsOn createClientContainer + targetContainerId createClientContainer.getContainerId() +} + +task waitClientContainer(type: DockerWaitContainer) { + dependsOn startClientContainer + targetContainerId createClientContainer.getContainerId() +} + +// Task to commit the source container to create a new image that will include the loaded data +def sourceContainerCommitTask = task commitSourceContainer() { + dependsOn waitClientContainer + + doLast { + def client = createDockerClient() + def containerId = createInitialElasticsearchContainer.getContainerId().get() + + client.commitCmd(containerId) + .withRepository("elasticsearch_rfs_source") + .withTag("latest") + .exec() + } +} + +task removeClientContainer(type: DockerRemoveContainer) { + dependsOn waitClientContainer + targetContainerId createClientContainer.getContainerId() +} +startClientTask.finalizedBy(removeClientContainer) + +// Task to stop and remove the primary container +task stopInitialElasticsearchContainer(type: DockerStopContainer) { + dependsOn commitSourceContainer + targetContainerId createInitialElasticsearchContainer.getContainerId() +} + +task removeInitialElasticsearchContainer(type: DockerRemoveContainer) { + dependsOn stopInitialElasticsearchContainer + targetContainerId createInitialElasticsearchContainer.getContainerId() +} +startSourceTask.finalizedBy(removeInitialElasticsearchContainer) + +def deleteNetworkTask = task deleteNetwork(type: Exec) { + mustRunAfter removeInitialElasticsearchContainer, removeClientContainer + commandLine 'docker', 'network', 'rm', myNetworkName + doLast { + println 'Custom network removed' + } + ignoreExitValue = true +} +createNetworkTask.finalizedBy(deleteNetworkTask) + +// Orchestration task +task orchestrateContainers { + dependsOn removeInitialElasticsearchContainer + doLast { + println 'Primary container modified, committed as a new image, and cleaned up.' + println 'Client container executed, waited for completion, and cleaned up.' + } +} + +task buildDockerImage_elasticsearchRFSSource { + dependsOn sourceContainerCommitTask +} \ No newline at end of file diff --git a/DocumentsFromSnapshotMigration/build.gradle b/DocumentsFromSnapshotMigration/build.gradle index d8528d190..fddb3ac1e 100644 --- a/DocumentsFromSnapshotMigration/build.gradle +++ b/DocumentsFromSnapshotMigration/build.gradle @@ -3,18 +3,34 @@ plugins { id 'java' id 'jacoco' id 'io.freefair.lombok' version '8.6' + id "com.avast.gradle.docker-compose" version "0.17.4" + id 'com.bmuschko.docker-remote-api' } +import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage +import groovy.transform.Canonical import org.opensearch.migrations.common.CommonUtils java.sourceCompatibility = JavaVersion.VERSION_11 java.targetCompatibility = JavaVersion.VERSION_11 +@Canonical +class DockerServiceProps { + String projectName = "" + String dockerImageName = "" + String inputDir = "" + Map buildArgs = [:] + List taskDependencies = [] +} + repositories { mavenCentral() } dependencies { + implementation platform('io.projectreactor:reactor-bom:2023.0.5') + testImplementation platform('io.projectreactor:reactor-bom:2023.0.5') + implementation project(":commonDependencyVersionConstraints") implementation project(":RFS") @@ -40,6 +56,8 @@ dependencies { testImplementation group: 'org.opensearch', name: 'opensearch-testcontainers' testImplementation group: 'org.testcontainers', name: 'testcontainers' + testRuntimeOnly group: 'org.junit.jupiter', name: 'junit-jupiter-engine' + testImplementation platform('io.projectreactor:reactor-bom:2023.0.5') } @@ -47,6 +65,11 @@ application { mainClassName = 'com.rfs.RfsMigrateDocuments' } +// Cleanup additional docker build directory +clean.doFirst { + delete project.file("./docker/build") +} + // Utility task to allow copying required libraries into a 'dependencies' folder for security scanning tasks.register('copyDependencies', Sync) { duplicatesStrategy = DuplicatesStrategy.EXCLUDE @@ -55,6 +78,71 @@ tasks.register('copyDependencies', Sync) { into "${buildDir}/dependencies" } +task copyDockerRuntimeJars (type: Sync) { + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + description = 'Copy runtime JARs and app jar to docker build directory' + + // Define the destination directory + def buildDir = project.file("./docker/build/runtimeJars") + into buildDir + + // Add all the required runtime JARs to be copied + from configurations.runtimeClasspath + from tasks.named('jar') + include '*.jar' +} + +DockerServiceProps[] dockerServices = [ + new DockerServiceProps([projectName:"reindexFromSnapshot", + dockerImageName:"reindex_from_snapshot", + inputDir:"./docker", + taskDependencies:["copyDockerRuntimeJars"]]), + new DockerServiceProps([projectName:"emptyElasticsearchSource_7_10", + dockerImageName:"empty_elasticsearch_source_7_10", + inputDir:"./docker/TestSource_ES_7_10"]), + new DockerServiceProps([projectName:"emptyElasticsearchSource_7_17", + dockerImageName:"empty_elasticsearch_source_7_17", + inputDir:"./docker/TestSource_ES_7_17"]), + new DockerServiceProps([projectName:"trafficGenerator", + dockerImageName:"osb_traffic_generator", + inputDir:"./docker/TrafficGenerator", + taskDependencies:[":TrafficCapture:dockerSolution:buildDockerImage_elasticsearchTestConsole"]]), +] as DockerServiceProps[] + +for (dockerService in dockerServices) { + task "buildDockerImage_${dockerService.projectName}" (type: DockerBuildImage) { + def hash = CommonUtils.calculateDockerHash(project.fileTree("docker/${dockerService.projectName}")) + for (dep in dockerService.taskDependencies) { + dependsOn dep + } + inputDir = project.file(dockerService.inputDir) + buildArgs = dockerService.buildArgs + images.add("migrations/${dockerService.dockerImageName}:${hash}") + images.add("migrations/${dockerService.dockerImageName}:${version}") + images.add("migrations/${dockerService.dockerImageName}:latest") + } +} + +apply from: 'build-preloaded-source-image.gradle' + +dockerCompose { + useComposeFiles = ['docker/docker-compose.yml'] + projectName = 'rfs-compose' +} + +// ../gradlew buildDockerImages +task buildDockerImages { + for (dockerService in dockerServices) { + dependsOn "buildDockerImage_${dockerService.projectName}" + } +} + +tasks.named("buildDockerImage_elasticsearchRFSSource") { + dependsOn(':TrafficCapture:dockerSolution:buildDockerImage_elasticsearchTestConsole') +} +tasks.getByName('composeUp') + .dependsOn(tasks.getByName('buildDockerImages')) + jacocoTestReport { reports { xml.required = true diff --git a/RFS/docker/Dockerfile b/DocumentsFromSnapshotMigration/docker/Dockerfile similarity index 100% rename from RFS/docker/Dockerfile rename to DocumentsFromSnapshotMigration/docker/Dockerfile diff --git a/RFS/docker/TestSource_ES_7_10/Dockerfile b/DocumentsFromSnapshotMigration/docker/TestSource_ES_7_10/Dockerfile similarity index 100% rename from RFS/docker/TestSource_ES_7_10/Dockerfile rename to DocumentsFromSnapshotMigration/docker/TestSource_ES_7_10/Dockerfile diff --git a/RFS/docker/TestSource_ES_7_10/container-start.sh b/DocumentsFromSnapshotMigration/docker/TestSource_ES_7_10/container-start.sh similarity index 100% rename from RFS/docker/TestSource_ES_7_10/container-start.sh rename to DocumentsFromSnapshotMigration/docker/TestSource_ES_7_10/container-start.sh diff --git a/RFS/docker/TestSource_ES_7_17/Dockerfile b/DocumentsFromSnapshotMigration/docker/TestSource_ES_7_17/Dockerfile similarity index 100% rename from RFS/docker/TestSource_ES_7_17/Dockerfile rename to DocumentsFromSnapshotMigration/docker/TestSource_ES_7_17/Dockerfile diff --git a/RFS/docker/TestSource_ES_7_17/container-start.sh b/DocumentsFromSnapshotMigration/docker/TestSource_ES_7_17/container-start.sh similarity index 100% rename from RFS/docker/TestSource_ES_7_17/container-start.sh rename to DocumentsFromSnapshotMigration/docker/TestSource_ES_7_17/container-start.sh diff --git a/RFS/docker/TrafficGenerator/Dockerfile b/DocumentsFromSnapshotMigration/docker/TrafficGenerator/Dockerfile similarity index 100% rename from RFS/docker/TrafficGenerator/Dockerfile rename to DocumentsFromSnapshotMigration/docker/TrafficGenerator/Dockerfile diff --git a/RFS/docker/TrafficGenerator/generateDataset.sh b/DocumentsFromSnapshotMigration/docker/TrafficGenerator/generateDataset.sh similarity index 100% rename from RFS/docker/TrafficGenerator/generateDataset.sh rename to DocumentsFromSnapshotMigration/docker/TrafficGenerator/generateDataset.sh diff --git a/RFS/docker/docker-compose.yml b/DocumentsFromSnapshotMigration/docker/docker-compose.yml similarity index 100% rename from RFS/docker/docker-compose.yml rename to DocumentsFromSnapshotMigration/docker/docker-compose.yml diff --git a/RFS/build.gradle b/RFS/build.gradle index f1a2fdc4c..8ab08919f 100644 --- a/RFS/build.gradle +++ b/RFS/build.gradle @@ -2,28 +2,14 @@ plugins { id 'application' id 'java' id 'jacoco' - id "com.avast.gradle.docker-compose" version "0.17.4" - id 'com.bmuschko.docker-remote-api' id 'io.freefair.lombok' version '8.6' id 'java-test-fixtures' id 'com.dorongold.task-tree' version '4.0.0' } -import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage -import groovy.transform.Canonical -import org.opensearch.migrations.common.CommonUtils - java.sourceCompatibility = JavaVersion.VERSION_11 java.targetCompatibility = JavaVersion.VERSION_11 -@Canonical -class DockerServiceProps { - String projectName = "" - String dockerImageName = "" - String inputDir = "" - Map buildArgs = [:] - List taskDependencies = [] -} repositories { mavenCentral() @@ -104,11 +90,6 @@ task migrateDocuments (type: JavaExec) { mainClass = 'com.rfs.RfsMigrateDocuments' } -// Cleanup additional docker build directory -clean.doFirst { - delete project.file("./docker/build") -} - // Utility task to allow copying required libraries into a 'dependencies' folder for security scanning tasks.register('copyDependencies', Sync) { duplicatesStrategy = DuplicatesStrategy.EXCLUDE @@ -126,71 +107,6 @@ jacocoTestReport { } } -task copyDockerRuntimeJars (type: Sync) { - duplicatesStrategy = DuplicatesStrategy.EXCLUDE - description = 'Copy runtime JARs and app jar to docker build directory' - - // Define the destination directory - def buildDir = project.file("./docker/build/runtimeJars") - into buildDir - - // Add all the required runtime JARs to be copied - from configurations.runtimeClasspath - from tasks.named('jar') - include '*.jar' -} - -DockerServiceProps[] dockerServices = [ - new DockerServiceProps([projectName:"reindexFromSnapshot", - dockerImageName:"reindex_from_snapshot", - inputDir:"./docker", - taskDependencies:["copyDockerRuntimeJars"]]), - new DockerServiceProps([projectName:"emptyElasticsearchSource_7_10", - dockerImageName:"empty_elasticsearch_source_7_10", - inputDir:"./docker/TestSource_ES_7_10"]), - new DockerServiceProps([projectName:"emptyElasticsearchSource_7_17", - dockerImageName:"empty_elasticsearch_source_7_17", - inputDir:"./docker/TestSource_ES_7_17"]), - new DockerServiceProps([projectName:"trafficGenerator", - dockerImageName:"osb_traffic_generator", - inputDir:"./docker/TrafficGenerator", - taskDependencies:[":TrafficCapture:dockerSolution:buildDockerImage_elasticsearchTestConsole"]]), -] as DockerServiceProps[] - -for (dockerService in dockerServices) { - task "buildDockerImage_${dockerService.projectName}" (type: DockerBuildImage) { - def hash = CommonUtils.calculateDockerHash(project.fileTree("docker/${dockerService.projectName}")) - for (dep in dockerService.taskDependencies) { - dependsOn dep - } - inputDir = project.file(dockerService.inputDir) - buildArgs = dockerService.buildArgs - images.add("migrations/${dockerService.dockerImageName}:${hash}") - images.add("migrations/${dockerService.dockerImageName}:${version}") - images.add("migrations/${dockerService.dockerImageName}:latest") - } -} - -apply from: 'build-preloaded-source-image.gradle' - -dockerCompose { - useComposeFiles = ['docker/docker-compose.yml'] - projectName = 'rfs-compose' -} - -// ../gradlew buildDockerImages -task buildDockerImages { - for (dockerService in dockerServices) { - dependsOn "buildDockerImage_${dockerService.projectName}" - } -} - -tasks.named("buildDockerImage_elasticsearchRFSSource") { - dependsOn(':TrafficCapture:dockerSolution:buildDockerImage_elasticsearchTestConsole') -} -tasks.getByName('composeUp') - .dependsOn(tasks.getByName('buildDockerImages')) - test { useJUnitPlatform { excludeTags 'longTest' diff --git a/build.gradle b/build.gradle index 22a461f09..df5c54e9d 100644 --- a/build.gradle +++ b/build.gradle @@ -4,7 +4,7 @@ plugins { task buildDockerImages() { dependsOn(':TrafficCapture:dockerSolution:buildDockerImages') - dependsOn(':RFS:buildDockerImages') + dependsOn(':DocumentsFromSnapshotMigration:buildDockerImages') } subprojects {