Merge pull request #1061 from AndreKurait/CaptureProxyBaseImage

Remove elasticsearch from default capture proxy base image
opensearch-project · Oct 17, 2024 · d8b4d6c · d8b4d6c
2 parents 30185a2 + 2bd40b8
commit d8b4d6c
Show file tree

Hide file tree

Showing 27 changed files with 247 additions and 165 deletions.
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -200,8 +200,20 @@ jobs:
           node-version: ${{ env.node-version }}
       - name: Install NPM dependencies
         run: npm ci
-      - name: Run CDK Jest Tests
-        run: npm test
+      - name: Mock Docker Images for CDK Tests
+        run: |
+          image=$(docker images --format '{{.Repository}}:{{.Tag}}' | head -n 1)
+          echo "Using image for mocked tags: $image"
+          docker tag $image migrations/capture_proxy:latest
+          docker tag $image migrations/capture_proxy_es:latest
+          docker tag $image opensearchproject/opensearch:2
+          docker tag $image migrations/elasticsearch_searchguard:latest
+          docker tag $image docker.io/apache/kafka:3.7.0
+          docker tag $image migrations/migration_console:latest
+          docker tag $image migrations/reindex_from_snapshot:latest
+          docker tag $image migrations/traffic_replayer:latest
+      - name: Run CDK Jest Tests (using mocked images)
+        run: npm run test:withoutBuildImages
 
   link-checker:
     runs-on: ubuntu-latest

diff --git a/DocumentsFromSnapshotMigration/build.gradle b/DocumentsFromSnapshotMigration/build.gradle
@@ -145,6 +145,6 @@ tasks.named('composeUp') {
 }
 
 tasks.named('slowTest') {
-    dependsOn(':TrafficCapture:dockerSolution:buildDockerImage_elasticsearchTestConsole')
+    dependsOn(':TrafficCapture:dockerSolution:buildDockerImage_elasticsearch_client_test_console')
     testLogging.showStandardStreams = false
 }
diff --git a/TrafficCapture/dockerSolution/build.gradle b/TrafficCapture/dockerSolution/build.gradle
@@ -7,14 +7,6 @@ plugins {
 import org.opensearch.migrations.common.CommonUtils
 import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage
 
-def calculateDockerHash = { projectName ->
-    CommonUtils.calculateDockerHash(project.fileTree("src/main/docker/${projectName}"))
-}
-
-clean.doFirst {
-    delete project.file("./src/main/docker/migrationConsole/build")
-}
-
 dependencies {
     constraints {
         implementation('software.amazon.awssdk:secretsmanager:2.25.19') {
@@ -24,90 +16,97 @@ dependencies {
 }
 
 def dockerFilesForExternalServices = [
-        "elasticsearchWithSearchGuard": "elasticsearch_searchguard",
-        "elasticsearchTestConsole": "elasticsearch_client_test_console",
-        "migrationConsole": "migration_console",
-        "otelCollector": "otel_collector",
+        "elasticsearch_searchguard": "elasticsearchWithSearchGuard",
+        "capture_proxy_base": "captureProxyBase",
+        "elasticsearch_client_test_console": "elasticsearchTestConsole",
+        "migration_console": "migrationConsole",
+        "otel_collector": "otelCollector",
         "grafana": "grafana"
 ]
-// Create the static docker files that aren't hosting migrations java code from this repo
-dockerFilesForExternalServices.each { projectName, dockerImageName ->
+dockerFilesForExternalServices.each { dockerImageName, projectName ->
     def escapedProjectName = projectName;
-    task("buildDockerImage_${escapedProjectName}", type: DockerBuildImage) {
+    task("buildDockerImage_${dockerImageName}", type: DockerBuildImage) {
         if (escapedProjectName == "migrationConsole") {
-            def dependencies = []
-            dependencies += stageConsoleDependenciesFromLibraries(project as Project,
-                    List.of(project(":libraries:kafkaCommandLineFormatter")),
-                    projectName, "src/main/docker/${escapedProjectName}")
-            dependencies +=stageConsoleDependenciesFromApplications(project as Project,
-                    List.of(project(":CreateSnapshot"),
-                            project(":MetadataMigration"),
-                    ), projectName, "src/main/docker/${escapedProjectName}")
-
-
-            def parentCopyTask = project.tasks.create("copyArtifact_${projectName}")
-            dependencies.forEach { d -> parentCopyTask.dependsOn(d) }
+            def libraries = [
+                project(":libraries:kafkaCommandLineFormatter")
+            ]
 
-            dependsOn "copyArtifact_${escapedProjectName}"
-            dependsOn "buildDockerImage_elasticsearchTestConsole"
+            def applications = [
+                project(":CreateSnapshot"),
+                project(":MetadataMigration")
+            ]
+            def syncTask = getMigrationConsoleSyncTask(project, dockerImageName, escapedProjectName, libraries, applications)
+            dependsOn syncTask
+            inputDir = syncTask.destinationDir
+        } else {
+            inputDir = project.file("src/main/docker/${escapedProjectName}")
         }
-        def hash = calculateDockerHash(projectName)
-        images.add("migrations/${dockerImageName}:$hash")
+        def hashNonce = CommonUtils.calculateDockerHash(project.fileTree(inputDir))
+        images.add("migrations/${dockerImageName}:$hashNonce")
         images.add("migrations/${dockerImageName}:latest")
-        inputDir = project.file("src/main/docker/${projectName}")
     }
 }
 
-
-static def List<Task> stageConsoleDependenciesFromLibraries(Project dockerBuildProject,
-                                                            List<Project> sourceArtifactProjects,
-                                                            String destProjectName, String destDir) {
-    return sourceArtifactProjects.collect { sourceArtifactProject ->
-        def applicationDestDir = "${destDir}/staging/${sourceArtifactProject.name}/";
-        def libCopyTask = dockerBuildProject.tasks.create("copyLibArtifacts_${destProjectName}_${sourceArtifactProject.name}", Sync) {
-            from { sourceArtifactProject.configurations.findByName("runtimeClasspath").files }
-            from { sourceArtifactProject.tasks.getByName('jar') }
-            into "${applicationDestDir}/lib"
+static Sync getMigrationConsoleSyncTask(Project project, String dockerImageName, String escapedProjectName, List<Project> libraries, List<Project> applications) {
+     // Create a single sync task to copy the required files
+    def destDir = "build/docker/${dockerImageName}_${escapedProjectName}"
+    def syncTask = project.tasks.create("syncArtifact_${dockerImageName}_${escapedProjectName}", Sync) {
+        into destDir
+        duplicatesStrategy = DuplicatesStrategy.EXCLUDE
+
+        // Applications and Standalone Libraries both have libraries, sync them
+        (libraries + applications).each { lib ->
+            def applicationDestDir = "staging/${lib.name}/"
+            from (lib.configurations.findByName("runtimeClasspath").files) {
+                into "${applicationDestDir}/lib"
+            }
+            from (lib.tasks.getByName('jar')) {
+                into "${applicationDestDir}/lib"
+            }
         }
-        libCopyTask.dependsOn(sourceArtifactProject.tasks.named("assemble"))
-        return libCopyTask
-    }
-}
 
-static def List<Task> stageConsoleDependenciesFromApplications(Project dockerBuildProject,
-                                                               List<Project> sourceArtifactProjects,
-                                                               String destProjectName, String destDir) {
-    def libDeps = stageConsoleDependenciesFromLibraries(dockerBuildProject, sourceArtifactProjects, destProjectName, destDir)
-    def appDeps = sourceArtifactProjects.collect { sourceArtifactProject ->
-        def applicationDestDir = "${destDir}/staging/${sourceArtifactProject.name}/";
-        def binCopyTask = dockerBuildProject.tasks.create("copyBinArtifacts_${destProjectName}_${sourceArtifactProject.name}", Sync) {
-            from { sourceArtifactProject.tasks.getByName('startScripts').outputs.files }
-            into "${applicationDestDir}/bin"
+        // Sync application start scripts
+        applications.each { app ->
+            def applicationDestDir = "staging/${app.name}/"
+            from (app.tasks.getByName('startScripts').outputs.files) {
+                into "${applicationDestDir}/bin"
+            }
         }
-        binCopyTask.dependsOn(sourceArtifactProject.tasks.named("assemble"))
-        return binCopyTask;
+
+        from "src/main/docker/${escapedProjectName}"
     }
-    return libDeps + appDeps;
+
+    // Collect assemble tasks from libraries and applications
+    def assembleTasks = (libraries + applications).collect { it.tasks.named("assemble") }
+    // Ensure the sync task depends on the assemble tasks
+    syncTask.dependsOn assembleTasks
+
+    // Migration Console base image is the test console
+    syncTask.dependsOn "buildDockerImage_elasticsearch_client_test_console"
+
+    return syncTask
 }
 
 def javaContainerServices = [
-        ":TrafficCapture:trafficCaptureProxyServer": "capture_proxy",
-        ":TrafficCapture:trafficReplayer": "traffic_replayer"
+        "capture_proxy": ":TrafficCapture:trafficCaptureProxyServer",
+        "capture_proxy_es": ":TrafficCapture:trafficCaptureProxyServer",
+        "traffic_replayer": ":TrafficCapture:trafficReplayer"
 ]
-def baseImageProjectOverrides = [
-        "trafficCaptureProxyServer": "elasticsearchWithSearchGuard"
+def baseImageOverrides = [
+        "capture_proxy": "capture_proxy_base",
+        "capture_proxy_es": "elasticsearch_searchguard",
 ]
-javaContainerServices.each { projectName, dockerImageName ->
+javaContainerServices.each { dockerImageName, projectName ->
     def artifactProject = project(projectName);
-    CommonUtils.copyArtifactFromProjectToProjectsDockerStaging(project as Project, artifactProject)
-    CommonUtils.createDockerfile(project, artifactProject, baseImageProjectOverrides, dockerFilesForExternalServices)
+    CommonUtils.copyArtifactFromProjectToProjectsDockerStaging(project as Project, artifactProject, dockerImageName)
+    CommonUtils.createDockerfile(project, artifactProject, baseImageOverrides[dockerImageName], dockerFilesForExternalServices, dockerImageName)
 }
 
-javaContainerServices.forEach { projectName, dockerImageName ->
+javaContainerServices.forEach { dockerImageName, projectName ->
     def escapedProjectName = project(projectName).name;
-    def dockerBuildDir = "build/docker/${escapedProjectName}"
-    task "buildDockerImage_${escapedProjectName}"(type: DockerBuildImage) {
-        dependsOn "createDockerfile_${escapedProjectName}"
+    def dockerBuildDir = "build/docker/${dockerImageName}_${escapedProjectName}"
+    task "buildDockerImage_${dockerImageName}"(type: DockerBuildImage) {
+        dependsOn "createDockerfile_${dockerImageName}"
         inputDir = project.file("${dockerBuildDir}")
         images.add("migrations/${dockerImageName}:${version}")
         images.add("migrations/${dockerImageName}:latest")
@@ -126,13 +125,13 @@ dockerCompose {
 }
 
 task buildDockerImages {
-    dependsOn buildDockerImage_elasticsearchWithSearchGuard
-    dependsOn buildDockerImage_migrationConsole
-    dependsOn buildDockerImage_otelCollector
+    dependsOn buildDockerImage_elasticsearch_searchguard
+    dependsOn buildDockerImage_migration_console
+    dependsOn buildDockerImage_otel_collector
     dependsOn buildDockerImage_grafana
-
-    dependsOn buildDockerImage_trafficCaptureProxyServer
-    dependsOn buildDockerImage_trafficReplayer
+    dependsOn buildDockerImage_traffic_replayer
+    dependsOn buildDockerImage_capture_proxy
+    dependsOn buildDockerImage_capture_proxy_es
 }
 
 tasks.getByName('composeUp')

diff --git a/TrafficCapture/dockerSolution/src/main/docker/captureProxyBase/Dockerfile b/TrafficCapture/dockerSolution/src/main/docker/captureProxyBase/Dockerfile
@@ -0,0 +1,56 @@
+# =======================
+# Stage 1: Certificate Generation
+# =======================
+FROM amazonlinux:2023 AS cert-builder
+
+# Install OpenSSL
+RUN yum update -y && \
+    yum install -y openssl && \
+    yum clean all
+
+# Generate self-signed certificates
+RUN mkdir -p /certs && \
+    openssl req -x509 -nodes -newkey rsa:4096 \
+    -keyout /certs/key.pem \
+    -out /certs/ca.pem \
+    -subj "/CN=localhost" \
+    -days 36500 && \
+    openssl req -new -key /certs/key.pem -out /certs/key.csr -subj "/CN=localhost" && \
+    openssl x509 -req -in /certs/key.csr -CA /certs/ca.pem -CAkey /certs/key.pem -CAcreateserial -out /certs/pub.pem -days 36500 -sha256
+
+# =======================
+# Stage 2: Final Image
+# =======================
+FROM amazoncorretto:11-al2023-headless
+
+# Set environment variables
+ENV CONFIG_HOME=/usr/share/captureProxy/config
+ENV PROXY_TLS_CONFIG_FILE=$CONFIG_HOME/proxy_tls.yml
+
+# Copy certificates from builder stage
+USER root
+RUN mkdir -p $CONFIG_HOME/certs
+COPY --from=cert-builder /certs/ca.pem $CONFIG_HOME/ca.pem
+COPY --from=cert-builder /certs/pub.pem $CONFIG_HOME/pub.pem
+COPY --from=cert-builder /certs/key.pem $CONFIG_HOME/key.pem
+RUN chown -R root:root $CONFIG_HOME/
+
+# Update the default Java truststore to trust the root-ca
+RUN keytool -import -alias root-ca -cacerts -file $CONFIG_HOME/ca.pem -storepass changeit -noprompt
+
+# Note: The following configurations are added to ensure compatibility with the es-7.10-oss base image.
+# The proxy_tls.yml file are configured to work seamlessly with the Capture Proxy.
+RUN echo 'plugins.security.ssl.http.enabled: true' >> $PROXY_TLS_CONFIG_FILE && \
+    echo 'plugins.security.ssl.http.pemcert_filepath: pub.pem' >> $PROXY_TLS_CONFIG_FILE && \
+    echo 'plugins.security.ssl.http.pemkey_filepath: key.pem' >> $PROXY_TLS_CONFIG_FILE && \
+    echo 'plugins.security.ssl.http.pemtrustedcas_filepath: ca.pem' >> $PROXY_TLS_CONFIG_FILE && \
+    echo "plugins.security.ssl.http.enabled_protocols: ['TLSv1.2', 'TLSv1.3']" >> $PROXY_TLS_CONFIG_FILE
+
+RUN chmod 644 $PROXY_TLS_CONFIG_FILE && \
+    chown root:root $PROXY_TLS_CONFIG_FILE
+
+# Default port
+EXPOSE 9200
+
+# Set the entrypoint
+CMD ["tail", "-f", "/dev/null"]
diff --git a/TrafficCapture/dockerSolution/src/main/docker/composeExtensions/proxy-multi.yml b/TrafficCapture/dockerSolution/src/main/docker/composeExtensions/proxy-multi.yml
@@ -2,7 +2,7 @@ version: '3.7'
 services:
 
   capture-proxy-es1:
-     image: 'migrations/capture_proxy:latest'
+     image: 'migrations/capture_proxy_es:latest'
      networks:
       - migrations
      ports:
@@ -22,7 +22,7 @@ services:
       - kafka
 
   capture-proxy-es2:
-    image: 'migrations/capture_proxy:latest'
+    image: 'migrations/capture_proxy_es:latest'
     networks:
       - migrations
     ports:

diff --git a/TrafficCapture/dockerSolution/src/main/docker/composeExtensions/proxy-single.yml b/TrafficCapture/dockerSolution/src/main/docker/composeExtensions/proxy-single.yml
@@ -4,7 +4,7 @@ services:
 
   # Run combined instance of Capture Proxy and Elasticsearch
 #  capture-proxy-es:
-#    image: 'migrations/capture_proxy:latest'
+#    image: 'migrations/capture_proxy_es:latest'
 #    networks:
 #      - migrations
 #    ports:
@@ -25,7 +25,7 @@ services:
       - migrations
     ports:
       - "9200:9200"
-    command: /runJavaWithClasspath.sh org.opensearch.migrations.trafficcapture.proxyserver.CaptureProxy  --kafkaConnection kafka:9092 --destinationUri  https://elasticsearch:9200  --insecureDestination --listenPort 9200 --sslConfigFile /usr/share/elasticsearch/config/proxy_tls.yml --otelCollectorEndpoint http://otel-collector:4317
+    command: /runJavaWithClasspath.sh org.opensearch.migrations.trafficcapture.proxyserver.CaptureProxy  --kafkaConnection kafka:9092 --destinationUri  https://elasticsearch:9200  --insecureDestination --listenPort 9200 --sslConfigFile /usr/share/captureProxy/config/proxy_tls.yml --otelCollectorEndpoint http://otel-collector:4317
     depends_on:
       - kafka
       - elasticsearch

diff --git a/TrafficCapture/trafficCaptureProxyServerTest/build.gradle b/TrafficCapture/trafficCaptureProxyServerTest/build.gradle
@@ -43,12 +43,12 @@ configurations {
 
 
 def dockerFilesForExternalServices = [
-        "nginx": "nginx-perf-test-webserver"
+        "nginx_perf_test_webserver": "nginx"
 ]
 
 // Create the static docker files that aren't hosting migrations java code from this repo
-dockerFilesForExternalServices.each { projectName, dockerImageName ->
-    task("buildDockerImage_${projectName}", type: DockerBuildImage) {
+dockerFilesForExternalServices.each { dockerImageName, projectName ->
+    task("buildDockerImage_${dockerImageName}", type: DockerBuildImage) {
         def hash = calculateDockerHash(projectName)
         images.add("migrations/${dockerImageName}:$hash")
         images.add("migrations/${dockerImageName}:latest")
@@ -57,41 +57,44 @@ dockerFilesForExternalServices.each { projectName, dockerImageName ->
 }
 
 def javaContainerServices = [
-        ":TrafficCapture:trafficCaptureProxyServerTest": "jmeter"
+        "jmeter": ":TrafficCapture:trafficCaptureProxyServerTest"
 ]
 def baseImageProjectOverrides = [
-        "nginx": "nginx-perf-test-webserver"
+        "nginx": "nginx_perf_test_webserver"
 ]
 
-def createContainerTasks = { projectName, dockerImageName ->
+def createContainerTasks = { dockerImageName, projectName ->
     def sourceArtifactProject = project(projectName);
-    CommonUtils.copyArtifactFromProjectToProjectsDockerStaging(project, sourceArtifactProject)
-    CommonUtils.createDockerfile(project, sourceArtifactProject, baseImageProjectOverrides, dockerFilesForExternalServices)
+    CommonUtils.copyArtifactFromProjectToProjectsDockerStaging(project, sourceArtifactProject, dockerImageName)
+    CommonUtils.createDockerfile(project, sourceArtifactProject, baseImageProjectOverrides[dockerImageName], dockerFilesForExternalServices, dockerImageName)
 }
 
 javaContainerServices.each(createContainerTasks)
 
-(javaContainerServices).forEach { projectName, dockerImageName ->
+(javaContainerServices).forEach { dockerImageName, projectName ->
     def escapedProjectName = project(projectName).name;
-    def dockerBuildDir = "build/docker/${escapedProjectName}"
-    task "buildDockerImage_${escapedProjectName}"(type: DockerBuildImage) {
-        dependsOn "createDockerfile_${escapedProjectName}"
+    def dockerBuildDir = "build/docker/${dockerImageName}_${escapedProjectName}"
+    task "buildDockerImage_${dockerImageName}"(type: DockerBuildImage) {
+        dependsOn "createDockerfile_${dockerImageName}"
         inputDir = project.file("${dockerBuildDir}")
         images.add("migrations/${dockerImageName}:${version}")
         images.add("migrations/${dockerImageName}:latest")
     }
 }
 
-
-
 dockerCompose {
     useComposeFiles.add("src/main/docker/docker-compose.yml")
 }
 
 task buildDockerImages {
-    dependsOn(':TrafficCapture:dockerSolution:buildDockerImage_trafficCaptureProxyServer')
-    dependsOn buildDockerImage_nginx
-    dependsOn buildDockerImage_trafficCaptureProxyServerTest
+    dependsOn ':TrafficCapture:dockerSolution:buildDockerImage_capture_proxy'
+    dependsOn buildDockerImage_nginx_perf_test_webserver
+    dependsOn buildDockerImage_jmeter
+}
+
+tasks.named('copyArtifact_jmeter') {
+    // Add dependency on inter-project dependencies from above dependencies
+    dependsOn ':TrafficCapture:trafficCaptureProxyServer:build'
 }
 
 tasks.getByName('composeUp')