Merge pull request guardian#4241 from guardian/an/modern-testcontainers

use testcontainers directly instead of docker-testkit
bbc · Jun 20, 2024 · 1222ebb · 1222ebb
2 parents ce2cde0 + 302b96f
commit 1222ebb
Show file tree

Hide file tree

Showing 8 changed files with 86 additions and 82 deletions.
diff --git a/build.sbt b/build.sbt
@@ -102,7 +102,8 @@ lazy val commonLib = project("common-lib").settings(
     "com.gu" %% "scanamo" % "1.0.0-M8",
     // Necessary to have a mix of play library versions due to scala-java8-compat incompatibility
     "com.typesafe.play" %% "play-ahc-ws" % "2.8.9",
-    "org.yaml" % "snakeyaml" % "1.31"
+    "org.yaml" % "snakeyaml" % "1.31",
+    "org.testcontainers" % "elasticsearch" % "1.19.2" % Test
   ),
   dependencyOverrides += "org.apache.thrift" % "libthrift" % "0.13.0"
 )
@@ -136,30 +137,31 @@ lazy val kahuna = playProject("kahuna", 9005).settings(
 
 lazy val leases = playProject("leases", 9012)
 
-lazy val mediaApi = playProject("media-api", 9001).settings(
-  libraryDependencies ++= Seq(
-    "org.apache.commons" % "commons-email" % "1.5",
-    "org.parboiled" %% "parboiled" % "2.1.5",
-    "org.http4s" %% "http4s-core" % "0.23.17",
-    "com.softwaremill.quicklens" %% "quicklens" % "1.4.11",
-    "com.whisk" %% "docker-testkit-scalatest" % "0.9.8" % Test,
-    "com.whisk" %% "docker-testkit-impl-spotify" % "0.9.8" % Test
+lazy val mediaApi = playProject("media-api", 9001)
+  .dependsOn(commonLib % "compile;test->test")
+  .settings(
+    libraryDependencies ++= Seq(
+      "org.apache.commons" % "commons-email" % "1.5",
+      "org.parboiled" %% "parboiled" % "2.1.5",
+      "org.http4s" %% "http4s-core" % "0.23.17",
+      "com.softwaremill.quicklens" %% "quicklens" % "1.4.11",
+    )
   )
-)
 
 lazy val metadataEditor = playProject("metadata-editor", 9007)
 
-lazy val thrall = playProject("thrall", 9002).settings(
-  pipelineStages := Seq(digest, gzip),
-  libraryDependencies ++= Seq(
-    "org.codehaus.groovy" % "groovy-json" % "3.0.7",
-    "com.yakaz.elasticsearch.plugins" % "elasticsearch-action-updatebyquery" % "2.2.0",
-    "com.amazonaws" % "amazon-kinesis-client" % "1.8.10",
-    "com.whisk" %% "docker-testkit-scalatest" % "0.9.8" % Test,
-    "com.whisk" %% "docker-testkit-impl-spotify" % "0.9.8" % Test,
-    "com.google.protobuf" % "protobuf-java" % "3.19.6"
+lazy val thrall = playProject("thrall", 9002)
+  .dependsOn(commonLib % "compile;test->test")
+  .settings(
+    pipelineStages := Seq(digest, gzip),
+    libraryDependencies ++= Seq(
+      "org.codehaus.groovy" % "groovy-json" % "3.0.7",
+      "com.yakaz.elasticsearch.plugins" % "elasticsearch-action-updatebyquery" % "2.2.0",
+      "com.amazonaws" % "amazon-kinesis-client" % "1.8.10",
+      "org.testcontainers" % "elasticsearch" % "1.19.2" % Test,
+      "com.google.protobuf" % "protobuf-java" % "3.19.6"
+    )
   )
-)
 
 lazy val usage = playProject("usage", 9009).settings(
   libraryDependencies ++= Seq(

diff --git a/common-lib/src/test/scala/com/gu/mediaservice/testlib/ElasticSearchDockerBase.scala b/common-lib/src/test/scala/com/gu/mediaservice/testlib/ElasticSearchDockerBase.scala
@@ -0,0 +1,46 @@
+package com.gu.mediaservice.testlib
+
+import org.scalatest.{BeforeAndAfterAll, Suite}
+import org.testcontainers.containers.wait.strategy.Wait
+import org.testcontainers.elasticsearch.ElasticsearchContainer
+
+import scala.util.Properties
+import scala.concurrent.duration._
+import scala.compat.java8.DurationConverters._
+import scala.jdk.CollectionConverters._
+
+
+trait ElasticSearchDockerBase extends BeforeAndAfterAll {
+  self: Suite =>
+  val useEsDocker = Properties.envOrElse("USE_DOCKER_FOR_TESTS", "true").toBoolean
+
+  val esContainer: Option[ElasticsearchContainer] = if (useEsDocker) {
+    {
+      val container = new ElasticsearchContainer("docker.elastic.co/elasticsearch/elasticsearch:7.16.2")
+        .withExposedPorts(9200)
+        .withAccessToHost(true)
+        .withEnv(Map(
+          "cluster.name" -> "media-service",
+          "xpack.security.enabled" -> "false",
+          "discovery.type" -> "single-node",
+          "network.host" -> "0.0.0.0"
+        ).asJava)
+        .waitingFor(Wait.forHttp("/")
+          .forPort(9200)
+          .forStatusCode(200)
+          .withStartupTimeout(180.seconds.toJava)
+        )
+      container.start()
+      Some(container)
+    }
+  } else None
+
+  val esPort = esContainer.map(_.getMappedPort(9200)).getOrElse(9200)
+  val esTestUrl = Properties.envOrElse("ES6_TEST_URL", s"http://localhost:$esPort")
+
+  override protected def afterAll(): Unit = {
+    super.afterAll()
+
+    esContainer foreach { _.stop() }
+  }
+}
diff --git a/media-api/test/lib/elasticsearch/ElasticSearchTest.scala b/media-api/test/lib/elasticsearch/ElasticSearchTest.scala
@@ -11,7 +11,6 @@ import com.gu.mediaservice.model.leases.DenySyndicationLease
 import com.gu.mediaservice.model.usage.PublishedUsageStatus
 import com.sksamuel.elastic4s.ElasticDsl
 import com.sksamuel.elastic4s.ElasticDsl._
-import com.whisk.docker.{DockerContainer, DockerReadyChecker}
 import lib.querysyntax._
 import lib.{MediaApiConfig, MediaApiMetrics}
 import org.joda.time.DateTime
@@ -25,6 +24,7 @@ import play.api.mvc.Security.AuthenticatedRequest
 
 import scala.concurrent.duration._
 import scala.concurrent.{Await, Future}
+import scala.concurrent.ExecutionContext.Implicits.global
 
 class ElasticSearchTest extends ElasticSearchTestBase with Eventually with ElasticSearchExecutions with MockitoSugar {
 
@@ -47,20 +47,12 @@ class ElasticSearchTest extends ElasticSearchTestBase with Eventually with Elast
       current = "Images_Current",
       migration = "Images_Migration"
     ),
-    url = es6TestUrl,
+    url = esTestUrl,
     shards = 1,
     replicas = 0
   )
 
 
-
-  def esContainer = if (useEsDocker) Some(DockerContainer("docker.elastic.co/elasticsearch/elasticsearch:7.16.2")
-    .withPorts(9200 -> Some(9200))
-    .withEnv("cluster.name=media-service", "xpack.security.enabled=false", "discovery.type=single-node", "network.host=0.0.0.0")
-    .withReadyChecker(
-      DockerReadyChecker.HttpResponseCode(9200, "/", Some("0.0.0.0")).within(10.minutes).looped(40, 1250.millis)
-    )
-  ) else None
   private lazy val ES = new ElasticSearch(mediaApiConfig, mediaApiMetrics, elasticConfig, () => List.empty, mock[Scheduler])
   lazy val client = ES.client
 

diff --git a/media-api/test/lib/elasticsearch/ElasticSearchTestBase.scala b/media-api/test/lib/elasticsearch/ElasticSearchTestBase.scala
@@ -1,39 +1,23 @@
 package lib.elasticsearch
 
-import java.util.UUID
-import com.gu.mediaservice.lib.logging.{LogMarker, MarkerMap}
 import com.gu.mediaservice.model._
-import com.whisk.docker.impl.spotify.DockerKitSpotify
-import com.whisk.docker.scalatest.DockerTestKit
-import com.whisk.docker.{DockerContainer, DockerKit}
+import com.gu.mediaservice.testlib.ElasticSearchDockerBase
 import org.joda.time.DateTime
 import org.scalatest.concurrent.PatienceConfiguration.{Interval, Timeout}
 import org.scalatest.concurrent.ScalaFutures
 import org.scalatest.funspec.AnyFunSpec
-import org.scalatest.time.{Milliseconds, Seconds, Span}
-import org.scalatest.BeforeAndAfterAll
 import org.scalatest.matchers.should.Matchers
+import org.scalatest.time.{Milliseconds, Seconds, Span}
 import play.api.libs.json.JsString
 
-import scala.concurrent.duration._
-import scala.util.Properties
+import java.util.UUID
 
-trait ElasticSearchTestBase extends AnyFunSpec with BeforeAndAfterAll with Matchers with ScalaFutures with Fixtures with DockerKit with DockerTestKit with DockerKitSpotify with ConditionFixtures {
 
+trait ElasticSearchTestBase extends AnyFunSpec with ElasticSearchDockerBase with Matchers with ScalaFutures with Fixtures with ConditionFixtures {
 
   val interval = Interval(Span(100, Milliseconds))
   val timeout = Timeout(Span(10, Seconds))
 
-  val useEsDocker = Properties.envOrElse("USE_DOCKER_FOR_TESTS", "true").toBoolean
-  val es6TestUrl = Properties.envOrElse("ES6_TEST_URL", "http://localhost:9200")
-
-  def esContainer: Option[DockerContainer]
-
-  final override def dockerContainers: List[DockerContainer] =
-    esContainer.toList ++ super.dockerContainers
-
-  final override val StartContainersTimeout = 1.minute
-
   lazy val images = Seq(
     createImage("getty-image-1", Agency("Getty Images")),
     createImage("getty-image-2", Agency("Getty Images")),

diff --git a/thrall/test/lib/elasticsearch/ElasticSearchTest.scala b/thrall/test/lib/elasticsearch/ElasticSearchTest.scala
@@ -1,18 +1,16 @@
 package lib.elasticsearch
 
-import java.util.UUID
 import com.gu.mediaservice.lib.logging.{LogMarker, MarkerMap}
 import com.gu.mediaservice.model
 import com.gu.mediaservice.model._
 import com.gu.mediaservice.model.leases.{LeasesByMedia, MediaLease}
 import com.gu.mediaservice.model.usage.{PublishedUsageStatus, SyndicatedUsageStatus}
-import com.gu.mediaservice.model.usage.Usage
-import com.sksamuel.elastic4s.ElasticDsl
 import com.sksamuel.elastic4s.ElasticDsl._
-import com.sksamuel.elastic4s.http._
 import org.joda.time.{DateTime, DateTimeZone}
-import play.api.libs.json.{JsDefined, JsLookupResult, JsObject, JsString, Json}
+import play.api.libs.json.{JsString, Json}
 
+import java.util.UUID
+import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.{Await, Future}
 
 class ElasticSearchTest extends ElasticSearchTestBase {

diff --git a/thrall/test/lib/elasticsearch/ElasticSearchTestBase.scala b/thrall/test/lib/elasticsearch/ElasticSearchTestBase.scala
@@ -3,29 +3,23 @@ package lib.elasticsearch
 import akka.actor.Scheduler
 import com.gu.mediaservice.lib.elasticsearch.{ElasticSearchAliases, ElasticSearchConfig}
 import com.gu.mediaservice.lib.logging.{LogMarker, MarkerMap}
+import com.gu.mediaservice.testlib.ElasticSearchDockerBase
 import com.sksamuel.elastic4s.ElasticDsl
 import com.sksamuel.elastic4s.ElasticDsl._
-import com.whisk.docker.impl.spotify.DockerKitSpotify
-import com.whisk.docker.scalatest.DockerTestKit
-import com.whisk.docker.{DockerContainer, DockerKit, DockerReadyChecker}
 import helpers.Fixtures
 import org.joda.time.DateTime
+import org.scalatest.BeforeAndAfterEach
 import org.scalatest.concurrent.{Eventually, ScalaFutures}
 import org.scalatest.freespec.AnyFreeSpec
 import org.scalatest.matchers.should.Matchers
-import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
 import org.scalatestplus.mockito.MockitoSugar
 import play.api.libs.json.{JsDefined, JsLookupResult, Json}
 
 import scala.concurrent.Await
-import scala.concurrent.ExecutionContext.global
+import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.duration._
-import scala.util.Properties
 
-trait ElasticSearchTestBase extends AnyFreeSpec with Matchers with Fixtures with BeforeAndAfterAll with BeforeAndAfterEach with Eventually with ScalaFutures with DockerKit with DockerTestKit with DockerKitSpotify with MockitoSugar {
-
-  val useEsDocker = Properties.envOrElse("USE_DOCKER_FOR_TESTS", "true").toBoolean
-  val esTestUrl = Properties.envOrElse("ES6_TEST_URL", "http://localhost:9200")
+trait ElasticSearchTestBase extends AnyFreeSpec with Matchers with Fixtures with ElasticSearchDockerBase with BeforeAndAfterEach with Eventually with ScalaFutures with MockitoSugar {
 
   val oneHundredMilliseconds = Duration(100, MILLISECONDS)
   val fiveSeconds = Duration(5, SECONDS)
@@ -45,14 +39,6 @@ trait ElasticSearchTestBase extends AnyFreeSpec with Matchers with Fixtures with
     replicas = 0
   )
 
-  val esContainer = if (useEsDocker) Some(DockerContainer("docker.elastic.co/elasticsearch/elasticsearch:7.16.2")
-    .withPorts(9200 -> Some(9200))
-    .withEnv("cluster.name=media-service", "xpack.security.enabled=false", "discovery.type=single-node", "network.host=0.0.0.0")
-    .withReadyChecker(
-      DockerReadyChecker.HttpResponseCode(9200, "/", Some("0.0.0.0")).within(10.minutes).looped(40, 1250.millis)
-    )
-  ) else None
-
   lazy val ES = new ElasticSearch(elasticSearchConfig, None, mock[Scheduler])
 
   override def beforeAll {
@@ -82,13 +68,9 @@ trait ElasticSearchTestBase extends AnyFreeSpec with Matchers with Fixtures with
 
   override def afterAll: Unit = {
     super.afterAll()
-  }
-
-  final override def dockerContainers: List[DockerContainer] =
-    esContainer.toList ++ super.dockerContainers
-
-  final override val StartContainersTimeout = 1.minute
 
+    esContainer foreach { _.stop() }
+  }
 
   def reloadedImage(id: String) = {
     implicit val logMarker: LogMarker = MarkerMap()

diff --git a/thrall/test/lib/elasticsearch/GoodToGoCheckTest.scala b/thrall/test/lib/elasticsearch/GoodToGoCheckTest.scala
@@ -2,11 +2,10 @@ package lib.elasticsearch
 
 import com.gu.mediaservice.lib.elasticsearch.MappingTest
 import com.gu.mediaservice.lib.logging.LogMarker
-import com.sksamuel.elastic4s.requests.count.CountResponse
 import com.sksamuel.elastic4s.ElasticDsl._
-import com.sksamuel.elastic4s.Response
 import org.joda.time.{DateTime, DateTimeZone, Period}
 
+import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.Future
 
 class GoodToGoCheckTest extends ElasticSearchTestBase {

diff --git a/thrall/test/lib/elasticsearch/ImageModelTest.scala b/thrall/test/lib/elasticsearch/ImageModelTest.scala
@@ -4,7 +4,8 @@ import com.gu.mediaservice.lib.elasticsearch.MappingTest
 import com.gu.mediaservice.lib.logging.{LogMarker, MarkerMap}
 
 import scala.collection.TraversableLike
-import scala.concurrent.{Await, Future}
+import scala.concurrent.Await
+import scala.concurrent.ExecutionContext.Implicits.global
 
 class ImageModelTest extends ElasticSearchTestBase {
   implicit val logMarker: LogMarker = MarkerMap()