support 'projecting' from replica bucket as a way of restoring images incorrectly hard deleted

common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala

@@ -242,7 +242,7 @@ object S3Ops {
   // TODO: Make this region aware - i.e. RegionUtils.getRegion(region).getServiceEndpoint(AmazonS3.ENDPOINT_PREFIX)
   val s3Endpoint = "s3.amazonaws.com"
 
-  def buildS3Client(config: CommonConfig, forceV2Sigs: Boolean = false, localstackAware: Boolean = true): AmazonS3 = {
+  def buildS3Client(config: CommonConfig, forceV2Sigs: Boolean = false, localstackAware: Boolean = true, region: String = "eu-west-1"): AmazonS3 = {
 
     val clientConfig = new ClientConfiguration()
     // Option to disable v4 signatures (https://github.com/aws/aws-sdk-java/issues/372) which is required by imgops
@@ -260,6 +260,6 @@ object S3Ops {
       case _ => AmazonS3ClientBuilder.standard().withClientConfiguration(clientConfig)
     }
 
-    config.withAWSCredentials(builder, localstackAware).build()
+    config.withAWSCredentials(builder, localstackAware).withRegion(region).build()
   }
 }

image-loader/app/lib/ImageLoaderConfig.scala

@@ -12,6 +12,8 @@ import scala.concurrent.duration.FiniteDuration
 class ImageLoaderConfig(resources: GridConfigResources) extends CommonConfig(resources) with StrictLogging {
   val imageBucket: String = string("s3.image.bucket")
 
+  val imageReplicaBucket: Option[String] = stringOpt("s3.image.replicaBucket")
+
   val thumbnailBucket: String = string("s3.thumb.bucket")
   val quarantineBucket: Option[String] = stringOpt("s3.quarantine.bucket")
   val uploadToQuarantineEnabled: Boolean = boolean("upload.quarantine.enabled")

image-loader/app/model/Projector.scala

@@ -32,7 +32,7 @@ object Projector {
   import Uploader.toImageUploadOpsCfg
 
   def apply(config: ImageLoaderConfig, imageOps: ImageOperations, processor: ImageProcessor, auth: Authentication)(implicit ec: ExecutionContext): Projector
-  = new Projector(toImageUploadOpsCfg(config), S3Ops.buildS3Client(config), imageOps, processor, auth)
+  = new Projector(toImageUploadOpsCfg(config), S3Ops.buildS3Client(config), S3Ops.buildS3Client(config, region = "us-west-1"), imageOps, processor, auth)
 }
 
 case class S3FileExtractedMetadata(
@@ -83,6 +83,7 @@ object S3FileExtractedMetadata {
 
 class Projector(config: ImageUploadOpsCfg,
                 s3: AmazonS3,
+                replicaS3: AmazonS3,
                 imageOps: ImageOperations,
                 processor: ImageProcessor,
                 auth: Authentication) extends GridLogging {
@@ -95,11 +96,21 @@ class Projector(config: ImageUploadOpsCfg,
       import ImageIngestOperations.fileKeyFromId
       val s3Key = fileKeyFromId(imageId)
 
-      if (!s3.doesObjectExist(config.originalFileBucket, s3Key))
-        throw new NoSuchImageExistsInS3(config.originalFileBucket, s3Key)
+      val (s3Client, bucket) = config.maybeReplicaBucket match {
+        case _ if s3.doesObjectExist(config.originalFileBucket, s3Key) =>
+          (s3, config.originalFileBucket)
+        case Some(replicaBucket) if replicaS3.doesObjectExist(replicaBucket, s3Key) =>
+          (replicaS3, replicaBucket)
+        case _ =>
+          logger.error(
+            logMarker,
+            s"Image with id $imageId does not exist at key $s3Key in S3 bucket ${config.originalFileBucket}${config.maybeReplicaBucket.map(rb => s" NOR replica bucket $rb")}"
+          )
+          throw new NoSuchImageExistsInS3(config.originalFileBucket, s3Key)
+      }
 
-      val s3Source = Stopwatch(s"object exists, getting s3 object at s3://${config.originalFileBucket}/$s3Key to perform Image projection"){
-        s3.getObject(config.originalFileBucket, s3Key)
+      val s3Source = Stopwatch(s"object exists, getting s3 object at s3://$bucket/$s3Key to perform Image projection"){
+        s3Client.getObject(bucket, s3Key)
       }(logMarker)
 
       try {

image-loader/app/model/Uploader.scala

@@ -67,6 +67,7 @@ case class ImageUploadOpsCfg(
   thumbQuality: Double,
   transcodedMimeTypes: List[MimeType],
   originalFileBucket: String,
+  maybeReplicaBucket: Option[String],
   thumbBucket: String
 )
 
@@ -89,6 +90,7 @@ object Uploader extends GridLogging {
       config.thumbQuality,
       config.transcodedMimeTypes,
       config.imageBucket,
+      config.imageReplicaBucket,
       config.thumbnailBucket
     )
   }

image-loader/test/scala/model/ImageUploadTest.scala

@@ -32,7 +32,7 @@ class ImageUploadTest extends AsyncFunSuite with Matchers with MockitoSugar {
   private implicit val logMarker: MockLogMarker = new MockLogMarker()
     // For mime type info, see https://github.com/guardian/grid/pull/2568
     val tempDir = new File("/tmp")
-    val mockConfig: ImageUploadOpsCfg = ImageUploadOpsCfg(tempDir, 256, 85d, List(Tiff), "img-bucket", "thumb-bucket")
+    val mockConfig: ImageUploadOpsCfg = ImageUploadOpsCfg(tempDir, 256, 85d, List(Tiff), "img-bucket", None, "thumb-bucket")
 
   /**
     * @todo: I flailed about until I found a path that worked, but

image-loader/test/scala/model/ProjectorTest.scala

@@ -37,11 +37,9 @@ class ProjectorTest extends AnyFreeSpec with Matchers with ScalaFutures with Moc
 
   private val imageOperations = new ImageOperations(ctxPath)
 
-  private val config = ImageUploadOpsCfg(new File("/tmp"), 256, 85d, Nil, "img-bucket", "thumb-bucket")
+  private val config = ImageUploadOpsCfg(new File("/tmp"), 256, 85d, Nil, "img-bucket", None, "thumb-bucket")
 
-  private val s3 = mock[AmazonS3]
-  private val auth = mock[Authentication]
-  private val projector = new Projector(config, s3, imageOperations, ImageProcessor.identity, auth)
+  private val projector = new Projector(config, mock[AmazonS3], mock[AmazonS3], imageOperations, ImageProcessor.identity, mock[Authentication])
 
   // FIXME temporary ignored as test is not executable in CI/CD machine
   // because graphic lib files like srgb.icc, cmyk.icc are in root directory instead of resources