add a POST /images/restore/:id endpoint to image-loader to restor…

…e image from replica bucket (wired into a new view in thrall)

common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala

@@ -45,6 +45,9 @@ class ImageIngestOperations(imageBucket: String, thumbnailBucket: String, config
   def deleteOriginal(id: String)(implicit logMarker: LogMarker): Future[Unit] = if(isVersionedS3) deleteVersionedImage(imageBucket, fileKeyFromId(id)) else deleteImage(imageBucket, fileKeyFromId(id))
   def deleteThumbnail(id: String)(implicit logMarker: LogMarker): Future[Unit] = deleteImage(thumbnailBucket, fileKeyFromId(id))
   def deletePng(id: String)(implicit logMarker: LogMarker): Future[Unit] = deleteImage(imageBucket, optimisedPngKeyFromId(id))
+
+  def doesOriginalExist(id: String): Boolean =
+    client.doesObjectExist(imageBucket, fileKeyFromId(id))
 }
 
 sealed trait ImageWrapper {

common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala

@@ -43,4 +43,5 @@ class S3ImageStorage(config: CommonConfig) extends S3(config) with ImageStorage
 		files.foreach(file => client.deleteObject(bucket, file.getKey))
 		logger.info(logMarker, s"Deleting images in folder $id from bucket $bucket")
 	}
+
 }

common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala

@@ -242,7 +242,7 @@ object S3Ops {
   // TODO: Make this region aware - i.e. RegionUtils.getRegion(region).getServiceEndpoint(AmazonS3.ENDPOINT_PREFIX)
   val s3Endpoint = "s3.amazonaws.com"
 
-  def buildS3Client(config: CommonConfig, forceV2Sigs: Boolean = false, localstackAware: Boolean = true): AmazonS3 = {
+  def buildS3Client(config: CommonConfig, forceV2Sigs: Boolean = false, localstackAware: Boolean = true, maybeRegionOverride: Option[String] = None): AmazonS3 = {
 
     val clientConfig = new ClientConfiguration()
     // Option to disable v4 signatures (https://github.com/aws/aws-sdk-java/issues/372) which is required by imgops
@@ -260,6 +260,6 @@ object S3Ops {
       case _ => AmazonS3ClientBuilder.standard().withClientConfiguration(clientConfig)
     }
 
-    config.withAWSCredentials(builder, localstackAware).build()
+    config.withAWSCredentials(builder, localstackAware).withRegion(maybeRegionOverride.getOrElse(config.awsRegion)).build()
   }
 }

image-loader/app/controllers/ImageLoaderController.scala

@@ -1,6 +1,9 @@
 package controllers
 
-import java.io.File
+import com.amazonaws.services.s3.AmazonS3
+import com.amazonaws.util.IOUtils
+
+import java.io.{File, FileOutputStream}
 import java.net.URI
 import com.drew.imaging.ImageProcessingException
 import com.gu.mediaservice.lib.argo.ArgoHelpers
@@ -9,23 +12,26 @@ import com.gu.mediaservice.lib.auth._
 import com.gu.mediaservice.lib.formatting.printDateTime
 import com.gu.mediaservice.lib.logging.{FALLBACK, LogMarker, MarkerMap}
 import com.gu.mediaservice.lib.{DateTimeUtils, ImageIngestOperations}
-import com.gu.mediaservice.model.UnsupportedMimeTypeException
+import com.gu.mediaservice.model.{UnsupportedMimeTypeException, UploadInfo}
 import com.gu.scanamo.error.ConditionNotMet
 import lib.FailureResponse.Response
 import lib.{FailureResponse, _}
-import lib.imaging.{NoSuchImageExistsInS3, UserImageLoaderException}
+import lib.imaging.{MimeTypeDetection, NoSuchImageExistsInS3, UserImageLoaderException}
 import lib.storage.ImageLoaderStore
-import model.{Projector, QuarantineUploader, StatusType, UploadStatus, UploadStatusRecord, Uploader}
+import model.{Projector, QuarantineUploader, S3FileExtractedMetadata, StatusType, UploadStatus, UploadStatusRecord, Uploader}
 import play.api.libs.json.Json
 import play.api.mvc._
 import model.upload.UploadRequest
 
 import java.time.Instant
 import com.gu.mediaservice.GridClient
+import com.gu.mediaservice.lib.ImageIngestOperations.fileKeyFromId
 import com.gu.mediaservice.lib.auth.Authentication.OnBehalfOfPrincipal
+import com.gu.mediaservice.lib.aws.S3Ops
 import com.gu.mediaservice.lib.play.RequestLoggingFilter
 
-import scala.concurrent.{ExecutionContext, Future}
+import scala.concurrent.duration.DurationInt
+import scala.concurrent.{Await, ExecutionContext, Future}
 import scala.util.{Failure, Success, Try}
 import scala.util.control.NonFatal
 
@@ -244,6 +250,61 @@ class ImageLoaderController(auth: Authentication,
     }
   }
 
+  lazy val replicaS3: AmazonS3 = S3Ops.buildS3Client(config, maybeRegionOverride = Some("us-west-1"))
+  def restoreFromReplica(imageId: String): Action[AnyContent] = AuthenticatedAndAuthorised.async { request =>
+
+    implicit val logMarker: LogMarker = MarkerMap(
+      "imageId" -> imageId,
+      "requestType" -> "image-projection",
+      "requestId" -> RequestLoggingFilter.getRequestId(request)
+    )
+
+    config.maybeImageReplicaBucket match {
+      case _ if store.doesOriginalExist(imageId) =>
+        Future.successful(Conflict("Image already exists in main bucket"))
+      case None =>
+        Future.successful(NotImplemented("No replica bucket configured"))
+      case Some(replicaBucket) if replicaS3.doesObjectExist(replicaBucket, fileKeyFromId(imageId)) =>
+        val s3Key = fileKeyFromId(imageId)
+
+        logger.info(logMarker, s"Restoring image $imageId from replica bucket $replicaBucket (key: $s3Key)")
+
+        val replicaObject = replicaS3.getObject(replicaBucket, s3Key)
+        val metadata = S3FileExtractedMetadata(replicaObject.getObjectMetadata)
+        val stream = replicaObject.getObjectContent
+        val tempFile = createTempFile(s"restoringReplica-$imageId")
+        val fos = new FileOutputStream(tempFile)
+        try {
+          IOUtils.copy(stream, fos)
+        } finally {
+          stream.close()
+        }
+
+        val future =  uploader.storeFile(UploadRequest(
+          imageId,
+          tempFile, //TODO could we give it the stream directly
+          mimeType = MimeTypeDetection.guessMimeType(tempFile) match {
+            case Left(unsupported) => throw unsupported
+            case right => right.toOption
+          },
+          metadata.uploadTime,
+          metadata.uploadedBy,
+          metadata.identifiers,
+          UploadInfo(metadata.uploadFileName)
+        ))
+
+        future.onComplete(_ => Try { deleteTempFile(tempFile) })
+
+        future.map {_ =>
+          logger.info(logMarker, s"Restored image $imageId from replica bucket $replicaBucket (key: $s3Key)")
+          Redirect(s"${config.kahunaUri}/images/$imageId")
+        }
+
+      case _ =>
+        Future.successful(NotFound("Image not found in replica bucket"))
+    }
+  }
+
   // Find this a better home if used more widely
   implicit class NonEmpty(s: String) {
     def nonEmptyOpt: Option[String] = if (s.isEmpty) None else Some(s)

image-loader/app/lib/ImageLoaderConfig.scala

@@ -12,6 +12,8 @@ import scala.concurrent.duration.FiniteDuration
 class ImageLoaderConfig(resources: GridConfigResources) extends CommonConfig(resources) with StrictLogging {
   val imageBucket: String = string("s3.image.bucket")
 
+  val maybeImageReplicaBucket: Option[String] = stringOpt("s3.image.replicaBucket")
+
   val thumbnailBucket: String = string("s3.thumb.bucket")
   val quarantineBucket: Option[String] = stringOpt("s3.quarantine.bucket")
   val uploadToQuarantineEnabled: Boolean = boolean("upload.quarantine.enabled")
@@ -23,7 +25,7 @@ class ImageLoaderConfig(resources: GridConfigResources) extends CommonConfig(res
 
   val rootUri: String = services.loaderBaseUri
   val apiUri: String = services.apiBaseUri
-  val loginUriTemplate: String = services.loginUriTemplate
+  val kahunaUri: String = services.kahunaBaseUri
 
   val transcodedMimeTypes: List[MimeType] = getStringSet("transcoded.mime.types").toList.map(MimeType(_))
   val supportedMimeTypes: List[MimeType] = List(Jpeg, Png) ::: transcodedMimeTypes

image-loader/conf/routes

@@ -1,14 +1,15 @@
 GET     /                                             controllers.ImageLoaderController.index
 POST    /images                                       controllers.ImageLoaderController.loadImage(uploadedBy: Option[String], identifiers: Option[String], uploadTime: Option[String], filename: Option[String])
 POST    /imports                                      controllers.ImageLoaderController.importImage(uri: String, uploadedBy: Option[String], identifiers: Option[String], uploadTime: Option[String], filename: Option[String])
++nocsrf
+POST    /images/restore/:id                           controllers.ImageLoaderController.restoreFromReplica(id: String)
 GET     /images/project/:imageId                      controllers.ImageLoaderController.projectImageBy(imageId: String)
 
 # Upload Status
 GET     /uploadStatus/:imageId                        controllers.UploadStatusController.getUploadStatus(imageId: String)
 POST    /uploadStatus/:imageId                        controllers.UploadStatusController.updateUploadStatus(imageId: String)
 
 
-
 # Management
 GET     /management/healthcheck                       com.gu.mediaservice.lib.management.Management.healthCheck
 GET     /management/manifest                          com.gu.mediaservice.lib.management.Management.manifest

thrall/app/ThrallComponents.scala

@@ -82,7 +82,7 @@ class ThrallComponents(context: Context) extends GridComponents(context, new Thr
   )
   val syncCheckerStream: Future[Done] = syncChecker.run()
 
-  val thrallController = new ThrallController(es, migrationSourceWithSender.send, messageSender, actorSystem, auth, config.services, controllerComponents, gridClient)
+  val thrallController = new ThrallController(es, store, migrationSourceWithSender.send, messageSender, actorSystem, auth, config.services, controllerComponents, gridClient)
   val healthCheckController = new HealthCheck(es, streamRunning.isCompleted, config, controllerComponents)
   val InnerServiceStatusCheckController = new InnerServiceStatusCheckController(auth, controllerComponents, config.services, wsClient)
 

thrall/app/controllers/ThrallController.scala

@@ -11,7 +11,7 @@ import com.gu.mediaservice.lib.elasticsearch.{NotRunning, Running}
 import com.gu.mediaservice.lib.logging.GridLogging
 import com.gu.mediaservice.model.{CompleteMigrationMessage, CreateMigrationIndexMessage, UpsertFromProjectionMessage}
 import lib.elasticsearch.ElasticSearch
-import lib.{MigrationRequest, OptionalFutureRunner, Paging}
+import lib.{MigrationRequest, OptionalFutureRunner, Paging, ThrallStore}
 import org.joda.time.{DateTime, DateTimeZone}
 import play.api.data.Form
 import play.api.data.Forms._
@@ -26,6 +26,7 @@ case class MigrateSingleImageForm(id: String)
 
 class ThrallController(
   es: ElasticSearch,
+  store: ThrallStore,
   sendMigrationRequest: MigrationRequest => Future[Boolean],
   messageSender: ThrallMessageSender,
   actorSystem: ActorSystem,
@@ -67,11 +68,12 @@ class ThrallController(
 
   def upsertProjectPage(imageId: Option[String]) = withLoginRedirectAsync { implicit request =>
     imageId match {
-      case Some(id) =>
+      case Some(id) if store.doesOriginalExist(id) =>
         gridClient.getProjectionDiff(id, auth.innerServiceCall).map {
           case None => NotFound("couldn't generate projection for that image!!")
           case Some(diff) => Ok(views.html.previewUpsertProject(id, Json.prettyPrint(diff)))
         }
+      case Some(_) => Future.successful(Redirect(routes.ThrallController.restoreFromReplica))
       case None => Future.successful(Ok(views.html.upsertProject()))
     }
   }
@@ -242,6 +244,10 @@ class ThrallController(
     }}
   }
 
+  def restoreFromReplica: Action[AnyContent] = withLoginRedirect {implicit request =>
+    Ok(views.html.restoreFromReplica(s"${services.loaderBaseUri}/images/restore")) //FIXME figure out imageId bit
+  }
+
   def reattemptMigrationFailures(filter: String, page: Int): Action[AnyContent] = withLoginRedirectAsync { implicit request =>
     Paging.withPaging(Some(page)) { paging =>
       es.migrationStatus match {

thrall/app/views/index.scala.html

@@ -45,6 +45,8 @@
 <body>
     <nav>
         <span><a href="@routes.ThrallController.upsertProjectPage(None)">Upsert from projection</a></span>
+        &nbsp;|&nbsp;
+        <span><a href="@routes.ThrallController.restoreFromReplica">Restore from replica</a></span>
     </nav>
     <main>
         <h1>Current elasticsearch indices</h1>

thrall/app/views/restoreFromReplica.scala.html

@@ -0,0 +1,31 @@
+@import helper._
+@(imageLoaderEndpoint:String)(implicit request: RequestHeader)
+
+<!DOCTYPE html>
+<html lang="en">
+    <head>
+        <meta charset="UTF-8">
+        <title>Restore from replica</title>
+        <link rel="stylesheet" href="@routes.Assets.versioned("stylesheets/main.css")" />
+    </head>
+    <body>
+        <nav>
+            <span>
+                <a href="@routes.ThrallController.index">&lt; back to Thrall Dashboard</a>
+            </span>
+        </nav>
+        <main>
+            <h1>Restore from replica</h1>
+            <p>
+                This option will re-ingest an image from the replica bucket (if configured). It should fail early if the image already exists in the main image bucket.
+            </p>
+            <p>Useful if the image has been hard-deleted.</p>
+
+            @form(Call("POST", imageLoaderEndpoint)) {
+                <label for="imageId">Image ID:</label>
+                <input type="text" id="imageId" name="imageId">
+                <input type="submit" value="Submit">
+            }
+        </main>
+    </body>
+</html>

thrall/conf/routes

@@ -4,6 +4,7 @@
 
 GET     /                                             controllers.ThrallController.index
 GET     /upsertProject                                controllers.ThrallController.upsertProjectPage(imageId: Option[String])
+GET     /restoreFromReplica                           controllers.ThrallController.restoreFromReplica
 GET     /migrationFailuresOverview                    controllers.ThrallController.migrationFailuresOverview()
 GET     /migrationFailures                            controllers.ThrallController.migrationFailures(filter: String, page: Option[Int])
 +nocsrf