diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala
index 02af178672..419bf41a52 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala
@@ -45,6 +45,9 @@ class ImageIngestOperations(imageBucket: String, thumbnailBucket: String, config
def deleteOriginal(id: String)(implicit logMarker: LogMarker): Future[Unit] = if(isVersionedS3) deleteVersionedImage(imageBucket, fileKeyFromId(id)) else deleteImage(imageBucket, fileKeyFromId(id))
def deleteThumbnail(id: String)(implicit logMarker: LogMarker): Future[Unit] = deleteImage(thumbnailBucket, fileKeyFromId(id))
def deletePng(id: String)(implicit logMarker: LogMarker): Future[Unit] = deleteImage(imageBucket, optimisedPngKeyFromId(id))
+
+ def doesOriginalExist(id: String): Boolean =
+ client.doesObjectExist(imageBucket, fileKeyFromId(id))
}
sealed trait ImageWrapper {
diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala
index 2eff0ccb20..45020a315c 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala
@@ -43,4 +43,5 @@ class S3ImageStorage(config: CommonConfig) extends S3(config) with ImageStorage
files.foreach(file => client.deleteObject(bucket, file.getKey))
logger.info(logMarker, s"Deleting images in folder $id from bucket $bucket")
}
+
}
diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
index 6b031db1fd..029df565a1 100644
--- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
+++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala
@@ -242,7 +242,7 @@ object S3Ops {
// TODO: Make this region aware - i.e. RegionUtils.getRegion(region).getServiceEndpoint(AmazonS3.ENDPOINT_PREFIX)
val s3Endpoint = "s3.amazonaws.com"
- def buildS3Client(config: CommonConfig, forceV2Sigs: Boolean = false, localstackAware: Boolean = true): AmazonS3 = {
+ def buildS3Client(config: CommonConfig, forceV2Sigs: Boolean = false, localstackAware: Boolean = true, maybeRegionOverride: Option[String] = None): AmazonS3 = {
val clientConfig = new ClientConfiguration()
// Option to disable v4 signatures (https://github.com/aws/aws-sdk-java/issues/372) which is required by imgops
@@ -260,6 +260,6 @@ object S3Ops {
case _ => AmazonS3ClientBuilder.standard().withClientConfiguration(clientConfig)
}
- config.withAWSCredentials(builder, localstackAware).build()
+ config.withAWSCredentials(builder, localstackAware).withRegion(maybeRegionOverride.getOrElse(config.awsRegion)).build()
}
}
diff --git a/image-loader/app/controllers/ImageLoaderController.scala b/image-loader/app/controllers/ImageLoaderController.scala
index 7c6615688e..045bbaffda 100644
--- a/image-loader/app/controllers/ImageLoaderController.scala
+++ b/image-loader/app/controllers/ImageLoaderController.scala
@@ -1,6 +1,9 @@
package controllers
-import java.io.File
+import com.amazonaws.services.s3.AmazonS3
+import com.amazonaws.util.IOUtils
+
+import java.io.{File, FileOutputStream}
import java.net.URI
import com.drew.imaging.ImageProcessingException
import com.gu.mediaservice.lib.argo.ArgoHelpers
@@ -9,23 +12,26 @@ import com.gu.mediaservice.lib.auth._
import com.gu.mediaservice.lib.formatting.printDateTime
import com.gu.mediaservice.lib.logging.{FALLBACK, LogMarker, MarkerMap}
import com.gu.mediaservice.lib.{DateTimeUtils, ImageIngestOperations}
-import com.gu.mediaservice.model.UnsupportedMimeTypeException
+import com.gu.mediaservice.model.{UnsupportedMimeTypeException, UploadInfo}
import com.gu.scanamo.error.ConditionNotMet
import lib.FailureResponse.Response
import lib.{FailureResponse, _}
-import lib.imaging.{NoSuchImageExistsInS3, UserImageLoaderException}
+import lib.imaging.{MimeTypeDetection, NoSuchImageExistsInS3, UserImageLoaderException}
import lib.storage.ImageLoaderStore
-import model.{Projector, QuarantineUploader, StatusType, UploadStatus, UploadStatusRecord, Uploader}
+import model.{Projector, QuarantineUploader, S3FileExtractedMetadata, StatusType, UploadStatus, UploadStatusRecord, Uploader}
import play.api.libs.json.Json
import play.api.mvc._
import model.upload.UploadRequest
import java.time.Instant
import com.gu.mediaservice.GridClient
+import com.gu.mediaservice.lib.ImageIngestOperations.fileKeyFromId
import com.gu.mediaservice.lib.auth.Authentication.OnBehalfOfPrincipal
+import com.gu.mediaservice.lib.aws.S3Ops
import com.gu.mediaservice.lib.play.RequestLoggingFilter
-import scala.concurrent.{ExecutionContext, Future}
+import scala.concurrent.duration.DurationInt
+import scala.concurrent.{Await, ExecutionContext, Future}
import scala.util.{Failure, Success, Try}
import scala.util.control.NonFatal
@@ -244,6 +250,61 @@ class ImageLoaderController(auth: Authentication,
}
}
+ lazy val replicaS3: AmazonS3 = S3Ops.buildS3Client(config, maybeRegionOverride = Some("us-west-1"))
+ def restoreFromReplica(imageId: String): Action[AnyContent] = AuthenticatedAndAuthorised.async { request =>
+
+ implicit val logMarker: LogMarker = MarkerMap(
+ "imageId" -> imageId,
+ "requestType" -> "image-projection",
+ "requestId" -> RequestLoggingFilter.getRequestId(request)
+ )
+
+ config.maybeImageReplicaBucket match {
+ case _ if store.doesOriginalExist(imageId) =>
+ Future.successful(Conflict("Image already exists in main bucket"))
+ case None =>
+ Future.successful(NotImplemented("No replica bucket configured"))
+ case Some(replicaBucket) if replicaS3.doesObjectExist(replicaBucket, fileKeyFromId(imageId)) =>
+ val s3Key = fileKeyFromId(imageId)
+
+ logger.info(logMarker, s"Restoring image $imageId from replica bucket $replicaBucket (key: $s3Key)")
+
+ val replicaObject = replicaS3.getObject(replicaBucket, s3Key)
+ val metadata = S3FileExtractedMetadata(replicaObject.getObjectMetadata)
+ val stream = replicaObject.getObjectContent
+ val tempFile = createTempFile(s"restoringReplica-$imageId")
+ val fos = new FileOutputStream(tempFile)
+ try {
+ IOUtils.copy(stream, fos)
+ } finally {
+ stream.close()
+ }
+
+ val future = uploader.storeFile(UploadRequest(
+ imageId,
+ tempFile, //TODO could we give it the stream directly
+ mimeType = MimeTypeDetection.guessMimeType(tempFile) match {
+ case Left(unsupported) => throw unsupported
+ case right => right.toOption
+ },
+ metadata.uploadTime,
+ metadata.uploadedBy,
+ metadata.identifiers,
+ UploadInfo(metadata.uploadFileName)
+ ))
+
+ future.onComplete(_ => Try { deleteTempFile(tempFile) })
+
+ future.map {_ =>
+ logger.info(logMarker, s"Restored image $imageId from replica bucket $replicaBucket (key: $s3Key)")
+ Redirect(s"${config.kahunaUri}/images/$imageId")
+ }
+
+ case _ =>
+ Future.successful(NotFound("Image not found in replica bucket"))
+ }
+ }
+
// Find this a better home if used more widely
implicit class NonEmpty(s: String) {
def nonEmptyOpt: Option[String] = if (s.isEmpty) None else Some(s)
diff --git a/image-loader/app/lib/ImageLoaderConfig.scala b/image-loader/app/lib/ImageLoaderConfig.scala
index bd78bcb6a3..8c85037fe9 100644
--- a/image-loader/app/lib/ImageLoaderConfig.scala
+++ b/image-loader/app/lib/ImageLoaderConfig.scala
@@ -12,6 +12,8 @@ import scala.concurrent.duration.FiniteDuration
class ImageLoaderConfig(resources: GridConfigResources) extends CommonConfig(resources) with StrictLogging {
val imageBucket: String = string("s3.image.bucket")
+ val maybeImageReplicaBucket: Option[String] = stringOpt("s3.image.replicaBucket")
+
val thumbnailBucket: String = string("s3.thumb.bucket")
val quarantineBucket: Option[String] = stringOpt("s3.quarantine.bucket")
val uploadToQuarantineEnabled: Boolean = boolean("upload.quarantine.enabled")
@@ -23,7 +25,7 @@ class ImageLoaderConfig(resources: GridConfigResources) extends CommonConfig(res
val rootUri: String = services.loaderBaseUri
val apiUri: String = services.apiBaseUri
- val loginUriTemplate: String = services.loginUriTemplate
+ val kahunaUri: String = services.kahunaBaseUri
val transcodedMimeTypes: List[MimeType] = getStringSet("transcoded.mime.types").toList.map(MimeType(_))
val supportedMimeTypes: List[MimeType] = List(Jpeg, Png) ::: transcodedMimeTypes
diff --git a/image-loader/conf/routes b/image-loader/conf/routes
index 91afab7858..6aafad8e35 100644
--- a/image-loader/conf/routes
+++ b/image-loader/conf/routes
@@ -1,6 +1,8 @@
GET / controllers.ImageLoaderController.index
POST /images controllers.ImageLoaderController.loadImage(uploadedBy: Option[String], identifiers: Option[String], uploadTime: Option[String], filename: Option[String])
POST /imports controllers.ImageLoaderController.importImage(uri: String, uploadedBy: Option[String], identifiers: Option[String], uploadTime: Option[String], filename: Option[String])
++nocsrf
+POST /images/restore/:id controllers.ImageLoaderController.restoreFromReplica(id: String)
GET /images/project/:imageId controllers.ImageLoaderController.projectImageBy(imageId: String)
# Upload Status
@@ -8,7 +10,6 @@ GET /uploadStatus/:imageId controllers.UploadStatusCo
POST /uploadStatus/:imageId controllers.UploadStatusController.updateUploadStatus(imageId: String)
-
# Management
GET /management/healthcheck com.gu.mediaservice.lib.management.Management.healthCheck
GET /management/manifest com.gu.mediaservice.lib.management.Management.manifest
diff --git a/thrall/app/ThrallComponents.scala b/thrall/app/ThrallComponents.scala
index 65580da4e0..5fdb002da3 100644
--- a/thrall/app/ThrallComponents.scala
+++ b/thrall/app/ThrallComponents.scala
@@ -82,7 +82,7 @@ class ThrallComponents(context: Context) extends GridComponents(context, new Thr
)
val syncCheckerStream: Future[Done] = syncChecker.run()
- val thrallController = new ThrallController(es, migrationSourceWithSender.send, messageSender, actorSystem, auth, config.services, controllerComponents, gridClient)
+ val thrallController = new ThrallController(es, store, migrationSourceWithSender.send, messageSender, actorSystem, auth, config.services, controllerComponents, gridClient)
val healthCheckController = new HealthCheck(es, streamRunning.isCompleted, config, controllerComponents)
val InnerServiceStatusCheckController = new InnerServiceStatusCheckController(auth, controllerComponents, config.services, wsClient)
diff --git a/thrall/app/controllers/ThrallController.scala b/thrall/app/controllers/ThrallController.scala
index 18345d6b7b..0e27a8f5e7 100644
--- a/thrall/app/controllers/ThrallController.scala
+++ b/thrall/app/controllers/ThrallController.scala
@@ -11,7 +11,7 @@ import com.gu.mediaservice.lib.elasticsearch.{NotRunning, Running}
import com.gu.mediaservice.lib.logging.GridLogging
import com.gu.mediaservice.model.{CompleteMigrationMessage, CreateMigrationIndexMessage, UpsertFromProjectionMessage}
import lib.elasticsearch.ElasticSearch
-import lib.{MigrationRequest, OptionalFutureRunner, Paging}
+import lib.{MigrationRequest, OptionalFutureRunner, Paging, ThrallStore}
import org.joda.time.{DateTime, DateTimeZone}
import play.api.data.Form
import play.api.data.Forms._
@@ -26,6 +26,7 @@ case class MigrateSingleImageForm(id: String)
class ThrallController(
es: ElasticSearch,
+ store: ThrallStore,
sendMigrationRequest: MigrationRequest => Future[Boolean],
messageSender: ThrallMessageSender,
actorSystem: ActorSystem,
@@ -67,11 +68,12 @@ class ThrallController(
def upsertProjectPage(imageId: Option[String]) = withLoginRedirectAsync { implicit request =>
imageId match {
- case Some(id) =>
+ case Some(id) if store.doesOriginalExist(id) =>
gridClient.getProjectionDiff(id, auth.innerServiceCall).map {
case None => NotFound("couldn't generate projection for that image!!")
case Some(diff) => Ok(views.html.previewUpsertProject(id, Json.prettyPrint(diff)))
}
+ case Some(_) => Future.successful(Redirect(routes.ThrallController.restoreFromReplica))
case None => Future.successful(Ok(views.html.upsertProject()))
}
}
@@ -242,6 +244,10 @@ class ThrallController(
}}
}
+ def restoreFromReplica: Action[AnyContent] = withLoginRedirect {implicit request =>
+ Ok(views.html.restoreFromReplica(s"${services.loaderBaseUri}/images/restore")) //FIXME figure out imageId bit
+ }
+
def reattemptMigrationFailures(filter: String, page: Int): Action[AnyContent] = withLoginRedirectAsync { implicit request =>
Paging.withPaging(Some(page)) { paging =>
es.migrationStatus match {
diff --git a/thrall/app/views/index.scala.html b/thrall/app/views/index.scala.html
index 98127610a1..9aa370ca33 100644
--- a/thrall/app/views/index.scala.html
+++ b/thrall/app/views/index.scala.html
@@ -45,6 +45,8 @@
Upsert from projection
+ |
+ Restore from replica
Current elasticsearch indices
diff --git a/thrall/app/views/restoreFromReplica.scala.html b/thrall/app/views/restoreFromReplica.scala.html
new file mode 100644
index 0000000000..a512108a8f
--- /dev/null
+++ b/thrall/app/views/restoreFromReplica.scala.html
@@ -0,0 +1,31 @@
+@import helper._
+@(imageLoaderEndpoint:String)(implicit request: RequestHeader)
+
+
+
+
+
+ Restore from replica
+
+
+
+
+
+ < back to Thrall Dashboard
+
+
+
+ Restore from replica
+
+ This option will re-ingest an image from the replica bucket (if configured). It should fail early if the image already exists in the main image bucket.
+
+ Useful if the image has been hard-deleted.
+
+ @form(Call("POST", imageLoaderEndpoint)) {
+ Image ID:
+
+
+ }
+
+
+
diff --git a/thrall/conf/routes b/thrall/conf/routes
index 4a4450ab29..5790b446d1 100644
--- a/thrall/conf/routes
+++ b/thrall/conf/routes
@@ -4,6 +4,7 @@
GET / controllers.ThrallController.index
GET /upsertProject controllers.ThrallController.upsertProjectPage(imageId: Option[String])
+GET /restoreFromReplica controllers.ThrallController.restoreFromReplica
GET /migrationFailuresOverview controllers.ThrallController.migrationFailuresOverview()
GET /migrationFailures controllers.ThrallController.migrationFailures(filter: String, page: Option[Int])
+nocsrf