diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala index 02af178672..419bf41a52 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala @@ -45,6 +45,9 @@ class ImageIngestOperations(imageBucket: String, thumbnailBucket: String, config def deleteOriginal(id: String)(implicit logMarker: LogMarker): Future[Unit] = if(isVersionedS3) deleteVersionedImage(imageBucket, fileKeyFromId(id)) else deleteImage(imageBucket, fileKeyFromId(id)) def deleteThumbnail(id: String)(implicit logMarker: LogMarker): Future[Unit] = deleteImage(thumbnailBucket, fileKeyFromId(id)) def deletePng(id: String)(implicit logMarker: LogMarker): Future[Unit] = deleteImage(imageBucket, optimisedPngKeyFromId(id)) + + def doesOriginalExist(id: String): Boolean = + client.doesObjectExist(imageBucket, fileKeyFromId(id)) } sealed trait ImageWrapper { diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala index 2eff0ccb20..45020a315c 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala @@ -43,4 +43,5 @@ class S3ImageStorage(config: CommonConfig) extends S3(config) with ImageStorage files.foreach(file => client.deleteObject(bucket, file.getKey)) logger.info(logMarker, s"Deleting images in folder $id from bucket $bucket") } + } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index 6b031db1fd..029df565a1 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -242,7 +242,7 @@ object S3Ops { // TODO: Make this region aware - i.e. RegionUtils.getRegion(region).getServiceEndpoint(AmazonS3.ENDPOINT_PREFIX) val s3Endpoint = "s3.amazonaws.com" - def buildS3Client(config: CommonConfig, forceV2Sigs: Boolean = false, localstackAware: Boolean = true): AmazonS3 = { + def buildS3Client(config: CommonConfig, forceV2Sigs: Boolean = false, localstackAware: Boolean = true, maybeRegionOverride: Option[String] = None): AmazonS3 = { val clientConfig = new ClientConfiguration() // Option to disable v4 signatures (https://github.com/aws/aws-sdk-java/issues/372) which is required by imgops @@ -260,6 +260,6 @@ object S3Ops { case _ => AmazonS3ClientBuilder.standard().withClientConfiguration(clientConfig) } - config.withAWSCredentials(builder, localstackAware).build() + config.withAWSCredentials(builder, localstackAware).withRegion(maybeRegionOverride.getOrElse(config.awsRegion)).build() } } diff --git a/image-loader/app/controllers/ImageLoaderController.scala b/image-loader/app/controllers/ImageLoaderController.scala index 7c6615688e..045bbaffda 100644 --- a/image-loader/app/controllers/ImageLoaderController.scala +++ b/image-loader/app/controllers/ImageLoaderController.scala @@ -1,6 +1,9 @@ package controllers -import java.io.File +import com.amazonaws.services.s3.AmazonS3 +import com.amazonaws.util.IOUtils + +import java.io.{File, FileOutputStream} import java.net.URI import com.drew.imaging.ImageProcessingException import com.gu.mediaservice.lib.argo.ArgoHelpers @@ -9,23 +12,26 @@ import com.gu.mediaservice.lib.auth._ import com.gu.mediaservice.lib.formatting.printDateTime import com.gu.mediaservice.lib.logging.{FALLBACK, LogMarker, MarkerMap} import com.gu.mediaservice.lib.{DateTimeUtils, ImageIngestOperations} -import com.gu.mediaservice.model.UnsupportedMimeTypeException +import com.gu.mediaservice.model.{UnsupportedMimeTypeException, UploadInfo} import com.gu.scanamo.error.ConditionNotMet import lib.FailureResponse.Response import lib.{FailureResponse, _} -import lib.imaging.{NoSuchImageExistsInS3, UserImageLoaderException} +import lib.imaging.{MimeTypeDetection, NoSuchImageExistsInS3, UserImageLoaderException} import lib.storage.ImageLoaderStore -import model.{Projector, QuarantineUploader, StatusType, UploadStatus, UploadStatusRecord, Uploader} +import model.{Projector, QuarantineUploader, S3FileExtractedMetadata, StatusType, UploadStatus, UploadStatusRecord, Uploader} import play.api.libs.json.Json import play.api.mvc._ import model.upload.UploadRequest import java.time.Instant import com.gu.mediaservice.GridClient +import com.gu.mediaservice.lib.ImageIngestOperations.fileKeyFromId import com.gu.mediaservice.lib.auth.Authentication.OnBehalfOfPrincipal +import com.gu.mediaservice.lib.aws.S3Ops import com.gu.mediaservice.lib.play.RequestLoggingFilter -import scala.concurrent.{ExecutionContext, Future} +import scala.concurrent.duration.DurationInt +import scala.concurrent.{Await, ExecutionContext, Future} import scala.util.{Failure, Success, Try} import scala.util.control.NonFatal @@ -244,6 +250,61 @@ class ImageLoaderController(auth: Authentication, } } + lazy val replicaS3: AmazonS3 = S3Ops.buildS3Client(config, maybeRegionOverride = Some("us-west-1")) + def restoreFromReplica(imageId: String): Action[AnyContent] = AuthenticatedAndAuthorised.async { request => + + implicit val logMarker: LogMarker = MarkerMap( + "imageId" -> imageId, + "requestType" -> "image-projection", + "requestId" -> RequestLoggingFilter.getRequestId(request) + ) + + config.maybeImageReplicaBucket match { + case _ if store.doesOriginalExist(imageId) => + Future.successful(Conflict("Image already exists in main bucket")) + case None => + Future.successful(NotImplemented("No replica bucket configured")) + case Some(replicaBucket) if replicaS3.doesObjectExist(replicaBucket, fileKeyFromId(imageId)) => + val s3Key = fileKeyFromId(imageId) + + logger.info(logMarker, s"Restoring image $imageId from replica bucket $replicaBucket (key: $s3Key)") + + val replicaObject = replicaS3.getObject(replicaBucket, s3Key) + val metadata = S3FileExtractedMetadata(replicaObject.getObjectMetadata) + val stream = replicaObject.getObjectContent + val tempFile = createTempFile(s"restoringReplica-$imageId") + val fos = new FileOutputStream(tempFile) + try { + IOUtils.copy(stream, fos) + } finally { + stream.close() + } + + val future = uploader.storeFile(UploadRequest( + imageId, + tempFile, //TODO could we give it the stream directly + mimeType = MimeTypeDetection.guessMimeType(tempFile) match { + case Left(unsupported) => throw unsupported + case right => right.toOption + }, + metadata.uploadTime, + metadata.uploadedBy, + metadata.identifiers, + UploadInfo(metadata.uploadFileName) + )) + + future.onComplete(_ => Try { deleteTempFile(tempFile) }) + + future.map {_ => + logger.info(logMarker, s"Restored image $imageId from replica bucket $replicaBucket (key: $s3Key)") + Redirect(s"${config.kahunaUri}/images/$imageId") + } + + case _ => + Future.successful(NotFound("Image not found in replica bucket")) + } + } + // Find this a better home if used more widely implicit class NonEmpty(s: String) { def nonEmptyOpt: Option[String] = if (s.isEmpty) None else Some(s) diff --git a/image-loader/app/lib/ImageLoaderConfig.scala b/image-loader/app/lib/ImageLoaderConfig.scala index bd78bcb6a3..8c85037fe9 100644 --- a/image-loader/app/lib/ImageLoaderConfig.scala +++ b/image-loader/app/lib/ImageLoaderConfig.scala @@ -12,6 +12,8 @@ import scala.concurrent.duration.FiniteDuration class ImageLoaderConfig(resources: GridConfigResources) extends CommonConfig(resources) with StrictLogging { val imageBucket: String = string("s3.image.bucket") + val maybeImageReplicaBucket: Option[String] = stringOpt("s3.image.replicaBucket") + val thumbnailBucket: String = string("s3.thumb.bucket") val quarantineBucket: Option[String] = stringOpt("s3.quarantine.bucket") val uploadToQuarantineEnabled: Boolean = boolean("upload.quarantine.enabled") @@ -23,7 +25,7 @@ class ImageLoaderConfig(resources: GridConfigResources) extends CommonConfig(res val rootUri: String = services.loaderBaseUri val apiUri: String = services.apiBaseUri - val loginUriTemplate: String = services.loginUriTemplate + val kahunaUri: String = services.kahunaBaseUri val transcodedMimeTypes: List[MimeType] = getStringSet("transcoded.mime.types").toList.map(MimeType(_)) val supportedMimeTypes: List[MimeType] = List(Jpeg, Png) ::: transcodedMimeTypes diff --git a/image-loader/conf/routes b/image-loader/conf/routes index 91afab7858..6aafad8e35 100644 --- a/image-loader/conf/routes +++ b/image-loader/conf/routes @@ -1,6 +1,8 @@ GET / controllers.ImageLoaderController.index POST /images controllers.ImageLoaderController.loadImage(uploadedBy: Option[String], identifiers: Option[String], uploadTime: Option[String], filename: Option[String]) POST /imports controllers.ImageLoaderController.importImage(uri: String, uploadedBy: Option[String], identifiers: Option[String], uploadTime: Option[String], filename: Option[String]) ++nocsrf +POST /images/restore/:id controllers.ImageLoaderController.restoreFromReplica(id: String) GET /images/project/:imageId controllers.ImageLoaderController.projectImageBy(imageId: String) # Upload Status @@ -8,7 +10,6 @@ GET /uploadStatus/:imageId controllers.UploadStatusCo POST /uploadStatus/:imageId controllers.UploadStatusController.updateUploadStatus(imageId: String) - # Management GET /management/healthcheck com.gu.mediaservice.lib.management.Management.healthCheck GET /management/manifest com.gu.mediaservice.lib.management.Management.manifest diff --git a/thrall/app/ThrallComponents.scala b/thrall/app/ThrallComponents.scala index 65580da4e0..5fdb002da3 100644 --- a/thrall/app/ThrallComponents.scala +++ b/thrall/app/ThrallComponents.scala @@ -82,7 +82,7 @@ class ThrallComponents(context: Context) extends GridComponents(context, new Thr ) val syncCheckerStream: Future[Done] = syncChecker.run() - val thrallController = new ThrallController(es, migrationSourceWithSender.send, messageSender, actorSystem, auth, config.services, controllerComponents, gridClient) + val thrallController = new ThrallController(es, store, migrationSourceWithSender.send, messageSender, actorSystem, auth, config.services, controllerComponents, gridClient) val healthCheckController = new HealthCheck(es, streamRunning.isCompleted, config, controllerComponents) val InnerServiceStatusCheckController = new InnerServiceStatusCheckController(auth, controllerComponents, config.services, wsClient) diff --git a/thrall/app/controllers/ThrallController.scala b/thrall/app/controllers/ThrallController.scala index 18345d6b7b..0e27a8f5e7 100644 --- a/thrall/app/controllers/ThrallController.scala +++ b/thrall/app/controllers/ThrallController.scala @@ -11,7 +11,7 @@ import com.gu.mediaservice.lib.elasticsearch.{NotRunning, Running} import com.gu.mediaservice.lib.logging.GridLogging import com.gu.mediaservice.model.{CompleteMigrationMessage, CreateMigrationIndexMessage, UpsertFromProjectionMessage} import lib.elasticsearch.ElasticSearch -import lib.{MigrationRequest, OptionalFutureRunner, Paging} +import lib.{MigrationRequest, OptionalFutureRunner, Paging, ThrallStore} import org.joda.time.{DateTime, DateTimeZone} import play.api.data.Form import play.api.data.Forms._ @@ -26,6 +26,7 @@ case class MigrateSingleImageForm(id: String) class ThrallController( es: ElasticSearch, + store: ThrallStore, sendMigrationRequest: MigrationRequest => Future[Boolean], messageSender: ThrallMessageSender, actorSystem: ActorSystem, @@ -67,11 +68,12 @@ class ThrallController( def upsertProjectPage(imageId: Option[String]) = withLoginRedirectAsync { implicit request => imageId match { - case Some(id) => + case Some(id) if store.doesOriginalExist(id) => gridClient.getProjectionDiff(id, auth.innerServiceCall).map { case None => NotFound("couldn't generate projection for that image!!") case Some(diff) => Ok(views.html.previewUpsertProject(id, Json.prettyPrint(diff))) } + case Some(_) => Future.successful(Redirect(routes.ThrallController.restoreFromReplica)) case None => Future.successful(Ok(views.html.upsertProject())) } } @@ -242,6 +244,10 @@ class ThrallController( }} } + def restoreFromReplica: Action[AnyContent] = withLoginRedirect {implicit request => + Ok(views.html.restoreFromReplica(s"${services.loaderBaseUri}/images/restore")) //FIXME figure out imageId bit + } + def reattemptMigrationFailures(filter: String, page: Int): Action[AnyContent] = withLoginRedirectAsync { implicit request => Paging.withPaging(Some(page)) { paging => es.migrationStatus match { diff --git a/thrall/app/views/index.scala.html b/thrall/app/views/index.scala.html index 98127610a1..9aa370ca33 100644 --- a/thrall/app/views/index.scala.html +++ b/thrall/app/views/index.scala.html @@ -45,6 +45,8 @@

Current elasticsearch indices

diff --git a/thrall/app/views/restoreFromReplica.scala.html b/thrall/app/views/restoreFromReplica.scala.html new file mode 100644 index 0000000000..a512108a8f --- /dev/null +++ b/thrall/app/views/restoreFromReplica.scala.html @@ -0,0 +1,31 @@ +@import helper._ +@(imageLoaderEndpoint:String)(implicit request: RequestHeader) + + + + + + Restore from replica + + + + +
+

Restore from replica

+

+ This option will re-ingest an image from the replica bucket (if configured). It should fail early if the image already exists in the main image bucket. +

+

Useful if the image has been hard-deleted.

+ + @form(Call("POST", imageLoaderEndpoint)) { + + + + } +
+ + diff --git a/thrall/conf/routes b/thrall/conf/routes index 4a4450ab29..5790b446d1 100644 --- a/thrall/conf/routes +++ b/thrall/conf/routes @@ -4,6 +4,7 @@ GET / controllers.ThrallController.index GET /upsertProject controllers.ThrallController.upsertProjectPage(imageId: Option[String]) +GET /restoreFromReplica controllers.ThrallController.restoreFromReplica GET /migrationFailuresOverview controllers.ThrallController.migrationFailuresOverview() GET /migrationFailures controllers.ThrallController.migrationFailures(filter: String, page: Option[Int]) +nocsrf