diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala index fd561fe09f..1b070331fd 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala @@ -20,19 +20,43 @@ class ImageIngestOperations(imageBucket: String, thumbnailBucket: String, config import ImageIngestOperations.{fileKeyFromId, optimisedPngKeyFromId} - def storeOriginal(id: String, file: File, mimeType: Option[MimeType], meta: Map[String, String] = Map.empty) - (implicit logMarker: LogMarker): Future[S3Object] = - storeImage(imageBucket, fileKeyFromId(id), file, mimeType, meta) - - def storeThumbnail(id: String, file: File, mimeType: Option[MimeType]) - (implicit logMarker: LogMarker): Future[S3Object] = - storeImage(thumbnailBucket, fileKeyFromId(id), file, mimeType) - - def storeOptimisedPng(id: String, file: File) + def store(storableImage: StorableImage) + (implicit logMarker: LogMarker): Future[S3Object] = storableImage match { + case s:StorableOriginalImage => storeOriginalImage(s) + case s:StorableThumbImage => storeThumbnailImage(s) + case s:StorableOptimisedImage => storeOptimisedImage(s) + } + + private def storeOriginalImage(storableImage: StorableOriginalImage) + (implicit logMarker: LogMarker): Future[S3Object] = + storeImage(imageBucket, fileKeyFromId(storableImage.id), storableImage.file, Some(storableImage.mimeType), storableImage.meta) + + private def storeThumbnailImage(storableImage: StorableThumbImage) + (implicit logMarker: LogMarker): Future[S3Object] = + storeImage(thumbnailBucket, fileKeyFromId(storableImage.id), storableImage.file, Some(storableImage.mimeType)) + + private def storeOptimisedImage(storableImage: StorableOptimisedImage) (implicit logMarker: LogMarker): Future[S3Object] = - storeImage(imageBucket, optimisedPngKeyFromId(id), file, Some(Png)) - + storeImage(imageBucket, optimisedPngKeyFromId(storableImage.id), storableImage.file, Some(storableImage.mimeType)) + def deleteOriginal(id: String): Future[Unit] = if(isVersionedS3) deleteVersionedImage(imageBucket, fileKeyFromId(id)) else deleteImage(imageBucket, fileKeyFromId(id)) def deleteThumbnail(id: String): Future[Unit] = deleteImage(thumbnailBucket, fileKeyFromId(id)) def deletePng(id: String): Future[Unit] = deleteImage(imageBucket, optimisedPngKeyFromId(id)) } + +sealed trait ImageWrapper { + val id: String + val file: File + val mimeType: MimeType + val meta: Map[String, String] +} +sealed trait StorableImage extends ImageWrapper + +case class StorableThumbImage(id: String, file: File, mimeType: MimeType, meta: Map[String, String] = Map.empty) extends StorableImage +case class StorableOriginalImage(id: String, file: File, mimeType: MimeType, meta: Map[String, String] = Map.empty) extends StorableImage +case class StorableOptimisedImage(id: String, file: File, mimeType: MimeType, meta: Map[String, String] = Map.empty) extends StorableImage +case class BrowserViewableImage(id: String, file: File, mimeType: MimeType, meta: Map[String, String] = Map.empty, mustUpload: Boolean = false) extends ImageWrapper { + def asStorableOptimisedImage = StorableOptimisedImage(id, file, mimeType, meta) + def asStorableThumbImage = StorableThumbImage(id, file, mimeType, meta) +} + diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala index a8fbfd9cf4..0c7ed5d767 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala @@ -2,7 +2,7 @@ package com.gu.mediaservice.lib import java.io.File -import com.gu.mediaservice.lib.aws.S3 +import com.gu.mediaservice.lib.aws.{S3, S3Ops} import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.lib.logging.LogMarker import com.gu.mediaservice.model.MimeType @@ -15,9 +15,15 @@ import scala.concurrent.Future class S3ImageStorage(config: CommonConfig) extends S3(config) with ImageStorage { private val log = LoggerFactory.getLogger(getClass) + private val cacheSetting = Some(cacheForever) def storeImage(bucket: String, id: String, file: File, mimeType: Option[MimeType], meta: Map[String, String] = Map.empty) - (implicit logMarker: LogMarker) = - store(bucket, id, file, mimeType, meta, Some(cacheForever)) + (implicit logMarker: LogMarker) = { + store(bucket, id, file, mimeType, meta, cacheSetting) + .map( _ => + // TODO this is just giving back the stuff we passed in and should be factored out. + S3Ops.projectFileAsS3Object(bucket, id, file, mimeType, meta, cacheSetting) + ) + } def deleteImage(bucket: String, id: String) = Future { client.deleteObject(bucket, id) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index d16509c6c9..7320fe5f37 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -116,7 +116,7 @@ class S3(config: CommonConfig) extends GridLogging { } def store(bucket: Bucket, id: Key, file: File, mimeType: Option[MimeType], meta: UserMetadata = Map.empty, cacheControl: Option[String] = None) - (implicit ex: ExecutionContext, logMarker: LogMarker): Future[S3Object] = + (implicit ex: ExecutionContext, logMarker: LogMarker): Future[Unit] = Future { val metadata = new ObjectMetadata mimeType.foreach(m => metadata.setContentType(m.name)) @@ -127,8 +127,6 @@ class S3(config: CommonConfig) extends GridLogging { Stopwatch(s"S3 client.putObject ($req)"){ client.putObject(req) } - - S3Ops.projectFileAsS3Object(bucket, id, file, mimeType, meta, cacheControl) } def list(bucket: Bucket, prefixDir: String) @@ -199,9 +197,9 @@ object S3Ops { new URI("http", bucketUrl, s"/$key", null) } - def projectFileAsS3Object(bucket: String, key: String, file: File, mimeType: Option[MimeType], meta: Map[String, String] = Map.empty, cacheControl: Option[String] = None): S3Object = { + def projectFileAsS3Object(url: URI, file: File, mimeType: Option[MimeType], meta: Map[String, String], cacheControl: Option[String]): S3Object = { S3Object( - objectUrl(bucket, key), + url, file.length, S3Metadata( meta, @@ -212,4 +210,8 @@ object S3Ops { ) ) } + + def projectFileAsS3Object(bucket: String, key: String, file: File, mimeType: Option[MimeType], meta: Map[String, String] = Map.empty, cacheControl: Option[String] = None): S3Object = { + projectFileAsS3Object(objectUrl(bucket, key), file, mimeType, meta, cacheControl) + } } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/imaging/ImageOperations.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/imaging/ImageOperations.scala index bdbf64741e..c8f12e0580 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/imaging/ImageOperations.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/imaging/ImageOperations.scala @@ -4,6 +4,8 @@ import java.io._ import org.im4java.core.IMOperation import com.gu.mediaservice.lib.Files._ +import com.gu.mediaservice.lib.StorableThumbImage +import com.gu.mediaservice.lib.imaging.ImageOperations.{optimisedMimeType, thumbMimeType} import com.gu.mediaservice.lib.imaging.im4jwrapper.ImageMagick.{addImage, format, runIdentifyCmd} import com.gu.mediaservice.lib.imaging.im4jwrapper.{ExifTool, ImageMagick} import com.gu.mediaservice.lib.logging.GridLogging @@ -57,7 +59,7 @@ class ImageOperations(playPath: String) extends GridLogging { def cropImage(sourceFile: File, sourceMimeType: Option[MimeType], bounds: Bounds, qual: Double = 100d, tempDir: File, iccColourSpace: Option[String], colourModel: Option[String], fileType: MimeType): Future[File] = { for { - outputFile <- createTempFile(s"crop-", s".${fileType.fileExtension}", tempDir) + outputFile <- createTempFile(s"crop-", s"${fileType.fileExtension}", tempDir) cropSource = addImage(sourceFile) qualified = quality(cropSource)(qual) corrected = correctColour(qualified)(iccColourSpace, colourModel) @@ -68,6 +70,7 @@ class ImageOperations(playPath: String) extends GridLogging { depthAdjusted = depth(cropped)(8) addOutput = addDestImage(depthAdjusted)(outputFile) _ <- runConvertCmd(addOutput, useImageMagick = sourceMimeType.contains(Tiff)) + _ <- checkForOutputFileChange(outputFile) } yield outputFile } @@ -114,31 +117,60 @@ class ImageOperations(playPath: String) extends GridLogging { val thumbUnsharpRadius = 0.5d val thumbUnsharpSigma = 0.5d val thumbUnsharpAmount = 0.8d - def createThumbnail(sourceFile: File, sourceMimeType: Option[MimeType], width: Int, qual: Double = 100d, - tempDir: File, iccColourSpace: Option[String], colourModel: Option[String]): Future[File] = { + + /** + * Given a source file containing a png (the 'browser viewable' file), + * construct a thumbnail file in the provided temp directory, and return + * the file with metadata about it. + * @param sourceFile File containing browser viewable (ie not too big or colourful) image + * @param sourceMimeType Mime time of browser viewable file + * @param width Desired with of thumbnail + * @param qual Desired quality of thumbnail + * @param tempDir Location to create thumbnail file + * @param iccColourSpace (Approximately) number of colours to use + * @param colourModel Colour model - eg RGB or CMYK + * @return The file created and the mimetype of the content of that file, in a future. + */ + def createThumbnail(sourceFile: File, + sourceMimeType: Option[MimeType], + width: Int, + qual: Double = 100d, + tempDir: File, + iccColourSpace: Option[String], + colourModel: Option[String]): Future[(File, MimeType)] = { + val cropSource = addImage(sourceFile) + val thumbnailed = thumbnail(cropSource)(width) + val corrected = correctColour(thumbnailed)(iccColourSpace, colourModel) + val converted = applyOutputProfile(corrected, optimised = true) + val stripped = stripMeta(converted) + val profiled = applyOutputProfile(stripped, optimised = true) + val unsharpened = unsharp(profiled)(thumbUnsharpRadius, thumbUnsharpSigma, thumbUnsharpAmount) + val qualified = quality(unsharpened)(qual) + val addOutput = {file:File => addDestImage(qualified)(file)} for { - outputFile <- createTempFile(s"thumb-", ".jpg", tempDir) - cropSource = addImage(sourceFile) - thumbnailed = thumbnail(cropSource)(width) - corrected = correctColour(thumbnailed)(iccColourSpace, colourModel) - converted = applyOutputProfile(corrected, optimised = true) - stripped = stripMeta(converted) - profiled = applyOutputProfile(stripped, optimised = true) - unsharpened = unsharp(profiled)(thumbUnsharpRadius, thumbUnsharpSigma, thumbUnsharpAmount) - qualified = quality(unsharpened)(qual) - addOutput = addDestImage(qualified)(outputFile) - _ <- runConvertCmd(addOutput, useImageMagick = sourceMimeType.contains(Tiff)) - } yield outputFile + outputFile <- createTempFile(s"thumb-", thumbMimeType.fileExtension, tempDir) + _ <- runConvertCmd(addOutput(outputFile), useImageMagick = sourceMimeType.contains(Tiff)) + } yield (outputFile, thumbMimeType) } - def transformImage(sourceFile: File, sourceMimeType: Option[MimeType], tempDir: File): Future[File] = { + /** + * Given a source file containing a file which requires optimising to make it suitable for viewing in + * a browser, construct a new image file in the provided temp directory, and return + * * the file with metadata about it. + * @param sourceFile File containing browser viewable (ie not too big or colourful) image + * @param sourceMimeType Mime time of browser viewable file + * @param tempDir Location to create optimised file + * @return The file created and the mimetype of the content of that file, in a future. + */ + def transformImage(sourceFile: File, sourceMimeType: Option[MimeType], tempDir: File): Future[(File, MimeType)] = { for { - outputFile <- createTempFile(s"transformed-", ".png", tempDir) + // png suffix is used by imagemagick to infer the required type + outputFile <- createTempFile(s"transformed-", optimisedMimeType.fileExtension, tempDir) transformSource = addImage(sourceFile) addOutput = addDestImage(transformSource)(outputFile) _ <- runConvertCmd(addOutput, useImageMagick = sourceMimeType.contains(Tiff)) _ <- checkForOutputFileChange(outputFile) - } yield outputFile + } yield (outputFile, optimisedMimeType) } // When a layered tiff is unpacked, the temp file (blah.something) is moved @@ -175,6 +207,8 @@ class ImageOperations(playPath: String) extends GridLogging { } object ImageOperations { + val thumbMimeType = Jpeg + val optimisedMimeType = Png def identifyColourModel(sourceFile: File, mimeType: MimeType)(implicit ec: ExecutionContext): Future[Option[String]] = { // TODO: use mimeType to lookup other properties once we support other formats diff --git a/docs/06-objects-of-interest/06.01-tiffs.md b/docs/06-objects-of-interest/06.01-tiffs.md new file mode 100644 index 0000000000..7bb4d48fb8 --- /dev/null +++ b/docs/06-objects-of-interest/06.01-tiffs.md @@ -0,0 +1,19 @@ +# TIFF files + +Tiff files have special handling for importing. + +Like all files, we upload the original, but create an (optional) optimised +PNG, and (mandatory) thumbnail JPEG version for use in the UI. + +For TIFF files, these pngs derive from the first image file extracted by +ImageMagick. The `convert` function will extract a file of the +specified type (PNG in our case), which, in a file named `xxx.tif` will be +`xxx.png`. + +However, there is additional complexity with tiff files which contain layered +content. Specifically, the layer files will explode as `xxx-n.jpg` or +`xxx-n.png` where `n` is a numeric index. + +Our code looks for this special case (see `ImageOperations.checkForOutputFileChange`) +and simply moves the `-n` layer file to the expected location. + diff --git a/image-loader/app/ImageLoaderComponents.scala b/image-loader/app/ImageLoaderComponents.scala index c896b4af8e..d5d03b7949 100644 --- a/image-loader/app/ImageLoaderComponents.scala +++ b/image-loader/app/ImageLoaderComponents.scala @@ -3,7 +3,7 @@ import com.gu.mediaservice.lib.play.GridComponents import controllers.ImageLoaderController import lib._ import lib.storage.ImageLoaderStore -import model.{Uploader, Projector} +import model.{Projector, Uploader} import play.api.ApplicationLoader.Context import router.Routes diff --git a/image-loader/app/lib/imaging/FileMetadataReader.scala b/image-loader/app/lib/imaging/FileMetadataReader.scala index 59978fd4a0..9bc49be092 100644 --- a/image-loader/app/lib/imaging/FileMetadataReader.scala +++ b/image-loader/app/lib/imaging/FileMetadataReader.scala @@ -12,9 +12,11 @@ import com.drew.metadata.jpeg.JpegDirectory import com.drew.metadata.png.PngDirectory import com.drew.metadata.xmp.XmpDirectory import com.drew.metadata.{Directory, Metadata} +import com.gu.mediaservice.lib.{ImageWrapper, StorableImage} import com.gu.mediaservice.lib.imaging.im4jwrapper.ImageMagick._ import com.gu.mediaservice.lib.metadata.ImageMetadataConverter import com.gu.mediaservice.model._ +import model.upload.UploadRequest import org.joda.time.{DateTime, DateTimeZone} import org.joda.time.format.ISODateTimeFormat import play.api.libs.json.JsValue @@ -54,16 +56,19 @@ object FileMetadataReader { for { metadata <- readMetadata(image) } - yield getMetadataWithICPTCHeaders(metadata, imageId) // FIXME: JPEG, JFIF, Photoshop, GPS, File + yield getMetadataWithIPTCHeaders(metadata, imageId) // FIXME: JPEG, JFIF, Photoshop, GPS, File - def fromICPTCHeadersWithColorInfo(image: File, imageId:String, mimeType: MimeType): Future[FileMetadata] = + def fromIPTCHeadersWithColorInfo(image: ImageWrapper): Future[FileMetadata] = + fromIPTCHeadersWithColorInfo(image.file, image.id, image.mimeType) + + def fromIPTCHeadersWithColorInfo(image: File, imageId:String, mimeType: MimeType): Future[FileMetadata] = for { metadata <- readMetadata(image) colourModelInformation <- getColorModelInformation(image, metadata, mimeType) } - yield getMetadataWithICPTCHeaders(metadata, imageId).copy(colourModelInformation = colourModelInformation) + yield getMetadataWithIPTCHeaders(metadata, imageId).copy(colourModelInformation = colourModelInformation) - private def getMetadataWithICPTCHeaders(metadata: Metadata, imageId:String): FileMetadata = + private def getMetadataWithIPTCHeaders(metadata: Metadata, imageId:String): FileMetadata = FileMetadata( iptc = exportDirectory(metadata, classOf[IptcDirectory]), exif = exportDirectory(metadata, classOf[ExifIFD0Directory]), @@ -217,12 +222,12 @@ object FileMetadataReader { "paletteSize" -> Option(metaDir.getDescription(PngDirectory.TAG_PALETTE_SIZE)), "iccProfileName" -> Option(metaDir.getDescription(PngDirectory.TAG_ICC_PROFILE_NAME)) ).flattenOptions - case _ => val metaDir = metadata.getFirstDirectoryOfType(classOf[ExifIFD0Directory]) + case _ => val metaDir = Option(metadata.getFirstDirectoryOfType(classOf[ExifIFD0Directory])) Map( "hasAlpha" -> hasAlpha, "colorType" -> maybeImageType, - "photometricInterpretation" -> Option(metaDir.getDescription(ExifDirectoryBase.TAG_PHOTOMETRIC_INTERPRETATION)), - "bitsPerSample" -> Option(metaDir.getDescription(ExifDirectoryBase.TAG_BITS_PER_SAMPLE)) + "photometricInterpretation" -> metaDir.map(_.getDescription(ExifDirectoryBase.TAG_PHOTOMETRIC_INTERPRETATION)), + "bitsPerSample" -> metaDir.map(_.getDescription(ExifDirectoryBase.TAG_BITS_PER_SAMPLE)) ).flattenOptions } diff --git a/image-loader/app/model/ImageUpload.scala b/image-loader/app/model/ImageUpload.scala deleted file mode 100644 index cf4d0ccb8b..0000000000 --- a/image-loader/app/model/ImageUpload.scala +++ /dev/null @@ -1,420 +0,0 @@ -package model - -import java.io.File -import java.net.URLEncoder -import java.nio.charset.StandardCharsets -import java.util.UUID - -import com.gu.mediaservice.lib.argo.ArgoHelpers -import com.gu.mediaservice.lib.auth.Authentication -import com.gu.mediaservice.lib.auth.Authentication.Principal -import com.gu.mediaservice.lib.aws.{S3Object, UpdateMessage} -import com.gu.mediaservice.lib.cleanup.{MetadataCleaners, SupplierProcessors} -import com.gu.mediaservice.lib.config.MetadataConfig -import com.gu.mediaservice.lib.formatting._ -import com.gu.mediaservice.lib.imaging.ImageOperations -import com.gu.mediaservice.lib.logging._ -import com.gu.mediaservice.lib.metadata.{FileMetadataHelper, ImageMetadataConverter} -import com.gu.mediaservice.lib.resource.FutureResources._ -import com.gu.mediaservice.model._ -import lib.{DigestedFile, ImageLoaderConfig, Notifications} -import lib.imaging.{FileMetadataReader, MimeTypeDetection} -import lib.storage.ImageLoaderStore -import org.joda.time.DateTime -import play.api.libs.json.{JsObject, Json} - -import scala.concurrent.{ExecutionContext, Future} -import scala.sys.process._ - -case class OptimisedPng(optimisedFileStoreFuture: Future[Option[S3Object]], isPng24: Boolean, - optimisedTempFile: Option[File]) - -case object OptimisedPng { - - def shouldOptimise(mimeType: Option[MimeType], fileMetadata: FileMetadata): Boolean = - mimeType match { - case Some(Png) => - fileMetadata.colourModelInformation.get("colorType") match { - case Some("True Color") => true - case Some("True Color with Alpha") => true - case _ => false - } - case Some(Tiff) => true - case _ => false - } -} - -object OptimisedPngOps { - - def build(file: File, - uploadRequest: UploadRequest, - fileMetadata: FileMetadata, - config: ImageUploadOpsCfg, - storeOrProject: (UploadRequest, File) => Future[S3Object]) - (implicit ec: ExecutionContext, logMarker: LogMarker): OptimisedPng = { - - val result = if (!OptimisedPng.shouldOptimise(uploadRequest.mimeType, fileMetadata)) { - OptimisedPng(Future(None), isPng24 = false, None) - } else { - val optimisedFile: File = toOptimisedFile(file, uploadRequest, config) - val pngStoreFuture: Future[Option[S3Object]] = Some(storeOrProject(uploadRequest, optimisedFile)) - .map(result => result.map(Option(_))) - .getOrElse(Future.successful(None)) - if (isTransformedFilePath(file.getAbsolutePath)) - file.delete - OptimisedPng(pngStoreFuture, isPng24 = true, Some(optimisedFile)) - } - result - } - - private def toOptimisedFile(file: File, uploadRequest: UploadRequest, config: ImageUploadOpsCfg) - (implicit logMarker: LogMarker): File = { - val optimisedFilePath = config.tempDir.getAbsolutePath + "/optimisedpng - " + uploadRequest.imageId + ".png" - Stopwatch("pngquant") { - Seq("pngquant", "--quality", "1-85", file.getAbsolutePath, "--output", optimisedFilePath).! - } - new File(optimisedFilePath) - } - - private def isTransformedFilePath(filePath: String): Boolean = filePath.contains("transformed-") - -} - -case class ImageUpload(uploadRequest: UploadRequest, image: Image) - -case object ImageUpload { - val metadataCleaners = new MetadataCleaners(MetadataConfig.allPhotographersMap) - - def createImage(uploadRequest: UploadRequest, source: Asset, thumbnail: Asset, png: Option[Asset], - fileMetadata: FileMetadata, metadata: ImageMetadata): Image = { - val usageRights = NoRights - Image( - uploadRequest.imageId, - uploadRequest.uploadTime, - uploadRequest.uploadedBy, - Some(uploadRequest.uploadTime), - uploadRequest.identifiers, - uploadRequest.uploadInfo, - source, - Some(thumbnail), - png, - fileMetadata, - None, - metadata, - metadata, - usageRights, - usageRights, - List(), - List() - ) - } -} - -class Uploader(val store: ImageLoaderStore, - val config: ImageLoaderConfig, - val imageOps: ImageOperations, - val notifications: Notifications) - (implicit val ec: ExecutionContext) extends ArgoHelpers { - - - import Uploader.{fromUploadRequestShared, toMetaMap, toImageUploadOpsCfg} - - - def fromUploadRequest(uploadRequest: UploadRequest) - (implicit logMarker: LogMarker): Future[ImageUpload] = { - val sideEffectDependencies = ImageUploadOpsDependencies(toImageUploadOpsCfg(config), imageOps, - storeSource, storeThumbnail, storeOptimisedPng) - val finalImage = fromUploadRequestShared(uploadRequest, sideEffectDependencies) - finalImage.map(img => Stopwatch("finalImage"){ImageUpload(uploadRequest, img)}) - } - - private def storeSource(uploadRequest: UploadRequest) - (implicit logMarker: LogMarker) = { - val meta = toMetaMap(uploadRequest) - store.storeOriginal( - uploadRequest.imageId, - uploadRequest.tempFile, - uploadRequest.mimeType, - meta - ) - } - - private def storeThumbnail(uploadRequest: UploadRequest, thumbFile: File) - (implicit logMarker: LogMarker) = store.storeThumbnail( - uploadRequest.imageId, - thumbFile, - Some(Jpeg) - ) - - private def storeOptimisedPng(uploadRequest: UploadRequest, optimisedPngFile: File) - (implicit logMarker: LogMarker) = { - store.storeOptimisedPng( - uploadRequest.imageId, - optimisedPngFile - ) - } - - def loadFile(digestedFile: DigestedFile, - user: Principal, - uploadedBy: Option[String], - identifiers: Option[String], - uploadTime: DateTime, - filename: Option[String], - requestId: UUID) - (implicit ec:ExecutionContext, - logMarker: LogMarker): Future[UploadRequest] = Future { - val DigestedFile(tempFile, id) = digestedFile - - // TODO: should error if the JSON parsing failed - val identifiersMap = identifiers.map(Json.parse(_).as[Map[String, String]]) getOrElse Map() - - MimeTypeDetection.guessMimeType(tempFile) match { - case util.Left(unsupported) => - logger.error(s"Unsupported mimetype", unsupported) - throw unsupported - case util.Right(mimeType) => - logger.info(s"Detected mimetype as $mimeType") - UploadRequest( - requestId = requestId, - imageId = id, - tempFile = tempFile, - mimeType = Some(mimeType), - uploadTime = uploadTime, - uploadedBy = uploadedBy.getOrElse(Authentication.getIdentity(user)), - identifiers = identifiersMap, - uploadInfo = UploadInfo(filename) - ) - } - } - - def storeFile(uploadRequest: UploadRequest) - (implicit ec:ExecutionContext, - logMarker: LogMarker): Future[JsObject] = { - - logger.info("Storing file") - - for { - imageUpload <- fromUploadRequest(uploadRequest) - updateMessage = UpdateMessage(subject = "image", image = Some(imageUpload.image)) - _ <- Future { notifications.publish(updateMessage) } - // TODO: centralise where all these URLs are constructed - uri = s"${config.apiUri}/images/${uploadRequest.imageId}" - } yield { - Json.obj("uri" -> uri) - } - - } - -} - -case class ImageUploadOpsCfg( - tempDir: File, - thumbWidth: Int, - thumbQuality: Double, - transcodedMimeTypes: List[MimeType], - originalFileBucket: String, - thumbBucket: String -) - -case class ImageUploadOpsDependencies( - config: ImageUploadOpsCfg, - imageOps: ImageOperations, - storeOrProjectOriginalFile: UploadRequest => Future[S3Object], - storeOrProjectThumbFile: (UploadRequest, File) => Future[S3Object], - storeOrProjectOptimisedPNG: (UploadRequest, File) => Future[S3Object] -) - -object Uploader extends GridLogging { - - def toImageUploadOpsCfg(config: ImageLoaderConfig): ImageUploadOpsCfg = { - ImageUploadOpsCfg( - config.tempDir, - config.thumbWidth, - config.thumbQuality, - config.transcodedMimeTypes, - config.imageBucket, - config.thumbnailBucket - ) - } - - def fromUploadRequestShared(uploadRequest: UploadRequest, deps: ImageUploadOpsDependencies) - (implicit ec: ExecutionContext, logMarker: LogMarker): Future[Image] = { - - import deps._ - - logger.info("Starting image ops") - val uploadedFile = uploadRequest.tempFile - - val fileMetadataFuture = toFileMetadata(uploadedFile, uploadRequest.imageId, uploadRequest.mimeType) - - logger.info("Have read file headers") - - fileMetadataFuture.flatMap(fileMetadata => { - uploadAndStoreImage(config, - storeOrProjectOriginalFile, - storeOrProjectThumbFile, - storeOrProjectOptimisedPNG, - uploadRequest, - deps, - uploadedFile, - fileMetadataFuture, - fileMetadata)(ec, addLogMarkers(fileMetadata.toLogMarker)) - }) - } - - private def uploadAndStoreImage(config: ImageUploadOpsCfg, - storeOrProjectOriginalFile: UploadRequest => Future[S3Object], - storeOrProjectThumbFile: (UploadRequest, File) => Future[S3Object], - storeOrProjectOptimisedPNG: (UploadRequest, File) => Future[S3Object], - uploadRequest: UploadRequest, - deps: ImageUploadOpsDependencies, - uploadedFile: File, - fileMetadataFuture: Future[FileMetadata], - fileMetadata: FileMetadata) - (implicit ec: ExecutionContext, logMarker: LogMarker) = { - logger.info("Have read file metadata") - logger.info("stored source file") - // FIXME: pass mimeType - val colourModelFuture = ImageOperations.identifyColourModel(uploadedFile, Jpeg) - val sourceDimensionsFuture = FileMetadataReader.dimensions(uploadedFile, uploadRequest.mimeType) - - // if the file needs pre-processing into a supported type of file, do it now and create the new upload request. - createOptimisedFileFuture(uploadRequest, deps).flatMap(uploadRequest => { - val sourceStoreFuture = storeOrProjectOriginalFile(uploadRequest) - val toOptimiseFile = uploadRequest.tempFile - val thumbFuture = createThumbFuture(fileMetadataFuture, colourModelFuture, uploadRequest, deps) - logger.info("thumbnail created") - - val optimisedPng = OptimisedPngOps.build( - toOptimiseFile, - uploadRequest, - fileMetadata, - config, - storeOrProjectOptimisedPNG)(ec, logMarker) - logger.info(s"optimised image ($toOptimiseFile) created") - - bracket(thumbFuture)(_.delete) { thumb => - // Run the operations in parallel - val thumbStoreFuture = storeOrProjectThumbFile(uploadRequest, thumb) - val thumbDimensionsFuture = FileMetadataReader.dimensions(thumb, Some(Jpeg)) - - val finalImage = toFinalImage( - sourceStoreFuture, - thumbStoreFuture, - sourceDimensionsFuture, - thumbDimensionsFuture, - fileMetadataFuture, - colourModelFuture, - optimisedPng, - uploadRequest - ) - logger.info(s"Deleting temp file ${uploadedFile.getAbsolutePath}") - uploadedFile.delete() - toOptimiseFile.delete() - - finalImage - } - }) - } - - def toMetaMap(uploadRequest: UploadRequest): Map[String, String] = { - val baseMeta = Map( - "uploaded_by" -> uploadRequest.uploadedBy, - "upload_time" -> printDateTime(uploadRequest.uploadTime) - ) ++ uploadRequest.identifiersMeta - - uploadRequest.uploadInfo.filename match { - case Some(f) => baseMeta ++ Map("file_name" -> URLEncoder.encode(f, StandardCharsets.UTF_8.name())) - case _ => baseMeta - } - } - - private def toFinalImage(sourceStoreFuture: Future[S3Object], - thumbStoreFuture: Future[S3Object], - sourceDimensionsFuture: Future[Option[Dimensions]], - thumbDimensionsFuture: Future[Option[Dimensions]], - fileMetadataFuture: Future[FileMetadata], - colourModelFuture: Future[Option[String]], - optimisedPng: OptimisedPng, - uploadRequest: UploadRequest) - (implicit ec: ExecutionContext, logMarker: LogMarker): Future[Image] = { - logger.info("Starting image ops") - for { - s3Source <- sourceStoreFuture - s3Thumb <- thumbStoreFuture - s3PngOption <- optimisedPng.optimisedFileStoreFuture - sourceDimensions <- sourceDimensionsFuture - thumbDimensions <- thumbDimensionsFuture - fileMetadata <- fileMetadataFuture - colourModel <- colourModelFuture - fullFileMetadata = fileMetadata.copy(colourModel = colourModel) - metadata = ImageMetadataConverter.fromFileMetadata(fullFileMetadata) - cleanMetadata = ImageUpload.metadataCleaners.clean(metadata) - - sourceAsset = Asset.fromS3Object(s3Source, sourceDimensions) - thumbAsset = Asset.fromS3Object(s3Thumb, thumbDimensions) - - pngAsset = if (optimisedPng.isPng24) - Some(Asset.fromS3Object(s3PngOption.get, sourceDimensions)) - else - None - - baseImage = ImageUpload.createImage(uploadRequest, sourceAsset, thumbAsset, pngAsset, fullFileMetadata, cleanMetadata) - - processedImage = SupplierProcessors.process(baseImage) - - // FIXME: dirty hack to sync the originalUsageRights and originalMetadata as well - finalImage = processedImage.copy( - originalMetadata = processedImage.metadata, - originalUsageRights = processedImage.usageRights - ) - } yield { - if (optimisedPng.isPng24) optimisedPng.optimisedTempFile.get.delete - logger.info("Ending image ops") - finalImage - } - } - - private def toFileMetadata(f: File, imageId: String, mimeType: Option[MimeType]): Future[FileMetadata] = { - mimeType match { - case Some(Png | Tiff) => FileMetadataReader.fromICPTCHeadersWithColorInfo(f, imageId, mimeType.get) - case _ => FileMetadataReader.fromIPTCHeaders(f, imageId) - } - } - - private def createThumbFuture(fileMetadataFuture: Future[FileMetadata], - colourModelFuture: Future[Option[String]], - uploadRequest: UploadRequest, - deps: ImageUploadOpsDependencies)(implicit ec: ExecutionContext) = { - import deps._ - for { - fileMetadata <- fileMetadataFuture - colourModel <- colourModelFuture - iccColourSpace = FileMetadataHelper.normalisedIccColourSpace(fileMetadata) - thumb <- imageOps - .createThumbnail(uploadRequest.tempFile, uploadRequest.mimeType, config.thumbWidth, - config.thumbQuality, config.tempDir, iccColourSpace, colourModel) - } yield thumb - } - - private def createOptimisedFileFuture(uploadRequest: UploadRequest, - deps: ImageUploadOpsDependencies)(implicit ec: ExecutionContext): Future[UploadRequest] = { - import deps._ - uploadRequest.mimeType match { - case Some(mime) if config.transcodedMimeTypes.contains(mime) => - for { - transformedImage <- imageOps.transformImage(uploadRequest.tempFile, uploadRequest.mimeType, config.tempDir) - } yield uploadRequest - // This file has been converted. - .copy(mimeType = Some(Jpeg)) - .copy(tempFile = transformedImage) - case _ => - Future.successful(uploadRequest) - } - } - - -} - - - diff --git a/image-loader/app/model/Projector.scala b/image-loader/app/model/Projector.scala index 35fc655a6d..55f7c6c8ab 100644 --- a/image-loader/app/model/Projector.scala +++ b/image-loader/app/model/Projector.scala @@ -5,7 +5,7 @@ import java.util.UUID import com.amazonaws.services.s3.AmazonS3 import com.amazonaws.services.s3.model.{ObjectMetadata, S3Object} -import com.gu.mediaservice.lib.ImageIngestOperations +import com.gu.mediaservice.lib.{ImageIngestOperations, StorableOptimisedImage, StorableOriginalImage, StorableThumbImage} import com.gu.mediaservice.lib.aws.S3Ops import com.gu.mediaservice.lib.imaging.ImageOperations import com.gu.mediaservice.lib.logging.LogMarker @@ -13,6 +13,7 @@ import com.gu.mediaservice.lib.net.URI import com.gu.mediaservice.model.{Image, Jpeg, Png, UploadInfo} import lib.imaging.{MimeTypeDetection, NoSuchImageExistsInS3} import lib.{DigestedFile, ImageLoaderConfig} +import model.upload.{OptimiseWithPngQuant, UploadRequest} import org.apache.tika.io.IOUtils import org.joda.time.{DateTime, DateTimeZone} import play.api.{Logger, MarkerContext} @@ -133,37 +134,36 @@ class ImageUploadProjectionOps(config: ImageUploadOpsCfg, fromUploadRequestShared(uploadRequest, dependenciesWithProjectionsOnly) } - private def projectOriginalFileAsS3Model(uploadRequest: UploadRequest) + private def projectOriginalFileAsS3Model(storableOriginalImage: StorableOriginalImage) (implicit ec: ExecutionContext)= Future { - val meta: Map[String, String] = toMetaMap(uploadRequest) - val key = ImageIngestOperations.fileKeyFromId(uploadRequest.imageId) + val key = ImageIngestOperations.fileKeyFromId(storableOriginalImage.id) S3Ops.projectFileAsS3Object( config.originalFileBucket, key, - uploadRequest.tempFile, - uploadRequest.mimeType, - meta + storableOriginalImage.file, + Some(storableOriginalImage.mimeType), + storableOriginalImage.meta ) } - private def projectThumbnailFileAsS3Model(uploadRequest: UploadRequest, thumbFile: File)(implicit ec: ExecutionContext) = Future { - val key = ImageIngestOperations.fileKeyFromId(uploadRequest.imageId) - val thumbMimeType = Some(Jpeg) + private def projectThumbnailFileAsS3Model(storableThumbImage: StorableThumbImage)(implicit ec: ExecutionContext) = Future { + val key = ImageIngestOperations.fileKeyFromId(storableThumbImage.id) + val thumbMimeType = Some(OptimiseWithPngQuant.optimiseMimeType) // this IS what we will generate. S3Ops.projectFileAsS3Object( config.thumbBucket, key, - thumbFile, + storableThumbImage.file, thumbMimeType ) } - private def projectOptimisedPNGFileAsS3Model(uploadRequest: UploadRequest, optimisedPngFile: File)(implicit ec: ExecutionContext) = Future { - val key = ImageIngestOperations.optimisedPngKeyFromId(uploadRequest.imageId) - val optimisedPngMimeType = Some(Png) + private def projectOptimisedPNGFileAsS3Model(storableOptimisedImage: StorableOptimisedImage)(implicit ec: ExecutionContext) = Future { + val key = ImageIngestOperations.optimisedPngKeyFromId(storableOptimisedImage.id) + val optimisedPngMimeType = Some(ImageOperations.thumbMimeType) // this IS what we will generate. S3Ops.projectFileAsS3Object( config.originalFileBucket, key, - optimisedPngFile, + storableOptimisedImage.file, optimisedPngMimeType ) } diff --git a/image-loader/app/model/Uploader.scala b/image-loader/app/model/Uploader.scala new file mode 100644 index 0000000000..8f86e7cc04 --- /dev/null +++ b/image-loader/app/model/Uploader.scala @@ -0,0 +1,340 @@ +package model + +import java.io.File +import java.net.URLEncoder +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.util.UUID + +import com.gu.mediaservice.lib.argo.ArgoHelpers +import com.gu.mediaservice.lib.auth.Authentication +import com.gu.mediaservice.lib.auth.Authentication.Principal +import com.gu.mediaservice.lib.{BrowserViewableImage, StorableOptimisedImage, StorableOriginalImage, StorableThumbImage} +import com.gu.mediaservice.lib.aws.{S3Object, UpdateMessage} +import com.gu.mediaservice.lib.cleanup.{MetadataCleaners, SupplierProcessors} +import com.gu.mediaservice.lib.config.MetadataConfig +import com.gu.mediaservice.lib.formatting._ +import com.gu.mediaservice.lib.imaging.ImageOperations +import com.gu.mediaservice.lib.logging._ +import com.gu.mediaservice.lib.metadata.{FileMetadataHelper, ImageMetadataConverter} +import com.gu.mediaservice.model._ +import lib.{DigestedFile, ImageLoaderConfig, Notifications} +import lib.imaging.{FileMetadataReader, MimeTypeDetection} +import lib.storage.ImageLoaderStore +import model.Uploader.{fromUploadRequestShared, toImageUploadOpsCfg} +import model.upload.{OptimiseOps, OptimiseWithPngQuant, UploadRequest} +import org.joda.time.DateTime +import play.api.libs.json.{JsObject, Json} + +import scala.concurrent.{ExecutionContext, Future} + +case class ImageUpload(uploadRequest: UploadRequest, image: Image) + +case object ImageUpload { + val metadataCleaners = new MetadataCleaners(MetadataConfig.allPhotographersMap) + + def createImage(uploadRequest: UploadRequest, source: Asset, thumbnail: Asset, png: Option[Asset], + fileMetadata: FileMetadata, metadata: ImageMetadata): Image = { + val usageRights = NoRights + Image( + uploadRequest.imageId, + uploadRequest.uploadTime, + uploadRequest.uploadedBy, + Some(uploadRequest.uploadTime), + uploadRequest.identifiers, + uploadRequest.uploadInfo, + source, + Some(thumbnail), + png, + fileMetadata, + None, + metadata, + metadata, + usageRights, + usageRights, + List(), + List() + ) + } +} + +case class ImageUploadOpsCfg( + tempDir: File, + thumbWidth: Int, + thumbQuality: Double, + transcodedMimeTypes: List[MimeType], + originalFileBucket: String, + thumbBucket: String +) + +case class ImageUploadOpsDependencies( + config: ImageUploadOpsCfg, + imageOps: ImageOperations, + storeOrProjectOriginalFile: StorableOriginalImage => Future[S3Object], + storeOrProjectThumbFile: StorableThumbImage => Future[S3Object], + storeOrProjectOptimisedImage: StorableOptimisedImage => Future[S3Object] +) + +object Uploader extends GridLogging { + + def toImageUploadOpsCfg(config: ImageLoaderConfig): ImageUploadOpsCfg = { + ImageUploadOpsCfg( + config.tempDir, + config.thumbWidth, + config.thumbQuality, + config.transcodedMimeTypes, + config.imageBucket, + config.thumbnailBucket + ) + } + + def fromUploadRequestShared(uploadRequest: UploadRequest, deps: ImageUploadOpsDependencies) + (implicit ec: ExecutionContext, logMarker: LogMarker): Future[Image] = { + + import deps._ + + logger.info("Starting image ops") + + val fileMetadataFuture = toFileMetadata(uploadRequest.tempFile, uploadRequest.imageId, uploadRequest.mimeType) + + logger.info("Have read file headers") + + fileMetadataFuture.flatMap(fileMetadata => { + uploadAndStoreImage( + storeOrProjectOriginalFile, + storeOrProjectThumbFile, + storeOrProjectOptimisedImage, + OptimiseWithPngQuant, + uploadRequest, + deps, + fileMetadata)(ec, addLogMarkers(fileMetadata.toLogMarker)) + }) + } + + private[model] def uploadAndStoreImage(storeOrProjectOriginalFile: StorableOriginalImage => Future[S3Object], + storeOrProjectThumbFile: StorableThumbImage => Future[S3Object], + storeOrProjectOptimisedFile: StorableOptimisedImage => Future[S3Object], + optimiseOps: OptimiseOps, + uploadRequest: UploadRequest, + deps: ImageUploadOpsDependencies, + fileMetadata: FileMetadata) + (implicit ec: ExecutionContext, logMarker: LogMarker) = { + val originalMimeType = uploadRequest.mimeType + .orElse(MimeTypeDetection.guessMimeType(uploadRequest.tempFile).toOption) + match { + case Some(a) => a + case None => throw new Exception("File of unknown and undetectable mime type") + } + + val makeNewDirInTempDirHere: File = Files.createTempDirectory(deps.config.tempDir.toPath, "upload").toFile + + val colourModelFuture = ImageOperations.identifyColourModel(uploadRequest.tempFile, originalMimeType) + val sourceDimensionsFuture = FileMetadataReader.dimensions(uploadRequest.tempFile, Some(originalMimeType)) + + val storableOriginalImage = StorableOriginalImage( + uploadRequest.imageId, + uploadRequest.tempFile, + originalMimeType, + toMetaMap(uploadRequest) + ) + val sourceStoreFuture = storeOrProjectOriginalFile(storableOriginalImage) + val eventualBrowserViewableImage = createBrowserViewableFileFuture(uploadRequest, makeNewDirInTempDirHere, deps) + + + val eventualImage = for { + browserViewableImage <- eventualBrowserViewableImage + s3Source <- sourceStoreFuture + optimisedFileMetadata <- FileMetadataReader.fromIPTCHeadersWithColorInfo(browserViewableImage) + thumbViewableImage <- createThumbFuture(optimisedFileMetadata, colourModelFuture, browserViewableImage, deps) + s3Thumb <- storeOrProjectThumbFile(thumbViewableImage) + maybeStorableOptimisedImage <- getStorableOptimisedImage(makeNewDirInTempDirHere, optimiseOps, browserViewableImage, optimisedFileMetadata) + s3PngOption <- maybeStorableOptimisedImage match { + case Some(storableOptimisedImage) => storeOrProjectOptimisedFile(storableOptimisedImage).map(a=>Some(a)) + case None => Future.successful(None) + } + sourceDimensions <- sourceDimensionsFuture + thumbDimensions <- FileMetadataReader.dimensions(thumbViewableImage.file, Some(Jpeg)) + colourModel <- colourModelFuture + } yield { + val fullFileMetadata = fileMetadata.copy(colourModel = colourModel) + val metadata = ImageMetadataConverter.fromFileMetadata(fullFileMetadata) + val cleanMetadata = ImageUpload.metadataCleaners.clean(metadata) + + val sourceAsset = Asset.fromS3Object(s3Source, sourceDimensions) + val thumbAsset = Asset.fromS3Object(s3Thumb, thumbDimensions) + + val pngAsset = s3PngOption.map(Asset.fromS3Object(_, sourceDimensions)) + val baseImage = ImageUpload.createImage(uploadRequest, sourceAsset, thumbAsset, pngAsset, fullFileMetadata, cleanMetadata) + + val processedImage = SupplierProcessors.process(baseImage) + + logger.info("Ending image ops") + // FIXME: dirty hack to sync the originalUsageRights and originalMetadata as well + processedImage.copy( + originalMetadata = processedImage.metadata, + originalUsageRights = processedImage.usageRights + ) + } + eventualImage.onComplete{ _ => + makeNewDirInTempDirHere.listFiles().map(f => f.delete()) + makeNewDirInTempDirHere.delete() + } + eventualImage + } + + private def getStorableOptimisedImage( + tempDir: File, + optimiseOps: OptimiseOps, + browserViewableImage: BrowserViewableImage, + optimisedFileMetadata: FileMetadata) + (implicit ec: ExecutionContext, logMarker: LogMarker): Future[Option[StorableOptimisedImage]] = { + if (optimiseOps.shouldOptimise(Some(browserViewableImage.mimeType), optimisedFileMetadata)) { + for { + (optimisedFile: File, optimisedMimeType: MimeType) <- optimiseOps.toOptimisedFile(browserViewableImage.file, browserViewableImage, tempDir) + } yield Some(browserViewableImage.copy(file = optimisedFile).copy(mimeType = optimisedMimeType).asStorableOptimisedImage) + } else if (browserViewableImage.mustUpload) { + Future.successful(Some(browserViewableImage.asStorableOptimisedImage)) + } else + Future.successful(None) + } + + def toMetaMap(uploadRequest: UploadRequest): Map[String, String] = { + val baseMeta = Map( + "uploaded_by" -> uploadRequest.uploadedBy, + "upload_time" -> printDateTime(uploadRequest.uploadTime) + ) ++ uploadRequest.identifiersMeta + + uploadRequest.uploadInfo.filename match { + case Some(f) => baseMeta ++ Map("file_name" -> URLEncoder.encode(f, StandardCharsets.UTF_8.name())) + case _ => baseMeta + } + } + + private def toFileMetadata(f: File, imageId: String, mimeType: Option[MimeType]): Future[FileMetadata] = { + mimeType match { + case Some(Png | Tiff) => FileMetadataReader.fromIPTCHeadersWithColorInfo(f, imageId, mimeType.get) + case _ => FileMetadataReader.fromIPTCHeaders(f, imageId) + } + } + + private def createThumbFuture(fileMetadata: FileMetadata, + colourModelFuture: Future[Option[String]], + browserViewableImage: BrowserViewableImage, + deps: ImageUploadOpsDependencies)(implicit ec: ExecutionContext) = { + import deps._ + for { + colourModel <- colourModelFuture + iccColourSpace = FileMetadataHelper.normalisedIccColourSpace(fileMetadata) + (thumb, thumbMimeType) <- imageOps + .createThumbnail(browserViewableImage.file, Some(browserViewableImage.mimeType), config.thumbWidth, + config.thumbQuality, config.tempDir, iccColourSpace, colourModel) + } yield browserViewableImage + .copy(file = thumb, mimeType = thumbMimeType) + .asStorableThumbImage + } + + private def createBrowserViewableFileFuture(uploadRequest: UploadRequest, + tempDir: File, + deps: ImageUploadOpsDependencies)(implicit ec: ExecutionContext): Future[BrowserViewableImage] = { + import deps._ + uploadRequest.mimeType match { + case Some(mime) if config.transcodedMimeTypes.contains(mime) => + for { + (file, mimeType) <- imageOps.transformImage(uploadRequest.tempFile, uploadRequest.mimeType, tempDir) + } yield BrowserViewableImage( + uploadRequest.imageId, + file = file, + mimeType = mimeType, + mustUpload = true + ) + case Some(mimeType) => + Future.successful( + BrowserViewableImage( + uploadRequest.imageId, + file = uploadRequest.tempFile, + mimeType = mimeType) + ) + case None => Future.failed(new Exception("This file is not an image with an identifiable mime type")) + } + } +} + +class Uploader(val store: ImageLoaderStore, + val config: ImageLoaderConfig, + val imageOps: ImageOperations, + val notifications: Notifications) + (implicit val ec: ExecutionContext) extends ArgoHelpers { + + + + + def fromUploadRequest(uploadRequest: UploadRequest) + (implicit logMarker: LogMarker): Future[ImageUpload] = { + val sideEffectDependencies = ImageUploadOpsDependencies(toImageUploadOpsCfg(config), imageOps, + storeSource, storeThumbnail, storeOptimisedImage) + val finalImage = fromUploadRequestShared(uploadRequest, sideEffectDependencies) + finalImage.map(img => Stopwatch("finalImage"){ImageUpload(uploadRequest, img)}) + } + + private def storeSource(storableOriginalImage: StorableOriginalImage) + (implicit logMarker: LogMarker) = store.store(storableOriginalImage) + + private def storeThumbnail(storableThumbImage: StorableThumbImage) + (implicit logMarker: LogMarker) = store.store(storableThumbImage) + + private def storeOptimisedImage(storableOptimisedImage: StorableOptimisedImage) + (implicit logMarker: LogMarker) = store.store(storableOptimisedImage) + + def loadFile(digestedFile: DigestedFile, + user: Principal, + uploadedBy: Option[String], + identifiers: Option[String], + uploadTime: DateTime, + filename: Option[String], + requestId: UUID) + (implicit ec:ExecutionContext, + logMarker: LogMarker): Future[UploadRequest] = Future { + val DigestedFile(tempFile, id) = digestedFile + + // TODO: should error if the JSON parsing failed + val identifiersMap = identifiers.map(Json.parse(_).as[Map[String, String]]) getOrElse Map() + + MimeTypeDetection.guessMimeType(tempFile) match { + case util.Left(unsupported) => + logger.error(s"Unsupported mimetype", unsupported) + throw unsupported + case util.Right(mimeType) => + logger.info(s"Detected mimetype as $mimeType") + UploadRequest( + requestId = requestId, + imageId = id, + tempFile = tempFile, + mimeType = Some(mimeType), + uploadTime = uploadTime, + uploadedBy = uploadedBy.getOrElse(Authentication.getIdentity(user)), + identifiers = identifiersMap, + uploadInfo = UploadInfo(filename) + ) + } + } + + def storeFile(uploadRequest: UploadRequest) + (implicit ec:ExecutionContext, + logMarker: LogMarker): Future[JsObject] = { + + logger.info("Storing file") + + for { + imageUpload <- fromUploadRequest(uploadRequest) + updateMessage = UpdateMessage(subject = "image", image = Some(imageUpload.image)) + _ <- Future { notifications.publish(updateMessage) } + // TODO: centralise where all these URLs are constructed + uri = s"${config.apiUri}/images/${uploadRequest.imageId}" + } yield { + Json.obj("uri" -> uri) + } + + } + +} + diff --git a/image-loader/app/model/upload/OptimiseOps.scala b/image-loader/app/model/upload/OptimiseOps.scala new file mode 100644 index 0000000000..6bd497227e --- /dev/null +++ b/image-loader/app/model/upload/OptimiseOps.scala @@ -0,0 +1,56 @@ +package model.upload + +import java.io.File + +import com.gu.mediaservice.lib.{ImageWrapper, StorableImage} +import com.gu.mediaservice.lib.logging.{LogMarker, Stopwatch} +import com.gu.mediaservice.model.{FileMetadata, MimeType, Png, Tiff} + +import scala.concurrent.{ExecutionContext, Future} +import scala.sys.process._ + +trait OptimiseOps { + def toOptimisedFile(file: File, imageWrapper: ImageWrapper, tempDir: File) + (implicit ec: ExecutionContext, logMarker: LogMarker): Future[(File, MimeType)] + def isTransformedFilePath(filePath: String): Boolean + def shouldOptimise(mimeType: Option[MimeType], fileMetadata: FileMetadata): Boolean + def optimiseMimeType: MimeType +} + +object OptimiseWithPngQuant extends OptimiseOps { + + override def optimiseMimeType: MimeType = Png + + def toOptimisedFile(file: File, imageWrapper: ImageWrapper, tempDir: File) + (implicit ec: ExecutionContext, logMarker: LogMarker): Future[(File, MimeType)] = Future { + + val optimisedFilePath = tempDir.getAbsolutePath + "/optimisedpng - " + imageWrapper.id + optimiseMimeType.fileExtension + Stopwatch("pngquant") { + val result = Seq("pngquant", "--quality", "1-85", file.getAbsolutePath, "--output", optimisedFilePath).! + if (result>0) + throw new Exception(s"pngquant failed to convert to optimised png file (rc = $result)") + } + + val optimisedFile = new File(optimisedFilePath) + if (optimisedFile.exists()) { + (optimisedFile, Png) + } else { + throw new Exception(s"Attempted to optimise PNG file ${optimisedFile.getPath}") + } + } + + def isTransformedFilePath(filePath: String): Boolean = filePath.contains("transformed-") + + def shouldOptimise(mimeType: Option[MimeType], fileMetadata: FileMetadata): Boolean = + mimeType match { + case Some(Png) => + fileMetadata.colourModelInformation.get("colorType") match { + case Some("True Color") => true + case Some("True Color with Alpha") => true + case _ => false + } + case Some(Tiff) => true + case _ => false + } +} + diff --git a/image-loader/app/model/UploadRequest.scala b/image-loader/app/model/upload/UploadRequest.scala similarity index 62% rename from image-loader/app/model/UploadRequest.scala rename to image-loader/app/model/upload/UploadRequest.scala index 3d4ff79fea..b256776a0a 100644 --- a/image-loader/app/model/UploadRequest.scala +++ b/image-loader/app/model/upload/UploadRequest.scala @@ -1,31 +1,31 @@ -package model +package model.upload import java.io.File import java.util.UUID -import com.gu.mediaservice.model.{UploadInfo, MimeType} +import com.gu.mediaservice.model.{MimeType, UploadInfo} import net.logstash.logback.marker.{LogstashMarker, Markers} import org.joda.time.format.ISODateTimeFormat import org.joda.time.{DateTime, DateTimeZone} - import scala.collection.JavaConverters._ case class UploadRequest( - requestId: UUID, - imageId: String, - tempFile: File, - mimeType: Option[MimeType], - uploadTime: DateTime, - uploadedBy: String, - identifiers: Map[String, String], - uploadInfo: UploadInfo -) { + requestId: UUID, + imageId: String, + tempFile: File, + mimeType: Option[MimeType], + uploadTime: DateTime, + uploadedBy: String, + identifiers: Map[String, String], + uploadInfo: UploadInfo + ) { + val identifiersMeta: Map[String, String] = identifiers.map { case (k, v) => (s"identifier!$k", v) } def toLogMarker: LogstashMarker = { val fallback = "none" - val markers = Map ( + val markers = Map( "requestId" -> requestId, "imageId" -> imageId, "mimeType" -> mimeType.getOrElse(fallback), @@ -37,4 +37,5 @@ case class UploadRequest( Markers.appendEntries(markers.asJava) } + } diff --git a/image-loader/test/resources/IndexedColor.png b/image-loader/test/resources/IndexedColor.png new file mode 100644 index 0000000000..4d6cf9e44c Binary files /dev/null and b/image-loader/test/resources/IndexedColor.png differ diff --git a/image-loader/test/resources/basn2c16_TrueColor_16bit.png b/image-loader/test/resources/basn2c16_TrueColor_16bit.png new file mode 100644 index 0000000000..50c1cb91a0 Binary files /dev/null and b/image-loader/test/resources/basn2c16_TrueColor_16bit.png differ diff --git a/image-loader/test/resources/bgan6a16_TrueColorWithAlpha_16bit.png b/image-loader/test/resources/bgan6a16_TrueColorWithAlpha_16bit.png new file mode 100644 index 0000000000..984a99525f Binary files /dev/null and b/image-loader/test/resources/bgan6a16_TrueColorWithAlpha_16bit.png differ diff --git a/image-loader/test/resources/rubbish.jpg b/image-loader/test/resources/rubbish.jpg new file mode 100644 index 0000000000..3bfa4367b1 Binary files /dev/null and b/image-loader/test/resources/rubbish.jpg differ diff --git a/image-loader/test/resources/srgb.icc b/image-loader/test/resources/srgb.icc new file mode 100644 index 0000000000..7f9d18d097 Binary files /dev/null and b/image-loader/test/resources/srgb.icc differ diff --git a/image-loader/test/resources/thisisnotanimage.jpg b/image-loader/test/resources/thisisnotanimage.jpg new file mode 100644 index 0000000000..e69de29bb2 diff --git a/image-loader/test/resources/thisisnotanimage.stupid b/image-loader/test/resources/thisisnotanimage.stupid new file mode 100644 index 0000000000..e69de29bb2 diff --git a/image-loader/test/resources/tiff_8bpc_flat.tif b/image-loader/test/resources/tiff_8bpc_flat.tif new file mode 100644 index 0000000000..e4e176485e Binary files /dev/null and b/image-loader/test/resources/tiff_8bpc_flat.tif differ diff --git a/image-loader/test/resources/tiff_8bpc_layered_withTransparency.tif b/image-loader/test/resources/tiff_8bpc_layered_withTransparency.tif new file mode 100644 index 0000000000..1580e32154 Binary files /dev/null and b/image-loader/test/resources/tiff_8bpc_layered_withTransparency.tif differ diff --git a/image-loader/test/scala/lib/imaging/FileMetadataReaderTest.scala b/image-loader/test/scala/lib/imaging/FileMetadataReaderTest.scala index b21143d44e..a05be8e859 100644 --- a/image-loader/test/scala/lib/imaging/FileMetadataReaderTest.scala +++ b/image-loader/test/scala/lib/imaging/FileMetadataReaderTest.scala @@ -596,7 +596,7 @@ class FileMetadataReaderTest extends FunSpec with Matchers with ScalaFutures { it("should read the correct metadata for a grayscale png") { val image = fileAt("schaik.com_pngsuite/basn0g08.png") - val metadataFuture = FileMetadataReader.fromICPTCHeadersWithColorInfo(image, "dummy", Png) + val metadataFuture = FileMetadataReader.fromIPTCHeadersWithColorInfo(image, "dummy", Png) whenReady(metadataFuture) { metadata => metadata.colourModelInformation should contain( "colorType" -> "Greyscale" @@ -606,7 +606,7 @@ class FileMetadataReaderTest extends FunSpec with Matchers with ScalaFutures { it("should read the correct metadata for a colour 8bit paletted png") { val image = fileAt("schaik.com_pngsuite/basn3p08.png") - val metadataFuture = FileMetadataReader.fromICPTCHeadersWithColorInfo(image, "dummy", Png) + val metadataFuture = FileMetadataReader.fromIPTCHeadersWithColorInfo(image, "dummy", Png) whenReady(metadataFuture) { metadata => metadata.colourModelInformation should contain( "colorType" -> "Indexed Color" @@ -616,7 +616,7 @@ class FileMetadataReaderTest extends FunSpec with Matchers with ScalaFutures { it("should read the correct metadata for a truecolour png without alpha channel") { val image = fileAt("schaik.com_pngsuite/basn2c08.png") - val metadataFuture = FileMetadataReader.fromICPTCHeadersWithColorInfo(image, "dummy", Png) + val metadataFuture = FileMetadataReader.fromIPTCHeadersWithColorInfo(image, "dummy", Png) whenReady(metadataFuture) { metadata => metadata.colourModelInformation should contain( "colorType" -> "True Color" @@ -626,7 +626,7 @@ class FileMetadataReaderTest extends FunSpec with Matchers with ScalaFutures { it("should read the correct metadata for a truecolour pnd with alpha channel") { val image = fileAt("schaik.com_pngsuite/basn6a08.png") - val metadataFuture = FileMetadataReader.fromICPTCHeadersWithColorInfo(image, "dummy", Png) + val metadataFuture = FileMetadataReader.fromIPTCHeadersWithColorInfo(image, "dummy", Png) whenReady(metadataFuture) { metadata => metadata.colourModelInformation should contain( "colorType" -> "True Color with Alpha" @@ -636,7 +636,7 @@ class FileMetadataReaderTest extends FunSpec with Matchers with ScalaFutures { it("should read the correct colour metadata for a greyscale tiff") { val image = fileAt("flower.tif") - val metadataFuture = FileMetadataReader.fromICPTCHeadersWithColorInfo(image, "dummy", Tiff) + val metadataFuture = FileMetadataReader.fromIPTCHeadersWithColorInfo(image, "dummy", Tiff) whenReady(metadataFuture) { metadata => metadata.colourModelInformation should contain( "photometricInterpretation" -> "BlackIsZero" @@ -646,7 +646,7 @@ class FileMetadataReaderTest extends FunSpec with Matchers with ScalaFutures { it("should read the correct colour metadata for an alpha tiff") { val image = fileAt("lighthouse.tif") - val metadataFuture = FileMetadataReader.fromICPTCHeadersWithColorInfo(image, "dummy", Tiff) + val metadataFuture = FileMetadataReader.fromIPTCHeadersWithColorInfo(image, "dummy", Tiff) whenReady(metadataFuture) { metadata => metadata.colourModelInformation should contain( "photometricInterpretation" -> "RGB" diff --git a/image-loader/test/scala/model/ImageUploadTest.scala b/image-loader/test/scala/model/ImageUploadTest.scala new file mode 100644 index 0000000000..c740a523f7 --- /dev/null +++ b/image-loader/test/scala/model/ImageUploadTest.scala @@ -0,0 +1,147 @@ +package model + +import java.io.File +import java.net.URI +import java.util.UUID + +import com.drew.imaging.ImageProcessingException +import com.gu.mediaservice.lib.{StorableImage, StorableOptimisedImage, StorableOriginalImage, StorableThumbImage} +import com.gu.mediaservice.lib.aws.{S3Metadata, S3Object, S3ObjectMetadata, S3Ops} +import com.gu.mediaservice.lib.imaging.ImageOperations +import com.gu.mediaservice.lib.logging.LogMarker +import com.gu.mediaservice.model.{FileMetadata, Jpeg, MimeType, Png, Tiff, UploadInfo} +import lib.imaging.MimeTypeDetection +import model.upload.{OptimiseWithPngQuant, UploadRequest} +import org.joda.time.DateTime +import org.scalatest.mockito.MockitoSugar +import org.scalatest.{Assertion, AsyncFunSuite, Matchers} +import test.lib.ResourceHelpers + +import scala.concurrent.{ExecutionContext, Future} +import scala.util.{Failure, Success} + +class ImageUploadTest extends AsyncFunSuite with Matchers with MockitoSugar { + private implicit val ec: ExecutionContext = ExecutionContext.Implicits.global + + class MockLogMarker extends LogMarker { + override def markerContents: Map[String, Any] = Map() + } + + private implicit val logMarker: MockLogMarker = new MockLogMarker() + // For mime type info, see https://github.com/guardian/grid/pull/2568 + val tempDir = new File("/tmp") + val mockConfig: ImageUploadOpsCfg = ImageUploadOpsCfg(tempDir, 256, 85d, List(Tiff), "img-bucket", "thumb-bucket") + + /** + * @todo: I flailed about until I found a path that worked, but + * what arcane magic System.getProperty relies upon, and exactly + * _how_ it will break in CI, I do not know + */ + val imageOps: ImageOperations = new ImageOperations(System.getProperty("user.dir")) + + private def imageUpload( + fileName: String, + expectedOriginalMimeType: MimeType, + expectOptimisedFile: Boolean = false): Future[Assertion] = { + + val uuid = UUID.randomUUID() + val randomId = UUID.randomUUID().toString + fileName + + val mockS3Meta = S3Metadata(Map.empty, S3ObjectMetadata(None, None, None)) + val mockS3Object = S3Object(new URI("innernets.com"), 12345, mockS3Meta) + + def mockStore = (a: StorableImage) => + Future.successful( + S3Ops.projectFileAsS3Object(new URI("http://madeupname/"), a.file, Some(a.mimeType), a.meta, None) + ) + + def storeOrProjectOriginalFile: StorableOriginalImage => Future[S3Object] = mockStore + def storeOrProjectThumbFile: StorableThumbImage => Future[S3Object] = mockStore + def storeOrProjectOptimisedPNG: StorableOptimisedImage => Future[S3Object] = mockStore + + val mockDependencies = ImageUploadOpsDependencies( + mockConfig, + imageOps, + storeOrProjectOriginalFile, + storeOrProjectThumbFile, + storeOrProjectOptimisedPNG + ) + + val tempFile = ResourceHelpers.fileAt(fileName) + val ul = UploadInfo(None) + + val uploadRequest = UploadRequest( + uuid, + randomId, + tempFile, + MimeTypeDetection.guessMimeType(tempFile).right.toOption, + DateTime.now(), + "uploadedBy", + Map(), + ul + ) + + val futureImage = Uploader.uploadAndStoreImage( + mockDependencies.storeOrProjectOriginalFile, + mockDependencies.storeOrProjectThumbFile, + mockDependencies.storeOrProjectOptimisedImage, + OptimiseWithPngQuant, + uploadRequest, + mockDependencies, + FileMetadata() + ) + + // Assertions; Failure will auto-fail + futureImage.map(i => { + // Assertions on original request + assert(i.id == randomId, "Correct id comes back") + assert(i.source.mimeType.contains(expectedOriginalMimeType), "Should have the correct mime type") + + // Assertions on generated thumbnail image + assert(i.thumbnail.isDefined, "Should always create a thumbnail") + assert(i.thumbnail.get.mimeType.get == Jpeg, "Should have correct thumb mime type") + + // Assertions on optional generated optimised png image + assert(i.optimisedPng.isDefined == expectOptimisedFile, "Should have optimised file") + assert(!expectOptimisedFile || i.optimisedPng.flatMap(p => p.mimeType).contains(Png), "Should have correct optimised mime type") + }) + } + + ignore("A jpg which is suitable for UI viewing") { + imageUpload("rubbish.jpg", Jpeg) + } + ignore("An opaque tiff file which requires optimising for UI") { + imageUpload("lighthouse.tif", Tiff, expectOptimisedFile = true) + } + ignore("A layered tiff file (will require renaming extracted file) which requires optimising for UI") { + imageUpload("tiff_8bpc_layered_withTransparency.tif", Tiff, expectOptimisedFile = true) + } + ignore("Another opaque tiff file which requires optimising for UI") { + imageUpload("tiff_8bpc_flat.tif", Tiff, expectOptimisedFile = true) + } + ignore("A png which is suitable for UI viewing") { + imageUpload("IndexedColor.png", Png) + } + ignore("A png which is not suitable (too many colours + transparency) for UI viewing") { + imageUpload("bgan6a16_TrueColorWithAlpha_16bit.png", Png, expectOptimisedFile = true) + } + ignore("A png which is not suitable (too many colours) for UI viewing") { + imageUpload("basn2c16_TrueColor_16bit.png", Png, expectOptimisedFile = true) + } + ignore("not an image but looks like one") { + imageUpload("thisisnotanimage.jpg", Png, expectOptimisedFile = true).transformWith{ + case Success(_) => fail("Should have thrown an error") + case Failure(e) => e match { + case e: ImageProcessingException => assert(e.getMessage == "File format could not be determined") + } + } + } + ignore("not an image and does not look like one") { + // this exception is thrown before the futures are resolved, and so does not need transformWith + val caught = the [Exception] thrownBy + imageUpload("thisisnotanimage.stupid", Png, expectOptimisedFile = true) + assert(caught.getMessage == "File of unknown and undetectable mime type") + } +} + +// todo add to tests - tiff with layers, but not true colour so does not need optimising. MK to provide