From 99f455b050795e1623c24062ddc6c2a2d432b6f5 Mon Sep 17 00:00:00 2001 From: "pawel.lesniewski" Date: Tue, 7 Jan 2025 16:21:08 +0100 Subject: [PATCH 1/2] Removed the scale factor for optimization and added an alternative method for obtaining image representations on devices running iOS versions below 17.0 --- .../Broadcast/Uploader/SampleUploader.swift | 37 +++++++++++++------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/Sources/LiveKit/Broadcast/Uploader/SampleUploader.swift b/Sources/LiveKit/Broadcast/Uploader/SampleUploader.swift index f499f6360..f31a58844 100644 --- a/Sources/LiveKit/Broadcast/Uploader/SampleUploader.swift +++ b/Sources/LiveKit/Broadcast/Uploader/SampleUploader.swift @@ -17,6 +17,7 @@ #if os(iOS) import Foundation +import UniformTypeIdentifiers #if canImport(ReplayKit) import ReplayKit @@ -28,6 +29,7 @@ private enum Constants { class SampleUploader { private static var imageContext = CIContext(options: nil) + private static var colorSpace = CGColorSpace(name: CGColorSpace.sRGB) @Atomic private var isReady = false private var connection: BroadcastUploadSocketConnection @@ -115,14 +117,12 @@ private extension SampleUploader { CVPixelBufferLockBaseAddress(imageBuffer, .readOnly) - let scaleFactor = 1.0 - let width = CVPixelBufferGetWidth(imageBuffer) / Int(scaleFactor) - let height = CVPixelBufferGetHeight(imageBuffer) / Int(scaleFactor) + let width = CVPixelBufferGetWidth(imageBuffer) + let height = CVPixelBufferGetHeight(imageBuffer) let orientation = CMGetAttachment(buffer, key: RPVideoSampleOrientationKey as CFString, attachmentModeOut: nil)?.uintValue ?? 0 - let scaleTransform = CGAffineTransform(scaleX: CGFloat(1.0 / scaleFactor), y: CGFloat(1.0 / scaleFactor)) - let bufferData = jpegData(from: imageBuffer, scale: scaleTransform) + let bufferData = jpegData(from: imageBuffer) CVPixelBufferUnlockBaseAddress(imageBuffer, .readOnly) @@ -144,16 +144,29 @@ private extension SampleUploader { return serializedMessage } - func jpegData(from buffer: CVPixelBuffer, scale scaleTransform: CGAffineTransform) -> Data? { - let image = CIImage(cvPixelBuffer: buffer).transformed(by: scaleTransform) + func jpegData(from buffer: CVPixelBuffer) -> Data? { - guard let colorSpace = image.colorSpace else { - return nil - } + let image = CIImage(cvPixelBuffer: buffer) + + if #available(iOS 17.0, *) { + return SampleUploader.imageContext.jpegRepresentation(of: image, + colorSpace: SampleUploader.colorSpace ?? CGColorSpaceCreateDeviceRGB(), + options: [kCGImageDestinationLossyCompressionQuality as CIImageRepresentationOption: 1.0]) + } else { + guard let cgImage = SampleUploader.imageContext.createCGImage(image, from: image.extent) else { + return nil + } - let options: [CIImageRepresentationOption: Float] = [kCGImageDestinationLossyCompressionQuality as CIImageRepresentationOption: 1.0] + let data = NSMutableData() + if let imageDestination = CGImageDestinationCreateWithData(data, AVFileType.jpg as CFString, 1, nil) { + CGImageDestinationAddImage(imageDestination, cgImage, [:] as CFDictionary) // Empty dictionary, it is possible to set lossy compression quality here + if CGImageDestinationFinalize(imageDestination) { + return data as Data + } + } + } - return SampleUploader.imageContext.jpegRepresentation(of: image, colorSpace: colorSpace, options: options) + return nil } } From 90520704425c12fc3eed8d13ae905a988bb27c5c Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Tue, 14 Jan 2025 00:04:30 +0900 Subject: [PATCH 2/2] Code clean-up & format --- .../Broadcast/Uploader/SampleUploader.swift | 36 +++++++++++-------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/Sources/LiveKit/Broadcast/Uploader/SampleUploader.swift b/Sources/LiveKit/Broadcast/Uploader/SampleUploader.swift index f31a58844..df1e1223b 100644 --- a/Sources/LiveKit/Broadcast/Uploader/SampleUploader.swift +++ b/Sources/LiveKit/Broadcast/Uploader/SampleUploader.swift @@ -17,7 +17,6 @@ #if os(iOS) import Foundation -import UniformTypeIdentifiers #if canImport(ReplayKit) import ReplayKit @@ -29,7 +28,7 @@ private enum Constants { class SampleUploader { private static var imageContext = CIContext(options: nil) - private static var colorSpace = CGColorSpace(name: CGColorSpace.sRGB) + private static var colorSpace = CGColorSpaceCreateDeviceRGB() @Atomic private var isReady = false private var connection: BroadcastUploadSocketConnection @@ -39,6 +38,9 @@ class SampleUploader { private let serialQueue: DispatchQueue + // Configure desired compression quality (0.0 = max compression, 1.0 = least compression) + public let compressionQuality: CGFloat = 1.0 + init(connection: BroadcastUploadSocketConnection) { self.connection = connection serialQueue = DispatchQueue(label: "io.livekit.broadcast.sampleUploader") @@ -145,28 +147,34 @@ private extension SampleUploader { } func jpegData(from buffer: CVPixelBuffer) -> Data? { - let image = CIImage(cvPixelBuffer: buffer) if #available(iOS 17.0, *) { - return SampleUploader.imageContext.jpegRepresentation(of: image, - colorSpace: SampleUploader.colorSpace ?? CGColorSpaceCreateDeviceRGB(), - options: [kCGImageDestinationLossyCompressionQuality as CIImageRepresentationOption: 1.0]) + return Self.imageContext.jpegRepresentation( + of: image, + colorSpace: Self.colorSpace, + options: [kCGImageDestinationLossyCompressionQuality as CIImageRepresentationOption: compressionQuality] + ) } else { - guard let cgImage = SampleUploader.imageContext.createCGImage(image, from: image.extent) else { + // Workaround for "unsupported file format 'public.heic'" + guard let cgImage = Self.imageContext.createCGImage(image, from: image.extent) else { return nil } let data = NSMutableData() - if let imageDestination = CGImageDestinationCreateWithData(data, AVFileType.jpg as CFString, 1, nil) { - CGImageDestinationAddImage(imageDestination, cgImage, [:] as CFDictionary) // Empty dictionary, it is possible to set lossy compression quality here - if CGImageDestinationFinalize(imageDestination) { - return data as Data - } + guard let imageDestination = CGImageDestinationCreateWithData(data, AVFileType.jpg as CFString, 1, nil) else { + return nil } - } - return nil + let options: [CFString: Any] = [kCGImageDestinationLossyCompressionQuality: compressionQuality] + CGImageDestinationAddImage(imageDestination, cgImage, options as CFDictionary) + + guard CGImageDestinationFinalize(imageDestination) else { + return nil + } + + return data as Data + } } }