From aded1f910169d9afa9c768c5b6e57e18da7140d5 Mon Sep 17 00:00:00 2001
From: creativecbr <pawel.lesniewski@techstep.io>
Date: Mon, 13 Jan 2025 17:38:20 +0100
Subject: [PATCH] Bug unsupported file format 'public.heic' (#538)

While implementing remote sharing using LiveKit, we encountered an error
on the iPad 5th generation running iPadOS 16.7.10 when converting a
buffer to a JPEG representation. The suggested solution is to avoid
using the native .jpgedRepresentation method, as it appears to contain a
bug, and instead handle the conversion manually. After applying this
approach, the error seems to be resolved.

The error logged in the Console app was:
```
Error    11:20:25.613787+0100    RemoteSharing    findWriterForTypeAndAlternateType:119: unsupported file format 'public.heic'
```

This issue did not occur on newer devices with higher iOS versions and
it worked correctly on newer devices like the iPhone SE (2nd generation)
with iOS 15.5. This suggests the problem is specific to platform
16.7.10.

This error caused the screen-sharing session to be interrupted after
several occurrences. After changes session was stable.

Please review this change and consider potential improvements.

PS: I have also removed the code responsible for scale factor buffering,
as I did not observe any usage of it. If this removal was a mistake,
please let me know.

---------

Co-authored-by: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com>
---
 .../Broadcast/Uploader/SampleUploader.swift   | 45 ++++++++++++++-----
 1 file changed, 33 insertions(+), 12 deletions(-)

diff --git a/Sources/LiveKit/Broadcast/Uploader/SampleUploader.swift b/Sources/LiveKit/Broadcast/Uploader/SampleUploader.swift
index f499f6360..df1e1223b 100644
--- a/Sources/LiveKit/Broadcast/Uploader/SampleUploader.swift
+++ b/Sources/LiveKit/Broadcast/Uploader/SampleUploader.swift
@@ -28,6 +28,7 @@ private enum Constants {
 
 class SampleUploader {
     private static var imageContext = CIContext(options: nil)
+    private static var colorSpace = CGColorSpaceCreateDeviceRGB()
 
     @Atomic private var isReady = false
     private var connection: BroadcastUploadSocketConnection
@@ -37,6 +38,9 @@ class SampleUploader {
 
     private let serialQueue: DispatchQueue
 
+    // Configure desired compression quality (0.0 = max compression, 1.0 = least compression)
+    public let compressionQuality: CGFloat = 1.0
+
     init(connection: BroadcastUploadSocketConnection) {
         self.connection = connection
         serialQueue = DispatchQueue(label: "io.livekit.broadcast.sampleUploader")
@@ -115,14 +119,12 @@ private extension SampleUploader {
 
         CVPixelBufferLockBaseAddress(imageBuffer, .readOnly)
 
-        let scaleFactor = 1.0
-        let width = CVPixelBufferGetWidth(imageBuffer) / Int(scaleFactor)
-        let height = CVPixelBufferGetHeight(imageBuffer) / Int(scaleFactor)
+        let width = CVPixelBufferGetWidth(imageBuffer)
+        let height = CVPixelBufferGetHeight(imageBuffer)
 
         let orientation = CMGetAttachment(buffer, key: RPVideoSampleOrientationKey as CFString, attachmentModeOut: nil)?.uintValue ?? 0
 
-        let scaleTransform = CGAffineTransform(scaleX: CGFloat(1.0 / scaleFactor), y: CGFloat(1.0 / scaleFactor))
-        let bufferData = jpegData(from: imageBuffer, scale: scaleTransform)
+        let bufferData = jpegData(from: imageBuffer)
 
         CVPixelBufferUnlockBaseAddress(imageBuffer, .readOnly)
 
@@ -144,16 +146,35 @@ private extension SampleUploader {
         return serializedMessage
     }
 
-    func jpegData(from buffer: CVPixelBuffer, scale scaleTransform: CGAffineTransform) -> Data? {
-        let image = CIImage(cvPixelBuffer: buffer).transformed(by: scaleTransform)
+    func jpegData(from buffer: CVPixelBuffer) -> Data? {
+        let image = CIImage(cvPixelBuffer: buffer)
 
-        guard let colorSpace = image.colorSpace else {
-            return nil
-        }
+        if #available(iOS 17.0, *) {
+            return Self.imageContext.jpegRepresentation(
+                of: image,
+                colorSpace: Self.colorSpace,
+                options: [kCGImageDestinationLossyCompressionQuality as CIImageRepresentationOption: compressionQuality]
+            )
+        } else {
+            // Workaround for "unsupported file format 'public.heic'"
+            guard let cgImage = Self.imageContext.createCGImage(image, from: image.extent) else {
+                return nil
+            }
 
-        let options: [CIImageRepresentationOption: Float] = [kCGImageDestinationLossyCompressionQuality as CIImageRepresentationOption: 1.0]
+            let data = NSMutableData()
+            guard let imageDestination = CGImageDestinationCreateWithData(data, AVFileType.jpg as CFString, 1, nil) else {
+                return nil
+            }
+
+            let options: [CFString: Any] = [kCGImageDestinationLossyCompressionQuality: compressionQuality]
+            CGImageDestinationAddImage(imageDestination, cgImage, options as CFDictionary)
 
-        return SampleUploader.imageContext.jpegRepresentation(of: image, colorSpace: colorSpace, options: options)
+            guard CGImageDestinationFinalize(imageDestination) else {
+                return nil
+            }
+
+            return data as Data
+        }
     }
 }