From 482146688a44446a499e3d4093a06144378ed2f4 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Wed, 29 May 2024 19:09:49 +0900 Subject: [PATCH] Pinch to zoom in & out (#384) New VideoView.pinchToZoom and pinchToZoomAutoZoomOut properties which work for iOS only. Seamless zoom in/out will work for supported devices such as .builtInTripleCamera. This is currently only for local tracks (capturing). https://github.com/livekit/client-sdk-swift/assets/548776/2bc5fabb-3184-4613-9d3e-c8a236a15189 --- Package.swift | 2 +- .../LiveKit/Extensions/AVCaptureDevice.swift | 38 +++++++++ Sources/LiveKit/Support/DeviceManager.swift | 21 ++--- .../Track/Capturers/CameraCapturer.swift | 6 +- Sources/LiveKit/Views/VideoView.swift | 82 ++++++++++++++++--- 5 files changed, 122 insertions(+), 27 deletions(-) create mode 100644 Sources/LiveKit/Extensions/AVCaptureDevice.swift diff --git a/Package.swift b/Package.swift index 1ce515f77..537cf9237 100644 --- a/Package.swift +++ b/Package.swift @@ -18,7 +18,7 @@ let package = Package( ], dependencies: [ // LK-Prefixed Dynamic WebRTC XCFramework - .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "114.5735.16"), + .package(url: "https://github.com/livekit/webrtc-xcframework.git", exact: "114.5735.18"), .package(url: "https://github.com/apple/swift-protobuf.git", from: "1.26.0"), .package(url: "https://github.com/apple/swift-log.git", from: "1.5.4"), // Only used for DocC generation diff --git a/Sources/LiveKit/Extensions/AVCaptureDevice.swift b/Sources/LiveKit/Extensions/AVCaptureDevice.swift new file mode 100644 index 000000000..7a0da3bc1 --- /dev/null +++ b/Sources/LiveKit/Extensions/AVCaptureDevice.swift @@ -0,0 +1,38 @@ +/* + * Copyright 2024 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import AVFoundation + +public extension AVCaptureDevice { + /// Helper extension to return the acual direction the camera is facing. + /// In macOS, the Facetime camera's position is .unspecified but this property will return .front for such cases. + var facingPosition: AVCaptureDevice.Position { + if deviceType == .builtInWideAngleCamera, position == .unspecified { + return .front + } + + return position + } +} + +public extension Collection where Element: AVCaptureDevice { + /// Helper extension to return only a single suggested device for each position. + func singleDeviceforEachPosition() -> [AVCaptureDevice] { + let front = first { $0.facingPosition == .front } + let back = first { $0.facingPosition == .back } + return [front, back].compactMap { $0 } + } +} diff --git a/Sources/LiveKit/Support/DeviceManager.swift b/Sources/LiveKit/Support/DeviceManager.swift index 414721503..ed0aa0708 100644 --- a/Sources/LiveKit/Support/DeviceManager.swift +++ b/Sources/LiveKit/Support/DeviceManager.swift @@ -28,25 +28,26 @@ class DeviceManager: Loggable { } // Async version, waits until inital device fetch is complete - public func devices(types: [AVCaptureDevice.DeviceType] = [.builtInWideAngleCamera]) async throws -> [AVCaptureDevice] { - try await devicesCompleter.wait().filter { types.contains($0.deviceType) } + public func devices() async throws -> [AVCaptureDevice] { + try await devicesCompleter.wait() } // Sync version - public func devices(types: [AVCaptureDevice.DeviceType] = [.builtInWideAngleCamera]) -> [AVCaptureDevice] { - _state.devices.filter { types.contains($0.deviceType) } + public func devices() -> [AVCaptureDevice] { + _state.devices } private lazy var discoverySession: AVCaptureDevice.DiscoverySession = { var deviceTypes: [AVCaptureDevice.DeviceType] #if os(iOS) + // In order of priority deviceTypes = [ - .builtInWideAngleCamera, // General purpose use - .builtInTelephotoCamera, - .builtInUltraWideCamera, - .builtInTripleCamera, - .builtInDualCamera, - .builtInDualWideCamera, + .builtInTripleCamera, // Virtual, switchOver: [2, 6], default: 2 + .builtInDualCamera, // Virtual, switchOver: [3], default: 1 + .builtInDualWideCamera, // Virtual, switchOver: [2], default: 2 + .builtInWideAngleCamera, // Physical, General purpose use + .builtInTelephotoCamera, // Physical + .builtInUltraWideCamera, // Physical ] // Xcode 15.0 Swift 5.9 (iOS 17) #if compiler(>=5.9) diff --git a/Sources/LiveKit/Track/Capturers/CameraCapturer.swift b/Sources/LiveKit/Track/Capturers/CameraCapturer.swift index e48102280..0c806096b 100644 --- a/Sources/LiveKit/Track/Capturers/CameraCapturer.swift +++ b/Sources/LiveKit/Track/Capturers/CameraCapturer.swift @@ -34,8 +34,8 @@ public class CameraCapturer: VideoCapturer { public var options: CameraCaptureOptions { _cameraCapturerState.options } @objc - public static func captureDevices(types: [AVCaptureDevice.DeviceType] = [.builtInWideAngleCamera]) async throws -> [AVCaptureDevice] { - try await DeviceManager.shared.devices(types: types) + public static func captureDevices() async throws -> [AVCaptureDevice] { + try await DeviceManager.shared.devices() } /// Checks whether both front and back capturing devices exist, and can be switched. @@ -80,7 +80,7 @@ public class CameraCapturer: VideoCapturer { var device: AVCaptureDevice? } - private var _cameraCapturerState: StateSync + var _cameraCapturerState: StateSync // Used to hide LKRTCVideoCapturerDelegate symbol private lazy var adapter: VideoCapturerDelegateAdapter = .init(cameraCapturer: self) diff --git a/Sources/LiveKit/Views/VideoView.swift b/Sources/LiveKit/Views/VideoView.swift index 42b9a686a..c66abc584 100644 --- a/Sources/LiveKit/Views/VideoView.swift +++ b/Sources/LiveKit/Views/VideoView.swift @@ -140,6 +140,18 @@ public class VideoView: NativeView, Loggable { set { _state.mutate { $0.transitionDuration = newValue } } } + @objc + public var isPinchToZoomEnabled: Bool { + get { _state.isPinchToZoomEnabled } + set { _state.mutate { $0.isPinchToZoomEnabled = newValue } } + } + + @objc + public var isAutoZoomResetEnabled: Bool { + get { _state.isAutoZoomResetEnabled } + set { _state.mutate { $0.isAutoZoomResetEnabled = newValue } } + } + @objc public var isDebugMode: Bool { get { _state.isDebugMode } @@ -192,9 +204,12 @@ public class VideoView: NativeView, Loggable { var renderTarget: RenderTarget = .primary var isSwapping: Bool = false var remainingRenderCountBeforeSwap: Int = 0 // Number of frames to be rendered on secondary until swap is initiated - var transitionMode: TransitionMode = .flip + var transitionMode: TransitionMode = .crossDissolve var transitionDuration: TimeInterval = 0.3 + var isPinchToZoomEnabled: Bool = false + var isAutoZoomResetEnabled: Bool = true + // Only used for rendering local tracks var captureOptions: VideoCaptureOptions? = nil var captureDevice: AVCaptureDevice? = nil @@ -219,6 +234,12 @@ public class VideoView: NativeView, Loggable { private var _currentFPS: Int = 0 private var _frameCount: Int = 0 + #if os(iOS) || os(visionOS) + private lazy var _pinchGestureRecognizer = UIPinchGestureRecognizer(target: self, action: #selector(_handlePinchGesture(_:))) + // This should be thread safe so it's not required to be guarded by the lock + private var _pinchStartZoomFactor: CGFloat = 0.0 + #endif + override public init(frame: CGRect = .zero) { // initial state _state = StateSync(State(viewSize: frame.size)) @@ -326,6 +347,15 @@ public class VideoView: NativeView, Loggable { } } + #if os(iOS) || os(visionOS) + let newIsPinchToZoomEnabled = newState.isPinchToZoomEnabled + if newIsPinchToZoomEnabled != oldState.isPinchToZoomEnabled { + Task.detached { @MainActor in + self._pinchGestureRecognizer.isEnabled = newIsPinchToZoomEnabled + } + } + #endif + if newState.isDebugMode != oldState.isDebugMode { // fps timer if newState.isDebugMode { @@ -359,8 +389,44 @@ public class VideoView: NativeView, Loggable { await self._renderTimer.restart() } + + #if os(iOS) || os(visionOS) + // Add pinch gesture recognizer + addGestureRecognizer(_pinchGestureRecognizer) + _pinchGestureRecognizer.isEnabled = _state.isPinchToZoomEnabled + #endif } + #if os(iOS) || os(visionOS) + @objc func _handlePinchGesture(_ sender: UIPinchGestureRecognizer) { + if let track = _state.track as? LocalVideoTrack, + let capturer = track.capturer as? CameraCapturer, + let device = capturer.device + { + if sender.state == .began { + _pinchStartZoomFactor = device.videoZoomFactor + } else { + do { + try device.lockForConfiguration() + defer { device.unlockForConfiguration() } + + if sender.state == .changed { + let minZoom = device.minAvailableVideoZoomFactor + let maxZoom = device.maxAvailableVideoZoomFactor + device.videoZoomFactor = (_pinchStartZoomFactor * sender.scale).clamped(to: minZoom ... maxZoom) + } else if sender.state == .ended || sender.state == .cancelled, _state.isAutoZoomResetEnabled { + // Zoom to default zoom factor + let defaultZoomFactor = LKRTCCameraVideoCapturer.defaultZoomFactor(forDeviceType: device.deviceType) + device.ramp(toVideoZoomFactor: defaultZoomFactor, withRate: 32.0) + } + } catch { + log("Failed to adjust videoZoomFactor", .warning) + } + } + } + } + #endif + @available(*, unavailable) required init?(coder _: NSCoder) { fatalError("init(coder:) has not been implemented") @@ -527,7 +593,7 @@ private extension VideoView { func _shouldMirror() -> Bool { switch _state.mirrorMode { - case .auto: return _state.captureDevice?.realPosition == .front + case .auto: return _state.captureDevice?.facingPosition == .front case .off: return false case .mirror: return true } @@ -661,7 +727,7 @@ extension VideoView: VideoRenderer { // Currently only for iOS #if os(iOS) - let (mode, duration, position) = _state.read { ($0.transitionMode, $0.transitionDuration, $0.captureDevice?.realPosition) } + let (mode, duration, position) = _state.read { ($0.transitionMode, $0.transitionDuration, $0.captureDevice?.facingPosition) } if let transitionOption = mode.toAnimationOption(fromPosition: position) { UIView.transition(with: self, duration: duration, options: transitionOption, animations: block, completion: nil) } else { @@ -801,16 +867,6 @@ private extension VideoView { } } -extension AVCaptureDevice { - var realPosition: AVCaptureDevice.Position { - if deviceType == .builtInWideAngleCamera, position == .unspecified { - return .front - } - - return position - } -} - #if os(iOS) extension VideoView.TransitionMode { func toAnimationOption(fromPosition position: AVCaptureDevice.Position? = nil) -> UIView.AnimationOptions? {