diff --git a/Sources/LiveKit/Core/Engine+SignalClientDelegate.swift b/Sources/LiveKit/Core/Engine+SignalClientDelegate.swift index d2d40f763..7a0c5f284 100644 --- a/Sources/LiveKit/Core/Engine+SignalClientDelegate.swift +++ b/Sources/LiveKit/Core/Engine+SignalClientDelegate.swift @@ -96,7 +96,7 @@ extension Engine: SignalClientDelegate { func signalClient(_: SignalClient, didUpdateConnectionQuality _: [Livekit_ConnectionQualityInfo]) {} func signalClient(_: SignalClient, didUpdateRemoteMute _: String, muted _: Bool) {} func signalClient(_: SignalClient, didUpdateTrackStreamStates _: [Livekit_StreamStateInfo]) {} - func signalClient(_: SignalClient, didUpdateTrack _: String, subscribedQualities _: [Livekit_SubscribedQuality]) {} + func signalClient(_: SignalClient, didUpdateTrack _: String, subscribedQualities _: [Livekit_SubscribedQuality], subscribedCodecs _: [Livekit_SubscribedCodec]) {} func signalClient(_: SignalClient, didUpdateSubscriptionPermission _: Livekit_SubscriptionPermissionUpdate) {} func signalClient(_: SignalClient, didReceiveLeave _: Bool, reason _: Livekit_DisconnectReason) {} } diff --git a/Sources/LiveKit/Core/Engine+WebRTC.swift b/Sources/LiveKit/Core/Engine+WebRTC.swift index 28735633d..75efec9de 100644 --- a/Sources/LiveKit/Core/Engine+WebRTC.swift +++ b/Sources/LiveKit/Core/Engine+WebRTC.swift @@ -75,6 +75,9 @@ extension Engine { static let audioProcessingModule: LKRTCDefaultAudioProcessingModule = .init() + static let videoSenderCapabilities = peerConnectionFactory.rtpSenderCapabilities(for: .video) + static let audioSenderCapabilities = peerConnectionFactory.rtpSenderCapabilities(for: .audio) + static let peerConnectionFactory: LKRTCPeerConnectionFactory = { logger.log("Initializing SSL...", type: Engine.self) @@ -153,7 +156,8 @@ extension Engine { static func createRtpEncodingParameters(rid: String? = nil, encoding: MediaEncoding? = nil, scaleDownBy: Double? = nil, - active: Bool = true) -> LKRTCRtpEncodingParameters + active: Bool = true, + scalabilityMode: ScalabilityMode? = nil) -> LKRTCRtpEncodingParameters { let result = DispatchQueue.liveKitWebRTC.sync { LKRTCRtpEncodingParameters() } @@ -173,6 +177,10 @@ extension Engine { } } + if let scalabilityMode { + result.scalabilityMode = scalabilityMode.rawStringValue + } + return result } } diff --git a/Sources/LiveKit/Core/Room+SignalClientDelegate.swift b/Sources/LiveKit/Core/Room+SignalClientDelegate.swift index bb79938a4..433b92561 100644 --- a/Sources/LiveKit/Core/Room+SignalClientDelegate.swift +++ b/Sources/LiveKit/Core/Room+SignalClientDelegate.swift @@ -33,10 +33,35 @@ extension Room: SignalClientDelegate { } } - func signalClient(_: SignalClient, didUpdateTrack trackSid: String, subscribedQualities: [Livekit_SubscribedQuality]) { - log("qualities: \(subscribedQualities.map { String(describing: $0) }.joined(separator: ", "))") + func signalClient(_: SignalClient, didUpdateTrack trackSid: String, subscribedQualities qualities: [Livekit_SubscribedQuality], subscribedCodecs codecs: [Livekit_SubscribedCodec]) { + log("[Publish/Backup] Qualities: \(qualities.map { String(describing: $0) }.joined(separator: ", ")), Codecs: \(codecs.map { String(describing: $0) }.joined(separator: ", "))") - localParticipant.onSubscribedQualitiesUpdate(trackSid: trackSid, subscribedQualities: subscribedQualities) + guard let publication = localParticipant.getTrackPublication(sid: trackSid) else { + log("Received subscribed quality update for an unknown track", .warning) + return + } + + Task { + if !codecs.isEmpty { + guard let videoTrack = publication.track as? LocalVideoTrack else { return } + let missingSubscribedCodecs = try videoTrack._set(subscribedCodecs: codecs) + + if !missingSubscribedCodecs.isEmpty { + log("Missing codecs: \(missingSubscribedCodecs)") + for missingSubscribedCodec in missingSubscribedCodecs { + do { + log("Publishing additional codec: \(missingSubscribedCodec)") + try await localParticipant.publish(additionalVideoCodec: missingSubscribedCodec, for: publication) + } catch { + log("Failed publishing additional codec: \(missingSubscribedCodec), error: \(error)", .error) + } + } + } + + } else { + localParticipant._set(subscribedQualities: qualities, forTrackSid: trackSid) + } + } } func signalClient(_: SignalClient, didReceiveJoinResponse joinResponse: Livekit_JoinResponse) { diff --git a/Sources/LiveKit/Core/SignalClient.swift b/Sources/LiveKit/Core/SignalClient.swift index 0197d537b..099fe70a5 100644 --- a/Sources/LiveKit/Core/SignalClient.swift +++ b/Sources/LiveKit/Core/SignalClient.swift @@ -286,11 +286,11 @@ private extension SignalClient { notify { $0.signalClient(self, didUpdateTrackStreamStates: states.streamStates) } case let .subscribedQualityUpdate(update): - // ignore 0.15.1 - if latestJoinResponse?.serverVersion == "0.15.1" { - return - } - notify { $0.signalClient(self, didUpdateTrack: update.trackSid, subscribedQualities: update.subscribedQualities) } + notify { $0.signalClient(self, + didUpdateTrack: update.trackSid, + subscribedQualities: update.subscribedQualities, + subscribedCodecs: update.subscribedCodecs) } + case let .subscriptionPermissionUpdate(permissionUpdate): notify { $0.signalClient(self, didUpdateSubscriptionPermission: permissionUpdate) } case let .refreshToken(token): diff --git a/Sources/LiveKit/Extensions/CustomStringConvertible.swift b/Sources/LiveKit/Extensions/CustomStringConvertible.swift index 5581032dd..5fcfa0af1 100644 --- a/Sources/LiveKit/Extensions/CustomStringConvertible.swift +++ b/Sources/LiveKit/Extensions/CustomStringConvertible.swift @@ -63,6 +63,12 @@ extension Livekit_SubscribedQuality: CustomStringConvertible { } } +extension Livekit_SubscribedCodec: CustomStringConvertible { + public var description: String { + "SubscribedCodec(codec: \(codec), qualities: \(qualities.map { String(describing: $0) }.joined(separator: ", "))" + } +} + extension Livekit_ServerInfo: CustomStringConvertible { public var description: String { "ServerInfo(edition: \(edition), " + diff --git a/Sources/LiveKit/Extensions/LKRTCRtpSender.swift b/Sources/LiveKit/Extensions/LKRTCRtpSender.swift new file mode 100644 index 000000000..e7e471b90 --- /dev/null +++ b/Sources/LiveKit/Extensions/LKRTCRtpSender.swift @@ -0,0 +1,68 @@ +/* + * Copyright 2023 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Foundation + +@_implementationOnly import WebRTC + +extension LKRTCRtpSender: Loggable { + // ... + func _set(subscribedQualities qualities: [Livekit_SubscribedQuality]) { + let _parameters = parameters + let encodings = _parameters.encodings + + var didUpdate = false + + // For SVC mode... + if let firstEncoding = encodings.first, + let _ = ScalabilityMode.fromString(firstEncoding.scalabilityMode) + { + let _enabled = qualities.highest != .off + if firstEncoding.isActive != _enabled { + firstEncoding.isActive = _enabled + didUpdate = true + } + } else { + // For Simulcast... + for e in qualities { + guard let rid = e.quality.asRID else { continue } + guard let encodingforRID = encodings.first(where: { $0.rid == rid }) else { continue } + + if encodingforRID.isActive != e.enabled { + didUpdate = true + encodingforRID.isActive = e.enabled + log("Setting layer \(e.quality) to \(e.enabled)", .info) + } + } + + // Non simulcast streams don't have RIDs, handle here. + if encodings.count == 1, qualities.count >= 1 { + let firstEncoding = encodings.first! + let firstQuality = qualities.first! + + if firstEncoding.isActive != firstQuality.enabled { + didUpdate = true + firstEncoding.isActive = firstQuality.enabled + log("Setting layer \(firstQuality.quality) to \(firstQuality.enabled)", .info) + } + } + } + + if didUpdate { + parameters = _parameters + } + } +} diff --git a/Sources/LiveKit/Extensions/RTCRtpTransceiver.swift b/Sources/LiveKit/Extensions/RTCRtpTransceiver.swift new file mode 100644 index 000000000..9f5d7f2bc --- /dev/null +++ b/Sources/LiveKit/Extensions/RTCRtpTransceiver.swift @@ -0,0 +1,45 @@ +/* + * Copyright 2023 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Foundation + +@_implementationOnly import WebRTC + +extension LKRTCRtpTransceiver: Loggable { + /// Attempts to set preferred video codec. + func set(preferredVideoCodec codec: VideoCodec, exceptCodec: VideoCodec? = nil) { + // Get list of supported codecs... + let allVideoCodecs = Engine.videoSenderCapabilities.codecs + + // Get the RTCRtpCodecCapability of the preferred codec + let preferredCodecCapability = allVideoCodecs.first { $0.name.lowercased() == codec.id } + + // Get list of capabilities other than the preferred one + let otherCapabilities = allVideoCodecs.filter { + $0.name.lowercased() != codec.id && $0.name.lowercased() != exceptCodec?.id + } + + // Bring preferredCodecCapability to the front and combine all capabilities + let combinedCapabilities = [preferredCodecCapability] + otherCapabilities + + // Codecs not set in codecPreferences will not be negotiated in the offer + codecPreferences = combinedCapabilities.compactMap { $0 } + + log("codecPreferences set: \(codecPreferences.map { String(describing: $0) }.joined(separator: ", "))") + + assert(codecPreferences.first?.name.lowercased() == codec.id, "Preferred codec is not first in the list") + } +} diff --git a/Sources/LiveKit/Participant/LocalParticipant.swift b/Sources/LiveKit/Participant/LocalParticipant.swift index a47c9a47e..a6cffc978 100644 --- a/Sources/LiveKit/Participant/LocalParticipant.swift +++ b/Sources/LiveKit/Participant/LocalParticipant.swift @@ -87,9 +87,9 @@ public class LocalParticipant: Participant { let publishOptions = (publishOptions as? VideoPublishOptions) ?? self.room._state.options.defaultVideoPublishOptions - let encodings = Utils.computeEncodings(dimensions: dimensions, - publishOptions: publishOptions, - isScreenShare: track.source == .screenShareVideo) + let encodings = Utils.computeVideoEncodings(dimensions: dimensions, + publishOptions: publishOptions, + isScreenShare: track.source == .screenShareVideo) self.log("[publish] using encodings: \(encodings)") transInit.sendEncodings = encodings @@ -98,9 +98,29 @@ public class LocalParticipant: Participant { self.log("[publish] using layers: \(videoLayers.map { String(describing: $0) }.joined(separator: ", "))") + var simulcastCodecs: [Livekit_SimulcastCodec] = [ + // Always add first codec... + Livekit_SimulcastCodec.with { + $0.cid = track.mediaTrack.trackId + if let preferredCodec = publishOptions.preferredCodec { + $0.codec = preferredCodec.id + } + }, + ] + + if let backupCodec = publishOptions.preferredBackupCodec { + // Add backup codec to simulcast codecs... + let lkSimulcastCodec = Livekit_SimulcastCodec.with { + $0.cid = "" + $0.codec = backupCodec.id + } + simulcastCodecs.append(lkSimulcastCodec) + } + populator.width = UInt32(dimensions.width) populator.height = UInt32(dimensions.height) populator.layers = videoLayers + populator.simulcastCodecs = simulcastCodecs self.log("[publish] requesting add track to server with \(populator)...") @@ -146,6 +166,13 @@ public class LocalParticipant: Participant { track.set(transport: publisher, rtpSender: transceiver.sender) if track is LocalVideoTrack { + if let firstCodecMime = addTrackResult.trackInfo.codecs.first?.mimeType, + let firstVideoCodec = try? VideoCodec.from(mimeType: firstCodecMime) + { + log("[Publish] First video codec: \(firstVideoCodec)") + track._videoCodec = firstVideoCodec + } + let publishOptions = (publishOptions as? VideoPublishOptions) ?? room._state.options.defaultVideoPublishOptions // if screen share or simulcast is enabled, // degrade resolution by using server's layer switching logic instead of WebRTC's logic @@ -157,6 +184,10 @@ public class LocalParticipant: Participant { // and set it back to sender.parameters transceiver.sender.parameters = params } + + if let preferredCodec = publishOptions.preferredCodec { + transceiver.set(preferredVideoCodec: preferredCodec) + } } try await room.engine.publisherShouldNegotiate() @@ -249,7 +280,13 @@ public class LocalParticipant: Participant { } if let publisher = engine.publisher, let sender = track.rtpSender { + // Remove all simulcast senders... + for simulcastSender in track._simulcastRtpSenders.values { + try publisher.remove(track: simulcastSender) + } + // Remove main sender... try publisher.remove(track: sender) + // Mark re-negotiation required... try await engine.publisherShouldNegotiate() } @@ -350,55 +387,13 @@ public class LocalParticipant: Participant { trackPermissions: trackPermissions) } - func onSubscribedQualitiesUpdate(trackSid: String, subscribedQualities: [Livekit_SubscribedQuality]) { - if !room._state.options.dynacast { - return - } - + func _set(subscribedQualities qualities: [Livekit_SubscribedQuality], forTrackSid trackSid: String) { guard let pub = getTrackPublication(sid: trackSid), let track = pub.track as? LocalVideoTrack, let sender = track.rtpSender else { return } - let parameters = sender.parameters - let encodings = parameters.encodings - - var hasChanged = false - for quality in subscribedQualities { - var rid: String - switch quality.quality { - case Livekit_VideoQuality.high: rid = "f" - case Livekit_VideoQuality.medium: rid = "h" - case Livekit_VideoQuality.low: rid = "q" - default: continue - } - - guard let encoding = encodings.first(where: { $0.rid == rid }) else { - continue - } - - if encoding.isActive != quality.enabled { - hasChanged = true - encoding.isActive = quality.enabled - log("setting layer \(quality.quality) to \(quality.enabled)", .info) - } - } - - // Non simulcast streams don't have rids, handle here. - if encodings.count == 1, subscribedQualities.count >= 1 { - let encoding = encodings[0] - let quality = subscribedQualities[0] - - if encoding.isActive != quality.enabled { - hasChanged = true - encoding.isActive = quality.enabled - log("setting layer \(quality.quality) to \(quality.enabled)", .info) - } - } - - if hasChanged { - sender.parameters = parameters - } + sender._set(subscribedQualities: qualities) } override func set(permissions newValue: ParticipantPermissions) -> Bool { @@ -546,3 +541,88 @@ public extension LocalParticipant { return nil } } + +// MARK: - Simulcast codecs + +extension LocalParticipant { + // Publish additional (backup) codec when requested by server + func publish(additionalVideoCodec subscribedCodec: Livekit_SubscribedCodec, + for localTrackPublication: LocalTrackPublication) async throws + { + let videoCodec = try subscribedCodec.toVideoCodec() + + log("[Publish/Backup] Additional video codec: \(videoCodec)...") + + guard let track = localTrackPublication.track as? LocalVideoTrack else { + throw EngineError.state(message: "Track is nil") + } + + if !videoCodec.isBackup { + throw EngineError.state(message: "Attempted to publish a non-backup video codec as backup") + } + + let publisher = try room.engine.requirePublisher() + + let publishOptions = (track.publishOptions as? VideoPublishOptions) ?? room._state.options.defaultVideoPublishOptions + + // Should be already resolved... + let dimensions = try await track.capturer.dimensionsCompleter.wait() + + let encodings = Utils.computeVideoEncodings(dimensions: dimensions, + publishOptions: publishOptions, + overrideVideoCodec: videoCodec) + log("[Publish/Backup] Using encodings \(encodings)...") + + // Add transceiver first... + + let transInit = DispatchQueue.liveKitWebRTC.sync { LKRTCRtpTransceiverInit() } + transInit.direction = .sendOnly + transInit.sendEncodings = encodings + + // Add transceiver to publisher pc... + let transceiver = try publisher.addTransceiver(with: track.mediaTrack, transceiverInit: transInit) + log("[Publish] Added transceiver...") + + // Set codec... + transceiver.set(preferredVideoCodec: videoCodec) + + let sender = transceiver.sender + + // Request a new track to the server + let addTrackResult = try await room.engine.signalClient.sendAddTrack(cid: sender.senderId, + name: track.name, + type: track.kind.toPBType(), + source: track.source.toPBType()) + { + $0.sid = localTrackPublication.sid + $0.simulcastCodecs = [ + Livekit_SimulcastCodec.with { sc in + sc.cid = sender.senderId + sc.codec = videoCodec.id + }, + ] + + $0.layers = dimensions.videoLayers(for: encodings) + } + + log("[Publish] server responded trackInfo: \(addTrackResult.trackInfo)") + + sender._set(subscribedQualities: subscribedCodec.qualities) + + // Attach multi-codec sender... + track._simulcastRtpSenders[videoCodec] = sender + + try await room.engine.publisherShouldNegotiate() + } +} + +// MARK: - Helper + +extension [Livekit_SubscribedQuality] { + /// Find the highest quality in the array + var highest: Livekit_VideoQuality { + reduce(Livekit_VideoQuality.off) { maxQuality, subscribedQuality in + subscribedQuality.enabled && subscribedQuality.quality > maxQuality ? subscribedQuality.quality : maxQuality + } + } +} diff --git a/Sources/LiveKit/Protocols/SignalClientDelegate.swift b/Sources/LiveKit/Protocols/SignalClientDelegate.swift index a5faeb724..531615091 100644 --- a/Sources/LiveKit/Protocols/SignalClientDelegate.swift +++ b/Sources/LiveKit/Protocols/SignalClientDelegate.swift @@ -32,7 +32,9 @@ protocol SignalClientDelegate: AnyObject { func signalClient(_ signalClient: SignalClient, didUpdateConnectionQuality connectionQuality: [Livekit_ConnectionQualityInfo]) func signalClient(_ signalClient: SignalClient, didUpdateRemoteMute trackSid: String, muted: Bool) func signalClient(_ signalClient: SignalClient, didUpdateTrackStreamStates streamStates: [Livekit_StreamStateInfo]) - func signalClient(_ signalClient: SignalClient, didUpdateTrack trackSid: String, subscribedQualities qualities: [Livekit_SubscribedQuality]) + func signalClient(_ signalClient: SignalClient, didUpdateTrack trackSid: String, + subscribedQualities qualities: [Livekit_SubscribedQuality], + subscribedCodecs codecs: [Livekit_SubscribedCodec]) func signalClient(_ signalClient: SignalClient, didUpdateSubscriptionPermission permission: Livekit_SubscriptionPermissionUpdate) func signalClient(_ signalClient: SignalClient, didUpdateToken token: String) func signalClient(_ signalClient: SignalClient, didReceiveLeave canReconnect: Bool, reason: Livekit_DisconnectReason) diff --git a/Sources/LiveKit/Protocols/TrackDelegate.swift b/Sources/LiveKit/Protocols/TrackDelegate.swift index bd82d9063..abc93eeb5 100644 --- a/Sources/LiveKit/Protocols/TrackDelegate.swift +++ b/Sources/LiveKit/Protocols/TrackDelegate.swift @@ -37,8 +37,8 @@ public protocol TrackDelegate: AnyObject { func track(_ track: Track, didUpdate muted: Bool, shouldSendSignal: Bool) /// Statistics for the track has been generated (v2). - @objc(track:didUpdateStatistics:) optional - func track(_ track: Track, didUpdateStatistics: TrackStatistics) + @objc(track:didUpdateStatistics:simulcastStatistics:) optional + func track(_ track: Track, didUpdateStatistics: TrackStatistics, simulcastStatistics: [VideoCodec: TrackStatistics]) } protocol TrackDelegateInternal: TrackDelegate { diff --git a/Sources/LiveKit/Support/Utils.swift b/Sources/LiveKit/Support/Utils.swift index f30e8dd71..120052257 100644 --- a/Sources/LiveKit/Support/Utils.swift +++ b/Sources/LiveKit/Support/Utils.swift @@ -215,19 +215,30 @@ class Utils { } } - static func computeEncodings( + static func computeVideoEncodings( dimensions: Dimensions, publishOptions: VideoPublishOptions?, - isScreenShare: Bool = false + isScreenShare: Bool = false, + overrideVideoCodec: VideoCodec? = nil ) -> [LKRTCRtpEncodingParameters] { let publishOptions = publishOptions ?? VideoPublishOptions() let preferredEncoding: VideoEncoding? = isScreenShare ? publishOptions.screenShareEncoding : publishOptions.encoding let encoding = preferredEncoding ?? dimensions.computeSuggestedPreset(in: dimensions.computeSuggestedPresets(isScreenShare: isScreenShare)) - guard publishOptions.simulcast else { - return [Engine.createRtpEncodingParameters(encoding: encoding, scaleDownBy: 1)] + let videoCodec = overrideVideoCodec ?? publishOptions.preferredCodec + + if let videoCodec, videoCodec.isSVC { + // SVC mode + logger.log("Using SVC mode", type: Utils.self) + return [Engine.createRtpEncodingParameters(encoding: encoding, scalabilityMode: .L3T3_KEY)] + } else if !publishOptions.simulcast { + // Not-simulcast mode + logger.log("Simulcast not enabled", type: Utils.self) + return [Engine.createRtpEncodingParameters(encoding: encoding)] } + // Continue to simulcast encoding computation... + let baseParameters = VideoParameters(dimensions: dimensions, encoding: encoding) diff --git a/Sources/LiveKit/Track/Capturers/VideoCapturer.swift b/Sources/LiveKit/Track/Capturers/VideoCapturer.swift index 2918cb440..0e63af735 100644 --- a/Sources/LiveKit/Track/Capturers/VideoCapturer.swift +++ b/Sources/LiveKit/Track/Capturers/VideoCapturer.swift @@ -65,7 +65,7 @@ public class VideoCapturer: NSObject, Loggable, VideoCapturerProtocol { weak var delegate: LKRTCVideoCapturerDelegate? - let dimensionsCompleter = AsyncCompleter(label: "Dimensions", timeOut: .defaultCaptureStart) + let dimensionsCompleter = AsyncCompleter(label: "Dimensions", timeOut: .defaultCaptureStart) struct State: Equatable { // Counts calls to start/stopCapturer so multiple Tracks can use the same VideoCapturer. @@ -80,8 +80,12 @@ public class VideoCapturer: NSObject, Loggable, VideoCapturerProtocol { log("[publish] \(String(describing: oldValue)) -> \(String(describing: dimensions))") delegates.notify { $0.capturer?(self, didUpdate: self.dimensions) } - log("[publish] dimensions: \(String(describing: dimensions))") - dimensionsCompleter.resume(returning: dimensions) + if let dimensions { + log("[publish] dimensions: \(String(describing: dimensions))") + dimensionsCompleter.resume(returning: dimensions) + } else { + dimensionsCompleter.cancel() + } } } diff --git a/Sources/LiveKit/Track/Track.swift b/Sources/LiveKit/Track/Track.swift index b572c9d33..62854e7b8 100644 --- a/Sources/LiveKit/Track/Track.swift +++ b/Sources/LiveKit/Track/Track.swift @@ -84,6 +84,9 @@ public class Track: NSObject, Loggable { @objc public var statistics: TrackStatistics? { _state.statistics } + @objc + public var simulcastStatistics: [VideoCodec: TrackStatistics] { _state.simulcastStatistics } + /// Dimensions of the video (only if video track) @objc public var dimensions: Dimensions? { _state.dimensions } @@ -110,6 +113,9 @@ public class Track: NSObject, Loggable { private(set) var rtpSender: LKRTCRtpSender? private(set) var rtpReceiver: LKRTCRtpReceiver? + var _videoCodec: VideoCodec? + var _simulcastRtpSenders: [VideoCodec: LKRTCRtpSender] = [:] + // Weak reference to all VideoViews attached to this track. Must be accessed from main thread. var videoRenderers = NSHashTable.weakObjects() // internal var rtcVideoRenderers = NSHashTable.weakObjects() @@ -125,6 +131,7 @@ public class Track: NSObject, Loggable { var trackState: TrackState = .stopped var muted: Bool = false var statistics: TrackStatistics? + var simulcastStatistics: [VideoCodec: TrackStatistics] = [:] var reportStatistics: Bool = false } @@ -167,8 +174,10 @@ public class Track: NSObject, Loggable { } } - if newState.statistics != oldState.statistics, let statistics = newState.statistics { - self.delegates.notify { $0.track?(self, didUpdateStatistics: statistics) } + if newState.statistics != oldState.statistics || newState.simulcastStatistics != oldState.simulcastStatistics, + let statistics = newState.statistics + { + self.delegates.notify { $0.track?(self, didUpdateStatistics: statistics, simulcastStatistics: newState.simulcastStatistics) } } } @@ -443,6 +452,8 @@ extension Track { Task { defer { statisticsTimer.resume() } + // Main tatistics + var statisticsReport: LKRTCStatisticsReport? let prevStatistics = _state.read { $0.statistics } @@ -457,7 +468,20 @@ extension Track { let trackStatistics = TrackStatistics(from: Array(statisticsReport.statistics.values), prevStatistics: prevStatistics) - _state.mutate { $0.statistics = trackStatistics } + // Simulcast statistics + + let prevSimulcastStatistics = _state.read { $0.simulcastStatistics } + var _simulcastStatistics: [VideoCodec: TrackStatistics] = [:] + for _sender in _simulcastRtpSenders { + let _report = await transport.statistics(for: _sender.value) + _simulcastStatistics[_sender.key] = TrackStatistics(from: Array(_report.statistics.values), + prevStatistics: prevSimulcastStatistics[_sender.key]) + } + + _state.mutate { + $0.statistics = trackStatistics + $0.simulcastStatistics = _simulcastStatistics + } } } } diff --git a/Sources/LiveKit/Track/VideoTrack.swift b/Sources/LiveKit/Track/VideoTrack.swift index 6e44012a7..22ce0244c 100644 --- a/Sources/LiveKit/Track/VideoTrack.swift +++ b/Sources/LiveKit/Track/VideoTrack.swift @@ -33,3 +33,41 @@ protocol VideoTrack_Internal where Self: Track { func remove(rtcVideoRenderer: LKRTCVideoRenderer) } + +extension VideoTrack { + // Update a single SubscribedCodec + func _set(subscribedCodec: Livekit_SubscribedCodec) throws -> Bool { + // ... + let videoCodec = try VideoCodec.from(id: subscribedCodec.codec) + + // Check if main sender is sending the codec... + if let rtpSender, videoCodec == _videoCodec { + rtpSender._set(subscribedQualities: subscribedCodec.qualities) + return true + } + + // Find simulcast sender for codec... + if let rtpSender = _simulcastRtpSenders[videoCodec] { + rtpSender._set(subscribedQualities: subscribedCodec.qualities) + return true + } + + return false + } + + // Update an array of SubscribedCodecs + func _set(subscribedCodecs: [Livekit_SubscribedCodec]) throws -> [Livekit_SubscribedCodec] { + // ... + var missingCodecs: [Livekit_SubscribedCodec] = [] + + for subscribedCodec in subscribedCodecs { + let didUpdate = try _set(subscribedCodec: subscribedCodec) + if !didUpdate { + log("Sender for codec \(subscribedCodec.codec) not found", .info) + missingCodecs.append(subscribedCodec) + } + } + + return missingCodecs + } +} diff --git a/Sources/LiveKit/TrackPublications/LocalTrackPublication.swift b/Sources/LiveKit/TrackPublications/LocalTrackPublication.swift index 4f2efc371..221d779c6 100644 --- a/Sources/LiveKit/TrackPublications/LocalTrackPublication.swift +++ b/Sources/LiveKit/TrackPublications/LocalTrackPublication.swift @@ -114,9 +114,9 @@ extension LocalTrackPublication { let publishOptions = (track.publishOptions as? VideoPublishOptions) ?? participant.room._state.options.defaultVideoPublishOptions // re-compute encodings - let encodings = Utils.computeEncodings(dimensions: dimensions, - publishOptions: publishOptions, - isScreenShare: track.source == .screenShareVideo) + let encodings = Utils.computeVideoEncodings(dimensions: dimensions, + publishOptions: publishOptions, + isScreenShare: track.source == .screenShareVideo) log("Computed encodings: \(encodings)") diff --git a/Sources/LiveKit/Types/Dimensions.swift b/Sources/LiveKit/Types/Dimensions.swift index 72f5eae4d..821ad171c 100644 --- a/Sources/LiveKit/Types/Dimensions.swift +++ b/Sources/LiveKit/Types/Dimensions.swift @@ -135,7 +135,7 @@ extension Dimensions { func encodings(from presets: [VideoParameters?]) -> [LKRTCRtpEncodingParameters] { var result: [LKRTCRtpEncodingParameters] = [] for (index, preset) in presets.compactMap({ $0 }).enumerated() { - guard let rid = VideoQuality.rids[safe: index] else { + guard let rid = VideoQuality.RIDs[safe: index] else { continue } @@ -148,17 +148,31 @@ extension Dimensions { result.append(parameters) } - return VideoQuality.rids.compactMap { rid in result.first(where: { $0.rid == rid }) } + return VideoQuality.RIDs.compactMap { rid in result.first(where: { $0.rid == rid }) } } func videoLayers(for encodings: [LKRTCRtpEncodingParameters]) -> [Livekit_VideoLayer] { - encodings.filter(\.isActive).map { encoding in - let scaleDownBy = encoding.scaleResolutionDownBy?.doubleValue ?? 1.0 - return Livekit_VideoLayer.with { - $0.width = UInt32((Double(self.width) / scaleDownBy).rounded(.up)) - $0.height = UInt32((Double(self.height) / scaleDownBy).rounded(.up)) - $0.quality = Livekit_VideoQuality.from(rid: encoding.rid) - $0.bitrate = encoding.maxBitrateBps?.uint32Value ?? 0 + if let firstEncoding = encodings.first, + let scalabilityMode = ScalabilityMode.fromString(firstEncoding.scalabilityMode) + { + return (0 ... (scalabilityMode.spatial - 1)).map { idx in + Livekit_VideoLayer.with { + $0.width = UInt32((Double(width) / pow(2, Double(idx))).rounded(.down)) + $0.height = UInt32((Double(height) / pow(2, Double(idx))).rounded(.down)) + $0.quality = Livekit_VideoQuality(rawValue: scalabilityMode.spatial - idx - 1) ?? .off + $0.bitrate = UInt32((Double(truncating: firstEncoding.maxBitrateBps ?? 0) / pow(3, Double(idx))).rounded(.up)) + } + } + + } else { + return encodings.filter(\.isActive).map { encoding in + let scaleDownBy = encoding.scaleResolutionDownBy?.doubleValue ?? 1.0 + return Livekit_VideoLayer.with { + $0.width = UInt32((Double(width) / scaleDownBy).rounded(.down)) + $0.height = UInt32((Double(height) / scaleDownBy).rounded(.down)) + $0.quality = Livekit_VideoQuality.from(rid: encoding.rid) ?? .high + $0.bitrate = encoding.maxBitrateBps?.uint32Value ?? 0 + } } } } diff --git a/Sources/LiveKit/Types/Options/VideoPublishOptions.swift b/Sources/LiveKit/Types/Options/VideoPublishOptions.swift index b4dbb7538..7623fcb3a 100644 --- a/Sources/LiveKit/Types/Options/VideoPublishOptions.swift +++ b/Sources/LiveKit/Types/Options/VideoPublishOptions.swift @@ -39,12 +39,20 @@ public class VideoPublishOptions: NSObject, PublishOptions { @objc public let screenShareSimulcastLayers: [VideoParameters] + @objc + public let preferredCodec: VideoCodec? + + @objc + public let preferredBackupCodec: VideoCodec? + public init(name: String? = nil, encoding: VideoEncoding? = nil, screenShareEncoding: VideoEncoding? = nil, simulcast: Bool = true, simulcastLayers: [VideoParameters] = [], - screenShareSimulcastLayers: [VideoParameters] = []) + screenShareSimulcastLayers: [VideoParameters] = [], + preferredCodec: VideoCodec? = nil, + preferredBackupCodec: VideoCodec? = nil) { self.name = name self.encoding = encoding @@ -52,6 +60,8 @@ public class VideoPublishOptions: NSObject, PublishOptions { self.simulcast = simulcast self.simulcastLayers = simulcastLayers self.screenShareSimulcastLayers = screenShareSimulcastLayers + self.preferredCodec = preferredCodec + self.preferredBackupCodec = preferredBackupCodec } // MARK: - Equal @@ -63,7 +73,9 @@ public class VideoPublishOptions: NSObject, PublishOptions { screenShareEncoding == other.screenShareEncoding && simulcast == other.simulcast && simulcastLayers == other.simulcastLayers && - screenShareSimulcastLayers == other.screenShareSimulcastLayers + screenShareSimulcastLayers == other.screenShareSimulcastLayers && + preferredCodec == other.preferredCodec && + preferredBackupCodec == other.preferredBackupCodec } override public var hash: Int { @@ -74,6 +86,8 @@ public class VideoPublishOptions: NSObject, PublishOptions { hasher.combine(simulcast) hasher.combine(simulcastLayers) hasher.combine(screenShareSimulcastLayers) + hasher.combine(preferredCodec) + hasher.combine(preferredBackupCodec) return hasher.finalize() } } diff --git a/Sources/LiveKit/Types/ScalabilityMode.swift b/Sources/LiveKit/Types/ScalabilityMode.swift new file mode 100644 index 000000000..dcd92d827 --- /dev/null +++ b/Sources/LiveKit/Types/ScalabilityMode.swift @@ -0,0 +1,55 @@ +/* + * Copyright 2023 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Foundation + +@objc +public enum ScalabilityMode: Int { + case L3T3 = 1 + case L3T3_KEY = 2 + case L3T3_KEY_SHIFT = 3 +} + +public extension ScalabilityMode { + static func fromString(_ rawString: String?) -> ScalabilityMode? { + switch rawString { + case "L3T3": return .L3T3 + case "L3T3_KEY": return .L3T3_KEY + case "L3T3_KEY_SHIFT": return .L3T3_KEY_SHIFT + default: return nil + } + } + + var rawStringValue: String { + switch self { + case .L3T3: return "L3T3" + case .L3T3_KEY: return "L3T3_KEY" + case .L3T3_KEY_SHIFT: return "L3T3_KEY_SHIFT" + } + } + + var spatial: Int { 3 } + + var temporal: Int { 3 } +} + +// MARK: - CustomStringConvertible + +extension ScalabilityMode: CustomStringConvertible { + public var description: String { + "ScalabilityMode(\(rawStringValue))" + } +} diff --git a/Sources/LiveKit/Types/TrackStatistics.swift b/Sources/LiveKit/Types/TrackStatistics.swift index c67c3cbce..6ea82c461 100644 --- a/Sources/LiveKit/Types/TrackStatistics.swift +++ b/Sources/LiveKit/Types/TrackStatistics.swift @@ -104,7 +104,7 @@ public extension TrackStatistics { extension OutboundRtpStreamStatistics { /// Index of the rid. var ridIndex: Int { - guard let rid, let idx = VideoQuality.rids.firstIndex(of: rid) else { + guard let rid, let idx = VideoQuality.RIDs.firstIndex(of: rid) else { return -1 } return idx diff --git a/Sources/LiveKit/Types/VideoCodec.swift b/Sources/LiveKit/Types/VideoCodec.swift new file mode 100644 index 000000000..72c267cc5 --- /dev/null +++ b/Sources/LiveKit/Types/VideoCodec.swift @@ -0,0 +1,87 @@ +/* + * Copyright 2023 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Foundation + +@objc +public class VideoCodec: NSObject, Identifiable { + public static func from(id: String) throws -> VideoCodec { + // Try to find codec from id... + guard let codec = all.first(where: { $0.id == id }) else { + throw EngineError.state(message: "Failed to create VideoCodec from id") + } + + return codec + } + + public static func from(mimeType: String) throws -> VideoCodec { + let parts = mimeType.lowercased().split(separator: "/") + var id = String(parts.first!) + if parts.count > 1 { + if parts[0] != "video" { throw EngineError.state(message: "MIME type must be video") } + id = String(parts[1]) + } + return try from(id: id) + } + + public static let h264 = VideoCodec(id: "h264", backup: true) + public static let vp8 = VideoCodec(id: "vp8", backup: true) + public static let vp9 = VideoCodec(id: "vp9", isSVC: true) + public static let av1 = VideoCodec(id: "av1", isSVC: true) + + public static let all: [VideoCodec] = [.h264, .vp8, .vp9, .av1] + public static let allBackup: [VideoCodec] = [.h264, .vp8] + + // codec Id + public let id: String + // Whether the codec can be used as `backup` + public let isBackup: Bool + // Whether the codec can be used as `backup` + public let isSVC: Bool + + // Internal only + init(id: String, + backup: Bool = false, + isSVC: Bool = false) + { + self.id = id + isBackup = backup + self.isSVC = isSVC + } + + // MARK: - Equal + + override public func isEqual(_ object: Any?) -> Bool { + guard let other = object as? Self else { return false } + return id == other.id + } + + override public var hash: Int { + var hasher = Hasher() + hasher.combine(id) + return hasher.finalize() + } + + override public var description: String { + "VideoCodec(id: \(id))" + } +} + +extension Livekit_SubscribedCodec { + func toVideoCodec() throws -> VideoCodec { + try VideoCodec.from(id: codec) + } +} diff --git a/Sources/LiveKit/Types/VideoQuality.swift b/Sources/LiveKit/Types/VideoQuality.swift index 6744b819c..2a65ef375 100644 --- a/Sources/LiveKit/Types/VideoQuality.swift +++ b/Sources/LiveKit/Types/VideoQuality.swift @@ -23,9 +23,10 @@ enum VideoQuality { } extension VideoQuality { - static let rids = ["q", "h", "f"] + static let RIDs = ["q", "h", "f"] } +// Make convertible between protobuf type. extension VideoQuality { private static let toPBTypeMap: [VideoQuality: Livekit_VideoQuality] = [ .low: .low, @@ -38,12 +39,40 @@ extension VideoQuality { } } +// Make convertible between RIDs. extension Livekit_VideoQuality { - static func from(rid: String?) -> Livekit_VideoQuality { + static func from(rid: String?) -> Livekit_VideoQuality? { switch rid { - case "h": return Livekit_VideoQuality.medium case "q": return Livekit_VideoQuality.low - default: return Livekit_VideoQuality.high + case "h": return Livekit_VideoQuality.medium + case "f": return Livekit_VideoQuality.high + default: return nil + } + } + + var asRID: String? { + switch self { + case .low: return "q" + case .medium: return "h" + case .high: return "f" + default: return nil } } } + +// Make comparable by the real quality index since the raw protobuf values are not in order. +// E.g. value of `.off` is `3` which is larger than `.high`. +extension Livekit_VideoQuality: Comparable { + private var _weightIndex: Int { + switch self { + case .low: return 1 + case .medium: return 2 + case .high: return 3 + default: return 0 + } + } + + static func < (lhs: Livekit_VideoQuality, rhs: Livekit_VideoQuality) -> Bool { + lhs._weightIndex < rhs._weightIndex + } +}