Merge pull request #1370 from nextcloud/screensharing-2023

Add screensharing support

.pyspelling.wordlist.txt

@@ -11,3 +11,4 @@ ld
 listable
 lu
 decrypted
+screensharing

BroadcastUploadExtension/Atomic.swift

@@ -0,0 +1,39 @@
+//
+//  Atomic.swift
+//  Broadcast Extension
+//
+//  Created by Maksym Shcheglov.
+//  https://www.onswiftwings.com/posts/atomic-property-wrapper/
+//
+// From https://github.com/jitsi/jitsi-meet-sdk-samples (Apache 2.0 license)
+// SPDX-License-Identifier: Apache-2.0
+
+import Foundation
+
+@propertyWrapper
+struct Atomic<Value> {
+
+    private var value: Value
+    private let lock = NSLock()
+
+    init(wrappedValue value: Value) {
+        self.value = value
+    }
+
+    var wrappedValue: Value {
+        get { load() }
+        set { store(newValue: newValue) }
+    }
+
+    func load() -> Value {
+        lock.lock()
+        defer { lock.unlock() }
+        return value
+    }
+
+    mutating func store(newValue: Value) {
+        lock.lock()
+        defer { lock.unlock() }
+        value = newValue
+    }
+}

BroadcastUploadExtension/BroadcastUploadExtension.entitlements

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>com.apple.security.application-groups</key>
+	<array>
+		<string>group.com.nextcloud.Talk</string>
+	</array>
+</dict>
+</plist>

BroadcastUploadExtension/DarwinNotificationCenter.swift

@@ -0,0 +1,60 @@
+//
+//  DarwinNotificationCenter.swift
+//  Broadcast Extension
+//
+//  Created by Alex-Dan Bumbu on 23/03/2021.
+//  Copyright © 2021 8x8, Inc. All rights reserved.
+//
+// From https://github.com/jitsi/jitsi-meet-sdk-samples (Apache 2.0 license)
+// SPDX-License-Identifier: Apache-2.0
+
+import Foundation
+
+@objcMembers public class DarwinNotificationCenter: NSObject {
+
+    static let shared = DarwinNotificationCenter()
+    static let broadcastStartedNotification = "TalkiOS_BroadcastStarted"
+    static let broadcastStoppedNotification = "TalkiOS_BroadcastStopped"
+
+    private let notificationCenter: CFNotificationCenter
+    private var handlers: [String: [() -> Void]] = [:]
+
+    override init() {
+        notificationCenter = CFNotificationCenterGetDarwinNotifyCenter()
+    }
+
+    func postNotification(_ name: String) {
+        CFNotificationCenterPostNotification(notificationCenter, CFNotificationName(rawValue: name as CFString), nil, nil, true)
+    }
+
+    func addObserver(notificationName: String, completionBlock: @escaping () -> Void) {
+        if handlers[notificationName] == nil {
+            handlers[notificationName] = []
+        }
+
+        handlers[notificationName]?.append(completionBlock)
+        let observer = Unmanaged.passUnretained(self).toOpaque()
+
+        // see: https://stackoverflow.com/a/33262376
+        let callback: CFNotificationCallback = { _, observer, name, _, _ in
+            if let observer, let name {
+                // Extract pointer to `self` from void pointer:
+                let mySelf = Unmanaged<DarwinNotificationCenter>.fromOpaque(observer).takeUnretainedValue()
+
+                if let handlers = mySelf.handlers[name.rawValue as String] {
+                    for handler in handlers {
+                        handler()
+                    }
+                }
+            }
+        }
+
+        CFNotificationCenterAddObserver(self.notificationCenter, observer, callback, notificationName as CFString, nil, .coalesce)
+    }
+
+    func removeObserver(_ name: String) {
+        let observer = Unmanaged.passUnretained(self).toOpaque()
+        let name = CFNotificationName(rawValue: name as CFString)
+        CFNotificationCenterRemoveObserver(notificationCenter, observer, name, nil)
+    }
+}

BroadcastUploadExtension/Info.plist

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>NSExtension</key>
+	<dict>
+		<key>NSExtensionPointIdentifier</key>
+		<string>com.apple.broadcast-services-upload</string>
+		<key>NSExtensionPrincipalClass</key>
+		<string>$(PRODUCT_MODULE_NAME).SampleHandler</string>
+		<key>RPBroadcastProcessMode</key>
+		<string>RPBroadcastProcessModeSampleBuffer</string>
+	</dict>
+</dict>
+</plist>

BroadcastUploadExtension/SampleHandler.swift

@@ -0,0 +1,102 @@
+//
+//  SampleHandler.swift
+//  Broadcast Extension
+//
+//  Created by Alex-Dan Bumbu on 04.06.2021.
+//
+// From https://github.com/jitsi/jitsi-meet-sdk-samples (Apache 2.0 license)
+// SPDX-License-Identifier: Apache-2.0
+
+import ReplayKit
+
+class SampleHandler: RPBroadcastSampleHandler {
+
+    private var clientConnection: SocketConnection?
+    private var uploader: SampleUploader?
+
+    private var frameCount: Int = 0
+
+    var socketFilePath: String {
+        let sharedContainer = FileManager.default.containerURL(forSecurityApplicationGroupIdentifier: groupIdentifier)
+        return sharedContainer?.appendingPathComponent("rtc_SSFD").path ?? ""
+    }
+
+    override init() {
+      super.init()
+        if let connection = SocketConnection(filePath: socketFilePath) {
+          clientConnection = connection
+          setupConnection()
+
+          uploader = SampleUploader(connection: connection)
+        }
+    }
+
+    override func broadcastStarted(withSetupInfo setupInfo: [String: NSObject]?) {
+        // User has requested to start the broadcast. Setup info from the UI extension can be supplied but optional.
+        frameCount = 0
+
+        DarwinNotificationCenter.shared.postNotification(DarwinNotificationCenter.broadcastStartedNotification)
+        openConnection()
+    }
+
+    override func broadcastPaused() {
+        // User has requested to pause the broadcast. Samples will stop being delivered.
+    }
+
+    override func broadcastResumed() {
+        // User has requested to resume the broadcast. Samples delivery will resume.
+    }
+
+    override func broadcastFinished() {
+        // User has requested to finish the broadcast.
+        DarwinNotificationCenter.shared.postNotification(DarwinNotificationCenter.broadcastStoppedNotification)
+        clientConnection?.close()
+    }
+
+    override func processSampleBuffer(_ sampleBuffer: CMSampleBuffer, with sampleBufferType: RPSampleBufferType) {
+        switch sampleBufferType {
+        case RPSampleBufferType.video:
+            // very simple mechanism for adjusting frame rate by using every third frame
+            frameCount += 1
+            if frameCount % 2 == 0 {
+                uploader?.send(sample: sampleBuffer)
+            }
+        default:
+            break
+        }
+    }
+}
+
+private extension SampleHandler {
+
+    func setupConnection() {
+        clientConnection?.didClose = { [weak self] error in
+            print("client connection did close \(String(describing: error))")
+
+            if let error = error {
+                self?.finishBroadcastWithError(error)
+            } else {
+                // the displayed failure message is more user friendly when using NSError instead of Error
+                let JMScreenSharingStopped = 10001
+                let localizedError = NSLocalizedString("Screensharing stopped", comment: "")
+                let customError = NSError(domain: RPRecordingErrorDomain, code: JMScreenSharingStopped, userInfo: [NSLocalizedDescriptionKey: localizedError])
+                self?.finishBroadcastWithError(customError)
+            }
+        }
+    }
+
+    func openConnection() {
+        let queue = DispatchQueue(label: "broadcast.connectTimer")
+        let timer = DispatchSource.makeTimerSource(queue: queue)
+        timer.schedule(deadline: .now(), repeating: .milliseconds(100), leeway: .milliseconds(500))
+        timer.setEventHandler { [weak self] in
+            guard self?.clientConnection?.open() == true else {
+                return
+            }
+
+            timer.cancel()
+        }
+
+        timer.resume()
+    }
+}

BroadcastUploadExtension/SampleUploader.swift

@@ -0,0 +1,148 @@
+//
+//  SampleUploader.swift
+//  Broadcast Extension
+//
+//  Created by Alex-Dan Bumbu on 22/03/2021.
+//  Copyright © 2021 8x8, Inc. All rights reserved.
+//
+// From https://github.com/jitsi/jitsi-meet-sdk-samples (Apache 2.0 license)
+// SPDX-License-Identifier: Apache-2.0
+
+import Foundation
+import ReplayKit
+
+private enum Constants {
+    static let bufferMaxLength = 10240
+}
+
+class SampleUploader {
+
+    private static var imageContext = CIContext(options: nil)
+
+    @Atomic private var isReady = false
+    private var connection: SocketConnection
+
+    private var dataToSend: Data?
+    private var byteIndex = 0
+
+    private let serialQueue: DispatchQueue
+
+    init(connection: SocketConnection) {
+        self.connection = connection
+        self.serialQueue = DispatchQueue(label: "talk.broadcast.sampleUploader")
+
+        setupConnection()
+    }
+
+    @discardableResult func send(sample buffer: CMSampleBuffer) -> Bool {
+        guard isReady else {
+            return false
+        }
+
+        isReady = false
+
+        dataToSend = prepare(sample: buffer)
+        byteIndex = 0
+
+        serialQueue.async { [weak self] in
+            self?.sendDataChunk()
+        }
+
+        return true
+    }
+}
+
+private extension SampleUploader {
+
+    func setupConnection() {
+        connection.didOpen = { [weak self] in
+            self?.isReady = true
+        }
+        connection.streamHasSpaceAvailable = { [weak self] in
+            self?.serialQueue.async {
+                if let success = self?.sendDataChunk() {
+                    self?.isReady = !success
+                }
+            }
+        }
+    }
+
+    @discardableResult func sendDataChunk() -> Bool {
+        guard let dataToSend = dataToSend else {
+            return false
+        }
+
+        var bytesLeft = dataToSend.count - byteIndex
+        var length = bytesLeft > Constants.bufferMaxLength ? Constants.bufferMaxLength : bytesLeft
+
+        length = dataToSend[byteIndex..<(byteIndex + length)].withUnsafeBytes {
+            guard let ptr = $0.bindMemory(to: UInt8.self).baseAddress else {
+                return 0
+            }
+
+            return connection.writeToStream(buffer: ptr, maxLength: length)
+        }
+
+        if length > 0 {
+            byteIndex += length
+            bytesLeft -= length
+
+            if bytesLeft == 0 {
+                self.dataToSend = nil
+                byteIndex = 0
+            }
+        } else {
+            print("writeBufferToStream failure")
+        }
+
+        return true
+    }
+
+    func prepare(sample buffer: CMSampleBuffer) -> Data? {
+        guard let imageBuffer = CMSampleBufferGetImageBuffer(buffer) else {
+            print("image buffer not available")
+            return nil
+        }
+
+        CVPixelBufferLockBaseAddress(imageBuffer, .readOnly)
+
+        let scaleFactor = 2.0
+        let width = CVPixelBufferGetWidth(imageBuffer)/Int(scaleFactor)
+        let height = CVPixelBufferGetHeight(imageBuffer)/Int(scaleFactor)
+        let orientation = CMGetAttachment(buffer, key: RPVideoSampleOrientationKey as CFString, attachmentModeOut: nil)?.uintValue ?? 0
+
+        let scaleTransform = CGAffineTransform(scaleX: CGFloat(1.0/scaleFactor), y: CGFloat(1.0/scaleFactor))
+        let bufferData = self.jpegData(from: imageBuffer, scale: scaleTransform)
+
+        CVPixelBufferUnlockBaseAddress(imageBuffer, .readOnly)
+
+        guard let messageData = bufferData else {
+            print("corrupted image buffer")
+            return nil
+        }
+
+        let httpResponse = CFHTTPMessageCreateResponse(nil, 200, nil, kCFHTTPVersion1_1).takeRetainedValue()
+        CFHTTPMessageSetHeaderFieldValue(httpResponse, "Content-Length" as CFString, String(messageData.count) as CFString)
+        CFHTTPMessageSetHeaderFieldValue(httpResponse, "Buffer-Width" as CFString, String(width) as CFString)
+        CFHTTPMessageSetHeaderFieldValue(httpResponse, "Buffer-Height" as CFString, String(height) as CFString)
+        CFHTTPMessageSetHeaderFieldValue(httpResponse, "Buffer-Orientation" as CFString, String(orientation) as CFString)
+
+        CFHTTPMessageSetBody(httpResponse, messageData as CFData)
+
+        let serializedMessage = CFHTTPMessageCopySerializedMessage(httpResponse)?.takeRetainedValue() as Data?
+
+        return serializedMessage
+    }
+
+    func jpegData(from buffer: CVPixelBuffer, scale scaleTransform: CGAffineTransform) -> Data? {
+        let image = CIImage(cvPixelBuffer: buffer).transformed(by: scaleTransform)
+
+        guard let colorSpace = image.colorSpace else {
+            return nil
+        }
+
+        let options: [CIImageRepresentationOption: Float] = [kCGImageDestinationLossyCompressionQuality as CIImageRepresentationOption: 1.0]
+
+        return SampleUploader.imageContext.jpegRepresentation(of: image, colorSpace: colorSpace, options: options)
+    }
+}