From 4d8d4188a7061a647550d4605604a2404e902de2 Mon Sep 17 00:00:00 2001 From: Ali <> Date: Tue, 16 Jun 2020 11:22:27 +0400 Subject: [PATCH] Temp --- .../Sources/PresentationCallManager.swift | 3 +- .../Sources/CallControllerNode.swift | 43 +- .../Sources/PresentationCall.swift | 8 +- .../Sources/OngoingCallContext.swift | 17 +- submodules/TgVoipWebrtc/BUILD | 10 +- submodules/TgVoipWebrtc/Impl/CodecsApple.h | 27 ++ submodules/TgVoipWebrtc/Impl/CodecsApple.mm | 169 ++++++++ submodules/TgVoipWebrtc/Impl/Controller.h | 8 +- submodules/TgVoipWebrtc/Impl/Controller.mm | 17 +- submodules/TgVoipWebrtc/Impl/Manager.cpp | 116 ++++++ submodules/TgVoipWebrtc/Impl/Manager.h | 45 +++ .../TgVoipWebrtc/Impl/MediaEngineWebrtc.mm | 1 - submodules/TgVoipWebrtc/Impl/MediaManager.cpp | 377 ++++++++++++++++++ submodules/TgVoipWebrtc/Impl/MediaManager.h | 100 +++++ .../TgVoipWebrtc/Impl/NetworkManager.cpp | 185 +++++++++ submodules/TgVoipWebrtc/Impl/NetworkManager.h | 77 ++++ submodules/TgVoipWebrtc/Impl/TgVoip.h | 24 +- submodules/TgVoipWebrtc/Impl/TgVoip.mm | 178 +++------ .../TgVoipWebrtc/Impl/ThreadLocalObject.cpp | 1 + .../TgVoipWebrtc/Impl/ThreadLocalObject.h | 64 +++ submodules/TgVoipWebrtc/Impl/VideoMetalView.h | 6 +- .../TgVoipWebrtc/Impl/VideoMetalView.mm | 71 ++-- .../TgVoip/OngoingCallThreadLocalContext.h | 3 +- .../Sources/OngoingCallThreadLocalContext.mm | 97 ++--- 24 files changed, 1391 insertions(+), 256 deletions(-) create mode 100644 submodules/TgVoipWebrtc/Impl/CodecsApple.h create mode 100644 submodules/TgVoipWebrtc/Impl/CodecsApple.mm create mode 100644 submodules/TgVoipWebrtc/Impl/Manager.cpp create mode 100644 submodules/TgVoipWebrtc/Impl/Manager.h create mode 100644 submodules/TgVoipWebrtc/Impl/MediaManager.cpp create mode 100644 submodules/TgVoipWebrtc/Impl/MediaManager.h create mode 100644 submodules/TgVoipWebrtc/Impl/NetworkManager.cpp create mode 100644 submodules/TgVoipWebrtc/Impl/NetworkManager.h create mode 100644 submodules/TgVoipWebrtc/Impl/ThreadLocalObject.cpp create mode 100644 submodules/TgVoipWebrtc/Impl/ThreadLocalObject.h diff --git a/submodules/AccountContext/Sources/PresentationCallManager.swift b/submodules/AccountContext/Sources/PresentationCallManager.swift index 0f179f4dc5..2f9bbae345 100644 --- a/submodules/AccountContext/Sources/PresentationCallManager.swift +++ b/submodules/AccountContext/Sources/PresentationCallManager.swift @@ -47,7 +47,8 @@ public protocol PresentationCall: class { func setCurrentAudioOutput(_ output: AudioSessionOutput) func debugInfo() -> Signal<(String, String), NoError> - func getVideoView(completion: @escaping (UIView?) -> Void) + func makeIncomingVideoView(completion: @escaping (UIView?) -> Void) + func makeOutgoingVideoView(completion: @escaping (UIView?) -> Void) } public protocol PresentationCallManager: class { diff --git a/submodules/TelegramCallsUI/Sources/CallControllerNode.swift b/submodules/TelegramCallsUI/Sources/CallControllerNode.swift index 13dfb0591e..f75ec12bd5 100644 --- a/submodules/TelegramCallsUI/Sources/CallControllerNode.swift +++ b/submodules/TelegramCallsUI/Sources/CallControllerNode.swift @@ -31,8 +31,9 @@ final class CallControllerNode: ASDisplayNode { private let imageNode: TransformImageNode private let dimNode: ASDisplayNode - private var videoView: UIView? - private var videoViewRequested: Bool = false + private var incomingVideoView: UIView? + private var outgoingVideoView: UIView? + private var videoViewsRequested: Bool = false private let backButtonArrowNode: ASImageNode private let backButtonNode: HighlightableButtonNode private let statusNode: CallControllerStatusNode @@ -265,16 +266,34 @@ final class CallControllerNode: ASDisplayNode { } } statusReception = reception - if !self.videoViewRequested { - self.videoViewRequested = true - self.call.getVideoView(completion: { [weak self] videoView in + if !self.videoViewsRequested { + self.videoViewsRequested = true + self.call.makeIncomingVideoView(completion: { [weak self] incomingVideoView in guard let strongSelf = self else { return } - if let videoView = videoView { + if let incomingVideoView = incomingVideoView { strongSelf.setCurrentAudioOutput?(.speaker) - strongSelf.videoView = videoView - strongSelf.containerNode.view.insertSubview(videoView, aboveSubview: strongSelf.dimNode.view) + strongSelf.incomingVideoView = incomingVideoView + strongSelf.containerNode.view.insertSubview(incomingVideoView, aboveSubview: strongSelf.dimNode.view) + if let (layout, navigationBarHeight) = strongSelf.validLayout { + strongSelf.containerLayoutUpdated(layout, navigationBarHeight: navigationBarHeight, transition: .immediate) + } + } + }) + + self.call.makeOutgoingVideoView(completion: { [weak self] outgoingVideoView in + guard let strongSelf = self else { + return + } + if let outgoingVideoView = outgoingVideoView { + strongSelf.setCurrentAudioOutput?(.speaker) + strongSelf.outgoingVideoView = outgoingVideoView + if let incomingVideoView = strongSelf.incomingVideoView { + strongSelf.containerNode.view.insertSubview(outgoingVideoView, aboveSubview: incomingVideoView) + } else { + strongSelf.containerNode.view.insertSubview(outgoingVideoView, aboveSubview: strongSelf.dimNode.view) + } if let (layout, navigationBarHeight) = strongSelf.validLayout { strongSelf.containerLayoutUpdated(layout, navigationBarHeight: navigationBarHeight, transition: .immediate) } @@ -388,8 +407,12 @@ final class CallControllerNode: ASDisplayNode { transition.updateFrame(node: self.containerNode, frame: CGRect(origin: CGPoint(), size: layout.size)) transition.updateFrame(node: self.dimNode, frame: CGRect(origin: CGPoint(), size: layout.size)) - if let videoView = self.videoView { - videoView.frame = CGRect(origin: CGPoint(), size: layout.size) + if let incomingVideoView = self.incomingVideoView { + incomingVideoView.frame = CGRect(origin: CGPoint(), size: layout.size) + } + if let outgoingVideoView = self.outgoingVideoView { + let outgoingSize = layout.size.aspectFitted(CGSize(width: 320.0, height: 320.0)) + outgoingVideoView.frame = CGRect(origin: CGPoint(x: layout.size.width - 16.0 - outgoingSize.width, y: layout.size.height - 16.0 - outgoingSize.height), size: outgoingSize) } if let keyPreviewNode = self.keyPreviewNode { diff --git a/submodules/TelegramCallsUI/Sources/PresentationCall.swift b/submodules/TelegramCallsUI/Sources/PresentationCall.swift index 24aac90b50..551e24666b 100644 --- a/submodules/TelegramCallsUI/Sources/PresentationCall.swift +++ b/submodules/TelegramCallsUI/Sources/PresentationCall.swift @@ -673,7 +673,11 @@ public final class PresentationCallImpl: PresentationCall { return self.debugInfoValue.get() } - public func getVideoView(completion: @escaping (UIView?) -> Void) { - self.ongoingContext?.getVideoView(completion: completion) + public func makeIncomingVideoView(completion: @escaping (UIView?) -> Void) { + self.ongoingContext?.makeIncomingVideoView(completion: completion) + } + + public func makeOutgoingVideoView(completion: @escaping (UIView?) -> Void) { + self.ongoingContext?.makeOutgoingVideoView(completion: completion) } } diff --git a/submodules/TelegramVoip/Sources/OngoingCallContext.swift b/submodules/TelegramVoip/Sources/OngoingCallContext.swift index a1080aa44d..75b1548b95 100644 --- a/submodules/TelegramVoip/Sources/OngoingCallContext.swift +++ b/submodules/TelegramVoip/Sources/OngoingCallContext.swift @@ -585,12 +585,23 @@ public final class OngoingCallContext { return (poll |> then(.complete() |> delay(0.5, queue: Queue.concurrentDefaultQueue()))) |> restart } - public func getVideoView(completion: @escaping (UIView?) -> Void) { + public func makeIncomingVideoView(completion: @escaping (UIView?) -> Void) { self.withContext { context in if let context = context as? OngoingCallThreadLocalContextWebrtc { - context.getRemoteCameraView(completion) + context.makeIncomingVideoView(completion) + } else { + completion(nil) + } + } + } + + public func makeOutgoingVideoView(completion: @escaping (UIView?) -> Void) { + self.withContext { context in + if let context = context as? OngoingCallThreadLocalContextWebrtc { + context.makeOutgoingVideoView(completion) + } else { + completion(nil) } - completion(nil) } } } diff --git a/submodules/TgVoipWebrtc/BUILD b/submodules/TgVoipWebrtc/BUILD index 1b7572c72f..199e7c4cfd 100644 --- a/submodules/TgVoipWebrtc/BUILD +++ b/submodules/TgVoipWebrtc/BUILD @@ -12,10 +12,10 @@ objc_library( "Sources/**/*.m", "Sources/**/*.mm", "Sources/**/*.h", - "Impl/*.h", - "Impl/*.cpp", - "Impl/*.mm", - "Impl/*.m", + "Impl/**/*.h", + "Impl/**/*.cpp", + "Impl/**/*.mm", + "Impl/**/*.m", ]), hdrs = glob([ "PublicHeaders/**/*.h", @@ -30,6 +30,8 @@ objc_library( "-DWEBRTC_IOS", "-DWEBRTC_MAC", "-DWEBRTC_POSIX", + "-DRTC_ENABLE_VP9", + "-DTGVOIP_NAMESPACE=tgvoip_webrtc", "-std=c++14", ], includes = [ diff --git a/submodules/TgVoipWebrtc/Impl/CodecsApple.h b/submodules/TgVoipWebrtc/Impl/CodecsApple.h new file mode 100644 index 0000000000..260ccfb9c6 --- /dev/null +++ b/submodules/TgVoipWebrtc/Impl/CodecsApple.h @@ -0,0 +1,27 @@ +#ifndef CODECS_APPLE_H +#define CODECS_APPLE_H + +#include "rtc_base/thread.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "api/media_stream_interface.h" + +#ifdef TGVOIP_NAMESPACE +namespace TGVOIP_NAMESPACE { +#endif + +class VideoCapturerInterface { +public: + virtual ~VideoCapturerInterface(); +}; + +std::unique_ptr makeVideoEncoderFactory(); +std::unique_ptr makeVideoDecoderFactory(); +rtc::scoped_refptr makeVideoSource(rtc::Thread *signalingThread, rtc::Thread *workerThread); +std::unique_ptr makeVideoCapturer(rtc::scoped_refptr source); + +#ifdef TGVOIP_NAMESPACE +} +#endif + +#endif diff --git a/submodules/TgVoipWebrtc/Impl/CodecsApple.mm b/submodules/TgVoipWebrtc/Impl/CodecsApple.mm new file mode 100644 index 0000000000..cfc67fee3d --- /dev/null +++ b/submodules/TgVoipWebrtc/Impl/CodecsApple.mm @@ -0,0 +1,169 @@ +#import "CodecsApple.h" + +#include "absl/strings/match.h" +#include "api/audio_codecs/audio_decoder_factory_template.h" +#include "api/audio_codecs/audio_encoder_factory_template.h" +#include "api/audio_codecs/opus/audio_decoder_opus.h" +#include "api/audio_codecs/opus/audio_encoder_opus.h" +#include "api/rtp_parameters.h" +#include "api/task_queue/default_task_queue_factory.h" +#include "media/base/codec.h" +#include "media/base/media_constants.h" +#include "media/engine/webrtc_media_engine.h" +#include "modules/audio_device/include/audio_device_default.h" +#include "rtc_base/task_utils/repeating_task.h" +#include "system_wrappers/include/field_trial.h" +#include "api/video/builtin_video_bitrate_allocator_factory.h" +#include "api/video/video_bitrate_allocation.h" + +#include "sdk/objc/components/video_codec/RTCVideoEncoderFactoryH264.h" +#include "sdk/objc/components/video_codec/RTCVideoDecoderFactoryH264.h" +#include "sdk/objc/components/video_codec/RTCDefaultVideoEncoderFactory.h" +#include "sdk/objc/components/video_codec/RTCDefaultVideoDecoderFactory.h" +#include "sdk/objc/native/api/video_encoder_factory.h" +#include "sdk/objc/native/api/video_decoder_factory.h" + +#include "sdk/objc/native/src/objc_video_track_source.h" +#include "api/video_track_source_proxy.h" +#include "sdk/objc/api/RTCVideoRendererAdapter.h" +#include "sdk/objc/native/api/video_frame.h" +#include "api/media_types.h" + +#import "VideoCameraCapturer.h" + +@interface VideoCapturerInterfaceImplReference : NSObject { + VideoCameraCapturer *_videoCapturer; +} + +@end + +@implementation VideoCapturerInterfaceImplReference + +- (instancetype)initWithSource:(rtc::scoped_refptr)source { + self = [super init]; + if (self != nil) { + assert([NSThread isMainThread]); + + _videoCapturer = [[VideoCameraCapturer alloc] initWithSource:source]; + + AVCaptureDevice *frontCamera = nil; + for (AVCaptureDevice *device in [VideoCameraCapturer captureDevices]) { + if (device.position == AVCaptureDevicePositionFront) { + frontCamera = device; + break; + } + } + + if (frontCamera == nil) { + return nil; + } + + NSArray *sortedFormats = [[VideoCameraCapturer supportedFormatsForDevice:frontCamera] sortedArrayUsingComparator:^NSComparisonResult(AVCaptureDeviceFormat* lhs, AVCaptureDeviceFormat *rhs) { + int32_t width1 = CMVideoFormatDescriptionGetDimensions(lhs.formatDescription).width; + int32_t width2 = CMVideoFormatDescriptionGetDimensions(rhs.formatDescription).width; + return width1 < width2 ? NSOrderedAscending : NSOrderedDescending; + }]; + + AVCaptureDeviceFormat *bestFormat = nil; + for (AVCaptureDeviceFormat *format in sortedFormats) { + CMVideoDimensions dimensions = CMVideoFormatDescriptionGetDimensions(format.formatDescription); + if (dimensions.width >= 1000 || dimensions.height >= 1000) { + bestFormat = format; + break; + } + } + + if (bestFormat == nil) { + assert(false); + return nil; + } + + AVFrameRateRange *frameRateRange = [[bestFormat.videoSupportedFrameRateRanges sortedArrayUsingComparator:^NSComparisonResult(AVFrameRateRange *lhs, AVFrameRateRange *rhs) { + if (lhs.maxFrameRate < rhs.maxFrameRate) { + return NSOrderedAscending; + } else { + return NSOrderedDescending; + } + }] lastObject]; + + if (frameRateRange == nil) { + assert(false); + return nil; + } + + [_videoCapturer startCaptureWithDevice:frontCamera format:bestFormat fps:27]; + } + return self; +} + +- (void)dealloc { + assert([NSThread isMainThread]); +} + +@end + +@interface VideoCapturerInterfaceImplHolder : NSObject + +@property (nonatomic) void *reference; + +@end + +@implementation VideoCapturerInterfaceImplHolder + +@end + +#ifdef TGVOIP_NAMESPACE +namespace TGVOIP_NAMESPACE { +#endif + +class VideoCapturerInterfaceImpl: public VideoCapturerInterface { +public: + VideoCapturerInterfaceImpl(rtc::scoped_refptr source) : + _source(source) { + _implReference = [[VideoCapturerInterfaceImplHolder alloc] init]; + VideoCapturerInterfaceImplHolder *implReference = _implReference; + dispatch_async(dispatch_get_main_queue(), ^{ + VideoCapturerInterfaceImplReference *value = [[VideoCapturerInterfaceImplReference alloc] initWithSource:source]; + if (value != nil) { + implReference.reference = (void *)CFBridgingRetain(value); + } + }); + } + + virtual ~VideoCapturerInterfaceImpl() { + VideoCapturerInterfaceImplHolder *implReference = _implReference; + dispatch_async(dispatch_get_main_queue(), ^{ + if (implReference.reference != nil) { + CFBridgingRelease(implReference.reference); + } + }); + } + +private: + rtc::scoped_refptr _source; + VideoCapturerInterfaceImplHolder *_implReference; +}; + +VideoCapturerInterface::~VideoCapturerInterface() { +} + +std::unique_ptr makeVideoEncoderFactory() { + return webrtc::ObjCToNativeVideoEncoderFactory([[RTCDefaultVideoEncoderFactory alloc] init]); +} + +std::unique_ptr makeVideoDecoderFactory() { + return webrtc::ObjCToNativeVideoDecoderFactory([[RTCDefaultVideoDecoderFactory alloc] init]); +} + +rtc::scoped_refptr makeVideoSource(rtc::Thread *signalingThread, rtc::Thread *workerThread) { + rtc::scoped_refptr objCVideoTrackSource(new rtc::RefCountedObject()); + return webrtc::VideoTrackSourceProxy::Create(signalingThread, workerThread, objCVideoTrackSource); +} + +std::unique_ptr makeVideoCapturer(rtc::scoped_refptr source) { + return std::make_unique(source); +} + +#ifdef TGVOIP_NAMESPACE +} +#endif diff --git a/submodules/TgVoipWebrtc/Impl/Controller.h b/submodules/TgVoipWebrtc/Impl/Controller.h index ac22631d8e..a1d7309390 100644 --- a/submodules/TgVoipWebrtc/Impl/Controller.h +++ b/submodules/TgVoipWebrtc/Impl/Controller.h @@ -53,13 +53,7 @@ private: webrtc::RepeatingTaskHandle repeatable; int64_t last_recv_time; int64_t last_send_time; - const bool is_outgoing; - const size_t init_timeout; - const size_t reconnect_timeout; - bool local_datasaving; - bool final_datasaving; - //message::NetworkType local_network_type; - //message::NetworkType final_network_type; + const bool isOutgoing; void PacketReceived(const rtc::CopyOnWriteBuffer &); void WriteableStateChanged(bool); diff --git a/submodules/TgVoipWebrtc/Impl/Controller.mm b/submodules/TgVoipWebrtc/Impl/Controller.mm index 8a1a0a1c3f..a426732710 100644 --- a/submodules/TgVoipWebrtc/Impl/Controller.mm +++ b/submodules/TgVoipWebrtc/Impl/Controller.mm @@ -1,30 +1,16 @@ #include "Controller.h" #include "modules/rtp_rtcp/source/rtp_utility.h" -#include "rtc_base/time_utils.h" -#include "rtc_base/message_handler.h" #include -/*std::map Controller::network_params = { - {message::NetworkType::nGprs, {6, 8, 6, 120, false, false, false}}, - {message::NetworkType::nEdge, {6, 16, 6, 120, false, false, false}}, - {message::NetworkType::n3gOrAbove, {6, 32, 16, 60, false, false, false}}, -}; -MediaEngineWebrtc::NetworkParams Controller::default_network_params = {6, 32, 16, 30, false, false, false}; -MediaEngineWebrtc::NetworkParams Controller::datasaving_network_params = {6, 8, 6, 120, false, false, true};*/ - Controller::Controller(bool is_outgoing, size_t init_timeout, size_t reconnect_timeout) : thread(rtc::Thread::Create()) , connector(std::make_unique(is_outgoing)) , state(State::Starting) , last_recv_time(rtc::TimeMillis()) , last_send_time(rtc::TimeMillis()) -, is_outgoing(is_outgoing) -, init_timeout(init_timeout * 1000) -, reconnect_timeout(reconnect_timeout * 1000) -, local_datasaving(false) -, final_datasaving(false) +, isOutgoing(is_outgoing) { connector->SignalReadyToSendStateChanged.connect(this, &Controller::WriteableStateChanged); connector->SignalPacketReceived.connect(this, &Controller::PacketReceived); @@ -104,7 +90,6 @@ void Controller::AttachVideoView(rtc::VideoSinkInterface *si }*/ void Controller::SetDataSaving(bool data_saving) { - local_datasaving = data_saving; } void Controller::SetMute(bool mute) { diff --git a/submodules/TgVoipWebrtc/Impl/Manager.cpp b/submodules/TgVoipWebrtc/Impl/Manager.cpp new file mode 100644 index 0000000000..502bcb8df2 --- /dev/null +++ b/submodules/TgVoipWebrtc/Impl/Manager.cpp @@ -0,0 +1,116 @@ +#include "Manager.h" + +#ifdef TGVOIP_NAMESPACE +namespace TGVOIP_NAMESPACE { +#endif + +Manager::Manager( + rtc::Thread *thread, + TgVoipEncryptionKey encryptionKey, + std::function stateUpdated, + std::function &)> signalingDataEmitted +) : +_thread(thread), +_encryptionKey(encryptionKey), +_networkThread(rtc::Thread::CreateWithSocketServer()), +_mediaThread(rtc::Thread::Create()), +_stateUpdated(stateUpdated), +_signalingDataEmitted(signalingDataEmitted) { + assert(_thread->IsCurrent()); + + _networkThread->Start(); + _mediaThread->Start(); +} + +Manager::~Manager() { + assert(_thread->IsCurrent()); +} + +void Manager::start() { + auto weakThis = std::weak_ptr(shared_from_this()); + _networkManager.reset(new ThreadLocalObject(_networkThread.get(), [networkThreadPtr = _networkThread.get(), encryptionKey = _encryptionKey, thread = _thread, weakThis]() { + return new NetworkManager( + networkThreadPtr, + encryptionKey, + [thread, weakThis](const NetworkManager::State &state) { + thread->Invoke(RTC_FROM_HERE, [weakThis, state]() { + auto strongThis = weakThis.lock(); + if (strongThis == nullptr) { + return; + } + TgVoipState mappedState; + if (state.isReadyToSendData) { + mappedState = TgVoipState::Estabilished; + } else { + mappedState = TgVoipState::Reconnecting; + } + strongThis->_stateUpdated(mappedState); + + strongThis->_mediaManager->perform([state](MediaManager *mediaManager) { + mediaManager->setIsConnected(state.isReadyToSendData); + }); + }); + }, + [thread, weakThis](const rtc::CopyOnWriteBuffer &packet) { + thread->PostTask(RTC_FROM_HERE, [weakThis, packet]() { + auto strongThis = weakThis.lock(); + if (strongThis == nullptr) { + return; + } + strongThis->_mediaManager->perform([packet](MediaManager *mediaManager) { + mediaManager->receivePacket(packet); + }); + }); + }, + [thread, weakThis](const std::vector &data) { + thread->PostTask(RTC_FROM_HERE, [weakThis, data]() { + auto strongThis = weakThis.lock(); + if (strongThis == nullptr) { + return; + } + strongThis->_signalingDataEmitted(data); + }); + } + ); + })); + bool isOutgoing = _encryptionKey.isOutgoing; + _mediaManager.reset(new ThreadLocalObject(_mediaThread.get(), [mediaThreadPtr = _mediaThread.get(), isOutgoing, thread = _thread, weakThis]() { + return new MediaManager( + mediaThreadPtr, + isOutgoing, + [thread, weakThis](const rtc::CopyOnWriteBuffer &packet) { + thread->PostTask(RTC_FROM_HERE, [weakThis, packet]() { + auto strongThis = weakThis.lock(); + if (strongThis == nullptr) { + return; + } + strongThis->_networkManager->perform([packet](NetworkManager *networkManager) { + networkManager->sendPacket(packet); + }); + }); + } + ); + })); +} + +void Manager::receiveSignalingData(const std::vector &data) { + _networkManager->perform([data](NetworkManager *networkManager) { + networkManager->receiveSignalingData(data); + }); +} + +void Manager::setIncomingVideoOutput(std::shared_ptr> sink) { + _mediaManager->perform([sink](MediaManager *mediaManager) { + mediaManager->setIncomingVideoOutput(sink); + }); +} + +void Manager::setOutgoingVideoOutput(std::shared_ptr> sink) { + _mediaManager->perform([sink](MediaManager *mediaManager) { + mediaManager->setOutgoingVideoOutput(sink); + }); +} + +#ifdef TGVOIP_NAMESPACE +} +#endif diff --git a/submodules/TgVoipWebrtc/Impl/Manager.h b/submodules/TgVoipWebrtc/Impl/Manager.h new file mode 100644 index 0000000000..eaac643681 --- /dev/null +++ b/submodules/TgVoipWebrtc/Impl/Manager.h @@ -0,0 +1,45 @@ +#ifndef TGVOIP_WEBRTC_MANAGER_H +#define TGVOIP_WEBRTC_MANAGER_H + +#include "ThreadLocalObject.h" +#include "NetworkManager.h" +#include "MediaManager.h" +#include "TgVoip.h" + +#ifdef TGVOIP_NAMESPACE +namespace TGVOIP_NAMESPACE { +#endif + +class Manager : public std::enable_shared_from_this { +public: + Manager( + rtc::Thread *thread, + TgVoipEncryptionKey encryptionKey, + std::function stateUpdated, + std::function &)> signalingDataEmitted + ); + ~Manager(); + + void start(); + void receiveSignalingData(const std::vector &data); + void setIncomingVideoOutput(std::shared_ptr> sink); + void setOutgoingVideoOutput(std::shared_ptr> sink); + +private: + rtc::Thread *_thread; + TgVoipEncryptionKey _encryptionKey; + std::unique_ptr _networkThread; + std::unique_ptr _mediaThread; + std::function _stateUpdated; + std::function &)> _signalingDataEmitted; + std::unique_ptr> _networkManager; + std::unique_ptr> _mediaManager; + +private: +}; + +#ifdef TGVOIP_NAMESPACE +} +#endif + +#endif diff --git a/submodules/TgVoipWebrtc/Impl/MediaEngineWebrtc.mm b/submodules/TgVoipWebrtc/Impl/MediaEngineWebrtc.mm index 90775e8011..7f1df11dc9 100644 --- a/submodules/TgVoipWebrtc/Impl/MediaEngineWebrtc.mm +++ b/submodules/TgVoipWebrtc/Impl/MediaEngineWebrtc.mm @@ -157,7 +157,6 @@ MediaEngineWebrtc::MediaEngineWebrtc(bool outgoing) media_deps.audio_encoder_factory = webrtc::CreateAudioEncoderFactory(); media_deps.audio_decoder_factory = webrtc::CreateAudioDecoderFactory(); - //auto video_encoder_factory = webrtc::ObjCToNativeVideoEncoderFactory([[RTCVideoEncoderFactoryH264 alloc] init]); auto video_encoder_factory = webrtc::ObjCToNativeVideoEncoderFactory([[RTCDefaultVideoEncoderFactory alloc] init]); int32_t outCodecId = 96; std::vector videoCodecs = AssignPayloadTypesAndDefaultCodecs(video_encoder_factory->GetSupportedFormats(), outCodecId); diff --git a/submodules/TgVoipWebrtc/Impl/MediaManager.cpp b/submodules/TgVoipWebrtc/Impl/MediaManager.cpp new file mode 100644 index 0000000000..367c35706b --- /dev/null +++ b/submodules/TgVoipWebrtc/Impl/MediaManager.cpp @@ -0,0 +1,377 @@ +#include "MediaManager.h" + +#include "absl/strings/match.h" +#include "api/audio_codecs/audio_decoder_factory_template.h" +#include "api/audio_codecs/audio_encoder_factory_template.h" +#include "api/audio_codecs/opus/audio_decoder_opus.h" +#include "api/audio_codecs/opus/audio_encoder_opus.h" +#include "api/rtp_parameters.h" +#include "api/task_queue/default_task_queue_factory.h" +#include "media/base/codec.h" +#include "media/base/media_constants.h" +#include "media/engine/webrtc_media_engine.h" +#include "modules/audio_device/include/audio_device_default.h" +#include "rtc_base/task_utils/repeating_task.h" +#include "system_wrappers/include/field_trial.h" +#include "api/video/builtin_video_bitrate_allocator_factory.h" +#include "api/video/video_bitrate_allocation.h" +#include "call/call.h" + +#if TARGET_OS_IPHONE + +#include "CodecsApple.h" + +#else +#error "Unsupported platform" +#endif + +#ifdef TGVOIP_NAMESPACE +namespace TGVOIP_NAMESPACE { +#endif + +static const uint32_t ssrcAudioIncoming = 1; +static const uint32_t ssrcAudioOutgoing = 2; +static const uint32_t ssrcVideoIncoming = 3; +static const uint32_t ssrcVideoOutgoing = 4; + +static void AddDefaultFeedbackParams(cricket::VideoCodec *codec) { + // Don't add any feedback params for RED and ULPFEC. + if (codec->name == cricket::kRedCodecName || codec->name == cricket::kUlpfecCodecName) + return; + codec->AddFeedbackParam(cricket::FeedbackParam(cricket::kRtcpFbParamRemb, cricket::kParamValueEmpty)); + codec->AddFeedbackParam( + cricket::FeedbackParam(cricket::kRtcpFbParamTransportCc, cricket::kParamValueEmpty)); + // Don't add any more feedback params for FLEXFEC. + if (codec->name == cricket::kFlexfecCodecName) + return; + codec->AddFeedbackParam(cricket::FeedbackParam(cricket::kRtcpFbParamCcm, cricket::kRtcpFbCcmParamFir)); + codec->AddFeedbackParam(cricket::FeedbackParam(cricket::kRtcpFbParamNack, cricket::kParamValueEmpty)); + codec->AddFeedbackParam(cricket::FeedbackParam(cricket::kRtcpFbParamNack, cricket::kRtcpFbNackParamPli)); + if (codec->name == cricket::kVp8CodecName && + webrtc::field_trial::IsEnabled("WebRTC-RtcpLossNotification")) { + codec->AddFeedbackParam(cricket::FeedbackParam(cricket::kRtcpFbParamLntf, cricket::kParamValueEmpty)); + } +} + +static std::vector AssignPayloadTypesAndDefaultCodecs(std::vector input_formats, int32_t &outCodecId) { + if (input_formats.empty()) + return std::vector(); + static const int kFirstDynamicPayloadType = 96; + static const int kLastDynamicPayloadType = 127; + int payload_type = kFirstDynamicPayloadType; + + input_formats.push_back(webrtc::SdpVideoFormat(cricket::kRedCodecName)); + input_formats.push_back(webrtc::SdpVideoFormat(cricket::kUlpfecCodecName)); + + /*if (IsFlexfecAdvertisedFieldTrialEnabled()) { + webrtc::SdpVideoFormat flexfec_format(kFlexfecCodecName); + // This value is currently arbitrarily set to 10 seconds. (The unit + // is microseconds.) This parameter MUST be present in the SDP, but + // we never use the actual value anywhere in our code however. + // TODO(brandtr): Consider honouring this value in the sender and receiver. + flexfec_format.parameters = {{kFlexfecFmtpRepairWindow, "10000000"}}; + input_formats.push_back(flexfec_format); + }*/ + + bool found = false; + bool useVP9 = true; + + std::vector output_codecs; + for (const webrtc::SdpVideoFormat& format : input_formats) { + cricket::VideoCodec codec(format); + codec.id = payload_type; + AddDefaultFeedbackParams(&codec); + output_codecs.push_back(codec); + + if (useVP9 && codec.name == cricket::kVp9CodecName) { + if (!found) { + outCodecId = codec.id; + found = true; + } + } + if (!useVP9 && codec.name == cricket::kH264CodecName) { + if (!found) { + outCodecId = codec.id; + found = true; + } + } + + // Increment payload type. + ++payload_type; + if (payload_type > kLastDynamicPayloadType) { + RTC_LOG(LS_ERROR) << "Out of dynamic payload types, skipping the rest."; + break; + } + + // Add associated RTX codec for non-FEC codecs. + if (!absl::EqualsIgnoreCase(codec.name, cricket::kUlpfecCodecName) && + !absl::EqualsIgnoreCase(codec.name, cricket::kFlexfecCodecName)) { + output_codecs.push_back( + cricket::VideoCodec::CreateRtxCodec(payload_type, codec.id)); + + // Increment payload type. + ++payload_type; + if (payload_type > kLastDynamicPayloadType) { + RTC_LOG(LS_ERROR) << "Out of dynamic payload types, skipping the rest."; + break; + } + } + } + return output_codecs; +} + +static absl::optional selectVideoCodec(std::vector &codecs) { + bool useVP9 = false; + + for (auto &codec : codecs) { + if (useVP9) { + if (codec.name == cricket::kVp9CodecName) { + return absl::optional(codec); + } + } else { + if (codec.name == cricket::kH264CodecName) { + return absl::optional(codec); + } + } + } + return absl::optional(); +} + +MediaManager::MediaManager( + rtc::Thread *thread, + bool isOutgoing, + std::function packetEmitted +) : +_packetEmitted(packetEmitted), +_thread(thread), +_eventLog(std::make_unique()), +_taskQueueFactory(webrtc::CreateDefaultTaskQueueFactory()), +_workerThread(rtc::Thread::Create()) { + _ssrcAudio.incoming = isOutgoing ? ssrcAudioIncoming : ssrcAudioOutgoing; + _ssrcAudio.outgoing = (!isOutgoing) ? ssrcAudioIncoming : ssrcAudioOutgoing; + _ssrcVideo.incoming = isOutgoing ? ssrcVideoIncoming : ssrcVideoOutgoing; + _ssrcVideo.outgoing = (!isOutgoing) ? ssrcVideoIncoming : ssrcVideoOutgoing; + + _audioNetworkInterface = std::unique_ptr(new MediaManager::NetworkInterfaceImpl(this, false)); + _videoNetworkInterface = std::unique_ptr(new MediaManager::NetworkInterfaceImpl(this, true)); + + _workerThread->Start(); + + webrtc::field_trial::InitFieldTrialsFromString( + "WebRTC-Audio-SendSideBwe/Enabled/" + "WebRTC-Audio-Allocation/min:6kbps,max:32kbps/" + "WebRTC-Audio-OpusMinPacketLossRate/Enabled-1/" + ); + + _videoBitrateAllocatorFactory = webrtc::CreateBuiltinVideoBitrateAllocatorFactory(); + + cricket::MediaEngineDependencies mediaDeps; + mediaDeps.task_queue_factory = _taskQueueFactory.get(); + mediaDeps.audio_encoder_factory = webrtc::CreateAudioEncoderFactory(); + mediaDeps.audio_decoder_factory = webrtc::CreateAudioDecoderFactory(); + + auto videoEncoderFactory = makeVideoEncoderFactory(); + int32_t outCodecId = 96; + std::vector videoCodecs = AssignPayloadTypesAndDefaultCodecs(videoEncoderFactory->GetSupportedFormats(), outCodecId); + + mediaDeps.video_encoder_factory = makeVideoEncoderFactory(); + mediaDeps.video_decoder_factory = makeVideoDecoderFactory(); + + mediaDeps.audio_processing = webrtc::AudioProcessingBuilder().Create(); + _mediaEngine = cricket::CreateMediaEngine(std::move(mediaDeps)); + _mediaEngine->Init(); + webrtc::Call::Config callConfig(_eventLog.get()); + callConfig.task_queue_factory = _taskQueueFactory.get(); + callConfig.trials = &_fieldTrials; + callConfig.audio_state = _mediaEngine->voice().GetAudioState(); + _call.reset(webrtc::Call::Create(callConfig)); + _audioChannel.reset(_mediaEngine->voice().CreateMediaChannel(_call.get(), cricket::MediaConfig(), cricket::AudioOptions(), webrtc::CryptoOptions::NoGcm())); + _videoChannel.reset(_mediaEngine->video().CreateMediaChannel(_call.get(), cricket::MediaConfig(), cricket::VideoOptions(), webrtc::CryptoOptions::NoGcm(), _videoBitrateAllocatorFactory.get())); + + _audioChannel->AddSendStream(cricket::StreamParams::CreateLegacy(_ssrcAudio.outgoing)); + + const uint32_t opusClockrate = 48000; + const uint16_t opusSdpPayload = 111; + const char *opusSdpName = "opus"; + const uint8_t opusSdpChannels = 2; + const uint32_t opusSdpBitrate = 0; + + const uint8_t opusMinBitrateKbps = 6; + const uint8_t opusMaxBitrateKbps = 32; + const uint8_t opusStartBitrateKbps = 6; + const uint8_t opusPTimeMs = 120; + const int opusExtensionSequence = 1; + + cricket::AudioCodec opusCodec(opusSdpPayload, opusSdpName, opusClockrate, opusSdpBitrate, opusSdpChannels); + opusCodec.AddFeedbackParam(cricket::FeedbackParam(cricket::kRtcpFbParamTransportCc)); + opusCodec.SetParam(cricket::kCodecParamMinBitrate, opusMinBitrateKbps); + opusCodec.SetParam(cricket::kCodecParamStartBitrate, opusStartBitrateKbps); + opusCodec.SetParam(cricket::kCodecParamMaxBitrate, opusMaxBitrateKbps); + opusCodec.SetParam(cricket::kCodecParamUseInbandFec, 1); + opusCodec.SetParam(cricket::kCodecParamPTime, opusPTimeMs); + + cricket::AudioSendParameters audioSendPrameters; + audioSendPrameters.codecs.push_back(opusCodec); + audioSendPrameters.extensions.emplace_back(webrtc::RtpExtension::kTransportSequenceNumberUri, opusExtensionSequence); + audioSendPrameters.options.echo_cancellation = false; + //audioSendPrameters.options.experimental_ns = false; + audioSendPrameters.options.noise_suppression = false; + audioSendPrameters.options.auto_gain_control = false; + audioSendPrameters.options.highpass_filter = false; + audioSendPrameters.options.typing_detection = false; + //audioSendPrameters.max_bandwidth_bps = 16000; + audioSendPrameters.rtcp.reduced_size = true; + audioSendPrameters.rtcp.remote_estimate = true; + _audioChannel->SetSendParameters(audioSendPrameters); + _audioChannel->SetInterface(_audioNetworkInterface.get(), webrtc::MediaTransportConfig()); + + cricket::AudioRecvParameters audioRecvParameters; + audioRecvParameters.codecs.emplace_back(opusSdpPayload, opusSdpName, opusClockrate, opusSdpBitrate, opusSdpChannels); + audioRecvParameters.extensions.emplace_back(webrtc::RtpExtension::kTransportSequenceNumberUri, opusExtensionSequence); + audioRecvParameters.rtcp.reduced_size = true; + audioRecvParameters.rtcp.remote_estimate = true; + _audioChannel->AddRecvStream(cricket::StreamParams::CreateLegacy(_ssrcAudio.incoming)); + _audioChannel->SetRecvParameters(audioRecvParameters); + _audioChannel->SetPlayout(true); + + _videoChannel->AddSendStream(cricket::StreamParams::CreateLegacy(_ssrcVideo.outgoing)); + + auto videoCodec = selectVideoCodec(videoCodecs); + if (videoCodec.has_value()) { + _nativeVideoSource = makeVideoSource(_thread, _workerThread.get()); + + auto codec = videoCodec.value(); + + codec.SetParam(cricket::kCodecParamMinBitrate, 64); + codec.SetParam(cricket::kCodecParamStartBitrate, 256); + codec.SetParam(cricket::kCodecParamMaxBitrate, 2500); + + _videoCapturer = makeVideoCapturer(_nativeVideoSource); + + cricket::VideoSendParameters videoSendParameters; + videoSendParameters.codecs.push_back(codec); + const int videoExtensionSequence = 1; + videoSendParameters.extensions.emplace_back(webrtc::RtpExtension::kTransportSequenceNumberUri, videoExtensionSequence); + //send_parameters.max_bandwidth_bps = 800000; + //send_parameters.rtcp.reduced_size = true; + videoSendParameters.rtcp.remote_estimate = true; + _videoChannel->SetSendParameters(videoSendParameters); + + _videoChannel->SetVideoSend(_ssrcVideo.outgoing, NULL, _nativeVideoSource.get()); + + _videoChannel->SetInterface(_videoNetworkInterface.get(), webrtc::MediaTransportConfig()); + + cricket::VideoRecvParameters videoRecvParameters; + videoRecvParameters.codecs.emplace_back(codec); + videoRecvParameters.extensions.emplace_back(webrtc::RtpExtension::kTransportSequenceNumberUri, videoExtensionSequence); + //recv_parameters.rtcp.reduced_size = true; + videoRecvParameters.rtcp.remote_estimate = true; + _videoChannel->AddRecvStream(cricket::StreamParams::CreateLegacy(_ssrcVideo.incoming)); + _videoChannel->SetRecvParameters(videoRecvParameters); + } +} + +MediaManager::~MediaManager() { + assert(_thread->IsCurrent()); + + _call->SignalChannelNetworkState(webrtc::MediaType::AUDIO, webrtc::kNetworkDown); + _call->SignalChannelNetworkState(webrtc::MediaType::VIDEO, webrtc::kNetworkDown); + + _audioChannel->OnReadyToSend(false); + _audioChannel->SetSend(false); + _audioChannel->SetAudioSend(_ssrcAudio.outgoing, false, nullptr, &_audioSource); + + _audioChannel->SetPlayout(false); + + _audioChannel->RemoveRecvStream(_ssrcAudio.incoming); + _audioChannel->RemoveSendStream(_ssrcAudio.outgoing); +} + +void MediaManager::setIsConnected(bool isConnected) { + if (isConnected) { + _call->SignalChannelNetworkState(webrtc::MediaType::AUDIO, webrtc::kNetworkUp); + _call->SignalChannelNetworkState(webrtc::MediaType::VIDEO, webrtc::kNetworkUp); + } else { + _call->SignalChannelNetworkState(webrtc::MediaType::AUDIO, webrtc::kNetworkDown); + _call->SignalChannelNetworkState(webrtc::MediaType::VIDEO, webrtc::kNetworkDown); + } + if (_audioChannel) { + _audioChannel->OnReadyToSend(isConnected); + _audioChannel->SetSend(isConnected); + _audioChannel->SetAudioSend(_ssrcAudio.outgoing, isConnected, nullptr, &_audioSource); + } + if (_videoChannel) { + _videoChannel->OnReadyToSend(isConnected); + _videoChannel->SetSend(isConnected); + } +} + +void MediaManager::receivePacket(const rtc::CopyOnWriteBuffer &packet) { + if (packet.size() < 1) { + return; + } + + uint8_t header = ((uint8_t *)packet.data())[0]; + rtc::CopyOnWriteBuffer unwrappedPacket = packet.Slice(1, packet.size() - 1); + + if (header == 0xba) { + if (_audioChannel) { + _audioChannel->OnPacketReceived(unwrappedPacket, -1); + } + } else if (header == 0xbf) { + if (_videoChannel) { + _videoChannel->OnPacketReceived(unwrappedPacket, -1); + } + } +} + +void MediaManager::notifyPacketSent(const rtc::SentPacket &sentPacket) { + _call->OnSentPacket(sentPacket); +} + +void MediaManager::setIncomingVideoOutput(std::shared_ptr> sink) { + _currentIncomingVideoSink = sink; + _videoChannel->SetSink(_ssrcVideo.incoming, sink.get()); +} + +void MediaManager::setOutgoingVideoOutput(std::shared_ptr> sink) { + _currentOutgoingVideoSink = sink; + _nativeVideoSource->AddOrUpdateSink(sink.get(), rtc::VideoSinkWants()); +} + +MediaManager::NetworkInterfaceImpl::NetworkInterfaceImpl(MediaManager *mediaManager, bool isVideo) : +_mediaManager(mediaManager), +_isVideo(isVideo) { +} + +bool MediaManager::NetworkInterfaceImpl::SendPacket(rtc::CopyOnWriteBuffer *packet, const rtc::PacketOptions& options) { + rtc::CopyOnWriteBuffer wrappedPacket; + uint8_t header = _isVideo ? 0xbf : 0xba; + wrappedPacket.AppendData(&header, 1); + wrappedPacket.AppendData(*packet); + + _mediaManager->_packetEmitted(wrappedPacket); + rtc::SentPacket sentPacket(options.packet_id, rtc::TimeMillis(), options.info_signaled_after_sent); + _mediaManager->notifyPacketSent(sentPacket); + return true; +} + +bool MediaManager::NetworkInterfaceImpl::SendRtcp(rtc::CopyOnWriteBuffer *packet, const rtc::PacketOptions& options) { + rtc::CopyOnWriteBuffer wrappedPacket; + uint8_t header = _isVideo ? 0xbf : 0xba; + wrappedPacket.AppendData(&header, 1); + wrappedPacket.AppendData(*packet); + + _mediaManager->_packetEmitted(wrappedPacket); + rtc::SentPacket sentPacket(options.packet_id, rtc::TimeMillis(), options.info_signaled_after_sent); + _mediaManager->notifyPacketSent(sentPacket); + return true; +} + +int MediaManager::NetworkInterfaceImpl::SetOption(cricket::MediaChannel::NetworkInterface::SocketType, rtc::Socket::Option, int) { + return -1; +} + +#ifdef TGVOIP_NAMESPACE +} +#endif diff --git a/submodules/TgVoipWebrtc/Impl/MediaManager.h b/submodules/TgVoipWebrtc/Impl/MediaManager.h new file mode 100644 index 0000000000..4003e84f12 --- /dev/null +++ b/submodules/TgVoipWebrtc/Impl/MediaManager.h @@ -0,0 +1,100 @@ +#ifndef TGVOIP_WEBRTC_MEDIA_MANAGER_H +#define TGVOIP_WEBRTC_MEDIA_MANAGER_H + +#include "rtc_base/thread.h" +#include "rtc_base/copy_on_write_buffer.h" +#include "rtc_base/third_party/sigslot/sigslot.h" +#include "api/transport/field_trial_based_config.h" +#include "pc/rtp_sender.h" + +#include +#include + +namespace webrtc { +class Call; +class RtcEventLogNull; +class TaskQueueFactory; +class VideoBitrateAllocatorFactory; +class VideoTrackSourceInterface; +}; + +namespace cricket { +class MediaEngineInterface; +class VoiceMediaChannel; +class VideoMediaChannel; +}; + +#ifdef TGVOIP_NAMESPACE +namespace TGVOIP_NAMESPACE { +#endif + +class VideoCapturerInterface; + +class MediaManager : public sigslot::has_slots<>, public std::enable_shared_from_this { +private: + struct SSRC { + uint32_t incoming; + uint32_t outgoing; + }; + + class NetworkInterfaceImpl : public cricket::MediaChannel::NetworkInterface { + public: + NetworkInterfaceImpl(MediaManager *mediaManager, bool isVideo); + bool SendPacket(rtc::CopyOnWriteBuffer *packet, const rtc::PacketOptions& options) override; + bool SendRtcp(rtc::CopyOnWriteBuffer *packet, const rtc::PacketOptions& options) override; + int SetOption(SocketType type, rtc::Socket::Option opt, int option) override; + + private: + MediaManager *_mediaManager; + bool _isVideo; + }; + + friend class MediaManager::NetworkInterfaceImpl; + +public: + MediaManager( + rtc::Thread *thread, + bool isOutgoing, + std::function packetEmitted + ); + ~MediaManager(); + + void setIsConnected(bool isConnected); + void receivePacket(const rtc::CopyOnWriteBuffer &packet); + void notifyPacketSent(const rtc::SentPacket &sentPacket); + void setIncomingVideoOutput(std::shared_ptr> sink); + void setOutgoingVideoOutput(std::shared_ptr> sink); + +protected: + std::function _packetEmitted; + +private: + rtc::Thread *_thread; + std::unique_ptr _eventLog; + std::unique_ptr _taskQueueFactory; + std::unique_ptr _workerThread; + + SSRC _ssrcAudio; + SSRC _ssrcVideo; + + std::unique_ptr _mediaEngine; + std::unique_ptr _call; + webrtc::FieldTrialBasedConfig _fieldTrials; + webrtc::LocalAudioSinkAdapter _audioSource; + std::unique_ptr _audioChannel; + std::unique_ptr _videoChannel; + std::unique_ptr _videoBitrateAllocatorFactory; + rtc::scoped_refptr _nativeVideoSource; + std::unique_ptr _videoCapturer; + std::shared_ptr> _currentIncomingVideoSink; + std::shared_ptr> _currentOutgoingVideoSink; + + std::unique_ptr _audioNetworkInterface; + std::unique_ptr _videoNetworkInterface; +}; + +#ifdef TGVOIP_NAMESPACE +} +#endif + +#endif diff --git a/submodules/TgVoipWebrtc/Impl/NetworkManager.cpp b/submodules/TgVoipWebrtc/Impl/NetworkManager.cpp new file mode 100644 index 0000000000..33175373e1 --- /dev/null +++ b/submodules/TgVoipWebrtc/Impl/NetworkManager.cpp @@ -0,0 +1,185 @@ +#include "NetworkManager.h" + +#include "p2p/base/basic_packet_socket_factory.h" +#include "p2p/client/basic_port_allocator.h" +#include "p2p/base/p2p_transport_channel.h" +#include "p2p/base/basic_async_resolver_factory.h" +#include "api/packet_socket_factory.h" +#include "rtc_base/task_utils/to_queued_task.h" +#include "p2p/base/ice_credentials_iterator.h" +#include "api/jsep_ice_candidate.h" + +#ifdef TGVOIP_NAMESPACE +namespace TGVOIP_NAMESPACE { +#endif + +NetworkManager::NetworkManager( + rtc::Thread *thread, + TgVoipEncryptionKey encryptionKey, + std::function stateUpdated, + std::function packetReceived, + std::function &)> signalingDataEmitted +) : +_thread(thread), +_encryptionKey(encryptionKey), +_stateUpdated(stateUpdated), +_packetReceived(packetReceived), +_signalingDataEmitted(signalingDataEmitted) { + assert(_thread->IsCurrent()); + + _socketFactory.reset(new rtc::BasicPacketSocketFactory(_thread)); + + _networkManager = std::make_unique(); + _portAllocator.reset(new cricket::BasicPortAllocator(_networkManager.get(), _socketFactory.get(), nullptr, nullptr)); + + uint32_t flags = cricket::PORTALLOCATOR_DISABLE_TCP; + //flags |= cricket::PORTALLOCATOR_DISABLE_UDP; + _portAllocator->set_flags(_portAllocator->flags() | flags); + _portAllocator->Initialize(); + + rtc::SocketAddress defaultStunAddress = rtc::SocketAddress("hlgkfjdrtjfykgulhijkljhulyo.uksouth.cloudapp.azure.com", 3478); + cricket::ServerAddresses stunServers; + stunServers.insert(defaultStunAddress); + std::vector turnServers; + turnServers.push_back(cricket::RelayServerConfig( + rtc::SocketAddress("hlgkfjdrtjfykgulhijkljhulyo.uksouth.cloudapp.azure.com", 3478), + "user", + "root", + cricket::PROTO_UDP + )); + _portAllocator->SetConfiguration(stunServers, turnServers, 2, webrtc::NO_PRUNE); + + _asyncResolverFactory = std::make_unique(); + _transportChannel.reset(new cricket::P2PTransportChannel("transport", 0, _portAllocator.get(), _asyncResolverFactory.get(), nullptr)); + + cricket::IceConfig iceConfig; + iceConfig.continual_gathering_policy = cricket::GATHER_CONTINUALLY; + _transportChannel->SetIceConfig(iceConfig); + + cricket::IceParameters localIceParameters( + "gcp3", + "zWDKozH8/3JWt8he3M/CMj5R", + false + ); + cricket::IceParameters remoteIceParameters( + "acp3", + "aWDKozH8/3JWt8he3M/CMj5R", + false + ); + + _transportChannel->SetIceParameters(_encryptionKey.isOutgoing ? localIceParameters : remoteIceParameters); + _transportChannel->SetIceRole(_encryptionKey.isOutgoing ? cricket::ICEROLE_CONTROLLING : cricket::ICEROLE_CONTROLLED); + + _transportChannel->SignalCandidateGathered.connect(this, &NetworkManager::candidateGathered); + _transportChannel->SignalGatheringState.connect(this, &NetworkManager::candidateGatheringState); + _transportChannel->SignalIceTransportStateChanged.connect(this, &NetworkManager::transportStateChanged); + _transportChannel->SignalReadPacket.connect(this, &NetworkManager::transportPacketReceived); + + _transportChannel->MaybeStartGathering(); + + _transportChannel->SetRemoteIceMode(cricket::ICEMODE_FULL); + _transportChannel->SetRemoteIceParameters((!_encryptionKey.isOutgoing) ? localIceParameters : remoteIceParameters); +} + +NetworkManager::~NetworkManager() { + assert(_thread->IsCurrent()); + + _transportChannel.reset(); + _asyncResolverFactory.reset(); + _portAllocator.reset(); + _networkManager.reset(); + _socketFactory.reset(); +} + +void NetworkManager::receiveSignalingData(const std::vector &data) { + rtc::ByteBufferReader reader((const char *)data.data(), data.size()); + uint32_t candidateCount = 0; + if (!reader.ReadUInt32(&candidateCount)) { + return; + } + std::vector candidates; + for (uint32_t i = 0; i < candidateCount; i++) { + uint32_t candidateLength = 0; + if (!reader.ReadUInt32(&candidateLength)) { + return; + } + std::string candidate; + if (!reader.ReadString(&candidate, candidateLength)) { + return; + } + candidates.push_back(candidate); + } + + for (auto &serializedCandidate : candidates) { + webrtc::JsepIceCandidate parseCandidate("", 0); + if (parseCandidate.Initialize(serializedCandidate, nullptr)) { + auto parsedCandidate = parseCandidate.candidate(); + _transportChannel->AddRemoteCandidate(parsedCandidate); + } + } +} + +void NetworkManager::sendPacket(const rtc::CopyOnWriteBuffer &packet) { + rtc::PacketOptions packetOptions; + _transportChannel->SendPacket((const char *)packet.data(), packet.size(), packetOptions, 0); +} + +void NetworkManager::candidateGathered(cricket::IceTransportInternal *transport, const cricket::Candidate &candidate) { + assert(_thread->IsCurrent()); + webrtc::JsepIceCandidate iceCandidate("", 0); + iceCandidate.SetCandidate(candidate); + std::string serializedCandidate; + if (!iceCandidate.ToString(&serializedCandidate)) { + return; + } + std::vector candidates; + candidates.push_back(serializedCandidate); + + rtc::ByteBufferWriter writer; + writer.WriteUInt32((uint32_t)candidates.size()); + for (auto string : candidates) { + writer.WriteUInt32((uint32_t)string.size()); + writer.WriteString(string); + } + std::vector data; + data.resize(writer.Length()); + memcpy(data.data(), writer.Data(), writer.Length()); + _signalingDataEmitted(data); +} + +void NetworkManager::candidateGatheringState(cricket::IceTransportInternal *transport) { + assert(_thread->IsCurrent()); +} + +void NetworkManager::transportStateChanged(cricket::IceTransportInternal *transport) { + assert(_thread->IsCurrent()); + + auto state = transport->GetIceTransportState(); + bool isConnected = false; + switch (state) { + case webrtc::IceTransportState::kConnected: + case webrtc::IceTransportState::kCompleted: + isConnected = true; + break; + default: + break; + } + NetworkManager::State emitState; + emitState.isReadyToSendData = isConnected; + _stateUpdated(emitState); +} + +void NetworkManager::transportReadyToSend(cricket::IceTransportInternal *transport) { + assert(_thread->IsCurrent()); +} + +void NetworkManager::transportPacketReceived(rtc::PacketTransportInternal *transport, const char *bytes, size_t size, const int64_t ×tamp, int unused) { + assert(_thread->IsCurrent()); + rtc::CopyOnWriteBuffer packet; + packet.AppendData(bytes, size); + _packetReceived(packet); +} + +#ifdef TGVOIP_NAMESPACE +} +#endif diff --git a/submodules/TgVoipWebrtc/Impl/NetworkManager.h b/submodules/TgVoipWebrtc/Impl/NetworkManager.h new file mode 100644 index 0000000000..9c9b93789e --- /dev/null +++ b/submodules/TgVoipWebrtc/Impl/NetworkManager.h @@ -0,0 +1,77 @@ +#ifndef TGVOIP_WEBRTC_NETWORK_MANAGER_H +#define TGVOIP_WEBRTC_NETWORK_MANAGER_H + +#include "rtc_base/thread.h" + +#include +#include + +#include "rtc_base/copy_on_write_buffer.h" +#include "api/candidate.h" +#include "TgVoip.h" + +namespace rtc { +class BasicPacketSocketFactory; +class BasicNetworkManager; +class PacketTransportInternal; +} + +namespace cricket { +class BasicPortAllocator; +class P2PTransportChannel; +class IceTransportInternal; +} + +namespace webrtc { +class BasicAsyncResolverFactory; +} + +#ifdef TGVOIP_NAMESPACE +namespace TGVOIP_NAMESPACE { +#endif + +class NetworkManager: public sigslot::has_slots<> { +public: + struct State { + bool isReadyToSendData; + }; + +public: + NetworkManager( + rtc::Thread *thread, + TgVoipEncryptionKey encryptionKey, + std::function stateUpdated, + std::function packetReceived, + std::function &)> signalingDataEmitted + ); + ~NetworkManager(); + + void receiveSignalingData(const std::vector &data); + void sendPacket(const rtc::CopyOnWriteBuffer &packet); + +private: + rtc::Thread *_thread; + TgVoipEncryptionKey _encryptionKey; + std::function _stateUpdated; + std::function _packetReceived; + std::function &)> _signalingDataEmitted; + + std::unique_ptr _socketFactory; + std::unique_ptr _networkManager; + std::unique_ptr _portAllocator; + std::unique_ptr _asyncResolverFactory; + std::unique_ptr _transportChannel; + +private: + void candidateGathered(cricket::IceTransportInternal *transport, const cricket::Candidate &candidate); + void candidateGatheringState(cricket::IceTransportInternal *transport); + void transportStateChanged(cricket::IceTransportInternal *transport); + void transportReadyToSend(cricket::IceTransportInternal *transport); + void transportPacketReceived(rtc::PacketTransportInternal *transport, const char *bytes, size_t size, const int64_t ×tamp, int unused); +}; + +#ifdef TGVOIP_NAMESPACE +} +#endif + +#endif diff --git a/submodules/TgVoipWebrtc/Impl/TgVoip.h b/submodules/TgVoipWebrtc/Impl/TgVoip.h index 220975e71b..bf1c29ae03 100644 --- a/submodules/TgVoipWebrtc/Impl/TgVoip.h +++ b/submodules/TgVoipWebrtc/Impl/TgVoip.h @@ -1,14 +1,19 @@ #ifndef __TGVOIP_H #define __TGVOIP_H -#define TGVOIP_NAMESPACE tgvoip_webrtc - #include #include #include #include -#import "VideoMetalView.h" +namespace rtc { +template +class VideoSinkInterface; +} + +namespace webrtc { +class VideoFrame; +} #ifdef TGVOIP_NAMESPACE namespace TGVOIP_NAMESPACE { @@ -131,7 +136,9 @@ public: std::vector const &endpoints, std::unique_ptr const &proxy, TgVoipNetworkType initialNetworkType, - TgVoipEncryptionKey const &encryptionKey + TgVoipEncryptionKey const &encryptionKey, + std::function stateUpdated, + std::function &)> signalingDataEmitted ); virtual ~TgVoip(); @@ -141,19 +148,16 @@ public: virtual void setAudioOutputGainControlEnabled(bool enabled) = 0; virtual void setEchoCancellationStrength(int strength) = 0; - virtual void AttachVideoView(VideoMetalView *videoView) = 0; + virtual void setIncomingVideoOutput(std::shared_ptr> sink) = 0; + virtual void setOutgoingVideoOutput(std::shared_ptr> sink) = 0; virtual std::string getLastError() = 0; virtual std::string getDebugInfo() = 0; virtual int64_t getPreferredRelayId() = 0; virtual TgVoipTrafficStats getTrafficStats() = 0; virtual TgVoipPersistentState getPersistentState() = 0; - - virtual void setOnStateUpdated(std::function onStateUpdated) = 0; - virtual void setOnSignalBarsUpdated(std::function onSignalBarsUpdated) = 0; - virtual void setOnCandidatesGathered(std::function &)> onCandidatesGathered) = 0; - virtual void addRemoteCandidates(const std::vector &candidates) = 0; + virtual void receiveSignalingData(const std::vector &data) = 0; virtual TgVoipFinalState stop() = 0; }; diff --git a/submodules/TgVoipWebrtc/Impl/TgVoip.mm b/submodules/TgVoipWebrtc/Impl/TgVoip.mm index 8ebefc7aa4..76619d0249 100644 --- a/submodules/TgVoipWebrtc/Impl/TgVoip.mm +++ b/submodules/TgVoipWebrtc/Impl/TgVoip.mm @@ -2,11 +2,15 @@ #include "TgVoip.h" -#include "Controller.h" +#include "rtc_base/logging.h" + +#include "Manager.h" #include #include +#import + #ifndef TGVOIP_USE_CUSTOM_CRYPTO /*extern "C" { #include @@ -76,9 +80,6 @@ namespace TGVOIP_NAMESPACE { #endif class TgVoipImpl : public TgVoip, public sigslot::has_slots<> { -private: - - public: TgVoipImpl( std::vector const &endpoints, @@ -86,88 +87,45 @@ public: std::unique_ptr const &proxy, TgVoipConfig const &config, TgVoipEncryptionKey const &encryptionKey, - TgVoipNetworkType initialNetworkType - ) { - + TgVoipNetworkType initialNetworkType, + std::function stateUpdated, + std::function &)> signalingDataEmitted + ) : + _stateUpdated(stateUpdated), + _signalingDataEmitted(signalingDataEmitted) { static dispatch_once_t onceToken; dispatch_once(&onceToken, ^{ rtc::LogMessage::LogToDebug(rtc::LS_INFO); rtc::LogMessage::SetLogToStderr(true); }); - - /*EncryptionKey encryptionKeyValue; - memcpy(encryptionKeyValue, encryptionKey.value.data(), 256);*/ - controller_ = new Controller(encryptionKey.isOutgoing, 5, 3); - - if (proxy != nullptr) { - controller_->SetProxy(rtc::ProxyType::PROXY_SOCKS5, rtc::SocketAddress(proxy->host, proxy->port), - proxy->login, proxy->password); - } - - controller_->SignalNewState.connect(this, &TgVoipImpl::controllerStateCallback); - controller_->SignalCandidatesGathered.connect(this, &TgVoipImpl::candidatesGathered); - controller_->Start(); - - for (const auto &endpoint : endpoints) { - rtc::SocketAddress addr(endpoint.host.ipv4, endpoint.port); - Controller::EndpointType type; - switch (endpoint.type) { - case TgVoipEndpointType::UdpRelay: - type = Controller::EndpointType::UDP; - break; - case TgVoipEndpointType::Lan: - case TgVoipEndpointType::Inet: - type = Controller::EndpointType::P2P; - break; - case TgVoipEndpointType::TcpRelay: - type = Controller::EndpointType::TCP; - break; - default: - type = Controller::EndpointType::UDP; - break; - } - //controller_->AddEndpoint(addr, endpoint.peerTag, type); - } - /*rtc::SocketAddress addr("192.168.8.118", 7325); - unsigned char peerTag[16]; - controller_->AddEndpoint(addr, peerTag, Controller::EndpointType::P2P);*/ - - setNetworkType(initialNetworkType); - - switch (config.dataSaving) { - case TgVoipDataSaving::Mobile: - controller_->SetDataSaving(true); - break; - case TgVoipDataSaving::Always: - controller_->SetDataSaving(true); - break; - default: - controller_->SetDataSaving(false); - break; - } + + _managerThread = rtc::Thread::Create(); + _managerThread->Start(); + _manager.reset(new ThreadLocalObject(_managerThread.get(), [managerThreadPtr = _managerThread.get(), encryptionKey = encryptionKey, stateUpdated, signalingDataEmitted](){ + return new Manager( + managerThreadPtr, + encryptionKey, + [stateUpdated](const TgVoipState &state) { + stateUpdated(state); + }, + [signalingDataEmitted](const std::vector &data) { + signalingDataEmitted(data); + } + ); + })); + _manager->perform([](Manager *manager) { + manager->start(); + }); } ~TgVoipImpl() override { - stop(); - } - - void setOnStateUpdated(std::function onStateUpdated) override { - std::lock_guard lock(m_onStateUpdated); - onStateUpdated_ = onStateUpdated; - } - - void setOnSignalBarsUpdated(std::function onSignalBarsUpdated) override { - std::lock_guard lock(m_onSignalBarsUpdated); - onSignalBarsUpdated_ = onSignalBarsUpdated; } - void setOnCandidatesGathered(std::function &)> onCandidatesGathered) override { - onCandidatesGathered_ = onCandidatesGathered; - } - - void addRemoteCandidates(const std::vector &candidates) override { - controller_->AddRemoteCandidates(candidates); - } + void receiveSignalingData(const std::vector &data) override { + _manager->perform([data](Manager *manager) { + manager->receiveSignalingData(data); + }); + }; void setNetworkType(TgVoipNetworkType networkType) override { /*message::NetworkType mappedType; @@ -218,11 +176,19 @@ public: } void setMuteMicrophone(bool muteMicrophone) override { - controller_->SetMute(muteMicrophone); + //controller_->SetMute(muteMicrophone); } - void AttachVideoView(VideoMetalView *videoView) override { - controller_->AttachVideoView([videoView getSink]); + void setIncomingVideoOutput(std::shared_ptr> sink) { + _manager->perform([sink](Manager *manager) { + manager->setIncomingVideoOutput(sink); + }); + } + + void setOutgoingVideoOutput(std::shared_ptr> sink) { + _manager->perform([sink](Manager *manager) { + manager->setOutgoingVideoOutput(sink); + }); } void setAudioOutputGainControlEnabled(bool enabled) override { @@ -255,13 +221,10 @@ public: TgVoipFinalState finalState = { }; - delete controller_; - controller_ = nullptr; - return finalState; } - void controllerStateCallback(Controller::State state) { + /*void controllerStateCallback(Controller::State state) { if (onStateUpdated_) { TgVoipState mappedState; switch (state) { @@ -287,44 +250,13 @@ public: onStateUpdated_(mappedState); } - } - - void candidatesGathered(const std::vector &candidates) { - onCandidatesGathered_(candidates); - } + }*/ private: -#ifdef TGVOIP_USE_CALLBACK_AUDIO_IO - TgVoipAudioDataCallbacks audioCallbacks; - - void play(const int16_t *data, size_t size) { - if (!audioCallbacks.output) - return; - int16_t buf[size]; - memcpy(buf, data, size * 2); - audioCallbacks.output(buf, size); - } - - void record(int16_t *data, size_t size) { - if (audioCallbacks.input) - audioCallbacks.input(data, size); - } - - void preprocessed(const int16_t *data, size_t size) { - if (!audioCallbacks.preprocessed) - return; - int16_t buf[size]; - memcpy(buf, data, size * 2); - audioCallbacks.preprocessed(buf, size); - } -#endif - -private: - Controller *controller_; - std::function onStateUpdated_; - std::function onSignalBarsUpdated_; - std::function &)> onCandidatesGathered_; - std::mutex m_onStateUpdated, m_onSignalBarsUpdated; + std::unique_ptr _managerThread; + std::unique_ptr> _manager; + std::function _stateUpdated; + std::function &)> _signalingDataEmitted; }; std::function globalLoggingFunction; @@ -368,7 +300,9 @@ TgVoip *TgVoip::makeInstance( std::vector const &endpoints, std::unique_ptr const &proxy, TgVoipNetworkType initialNetworkType, - TgVoipEncryptionKey const &encryptionKey + TgVoipEncryptionKey const &encryptionKey, + std::function stateUpdated, + std::function &)> signalingDataEmitted ) { return new TgVoipImpl( endpoints, @@ -376,7 +310,9 @@ TgVoip *TgVoip::makeInstance( proxy, config, encryptionKey, - initialNetworkType + initialNetworkType, + stateUpdated, + signalingDataEmitted ); } diff --git a/submodules/TgVoipWebrtc/Impl/ThreadLocalObject.cpp b/submodules/TgVoipWebrtc/Impl/ThreadLocalObject.cpp new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/submodules/TgVoipWebrtc/Impl/ThreadLocalObject.cpp @@ -0,0 +1 @@ + diff --git a/submodules/TgVoipWebrtc/Impl/ThreadLocalObject.h b/submodules/TgVoipWebrtc/Impl/ThreadLocalObject.h new file mode 100644 index 0000000000..f4c24960f5 --- /dev/null +++ b/submodules/TgVoipWebrtc/Impl/ThreadLocalObject.h @@ -0,0 +1,64 @@ +#ifndef TGVOIP_WEBRTC_THREAD_LOCAL_OBJECT_H +#define TGVOIP_WEBRTC_THREAD_LOCAL_OBJECT_H + +#include "rtc_base/thread.h" + +#include +#include + +#ifdef TGVOIP_NAMESPACE +namespace TGVOIP_NAMESPACE { +#endif + +template +class ThreadLocalObject { +private: + template + class ValueHolder { + public: + std::shared_ptr _value; + }; + +public: + ThreadLocalObject(rtc::Thread *thread, std::function generator) : + _thread(thread) { + assert(_thread != nullptr); + _valueHolder = new ThreadLocalObject::ValueHolder(); + //ValueHolder *valueHolder = _valueHolder; + _thread->Invoke(RTC_FROM_HERE, [this, generator](){ + this->_valueHolder->_value.reset(generator()); + }); + } + + ~ThreadLocalObject() { + ValueHolder *valueHolder = _valueHolder; + _thread->Invoke(RTC_FROM_HERE, [this](){ + this->_valueHolder->_value.reset(); + }); + delete valueHolder; + } + + template + void perform(FunctorT&& functor) { + //ValueHolder *valueHolder = _valueHolder; + /*_thread->PostTask(RTC_FROM_HERE, [valueHolder, f = std::forward>(f)](){ + T *value = valueHolder->_value; + assert(value != nullptr); + f(*value); + });*/ + _thread->Invoke(RTC_FROM_HERE, [this, f = std::forward(functor)](){ + assert(_valueHolder->_value != nullptr); + f(_valueHolder->_value.get()); + }); + } + +private: + rtc::Thread *_thread; + ValueHolder *_valueHolder; +}; + +#ifdef TGVOIP_NAMESPACE +} +#endif + +#endif diff --git a/submodules/TgVoipWebrtc/Impl/VideoMetalView.h b/submodules/TgVoipWebrtc/Impl/VideoMetalView.h index 3425ec74f8..eb332b65e9 100644 --- a/submodules/TgVoipWebrtc/Impl/VideoMetalView.h +++ b/submodules/TgVoipWebrtc/Impl/VideoMetalView.h @@ -6,6 +6,8 @@ #import "api/media_stream_interface.h" +#include + @class RTCVideoFrame; @interface VideoMetalView : UIView @@ -17,9 +19,7 @@ - (void)setSize:(CGSize)size; - (void)renderFrame:(nullable RTCVideoFrame *)frame; -- (void)addToTrack:(rtc::scoped_refptr)track; - -- (rtc::VideoSinkInterface *)getSink; +- (std::shared_ptr>)getSink; @end diff --git a/submodules/TgVoipWebrtc/Impl/VideoMetalView.mm b/submodules/TgVoipWebrtc/Impl/VideoMetalView.mm index 32616ffb67..125fe45d4d 100644 --- a/submodules/TgVoipWebrtc/Impl/VideoMetalView.mm +++ b/submodules/TgVoipWebrtc/Impl/VideoMetalView.mm @@ -23,26 +23,22 @@ class VideoRendererAdapterImpl : public rtc::VideoSinkInterface { public: - VideoRendererAdapterImpl(VideoMetalView *adapter) { - adapter_ = adapter; - size_ = CGSizeZero; + VideoRendererAdapterImpl(void (^frameReceived)(CGSize, RTCVideoFrame *)) { + _frameReceived = [frameReceived copy]; } void OnFrame(const webrtc::VideoFrame& nativeVideoFrame) override { RTCVideoFrame* videoFrame = NativeToObjCVideoFrame(nativeVideoFrame); - CGSize current_size = (videoFrame.rotation % 180 == 0) ? CGSizeMake(videoFrame.width, videoFrame.height) : CGSizeMake(videoFrame.height, videoFrame.width); + CGSize currentSize = (videoFrame.rotation % 180 == 0) ? CGSizeMake(videoFrame.width, videoFrame.height) : CGSizeMake(videoFrame.height, videoFrame.width); - if (!CGSizeEqualToSize(size_, current_size)) { - size_ = current_size; - [adapter_ setSize:size_]; + if (_frameReceived) { + _frameReceived(currentSize, videoFrame); } - [adapter_ renderFrame:videoFrame]; } private: - __weak VideoMetalView *adapter_; - CGSize size_; + void (^_frameReceived)(CGSize, RTCVideoFrame *); }; @interface VideoMetalView () { @@ -54,7 +50,8 @@ private: CGSize _videoFrameSize; int64_t _lastFrameTimeNs; - std::unique_ptr _sink; + CGSize _currentSize; + std::shared_ptr _sink; } @end @@ -66,7 +63,23 @@ private: if (self) { [self configure]; - _sink.reset(new VideoRendererAdapterImpl(self)); + _currentSize = CGSizeZero; + + __weak VideoMetalView *weakSelf = self; + _sink.reset(new VideoRendererAdapterImpl(^(CGSize size, RTCVideoFrame *videoFrame) { + dispatch_async(dispatch_get_main_queue(), ^{ + __strong VideoMetalView *strongSelf = weakSelf; + if (strongSelf == nil) { + return; + } + if (!CGSizeEqualToSize(size, strongSelf->_currentSize)) { + strongSelf->_currentSize = size; + [strongSelf setSize:size]; + } + + [strongSelf renderFrame:videoFrame]; + }); + })); } return self; } @@ -239,23 +252,19 @@ private: #pragma mark - RTCVideoRenderer - (void)setSize:(CGSize)size { - __weak VideoMetalView *weakSelf = self; - dispatch_async(dispatch_get_main_queue(), ^{ - __strong VideoMetalView *strongSelf = weakSelf; - if (strongSelf == nil) { - return; - } - - strongSelf->_videoFrameSize = size; - CGSize drawableSize = [strongSelf drawableSize]; - - strongSelf->_metalView.drawableSize = drawableSize; - [strongSelf setNeedsLayout]; - //[strongSelf.delegate videoView:self didChangeVideoSize:size]; - }); + assert([NSThread isMainThread]); + + _videoFrameSize = size; + CGSize drawableSize = [self drawableSize]; + + _metalView.drawableSize = drawableSize; + [self setNeedsLayout]; + //[strongSelf.delegate videoView:self didChangeVideoSize:size]; } - (void)renderFrame:(nullable RTCVideoFrame *)frame { + assert([NSThread isMainThread]); + if (!self.isEnabled) { return; } @@ -267,12 +276,10 @@ private: _videoFrame = frame; } -- (void)addToTrack:(rtc::scoped_refptr)track { - track->AddOrUpdateSink(_sink.get(), rtc::VideoSinkWants()); -} - -- (rtc::VideoSinkInterface *)getSink { - return _sink.get(); +- (std::shared_ptr>)getSink { + assert([NSThread isMainThread]); + + return _sink; } @end diff --git a/submodules/TgVoipWebrtc/PublicHeaders/TgVoip/OngoingCallThreadLocalContext.h b/submodules/TgVoipWebrtc/PublicHeaders/TgVoip/OngoingCallThreadLocalContext.h index 6ecd994552..f90f9b961c 100644 --- a/submodules/TgVoipWebrtc/PublicHeaders/TgVoip/OngoingCallThreadLocalContext.h +++ b/submodules/TgVoipWebrtc/PublicHeaders/TgVoip/OngoingCallThreadLocalContext.h @@ -76,7 +76,8 @@ typedef NS_ENUM(int32_t, OngoingCallDataSavingWebrtc) { - (void)setIsMuted:(bool)isMuted; - (void)setNetworkType:(OngoingCallNetworkTypeWebrtc)networkType; -- (void)getRemoteCameraView:(void (^_Nonnull)(UIView * _Nullable))completion; +- (void)makeIncomingVideoView:(void (^_Nonnull)(UIView * _Nullable))completion; +- (void)makeOutgoingVideoView:(void (^_Nonnull)(UIView * _Nullable))completion; - (void)addSignalingData:(NSData * _Nonnull)data; @end diff --git a/submodules/TgVoipWebrtc/Sources/OngoingCallThreadLocalContext.mm b/submodules/TgVoipWebrtc/Sources/OngoingCallThreadLocalContext.mm index f8fd37445e..08859c9538 100644 --- a/submodules/TgVoipWebrtc/Sources/OngoingCallThreadLocalContext.mm +++ b/submodules/TgVoipWebrtc/Sources/OngoingCallThreadLocalContext.mm @@ -1,6 +1,7 @@ #import #import "TgVoip.h" +#import "VideoMetalView.h" using namespace TGVOIP_NAMESPACE; @@ -189,41 +190,35 @@ static void (*InternalVoipLoggingFunction)(NSString *) = NULL; .isOutgoing = isOutgoing, }; + __weak OngoingCallThreadLocalContextWebrtc *weakSelf = self; _tgVoip = TgVoip::makeInstance( config, { derivedStateValue }, endpoints, proxyValue, callControllerNetworkTypeForType(networkType), - encryptionKey + encryptionKey, + [weakSelf, queue](TgVoipState state) { + [queue dispatch:^{ + __strong OngoingCallThreadLocalContextWebrtc *strongSelf = weakSelf; + if (strongSelf) { + [strongSelf controllerStateChanged:state]; + } + }]; + }, + [weakSelf, queue](const std::vector &data) { + NSData *mappedData = [[NSData alloc] initWithBytes:data.data() length:data.size()]; + [queue dispatch:^{ + __strong OngoingCallThreadLocalContextWebrtc *strongSelf = weakSelf; + if (strongSelf) { + [strongSelf signalingDataEmitted:mappedData]; + } + }]; + } ); _state = OngoingCallStateInitializing; _signalBars = -1; - - __weak OngoingCallThreadLocalContextWebrtc *weakSelf = self; - _tgVoip->setOnStateUpdated([weakSelf](TgVoipState state) { - __strong OngoingCallThreadLocalContextWebrtc *strongSelf = weakSelf; - if (strongSelf) { - [strongSelf controllerStateChanged:state]; - } - }); - _tgVoip->setOnSignalBarsUpdated([weakSelf](int signalBars) { - __strong OngoingCallThreadLocalContextWebrtc *strongSelf = weakSelf; - if (strongSelf) { - [strongSelf signalBarsChanged:signalBars]; - } - }); - _tgVoip->setOnCandidatesGathered([weakSelf](const std::vector &candidates) { - __strong OngoingCallThreadLocalContextWebrtc *strongSelf = weakSelf; - if (strongSelf) { - NSMutableArray *mappedCandidates = [[NSMutableArray alloc] init]; - for (auto &candidate : candidates) { - [mappedCandidates addObject:[[NSString alloc] initWithCString:candidate.c_str() encoding:NSUTF8StringEncoding]]; - } - [strongSelf candidatesGathered:mappedCandidates]; - } - }); } return self; } @@ -320,27 +315,18 @@ static void (*InternalVoipLoggingFunction)(NSString *) = NULL; } } -- (void)candidatesGathered:(NSArray *)candidates { +- (void)signalingDataEmitted:(NSData *)data { if (_sendSignalingData) { - NSData *data = [NSKeyedArchiver archivedDataWithRootObject:@{ - @"type": @"candidates", - @"data": candidates - }]; _sendSignalingData(data); } } - (void)addSignalingData:(NSData *)data { - NSDictionary *dict = [NSKeyedUnarchiver unarchiveObjectWithData:data]; - NSString *type = dict[@"type"]; - if ([type isEqualToString:@"candidates"]) { - if (_tgVoip) { - std::vector candidates; - for (NSString *string in dict[@"data"]) { - candidates.push_back([string UTF8String]); - } - _tgVoip->addRemoteCandidates(candidates); - } + if (_tgVoip) { + std::vector mappedData; + mappedData.resize(data.length); + [data getBytes:mappedData.data() length:data.length]; + _tgVoip->receiveSignalingData(mappedData); } } @@ -359,17 +345,38 @@ static void (*InternalVoipLoggingFunction)(NSString *) = NULL; } } -- (void)getRemoteCameraView:(void (^_Nonnull)(UIView * _Nullable))completion { +- (void)makeIncomingVideoView:(void (^_Nonnull)(UIView * _Nullable))completion { if (_tgVoip) { + __weak OngoingCallThreadLocalContextWebrtc *weakSelf = self; dispatch_async(dispatch_get_main_queue(), ^{ VideoMetalView *remoteRenderer = [[VideoMetalView alloc] initWithFrame:CGRectZero]; remoteRenderer.videoContentMode = UIViewContentModeScaleAspectFill; - _tgVoip->AttachVideoView(remoteRenderer); + std::shared_ptr> sink = [remoteRenderer getSink]; + __strong OngoingCallThreadLocalContextWebrtc *strongSelf = weakSelf; + if (strongSelf) { + strongSelf->_tgVoip->setIncomingVideoOutput(sink); + } - dispatch_async(dispatch_get_main_queue(), ^{ - completion(remoteRenderer); - }); + completion(remoteRenderer); + }); + } +} + +- (void)makeOutgoingVideoView:(void (^_Nonnull)(UIView * _Nullable))completion { + if (_tgVoip) { + __weak OngoingCallThreadLocalContextWebrtc *weakSelf = self; + dispatch_async(dispatch_get_main_queue(), ^{ + VideoMetalView *remoteRenderer = [[VideoMetalView alloc] initWithFrame:CGRectZero]; + remoteRenderer.videoContentMode = UIViewContentModeScaleAspectFill; + + std::shared_ptr> sink = [remoteRenderer getSink]; + __strong OngoingCallThreadLocalContextWebrtc *strongSelf = weakSelf; + if (strongSelf) { + strongSelf->_tgVoip->setOutgoingVideoOutput(sink); + } + + completion(remoteRenderer); }); } }