From 7c6651db3484bd8b2e68e39168da03bc1d2ca3e5 Mon Sep 17 00:00:00 2001 From: Ilya Laktyushin Date: Sat, 6 Apr 2024 18:08:20 +0400 Subject: [PATCH] [WIP] Stickers editor --- .../Sources/UniversalVideoNode.swift | 4 ++ .../Sources/AnimatedStickerNode.swift | 3 ++ .../Sources/DrawingStickerEntityView.swift | 10 +++++ .../Sources/MediaEditorValues.swift | 3 +- .../Sources/MediaEditorVideoExport.swift | 7 +++- .../MediaEditorVideoFFMpegWriter.swift | 41 +++++++++++-------- .../Sources/MediaEditorScreen.swift | 14 +++++++ 7 files changed, 63 insertions(+), 19 deletions(-) diff --git a/submodules/AccountContext/Sources/UniversalVideoNode.swift b/submodules/AccountContext/Sources/UniversalVideoNode.swift index 1828a29695..c9333755ff 100644 --- a/submodules/AccountContext/Sources/UniversalVideoNode.swift +++ b/submodules/AccountContext/Sources/UniversalVideoNode.swift @@ -103,6 +103,10 @@ public final class UniversalVideoNode: ASDisplayNode { public private(set) var ownsContentNode: Bool = false public var ownsContentNodeUpdated: ((Bool) -> Void)? + public var duration: Double { + return self.content.duration + } + private let _status = Promise() public var status: Signal { return self._status.get() diff --git a/submodules/AnimatedStickerNode/Sources/AnimatedStickerNode.swift b/submodules/AnimatedStickerNode/Sources/AnimatedStickerNode.swift index 7de93b69d0..b6b3a76dd3 100644 --- a/submodules/AnimatedStickerNode/Sources/AnimatedStickerNode.swift +++ b/submodules/AnimatedStickerNode/Sources/AnimatedStickerNode.swift @@ -201,6 +201,7 @@ public final class DefaultAnimatedStickerNodeImpl: ASDisplayNode, AnimatedSticke public var frameUpdated: (Int, Int) -> Void = { _, _ in } public private(set) var currentFrameIndex: Int = 0 public private(set) var currentFrameCount: Int = 0 + public private(set) var currentFrameRate: Int = 0 private var playFromIndex: Int? public var frameColorUpdated: ((UIColor) -> Void)? @@ -537,6 +538,7 @@ public final class DefaultAnimatedStickerNodeImpl: ASDisplayNode, AnimatedSticke strongSelf.frameUpdated(frame.index, frame.totalFrames) strongSelf.currentFrameIndex = frame.index strongSelf.currentFrameCount = frame.totalFrames + strongSelf.currentFrameRate = frameRate if frame.isLastFrame { var stopped = false @@ -652,6 +654,7 @@ public final class DefaultAnimatedStickerNodeImpl: ASDisplayNode, AnimatedSticke strongSelf.frameUpdated(frame.index, frame.totalFrames) strongSelf.currentFrameIndex = frame.index strongSelf.currentFrameCount = frame.totalFrames; + strongSelf.currentFrameRate = frameRate if frame.isLastFrame { var stopped = false diff --git a/submodules/DrawingUI/Sources/DrawingStickerEntityView.swift b/submodules/DrawingUI/Sources/DrawingStickerEntityView.swift index cd4bb1e43b..a6b7dd50b1 100644 --- a/submodules/DrawingUI/Sources/DrawingStickerEntityView.swift +++ b/submodules/DrawingUI/Sources/DrawingStickerEntityView.swift @@ -86,6 +86,16 @@ public class DrawingStickerEntityView: DrawingEntityView { var currentSize: CGSize? public var updated: () -> Void = {} + public var duration: Double? { + if let animationNode = self.animationNode, animationNode.currentFrameCount > 1 { + return Double(animationNode.currentFrameCount) / Double(animationNode.currentFrameRate) + } else if let videoNode = self.videoNode { + return videoNode.duration + } else { + return nil + } + } + init(context: AccountContext, entity: DrawingStickerEntity) { self.imageNode = TransformImageNode() diff --git a/submodules/TelegramUI/Components/MediaEditor/Sources/MediaEditorValues.swift b/submodules/TelegramUI/Components/MediaEditor/Sources/MediaEditorValues.swift index 2b466434e1..5c738d60f8 100644 --- a/submodules/TelegramUI/Components/MediaEditor/Sources/MediaEditorValues.swift +++ b/submodules/TelegramUI/Components/MediaEditor/Sources/MediaEditorValues.swift @@ -1637,6 +1637,7 @@ public func recommendedVideoExportConfiguration(values: MediaEditorValues, durat videoSettings: videoSettings, audioSettings: audioSettings, values: values, - frameRate: frameRate + frameRate: frameRate, + preferredDuration: isSticker ? duration: nil ) } diff --git a/submodules/TelegramUI/Components/MediaEditor/Sources/MediaEditorVideoExport.swift b/submodules/TelegramUI/Components/MediaEditor/Sources/MediaEditorVideoExport.swift index 638b12d10d..d6491640f5 100644 --- a/submodules/TelegramUI/Components/MediaEditor/Sources/MediaEditorVideoExport.swift +++ b/submodules/TelegramUI/Components/MediaEditor/Sources/MediaEditorVideoExport.swift @@ -56,17 +56,20 @@ public final class MediaEditorVideoExport { public var audioSettings: [String: Any] public var values: MediaEditorValues public var frameRate: Float + public var preferredDuration: Double? public init( videoSettings: [String: Any], audioSettings: [String: Any], values: MediaEditorValues, - frameRate: Float + frameRate: Float, + preferredDuration: Double? = nil ) { self.videoSettings = videoSettings self.audioSettings = audioSettings self.values = values self.frameRate = frameRate + self.preferredDuration = preferredDuration } var isSticker: Bool { @@ -284,7 +287,7 @@ public final class MediaEditorVideoExport { let duration: CMTime if self.configuration.isSticker { - duration = CMTime(seconds: 3.0, preferredTimescale: CMTimeScale(NSEC_PER_SEC)) + duration = CMTime(seconds: self.configuration.preferredDuration ?? 3.0, preferredTimescale: CMTimeScale(NSEC_PER_SEC)) } else if let mainAsset { if let trimmedDuration = self.configuration.timeRange?.duration { duration = trimmedDuration diff --git a/submodules/TelegramUI/Components/MediaEditor/Sources/MediaEditorVideoFFMpegWriter.swift b/submodules/TelegramUI/Components/MediaEditor/Sources/MediaEditorVideoFFMpegWriter.swift index c9b703df92..b42e99a435 100644 --- a/submodules/TelegramUI/Components/MediaEditor/Sources/MediaEditorVideoFFMpegWriter.swift +++ b/submodules/TelegramUI/Components/MediaEditor/Sources/MediaEditorVideoFFMpegWriter.swift @@ -3,6 +3,7 @@ import UIKit import CoreMedia import FFMpegBinding import ImageDCT +import Accelerate final class MediaEditorVideoFFMpegWriter: MediaEditorVideoExportWriter { public static let registerFFMpegGlobals: Void = { @@ -12,6 +13,15 @@ final class MediaEditorVideoFFMpegWriter: MediaEditorVideoExportWriter { let ffmpegWriter = FFMpegVideoWriter() var pool: CVPixelBufferPool? + + let conversionInfo: vImage_ARGBToYpCbCr + + init() { + var pixelRange = vImage_YpCbCrPixelRange( Yp_bias: 16, CbCr_bias: 128, YpRangeMax: 235, CbCrRangeMax: 240, YpMax: 235, YpMin: 16, CbCrMax: 240, CbCrMin: 16) + var conversionInfo = vImage_ARGBToYpCbCr() + let _ = vImageConvert_ARGBToYpCbCr_GenerateConversion(kvImage_ARGBToYpCbCrMatrix_ITU_R_709_2, &pixelRange, &conversionInfo, kvImageARGB8888, kvImage420Yp8_Cb8_Cr8, vImage_Flags(kvImageNoFlags)) + self.conversionInfo = conversionInfo + } func setup(configuration: MediaEditorVideoExport.Configuration, outputPath: String) { let _ = MediaEditorVideoFFMpegWriter.registerFFMpegGlobals @@ -91,28 +101,27 @@ final class MediaEditorVideoFFMpegWriter: MediaEditorVideoExportWriter { } func appendPixelBuffer(_ buffer: CVPixelBuffer, at time: CMTime) -> Bool { - let width = Int32(CVPixelBufferGetWidth(buffer)) - let height = Int32(CVPixelBufferGetHeight(buffer)) - let bytesPerRow = Int32(CVPixelBufferGetBytesPerRow(buffer)) + let width = CVPixelBufferGetWidth(buffer) + let height = CVPixelBufferGetHeight(buffer) + let bytesPerRow = CVPixelBufferGetBytesPerRow(buffer) - let frame = FFMpegAVFrame(pixelFormat: .YUVA, width: width, height: height) + let frame = FFMpegAVFrame(pixelFormat: .YUVA, width: Int32(width), height: Int32(height)) CVPixelBufferLockBaseAddress(buffer, CVPixelBufferLockFlags.readOnly) let src = CVPixelBufferGetBaseAddress(buffer) - splitRGBAIntoYUVAPlanes( - src, - frame.data[0], - frame.data[1], - frame.data[2], - frame.data[3], - width, - height, - bytesPerRow, - true, - true - ) + + var srcBuffer = vImage_Buffer(data: src, height: vImagePixelCount(height), width: vImagePixelCount(width), rowBytes: bytesPerRow) + var yBuffer = vImage_Buffer(data: frame.data[0], height: vImagePixelCount(height), width: vImagePixelCount(width), rowBytes: width) + var uBuffer = vImage_Buffer(data: frame.data[1], height: vImagePixelCount(height / 2), width: vImagePixelCount(width / 2), rowBytes: width / 2) + var vBuffer = vImage_Buffer(data: frame.data[2], height: vImagePixelCount(height / 2), width: vImagePixelCount(width / 2), rowBytes: width / 2) + var aBuffer = vImage_Buffer(data: frame.data[3], height: vImagePixelCount(height), width: vImagePixelCount(width), rowBytes: width) + + var outInfo = self.conversionInfo + let _ = vImageConvert_ARGB8888To420Yp8_Cb8_Cr8(&srcBuffer, &yBuffer, &uBuffer, &vBuffer, &outInfo, [ 3, 2, 1, 0 ], vImage_Flags(kvImageDoNotTile)) + vImageExtractChannel_ARGB8888(&srcBuffer, &aBuffer, 3, vImage_Flags(kvImageDoNotTile)) + CVPixelBufferUnlockBaseAddress(buffer, CVPixelBufferLockFlags.readOnly) return self.ffmpegWriter.encode(frame) diff --git a/submodules/TelegramUI/Components/MediaEditorScreen/Sources/MediaEditorScreen.swift b/submodules/TelegramUI/Components/MediaEditorScreen/Sources/MediaEditorScreen.swift index 275e4b3ea1..4fc58e4315 100644 --- a/submodules/TelegramUI/Components/MediaEditorScreen/Sources/MediaEditorScreen.swift +++ b/submodules/TelegramUI/Components/MediaEditorScreen/Sources/MediaEditorScreen.swift @@ -6847,6 +6847,20 @@ public final class MediaEditorScreen: ViewController, UIDropInteractionDelegate if case let .video(video, _) = exportSubject { duration = video.duration.seconds } + if isSticker { + duration = 3.0 + var stickerDurations: [Double] = [] + self.node.entitiesView.eachView { entityView in + if let stickerEntityView = entityView as? DrawingStickerEntityView { + if let duration = stickerEntityView.duration, duration > 0.0 { + stickerDurations.append(duration) + } + } + } + if !stickerDurations.isEmpty { + duration = stickerDurations.max() ?? 3.0 + } + } let configuration = recommendedVideoExportConfiguration(values: mediaEditor.values, duration: duration, forceFullHd: true, frameRate: 60.0, isSticker: isSticker) let outputPath = NSTemporaryDirectory() + "\(Int64.random(in: 0 ..< .max)).\(fileExtension)" let videoExport = MediaEditorVideoExport(postbox: self.context.account.postbox, subject: exportSubject, configuration: configuration, outputPath: outputPath, textScale: 2.0)