From 1e2695839a26f43b69081a905811fd0dff5d5ef9 Mon Sep 17 00:00:00 2001 From: Ilya Laktyushin Date: Thu, 27 Jan 2022 17:56:25 +0300 Subject: [PATCH] Video Stickers Fixes --- .../Sources/VideoStickerFrameSource.swift | 4 +- .../FFMpegMediaVideoFrameDecoder.swift | 40 +++++++++---------- .../CachedResourceRepresentations.swift | 2 +- .../Sources/AnimatedStickerUtils.swift | 2 +- .../ChatMediaInputStickerGridItem.swift | 2 +- .../ChatMediaInputStickerPackItem.swift | 22 +++++++--- .../PublicHeaders/YuvConversion/YUV.h | 2 +- submodules/YuvConversion/Sources/YUV.m | 31 ++++++++------ 8 files changed, 58 insertions(+), 47 deletions(-) diff --git a/submodules/AnimatedStickerNode/Sources/VideoStickerFrameSource.swift b/submodules/AnimatedStickerNode/Sources/VideoStickerFrameSource.swift index ec870feac1..66046ae465 100644 --- a/submodules/AnimatedStickerNode/Sources/VideoStickerFrameSource.swift +++ b/submodules/AnimatedStickerNode/Sources/VideoStickerFrameSource.swift @@ -299,13 +299,13 @@ final class VideoStickerDirectFrameSource: AnimatedStickerFrameSource { self.height = height self.bytesPerRow = DeviceGraphicsContextSettings.shared.bytesPerRow(forWidth: Int(self.width)) self.currentFrame = 0 - + self.cache = cachePathPrefix.flatMap { cachePathPrefix in VideoStickerFrameSourceCache(queue: queue, pathPrefix: cachePathPrefix, width: width, height: height) } self.source = SoftwareVideoSource(path: path, hintVP9: true) - self.frameRate = self.source.getFramerate() + self.frameRate = min(30, self.source.getFramerate()) self.frameCount = (self.cache?.frameCount).flatMap { Int($0) } ?? 0 } diff --git a/submodules/MediaPlayer/Sources/FFMpegMediaVideoFrameDecoder.swift b/submodules/MediaPlayer/Sources/FFMpegMediaVideoFrameDecoder.swift index 2dd62a4297..b28f899c46 100644 --- a/submodules/MediaPlayer/Sources/FFMpegMediaVideoFrameDecoder.swift +++ b/submodules/MediaPlayer/Sources/FFMpegMediaVideoFrameDecoder.swift @@ -251,11 +251,7 @@ public final class FFMpegMediaVideoFrameDecoder: MediaTrackFrameDecoder { case .YUV: pixelFormat = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange case .YUVA: -// if #available(iOS 13.0, *) { -// pixelFormat = kCVPixelFormatType_420YpCbCr8VideoRange_8A_TriPlanar -// } else { - pixelFormat = kCVPixelFormatType_32ARGB -// } + pixelFormat = kCVPixelFormatType_32ARGB default: pixelFormat = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange } @@ -286,22 +282,6 @@ public final class FFMpegMediaVideoFrameDecoder: MediaTrackFrameDecoder { return nil } - let srcPlaneSize = Int(frame.lineSize[1]) * Int(frame.height / 2) - let uvPlaneSize = srcPlaneSize * 2 - - let uvPlane: UnsafeMutablePointer - if let (existingUvPlane, existingUvPlaneSize) = self.uvPlane, existingUvPlaneSize == uvPlaneSize { - uvPlane = existingUvPlane - } else { - if let (existingDstPlane, _) = self.uvPlane { - free(existingDstPlane) - } - uvPlane = malloc(uvPlaneSize)!.assumingMemoryBound(to: UInt8.self) - self.uvPlane = (uvPlane, uvPlaneSize) - } - - fillDstPlane(uvPlane, frame.data[1]!, frame.data[2]!, srcPlaneSize) - let status = CVPixelBufferLockBaseAddress(pixelBuffer, []) if status != kCVReturnSuccess { return nil @@ -310,8 +290,24 @@ public final class FFMpegMediaVideoFrameDecoder: MediaTrackFrameDecoder { var base: UnsafeMutableRawPointer if pixelFormat == kCVPixelFormatType_32ARGB { let bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer) - decodeYUVAPlanesToRGBA(frame.data[0], uvPlane, frame.data[3], CVPixelBufferGetBaseAddress(pixelBuffer)?.assumingMemoryBound(to: UInt8.self), Int32(frame.width), Int32(frame.height), Int32(bytesPerRow)) + decodeYUVAPlanesToRGBA(frame.data[0], Int32(frame.lineSize[0]), frame.data[1], Int32(frame.lineSize[1]), frame.data[2], Int32(frame.lineSize[2]), frame.data[3], CVPixelBufferGetBaseAddress(pixelBuffer)?.assumingMemoryBound(to: UInt8.self), Int32(frame.width), Int32(frame.height), Int32(bytesPerRow)) } else { + let srcPlaneSize = Int(frame.lineSize[1]) * Int(frame.height / 2) + let uvPlaneSize = srcPlaneSize * 2 + + let uvPlane: UnsafeMutablePointer + if let (existingUvPlane, existingUvPlaneSize) = self.uvPlane, existingUvPlaneSize == uvPlaneSize { + uvPlane = existingUvPlane + } else { + if let (existingDstPlane, _) = self.uvPlane { + free(existingDstPlane) + } + uvPlane = malloc(uvPlaneSize)!.assumingMemoryBound(to: UInt8.self) + self.uvPlane = (uvPlane, uvPlaneSize) + } + + fillDstPlane(uvPlane, frame.data[1]!, frame.data[2]!, srcPlaneSize) + let bytesPerRowY = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0) let bytesPerRowUV = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1) let bytesPerRowA = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 2) diff --git a/submodules/MediaResources/Sources/CachedResourceRepresentations.swift b/submodules/MediaResources/Sources/CachedResourceRepresentations.swift index 2a5099e420..a789c38baf 100644 --- a/submodules/MediaResources/Sources/CachedResourceRepresentations.swift +++ b/submodules/MediaResources/Sources/CachedResourceRepresentations.swift @@ -301,7 +301,7 @@ public final class CachedVideoStickerRepresentation: CachedMediaResourceRepresen public let height: Int32 public var uniqueId: String { - let version: Int = 0 + let version: Int = 1 return "video-sticker-\(self.width)x\(self.height)-v\(version)" } diff --git a/submodules/TelegramAnimatedStickerNode/Sources/AnimatedStickerUtils.swift b/submodules/TelegramAnimatedStickerNode/Sources/AnimatedStickerUtils.swift index 1f45bc7c4f..d6226d4289 100644 --- a/submodules/TelegramAnimatedStickerNode/Sources/AnimatedStickerUtils.swift +++ b/submodules/TelegramAnimatedStickerNode/Sources/AnimatedStickerUtils.swift @@ -288,7 +288,7 @@ public func cacheVideoStickerFrames(path: String, size: CGSize, cacheKey: String let _ = file.write(data, count: length) } - var fps: Int32 = Int32(source.getFramerate()) + var fps: Int32 = Int32(min(30, source.getFramerate())) var frameCount: Int32 = 0 writeData(&fps, length: 4) writeData(&frameCount, length: 4) diff --git a/submodules/TelegramUI/Sources/ChatMediaInputStickerGridItem.swift b/submodules/TelegramUI/Sources/ChatMediaInputStickerGridItem.swift index 069bed8229..f59ce023b7 100644 --- a/submodules/TelegramUI/Sources/ChatMediaInputStickerGridItem.swift +++ b/submodules/TelegramUI/Sources/ChatMediaInputStickerGridItem.swift @@ -282,7 +282,7 @@ final class ChatMediaInputStickerGridItemNode: GridItemNode { let dimensions = item.stickerItem.file.dimensions ?? PixelDimensions(width: 512, height: 512) let fittedSize = item.large ? CGSize(width: 384.0, height: 384.0) : CGSize(width: 160.0, height: 160.0) if item.stickerItem.file.isVideoSticker { - self.imageNode.setSignal(chatMessageSticker(account: item.account, file: item.stickerItem.file, small: !item.large, synchronousLoad: synchronousLoads && isVisible)) + self.imageNode.setSignal(chatMessageSticker(account: item.account, file: item.stickerItem.file, small: false, synchronousLoad: synchronousLoads && isVisible)) } else { self.imageNode.setSignal(chatMessageAnimatedSticker(postbox: item.account.postbox, file: item.stickerItem.file, small: false, size: dimensions.cgSize.aspectFitted(fittedSize))) } diff --git a/submodules/TelegramUI/Sources/ChatMediaInputStickerPackItem.swift b/submodules/TelegramUI/Sources/ChatMediaInputStickerPackItem.swift index 2e18981162..823cbb0a39 100644 --- a/submodules/TelegramUI/Sources/ChatMediaInputStickerPackItem.swift +++ b/submodules/TelegramUI/Sources/ChatMediaInputStickerPackItem.swift @@ -214,7 +214,6 @@ final class ChatMediaInputStickerPackItemNode: ListViewItemNode { if self.currentThumbnailItem != thumbnailItem { self.currentThumbnailItem = thumbnailItem - let thumbnailDimensions = PixelDimensions(width: 512, height: 512) if let thumbnailItem = thumbnailItem { switch thumbnailItem { case let .still(representation): @@ -236,6 +235,7 @@ final class ChatMediaInputStickerPackItemNode: ListViewItemNode { animatedStickerNode = AnimatedStickerNode() animatedStickerNode.started = { [weak self] in self?.imageNode.isHidden = true + self?.removePlaceholder(animated: false) } self.animatedStickerNode = animatedStickerNode if let placeholderNode = self.placeholderNode { @@ -251,13 +251,23 @@ final class ChatMediaInputStickerPackItemNode: ListViewItemNode { self.stickerFetchedDisposable.set(fetchedMediaResource(mediaBox: account.postbox.mediaBox, reference: resourceReference).start()) } } - - if let placeholderNode = self.placeholderNode { - let imageSize = boundingImageSize - placeholderNode.update(backgroundColor: nil, foregroundColor: theme.chat.inputMediaPanel.stickersSectionTextColor.blitOver(theme.chat.inputPanel.panelBackgroundColor, alpha: 0.4), shimmeringColor: theme.chat.inputMediaPanel.panelHighlightedIconBackgroundColor.withMultipliedAlpha(0.2), data: info.immediateThumbnailData, size: imageSize, imageSize: thumbnailDimensions.cgSize) + + self.updateIsHighlighted() + } + + if let placeholderNode = self.placeholderNode { + var imageSize = PixelDimensions(width: 512, height: 512) + var immediateThumbnailData: Data? + if let data = info.immediateThumbnailData { + if info.flags.contains(.isVideo) { + imageSize = PixelDimensions(width: 100, height: 100) + } + immediateThumbnailData = data + } else if let data = item?.file.immediateThumbnailData { + immediateThumbnailData = data } - self.updateIsHighlighted() + placeholderNode.update(backgroundColor: nil, foregroundColor: theme.chat.inputMediaPanel.stickersSectionTextColor.blitOver(theme.chat.inputPanel.panelBackgroundColor, alpha: 0.4), shimmeringColor: theme.chat.inputMediaPanel.panelHighlightedIconBackgroundColor.withMultipliedAlpha(0.2), data: immediateThumbnailData, size: boundingImageSize, imageSize: imageSize.cgSize) } self.containerNode.frame = CGRect(origin: CGPoint(x: 0.0, y: 0.0), size: expandedBoundingSize) diff --git a/submodules/YuvConversion/PublicHeaders/YuvConversion/YUV.h b/submodules/YuvConversion/PublicHeaders/YuvConversion/YUV.h index 18b1f3bfff..dfdf99fb01 100644 --- a/submodules/YuvConversion/PublicHeaders/YuvConversion/YUV.h +++ b/submodules/YuvConversion/PublicHeaders/YuvConversion/YUV.h @@ -4,4 +4,4 @@ void encodeRGBAToYUVA(uint8_t *yuva, uint8_t const *argb, int width, int height, void resizeAndEncodeRGBAToYUVA(uint8_t *yuva, uint8_t const *argb, int width, int height, int bytesPerRow, int originalWidth, int originalHeight, int originalBytesPerRow); void decodeYUVAToRGBA(uint8_t const *yuva, uint8_t *argb, int width, int height, int bytesPerRow); -void decodeYUVAPlanesToRGBA(uint8_t const *yPlane, uint8_t const *uvPlane, uint8_t const *alphaPlane, uint8_t *argb, int width, int height, int bytesPerRow); +void decodeYUVAPlanesToRGBA(uint8_t const *srcYpData, int srcYpBytesPerRow, uint8_t const *srcCbData, int srcCbBytesPerRow, uint8_t const *srcCrData, int srcCrBytesPerRow, uint8_t const *alphaData, uint8_t *argb, int width, int height, int bytesPerRow); diff --git a/submodules/YuvConversion/Sources/YUV.m b/submodules/YuvConversion/Sources/YUV.m index 7a51d1b892..b2445ba58f 100644 --- a/submodules/YuvConversion/Sources/YUV.m +++ b/submodules/YuvConversion/Sources/YUV.m @@ -169,7 +169,7 @@ void decodeYUVAToRGBA(uint8_t const *yuva, uint8_t *argb, int width, int height, } } -void decodeYUVAPlanesToRGBA(uint8_t const *yPlane, uint8_t const *uvPlane, uint8_t const *alphaPlane, uint8_t *argb, int width, int height, int bytesPerRow) { +void decodeYUVAPlanesToRGBA(uint8_t const *srcYpData, int srcYpBytesPerRow, uint8_t const *srcCbData, int srcCbBytesPerRow, uint8_t const *srcCrData, int srcCrBytesPerRow, uint8_t const *alphaData, uint8_t *argb, int width, int height, int bytesPerRow) { static vImage_YpCbCrToARGB info; static dispatch_once_t onceToken; dispatch_once(&onceToken, ^{ @@ -180,31 +180,36 @@ void decodeYUVAPlanesToRGBA(uint8_t const *yPlane, uint8_t const *uvPlane, uint8 vImage_Error error = kvImageNoError; vImage_Buffer srcYp; - srcYp.data = (void *)(yPlane + 0); + srcYp.data = (void *)(srcYpData); srcYp.width = width; srcYp.height = height; - srcYp.rowBytes = width * 1; + srcYp.rowBytes = srcYpBytesPerRow; - vImage_Buffer srcCbCr; - srcCbCr.data = (void *)(uvPlane); - srcCbCr.width = width; - srcCbCr.height = height; - srcCbCr.rowBytes = width * 1; + vImage_Buffer srcCb; + srcCb.data = (void *)(srcCbData); + srcCb.width = width / 2; + srcCb.height = height; + srcCb.rowBytes = srcCbBytesPerRow; + + vImage_Buffer srcCr; + srcCr.data = (void *)(srcCrData); + srcCr.width = width / 2; + srcCr.height = height; + srcCr.rowBytes = srcCrBytesPerRow; vImage_Buffer dest; dest.data = (void *)argb; dest.width = width; dest.height = height; dest.rowBytes = bytesPerRow; + error = vImageConvert_420Yp8_Cb8_Cr8ToARGB8888(&srcYp, &srcCb, &srcCr, &dest, &info, NULL, 0xff, kvImageDoNotTile); - error = vImageConvert_420Yp8_CbCr8ToARGB8888(&srcYp, &srcCbCr, &dest, &info, NULL, 0xff, kvImageDoNotTile); - - int i = 0; for (int y = 0; y < height; y += 1) { uint8_t *argbRow = argb + y * bytesPerRow; + int alphaRow = y * srcYpBytesPerRow; + for (int x = 0; x < width; x += 1) { - argbRow[x * 4] = alphaPlane[i]; - i += 1; + argbRow[x * 4] = alphaData[alphaRow + x]; } }