From 829408494ddb9da4f0b3d674324529ecc4b0271b Mon Sep 17 00:00:00 2001 From: huiping_guo Date: Thu, 10 Feb 2022 17:49:04 +0900 Subject: [PATCH 1/3] =?UTF-8?q?thread=20block=E3=81=95=E3=82=8C=E3=81=9F?= =?UTF-8?q?=E3=81=A8=E3=81=8D=E3=81=AB=E3=83=A1=E3=83=A2=E3=83=AA=E3=83=BC?= =?UTF-8?q?=E4=BD=BF=E7=94=A8=E9=87=8F=E6=80=A5=E3=81=AB=E5=A2=97=E3=81=88?= =?UTF-8?q?=E3=81=AA=E3=81=84=E3=82=88=E3=81=86=E3=81=AB=E3=81=97=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Sources/Kitsunebi/AnimationView.swift | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/Sources/Kitsunebi/AnimationView.swift b/Sources/Kitsunebi/AnimationView.swift index 5c75857..75a23fc 100644 --- a/Sources/Kitsunebi/AnimationView.swift +++ b/Sources/Kitsunebi/AnimationView.swift @@ -18,7 +18,7 @@ open class PlayerView: UIView { override open class var layerClass: Swift.AnyClass { return CAMetalLayer.self } - private var gpuLayer: LayerClass { self.layer as! LayerClass } + private lazy var gpuLayer: LayerClass = { fatalError("gpuLayer must be init") }() private let renderQueue: DispatchQueue = .global(qos: .userInitiated) private let commandQueue: MTLCommandQueue private let textureCache: CVMetalTextureCache @@ -60,6 +60,8 @@ open class PlayerView: UIView { super.init(frame: frame) applicationHandler.delegate = self backgroundColor = .clear + + gpuLayer = self.layer as! LayerClass gpuLayer.isOpaque = false gpuLayer.drawsAsynchronously = true gpuLayer.contentsGravity = .resizeAspectFill @@ -84,6 +86,8 @@ open class PlayerView: UIView { super.init(coder: aDecoder) applicationHandler.delegate = self backgroundColor = .clear + + gpuLayer = self.layer as! LayerClass gpuLayer.isOpaque = false gpuLayer.drawsAsynchronously = true gpuLayer.contentsGravity = .resizeAspectFill @@ -192,16 +196,13 @@ open class PlayerView: UIView { extension PlayerView: VideoEngineUpdateDelegate { internal func didOutputFrame(_ frame: Frame) { guard applicationHandler.isActive else { return } - DispatchQueue.main.async { [weak self] in - /// `gpuLayer` must access within main-thread. - guard let nextDrawable = self?.gpuLayer.nextDrawable() else { return } - self?.gpuLayer.drawableSize = frame.size - self?.renderQueue.async { [weak self] in - do { - try self?.renderImage(with: frame, to: nextDrawable) - } catch { - self?.clear(nextDrawable: nextDrawable) - } + guard let nextDrawable = gpuLayer.nextDrawable() else { return } + gpuLayer.drawableSize = frame.size + renderQueue.async { [weak self] in + do { + try self?.renderImage(with: frame, to: nextDrawable) + } catch { + self?.clear(nextDrawable: nextDrawable) } } } From 28c0d3c9678ae2d7311e02d05f21eb6677b9754a Mon Sep 17 00:00:00 2001 From: huiping_guo Date: Thu, 10 Feb 2022 22:37:44 +0900 Subject: [PATCH 2/3] =?UTF-8?q?=E3=81=BB=E3=81=8B=E3=81=AEmain=20thread?= =?UTF-8?q?=E4=BD=BF=E3=81=A3=E3=81=A6=E3=82=8B=E3=81=A8=E3=81=93=E3=82=8D?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E3=81=A8=E3=82=B3=E3=83=A1=E3=83=B3=E3=83=88?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Sources/Kitsunebi/AnimationView.swift | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/Sources/Kitsunebi/AnimationView.swift b/Sources/Kitsunebi/AnimationView.swift index 75a23fc..d3f8be8 100644 --- a/Sources/Kitsunebi/AnimationView.swift +++ b/Sources/Kitsunebi/AnimationView.swift @@ -18,6 +18,7 @@ open class PlayerView: UIView { override open class var layerClass: Swift.AnyClass { return CAMetalLayer.self } + // self.layer main thread使う必要あるので、事前に持つことでmain thread以外のthreadでも使えるように private lazy var gpuLayer: LayerClass = { fatalError("gpuLayer must be init") }() private let renderQueue: DispatchQueue = .global(qos: .userInitiated) private let commandQueue: MTLCommandQueue @@ -126,12 +127,9 @@ open class PlayerView: UIView { } private func clear() { - DispatchQueue.main.async { [weak self] in - /// `gpuLayer` must access within main-thread. + renderQueue.async { [weak self] in guard let nextDrawable = self?.gpuLayer.nextDrawable() else { return } - self?.renderQueue.async { [weak self] in - self?.clear(nextDrawable: nextDrawable) - } + self?.clear(nextDrawable: nextDrawable) } } @@ -196,13 +194,14 @@ open class PlayerView: UIView { extension PlayerView: VideoEngineUpdateDelegate { internal func didOutputFrame(_ frame: Frame) { guard applicationHandler.isActive else { return } - guard let nextDrawable = gpuLayer.nextDrawable() else { return } - gpuLayer.drawableSize = frame.size + renderQueue.async { [weak self] in + guard let self = self, let nextDrawable = self.gpuLayer.nextDrawable() else { return } + self.gpuLayer.drawableSize = frame.size do { - try self?.renderImage(with: frame, to: nextDrawable) + try self.renderImage(with: frame, to: nextDrawable) } catch { - self?.clear(nextDrawable: nextDrawable) + self.clear(nextDrawable: nextDrawable) } } } From 4e7ccf9e4af422ca3da975689a594b0c88365a40 Mon Sep 17 00:00:00 2001 From: huiping_guo Date: Wed, 26 Jan 2022 22:43:19 +0900 Subject: [PATCH 3/3] =?UTF-8?q?=E5=8B=95=E7=94=BB=E5=86=8D=E7=94=9Fvideo?= =?UTF-8?q?=20range=E3=81=8Cfull=20range=E3=81=A8=E3=81=97=E3=81=A6?= =?UTF-8?q?=E4=BD=BF=E3=82=8F=E3=82=8C=E3=81=9F=E5=95=8F=E9=A1=8C=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3=20*=20mp4=E5=86=8D=E7=94=9F=E3=81=A7=E5=8D=8A?= =?UTF-8?q?=E9=80=8F=E6=98=8E=E3=81=AB=E3=81=AA=E3=82=8B=E5=95=8F=E9=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Sources/Kitsunebi/AnimationView.swift | 28 +++++++++++--- Sources/Kitsunebi/Asset.swift | 19 ++++++--- Sources/Kitsunebi/MTLDevice+.swift | 2 +- Sources/Kitsunebi/VideoEngine.swift | 8 ++-- Sources/Kitsunebi/default.metal | 55 +++++++++++++++++++++++---- 5 files changed, 91 insertions(+), 21 deletions(-) diff --git a/Sources/Kitsunebi/AnimationView.swift b/Sources/Kitsunebi/AnimationView.swift index d3f8be8..778ba88 100644 --- a/Sources/Kitsunebi/AnimationView.swift +++ b/Sources/Kitsunebi/AnimationView.swift @@ -23,7 +23,8 @@ open class PlayerView: UIView { private let renderQueue: DispatchQueue = .global(qos: .userInitiated) private let commandQueue: MTLCommandQueue private let textureCache: CVMetalTextureCache - private let pipelineState: MTLRenderPipelineState + private let mp4VideoRangePipelineState: MTLRenderPipelineState + private let hevcVideoRangePipelineState: MTLRenderPipelineState private var applicationHandler = ApplicationHandler() public weak var delegate: PlayerViewDelegate? = nil @@ -52,12 +53,16 @@ open class PlayerView: UIView { guard let metalLib = try? device.makeLibrary(URL: Bundle.module.defaultMetalLibraryURL) else { return nil } - guard let pipelineState = try? device.makeRenderPipelineState(metalLib: metalLib) else { + guard let mp4VideoRangePipelineState = try? device.makeRenderPipelineState(metalLib: metalLib, fragmentFunctionName: "mp4VideoRangeFragmentShader") else { + return nil + } + guard let hevcVideoRangePipelineState = try? device.makeRenderPipelineState(metalLib: metalLib, fragmentFunctionName: "hevcVideoRangeFragmentShader") else { return nil } self.commandQueue = commandQueue self.textureCache = textureCache - self.pipelineState = pipelineState + self.mp4VideoRangePipelineState = mp4VideoRangePipelineState + self.hevcVideoRangePipelineState = hevcVideoRangePipelineState super.init(frame: frame) applicationHandler.delegate = self backgroundColor = .clear @@ -78,12 +83,16 @@ open class PlayerView: UIView { guard let metalLib = try? device.makeLibrary(URL: Bundle.module.defaultMetalLibraryURL) else { return nil } - guard let pipelineState = try? device.makeRenderPipelineState(metalLib: metalLib) else { + guard let mp4VideoRangePipelineState = try? device.makeRenderPipelineState(metalLib: metalLib, fragmentFunctionName: "mp4VideoRangeFragmentShader") else { + return nil + } + guard let hevcVideoRangePipelineState = try? device.makeRenderPipelineState(metalLib: metalLib, fragmentFunctionName: "hevcVideoRangeFragmentShader") else { return nil } self.commandQueue = commandQueue self.textureCache = textureCache - self.pipelineState = pipelineState + self.mp4VideoRangePipelineState = mp4VideoRangePipelineState + self.hevcVideoRangePipelineState = hevcVideoRangePipelineState super.init(coder: aDecoder) applicationHandler.delegate = self backgroundColor = .clear @@ -103,7 +112,16 @@ open class PlayerView: UIView { private func renderImage(with frame: Frame, to nextDrawable: CAMetalDrawable) throws { let (baseYTexture, baseCbCrTexture, alphaYTexture) = try makeTexturesFrom(frame) + + let pipelineState: MTLRenderPipelineState + switch frame { + case .yCbCrWithA(_, _): + pipelineState = mp4VideoRangePipelineState + case .yCbCrA(_): + pipelineState = hevcVideoRangePipelineState + } + let renderDesc = MTLRenderPassDescriptor() renderDesc.colorAttachments[0].texture = nextDrawable.texture renderDesc.colorAttachments[0].loadAction = .clear diff --git a/Sources/Kitsunebi/Asset.swift b/Sources/Kitsunebi/Asset.swift index 95c669b..59112a1 100644 --- a/Sources/Kitsunebi/Asset.swift +++ b/Sources/Kitsunebi/Asset.swift @@ -8,16 +8,25 @@ import AVFoundation final class Asset { - private let outputSettings: [String: Any] = [ - kCVPixelBufferMetalCompatibilityKey as String: true - ] + private var outputSettings: [String: Any] { + if let pixelFormatType = pixelFormatType { + return [ + kCVPixelBufferPixelFormatTypeKey as String: pixelFormatType, + kCVPixelBufferMetalCompatibilityKey as String: true + ] + } + + return [kCVPixelBufferMetalCompatibilityKey as String: true] + } let asset: AVURLAsset + private let pixelFormatType: OSType? private var reader: AVAssetReader? = nil private var output: AVAssetReaderTrackOutput? = nil var status: AVAssetReader.Status? { reader?.status } - init(url: URL) { - asset = AVURLAsset(url: url) + init(url: URL, pixelFormatType: OSType? = nil) { + self.asset = AVURLAsset(url: url) + self.pixelFormatType = pixelFormatType } func reset() throws { diff --git a/Sources/Kitsunebi/MTLDevice+.swift b/Sources/Kitsunebi/MTLDevice+.swift index 8ecc1c7..08f745f 100644 --- a/Sources/Kitsunebi/MTLDevice+.swift +++ b/Sources/Kitsunebi/MTLDevice+.swift @@ -51,7 +51,7 @@ extension MTLDevice { metalLib: MTLLibrary, pixelFormat: MTLPixelFormat = .bgra8Unorm, vertexFunctionName: String = "vertexShader", - fragmentFunctionName: String = "fragmentShader" + fragmentFunctionName: String ) throws -> MTLRenderPipelineState { let pipelineDesc = MTLRenderPipelineDescriptor() pipelineDesc.vertexFunction = metalLib.makeFunction(name: vertexFunctionName) diff --git a/Sources/Kitsunebi/VideoEngine.swift b/Sources/Kitsunebi/VideoEngine.swift index fd4cbd6..1fe1f20 100644 --- a/Sources/Kitsunebi/VideoEngine.swift +++ b/Sources/Kitsunebi/VideoEngine.swift @@ -37,8 +37,9 @@ internal class VideoEngine: NSObject { private lazy var currentFrameIndex: Int = 0 public init(base baseVideoURL: URL, alpha alphaVideoURL: URL, fps: Int) { - let baseAsset = Asset(url: baseVideoURL) - let alphaAsset = Asset(url: alphaVideoURL) + // video range, full range両方くる可能性があるので、video rangeに統一 + let baseAsset = Asset(url: baseVideoURL, pixelFormatType: kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange) + let alphaAsset = Asset(url: alphaVideoURL, pixelFormatType: kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange) asset = .yCbCrWithA(yCbCr: baseAsset, a: alphaAsset) fpsKeeper = FPSKeeper(fps: fps) super.init() @@ -46,7 +47,8 @@ internal class VideoEngine: NSObject { } public init(hevcWithAlpha hevcWithAlphaVideoURL: URL, fps: Int) { - let hevcWithAlphaAsset = Asset(url: hevcWithAlphaVideoURL) + // video range, full range両方くる可能性があるので、video rangeに統一 + let hevcWithAlphaAsset = Asset(url: hevcWithAlphaVideoURL, pixelFormatType: kCVPixelFormatType_420YpCbCr8VideoRange_8A_TriPlanar) asset = .yCbCrA(yCbCrA: hevcWithAlphaAsset) fpsKeeper = FPSKeeper(fps: fps) super.init() diff --git a/Sources/Kitsunebi/default.metal b/Sources/Kitsunebi/default.metal index 3059d73..09e95f3 100644 --- a/Sources/Kitsunebi/default.metal +++ b/Sources/Kitsunebi/default.metal @@ -23,7 +23,39 @@ vertex ColorInOut vertexShader(uint vid [[ vertex_id ]]) { return vertices[vid]; } -fragment float4 fragmentShader(ColorInOut in [[ stage_in ]], +fragment float4 mp4VideoRangeFragmentShader(ColorInOut in [[ stage_in ]], + texture2d baseYTexture [[ texture(0) ]], + texture2d alphaYTexture [[ texture(1) ]], + texture2d baseCbCrTexture [[ texture(2) ]]) { + constexpr sampler colorSampler; + const float4x4 ycbcrToRGBTransform = float4x4( + float4(+1.0000f, +1.0000f, +1.0000f, +0.0000f), + float4(+0.0000f, -0.3441f, +1.7720f, +0.0000f), + float4(+1.4020f, -0.7141f, +0.0000f, +0.0000f), + float4(-0.7010f, +0.5291f, -0.8860f, +1.0000f) + ); + float4 baseYUVColor = float4(baseYTexture.sample(colorSampler, in.texCoords).r, + baseCbCrTexture.sample(colorSampler, in.texCoords).rg, + 1.0f); + // yuv video range to full range + baseYUVColor.r = (baseYUVColor.r - (16.0f/255.0f)) * (255.0f/(235.0f-16.0f)); + baseYUVColor.g = (baseYUVColor.g - (16.0f/255.0f)) * (255.0f/(240.0f-16.0f)); + baseYUVColor.b = (baseYUVColor.b - (16.0f/255.0f)) * (255.0f/(240.0f-16.0f)); + + // yuv to rgb + float4 baseColor = ycbcrToRGBTransform * baseYUVColor; + + + // get alpha value + float alphaColor = alphaYTexture.sample(colorSampler, in.texCoords).r; + // video range to full range + alphaColor = (alphaColor - (16.0f/255.0f)) * (255.0f/(235.0f-16.0f)); + + return float4(baseColor.r, baseColor.g, baseColor.b, alphaColor); +} + + +fragment float4 hevcVideoRangeFragmentShader(ColorInOut in [[ stage_in ]], texture2d baseYTexture [[ texture(0) ]], texture2d alphaYTexture [[ texture(1) ]], texture2d baseCbCrTexture [[ texture(2) ]]) { @@ -35,13 +67,22 @@ fragment float4 fragmentShader(ColorInOut in [[ stage_in ]], float4(-0.7010f, +0.5291f, -0.8860f, +1.0000f) ); - float4 baseColor = ycbcrToRGBTransform * float4(baseYTexture.sample(colorSampler, in.texCoords).r, - baseCbCrTexture.sample(colorSampler, in.texCoords).rg, - 1.0); + float4 baseYUVColor = float4(baseYTexture.sample(colorSampler, in.texCoords).r, + baseCbCrTexture.sample(colorSampler, in.texCoords).rg, + 1.0f); - float4 alphaColor = alphaYTexture.sample(colorSampler, in.texCoords).r; + // yuv video range to full range + baseYUVColor.r = (baseYUVColor.r - (16.0f/255.0f)) * (255.0f/(235.0f-16.0f)); + baseYUVColor.g = (baseYUVColor.g - (16.0f/255.0f)) * (255.0f/(240.0f-16.0f)); + baseYUVColor.b = (baseYUVColor.b - (16.0f/255.0f)) * (255.0f/(240.0f-16.0f)); - return float4(baseColor.r, baseColor.g, baseColor.b, alphaColor.r); -} + // yuv to rgb + float4 baseColor = ycbcrToRGBTransform * baseYUVColor; + + // kCVPixelFormatType_420YpCbCr8VideoRange_8A_TriPlanar + // alphaはfull rangeのため、変更必要ない + float alphaColor = alphaYTexture.sample(colorSampler, in.texCoords).r; + return float4(baseColor.r, baseColor.g, baseColor.b, alphaColor); +}