From 98bf4d524f863f8d10c14c75d5aedbe94dba9835 Mon Sep 17 00:00:00 2001 From: James Hurley Date: Wed, 29 Apr 2015 16:07:12 -0700 Subject: [PATCH 1/2] OSX Port. --- api/iOS/VCSimpleSession.mm | 8 +- filters/Basic/BasicVideoFilterBGRA.cpp | 94 ++++- filters/Basic/BasicVideoFilterBGRA.h | 1 + .../Basic/BasicVideoFilterBGRAinYUVAout.cpp | 38 +- filters/Basic/GrayscaleVideoFilter.cpp | 33 +- filters/Basic/InvertColorsVideoFilter.cpp | 43 ++- filters/Basic/SepiaVideoFilter.cpp | 43 ++- filters/IVideoFilter.hpp | 3 +- .../GLESVideoMixer.h => Apple/GLVideoMixer.h} | 42 +- .../GLVideoMixer.mm} | 363 ++++++++++++------ mixers/GenericAudioMixer.cpp | 2 +- sources/Apple/PixelBufferSource.mm | 9 +- sources/iOS/MicSource.mm | 6 +- system/util.h | 5 +- transforms/{iOS => Apple}/AACEncode.cpp | 28 +- transforms/{iOS => Apple}/AACEncode.h | 2 +- transforms/Apple/H264Encode.mm | 48 ++- transforms/IMetaData.hpp | 8 +- 18 files changed, 521 insertions(+), 255 deletions(-) rename mixers/{iOS/GLESVideoMixer.h => Apple/GLVideoMixer.h} (86%) rename mixers/{iOS/GLESVideoMixer.mm => Apple/GLVideoMixer.mm} (62%) rename transforms/{iOS => Apple}/AACEncode.cpp (96%) rename transforms/{iOS => Apple}/AACEncode.h (98%) diff --git a/api/iOS/VCSimpleSession.mm b/api/iOS/VCSimpleSession.mm index 4c236c0f..cc704752 100644 --- a/api/iOS/VCSimpleSession.mm +++ b/api/iOS/VCSimpleSession.mm @@ -38,11 +38,11 @@ of this software and associated documentation files (the "Software"), to deal # include # include # include +# include +# include # ifdef TARGET_OS_IPHONE # include # include -# include -# include # include # else /* OS X */ @@ -681,7 +681,7 @@ - (void) setupGraph { // Add video mixer - m_videoMixer = std::make_shared(self.videoSize.width, + m_videoMixer = std::make_shared(self.videoSize.width, self.videoSize.height, frameDuration); @@ -763,7 +763,7 @@ - (void) addEncodersAndPacketizers { // Add encoders - m_aacEncoder = std::make_shared(self.audioSampleRate, self.audioChannelCount, 96000); + m_aacEncoder = std::make_shared(self.audioSampleRate, self.audioChannelCount, 96000); if(SYSTEM_VERSION_GREATER_THAN_OR_EQUAL_TO(@"8.0")) { // If >= iOS 8.0 use the VideoToolbox encoder that does not write to disk. m_h264Encoder = std::make_shared(self.videoSize.width, diff --git a/filters/Basic/BasicVideoFilterBGRA.cpp b/filters/Basic/BasicVideoFilterBGRA.cpp index cb3ace41..1a5a1d19 100644 --- a/filters/Basic/BasicVideoFilterBGRA.cpp +++ b/filters/Basic/BasicVideoFilterBGRA.cpp @@ -1,18 +1,28 @@ #include -#include -#ifdef TARGET_OS_IPHONE - -#include -#include -#include -#include +#ifdef __APPLE__ +# include +# if (TARGET_OS_IPHONE) +# include +# include +# include +# else +# include +# include +# include +# define glDeleteVertexArraysOES glDeleteVertexArrays +# define glGenVertexArraysOES glGenVertexArrays +# define glBindVertexArrayOES glBindVertexArray +# endif #endif + +#include + namespace videocore { namespace filters { bool BasicVideoFilterBGRA::s_registered = BasicVideoFilterBGRA::registerFilter(); @@ -31,15 +41,17 @@ namespace videocore { namespace filters { } BasicVideoFilterBGRA::~BasicVideoFilterBGRA() { - glDeleteProgram(m_program); - glDeleteVertexArraysOES(1, &m_vao); + if(m_initialized) { + glDeleteProgram(m_program); + glDeleteVertexArraysOES(1, &m_vao); + } } const char * const BasicVideoFilterBGRA::vertexKernel() const { - KERNEL(GL_ES2_3, m_language, + FKERNEL(GL_ES2_3, m_language, attribute vec2 aPos; attribute vec2 aCoord; varying vec2 vCoord; @@ -47,9 +59,20 @@ namespace videocore { namespace filters { void main(void) { gl_Position = uMat * vec4(aPos,0.,1.); vCoord = aCoord; - } + }, + "" + ) + FKERNEL(GL_3, m_language, + in vec2 aPos; + in vec2 aCoord; + out vec2 vCoord; + uniform mat4 uMat; + void main(void) { + gl_Position = uMat * vec4(aPos,0.,1.); + vCoord = aCoord; + }, + "#version 150" ) - return nullptr; } @@ -57,13 +80,27 @@ namespace videocore { namespace filters { BasicVideoFilterBGRA::pixelKernel() const { - KERNEL(GL_ES2_3, m_language, + FKERNEL(GL_ES2_3, m_language, precision mediump float; varying vec2 vCoord; uniform sampler2D uTex0; void main(void) { gl_FragData[0] = texture2D(uTex0, vCoord); - } + }, + "" + ) + FKERNEL(GL_3, m_language, + + in vec2 vCoord; + uniform sampler2DRect uTex0; + out vec4 fragColor; + uniform vec2 uImageSize; + void main(void) { + + vec4 color = texture(uTex0, vCoord * uImageSize); + fragColor = color; + }, + "#version 150" ) return nullptr; @@ -90,6 +127,26 @@ namespace videocore { namespace filters { } break; case GL_3: + { + setProgram(build_program(vertexKernel(), pixelKernel())); + + glGenVertexArraysOES(1, &m_vao); + glBindVertexArrayOES(m_vao); + glUseProgram(m_program); + m_uMatrix = glGetUniformLocation(m_program, "uMat"); + int attrpos = glGetAttribLocation(m_program, "aPos"); + int attrtex = glGetAttribLocation(m_program, "aCoord"); + int unitex = glGetUniformLocation(m_program, "uTex0"); + m_uImageSize = glGetUniformLocation(m_program, "uImageSize"); + + glUniform1i(unitex, 0); + glEnableVertexAttribArray(attrpos); + glEnableVertexAttribArray(attrtex); + glVertexAttribPointer(attrpos, BUFFER_SIZE_POSITION, GL_FLOAT, GL_FALSE, BUFFER_STRIDE, BUFFER_OFFSET_POSITION); + glVertexAttribPointer(attrtex, BUFFER_SIZE_POSITION, GL_FLOAT, GL_FALSE, BUFFER_STRIDE, BUFFER_OFFSET_TEXTURE); + m_initialized = true; + + } break; } } @@ -109,6 +166,15 @@ namespace videocore { namespace filters { glUniformMatrix4fv(m_uMatrix, 1, GL_FALSE, &m_matrix[0][0]); break; case GL_3: + if(!m_bound) { + if(!initialized()) { + initialize(); + } + glUseProgram(m_program); + glBindVertexArrayOES(m_vao); + } + glUniformMatrix4fv(m_uMatrix, 1, GL_FALSE, &m_matrix[0][0]); + glUniform2f(m_uImageSize, m_dimensions.w, m_dimensions.h); break; } } diff --git a/filters/Basic/BasicVideoFilterBGRA.h b/filters/Basic/BasicVideoFilterBGRA.h index 75661156..b8ad64bd 100644 --- a/filters/Basic/BasicVideoFilterBGRA.h +++ b/filters/Basic/BasicVideoFilterBGRA.h @@ -53,6 +53,7 @@ namespace videocore { unsigned int m_vao; unsigned int m_uMatrix; + unsigned int m_uImageSize; bool m_initialized; bool m_bound; diff --git a/filters/Basic/BasicVideoFilterBGRAinYUVAout.cpp b/filters/Basic/BasicVideoFilterBGRAinYUVAout.cpp index 8b4b4e0c..851e6bea 100644 --- a/filters/Basic/BasicVideoFilterBGRAinYUVAout.cpp +++ b/filters/Basic/BasicVideoFilterBGRAinYUVAout.cpp @@ -1,15 +1,21 @@ #include #ifdef __APPLE__ -#include -# ifdef TARGET_OS_IPHONE -# include -# include -# include -# include +# include +# if (TARGET_OS_IPHONE) +# include +# include +# include +# else +# include +# include +# include +# define glDeleteVertexArraysOES glDeleteVertexArrays +# define glGenVertexArraysOES glGenVertexArrays +# define glBindVertexArrayOES glBindVertexArray # endif #endif - +#include namespace videocore { namespace filters { bool BasicVideoFilterBGRAinYUVAout::s_registered = BasicVideoFilterBGRAinYUVAout::registerFilter(); @@ -29,14 +35,14 @@ namespace videocore { namespace filters { BasicVideoFilterBGRAinYUVAout::~BasicVideoFilterBGRAinYUVAout() { glDeleteProgram(m_program); - glDeleteVertexArrays(1, &m_vao); + glDeleteVertexArraysOES(1, &m_vao); } const char * const BasicVideoFilterBGRAinYUVAout::vertexKernel() const { - KERNEL(GL_ES2_3, m_language, + FKERNEL(GL_ES2_3, m_language, attribute vec2 aPos; attribute vec2 aCoord; varying vec2 vCoord; @@ -44,7 +50,7 @@ namespace videocore { namespace filters { void main(void) { gl_Position = uMat * vec4(aPos,0.,1.); vCoord = aCoord; - } + }, "" ) return nullptr; @@ -54,7 +60,7 @@ namespace videocore { namespace filters { BasicVideoFilterBGRAinYUVAout::pixelKernel() const { - KERNEL(GL_ES2_3, m_language, + FKERNEL(GL_ES2_3, m_language, precision mediump float; varying vec2 vCoord; uniform sampler2D uTex0; @@ -64,7 +70,7 @@ namespace videocore { namespace filters { 0.0625, 0.500, 0.500, 1.0 ); void main(void) { gl_FragData[0] = texture2D(uTex0, vCoord) * RGBtoYUV; - } + }, "" ) return nullptr; @@ -76,8 +82,8 @@ namespace videocore { namespace filters { case GL_ES2_3: case GL_2: { setProgram(build_program(vertexKernel(), pixelKernel())); - glGenVertexArrays(1, &m_vao); - glBindVertexArray(m_vao); + glGenVertexArraysOES(1, &m_vao); + glBindVertexArrayOES(m_vao); m_uMatrix = glGetUniformLocation(m_program, "uMat"); int attrpos = glGetAttribLocation(m_program, "aPos"); int attrtex = glGetAttribLocation(m_program, "aCoord"); @@ -105,7 +111,7 @@ namespace videocore { namespace filters { initialize(); } glUseProgram(m_program); - glBindVertexArray(m_vao); + glBindVertexArrayOES(m_vao); } glUniformMatrix4fv(m_uMatrix, 1, GL_FALSE, &m_matrix[0][0]); break; @@ -119,4 +125,4 @@ namespace videocore { namespace filters { m_bound = false; } } -} \ No newline at end of file +} diff --git a/filters/Basic/GrayscaleVideoFilter.cpp b/filters/Basic/GrayscaleVideoFilter.cpp index a3b855e9..7e1b583d 100644 --- a/filters/Basic/GrayscaleVideoFilter.cpp +++ b/filters/Basic/GrayscaleVideoFilter.cpp @@ -1,17 +1,22 @@ #include -#include - - -#ifdef TARGET_OS_IPHONE - -#include -#include -#include -#include - +#ifdef __APPLE__ +# include +# if (TARGET_OS_IPHONE) +# include +# include +# include +# else +# include +# include +# include +# define glDeleteVertexArraysOES glDeleteVertexArrays +# define glGenVertexArraysOES glGenVertexArrays +# define glBindVertexArrayOES glBindVertexArray +# endif #endif +#include namespace videocore { namespace filters { @@ -39,7 +44,7 @@ namespace videocore { namespace filters { GrayscaleVideoFilter::vertexKernel() const { - KERNEL(GL_ES2_3, m_language, + FKERNEL(GL_ES2_3, m_language, attribute vec2 aPos; attribute vec2 aCoord; varying vec2 vCoord; @@ -47,7 +52,7 @@ namespace videocore { namespace filters { void main(void) { gl_Position = uMat * vec4(aPos,0.,1.); vCoord = aCoord; - } + }, "" ) return nullptr; @@ -57,7 +62,7 @@ namespace videocore { namespace filters { GrayscaleVideoFilter::pixelKernel() const { - KERNEL(GL_ES2_3, m_language, + FKERNEL(GL_ES2_3, m_language, precision mediump float; varying vec2 vCoord; uniform sampler2D uTex0; @@ -65,7 +70,7 @@ namespace videocore { namespace filters { vec4 color = texture2D(uTex0, vCoord); float gray = dot(color.rgb, vec3(0.3, 0.59, 0.11)); gl_FragColor = vec4(gray, gray, gray, color.a); - } + }, "" ) return nullptr; diff --git a/filters/Basic/InvertColorsVideoFilter.cpp b/filters/Basic/InvertColorsVideoFilter.cpp index 8e47d17f..a60a051b 100644 --- a/filters/Basic/InvertColorsVideoFilter.cpp +++ b/filters/Basic/InvertColorsVideoFilter.cpp @@ -1,18 +1,23 @@ #include -#include - - -#ifdef TARGET_OS_IPHONE - -#include -#include -#include -#include - +#ifdef __APPLE__ +# include +# if (TARGET_OS_IPHONE) +# include +# include +# include +# else +# include +# include +# include +# define glDeleteVertexArraysOES glDeleteVertexArrays +# define glGenVertexArraysOES glGenVertexArrays +# define glBindVertexArrayOES glBindVertexArray +# endif #endif +#include namespace videocore { namespace filters { bool InvertColorsVideoFilter::s_registered = InvertColorsVideoFilter::registerFilter(); @@ -32,14 +37,14 @@ namespace videocore { namespace filters { InvertColorsVideoFilter::~InvertColorsVideoFilter() { glDeleteProgram(m_program); - glDeleteVertexArrays(1, &m_vao); + glDeleteVertexArraysOES(1, &m_vao); } const char * const InvertColorsVideoFilter::vertexKernel() const { - KERNEL(GL_ES2_3, m_language, + FKERNEL(GL_ES2_3, m_language, attribute vec2 aPos; attribute vec2 aCoord; varying vec2 vCoord; @@ -47,7 +52,7 @@ namespace videocore { namespace filters { void main(void) { gl_Position = uMat * vec4(aPos,0.,1.); vCoord = aCoord; - } + }, "" ) return nullptr; @@ -57,14 +62,14 @@ namespace videocore { namespace filters { InvertColorsVideoFilter::pixelKernel() const { - KERNEL(GL_ES2_3, m_language, + FKERNEL(GL_ES2_3, m_language, precision mediump float; varying vec2 vCoord; uniform sampler2D uTex0; void main(void) { vec4 color = texture2D(uTex0, vCoord); gl_FragColor = vec4(1.0 - color.r, 1.0 - color.g, 1.0 - color.b, color.a); - } + }, "" ) return nullptr; @@ -76,8 +81,8 @@ namespace videocore { namespace filters { case GL_ES2_3: case GL_2: { setProgram(build_program(vertexKernel(), pixelKernel())); - glGenVertexArrays(1, &m_vao); - glBindVertexArray(m_vao); + glGenVertexArraysOES(1, &m_vao); + glBindVertexArrayOES(m_vao); m_uMatrix = glGetUniformLocation(m_program, "uMat"); int attrpos = glGetAttribLocation(m_program, "aPos"); int attrtex = glGetAttribLocation(m_program, "aCoord"); @@ -105,7 +110,7 @@ namespace videocore { namespace filters { initialize(); } glUseProgram(m_program); - glBindVertexArray(m_vao); + glBindVertexArrayOES(m_vao); } glUniformMatrix4fv(m_uMatrix, 1, GL_FALSE, &m_matrix[0][0]); break; @@ -119,4 +124,4 @@ namespace videocore { namespace filters { m_bound = false; } } -} \ No newline at end of file +} diff --git a/filters/Basic/SepiaVideoFilter.cpp b/filters/Basic/SepiaVideoFilter.cpp index 59cd2ba5..26159add 100644 --- a/filters/Basic/SepiaVideoFilter.cpp +++ b/filters/Basic/SepiaVideoFilter.cpp @@ -1,17 +1,22 @@ #include -#include - - -#ifdef TARGET_OS_IPHONE - -#include -#include -#include -#include - +#ifdef __APPLE__ +# include +# if (TARGET_OS_IPHONE) +# include +# include +# include +# else +# include +# include +# include +# define glDeleteVertexArraysOES glDeleteVertexArrays +# define glGenVertexArraysOES glGenVertexArrays +# define glBindVertexArrayOES glBindVertexArray +# endif #endif +#include namespace videocore { namespace filters { @@ -32,14 +37,14 @@ namespace videocore { namespace filters { SepiaVideoFilter::~SepiaVideoFilter() { glDeleteProgram(m_program); - glDeleteVertexArrays(1, &m_vao); + glDeleteVertexArraysOES(1, &m_vao); } const char * const SepiaVideoFilter::vertexKernel() const { - KERNEL(GL_ES2_3, m_language, + FKERNEL(GL_ES2_3, m_language, attribute vec2 aPos; attribute vec2 aCoord; varying vec2 vCoord; @@ -47,7 +52,7 @@ namespace videocore { namespace filters { void main(void) { gl_Position = uMat * vec4(aPos,0.,1.); vCoord = aCoord; - } + }, "" ) return nullptr; @@ -57,7 +62,7 @@ namespace videocore { namespace filters { SepiaVideoFilter::pixelKernel() const { - KERNEL(GL_ES2_3, m_language, + FKERNEL(GL_ES2_3, m_language, precision mediump float; varying vec2 vCoord; uniform sampler2D uTex0; @@ -68,7 +73,7 @@ namespace videocore { namespace filters { vec3 sepiaColor = vec3(gray) * SEPIA; color.rgb = mix(color.rgb, sepiaColor, 0.75); gl_FragColor = color; - } + }, "" ) return nullptr; @@ -80,8 +85,8 @@ namespace videocore { namespace filters { case GL_ES2_3: case GL_2: { setProgram(build_program(vertexKernel(), pixelKernel())); - glGenVertexArrays(1, &m_vao); - glBindVertexArray(m_vao); + glGenVertexArraysOES(1, &m_vao); + glBindVertexArrayOES(m_vao); m_uMatrix = glGetUniformLocation(m_program, "uMat"); int attrpos = glGetAttribLocation(m_program, "aPos"); int attrtex = glGetAttribLocation(m_program, "aCoord"); @@ -109,7 +114,7 @@ namespace videocore { namespace filters { initialize(); } glUseProgram(m_program); - glBindVertexArray(m_vao); + glBindVertexArrayOES(m_vao); } glUniformMatrix4fv(m_uMatrix, 1, GL_FALSE, &m_matrix[0][0]); break; @@ -123,4 +128,4 @@ namespace videocore { namespace filters { m_bound = false; } } -} \ No newline at end of file +} diff --git a/filters/IVideoFilter.hpp b/filters/IVideoFilter.hpp index 2e804676..7ad693a5 100644 --- a/filters/IVideoFilter.hpp +++ b/filters/IVideoFilter.hpp @@ -34,7 +34,8 @@ #include #include -#define KERNEL(_language, _target, _kernelstr) if(_language == _target){ do { return # _kernelstr ; } while(0); } +#define FKERNEL1(_language, _target, _kernelstr, _pre) if(_language == _target){ do { return _pre "\n" # _kernelstr ; } while(0); } +#define FKERNEL(_language, _target, _kernelstr, _pre) FKERNEL1(_language, _target, _kernelstr, _pre) namespace videocore { diff --git a/mixers/iOS/GLESVideoMixer.h b/mixers/Apple/GLVideoMixer.h similarity index 86% rename from mixers/iOS/GLESVideoMixer.h rename to mixers/Apple/GLVideoMixer.h index a3f3505e..7b2ef8d8 100644 --- a/mixers/iOS/GLESVideoMixer.h +++ b/mixers/Apple/GLVideoMixer.h @@ -24,8 +24,8 @@ */ -#ifndef __videocore__GLESVideoMixer__ -#define __videocore__GLESVideoMixer__ +#ifndef __videocore__GLVideoMixer__ +#define __videocore__GLVideoMixer__ #include #include @@ -41,7 +41,12 @@ #include #include -namespace videocore { namespace iOS { + +#if !TARGET_OS_IPHONE +typedef CVOpenGLTextureCacheRef CVOpenGLESTextureCacheRef; +typedef CVOpenGLTextureRef CVOpenGLESTextureRef; +#endif +namespace videocore { namespace Apple { struct SourceBuffer { @@ -49,6 +54,7 @@ namespace videocore { namespace iOS { ~SourceBuffer() { }; void setBuffer(Apple::PixelBufferRef ref, CVOpenGLESTextureCacheRef textureCache, JobQueue& jobQueue, void* glContext); + size_t bufferId() const { return m_currentBufferBacking->bufid; }; CVOpenGLESTextureRef currentTexture() const { return m_currentTexture; }; Apple::PixelBufferRef currentBuffer() const { return m_currentBuffer; }; @@ -56,24 +62,27 @@ namespace videocore { namespace iOS { void setBlends(bool blends) { m_blends = blends; }; private: typedef struct __Buffer_ { - __Buffer_(Apple::PixelBufferRef buf) : texture(nullptr), buffer(buf) {}; + __Buffer_(Apple::PixelBufferRef buf) : texture(nullptr), buffer(buf) { static int sbufid=0; bufid=sbufid++; }; ~__Buffer_() { if(texture) { CFRelease(texture); } }; Apple::PixelBufferRef buffer; CVOpenGLESTextureRef texture; std::chrono::steady_clock::time_point time; + size_t bufid; } Buffer_; std::map< CVPixelBufferRef, Buffer_ > m_pixelBuffers; Apple::PixelBufferRef m_currentBuffer; CVOpenGLESTextureRef m_currentTexture; + Buffer_* m_currentBufferBacking; bool m_blends; }; /* * Takes CVPixelBufferRef inputs and outputs a single CVPixelBufferRef that has been composited from the various sources. * Sources must output VideoBufferMetadata with their buffers. This compositor uses homogeneous coordinates. */ - class GLESVideoMixer : public IVideoMixer + static const int kOutPBCount = 2; + class GLVideoMixer : public IVideoMixer { public: @@ -86,14 +95,14 @@ namespace videocore { namespace iOS { * The parameter of this method will be a pointer to its EAGLContext. This is useful for * applications that may be capturing GLES data and do not wish to capture the mixer. */ - GLESVideoMixer(int frame_w, + GLVideoMixer(int frame_w, int frame_h, double frameDuration, CVPixelBufferPoolRef pixelBufferPool = nullptr, std::function excludeContext = nullptr); /*! Destructor */ - ~GLESVideoMixer(); + ~GLVideoMixer(); /*! IMixer::registerSource */ void registerSource(std::shared_ptr source, @@ -156,6 +165,9 @@ namespace videocore { namespace iOS { void setupGLES(std::function excludeContext); + void compose(int fbo, + std::chrono::steady_clock::time_point currentTime, + std::chrono::steady_clock::time_point lastMixTime); private: @@ -169,17 +181,21 @@ namespace videocore { namespace iOS { std::thread m_mixThread; std::mutex m_mutex; - std::condition_variable m_mixThreadCond; - + + dispatch_semaphore_t m_mixThreadSem; CVPixelBufferPoolRef m_pixelBufferPool; - CVPixelBufferRef m_pixelBuffer[2]; + CVPixelBufferRef m_pixelBuffer[kOutPBCount]; +#if TARGET_OS_IPHONE CVOpenGLESTextureCacheRef m_textureCache; - CVOpenGLESTextureRef m_texture[2]; - + CVOpenGLESTextureRef m_texture[kOutPBCount]; +#else + CVOpenGLTextureCacheRef m_textureCache; + CVOpenGLTextureRef m_texture[kOutPBCount]; +#endif void* m_callbackSession; void* m_glesCtx; - unsigned m_vbo, m_vao, m_fbo[2], m_prog, m_uMat; + unsigned m_vbo, m_vao, m_fbo[kOutPBCount], m_prog, m_uMat; int m_frameW; diff --git a/mixers/iOS/GLESVideoMixer.mm b/mixers/Apple/GLVideoMixer.mm similarity index 62% rename from mixers/iOS/GLESVideoMixer.mm rename to mixers/Apple/GLVideoMixer.mm index d8c6d641..1cdd98d2 100644 --- a/mixers/iOS/GLESVideoMixer.mm +++ b/mixers/Apple/GLVideoMixer.mm @@ -24,17 +24,31 @@ of this software and associated documentation files (the "Software"), to deal */ -#include +#include #include #include #import -#import -#import -#import -#import -#import -#import +#ifdef __APPLE__ +# include +# include +# if (TARGET_OS_IPHONE) +# include +# include +# import +# else +# include +# include +# import +# include +# define glDeleteVertexArraysOES glDeleteVertexArraysAPPLE +# define glGenVertexArraysOES glGenVertexArraysAPPLE +# define glBindVertexArrayOES glBindVertexArrayAPPLE +# define CVOpenGLESTextureCacheFlush CVOpenGLTextureCacheFlush +# define CVOpenGLESTextureGetName CVOpenGLTextureGetName +# define CVOpenGLESTextureGetTarget CVOpenGLTextureGetTarget +# endif +#endif #include @@ -44,6 +58,7 @@ of this software and associated documentation files (the "Software"), to deal // Convenience macro to dispatch an OpenGL ES job to the created videocore::JobQueue +#if TARGET_OS_IPHONE #define PERF_GL(x, dispatch) do {\ m_glJobQueue.dispatch([=](){\ EAGLContext* cur = [EAGLContext currentContext];\ @@ -54,6 +69,20 @@ of this software and associated documentation files (the "Software"), to deal [EAGLContext setCurrentContext:cur];\ });\ } while(0) +#else +#define PERF_GL(x, dispatch) do {\ +m_glJobQueue.dispatch([=](){\ +CGLContextObj cur = CGLGetCurrentContext();\ +if(m_glesCtx) {\ +CGLSetCurrentContext((CGLContextObj)m_glesCtx);\ +CGLLockContext((CGLContextObj)m_glesCtx);\ +}\ +x ;\ +CGLUnlockContext((CGLContextObj)m_glesCtx);\ +CGLSetCurrentContext(cur);\ +});\ +} while(0) +#endif // Dispatch and execute synchronously #define PERF_GL_sync(x) PERF_GL((x), enqueue_sync); // Dispatch and execute asynchronously @@ -61,16 +90,17 @@ of this software and associated documentation files (the "Software"), to deal @interface GLESObjCCallback : NSObject { - videocore::iOS::GLESVideoMixer* _mixer; + videocore::Apple::GLVideoMixer* _mixer; } -- (void) setMixer: (videocore::iOS::GLESVideoMixer*) mixer; +- (void) setMixer: (videocore::Apple::GLVideoMixer*) mixer; @end @implementation GLESObjCCallback - (instancetype) init { if((self = [super init])) { +#if TARGET_OS_IPHONE [[NSNotificationCenter defaultCenter] addObserver:self selector:@selector(notification:) name:UIApplicationDidEnterBackgroundNotification object:nil]; [[NSNotificationCenter defaultCenter] addObserver:self selector:@selector(notification:) name:UIApplicationWillEnterForegroundNotification object:nil]; - +#endif } return self; } @@ -79,6 +109,7 @@ - (void) dealloc { [super dealloc]; } - (void) notification: (NSNotification*) notification { +#if TARGET_OS_IPHONE if([notification.name isEqualToString:UIApplicationDidEnterBackgroundNotification]) { _mixer->mixPaused(true); @@ -88,13 +119,14 @@ - (void) notification: (NSNotification*) notification { _mixer->mixPaused(false); } +#endif } -- (void) setMixer: (videocore::iOS::GLESVideoMixer*) mixer +- (void) setMixer: (videocore::Apple::GLVideoMixer*) mixer { _mixer = mixer; } @end -namespace videocore { namespace iOS { +namespace videocore { namespace Apple { // ------------------------------------------------------------------------- // @@ -125,32 +157,45 @@ - (void) setMixer: (videocore::iOS::GLESVideoMixer*) mixer bool is32bit = true; is32bit = (format != kCVPixelFormatType_16LE565); - + CVOpenGLESTextureRef texture = nullptr; + const CVPixelBufferRef inCvpb = ref->cvBuffer(); + + int w = (int)CVPixelBufferGetWidth(inCvpb); + int h = (int)CVPixelBufferGetHeight(inCvpb); + + +#if TARGET_OS_IPHONE + CVReturn ret = CVOpenGLESTextureCacheCreateTextureFromImage(kCFAllocatorDefault, textureCache, - ref->cvBuffer(), + inCvpb, NULL, GL_TEXTURE_2D, is32bit ? GL_RGBA : GL_RGB, - (GLsizei)ref->width(), - (GLsizei)ref->height(), + (GLsizei)w, + (GLsizei)h, is32bit ? GL_BGRA : GL_RGB, is32bit ? GL_UNSIGNED_BYTE : GL_UNSIGNED_SHORT_5_6_5, 0, &texture); +#else + CVReturn ret; + ret = CVOpenGLTextureCacheCreateTextureFromImage(kCFAllocatorDefault, textureCache, inCvpb, NULL, &texture); +#endif ref->unlock(true); if(ret == noErr && texture) { - glBindTexture(GL_TEXTURE_2D, CVOpenGLESTextureGetName(texture)); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + GLint target = CVOpenGLESTextureGetTarget(texture); + glBindTexture(target, CVOpenGLESTextureGetName(texture)); + glTexParameteri(target, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + glTexParameteri(target, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(target, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(target, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); - auto iit = this->m_pixelBuffers.emplace(ref->cvBuffer(), ref).first; + auto iit = this->m_pixelBuffers.emplace(inCvpb, ref).first; iit->second.texture = texture; - + this->m_currentBufferBacking = &iit->second; this->m_currentBuffer = ref; this->m_currentTexture = texture; iit->second.time = now; @@ -161,7 +206,8 @@ - (void) setMixer: (videocore::iOS::GLESVideoMixer*) mixer }); flush = true; } else { - + + m_currentBufferBacking = &it->second; m_currentBuffer = ref; m_currentTexture = it->second.texture; it->second.time = now; @@ -192,11 +238,11 @@ - (void) setMixer: (videocore::iOS::GLESVideoMixer*) mixer // // // ------------------------------------------------------------------------- - GLESVideoMixer::GLESVideoMixer( int frame_w, - int frame_h, - double frameDuration, - CVPixelBufferPoolRef pool, - std::function excludeContext ) + GLVideoMixer::GLVideoMixer( int frame_w, + int frame_h, + double frameDuration, + CVPixelBufferPoolRef pool, + std::function excludeContext ) : m_bufferDuration(frameDuration), m_glesCtx(nullptr), m_frameW(frame_w), @@ -217,20 +263,27 @@ - (void) setMixer: (videocore::iOS::GLESVideoMixer*) mixer m_zRange.first = INT_MAX; m_zRange.second = INT_MIN; + m_mixThreadSem = dispatch_semaphore_create(0); + m_callbackSession = [[GLESObjCCallback alloc] init]; [(GLESObjCCallback*)m_callbackSession setMixer:this]; } - GLESVideoMixer::~GLESVideoMixer() + GLVideoMixer::~GLVideoMixer() { m_output.reset(); m_exiting = true; - m_mixThreadCond.notify_all(); - DLog("GLESVideoMixer::~GLESVideoMixer()"); + + dispatch_semaphore_signal(m_mixThreadSem); + DLog("GLVideoMixer::~GLVideoMixer()"); + if(m_mixThread.joinable()) { + m_mixThread.join(); + } + PERF_GL_sync({ //glDeleteProgram(m_prog); - glDeleteFramebuffers(2, m_fbo); + glDeleteFramebuffers(kOutPBCount, m_fbo); glDeleteBuffers(1, &m_vbo); //glDeleteVertexArraysOES(1, &m_vao); GLuint textures[2] ; @@ -255,17 +308,19 @@ - (void) setMixer: (videocore::iOS::GLESVideoMixer*) mixer } m_glJobQueue.mark_exiting(); m_glJobQueue.enqueue_sync([](){}); - + [(id)m_callbackSession release]; + dispatch_release(m_mixThreadSem); } void - GLESVideoMixer::start() { + GLVideoMixer::start() { m_mixThread = std::thread([this](){ this->mixThread(); }); } void - GLESVideoMixer::setupGLES(std::function excludeContext) + GLVideoMixer::setupGLES(std::function excludeContext) { +#if TARGET_OS_IPHONE if(SYSTEM_VERSION_GREATER_THAN_OR_EQUAL_TO(@"7.0")) { m_glesCtx = [[EAGLContext alloc] initWithAPI:kEAGLRenderingAPIOpenGLES3]; } @@ -281,7 +336,24 @@ - (void) setMixer: (videocore::iOS::GLESVideoMixer*) mixer if(excludeContext) { excludeContext(m_glesCtx); } - +#else + CGLPixelFormatObj pix; + CGLError errorCode; + GLint num; // stores the number of possible pixel formats + CGLPixelFormatAttribute attributes[] = { + kCGLPFAAccelerated, // no software rendering + kCGLPFAOpenGLProfile, // core profile with the version stated below + (CGLPixelFormatAttribute) kCGLOGLPVersion_3_2_Core, + (CGLPixelFormatAttribute) 0 + }; + errorCode = CGLChoosePixelFormat( attributes, &pix, &num ); + CGLContextObj ctx; + + errorCode = CGLCreateContext(pix, nullptr, &ctx); + CGLSetCurrentContext(ctx); + CGLLockContext(ctx); + m_glesCtx = (void*)ctx; +#endif // // Shared-memory FBOs // @@ -303,8 +375,11 @@ - (void) setMixer: (videocore::iOS::GLESVideoMixer*) mixer NSDictionary* pixelBufferOptions = @{ (NSString*) kCVPixelBufferPixelFormatTypeKey : @(kCVPixelFormatType_32BGRA), (NSString*) kCVPixelBufferWidthKey : @(m_frameW), (NSString*) kCVPixelBufferHeightKey : @(m_frameH), +#if TARGET_OS_IPHONE (NSString*) kCVPixelBufferOpenGLESCompatibilityKey : @YES, - (NSString*) kCVPixelBufferIOSurfacePropertiesKey : @{}}; + (NSString*) kCVPixelBufferIOSurfacePropertiesKey : @{} +#endif + }; CVPixelBufferCreate(kCFAllocatorDefault, m_frameW, m_frameH, kCVPixelFormatType_32BGRA, (CFDictionaryRef)pixelBufferOptions, &m_pixelBuffer[0]); CVPixelBufferCreate(kCFAllocatorDefault, m_frameW, m_frameH, kCVPixelFormatType_32BGRA, (CFDictionaryRef)pixelBufferOptions, &m_pixelBuffer[1]); @@ -315,23 +390,41 @@ - (void) setMixer: (videocore::iOS::GLESVideoMixer*) mixer } } + +#if TARGET_OS_IPHONE CVOpenGLESTextureCacheCreate(kCFAllocatorDefault, NULL, (EAGLContext*)this->m_glesCtx, NULL, &this->m_textureCache); +#else + CVOpenGLTextureCacheCreate(kCFAllocatorDefault, + nullptr, + (CGLContextObj)m_glesCtx, + pix, + nullptr, + &m_textureCache); + CGLDestroyPixelFormat( pix ); +#endif + glGenFramebuffers(2, this->m_fbo); for(int i = 0 ; i < 2 ; ++i) { +#if TARGET_OS_IPHONE CVOpenGLESTextureCacheCreateTextureFromImage(kCFAllocatorDefault, this->m_textureCache, this->m_pixelBuffer[i], NULL, GL_TEXTURE_2D, GL_RGBA, m_frameW, m_frameH, GL_BGRA, GL_UNSIGNED_BYTE, 0, &m_texture[i]); - - glBindTexture(GL_TEXTURE_2D, CVOpenGLESTextureGetName(m_texture[i])); - glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); - glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); - glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); - glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); +#else + CVOpenGLTextureCacheCreateTextureFromImage(kCFAllocatorDefault, this->m_textureCache, this->m_pixelBuffer[i], NULL, &m_texture[i]); +#endif + GLenum textarget = CVOpenGLESTextureGetTarget(m_texture[i]); + GLint texname = CVOpenGLESTextureGetName(m_texture[i]); + glBindTexture(textarget, texname); + glTexParameterf(textarget, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + glTexParameterf(textarget, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameterf(textarget, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameterf(textarget, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); glBindFramebuffer(GL_FRAMEBUFFER, m_fbo[i]); - glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, CVOpenGLESTextureGetName(m_texture[i]), 0); + glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, textarget, texname, 0); + GL_FRAMEBUFFER_STATUS(__LINE__); } + GL_ERRORS(__LINE__); - GL_FRAMEBUFFER_STATUS(__LINE__); glGenBuffers(1, &this->m_vbo); glBindBuffer(GL_ARRAY_BUFFER, this->m_vbo); @@ -342,11 +435,13 @@ - (void) setMixer: (videocore::iOS::GLESVideoMixer*) mixer glDisable(GL_SCISSOR_TEST); glViewport(0, 0, m_frameW,m_frameH); glClearColor(0.05,0.05,0.07,1.0); + CVOpenGLESTextureCacheFlush(m_textureCache, 0); + GL_ERRORS(__LINE__); } void - GLESVideoMixer::registerSource(std::shared_ptr source, - size_t bufferSize) + GLVideoMixer::registerSource(std::shared_ptr source, + size_t bufferSize) { const auto hash = std::hash< std::shared_ptr > () (source); bool registered = false; @@ -367,9 +462,9 @@ - (void) setMixer: (videocore::iOS::GLESVideoMixer*) mixer } } void - GLESVideoMixer::releaseBuffer(std::weak_ptr source) + GLVideoMixer::releaseBuffer(std::weak_ptr source) { - DLog("GLESVideoMixer::releaseBuffer"); + DLog("GLVideoMixer::releaseBuffer"); const auto h = hash(source); auto it = m_sourceBuffers.find(h) ; if(it != m_sourceBuffers.end()) { @@ -378,9 +473,9 @@ - (void) setMixer: (videocore::iOS::GLESVideoMixer*) mixer } void - GLESVideoMixer::unregisterSource(std::shared_ptr source) + GLVideoMixer::unregisterSource(std::shared_ptr source) { - DLog("GLESVideoMixer::unregisterSource"); + DLog("GLVideoMixer::unregisterSource"); releaseBuffer(source); auto it = m_sources.begin(); @@ -413,9 +508,9 @@ - (void) setMixer: (videocore::iOS::GLESVideoMixer*) mixer } void - GLESVideoMixer::pushBuffer(const uint8_t *const data, - size_t size, - videocore::IMetadata &metadata) + GLVideoMixer::pushBuffer(const uint8_t *const data, + size_t size, + videocore::IMetadata &metadata) { if(m_paused.load()) { return; @@ -439,7 +534,7 @@ - (void) setMixer: (videocore::iOS::GLESVideoMixer*) mixer auto inPixelBuffer = *(Apple::PixelBufferRef*)data ; - + m_sourceBuffers[h].setBuffer(inPixelBuffer, this->m_textureCache, m_glJobQueue, m_glesCtx); m_sourceBuffers[h].setBlends(md.getData()); @@ -450,12 +545,12 @@ - (void) setMixer: (videocore::iOS::GLESVideoMixer*) mixer this->m_sourceMats[h] = mat; } void - GLESVideoMixer::setOutput(std::shared_ptr output) + GLVideoMixer::setOutput(std::shared_ptr output) { m_output = output; } const std::size_t - GLESVideoMixer::hash(std::weak_ptr source) const + GLVideoMixer::hash(std::weak_ptr source) const { const auto l = source.lock(); if (l) { @@ -464,7 +559,7 @@ - (void) setMixer: (videocore::iOS::GLESVideoMixer*) mixer return 0; } void - GLESVideoMixer::mixThread() + GLVideoMixer::mixThread() { const auto us = std::chrono::microseconds(static_cast(m_bufferDuration * 1000000.)); const auto us_25 = std::chrono::microseconds(static_cast(m_bufferDuration * 250000.)); @@ -477,10 +572,9 @@ - (void) setMixer: (videocore::iOS::GLESVideoMixer*) mixer bool locked[2] = {false}; m_nextMixTime = m_epoch; - + auto lastMixTime = m_epoch; while(!m_exiting.load()) { - std::unique_lock l(m_mutex); const auto now = std::chrono::steady_clock::now(); if(now >= (m_nextMixTime)) { @@ -500,89 +594,106 @@ - (void) setMixer: (videocore::iOS::GLESVideoMixer*) mixer locked[current_fb] = true; m_mixing = true; - PERF_GL_async({ - glPushGroupMarkerEXT(0, "Videocore.Mix"); - glBindFramebuffer(GL_FRAMEBUFFER, this->m_fbo[current_fb]); - - IVideoFilter* currentFilter = nil; - glClear(GL_COLOR_BUFFER_BIT); - for ( int i = m_zRange.first ; i <= m_zRange.second ; ++i) { - - for ( auto it = this->m_layerMap[i].begin() ; it != this->m_layerMap[i].end() ; ++ it) { - CVOpenGLESTextureRef texture = NULL; - auto filterit = m_sourceFilters.find(*it); - if(filterit == m_sourceFilters.end()) { - IFilter* filter = m_filterFactory.filter("com.videocore.filters.bgra"); - m_sourceFilters[*it] = dynamic_cast(filter); - } - if(currentFilter != m_sourceFilters[*it]) { - if(currentFilter) { - currentFilter->unbind(); - } - currentFilter = m_sourceFilters[*it]; - - if(currentFilter && !currentFilter->initialized()) { - currentFilter->initialize(); - } - } - - auto iTex = this->m_sourceBuffers.find(*it); - if(iTex == this->m_sourceBuffers.end()) continue; - - texture = iTex->second.currentTexture(); - - // TODO: Add blending. - if(iTex->second.blends()) { - glEnable(GL_BLEND); - glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA); - } - if(texture && currentFilter) { - currentFilter->incomingMatrix(this->m_sourceMats[*it]); - currentFilter->bind(); - glBindTexture(GL_TEXTURE_2D, CVOpenGLESTextureGetName(texture)); - glDrawArrays(GL_TRIANGLES, 0, 6); - } else { - DLog("Null texture!"); - } - if(iTex->second.blends()) { - glDisable(GL_BLEND); - } - } + + PERF_GL_sync({ + compose(current_fb, currentTime, lastMixTime); + }); + + current_fb = (current_fb + 1) % kOutPBCount; + lastMixTime = currentTime; + } + + while(std::chrono::steady_clock::now() >= m_nextMixTime) { + m_nextMixTime += us; + } + dispatch_semaphore_wait(m_mixThreadSem, dispatch_time(DISPATCH_TIME_NOW, + std::chrono::duration_cast( + m_nextMixTime - + std::chrono::steady_clock::now()) + .count())); + + } + } + void + GLVideoMixer::compose(int fbo, std::chrono::steady_clock::time_point currentTime, std::chrono::steady_clock::time_point lastMixTime) + { + glPushGroupMarkerEXT(0, "Videocore.Mix"); + glBindFramebuffer(GL_FRAMEBUFFER, this->m_fbo[fbo]); + + IVideoFilter* currentFilter = nil; + glClear(GL_COLOR_BUFFER_BIT); + for ( int i = m_zRange.first ; i <= m_zRange.second ; ++i) { + + for ( auto it = this->m_layerMap[i].begin() ; it != this->m_layerMap[i].end() ; ++ it) { + CVOpenGLESTextureRef texture = NULL; + auto filterit = m_sourceFilters.find(*it); + if(filterit == m_sourceFilters.end()) { + IFilter* filter = m_filterFactory.filter("com.videocore.filters.bgra"); + m_sourceFilters[*it] = dynamic_cast(filter); + } + if(currentFilter != m_sourceFilters[*it]) { + if(currentFilter) { + currentFilter->unbind(); } - glFlush(); - glPopGroupMarkerEXT(); - + currentFilter = m_sourceFilters[*it]; - auto lout = this->m_output.lock(); - if(lout) { - - MetaData<'vide'> md(std::chrono::duration_cast(currentTime - m_epoch).count()); - lout->pushBuffer((uint8_t*)this->m_pixelBuffer[!current_fb], sizeof(this->m_pixelBuffer[!current_fb]), md); + if(currentFilter && !currentFilter->initialized()) { + currentFilter->setFilterLanguage(TARGET_OS_IPHONE ? GL_ES2_3 : GL_3); + currentFilter->initialize(); } - this->m_mixing = false; - - }); - current_fb = !current_fb; + } + + auto iTex = this->m_sourceBuffers.find(*it); + if(iTex == this->m_sourceBuffers.end()) continue; + + texture = iTex->second.currentTexture(); + + // TODO: Add blending. + if(iTex->second.blends()) { + glEnable(GL_BLEND); + glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA); + } + if(texture && currentFilter) { + currentFilter->incomingMatrix(this->m_sourceMats[*it]); + currentFilter->imageDimensions(iTex->second.currentBuffer()->width(), iTex->second.currentBuffer()->height()); + currentFilter->bind(); + glBindTexture(CVOpenGLESTextureGetTarget(texture), CVOpenGLESTextureGetName(texture)); + glDrawArrays(GL_TRIANGLES, 0, 6); + } else { + DLog("Null texture!"); + } + if(iTex->second.blends()) { + glDisable(GL_BLEND); + } } + } + glFlush(); + glPopGroupMarkerEXT(); + + + auto lout = this->m_output.lock(); + if(lout) { - m_mixThreadCond.wait_until(l, m_nextMixTime); - + MetaData<'vide'> md(std::chrono::duration_cast(currentTime - m_epoch).count()); + lout->pushBuffer((uint8_t*)this->m_pixelBuffer[!fbo], sizeof(this->m_pixelBuffer[!fbo]), md); } + this->m_mixing = false; + } void - GLESVideoMixer::mixPaused(bool paused) + GLVideoMixer::mixPaused(bool paused) { m_paused = paused; } void - GLESVideoMixer::setSourceFilter(std::weak_ptr source, IVideoFilter *filter) { + GLVideoMixer::setSourceFilter(std::weak_ptr source, IVideoFilter *filter) { auto h = hash(source); m_sourceFilters[h] = filter; } void - GLESVideoMixer::sync() { + GLVideoMixer::sync() { m_syncPoint = std::chrono::steady_clock::now(); m_shouldSync = true; //if(m_syncPoint >= (m_nextMixTime)) { diff --git a/mixers/GenericAudioMixer.cpp b/mixers/GenericAudioMixer.cpp index b7b6a446..25925ef3 100644 --- a/mixers/GenericAudioMixer.cpp +++ b/mixers/GenericAudioMixer.cpp @@ -26,7 +26,7 @@ #include #include #include - +#include #ifndef INT16_MAX #define INT16_MAX 0x7FFF diff --git a/sources/Apple/PixelBufferSource.mm b/sources/Apple/PixelBufferSource.mm index 68b49e2e..b9f0abb1 100644 --- a/sources/Apple/PixelBufferSource.mm +++ b/sources/Apple/PixelBufferSource.mm @@ -26,7 +26,7 @@ #include #include - +#import namespace videocore { namespace Apple { PixelBufferSource::PixelBufferSource(int width, int height, OSType pixelFormat ) @@ -38,9 +38,11 @@ NSDictionary* pixelBufferOptions = @{ (NSString*) kCVPixelBufferPixelFormatTypeKey : @(kCVPixelFormatType_32BGRA), (NSString*) kCVPixelBufferWidthKey : @(width), (NSString*) kCVPixelBufferHeightKey : @(height), +#if TARGET_OS_IPHONE (NSString*) kCVPixelBufferOpenGLESCompatibilityKey : @YES, - (NSString*) kCVPixelBufferIOSurfacePropertiesKey : @{}}; - + (NSString*) kCVPixelBufferIOSurfacePropertiesKey : @{} +#endif + }; ret = CVPixelBufferCreate(kCFAllocatorDefault, width, height, pixelFormat, (__bridge CFDictionaryRef)pixelBufferOptions, &pb); } if(!ret) { @@ -74,6 +76,7 @@ md.setData(4, mat, true, shared_from_this()); auto pixelBuffer = std::make_shared((CVPixelBufferRef)m_pixelBuffer, false); outp->pushBuffer((const uint8_t*)&pixelBuffer, sizeof(pixelBuffer), md); + DLog("Pb bpr: %zu",CVPixelBufferGetBytesPerRow((CVPixelBufferRef)m_pixelBuffer)); } } void diff --git a/sources/iOS/MicSource.mm b/sources/iOS/MicSource.mm index 95055522..b958f75f 100644 --- a/sources/iOS/MicSource.mm +++ b/sources/iOS/MicSource.mm @@ -96,7 +96,7 @@ static OSStatus handleInputBuffer(void *inRefCon, if(granted) { [session setCategory:AVAudioSessionCategoryPlayAndRecord withOptions:AVAudioSessionCategoryOptionDefaultToSpeaker | AVAudioSessionCategoryOptionMixWithOthers error:nil]; - //[session setMode:AVAudioSessionModeVideoChat error:nil]; + [session setMode:AVAudioSessionModeVideoChat error:nil]; [session setActive:YES error:nil]; AudioComponentDescription acd; @@ -197,7 +197,9 @@ static OSStatus handleInputBuffer(void *inRefCon, MicSource::setOutput(std::shared_ptr output) { m_output = output; auto mixer = std::dynamic_pointer_cast(output); - mixer->registerSource(shared_from_this()); + if(mixer) { + mixer->registerSource(shared_from_this()); + } } } } diff --git a/system/util.h b/system/util.h index 217b40e1..cda9e4c1 100644 --- a/system/util.h +++ b/system/util.h @@ -28,9 +28,10 @@ #ifdef DEBUG -#define DLog(...) printf(__VA_ARGS__); +#define DLogLvl(lvl, fmt, ...) printf( "[VIDEOCORE][%d] " fmt "\n", lvl, ##__VA_ARGS__); +#define DLog(fmt, ...) DLogLvl(0, fmt, ##__VA_ARGS__) #else -#define DLog(...) {} +#define DLog(fmt, ...) {} #endif diff --git a/transforms/iOS/AACEncode.cpp b/transforms/Apple/AACEncode.cpp similarity index 96% rename from transforms/iOS/AACEncode.cpp rename to transforms/Apple/AACEncode.cpp index b7b0dec5..bd0c1d5c 100644 --- a/transforms/iOS/AACEncode.cpp +++ b/transforms/Apple/AACEncode.cpp @@ -22,11 +22,12 @@ THE SOFTWARE. */ -#include +#include #include +#include -namespace videocore { namespace iOS { - +namespace videocore { namespace Apple { +#if TARGET_OS_IPHONE Boolean IsAACHardwareEncoderAvailable(void) { Boolean isAvailable = false; @@ -53,10 +54,10 @@ namespace videocore { namespace iOS { if (encoderDescriptions[i].mSubType == kAudioFormatMPEG4AAC && encoderDescriptions[i].mManufacturer == kAppleHardwareAudioCodecManufacturer) isAvailable = true; } - + return isAvailable; } - +#endif struct UserData { uint8_t* data; int size; @@ -119,16 +120,25 @@ namespace videocore { namespace iOS { UInt32 outputPacketSize = 0; const OSType subtype = kAudioFormatMPEG4AAC; + AudioClassDescription requestedCodecs[2] = { { kAudioEncoderComponentType, subtype, +#if TARGET_OS_IPHONE kAppleSoftwareAudioCodecManufacturer +#else + 'appl' +#endif }, { kAudioEncoderComponentType, subtype, +#if TARGET_OS_IPHONE kAppleHardwareAudioCodecManufacturer +#else + 'appl' +#endif } }; @@ -284,12 +294,20 @@ namespace videocore { namespace iOS { { kAudioEncoderComponentType, subtype, +#if TARGET_OS_IPHONE kAppleSoftwareAudioCodecManufacturer +#else + 'appl' +#endif }, { kAudioEncoderComponentType, subtype, +#if TARGET_OS_IPHONE kAppleHardwareAudioCodecManufacturer +#else + 'appl' +#endif } }; AudioConverterNewSpecific(&m_in, &m_out, 2,requestedCodecs, &m_audioConverter); diff --git a/transforms/iOS/AACEncode.h b/transforms/Apple/AACEncode.h similarity index 98% rename from transforms/iOS/AACEncode.h rename to transforms/Apple/AACEncode.h index f19fcd67..f86bb1b7 100644 --- a/transforms/iOS/AACEncode.h +++ b/transforms/Apple/AACEncode.h @@ -38,7 +38,7 @@ #include #include -namespace videocore { namespace iOS { +namespace videocore { namespace Apple { class AACEncode : public IEncoder { diff --git a/transforms/Apple/H264Encode.mm b/transforms/Apple/H264Encode.mm index 32192938..b9791958 100644 --- a/transforms/Apple/H264Encode.mm +++ b/transforms/Apple/H264Encode.mm @@ -22,6 +22,10 @@ #include #include #include +#include + +#import +#include #if TARGET_OS_IPHONE #if defined(__IPHONE_8_0) && __IPHONE_OS_VERSION_MAX_ALLOWED >= __IPHONE_8_0 @@ -116,9 +120,12 @@ void vtCallback(void *outputCallbackRefCon, if(m_compressionSession) { m_encodeMutex.lock(); VTCompressionSessionRef session = (VTCompressionSessionRef)m_compressionSession; - - CMTime pts = CMTimeMake(metadata.timestampDelta + m_ctsOffset, 1000.); // timestamp is in ms. - CMTime dur = CMTimeMake(1, m_fps); + + CVPixelBufferRef buffer = (CVPixelBufferRef)data; + + const uint64_t _pts = metadata.pts + m_ctsOffset; + CMTime pts = CMTimeMake(_pts, 1000.); // timestamp is in ms. + CMTime dur = CMTimeMake(1, (1. / metadata.duration)); VTEncodeInfoFlags flags; @@ -132,9 +139,9 @@ void vtCallback(void *outputCallbackRefCon, CFDictionaryAddValue(frameProps, kVTEncodeFrameOptionKey_ForceKeyFrame, kCFBooleanTrue); } - - VTCompressionSessionEncodeFrame(session, (CVPixelBufferRef)data, pts, dur, frameProps, NULL, &flags); - + + OSStatus ret = VTCompressionSessionEncodeFrame(session, buffer, pts, dur, frameProps, NULL, &flags); + if(m_forceKeyframe) { CFRelease(frameProps); m_forceKeyframe = false; @@ -162,10 +169,10 @@ void vtCallback(void *outputCallbackRefCon, CFStringRef key = kVTVideoEncoderSpecification_EncoderID; CFStringRef value = CFSTR("com.apple.videotoolbox.videoencoder.h264.gva"); - CFStringRef bkey = CFSTR("EnableHardwareAcceleratedVideoEncoder"); + CFStringRef bkey = kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder; CFBooleanRef bvalue = kCFBooleanTrue; - CFStringRef ckey = CFSTR("RequireHardwareAcceleratedVideoEncoder"); + CFStringRef ckey = kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder; CFBooleanRef cvalue = kCFBooleanTrue; encoderSpecifications = CFDictionaryCreateMutable( @@ -183,8 +190,7 @@ void vtCallback(void *outputCallbackRefCon, @autoreleasepool { - NSDictionary* pixelBufferOptions = @{ (NSString*) kCVPixelBufferPixelFormatTypeKey : //@(kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange), - @(kCVPixelFormatType_32BGRA), + NSDictionary* pixelBufferOptions = @{ (NSString*) kCVPixelBufferPixelFormatTypeKey : @(kCVPixelFormatType_32BGRA), (NSString*) kCVPixelBufferWidthKey : @(m_frameW), (NSString*) kCVPixelBufferHeightKey : @(m_frameH), (NSString*) kCVPixelBufferOpenGLESCompatibilityKey : @YES, @@ -259,6 +265,7 @@ void vtCallback(void *outputCallbackRefCon, if(m_compressionSession) { VTCompressionSessionInvalidate((VTCompressionSessionRef)m_compressionSession); CFRelease((VTCompressionSessionRef)m_compressionSession); + m_compressionSession = nullptr; } #endif } @@ -268,8 +275,25 @@ void vtCallback(void *outputCallbackRefCon, #if VERSION_OK auto l = m_output.lock(); if(l && data && size > 0) { + //DLog("Received [pts:%lld dts:%lld]", pts, dts); + // DLog("NALU: %d [%x %x %x %x][%x %x %x %x] pts: %lld dts: %lld", data[4] & 0x1F, data[0], data[1], data[2], data[3], + // data[4],data[5],data[6],data[7],pts, dts); + int nalu_type = INT32_MAX; + uint8_t* p = (uint8_t*)&data[0]; + int32_t nsize = 0; + while(p < (data+size)) { + nsize = htonl(*(int32_t*)p); + nalu_type = p[4] & 0x1f; + //DLog("nalu[%x][%d]", nalu_type, nsize); + if(nalu_type <= 5 || nalu_type == 7 || nalu_type == 8) { + break; + } + p += nsize + sizeof(nsize); + } + //DLog("NALU: %d", p[4] & 0x1f); videocore::VideoBufferMetadata md(pts, dts); - l->pushBuffer(data, size, md); + l->pushBuffer(p, (data+size)-p, md); + } #endif @@ -343,4 +367,4 @@ void vtCallback(void *outputCallbackRefCon, return nullptr; } } -} \ No newline at end of file +} diff --git a/transforms/IMetaData.hpp b/transforms/IMetaData.hpp index a509f101..3ab070b9 100644 --- a/transforms/IMetaData.hpp +++ b/transforms/IMetaData.hpp @@ -26,7 +26,6 @@ #include #include #include -#include #include @@ -34,9 +33,10 @@ namespace videocore { struct IMetadata { + IMetadata(double pts, double dts, double duration) : pts(pts), dts(dts), duration(duration) {}; IMetadata(double pts, double dts) : pts(pts), dts(dts) {}; - IMetadata(double ts) : pts(ts), dts(ts) {}; - IMetadata() : pts(0.), dts(0.) {}; + IMetadata(double ts) : pts(ts), dts(ts), duration(0.) {}; + IMetadata() : pts(0.), dts(0.), duration(0.) {}; virtual ~IMetadata() {}; @@ -46,11 +46,13 @@ namespace videocore double timestampDelta __attribute__((deprecated)); }; double dts; + double duration; }; template struct MetaData : public IMetadata { + MetaData(double pts, double dts, double duration) : IMetadata(pts, dts, duration) {}; MetaData(double pts, double dts) : IMetadata(pts, dts) {}; MetaData(double ts) : IMetadata(ts) {}; MetaData() : IMetadata() {}; From cf20041c184cdb192bca1ba04218b3232527e7e5 Mon Sep 17 00:00:00 2001 From: James Hurley Date: Wed, 29 Apr 2015 16:18:20 -0700 Subject: [PATCH 2/2] fix mac build errors --- mixers/Apple/GLVideoMixer.mm | 7 +++---- sources/iOS/GLESUtil.h | 19 +++++++++++++------ transforms/Apple/H264Encode.mm | 5 ++++- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/mixers/Apple/GLVideoMixer.mm b/mixers/Apple/GLVideoMixer.mm index 1cdd98d2..5359130a 100644 --- a/mixers/Apple/GLVideoMixer.mm +++ b/mixers/Apple/GLVideoMixer.mm @@ -161,11 +161,10 @@ - (void) setMixer: (videocore::Apple::GLVideoMixer*) mixer CVOpenGLESTextureRef texture = nullptr; const CVPixelBufferRef inCvpb = ref->cvBuffer(); - int w = (int)CVPixelBufferGetWidth(inCvpb); - int h = (int)CVPixelBufferGetHeight(inCvpb); - #if TARGET_OS_IPHONE + int w = (int)CVPixelBufferGetWidth(inCvpb); + int h = (int)CVPixelBufferGetHeight(inCvpb); CVReturn ret = CVOpenGLESTextureCacheCreateTextureFromImage(kCFAllocatorDefault, textureCache, @@ -419,7 +418,7 @@ - (void) setMixer: (videocore::Apple::GLVideoMixer*) mixer glTexParameterf(textarget, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); glBindFramebuffer(GL_FRAMEBUFFER, m_fbo[i]); glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, textarget, texname, 0); - + GL_FRAMEBUFFER_STATUS(__LINE__); } diff --git a/sources/iOS/GLESUtil.h b/sources/iOS/GLESUtil.h index fb755351..478ad003 100644 --- a/sources/iOS/GLESUtil.h +++ b/sources/iOS/GLESUtil.h @@ -25,10 +25,17 @@ #ifndef videocore_GLESUtil_h #define videocore_GLESUtil_h -#import -#import -#import - +#ifdef __APPLE__ +# include +# if TARGET_OS_IPHONE +# import +# import +# import +# else +# include +# include +# endif +#endif #define BUFFER_OFFSET(i) ((void*)(i)) #define BUFFER_OFFSET_POSITION BUFFER_OFFSET(0) #define BUFFER_OFFSET_TEXTURE BUFFER_OFFSET(8) @@ -62,9 +69,9 @@ switch(status)\ case GL_FRAMEBUFFER_INCOMPLETE_ATTACHMENT:\ DLog("OGL(" __FILE__ "):: %d: Incomplete attachment\n", line);\ break;\ -case GL_FRAMEBUFFER_INCOMPLETE_DIMENSIONS:\ +/*case GL_FRAMEBUFFER_INCOMPLETE_DIMENSIONS:\ DLog("OGL(" __FILE__ "):: %d: Incomplete dimensions\n", line);\ -break;\ +break;*/\ case GL_FRAMEBUFFER_INCOMPLETE_MISSING_ATTACHMENT:\ DLog("OGL(" __FILE__ "):: %d: Incomplete missing attachment\n", line);\ break;\ diff --git a/transforms/Apple/H264Encode.mm b/transforms/Apple/H264Encode.mm index b9791958..ff6db534 100644 --- a/transforms/Apple/H264Encode.mm +++ b/transforms/Apple/H264Encode.mm @@ -193,8 +193,11 @@ void vtCallback(void *outputCallbackRefCon, NSDictionary* pixelBufferOptions = @{ (NSString*) kCVPixelBufferPixelFormatTypeKey : @(kCVPixelFormatType_32BGRA), (NSString*) kCVPixelBufferWidthKey : @(m_frameW), (NSString*) kCVPixelBufferHeightKey : @(m_frameH), +#if TARGET_OS_IPHONE (NSString*) kCVPixelBufferOpenGLESCompatibilityKey : @YES, - (NSString*) kCVPixelBufferIOSurfacePropertiesKey : @{}}; + (NSString*) kCVPixelBufferIOSurfacePropertiesKey : @{} +#endif + }; err = VTCompressionSessionCreate( kCFAllocatorDefault,