Post‐Processing Local Video

Local video capture can be post processed by manually creating a video track with LocalParticipant.createLocalTrack() and passing a VideoProcessor argument. The following example shows how to create a basic monochrome filter for your video.

class MonochromeVideoProcessor : VideoProcessor {

    // Where the processed frames should be outputted to.
    private var sink: VideoSink? = null

    // Initialize any resources for video processing here.
    override fun onCapturerStarted(success: Boolean) {
    }

    // Release any resources associated with this capturer.
    override fun onCapturerStopped() {
    }

    // Called for each frame captured.
    override fun onFrameCaptured(frame: VideoFrame) {
        val processedFrame = monochromeFilter(frame)
        sink?.onFrame(processedFrame)
        
        // Make sure to release when you're done with frame.
        processedFrame.release()
    }

    // A basic monochrome filter can be achieved by setting the UV planes
    // to zero on an I420 image.
    private fun monochromeFilter(frame: VideoFrame): VideoFrame {
        val buffer = frame.buffer.toI420() ?: return frame

        fun clearUV(byteBuffer: ByteBuffer) {
            // In I420, 0x00 represents -128, so need to use 0x80 for proper zero.
            val zeroArray = ByteArray(1024) { 0x80.toByte() }
            while (byteBuffer.remaining() > 1024) {
                byteBuffer.put(zeroArray)
            }

            if (byteBuffer.remaining() > 0) {
                val lastZeroArray = ByteArray(byteBuffer.remaining()) { 0x80.toByte() }
                byteBuffer.put(lastZeroArray)
            }

            byteBuffer.rewind()
        }

        clearUV(buffer.dataU)
        clearUV(buffer.dataV)

        return VideoFrame(buffer, frame.rotation, frame.timestampNs)

    }

    override fun setSink(sink: VideoSink?) {
        this.sink = sink
    }
}

fun publishProcessedVideo() {
    val videoTrack = localParticipant.createVideoTrack(
        videoProcessor = ExampleVideoProcessor(),
    )
    videoTrack.startCapture()
    localParticipant.publishVideoTrack(
        track = videoTrack,
    )
}