Tweet
Tweet

Slide 1

Slide 1 text

Demuxed 2018 @tmm1 FFmpeg Video decoding with libavformat libavcodec libavutil libavfilter iOS on and VideoToolbox Apple TV iPhone iPad MediaCodec Android TV Fire TV Android

Slide 2

Slide 2 text

Aman Gupta @tmm1 @tmm1 [email protected] Channels app founder 2015 GitHub Employee #18 2011 ruby-core committer 2013 FFmpeg committer 2017

Slide 3

Slide 3 text

✋"# FFmpeg $ ffmpeg -i …

Slide 4

Slide 4 text

✋"# FFmpeg libavformat libavcodec libavutil libavfilter

Slide 5

Slide 5 text

✋"# FFmpeg libavformat libavcodec libavutil libavfilter git:// git:// git:// git://

Slide 6

Slide 6 text

FFmpeg libavformat libavcodec

Slide 7

Slide 7 text

FFmpeg libavformat muxers + demuxers hls, dash, mpegts, … data stream → AVPacket* libavcodec

Slide 8

Slide 8 text

FFmpeg libavformat protocols http, tcp, rtmp, udp, … muxers + demuxers hls, dash, mpegts, … data stream → AVPacket* libavcodec

Slide 9

Slide 9 text

FFmpeg libavformat protocols http, tcp, rtmp, udp, … muxers + demuxers hls, dash, mpegts, … data stream → AVPacket* libavcodec encoders decoders + hwaccels AVFrame* → AVPacket* AVPacket* → AVFrame* mpeg2, h264, hevc, …

Slide 10

Slide 10 text

FFmpeg iOS on VideoToolbox with

Slide 11

Slide 11 text

SOFTWARE DECODING AVCodec *codec = avcodec_find_decoder(AV_CODEC_ID_H264); AVCodecContext *avctx = avcodec_alloc_context3(codec); avcodec_open2(avctx, codec, NULL);

Slide 12

Slide 12 text

SOFTWARE DECODING AVCodec *codec = avcodec_find_decoder(AV_CODEC_ID_H264); AVCodecContext *avctx = avcodec_alloc_context3(codec); avcodec_open2(avctx, codec, NULL); { find decoder and create instance

Slide 13

Slide 13 text

SOFTWARE DECODING AVCodec *codec = avcodec_find_decoder(AV_CODEC_ID_H264); AVCodecContext *avctx = avcodec_alloc_context3(codec); avcodec_open2(avctx, codec, NULL); { find decoder and create instance { prepare decoder for data

Slide 14

Slide 14 text

SOFTWARE DECODING AVCodec *codec = avcodec_find_decoder(AV_CODEC_ID_H264); AVCodecContext *avctx = avcodec_alloc_context3(codec); avcodec_open2(avctx, codec, NULL); AVCodec *codec = avcodec_find_decoder(AV_CODEC_ID_H264); AVCodecContext *avctx = avcodec_alloc_context3(codec); AVBufferRef *device_ref = NULL; av_hwdevice_ctx_create( &device_ref, AV_HWDEVICE_TYPE_VIDEOTOOLBOX, NULL, NULL, 0); avctx->hw_device_ctx = device_ref; enum AVPixelFormat get_vt_format(struct AVCodecContext *avctx, const enum AVPixelFormat *fmt) { return AV_PIX_FMT_VIDEOTOOLBOX; } avctx->get_format = get_vt_format; avcodec_open2(avctx, codec, NULL); HARDWARE DECODING: VIDEOTOOLBOX

Slide 15

Slide 15 text

SOFTWARE DECODING AVCodec *codec = avcodec_find_decoder(AV_CODEC_ID_H264); AVCodecContext *avctx = avcodec_alloc_context3(codec); avcodec_open2(avctx, codec, NULL); AVCodec *codec = avcodec_find_decoder(AV_CODEC_ID_H264); AVCodecContext *avctx = avcodec_alloc_context3(codec); AVBufferRef *device_ref = NULL; av_hwdevice_ctx_create( &device_ref, AV_HWDEVICE_TYPE_VIDEOTOOLBOX, NULL, NULL, 0); avctx->hw_device_ctx = device_ref; enum AVPixelFormat get_vt_format(struct AVCodecContext *avctx, const enum AVPixelFormat *fmt) { return AV_PIX_FMT_VIDEOTOOLBOX; } avctx->get_format = get_vt_format; avcodec_open2(avctx, codec, NULL); HARDWARE DECODING: VIDEOTOOLBOX { create and attach HWDEVICE context

Slide 16

Slide 16 text

SOFTWARE DECODING AVCodec *codec = avcodec_find_decoder(AV_CODEC_ID_H264); AVCodecContext *avctx = avcodec_alloc_context3(codec); avcodec_open2(avctx, codec, NULL); AVCodec *codec = avcodec_find_decoder(AV_CODEC_ID_H264); AVCodecContext *avctx = avcodec_alloc_context3(codec); AVBufferRef *device_ref = NULL; av_hwdevice_ctx_create( &device_ref, AV_HWDEVICE_TYPE_VIDEOTOOLBOX, NULL, NULL, 0); avctx->hw_device_ctx = device_ref; enum AVPixelFormat get_vt_format(struct AVCodecContext *avctx, const enum AVPixelFormat *fmt) { return AV_PIX_FMT_VIDEOTOOLBOX; } avctx->get_format = get_vt_format; avcodec_open2(avctx, codec, NULL); HARDWARE DECODING: VIDEOTOOLBOX { request videotoolbox pixel format

Slide 17

Slide 17 text

SOFTWARE FRAMES AVFrame *frame = av_frame_alloc(); int ret = avcodec_receive_frame(avctx, frame); assert(frame->imgfmt == AV_PIX_FMT_YUV420P);

Slide 18

Slide 18 text

SOFTWARE FRAMES AVFrame *frame = av_frame_alloc(); int ret = avcodec_receive_frame(avctx, frame); assert(frame->imgfmt == AV_PIX_FMT_YUV420P); { receive decoded frame

Slide 19

Slide 19 text

SOFTWARE FRAMES AVFrame *frame = av_frame_alloc(); int ret = avcodec_receive_frame(avctx, frame); assert(frame->imgfmt == AV_PIX_FMT_YUV420P); AVFrame *frame = av_frame_alloc(); int ret = avcodec_receive_frame(avctx, frame); assert(frame->imgfmt == AV_PIX_FMT_VIDEOTOOLBOX); HARDWARE FRAMES: VIDEOTOOLBOX

Slide 20

Slide 20 text

SOFTWARE FRAMES AVFrame *frame = av_frame_alloc(); int ret = avcodec_receive_frame(avctx, frame); assert(frame->imgfmt == AV_PIX_FMT_YUV420P); AVFrame *frame = av_frame_alloc(); int ret = avcodec_receive_frame(avctx, frame); assert(frame->imgfmt == AV_PIX_FMT_VIDEOTOOLBOX); HARDWARE FRAMES: VIDEOTOOLBOX { platform specific pixel format { generic pixel format

Slide 21

Slide 21 text

SOFTWARE FRAMES AVFrame *frame = av_frame_alloc(); int ret = avcodec_receive_frame(avctx, frame); assert(frame->imgfmt == AV_PIX_FMT_YUV420P); AVFrame *frame = av_frame_alloc(); int ret = avcodec_receive_frame(avctx, frame); assert(frame->imgfmt == AV_PIX_FMT_VIDEOTOOLBOX); CVPixelBufferRef img = (CVPixelBufferRef)frame->planes[3]; HARDWARE FRAMES: VIDEOTOOLBOX { unwrap to access iOS pixel buffer

Slide 22

Slide 22 text

SOFTWARE FRAMES AVFrame *frame = av_frame_alloc(); int ret = avcodec_receive_frame(avctx, frame); assert(frame->imgfmt == AV_PIX_FMT_YUV420P); AVFrame *frame = av_frame_alloc(); int ret = avcodec_receive_frame(avctx, frame); assert(frame->imgfmt == AV_PIX_FMT_VIDEOTOOLBOX); CVPixelBufferRef img = (CVPixelBufferRef)frame->planes[3]; HARDWARE FRAMES: VIDEOTOOLBOX { unwrap to access iOS pixel buffer // use pixel buffer to: // - render to UIImage on screen // - read video pixel data // - modify pixel data // - upload video frame to OpenGL tex // // or: // - convert back to generic software frame // - filter/render like with software decode

Slide 23

Slide 23 text

SOFTWARE FRAMES AVFrame *frame = av_frame_alloc(); int ret = avcodec_receive_frame(avctx, frame); assert(frame->imgfmt == AV_PIX_FMT_YUV420P); AVFrame *frame = av_frame_alloc(); int ret = avcodec_receive_frame(avctx, frame); assert(frame->imgfmt == AV_PIX_FMT_VIDEOTOOLBOX); CVPixelBufferRef img = (CVPixelBufferRef)frame->planes[3]; int planes_nb = CVPixelBufferGetPlaneCount(img); CVPixelBufferLockBaseAddress(img, 0); for (int i = 0; i < planes_nb; i++) { size_t height = CVPixelBufferGetHeightOfPlane(img,i); size_t rowsize = CVPixelBufferGetBytesPerRowOfPlane(img,i); uint8_t *rowdata = CVPixelBufferGetBaseAddressOfPlane(img,i); // modify rowdata } CVPixelBufferUnlockBaseAddress(img, 0); HARDWARE FRAMES: VIDEOTOOLBOX { read/write access to underlying memory

Slide 24

Slide 24 text

SOFTWARE FRAMES AVFrame *frame = av_frame_alloc(); int ret = avcodec_receive_frame(avctx, frame); assert(frame->imgfmt == AV_PIX_FMT_YUV420P); AVFrame *frame = av_frame_alloc(); int ret = avcodec_receive_frame(avctx, frame); assert(frame->imgfmt == AV_PIX_FMT_VIDEOTOOLBOX); CVPixelBufferRef img = (CVPixelBufferRef)frame->planes[3]; int planes_nb = CVPixelBufferGetPlaneCount(img); for (int i = 0; i < planes_nb; i++) { CVOpenGLESTextureCacheCreateTextureFromImage( ... ); GLuint tex = CVOpenGLESTextureGetName(plane); // pass to GL shader for rendering } HARDWARE FRAMES: VIDEOTOOLBOX { transfer each plane to a OpenGL texture

Slide 25

Slide 25 text

SOFTWARE FRAMES AVFrame *frame = av_frame_alloc(); int ret = avcodec_receive_frame(avctx, frame); assert(frame->imgfmt == AV_PIX_FMT_YUV420P); AVFrame *frame = av_frame_alloc(); int ret = avcodec_receive_frame(avctx, frame); assert(frame->imgfmt == AV_PIX_FMT_VIDEOTOOLBOX); AVFrame *swframe = av_frame_alloc(); av_hwframe_transfer_data(swframe, frame, 0); assert(swframe->imgfmt == AV_PIX_FMT_YUV420P); HARDWARE FRAMES: VIDEOTOOLBOX { convert back to a regular software frame

Slide 26

Slide 26 text

FFmpeg on with MediaCodec

Slide 27

Slide 27 text

SOFTWARE DECODING AVCodec *codec = avcodec_find_decoder(AV_CODEC_ID_H264); AVCodecContext *avctx = avcodec_alloc_context3(codec); avcodec_open2(avctx, codec, NULL);

Slide 28

Slide 28 text

SOFTWARE DECODING AVCodec *codec = avcodec_find_decoder(AV_CODEC_ID_H264); AVCodecContext *avctx = avcodec_alloc_context3(codec); avcodec_open2(avctx, codec, NULL); JavaVM *vm = ...; // via JNI_OnLoad() etc av_jni_set_java_vm(vm, NULL); AVCodec *codec = avcodec_find_decoder_by_name(“h264_mediacodec”) AVCodecContext *avctx = avcodec_alloc_context3(codec); avcodec_open2(avctx, codec, NULL); HARDWARE DECODING: MEDIACODEC

Slide 29

Slide 29 text

SOFTWARE DECODING AVCodec *codec = avcodec_find_decoder(AV_CODEC_ID_H264); AVCodecContext *avctx = avcodec_alloc_context3(codec); avcodec_open2(avctx, codec, NULL); JavaVM *vm = ...; // via JNI_OnLoad() etc av_jni_set_java_vm(vm, NULL); AVCodec *codec = avcodec_find_decoder_by_name(“h264_mediacodec”) AVCodecContext *avctx = avcodec_alloc_context3(codec); avcodec_open2(avctx, codec, NULL); HARDWARE DECODING: MEDIACODEC { allow FFmpeg to access Android Java APIs

Slide 30

Slide 30 text

SOFTWARE DECODING AVCodec *codec = avcodec_find_decoder(AV_CODEC_ID_H264); AVCodecContext *avctx = avcodec_alloc_context3(codec); avcodec_open2(avctx, codec, NULL); JavaVM *vm = ...; // via JNI_OnLoad() etc av_jni_set_java_vm(vm, NULL); AVCodec *codec = avcodec_find_decoder_by_name(“h264_mediacodec") AVCodecContext *avctx = avcodec_alloc_context3(codec); avcodec_open2(avctx, codec, NULL); HARDWARE DECODING: MEDIACODEC { implemented as a separate decoder

Slide 31

Slide 31 text

SOFTWARE FRAMES AVFrame *frame = av_frame_alloc(); int ret = avcodec_receive_frame(avctx, frame); assert(frame->imgfmt == AV_PIX_FMT_YUV420P); HARDWARE FRAMES: MEDIACODEC AVFrame *frame = av_frame_alloc(); int ret = avcodec_receive_frame(avctx, frame); assert(frame->imgfmt == AV_PIX_FMT_NV12);

Slide 32

Slide 32 text

SOFTWARE FRAMES AVFrame *frame = av_frame_alloc(); int ret = avcodec_receive_frame(avctx, frame); assert(frame->imgfmt == AV_PIX_FMT_YUV420P); HARDWARE FRAMES: MEDIACODEC AVFrame *frame = av_frame_alloc(); int ret = avcodec_receive_frame(avctx, frame); assert(frame->imgfmt == AV_PIX_FMT_NV12); { generic pixel format { generic pixel format (decoded frame is copied back)

Slide 33

Slide 33 text

SOFTWARE DECODING AVCodec *codec = avcodec_find_decoder(AV_CODEC_ID_H264); AVCodecContext *avctx = avcodec_alloc_context3(codec); avcodec_open2(avctx, codec, NULL); JavaVM *vm = ...; // via JNI_OnLoad() etc av_jni_set_java_vm(vm, NULL); AVCodec *codec = avcodec_find_decoder_by_name(“h264_mediacodec") AVCodecContext *avctx = avcodec_alloc_context3(codec); jobject surface = ...; // android.view.Surface AVBufferRef *device_ref = av_hwdevice_ctx_alloc( AV_HWDEVICE_TYPE_MEDIACODEC); AVHWDeviceContext *ctx = (void *)device_ref->data; AVMediaCodecDeviceContext *hwctx = ctx->hwctx; hwctx->surface = (void *)(intptr_t)surface; av_hwdevice_ctx_init(device_ref); avctx->hw_device_ctx = device_ref; avcodec_open2(avctx, codec, NULL); HARDWARE DECODING: MEDIACODEC SURFACE

Slide 34

Slide 34 text

SOFTWARE DECODING AVCodec *codec = avcodec_find_decoder(AV_CODEC_ID_H264); AVCodecContext *avctx = avcodec_alloc_context3(codec); avcodec_open2(avctx, codec, NULL); JavaVM *vm = ...; // via JNI_OnLoad() etc av_jni_set_java_vm(vm, NULL); AVCodec *codec = avcodec_find_decoder_by_name(“h264_mediacodec") AVCodecContext *avctx = avcodec_alloc_context3(codec); jobject surface = ...; // android.view.Surface AVBufferRef *device_ref = av_hwdevice_ctx_alloc( AV_HWDEVICE_TYPE_MEDIACODEC); AVHWDeviceContext *ctx = (void *)device_ref->data; AVMediaCodecDeviceContext *hwctx = ctx->hwctx; hwctx->surface = (void *)(intptr_t)surface; av_hwdevice_ctx_init(device_ref); avctx->hw_device_ctx = device_ref; avcodec_open2(avctx, codec, NULL); HARDWARE DECODING: MEDIACODEC SURFACE { create HWDEVICE context to pass in Surface

Slide 35

Slide 35 text

SOFTWARE FRAMES AVFrame *frame = av_frame_alloc(); int ret = avcodec_receive_frame(avctx, frame); assert(frame->imgfmt == AV_PIX_FMT_YUV420P); HARDWARE FRAMES: MEDIACODEC SURFACE AVFrame *frame = av_frame_alloc(); int ret = avcodec_receive_frame(avctx, frame); assert(frame->imgfmt == AV_PIX_FMT_MEDIACODEC);

Slide 36

Slide 36 text

SOFTWARE FRAMES AVFrame *frame = av_frame_alloc(); int ret = avcodec_receive_frame(avctx, frame); assert(frame->imgfmt == AV_PIX_FMT_YUV420P); HARDWARE FRAMES: MEDIACODEC SURFACE AVFrame *frame = av_frame_alloc(); int ret = avcodec_receive_frame(avctx, frame); assert(frame->imgfmt == AV_PIX_FMT_MEDIACODEC); { platform specific pixel format { generic pixel format

Slide 37

Slide 37 text

SOFTWARE FRAMES AVFrame *frame = av_frame_alloc(); int ret = avcodec_receive_frame(avctx, frame); assert(frame->imgfmt == AV_PIX_FMT_YUV420P); HARDWARE FRAMES: MEDIACODEC SURFACE AVFrame *frame = av_frame_alloc(); int ret = avcodec_receive_frame(avctx, frame); assert(frame->imgfmt == AV_PIX_FMT_MEDIACODEC); AVMediaCodecBuffer *buffer = (AVMediaCodecBuffer *)frame->planes[3]; { unwrap to access MediaCodec Output Buffer

Slide 38

Slide 38 text

SOFTWARE FRAMES AVFrame *frame = av_frame_alloc(); int ret = avcodec_receive_frame(avctx, frame); assert(frame->imgfmt == AV_PIX_FMT_YUV420P); HARDWARE FRAMES: MEDIACODEC SURFACE AVFrame *frame = av_frame_alloc(); int ret = avcodec_receive_frame(avctx, frame); assert(frame->imgfmt == AV_PIX_FMT_MEDIACODEC); AVMediaCodecBuffer *buffer = (AVMediaCodecBuffer *)frame->planes[3]; // drop frame av_mediacodec_release_buffer(buffer, 0); // render to surface av_mediacodec_release_buffer(buffer, 1); // render at clock time av_mediacodec_render_buffer_at_time(buffer, nanotime); { render Output Buffer to screen

Slide 39

Slide 39 text

iOS STATE OF THE FFmpeg ☑ videotoolbox encoder h264, hevc ☑ videotoolbox hwaccel decoder h264, hevc ⬜ videotoolbox decoder (async) ☑ audiotoolbox encoder aac, alac, ilbc, pcm ☑ audiotoolbox decoder aac, ac3, eac3, mp3, … ⬜ mediacodec video encoder ☑ mediacodec video decoder mpeg2, h264, hevc ⬜ mediacodec audio encoder ⬜ mediacodec audio decoder ⬜ replace JNI with NDK (API 21+) ⬜ use async NDK decode (API 27+)

Slide 40

Slide 40 text

HELP ME IMPROVE FFMPEG ON MOBILE PLATFORMS! Aman Gupta @tmm1 @tmm1 [email protected] Demuxed 2018 Thank you