在上一篇文章中，我们了解了音频的解码和播放过程，这篇我们一起来看一下ijk播放器在Android平台上视频是如何进行解码和播放的。在Android上ijk支持视频的软件和硬解，我们先看一下软解及其播放过程。
1.线程创建
其实在之前的文章里面已经贴过视频渲染线程和视频解码线程的创建相关的代码了，但为了完整的流程，这里还是贴一下。
视频渲染线程创建
c
static VideoState *stream_open(FFPlayer *ffp, const char *filename, AVInputFormat *iformat)
{
//创建video刷新线程
is->video_refresh_tid = SDL_CreateThreadEx(&is->_video_refresh_tid, video_refresh_thread, ffp, "ff_vout");
}
// 渲染线程入口函数
static int video_refresh_thread(void *arg)
{
FFPlayer *ffp = arg;
VideoState *is = ffp->is;
double remaining_time = 0.0;
while (!is->abort_request) {
if (remaining_time > 0.0)
av_usleep((int)(int64_t)(remaining_time * 1000000.0));
remaining_time = REFRESH_RATE;
if (is->show_mode != SHOW_MODE_NONE && (!is->paused || is->force_refresh))
video_refresh(ffp, &remaining_time);
}
return 0;
}视频解码线程创建
c
static int stream_component_open(FFPlayer *ffp, int stream_index)
{
..........
case AVMEDIA_TYPE_VIDEO:
is->video_stream = stream_index;
is->video_st = ic->streams[stream_index];
if (ffp->async_init_decoder) {
while (!is->initialized_decoder) {
SDL_Delay(5);
}
if (ffp->node_vdec) {
is->viddec.avctx = avctx;
ret = ffpipeline_config_video_decoder(ffp->pipeline, ffp);
}
if (ret || !ffp->node_vdec) {
decoder_init(&is->viddec, avctx, &is->videoq, is->continue_read_thread);
ffp->node_vdec = ffpipeline_open_video_decoder(ffp->pipeline, ffp);
if (!ffp->node_vdec)
goto fail;
}
} else {
decoder_init(&is->viddec, avctx, &is->videoq, is->continue_read_thread);
//初始化解码器，如果支持硬解走硬解，否则使用软解
ffp->node_vdec = ffpipeline_open_video_decoder(ffp->pipeline, ffp);
if (!ffp->node_vdec)
goto fail;
}
//创建video解码线程
if ((ret = decoder_start(&is->viddec, video_thread, ffp, "ff_video_dec")) < 0)
goto out;
}
// 视频解码线程入口函数
static int video_thread(void *arg)
{
FFPlayer *ffp = (FFPlayer *)arg;
int ret = 0;
if (ffp->node_vdec) {
ret = ffpipenode_run_sync(ffp->node_vdec);
}
return ret;
}解码相关接口配置
c
IJKFF_Pipenode* ffpipeline_open_video_decoder(IJKFF_Pipeline *pipeline, FFPlayer *ffp)
{
return pipeline->func_open_video_decoder(pipeline, ffp);
}
static IJKFF_Pipenode *func_open_video_decoder(IJKFF_Pipeline *pipeline, FFPlayer *ffp)
{
IJKFF_Pipeline_Opaque *opaque = pipeline->opaque;
IJKFF_Pipenode *node = NULL;
//优先使用硬解码器，会先查看视频流格式是否符合，不不符合或者没有开启硬解码，使用软解码
if (ffp->mediacodec_all_videos || ffp->mediacodec_avc || ffp->mediacodec_hevc || ffp->mediacodec_mpeg2)
node = ffpipenode_create_video_decoder_from_android_mediacodec(ffp, pipeline, opaque->weak_vout);
if (!node) {
node = ffpipenode_create_video_decoder_from_ffplay(ffp);
}
return node;
}这里我们先看软解码所配置的接口。
c
IJKFF_Pipenode *ffpipenode_create_video_decoder_from_ffplay(FFPlayer *ffp)
{
IJKFF_Pipenode *node = ffpipenode_alloc(sizeof(IJKFF_Pipenode_Opaque));
if (!node)
return node;
IJKFF_Pipenode_Opaque *opaque = node->opaque;
opaque->ffp = ffp;
node->func_destroy = func_destroy;
node->func_run_sync = func_run_sync;
ffp_set_video_codec_info(ffp, AVCODEC_MODULE_NAME, avcodec_get_name(ffp->is->viddec.avctx->codec_id));
ffp->stat.vdec_type = FFP_PROPV_DECODER_AVCODEC;
return node;
}2.视频解码
执行之前配置的接口。
c
static int video_thread(void *arg)
{
FFPlayer *ffp = (FFPlayer *)arg;
int ret = 0;
if (ffp->node_vdec) {
ret = ffpipenode_run_sync(ffp->node_vdec);
}
return ret;
}下面是软解码的过程，视频的软解码与音频的类似，都是调用decoder_decode_frame获取AVFrame，然后再放入到pictq中等待被渲染。
c
static int func_run_sync(IJKFF_Pipenode *node)
{
IJKFF_Pipenode_Opaque *opaque = node->opaque;
return ffp_video_thread(opaque->ffp);
}
int ffp_video_thread(FFPlayer *ffp)
{
return ffplay_video_thread(ffp);
}
static int ffplay_video_thread(void *arg)
{
FFPlayer *ffp = arg;
VideoState *is = ffp->is;
AVFrame *frame = av_frame_alloc();
double pts;
double duration;
int ret;
AVRational tb = is->video_st->time_base;
AVRational frame_rate = av_guess_frame_rate(is->ic, is->video_st, NULL);
int64_t dst_pts = -1;
int64_t last_dst_pts = -1;
int retry_convert_image = 0;
int convert_frame_count = 0;
ffp_notify_msg2(ffp, FFP_MSG_VIDEO_ROTATION_CHANGED, ffp_get_video_rotate_degrees(ffp));
if (!frame) {
return AVERROR(ENOMEM);
}
for (;;) {
//填充frame
ret = get_video_frame(ffp, frame);
if (ret < 0)
goto the_end;
if (!ret)
continue;
duration = (frame_rate.num && frame_rate.den ? av_q2d((AVRational){frame_rate.den, frame_rate.num}) : 0);
pts = (frame->pts == AV_NOPTS_VALUE) ? NAN : frame->pts * av_q2d(tb);
//添加到帧队列
ret = queue_picture(ffp, frame, pts, duration, frame->pkt_pos, is->viddec.pkt_serial);
av_frame_unref(frame);
if (ret < 0)
goto the_end;
}
the_end:
av_log(NULL, AV_LOG_INFO, "convert image convert_frame_count = %d\n", convert_frame_count);
av_frame_free(&frame);
return 0;
}获取视频帧
c
static int get_video_frame(FFPlayer *ffp, AVFrame *frame)
{
VideoState *is = ffp->is;
int got_picture;
ffp_video_statistic_l(ffp);
//执行avcodec_send_packet和avcodec_receive_frame，获取解码后的frame
if ((got_picture = decoder_decode_frame(ffp, &is->viddec, frame, NULL)) < 0)
return -1;
if (got_picture) {
double dpts = NAN;
if (frame->pts != AV_NOPTS_VALUE)
dpts = av_q2d(is->video_st->time_base) * frame->pts;
frame->sample_aspect_ratio = av_guess_sample_aspect_ratio(is->ic, is->video_st, frame);
}
return got_picture;
}添加到视频帧队列
在添加到帧队列前还需要做两件事情：
1.如果从队列中取出的可写Frame还未分配过或画面宽高发生变化需要为其重新分配SDL_VoutOverlay。
2.转换为目标像素格式，并填充到Frame中。
c
static int queue_picture(FFPlayer *ffp, AVFrame *src_frame, double pts, double duration, int64_t pos, int serial)
{
VideoState *is = ffp->is;
Frame *vp;
int video_accurate_seek_fail = 0;
int64_t video_seek_pos = 0;
int64_t now = 0;
int64_t deviation = 0;
int64_t deviation2 = 0;
int64_t deviation3 = 0;
//从pictq中取出可写frame
if (!(vp = frame_queue_peek_writable(&is->pictq)))
return -1;
vp->sar = src_frame->sample_aspect_ratio;
/* alloc or resize hardware picture buffer */
if (!vp->bmp || !vp->allocated ||
vp->width != src_frame->width ||
vp->height != src_frame->height ||
vp->format != src_frame->format) {
if (vp->width != src_frame->width || vp->height != src_frame->height)
ffp_notify_msg3(ffp, FFP_MSG_VIDEO_SIZE_CHANGED, src_frame->width, src_frame->height);
//设置格式宽高
vp->allocated = 0;
vp->width = src_frame->width;
vp->height = src_frame->height;
vp->format = src_frame->format;
/* the allocation must be done in the main thread to avoid
locking problems. */
//分配picture
alloc_picture(ffp, src_frame->format);
if (is->videoq.abort_request)
return -1;
}
/* if the frame is not skipped, then display it */
if (vp->bmp) {
/* get a pointer on the bitmap */
SDL_VoutLockYUVOverlay(vp->bmp);
// FIXME: set swscale options
// 进行像素数据填充
if (SDL_VoutFillFrameYUVOverlay(vp->bmp, src_frame) < 0) {
av_log(NULL, AV_LOG_FATAL, "Cannot initialize the conversion context\n");
exit(1);
}
/* update the bitmap content */
SDL_VoutUnlockYUVOverlay(vp->bmp);
vp->pts = pts;
vp->duration = duration;
vp->pos = pos;
vp->serial = serial;
vp->sar = src_frame->sample_aspect_ratio;
vp->bmp->sar_num = vp->sar.num;
vp->bmp->sar_den = vp->sar.den;
//添加可读frame
//至此我们能拿到渲染一帧画面的数据
frame_queue_push(&is->pictq);
if (!is->viddec.first_frame_decoded) {
ALOGD("Video: first frame decoded\n");
ffp_notify_msg1(ffp, FFP_MSG_VIDEO_DECODED_START);
is->viddec.first_frame_decoded_time = SDL_GetTickHR();
is->viddec.first_frame_decoded = 1;
}
}
return 0;
}分配SDL_VoutOverlay
在ijk中默认是使用软解码，显示的格式默认为SDL_FCC_RV32，并使用ANativeWindow拷贝像素方式实现视频的绘制。
c
static void alloc_picture(FFPlayer *ffp, int frame_format)
{
av_log(NULL, AV_LOG_WARNING, "hyc alloc_picture frame_format=%d",frame_format);
VideoState *is = ffp->is;
Frame *vp;
vp = &is->pictq.queue[is->pictq.windex];
// 手动清除，软件绘制没有做清理操作
free_picture(vp);
//设置SDL_Vout 的format即需要显示的格式默认为SDL_FCC_RV32
SDL_VoutSetOverlayFormat(ffp->vout, ffp->overlay_format);
//创建Overlay
vp->bmp = SDL_Vout_CreateOverlay(vp->width, vp->height,
frame_format,
ffp->vout);
SDL_LockMutex(is->pictq.mutex);
vp->allocated = 1;
SDL_CondSignal(is->pictq.cond);
SDL_UnlockMutex(is->pictq.mutex);
}
static SDL_VoutOverlay *func_create_overlay(int width, int height, int frame_format, SDL_Vout *vout)
{
SDL_LockMutex(vout->mutex);
SDL_VoutOverlay *overlay = func_create_overlay_l(width, height, frame_format, vout);
SDL_UnlockMutex(vout->mutex);
return overlay;
}
static SDL_VoutOverlay *func_create_overlay_l(int width, int height, int frame_format, SDL_Vout *vout)
{
//在获取frame的时候设置 frame_format，硬解码为IJK_AV_PIX_FMT__ANDROID_MEDIACODEC
switch (frame_format) {
case IJK_AV_PIX_FMT__ANDROID_MEDIACODEC:
return SDL_VoutAMediaCodec_CreateOverlay(width, height, vout);
default:
return SDL_VoutFFmpeg_CreateOverlay(width, height, frame_format, vout);
}
}创建SDL_VoutOverlay时，根据需要显示的像素类型，去选择FFmpeg中的对应的像素格式，并分配好包含这种像素格式的AVFrame.
c
SDL_VoutOverlay *SDL_VoutFFmpeg_CreateOverlay(int width, int height, int frame_format, SDL_Vout *display)
{
//需要显示的格式
Uint32 overlay_format = display->overlay_format;
switch (overlay_format) {
// 使用opengl进行渲染
//根据像素格式重新设置overlay_format格式
case SDL_FCC__GLES2: {
switch (frame_format) {
case AV_PIX_FMT_YUV444P10LE:
overlay_format = SDL_FCC_I444P10LE;
break;
case AV_PIX_FMT_YUV420P:
case AV_PIX_FMT_YUVJ420P:
default:
#if defined(__ANDROID__)
overlay_format = SDL_FCC_YV12;
#else
overlay_format = SDL_FCC_I420;
#endif
break;
}
break;
}
}
SDL_VoutOverlay *overlay = SDL_VoutOverlay_CreateInternal(sizeof(SDL_VoutOverlay_Opaque));
if (!overlay) {
ALOGE("overlay allocation failed");
return NULL;
}
//填充SDL_VoutOverlay_Opaque的数据
SDL_VoutOverlay_Opaque *opaque = overlay->opaque;
opaque->mutex = SDL_CreateMutex();
opaque->sws_flags = SWS_BILINEAR;
overlay->opaque_class = &g_vout_overlay_ffmpeg_class;
overlay->format = overlay_format;
overlay->pitches = opaque->pitches;
overlay->pixels = opaque->pixels;
overlay->w = width;
overlay->h = height;
overlay->free_l = func_free_l;
overlay->lock = func_lock;
overlay->unlock = func_unlock;
overlay->func_fill_frame = func_fill_frame;
enum AVPixelFormat ff_format = AV_PIX_FMT_NONE;
int buf_width = width;
int buf_height = height;
//根据overlay_format转换为FFmpeg中的像素格式和buf_width对齐
switch (overlay_format) {
case SDL_FCC_I420:
case SDL_FCC_YV12: {
//SDL_FCC_I420与SDL_FCC_YV12 相同，都属于yuv，y u v分别存储，4个y对应1个uv分量
// U、V 平面的 strides, width 和 height 都是 Y 平面的一半
//I420 现存u再存v，YV12 先存v再存u
ff_format = AV_PIX_FMT_YUV420P;
// FIXME: need runtime config
#if defined(__ANDROID__)
// 16 bytes align pitch for arm-neon image-convert
buf_width = IJKALIGN(width, 16); // 1 bytes per pixel for Y-plane
#elif defined(__APPLE__)
// 2^n align for width
buf_width = width;
if (width > 0)
buf_width = 1 << (sizeof(int) * 8 - __builtin_clz(width));
#else
buf_width = IJKALIGN(width, 16); // unknown platform
#endif
//存在3个分量
opaque->planes = 3;
break;
}
case SDL_FCC_I444P10LE: {
ff_format = AV_PIX_FMT_YUV444P10LE;
// FIXME: need runtime config
#if defined(__ANDROID__)
// 16 bytes align pitch for arm-neon image-convert
buf_width = IJKALIGN(width, 16); // 1 bytes per pixel for Y-plane
#elif defined(__APPLE__)
// 2^n align for width
buf_width = width;
if (width > 0)
buf_width = 1 << (sizeof(int) * 8 - __builtin_clz(width));
#else
buf_width = IJKALIGN(width, 16); // unknown platform
#endif
opaque->planes = 3;
break;
}
case SDL_FCC_RV16: {
ff_format = AV_PIX_FMT_RGB565;
buf_width = IJKALIGN(width, 8); // 2 bytes per pixel
opaque->planes = 1;
break;
}
case SDL_FCC_RV24: {
ff_format = AV_PIX_FMT_RGB24;
#if defined(__ANDROID__)
// 16 bytes align pitch for arm-neon image-convert
buf_width = IJKALIGN(width, 16); // 1 bytes per pixel for Y-plane
#elif defined(__APPLE__)
buf_width = width;
#else
buf_width = IJKALIGN(width, 16); // unknown platform
#endif
opaque->planes = 1;
break;
}
case SDL_FCC_RV32: {
ff_format = AV_PIX_FMT_0BGR32;
buf_width = IJKALIGN(width, 4); // 4 bytes per pixel
opaque->planes = 1;
break;
}
default:
ALOGE("SDL_VoutFFmpeg_CreateOverlay(...): unknown format %.4s(0x%x)\n", (char*)&overlay_format, overlay_format);
goto fail;
}
//分配对应格式的AVFrame
opaque->managed_frame = opaque_setup_frame(opaque, ff_format, buf_width, buf_height);
if (!opaque->managed_frame) {
ALOGE("overlay->opaque->frame allocation failed\n");
goto fail;
}
//进行指针赋值
overlay_fill(overlay, opaque->managed_frame, opaque->planes);
return overlay;
fail:
func_free_l(overlay);
return NULL;
}分配AVFrame
在我们取出带像素数据的AVFrame后，还没办法立马将其渲染消费掉，但是代码里面是添加到队列之后就会进行回收操作，所以我们需要对入队列的帧数据分配我们自己的AVFrame。在分配AVFrame时会创建两个Frame对象，一种用于不需要转换直接从原始Frame中进行拷贝，一种则需要进行转换，并还需要进行Buffer的分配。
c
static AVFrame *opaque_setup_frame(SDL_VoutOverlay_Opaque* opaque, enum AVPixelFormat format, int width, int height)
{
// 需要转换像素时使用
AVFrame *managed_frame = av_frame_alloc();
if (!managed_frame) {
return NULL;
}
// 不需要转换像素时使用
AVFrame *linked_frame = av_frame_alloc();
if (!linked_frame) {
av_frame_free(&managed_frame);
return NULL;
}
/*-
* Lazily allocate frame buffer in opaque_obtain_managed_frame_buffer
*
* For refererenced frame management, we use buffer allocated by decoder
*
int frame_bytes = avpicture_get_size(format, width, height);
AVBufferRef *frame_buffer_ref = av_buffer_alloc(frame_bytes);
if (!frame_buffer_ref)
return NULL;
opaque->frame_buffer = frame_buffer_ref;
*/
managed_frame->format = format;
managed_frame->width = width;
managed_frame->height = height;
// 初始化managed_frame，未填充数据
av_image_fill_arrays(managed_frame->data, managed_frame->linesize ,NULL,
format, width, height, 1);
opaque->managed_frame = managed_frame;
opaque->linked_frame = linked_frame;
return managed_frame;
}像素数据填充
在分配好SDL_VoutOverlay后，还需要为其填充可渲染的像素数据。在这个过程中会根据解码后的AVFrame的像素格式和目标像素格式进行转换。这里如果使用OpenGL进行绘制，并且像素格式是YUV420p的，那么可以不需要转换。如果不转换的话就使用刚才分配好的linked_frame,将需要渲染的帧数据直接链接过去。如果进行转换，那么还需要为managed_frame分配好buffer。
c
static int func_fill_frame(SDL_VoutOverlay *overlay, const AVFrame *frame)
{
assert(overlay);
SDL_VoutOverlay_Opaque *opaque = overlay->opaque;
AVFrame swscale_dst_pic = { { 0 } };
av_frame_unref(opaque->linked_frame);
//获取需要展示的数据和当前的数据格式，进行像素格式转换
int need_swap_uv = 0;
int use_linked_frame = 0;
enum AVPixelFormat dst_format = AV_PIX_FMT_NONE;
//这里如果使用OpenGL进行绘制，并且像素格式是YUV420p的，那么可以不需要转换
switch (overlay->format) {
case SDL_FCC_YV12:
//vu需要转换为uv
need_swap_uv = 1;
// no break;
case SDL_FCC_I420:
if (frame->format == AV_PIX_FMT_YUV420P || frame->format == AV_PIX_FMT_YUVJ420P) {
// ALOGE("direct draw frame");
//不需要进行像素转换
use_linked_frame = 1;
dst_format = frame->format;
} else {
// ALOGE("copy draw frame");
dst_format = AV_PIX_FMT_YUV420P;
}
break;
case SDL_FCC_I444P10LE:
if (frame->format == AV_PIX_FMT_YUV444P10LE) {
// ALOGE("direct draw frame");
use_linked_frame = 1;
dst_format = frame->format;
} else {
// ALOGE("copy draw frame");
dst_format = AV_PIX_FMT_YUV444P10LE;
}
break;
case SDL_FCC_RV32:
dst_format = AV_PIX_FMT_0BGR32;
break;
case SDL_FCC_RV24:
dst_format = AV_PIX_FMT_RGB24;
break;
case SDL_FCC_RV16:
dst_format = AV_PIX_FMT_RGB565;
break;
default:
ALOGE("SDL_VoutFFmpeg_ConvertPicture: unexpected overlay format %s(%d)",
(char*)&overlay->format, overlay->format);
return -1;
}
// setup frame
//如果不需要转换像素格式
if (use_linked_frame) {
// linked frame
av_frame_ref(opaque->linked_frame, frame);
overlay_fill(overlay, opaque->linked_frame, opaque->planes);
//uv位置转换
if (need_swap_uv)
FFSWAP(Uint8*, overlay->pixels[1], overlay->pixels[2]);
} else {
//需要转换像素格式
// managed frame
AVFrame* managed_frame = opaque_obtain_managed_frame_buffer(opaque);
if (!managed_frame) {
ALOGE("OOM in opaque_obtain_managed_frame_buffer");
return -1;
}
overlay_fill(overlay, opaque->managed_frame, opaque->planes);
// setup frame managed
for (int i = 0; i < overlay->planes; ++i) {
swscale_dst_pic.data[i] = overlay->pixels[i];
swscale_dst_pic.linesize[i] = overlay->pitches[i];
}
if (need_swap_uv)
FFSWAP(Uint8*, swscale_dst_pic.data[1], swscale_dst_pic.data[2]);
}
// swscale / direct draw
/*
ALOGE("ijk_image_convert w=%d, h=%d, df=%d, dd=%d, dl=%d, sf=%d, sd=%d, sl=%d",
(int)frame->width,
(int)frame->height,
(int)dst_format,
(int)swscale_dst_pic.data[0],
(int)swscale_dst_pic.linesize[0],
(int)frame->format,
(int)(const uint8_t**) frame->data,
(int)frame->linesize);
*/
//如果需要转换先尝试使用libyuv中进行yuv转rgb，如果转换失败，使用sws_scale进行转换
if (use_linked_frame) {
// do nothing
} else if (ijk_image_convert(frame->width, frame->height,
dst_format, swscale_dst_pic.data, swscale_dst_pic.linesize,
frame->format, (const uint8_t**) frame->data, frame->linesize)) {
opaque->img_convert_ctx = sws_getCachedContext(opaque->img_convert_ctx,
frame->width, frame->height, frame->format, frame->width, frame->height,
dst_format, opaque->sws_flags, NULL, NULL, NULL);
if (opaque->img_convert_ctx == NULL) {
ALOGE("sws_getCachedContext failed");
return -1;
}
sws_scale(opaque->img_convert_ctx, (const uint8_t**) frame->data, frame->linesize,
0, frame->height, swscale_dst_pic.data, swscale_dst_pic.linesize);
if (!opaque->no_neon_warned) {
opaque->no_neon_warned = 1;
ALOGE("non-neon image convert %s -> %s", av_get_pix_fmt_name(frame->format), av_get_pix_fmt_name(dst_format));
}
}
// TODO: 9 draw black if overlay is larger than screen
return 0;
}分配buffer
c
static AVFrame *opaque_obtain_managed_frame_buffer(SDL_VoutOverlay_Opaque* opaque)
{
if (opaque->frame_buffer != NULL)
return opaque->managed_frame;
AVFrame *managed_frame = opaque->managed_frame;
int frame_bytes = av_image_get_buffer_size(managed_frame->format, managed_frame->width, managed_frame->height, 1);
// 为managed_frame分配buffer
AVBufferRef *frame_buffer_ref = av_buffer_alloc(frame_bytes);
if (!frame_buffer_ref)
return NULL;
av_image_fill_arrays(managed_frame->data, managed_frame->linesize,
frame_buffer_ref->data, managed_frame->format, managed_frame->width, managed_frame->height, 1);
opaque->frame_buffer = frame_buffer_ref;
return opaque->managed_frame;
}将数据指向SDL_VoutOverlay
分配好了buffer之后，将SDL_VoutOverlay的相关像素数据指向AVFrame,后续就可以从这里拿到像素数据了。
c
static void overlay_fill(SDL_VoutOverlay *overlay, AVFrame *frame, int planes)
{
overlay->planes = planes;
for (int i = 0; i < AV_NUM_DATA_POINTERS; ++i) {
overlay->pixels[i] = frame->data[i];
overlay->pitches[i] = frame->linesize[i];
}
}像素转换
在android平台上，默认的目标像素是AV_PIX_FMT_0BGR32,如果视频像素格式是YUV420P，那么使用libyuv进行转换。
c
int ijk_image_convert(int width, int height,
enum AVPixelFormat dst_format, uint8_t **dst_data, int *dst_linesize,
enum AVPixelFormat src_format, const uint8_t **src_data, const int *src_linesize)
{
#if defined(__ANDROID__)
switch (src_format) {
case AV_PIX_FMT_YUV420P:
case AV_PIX_FMT_YUVJ420P: // FIXME: 9 not equal to AV_PIX_FMT_YUV420P, but a workaround
switch (dst_format) {
case AV_PIX_FMT_RGB565:
return I420ToRGB565(
src_data[0], src_linesize[0],
src_data[1], src_linesize[1],
src_data[2], src_linesize[2],
dst_data[0], dst_linesize[0],
width, height);
case AV_PIX_FMT_0BGR32:
return I420ToABGR(
src_data[0], src_linesize[0],
src_data[1], src_linesize[1],
src_data[2], src_linesize[2],
dst_data[0], dst_linesize[0],
width, height);
default:
break;
}
break;
default:
break;
}
#endif
return -1;
}3.视频渲染
这里我们根据上面的像素类型进行绘制流程跟踪。渲染过程就是对SDL_VoutOverlay中像素数据进行拷贝的过程。
c
static void video_image_display2(FFPlayer *ffp)
{
VideoState *is = ffp->is;
Frame *vp;
Frame *sp = NULL;
vp = frame_queue_peek_last(&is->pictq);
if (vp->bmp) {
if (ffp->render_wait_start && !ffp->start_on_prepared && is->pause_req) {
if (!ffp->first_video_frame_rendered) {
ffp->first_video_frame_rendered = 1;
ffp_notify_msg1(ffp, FFP_MSG_VIDEO_RENDERING_START);
}
//暂停
while (is->pause_req && !is->abort_request) {
SDL_Delay(20);
}
}
//执行绘制
SDL_VoutDisplayYUVOverlay(ffp->vout, vp->bmp);
ffp->stat.vfps = SDL_SpeedSamplerAdd(&ffp->vfps_sampler, FFP_SHOW_VFPS_FFPLAY, "vfps[ffplay]");
if (!ffp->first_video_frame_rendered) {
ffp->first_video_frame_rendered = 1;
ffp_notify_msg1(ffp, FFP_MSG_VIDEO_RENDERING_START);
}
if (is->latest_video_seek_load_serial == vp->serial) {
int latest_video_seek_load_serial = __atomic_exchange_n(&(is->latest_video_seek_load_serial), -1, memory_order_seq_cst);
if (latest_video_seek_load_serial == vp->serial) {
ffp->stat.latest_seek_load_duration = (av_gettime() - is->latest_seek_load_start_at) / 1000;
if (ffp->av_sync_type == AV_SYNC_VIDEO_MASTER) {
ffp_notify_msg2(ffp, FFP_MSG_VIDEO_SEEK_RENDERING_START, 1);
} else {
ffp_notify_msg2(ffp, FFP_MSG_VIDEO_SEEK_RENDERING_START, 0);
}
}
}
}
}目前overlay->format是SDL_FCC_RV32，并且vout->overlay_format的值也是这个。所以上面的判断条件都不符合，会进入最后的Window绘制流程。
c
int SDL_VoutDisplayYUVOverlay(SDL_Vout *vout, SDL_VoutOverlay *overlay)
{
if (vout && overlay && vout->display_overlay)
return vout->display_overlay(vout, overlay);
return -1;
}
static int func_display_overlay(SDL_Vout *vout, SDL_VoutOverlay *overlay)
{
SDL_LockMutex(vout->mutex);
int retval = func_display_overlay_l(vout, overlay);
SDL_UnlockMutex(vout->mutex);
return retval;
}
static int func_display_overlay_l(SDL_Vout *vout, SDL_VoutOverlay *overlay)
{
SDL_Vout_Opaque *opaque = vout->opaque;
ANativeWindow *native_window = opaque->native_window;
if (!native_window) {
if (!opaque->null_native_window_warned) {
opaque->null_native_window_warned = 1;
ALOGW("func_display_overlay_l: NULL native_window");
}
return -1;
} else {
opaque->null_native_window_warned = 1;
}
if (!overlay) {
ALOGE("func_display_overlay_l: NULL overlay");
return -1;
}
if (overlay->w <= 0 || overlay->h <= 0) {
ALOGE("func_display_overlay_l: invalid overlay dimensions(%d, %d)", overlay->w, overlay->h);
return -1;
}
switch(overlay->format) {
case SDL_FCC__AMC: {
// only ANativeWindow support
IJK_EGL_terminate(opaque->egl);
//渲染为true就会渲染到surface
return SDL_VoutOverlayAMediaCodec_releaseFrame_l(overlay, NULL, true);
}
case SDL_FCC_RV24:
case SDL_FCC_I420:
case SDL_FCC_I444P10LE: {
// only GLES support
if (opaque->egl)
return IJK_EGL_display(opaque->egl, native_window, overlay);
break;
}
case SDL_FCC_YV12:
case SDL_FCC_RV16:
case SDL_FCC_RV32: {
// both GLES & ANativeWindow support
// 使用OpenGL进行绘制
if (vout->overlay_format == SDL_FCC__GLES2 && opaque->egl)
return IJK_EGL_display(opaque->egl, native_window, overlay);
break;
}
}
// fallback to ANativeWindow
IJK_EGL_terminate(opaque->egl);
return SDL_Android_NativeWindow_display_l(native_window, overlay);
}ANativeWindow绘制
绘制流程如下：
- 获取当前像素buffer所对应的像素格式，获取当前
ANativeWindow对应的像素格式 - 判断大小与格式是否相同，如果不同，那么调用
ANativeWindow_setBuffersGeometry进行配置 - 根据目标像素格式将
SDL_VoutOverlay的像素数据拷贝到ANativeWindow中
c
int SDL_Android_NativeWindow_display_l(ANativeWindow *native_window, SDL_VoutOverlay *overlay)
{
int retval;
if (!native_window)
return -1;
if (!overlay) {
ALOGE("SDL_Android_NativeWindow_display_l: NULL overlay");
return -1;
}
if (overlay->w <= 0 || overlay->h <= 0) {
ALOGE("SDL_Android_NativeWindow_display_l: invalid overlay dimensions(%d, %d)", overlay->w, overlay->h);
return -1;
}
int curr_w = ANativeWindow_getWidth(native_window);
int curr_h = ANativeWindow_getHeight(native_window);
int curr_format = ANativeWindow_getFormat(native_window);
int buff_w = IJKALIGN(overlay->w, 2);
int buff_h = IJKALIGN(overlay->h, 2);
// 获取当前像素buffer所对应的像素格式
AndroidHalFourccDescriptor *overlayDesc = native_window_get_desc(overlay->format);
if (!overlayDesc) {
ALOGE("SDL_Android_NativeWindow_display_l: unknown overlay format: %d", overlay->format);
return -1;
}
// 获取当前ANativeWindow对应的像素格式
AndroidHalFourccDescriptor *voutDesc = native_window_get_desc(curr_format);
if (!voutDesc || voutDesc->hal_format != overlayDesc->hal_format) {
ALOGD("ANativeWindow_setBuffersGeometry: w=%d, h=%d, f=%.4s(0x%x) => w=%d, h=%d, f=%.4s(0x%x)",
curr_w, curr_h, (char*) &curr_format, curr_format,
buff_w, buff_h, (char*) &overlay->format, overlay->format);
//如果当前window的像素格式与配置的像素格式不同，进行配置，可能会失败
retval = ANativeWindow_setBuffersGeometry(native_window, buff_w, buff_h, overlayDesc->hal_format);
if (retval < 0) {
ALOGE("SDL_Android_NativeWindow_display_l: ANativeWindow_setBuffersGeometry: failed %d", retval);
return retval;
}
if (!voutDesc) {
ALOGE("SDL_Android_NativeWindow_display_l: unknown hal format %d", curr_format);
return -1;
}
}
ANativeWindow_Buffer out_buffer;
retval = ANativeWindow_lock(native_window, &out_buffer, NULL);
if (retval < 0) {
ALOGE("SDL_Android_NativeWindow_display_l: ANativeWindow_lock: failed %d", retval);
return retval;
}
//二次确认宽度是否设置成功
if (out_buffer.width != buff_w || out_buffer.height != buff_h) {
ALOGE("unexpected native window buffer (%p)(w:%d, h:%d, fmt:'%.4s'0x%x), expecting (w:%d, h:%d, fmt:'%.4s'0x%x)",
native_window,
out_buffer.width, out_buffer.height, (char*)&out_buffer.format, out_buffer.format,
buff_w, buff_h, (char*)&overlay->format, overlay->format);
// TODO: 8 set all black
ANativeWindow_unlockAndPost(native_window);
ANativeWindow_setBuffersGeometry(native_window, buff_w, buff_h, overlayDesc->hal_format);
return -1;
}
//进行像素数据拷贝，也即渲染
int render_ret = voutDesc->render(&out_buffer, overlay);
if (render_ret < 0) {
// TODO: 8 set all black
// return after unlock image;
}
retval = ANativeWindow_unlockAndPost(native_window);
if (retval < 0) {
ALOGE("SDL_Android_NativeWindow_display_l: ANativeWindow_unlockAndPost: failed %d", retval);
return retval;
}
return render_ret;
}像素拷贝
目前我们的overlay_format是SDL_FCC_RV32，根据配置其拷贝函数为android_render_on_rgb8888。在配置中
c
static AndroidHalFourccDescriptor g_hal_fcc_map[] = {
// YV12
{ HAL_PIXEL_FORMAT_YV12, "HAL_YV12", HAL_PIXEL_FORMAT_YV12, android_render_on_yv12 },
{ SDL_FCC_YV12, "YV12", HAL_PIXEL_FORMAT_YV12, android_render_on_yv12 },
// RGB565
{ HAL_PIXEL_FORMAT_RGB_565, "HAL_RGB_565", HAL_PIXEL_FORMAT_RGB_565, android_render_on_rgb565 },
{ SDL_FCC_RV16, "RV16", HAL_PIXEL_FORMAT_RGB_565, android_render_on_rgb565 },
// RGB8888
{ HAL_PIXEL_FORMAT_RGBX_8888, "HAL_RGBX_8888", HAL_PIXEL_FORMAT_RGBX_8888, android_render_on_rgb8888 },
{ HAL_PIXEL_FORMAT_RGBA_8888, "HAL_RGBA_8888", HAL_PIXEL_FORMAT_RGBA_8888, android_render_on_rgb8888 },
{ HAL_PIXEL_FORMAT_BGRA_8888, "HAL_BGRA_8888", HAL_PIXEL_FORMAT_BGRA_8888, android_render_on_rgb8888 },
{ SDL_FCC_RV32, "RV32", HAL_PIXEL_FORMAT_RGBX_8888, android_render_on_rgb8888 },
};
AndroidHalFourccDescriptor *native_window_get_desc(int fourcc_or_hal)
{
for (int i = 0; i < NELEM(g_hal_fcc_map); ++i) {
AndroidHalFourccDescriptor *desc = &g_hal_fcc_map[i];
if (desc->fcc_or_hal == fourcc_or_hal)
return desc;
}
return NULL;
}
static int android_render_on_rgb8888(ANativeWindow_Buffer *out_buffer, const SDL_VoutOverlay *overlay)
{
assert(out_buffer);
assert(overlay);
switch (overlay->format) {
case SDL_FCC_RV32: {
return android_render_rgb32_on_rgb8888(out_buffer, overlay);
}
}
return -1;
}
static int android_render_rgb32_on_rgb8888(ANativeWindow_Buffer *out_buffer, const SDL_VoutOverlay *overlay)
{
return android_render_rgb_on_rgb(out_buffer, overlay, 32);
}
static int android_render_rgb_on_rgb(ANativeWindow_Buffer *out_buffer, const SDL_VoutOverlay *overlay, int bpp)
{
// SDLTRACE("SDL_VoutAndroid: android_render_rgb_on_rgb(%p)", overlay);
assert(overlay->format == SDL_FCC_RV16);
assert(overlay->planes == 1);
//stride=width+对齐
int min_height = IJKMIN(out_buffer->height, overlay->h);
int dst_stride = out_buffer->stride;
int src_line_size = overlay->pitches[0];
int dst_line_size = dst_stride * bpp / 8;//一行的字节数
uint8_t *dst_pixels = out_buffer->bits;
// 像素数据
const uint8_t *src_pixels = overlay->pixels[0];
if (dst_line_size == src_line_size) {
int plane_size = src_line_size * min_height;
// ALOGE("android_render_rgb_on_rgb (pix-match) %p %p %d", dst_pixels, src_pixels, plane_size);
memcpy(dst_pixels, src_pixels, plane_size);
} else {
// TODO: 9 padding
int bytewidth = IJKMIN(dst_line_size, src_line_size);
// ALOGE("android_render_rgb_on_rgb (pix-mismatch) %p %d %p %d %d %d", dst_pixels, dst_line_size, src_pixels, src_line_size, bytewidth, min_height);
av_image_copy_plane(dst_pixels, dst_line_size, src_pixels, src_line_size, bytewidth, min_height);
}
return 0;
}至此，ijk在android平台上默认的视频解码和渲染流程都代码已经跟踪完毕，下面来整体回顾一下简要流程：
1.调用decoder_decode_frame获取软解码视频帧AVFrame，里面包含了像素数据
2.将AVFrame包装为SDL_VoutOverlay，按目标格式进行像素转换，SDL_VoutOverlay中pixels将指向可渲染的像素数据，并加入到队列中。
3.渲染线程不断的从队列中获取出SDL_VoutOverlay
4.根据像素格式进行渲染，即从SDL_VoutOverlay所指向的像素copy到ANativeWindow中
以上是个人见解，如果错误或不同见解，欢迎指正和交流。