1 前言
与IJKPLAYER处理AudioTrack播放类似，OpenSL ES的接入需要满足SDL_Aout的接口规范，所不同的是OpenSL ES播放是在native完成的，调用的是NDK接口OpenSL ES的播放能力。关于OpenSL ES的详细介绍，请参考官方文档 OpenSL ES 一文。
Pipeline及SDL_Aout结构体及相关创建，与AudioTrack一致，请参考前文IJKPLAYER源码分析-AudioTrack播放-CSDN博客
2 接口
2.1 创建SDL_Aout
创建OpenSL ES的SDL_Aout对象，调用链如下：
ijkmp_android_create() => ffpipeline_create_from_android() => func_open_audio_output() => SDL_AoutAndroid_CreateForOpenSLES()
使能opensles选项，缺省为0，即使用AudioTrack播放：
{ "opensles", "OpenSL ES: enable",
OPTION_OFFSET(opensles), OPTION_INT(0, 0, 1) },
若使能了opensles选项，走OpenSL ES播放，相反则走AudioTrack播放：
static SDL_Aout *func_open_audio_output(IJKFF_Pipeline *pipeline, FFPlayer *ffp)
{
SDL_Aout *aout = NULL;
if (ffp->opensles) {
aout = SDL_AoutAndroid_CreateForOpenSLES();
} else {
aout = SDL_AoutAndroid_CreateForAudioTrack();
}
if (aout)
SDL_AoutSetStereoVolume(aout, pipeline->opaque->left_volume, pipeline->opaque->right_volume);
return aout;
}
OpenSL ES的SDL_Aout对象创建具体在此，遵循SDL_Aout接口规范：
SDL_Aout *SDL_AoutAndroid_CreateForOpenSLES()
{
SDLTRACE("%s\n", __func__);
SDL_Aout *aout = SDL_Aout_CreateInternal(sizeof(SDL_Aout_Opaque));
if (!aout)
return NULL;
SDL_Aout_Opaque *opaque = aout->opaque;
opaque->wakeup_cond = SDL_CreateCond();
opaque->wakeup_mutex = SDL_CreateMutex();
int ret = 0;
SLObjectItf slObject = NULL;
ret = slCreateEngine(&slObject, 0, NULL, 0, NULL, NULL);
CHECK_OPENSL_ERROR(ret, "%s: slCreateEngine() failed", __func__);
opaque->slObject = slObject;
ret = (*slObject)->Realize(slObject, SL_BOOLEAN_FALSE);
CHECK_OPENSL_ERROR(ret, "%s: slObject->Realize() failed", __func__);
SLEngineItf slEngine = NULL;
ret = (*slObject)->GetInterface(slObject, SL_IID_ENGINE, &slEngine);
CHECK_OPENSL_ERROR(ret, "%s: slObject->GetInterface() failed", __func__);
opaque->slEngine = slEngine;
SLObjectItf slOutputMixObject = NULL;
const SLInterfaceID ids1[] = {SL_IID_VOLUME};
const SLboolean req1[] = {SL_BOOLEAN_FALSE};
ret = (*slEngine)->CreateOutputMix(slEngine, &slOutputMixObject, 1, ids1, req1);
CHECK_OPENSL_ERROR(ret, "%s: slEngine->CreateOutputMix() failed", __func__);
opaque->slOutputMixObject = slOutputMixObject;
ret = (*slOutputMixObject)->Realize(slOutputMixObject, SL_BOOLEAN_FALSE);
CHECK_OPENSL_ERROR(ret, "%s: slOutputMixObject->Realize() failed", __func__);
aout->free_l = aout_free_l;
aout->opaque_class = &g_opensles_class;
aout->open_audio = aout_open_audio;
aout->pause_audio = aout_pause_audio;
aout->flush_audio = aout_flush_audio;
aout->close_audio = aout_close_audio;
aout->set_volume = aout_set_volume;
aout->func_get_latency_seconds = aout_get_latency_seconds;
return aout;
fail:
aout_free_l(aout);
return NULL;
}
2.2 func_get_latency_seconds接口
- 此接口是OpenSL ES相比于AudioTrack新增的1个接口；
- 作用是：用来计算OpenSL ES底层buffer缓存了多少ms的音频数据，音视频同步时用来纠正音频的时钟；
与AudioTrack相比，OpenSL ES增加了func_get_latency_seconds接口：
aout->func_get_latency_seconds = aout_get_latency_seconds;
此接口的具体实现：
static double aout_get_latency_seconds(SDL_Aout *aout)
{
SDL_Aout_Opaque *opaque = aout->opaque;
SLAndroidSimpleBufferQueueState state = {0};
SLresult slRet = (*opaque->slBufferQueueItf)->GetState(opaque->slBufferQueueItf, &state);
if (slRet != SL_RESULT_SUCCESS) {
ALOGE("%s failed\n", __func__);
return ((double)opaque->milli_per_buffer) * OPENSLES_BUFFERS / 1000;
}
// assume there is always a buffer in coping
// state.count表示已用buffer个数
double latency = ((double)opaque->milli_per_buffer) * state.count / 1000;
return latency;
}
以上就是OpenSL ES的SDL_Aout创建过程，大致与AudioTrack类似，遵循了SDL_Aout的接口规范，所不同的是OpenSL ES增加了func_get_latency_seconds接口，用来计算底层缓存的音频ms数；
3 open_audio
3.1 初始化
OpenSL ES的打开，即OpenSL ES的初始化，主要做以下事情：
OpenSL ES的open_audio接口，大致功能与AudioTrack类似；
将音频源的采样参数(采样率、通道数、采样位深)告诉OpenSL ES，并调用CreateAudioPlayer()创建SLObjectItf类型的播放器实例；
使用播放器实例，查询SLPlayItf实例/SLVolumeItf实例/SLAndroidSimpleBufferQueueItf实例；
使用播放器实例SLAndroidSimpleBufferQueueItf类型队列实例，并给该队列注册callback；
根据最终的音频参数(采样率、通道数、采样位深)，以及OpenSL ES每个buffer所能容纳的音频PCM数据量(10ms),，计算出最终的buffer总容量，并分配buffer内存；
启动1个audio_thread线程，此线程的作用与AudioTrack的audio_thread作用一致，异步执行所有关于音频的操作，取得音频的PCM数据并喂给OpenSL ES；
将最终的音频参数保存在全局变量is->audio_tgt中，后续若音频参数发生变更，需要重采样并且重置is->audio_tgt的值；
设置OpenSL ES的缺省延迟时间，即OpenSL ES最多缓存了多少秒的PCM数据，此值在音视频同步时纠正音频的时钟有重要用处；
// 设置缺省时延，若有func_set_default_latency_seconds回调则通过回调更新，没有则设置变量minimal_latency_seconds的值
SDL_AoutSetDefaultLatencySeconds(ffp->aout, ((double)(2 * spec.size)) / audio_hw_params->bytes_per_sec);
3.2 pcm buffer
定义OpenSL ES每个buffer的音频容量，即能装下10ms的PCM数据，一共由255个音频buffer组成，即10 * 255 = 2550ms的数据。
2个参数的具体宏定义请参照：
#define OPENSLES_BUFFERS 255 /* maximum number of buffers */
#define OPENSLES_BUFLEN 10 /* ms */
计算最终OpenGL ES的buffer容量，并分配buffer：
// 对于opensl es来说，播放的是pcm数据，每个pcm为1帧
opaque->bytes_per_frame = format_pcm->numChannels * format_pcm->bitsPerSample / 8;
// 对于opensl es来说，每次播放10ms的pcm数据
opaque->milli_per_buffer = OPENSLES_BUFLEN;
// 每OPENSLES_BUFLEN=10ms有多少pcm帧
opaque->frames_per_buffer = opaque->milli_per_buffer * format_pcm->samplesPerSec / 1000000; // samplesPerSec is in milli
opaque->bytes_per_buffer = opaque->bytes_per_frame * opaque->frames_per_buffer;
opaque->buffer_capacity = OPENSLES_BUFFERS * opaque->bytes_per_buffer;
ALOGI("OpenSL-ES: bytes_per_frame = %d bytes\n", (int)opaque->bytes_per_frame);
ALOGI("OpenSL-ES: milli_per_buffer = %d ms\n", (int)opaque->milli_per_buffer);
ALOGI("OpenSL-ES: frame_per_buffer = %d frames\n", (int)opaque->frames_per_buffer);
ALOGI("OpenSL-ES: bytes_per_buffer = %d bytes\n", (int)opaque->bytes_per_buffer);
ALOGI("OpenSL-ES: buffer_capacity = %d bytes\n", (int)opaque->buffer_capacity);
// 分配最终的pcm缓冲区
opaque->buffer = malloc(opaque->buffer_capacity);
CHECK_COND_ERROR(opaque->buffer, "%s: failed to alloc buffer %d\n", __func__, (int)opaque->buffer_capacity);
// (*opaque->slPlayItf)->SetPositionUpdatePeriod(opaque->slPlayItf, 1000);
// enqueue empty buffer to start play
memset(opaque->buffer, 0, opaque->buffer_capacity);
for(int i = 0; i < OPENSLES_BUFFERS; ++i) {
ret = (*opaque->slBufferQueueItf)->Enqueue(opaque->slBufferQueueItf, opaque->buffer + i * opaque->bytes_per_buffer, opaque->bytes_per_buffer);
CHECK_OPENSL_ERROR(ret, "%s: slBufferQueueItf->Enqueue(000...) failed", __func__);
}
值得一提的是，OpenGL ES在此所支持的音频采样参数如下：
CHECK_COND_ERROR((desired->format == AUDIO_S16SYS), "%s: not AUDIO_S16SYS", __func__);
CHECK_COND_ERROR((desired->channels == 2 || desired->channels == 1), "%s: not 1,2 channel", __func__);
CHECK_COND_ERROR((desired->freq >= 8000 && desired->freq <= 48000), "%s: unsupport freq %d Hz", __func__, desired->freq);
3.3 调用流程
打开OpenSL ES的调用链，其实是和AudioTrack一致，因为他们遵循了同样的接口规范SDL_Aout，具体如下：
read_thread() => stream_component_open() => audio_open() => SDL_AoutOpenAudio() => aout_open_audio()
最后，走到aout_open_audio方法：
static int aout_open_audio(SDL_Aout *aout, const SDL_AudioSpec *desired, SDL_AudioSpec *obtained)
{
SDLTRACE("%s\n", __func__);
assert(desired);
SDLTRACE("aout_open_audio()\n");
SDL_Aout_Opaque *opaque = aout->opaque;
SLEngineItf slEngine = opaque->slEngine;
SLDataFormat_PCM *format_pcm = &opaque->format_pcm;
int ret = 0;
opaque->spec = *desired;
// config audio src
SLDataLocator_AndroidSimpleBufferQueue loc_bufq = {
SL_DATALOCATOR_ANDROIDSIMPLEBUFFERQUEUE,
OPENSLES_BUFFERS
};
int native_sample_rate = audiotrack_get_native_output_sample_rate(NULL);
ALOGI("OpenSL-ES: native sample rate %d Hz\n", native_sample_rate);
// opensl es仅支持以下参数的audio pcm播放
CHECK_COND_ERROR((desired->format == AUDIO_S16SYS), "%s: not AUDIO_S16SYS", __func__);
CHECK_COND_ERROR((desired->channels == 2 || desired->channels == 1), "%s: not 1,2 channel", __func__);
CHECK_COND_ERROR((desired->freq >= 8000 && desired->freq <= 48000), "%s: unsupport freq %d Hz", __func__, desired->freq);
if (SDL_Android_GetApiLevel() < IJK_API_21_LOLLIPOP &&
native_sample_rate > 0 &&
desired->freq < native_sample_rate) {
// Don't try to play back a sample rate higher than the native one,
// since OpenSL ES will try to use the fast path, which AudioFlinger
// will reject (fast path can't do resampling), and will end up with
// too small buffers for the resampling. See http://b.android.com/59453
// for details. This bug is still present in 4.4. If it is fixed later
// this workaround could be made conditional.
//
// by VLC/android_opensles.c
ALOGW("OpenSL-ES: force resample %lu to native sample rate %d\n",
(unsigned long) format_pcm->samplesPerSec / 1000,
(int) native_sample_rate);
format_pcm->samplesPerSec = native_sample_rate * 1000;
}
format_pcm->formatType = SL_DATAFORMAT_PCM;
format_pcm->numChannels = desired->channels;
format_pcm->samplesPerSec = desired->freq * 1000; // milli Hz
// format_pcm->numChannels = 2;
// format_pcm->samplesPerSec = SL_SAMPLINGRATE_44_1;
format_pcm->bitsPerSample = SL_PCMSAMPLEFORMAT_FIXED_16;
format_pcm->containerSize = SL_PCMSAMPLEFORMAT_FIXED_16;
switch (desired->channels) {
case 2:
format_pcm->channelMask = SL_SPEAKER_FRONT_LEFT | SL_SPEAKER_FRONT_RIGHT;
break;
case 1:
format_pcm->channelMask = SL_SPEAKER_FRONT_CENTER;
break;
default:
ALOGE("%s, invalid channel %d", __func__, desired->channels);
goto fail;
}
format_pcm->endianness = SL_BYTEORDER_LITTLEENDIAN;
SLDataSource audio_source = {&loc_bufq, format_pcm};
// config audio sink
SLDataLocator_OutputMix loc_outmix = {
SL_DATALOCATOR_OUTPUTMIX,
opaque->slOutputMixObject
};
SLDataSink audio_sink = {&loc_outmix, NULL};
SLObjectItf slPlayerObject = NULL;
const SLInterfaceID ids2[] = { SL_IID_ANDROIDSIMPLEBUFFERQUEUE, SL_IID_VOLUME, SL_IID_PLAY };
static const SLboolean req2[] = { SL_BOOLEAN_TRUE, SL_BOOLEAN_TRUE, SL_BOOLEAN_TRUE };
// 在此将audio的采样参数传递给opensl es
ret = (*slEngine)->CreateAudioPlayer(slEngine, &slPlayerObject, &audio_source,
&audio_sink, sizeof(ids2) / sizeof(*ids2),
ids2, req2);
CHECK_OPENSL_ERROR(ret, "%s: slEngine->CreateAudioPlayer() failed", __func__);
opaque->slPlayerObject = slPlayerObject;
ret = (*slPlayerObject)->Realize(slPlayerObject, SL_BOOLEAN_FALSE);
CHECK_OPENSL_ERROR(ret, "%s: slPlayerObject->Realize() failed", __func__);
ret = (*slPlayerObject)->GetInterface(slPlayerObject, SL_IID_PLAY, &opaque->slPlayItf);
CHECK_OPENSL_ERROR(ret, "%s: slPlayerObject->GetInterface(SL_IID_PLAY) failed", __func__);
ret = (*slPlayerObject)->GetInterface(slPlayerObject, SL_IID_VOLUME, &opaque->slVolumeItf);
CHECK_OPENSL_ERROR(ret, "%s: slPlayerObject->GetInterface(SL_IID_VOLUME) failed", __func__);
ret = (*slPlayerObject)->GetInterface(slPlayerObject, SL_IID_ANDROIDSIMPLEBUFFERQUEUE, &opaque->slBufferQueueItf);
CHECK_OPENSL_ERROR(ret, "%s: slPlayerObject->GetInterface(SL_IID_ANDROIDSIMPLEBUFFERQUEUE) failed", __func__);
ret = (*opaque->slBufferQueueItf)->RegisterCallback(opaque->slBufferQueueItf, aout_opensles_callback, (void*)aout);
CHECK_OPENSL_ERROR(ret, "%s: slBufferQueueItf->RegisterCallback() failed", __func__);
// set the player's state to playing
// ret = (*opaque->slPlayItf)->SetPlayState(opaque->slPlayItf, SL_PLAYSTATE_PLAYING);
// CHECK_OPENSL_ERROR(ret, "%s: slBufferQueueItf->slPlayItf() failed", __func__);
// 对于opensl es来说，播放的是pcm数据，每个pcm为1帧
opaque->bytes_per_frame = format_pcm->numChannels * format_pcm->bitsPerSample / 8;
// 对于opensl es来说，每次播放是10ms的pcm数据
opaque->milli_per_buffer = OPENSLES_BUFLEN;
// 每OPENSLES_BUFLEN 10ms有多少pcm帧
opaque->frames_per_buffer = opaque->milli_per_buffer * format_pcm->samplesPerSec / 1000000; // samplesPerSec is in milli
opaque->bytes_per_buffer = opaque->bytes_per_frame * opaque->frames_per_buffer;
opaque->buffer_capacity = OPENSLES_BUFFERS * opaque->bytes_per_buffer;
ALOGI("OpenSL-ES: bytes_per_frame = %d bytes\n", (int)opaque->bytes_per_frame);
ALOGI("OpenSL-ES: milli_per_buffer = %d ms\n", (int)opaque->milli_per_buffer);
ALOGI("OpenSL-ES: frame_per_buffer = %d frames\n", (int)opaque->frames_per_buffer);
ALOGI("OpenSL-ES: bytes_per_buffer = %d bytes\n", (int)opaque->bytes_per_buffer);
ALOGI("OpenSL-ES: buffer_capacity = %d bytes\n", (int)opaque->buffer_capacity);
// 根据计算出来的buffer_capacity分配buffer
opaque->buffer = malloc(opaque->buffer_capacity);
CHECK_COND_ERROR(opaque->buffer, "%s: failed to alloc buffer %d\n", __func__, (int)opaque->buffer_capacity);
// (*opaque->slPlayItf)->SetPositionUpdatePeriod(opaque->slPlayItf, 1000);
// enqueue empty buffer to start play
memset(opaque->buffer, 0, opaque->buffer_capacity);
for(int i = 0; i < OPENSLES_BUFFERS; ++i) {
ret = (*opaque->slBufferQueueItf)->Enqueue(opaque->slBufferQueueItf, opaque->buffer + i * opaque->bytes_per_buffer, opaque->bytes_per_buffer);
CHECK_OPENSL_ERROR(ret, "%s: slBufferQueueItf->Enqueue(000...) failed", __func__);
}
opaque->pause_on = 1;
opaque->abort_request = 0;
opaque->audio_tid = SDL_CreateThreadEx(&opaque->_audio_tid, aout_thread, aout, "ff_aout_opensles");
CHECK_COND_ERROR(opaque->audio_tid, "%s: failed to SDL_CreateThreadEx", __func__);
if (obtained) {
*obtained = *desired;
// opensl es音频硬件的缓冲区容量
obtained->size = opaque->buffer_capacity;
obtained->freq = format_pcm->samplesPerSec / 1000;
}
return opaque->buffer_capacity;
fail:
aout_close_audio(aout);
return -1;
}
以上就是OpenSL ES的完整初始化流程。
4 audio_thread
该线程主要做以下事情：
- 响应业务侧对声音的操作，异步处理诸如play()/pause()/flush()/setVolume()等操作；
- 通过sdl_audio_callback回调，取得PCM数据，再喂给OpenSL ES播放；
- 每次取的PCM数据数是opaque->bytes_per_buffer，即OPENSLES_BUFLEN=10ms的PCM数据；
4.1 执行操作
在此临界区执行对声音的操作，播放/暂停/调节音量/flush()等：
SLAndroidSimpleBufferQueueState slState = {0};
SLresult slRet = (*slBufferQueueItf)->GetState(slBufferQueueItf, &slState);
if (slRet != SL_RESULT_SUCCESS) {
ALOGE("%s: slBufferQueueItf->GetState() failed\n", __func__);
SDL_UnlockMutex(opaque->wakeup_mutex);
}
SDL_LockMutex(opaque->wakeup_mutex);
if (!opaque->abort_request && (opaque->pause_on || slState.count >= OPENSLES_BUFFERS)) {
while (!opaque->abort_request && (opaque->pause_on || slState.count >= OPENSLES_BUFFERS)) {
if (!opaque->pause_on) {
(*slPlayItf)->SetPlayState(slPlayItf, SL_PLAYSTATE_PLAYING);
}
SDL_CondWaitTimeout(opaque->wakeup_cond, opaque->wakeup_mutex, 1000);
SLresult slRet = (*slBufferQueueItf)->GetState(slBufferQueueItf, &slState);
if (slRet != SL_RESULT_SUCCESS) {
ALOGE("%s: slBufferQueueItf->GetState() failed\n", __func__);
SDL_UnlockMutex(opaque->wakeup_mutex);
}
if (opaque->pause_on)
(*slPlayItf)->SetPlayState(slPlayItf, SL_PLAYSTATE_PAUSED);
}
if (!opaque->abort_request && !opaque->pause_on) {
(*slPlayItf)->SetPlayState(slPlayItf, SL_PLAYSTATE_PLAYING);
}
}
if (opaque->need_flush) {
opaque->need_flush = 0;
(*slBufferQueueItf)->Clear(slBufferQueueItf);
}
if (opaque->need_set_volume) {
opaque->need_set_volume = 0;
SLmillibel level = android_amplification_to_sles((opaque->left_volume + opaque->right_volume) / 2);
ALOGI("slVolumeItf->SetVolumeLevel((%f, %f) -> %d)\n", opaque->left_volume, opaque->right_volume, (int)level);
slRet = (*slVolumeItf)->SetVolumeLevel(slVolumeItf, level);
if (slRet != SL_RESULT_SUCCESS) {
ALOGE("slVolumeItf->SetVolumeLevel failed %d\n", (int)slRet);
// just ignore error
}
}
SDL_UnlockMutex(opaque->wakeup_mutex);
......
4.2 sdl_audio_callback
此处与AudioTrack逻辑一致，请参考 IJKPLAYER源码分析-AudioTrack播放-CSDN博客
- 确保通过此callback取得opaque->bytes_per_buffer字节的PCM数据即可，无论是静音PCM数据抑或真实的可播放的PCM数据；
4.3 喂PCM数据
再将取得的PCM数据喂给OpenSL ES播放即可：
......
next_buffer = opaque->buffer + next_buffer_index * bytes_per_buffer;
next_buffer_index = (next_buffer_index + 1) % OPENSLES_BUFFERS;
audio_cblk(userdata, next_buffer, bytes_per_buffer);
if (opaque->need_flush) {
(*slBufferQueueItf)->Clear(slBufferQueueItf);
opaque->need_flush = false;
}
if (opaque->need_flush) {
ALOGE("flush");
opaque->need_flush = 0;
(*slBufferQueueItf)->Clear(slBufferQueueItf);
} else {
// 每次送给opensl es的Audio样本byte数为OPENSLES_BUFLEN=10ms所采集的PCM样本
slRet = (*slBufferQueueItf)->Enqueue(slBufferQueueItf, next_buffer, bytes_per_buffer);
if (slRet == SL_RESULT_SUCCESS) {
// do nothing
} else if (slRet == SL_RESULT_BUFFER_INSUFFICIENT) {
// don't retry, just pass through
ALOGE("SL_RESULT_BUFFER_INSUFFICIENT\n");
} else {
ALOGE("slBufferQueueItf->Enqueue() = %d\n", (int)slRet);
break;
}
}