Unreal定制IAudioCaptureStream接口获取Microphone实时PCM数据输入

Unreal定制IAudioCaptureStream接口获取Microphone实时PCM数据输入

• • 第一步：引入AudioCaptureCore模块
  • 第二步：创建IAudioCaptureStream接口
  • 第三步：开始录制PCM音频流
  • 全部代码
  • 蓝图代码

Unreal的UAudioCaptureComponent组件只能在录制一段时间以后一次性获取PCM数据，不能实时获取WAVE音频流。

借助IAudioCaptureStream接口可以获得实时PCM音频流输入。

第一步：引入AudioCaptureCore模块

Project.Build.cs中引入模块

	PublicDependencyModuleNames.AddRange(new string[] { 
      "Core", 
      "CoreUObject",
      "Engine", 
      "InputCore",
      ...
      "AudioCaptureCore"
    });

第二步：创建IAudioCaptureStream接口

通过IAudioCaptureFactory接口创建IAudioCaptureStream，IAudioCaptureFactory有不同平台的实现方法，但我没找到快速创建方法，源代码中有一段私有创建方式，但没公开。

TUniquePtr<IAudioCaptureStream> AudioCapture;

  IModularFeatures::Get().LockModularFeatureList();
  TArray<IAudioCaptureFactory*> AudioCaptureStreamFactories = IModularFeatures::Get().GetModularFeatureImplementations<IAudioCaptureFactory>(IAudioCaptureFactory::GetModularFeatureName());
  IModularFeatures::Get().UnlockModularFeatureList();

  // For now, just return the first audio capture stream implemented. We can make this configurable at a later point.
  if (AudioCaptureStreamFactories.Num() > 0 && AudioCaptureStreamFactories[0] != nullptr)
  {
    AudioCapture = AudioCaptureStreamFactories[0]->CreateNewAudioCaptureStream();
    if (!AudioCapture.IsValid())
    {
      GEngine->AddOnScreenDebugMessage(-1, 5.f, FColor::Red, TEXT("CreateNewAudioCaptureStream return null"));
    }
  }
  else {
    GEngine->AddOnScreenDebugMessage(-1, 5.f, FColor::Red, TEXT("no Audio Capture Stream Factories"));
  }

第三步：开始录制PCM音频流

bool bRecording;

bool UFxAudioCaptureComponent::StartRecord()
{
  if (AudioCapture.IsValid())
  {
    FAudioCaptureDeviceParams Params;
    /*
     * 设置声卡不支持的采样数和通道数开始音频流不会成功，这里不能修改
     * Params.NumInputChannels = 1;
     * Params.SampleRate = 16000;
     * 
     * 可以修改PCM数据格式，默认是32位浮点数FLOATING_POINT_32
     * 我这里修改为32位整数PCM_32
     */
    Params.PCMAudioEncoding = EPCMAudioEncoding::PCM_32;
    // 使用 TFunction 包装成员函数
    FOnAudioCaptureFunction OnCapture = [this](const void* AudioData, int32 NumFrames, int32 NumChannels, int32 SampleRate, double StreamTime, bool bOverFlow)
      {
        this->OnAudioCapture(AudioData, NumFrames, NumChannels, SampleRate, StreamTime, bOverFlow);
      };
    bool r = AudioCapture->OpenAudioCaptureStream(Params, MoveTemp(OnCapture), 1600);
    if (r) {
      r = AudioCapture->StartStream();
      if (!r) {
        GEngine->AddOnScreenDebugMessage(-1, 5.f, FColor::Red, TEXT("StartStream return false"));
      }
    }
    else {
      GEngine->AddOnScreenDebugMessage(-1, 5.f, FColor::Red, TEXT("OpenAudioCaptureStream return false"));
    }
    bRecording = r;
  }

  return bRecording;
}

void UFxAudioCaptureComponent::OnAudioCapture(const void* InAudio, int32 NumFrames, int32 NumChannels, int32 SampleRate, double StreamTime, bool bOverFlow)
{
  // 这里可以传过来音频数据实际通道数和采样率，音频数据具体格式（位数）需要自己在打开音频流时记录
  GEngine->AddOnScreenDebugMessage(-1, 5.f, FColor::Red, FString::Printf(TEXT("OnAudioCapture - %d,%d,%d,%f"), NumFrames, NumChannels, SampleRate, (float)StreamTime));
}

全部代码

以下代码是用作捕获默认MIC输入，并转为16KHz Mono 16Bit PCM数据，用于语音转文本。

#pragma once

#include "CoreMinimal.h"
#include "Components/ActorComponent.h"
#include "AudioCaptureDeviceInterface.h"
#include "FxAudioCaptureComponent.generated.h"

using namespace Audio;

UCLASS(ClassGroup = (Custom), meta = (BlueprintSpawnableComponent))
class UFxAudioCaptureComponent : public UActorComponent
{
  GENERATED_BODY()

public:
  UFxAudioCaptureComponent();

  UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "XML")
    int ID;
  UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "XML")
    float Intensity;
  UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "XML")
    int DataNum;
  UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "XML")
    TArray<uint8> AudioData;

protected:
  virtual void BeginPlay() override;
  virtual void EndPlay(const EEndPlayReason::Type EndPlayReason);
  TArray<uint8> ResampleAndConvert16KHzMono16Bit(const uint8_t* inputData, int dataNum, int inputChannels, int inputBitsPerSample, int inputSampleRate);

public:
  UFUNCTION(BlueprintCallable, Category = "FxAudioCapture")
    bool StartRecord(float seconds = 10.0f);
  UFUNCTION(BlueprintCallable, Category = "FxAudioCapture")
    void StopRecord();
  UFUNCTION(BlueprintCallable, Category = "FxAudioCapture")
    int RemoveRecordData(int Length);
  UFUNCTION(BlueprintPure, Category = "FxAudioCapture")
    bool IsRecording();

private:
  bool bRecording;
  float RecordSeconds;
  TUniquePtr<IAudioCaptureStream> AudioCapture;
  void OnAudioCapture(const void* InAudio, int32 NumFrames, int32 NumChannels, int32 SampleRate, double StreamTime, bool bOverFlow);
};

#include "FxAudioCaptureComponent.h"
#include "AudioCaptureDeviceInterface.h"
#include "AudioCaptureCore.h"
#include "AudioMixer.h"

UFxAudioCaptureComponent::UFxAudioCaptureComponent()
  : bRecording(false)
{
  PrimaryComponentTick.bCanEverTick = true;
}

void UFxAudioCaptureComponent::BeginPlay()
{
  Super::BeginPlay();

  IModularFeatures::Get().LockModularFeatureList();
  TArray<IAudioCaptureFactory*> AudioCaptureStreamFactories = IModularFeatures::Get().GetModularFeatureImplementations<IAudioCaptureFactory>(IAudioCaptureFactory::GetModularFeatureName());
  IModularFeatures::Get().UnlockModularFeatureList();

  // For now, just return the first audio capture stream implemented. We can make this configurable at a later point.
  if (AudioCaptureStreamFactories.Num() > 0 && AudioCaptureStreamFactories[0] != nullptr)
  {
    AudioCapture = AudioCaptureStreamFactories[0]->CreateNewAudioCaptureStream();
    if (!AudioCapture.IsValid())
    {
      GEngine->AddOnScreenDebugMessage(-1, 5.f, FColor::Red, TEXT("CreateNewAudioCaptureStream return null"));
    }
  }
  else {
    GEngine->AddOnScreenDebugMessage(-1, 5.f, FColor::Red, TEXT("no Audio Capture Stream Factories"));
  }
}
void UFxAudioCaptureComponent::EndPlay(const EEndPlayReason::Type EndPlayReason)
{
  Super::EndPlay(EndPlayReason);

  StopRecord();
}
bool UFxAudioCaptureComponent::StartRecord(float seconds)
{
  StopRecord();
  RecordSeconds = seconds;
  if (AudioCapture.IsValid())
  {
    FAudioCaptureDeviceParams Params;
    /*
     * 设置声卡不支持的采样数和通道数开始音频流不会成功，这里不能修改
     * Params.NumInputChannels = 1;
     * Params.SampleRate = 16000;
     * 
     * 可以修改PCM数据格式，默认是32位浮点数FLOATING_POINT_32
     * 我这里修改为32位整数PCM_32
     */
    Params.PCMAudioEncoding = EPCMAudioEncoding::PCM_32;
    // 使用 TFunction 包装成员函数
    FOnAudioCaptureFunction OnCapture = [this](const void* AudioData, int32 NumFrames, int32 NumChannels, int32 SampleRate, double StreamTime, bool bOverFlow)
      {
        this->OnAudioCapture(AudioData, NumFrames, NumChannels, SampleRate, StreamTime, bOverFlow);
      };
    bool r = AudioCapture->OpenAudioCaptureStream(Params, MoveTemp(OnCapture), 1600);
    if (r) {
      r = AudioCapture->StartStream();
      if (!r) {
        GEngine->AddOnScreenDebugMessage(-1, 5.f, FColor::Red, TEXT("StartStream return false"));
      }
    }
    else {
      GEngine->AddOnScreenDebugMessage(-1, 5.f, FColor::Red, TEXT("OpenAudioCaptureStream return false"));
    }
    bRecording = r;
  }

  return IsRecording();
}
void UFxAudioCaptureComponent::StopRecord()
{
  if (bRecording && AudioCapture.IsValid())
  {
    AudioCapture->StopStream();
    AudioCapture->CloseStream();
  }
  bRecording = false;
}

void UFxAudioCaptureComponent::OnAudioCapture(const void* InAudio, int32 NumFrames, int32 NumChannels, int32 SampleRate, double StreamTime, bool bOverFlow)
{
  // GEngine->AddOnScreenDebugMessage(-1, 5.f, FColor::Red, FString::Printf(TEXT("OnAudioCapture - %d,%d,%d,%f"), NumFrames, NumChannels, SampleRate, (float)StreamTime));

  const uint8* data = static_cast<const uint8*>(InAudio);
  int32 bitsPerSample = 32; // PCM样本位数
  int32 sampleSizeInBytes = bitsPerSample / 8; // 每个样本占2字节
  DataNum = NumFrames * NumChannels * sampleSizeInBytes;

  if (16 == bitsPerSample) {
    const int16* p = static_cast<const int16*>(InAudio);
    float val = 0;
    for (int i = 0; i < NumFrames; ++i) {
      int id = i * NumChannels;
      val += FMath::Abs((float)(p[id]) / 32768.0f);
    }
    Intensity = val / NumFrames;
  }
  else if (32 == bitsPerSample) {
    const int32* p = static_cast<const int32*>(InAudio);
    float val = 0;
    for (int i = 0; i < NumFrames; ++i) {
      int id = i * NumChannels;
      val += FMath::Abs((float)(p[id]) / 2147483647.0f);
    }
    Intensity = val / NumFrames;
  }

  // 按照16Khz和Mono重采样数据
  TArray<uint8> Array = ResampleAndConvert16KHzMono16Bit(data, DataNum, NumChannels, bitsPerSample, SampleRate);
  // 拷贝数据
  AudioData.Append(Array.GetData(), Array.Num());

  // 计算录制缓冲区大小，自动停止录制
  int maxBuffSize = sampleSizeInBytes * NumChannels * SampleRate * RecordSeconds;
  if (AudioData.Num() >= maxBuffSize) {
    StopRecord();
  }
}

int UFxAudioCaptureComponent::RemoveRecordData(int Length)
{
  if (Length < AudioData.Num()) {
    Length = AudioData.Num();
  }
  if (Length > 0) {
    AudioData.RemoveAt(0, Length);
  }
  return Length;
}
bool UFxAudioCaptureComponent::IsRecording() {
  if (!AudioCapture.IsValid()) {
    return false;
  }
  if (!bRecording) {
    return false;
  }

  return true;
}
TArray<uint8> UFxAudioCaptureComponent::ResampleAndConvert16KHzMono16Bit(const uint8_t* inputData, int dataNum, int inputChannels, int inputBitsPerSample, int inputSampleRate) {
  int targetSampleRate = 16000;
  int bytesPerSample = inputBitsPerSample / 8; // 每个样本的字节数
  int numSamples = dataNum / (bytesPerSample * inputChannels);
  // 计算重采样的步长
  double resampleRate = static_cast<double>(inputSampleRate) / targetSampleRate;

  // 临时存储单声道数据
  std::vector<int32_t> monoSamples;

  for (int i = 0; i < numSamples; ++i) {
    int32_t sampleValue = 0;

    // 如果是多声道，转换为单声道
    if (inputChannels > 1) {
      int64 monoValue = 0;
      for (int ch = 0; ch < inputChannels; ++ch) {
        int32_t channelValue = 0;
        for (int j = 0; j < bytesPerSample; ++j) {
          channelValue |= (inputData[i * bytesPerSample * inputChannels + ch * bytesPerSample + j] << (j * 8));
        }
        monoValue += channelValue;
      }
      // 取平均值以避免溢出
      sampleValue = monoValue / inputChannels;
    }
    else {
      // 处理多字节样本
      for (int j = 0; j < bytesPerSample; ++j) {
        sampleValue |= (inputData[i * bytesPerSample * inputChannels + j] << (j * 8));
      }
    }

    monoSamples.push_back(sampleValue);
  }

  // 重采样
  std::vector<int32_t> resampledSamples;
  int targetNumSamples = static_cast<int>(numSamples / resampleRate);
  for (int i = 0; i < targetNumSamples; ++i) {
    double srcIndex = i * resampleRate;
    int srcIndexInt = static_cast<int>(srcIndex);
    double frac = srcIndex - srcIndexInt;

    // 线性插值
    int32_t sample1 = monoSamples[srcIndexInt];
    int32_t sample2 = monoSamples[std::min(srcIndexInt + 1, numSamples - 1)];
    int32_t resampledValue = static_cast<int32_t>((1.0 - frac) * sample1 + frac * sample2);
    resampledSamples.push_back(resampledValue);
  }

  TArray<uint8> Array;
  if (32 == inputBitsPerSample) {
    for (int32_t sample : resampledSamples) {
      // 将32位样本转换为16位样本
      int16_t sample16Bit = static_cast<int16_t>(sample >> 16);
      // 将16位样本存储到Array中
      Array.Push(static_cast<uint8_t>(sample16Bit & 0xFF));
      Array.Push(static_cast<uint8_t>((sample16Bit >> 8) & 0xFF));
    }
  }
  else if (16 == inputBitsPerSample) {
    for (int32_t sample : resampledSamples) {
      // 将32位样本转换为16位样本
      int16_t sample16Bit = static_cast<int16_t>(sample);

      // 将16位样本存储到Array中
      Array.Push(static_cast<uint8_t>(sample & 0xFF));
      Array.Push(static_cast<uint8_t>((sample >> 8) & 0xFF));
    }
  }

  return Array;
}

蓝图代码

TArray AudioData;存储的是16KHzMono16Bit音频数据，每次处理一段PCM数据后，调用RemoveRecordData(int Length)移除已处理数据，这样可以不断循环，实时处理音频流。