Unreal的Audio::IAudioCaptureStream在Android中录制数据异常

修改OpenAudioCaptureStream启动参数为PCM_32，在PC上正常，在Android系统，读取的的数据计算出的音量值在0.4-0.6之间跳动，数据异常。

csharp 复制代码

    Audio::FAudioCaptureDeviceParams Params;
    /*
     * 设置声卡不支持的采样数和通道数开始音频流不会成功，这里不能修改
     * Params.NumInputChannels = 1;
     * Params.SampleRate = 16000;
     * 
     * 可以修改PCM数据格式，默认是32位浮点数FLOATING_POINT_32
     * 我这里修改为32位整数PCM_32
     */
    Params.PCMAudioEncoding = Audio::EPCMAudioEncoding::PCM_32;
    // 使用 TFunction 包装成员函数
    Audio::FOnAudioCaptureFunction OnCapture = [this](const void* InAudio, int32 NumFrames, int32 NumChannels, int32 SampleRate, double StreamTime, bool bOverFlow)
      {
        this->OnAudioCapture(InAudio, NumFrames, NumChannels, SampleRate, StreamTime, bOverFlow);
      };
    bool r = AudioCapture->OpenAudioCaptureStream(Params, MoveTemp(OnCapture), 4800);

修改为FLOATING_POINT_32，按照float值读取数据则是正常的。

csharp 复制代码

    Audio::FAudioCaptureDeviceParams Params;
    /*
     * 设置声卡不支持的采样数和通道数开始音频流不会成功，这里不能修改
     * Params.NumInputChannels = 1;
     * Params.SampleRate = 16000;
     * 
     * 可以修改PCM数据格式，默认是32位浮点数FLOATING_POINT_32
     * 修改为32位整数PCM_32，在Android系统有问题，还是修改为FLOATING_POINT_32
     */
    Params.PCMAudioEncoding = Audio::EPCMAudioEncoding::FLOATING_POINT_32;
    // 使用 TFunction 包装成员函数
    Audio::FOnAudioCaptureFunction OnCapture = [this](const void* InAudio, int32 NumFrames, int32 NumChannels, int32 SampleRate, double StreamTime, bool bOverFlow)
      {
        this->OnAudioCapture(InAudio, NumFrames, NumChannels, SampleRate, StreamTime, bOverFlow);
      };
    bool r = AudioCapture->OpenAudioCaptureStream(Params, MoveTemp(OnCapture), 1920); // 48000采样率可以在重采样是整除3

全部代码

FxAudioCaptureComponent.h

csharp 复制代码

#pragma once

#include "CoreMinimal.h"
#include "Components/ActorComponent.h"
#include "AudioCaptureDeviceInterface.h"
#include "HAL/ThreadSafeCounter.h"
#include "HAL/Thread.h"
#include "FxAudioCaptureComponent.generated.h"

UCLASS(ClassGroup = (Custom), meta = (BlueprintSpawnableComponent))
class UFxAudioCaptureComponent : public UActorComponent
{
  GENERATED_BODY()

public:
  UFxAudioCaptureComponent();

  UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "XML")
    int ID;
  UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "XML")
    float Intensity;
  UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "XML")
    bool bMobile;

protected:
  virtual void BeginPlay() override;
  virtual void EndPlay(const EEndPlayReason::Type EndPlayReason);
  // Called every frame
  virtual void TickComponent(float DeltaTime, ELevelTick TickType, FActorComponentTickFunction* ThisTickFunction) override;

  TArray<float> ResampleAndConvert16KHzMono16Bit(const float* inputData, int inputNumSamples, int inputChannels, int inputSampleRate);

public:
  UFUNCTION(BlueprintPure, Category = "FxAudioCapture")
    int GetAudioDataSize();
  UFUNCTION(BlueprintCallable, Category = "FxAudioCapture")
    TArray<uint8> CopyAudioData(int Length, bool bRemove = true);
  UFUNCTION(BlueprintCallable, Category = "FxAudioCapture")
    bool StartRecord(float seconds = 10.0f);
  UFUNCTION(BlueprintCallable, Category = "FxAudioCapture")
    void StopRecord();
  UFUNCTION(BlueprintPure, Category = "FxAudioCapture")
    bool IsRecording();

private:
  FCriticalSection Mutex;
  TArray<float> m_audioData;

  bool bRecording;
  float RecordSeconds;
  TUniquePtr<Audio::IAudioCaptureStream> AudioCapture;
  void OnAudioCapture(const void* InAudio, int32 NumFrames, int32 NumChannels, int32 SampleRate, double StreamTime, bool bOverFlow);
};

FxAudioCaptureComponent.cpp

csharp 复制代码

#include "FxAudioCaptureComponent.h"
#include "AudioCaptureDeviceInterface.h"
#include "AudioCaptureCore.h"
#include "AudioMixer.h"

UFxAudioCaptureComponent::UFxAudioCaptureComponent()
  : bRecording(false)
{
  PrimaryComponentTick.bCanEverTick = true;
}

void UFxAudioCaptureComponent::BeginPlay()
{
  Super::BeginPlay();

  IModularFeatures::Get().LockModularFeatureList();
  TArray<Audio::IAudioCaptureFactory*> AudioCaptureStreamFactories = IModularFeatures::Get().GetModularFeatureImplementations<Audio::IAudioCaptureFactory>(Audio::IAudioCaptureFactory::GetModularFeatureName());
  IModularFeatures::Get().UnlockModularFeatureList();

  // For now, just return the first audio capture stream implemented. We can make this configurable at a later point.
  if (AudioCaptureStreamFactories.Num() > 0 && AudioCaptureStreamFactories[0] != nullptr)
  {
    AudioCapture = AudioCaptureStreamFactories[0]->CreateNewAudioCaptureStream();
    if (!AudioCapture.IsValid())
    {
      GEngine->AddOnScreenDebugMessage(-1, 5.f, FColor::Red, TEXT("CreateNewAudioCaptureStream return null"));
    }
  }
  else {
    GEngine->AddOnScreenDebugMessage(-1, 5.f, FColor::Red, TEXT("no Audio Capture Stream Factories"));
  }
}
void UFxAudioCaptureComponent::EndPlay(const EEndPlayReason::Type EndPlayReason)
{
  Super::EndPlay(EndPlayReason);

  StopRecord();
}
void UFxAudioCaptureComponent::TickComponent(float DeltaTime, ELevelTick TickType, FActorComponentTickFunction* ThisTickFunction)
{
  Super::TickComponent(DeltaTime, TickType, ThisTickFunction);
}
int UFxAudioCaptureComponent::GetAudioDataSize()
{
  int len = 0;
  Mutex.Lock();
  len = m_audioData.Num();
  Mutex.Unlock();
  return len;
}
TArray<uint8> UFxAudioCaptureComponent::CopyAudioData(int numSamples, bool bRemove)
{
  TArray<uint8> Array;
  Mutex.Lock();
  if (0 < numSamples && numSamples <= m_audioData.Num()) {
    for (int i = 0; i < numSamples; ++i) {
      int16_t sample16Bit = static_cast<int16_t>(m_audioData[i] * 32767.0f);
      // 将16位样本存储到Array中
      Array.Push(static_cast<uint8_t>(sample16Bit & 0xFF));
      Array.Push(static_cast<uint8_t>((sample16Bit >> 8) & 0xFF));
    }
    if (bRemove) {
      m_audioData.RemoveAt(0, numSamples);
    }
  }
  Mutex.Unlock();
  return Array;
}
bool UFxAudioCaptureComponent::StartRecord(float seconds)
{
  StopRecord();
  RecordSeconds = seconds;
  if (AudioCapture.IsValid())
  {
    Audio::FAudioCaptureDeviceParams Params;
    /*
     * 设置声卡不支持的采样数和通道数开始音频流不会成功，这里不能修改
     * Params.NumInputChannels = 1;
     * Params.SampleRate = 16000;
     * 
     * 可以修改PCM数据格式，默认是32位浮点数FLOATING_POINT_32
     * 修改为32位整数PCM_32，在Android系统有问题，还是修改为FLOATING_POINT_32
     */
    Params.PCMAudioEncoding = Audio::EPCMAudioEncoding::FLOATING_POINT_32;
    // 使用 TFunction 包装成员函数
    Audio::FOnAudioCaptureFunction OnCapture = [this](const void* InAudio, int32 NumFrames, int32 NumChannels, int32 SampleRate, double StreamTime, bool bOverFlow)
      {
        this->OnAudioCapture(InAudio, NumFrames, NumChannels, SampleRate, StreamTime, bOverFlow);
      };
    bool r = AudioCapture->OpenAudioCaptureStream(Params, MoveTemp(OnCapture), 1920); // 48000采样率可以在重采样是整除3
    if (r) {
      r = AudioCapture->StartStream();
      if (!r) {
        GEngine->AddOnScreenDebugMessage(-1, 5.f, FColor::Red, TEXT("StartStream return false"));
      }
    }
    else {
      GEngine->AddOnScreenDebugMessage(-1, 5.f, FColor::Red, TEXT("OpenAudioCaptureStream return false"));
    }
    bRecording = r;
  }

  return IsRecording();
}
void UFxAudioCaptureComponent::StopRecord()
{
  if (bRecording && AudioCapture.IsValid())
  {
    AudioCapture->StopStream();
    AudioCapture->CloseStream();
  }
  bRecording = false;
}

void UFxAudioCaptureComponent::OnAudioCapture(const void* InAudio, int32 NumFrames, int32 NumChannels, int32 SampleRate, double StreamTime, bool bOverFlow)
{
  // GEngine->AddOnScreenDebugMessage(-1, 5.f, FColor::Red, FString::Printf(TEXT("OnAudioCapture - %d,%d,%d,%f,%d"), NumFrames, NumChannels, SampleRate, (float)StreamTime, bOverFlow ? 1 : 0));
  // 按照16Khz和Mono重采样数据
  TArray<float> data = ResampleAndConvert16KHzMono16Bit(static_cast<const float*>(InAudio), NumFrames, NumChannels, SampleRate);
  // 计算强度
  float total = 0;
  for (float val : data) {
    total += FMath::Abs(val);
  }
  Intensity = total / (float)(data.Num());
  // 拷贝数据
  Mutex.Lock();
  m_audioData.Append(data.GetData(), data.Num());
  Mutex.Unlock();
}

bool UFxAudioCaptureComponent::IsRecording() {
  if (!AudioCapture.IsValid()) {
    return false;
  }
  if (!bRecording) {
    return false;
  }
  if (!AudioCapture->IsCapturing()) {
    return false;
  }

  return true;
}
TArray<float> UFxAudioCaptureComponent::ResampleAndConvert16KHzMono16Bit(const float* inputData, int inputNumSamples, int inputChannels, int inputSampleRate)
{
  int targetSampleRate = 16000;
  // 计算重采样的步长
  double resampleRate = static_cast<double>(inputSampleRate) / targetSampleRate;

  // 临时存储单声道数据
  std::vector<float> monoSamples;

  int id = 0;
  for (int i = 0; i < inputNumSamples; ++i) {
    float sampleValue = 0;
    // 如果是多声道，转换为单声道
    if (inputChannels > 1) {
      float monoValue = 0;
      for (int j = 0; j < inputChannels; ++j) {
        monoValue += inputData[id++];
      }
      // 取平均值以避免溢出
      sampleValue = monoValue / inputChannels;
    }
    else {
      sampleValue = inputData[id++];
    }

    monoSamples.push_back(sampleValue);
  }

  // 重采样
  TArray<float> resampledSamples;
  int targetNumSamples = static_cast<int>(inputNumSamples / resampleRate);
  for (int i = 0; i < targetNumSamples; ++i) {
    double srcIndex = i * resampleRate;
    int srcIndexInt = static_cast<int>(srcIndex);
    double frac = srcIndex - srcIndexInt;

    // 线性插值
    float sample1 = monoSamples[srcIndexInt];
    float sample2 = monoSamples[std::min(srcIndexInt + 1, inputNumSamples - 1)];
    float resampledValue = (1.0 - frac) * sample1 + frac * sample2;
    resampledSamples.Push(resampledValue);
  }

  return resampledSamples;
}