Unity使用sherpa-onnx实现关键词检测

使用模型 sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01

效果图

具体代码

using uMicrophoneWebGL;
using UnityEngine;

[RequireComponent(typeof(MicrophoneWebGL))]
public class KeywordSpottingSample : MonoBehaviour
{
    MicrophoneWebGL microphone;
    public KeywordSpotting keywordSpotting;

    // Start is called before the first frame update
    void Start()
    {
        keywordSpotting.Init();
        microphone = GetComponent<MicrophoneWebGL>();
        microphone.dataEvent.AddListener(OnAudioData);
    }

    public void OnAudioData(float[] data)
    {
        if (keywordSpotting != null)
        {
            keywordSpotting.AcceptData(data);
        }
    }

    float timer = 0f;
    float interval = 0.2f;
    string keyword;
    private void Update()
    {
        if (keywordSpotting != null && keywordSpotting.initDone)
        {
            timer += Time.deltaTime;
            if (timer >= interval)
            {
                keyword = keywordSpotting.Recognize();
                if (!string.IsNullOrEmpty(keyword))
                {
                    Debug.Log("keyword:" + keyword);
                } 
                timer = 0f;
            } 
        }
    }
}

using System.IO;
using SherpaOnnx;
using UnityEngine;

/// <summary>
/// 关键字识别
/// </summary>
public class KeywordSpotting : MonoBehaviour
{
    KeywordSpotter keywordSpotter;
    string pathRoot;
    string modelPath;
    OnlineStream onlineStream;
    int sampleRate = 16000;
    public bool initDone = false;

    public void Init()
    {
        pathRoot = Util.GetPath() + "/models";

        //需要将此文件夹拷贝到models
        modelPath = pathRoot + "/sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01";
        KeywordSpotterConfig config = new KeywordSpotterConfig();
        config.FeatConfig.SampleRate = 16000;
        config.FeatConfig.FeatureDim = 80;

        config.ModelConfig.Transducer.Encoder = Path.Combine(modelPath, "encoder-epoch-12-avg-2-chunk-16-left-64.onnx");
        config.ModelConfig.Transducer.Decoder = Path.Combine(modelPath, "decoder-epoch-12-avg-2-chunk-16-left-64.onnx");
        config.ModelConfig.Transducer.Joiner = Path.Combine(modelPath, "joiner-epoch-12-avg-2-chunk-16-left-64.onnx");

        config.ModelConfig.Tokens = Path.Combine(modelPath, "tokens.txt");
        config.ModelConfig.Provider = "cpu";
        config.ModelConfig.NumThreads = 1;
        config.ModelConfig.Debug = 0;
        config.KeywordsFile = Path.Combine(modelPath, "keywords.txt");
        keywordSpotter = new KeywordSpotter(config);
        onlineStream = keywordSpotter.CreateStream();
        initDone = true;
    }

    public void AcceptData(float[] data)
    {
        onlineStream.AcceptWaveform(sampleRate, data);
    }

    KeywordResult result;
    public string Recognize()
    { 
        while (keywordSpotter.IsReady(onlineStream))
        {
            keywordSpotter.Decode(onlineStream);
            result = keywordSpotter.GetResult(onlineStream);
            if (result.Keyword != string.Empty)
            {
                Debug.Log("关键字: " + result.Keyword);
                // Remember to call Reset() right after detecting a keyword
                keywordSpotter.Reset(onlineStream);
                return result.Keyword;
            }
        }
        return string.Empty;
    }
}

最后是工程地址

https://github.com/xue-fei/sherpa-onnx-unity