WakeWordDetector.h

Go to the documentation of this file.

#pragma once
 
#include <algorithm>
#include <cmath>
 
#include "AudioTools/CoreAudio/AudioOutput.h"
#include "AudioTools/CoreAudio/AudioBasic/Collections/Vector.h"
#include "AudioTools/CoreAudio/Buffers.h"
#include "AudioTools/FFT/AudioFFT.h"
 
namespace audio_tools {
 
/*
 * @brief Frame holding the top N dominant frequencies (in Hz) of an FFT window.
 *
 * Used as a compact representation of the dominant frequency content in a frame
 * of audio.
 */
template <size_t N>
struct FrequencyFrame {
  uint16_t top_freqs[N];  
};
 
template <typename T = int16_t, size_t N = 3>
class WakeWordDetector : public AudioOutput {
 public:
  struct Template {
    Vector<FrequencyFrame<N>>
        frames;  
    float threshold_percent;  
    const char* name;         
    float last_match_percent =
        0.0f;  
  };
 
  using WakeWordCallback = void (*)(const char* name);
 
  WakeWordDetector(AudioFFTBase& fft)
      : p_fft(&fft) {
    _frame_pos = 0;
  }
 
  void startRecording() {
    _recent_frames.clear();
    _is_recording = true;
  }
 
  Vector<FrequencyFrame<N>> stopRecording() {
    _is_recording = false;
    return _recent_frames;
  }
 
  bool isRecording() const { return _is_recording; }
 
  // Note: takes a non-const reference because AudioTools' Vector::size() is
  // not const-qualified.
  void addTemplate(Vector<FrequencyFrame<N>>& frames,
                   float threshold_percent, const char* name) {
    Template t;
    t.frames = frames;
    t.threshold_percent = threshold_percent;
    t.name = name;
    t.last_match_percent = 0.0f;
    _templates.push_back(t);
    if (frames.size() > _max_template_len) _max_template_len = frames.size();
  }
 
  void setWakeWordCallback(WakeWordCallback cb) { _callback = cb; }
 
  size_t write(const uint8_t* buf, size_t size) override {
    // Reasserted on every write (instead of only in the constructor) because
    // AudioFFTBase::begin(AudioFFTConfig) replaces the whole config object
    // (cfg = info), which would silently wipe out ref/callback if they had
    // only been set once, before the user's fft.begin(cfg) call.
    auto& fft_cfg = p_fft->config();
    if (fft_cfg.ref != this || fft_cfg.callback != fftResult) {
      fft_cfg.ref = this;
      fft_cfg.callback = fftResult;
    }
    return p_fft->write(buf, size);
  }
 
  static void fftResult(AudioFFTBase& fft) {
    // This static method must access instance data via fft.config().ref
    auto* self = static_cast<WakeWordDetector<T,N>*>(fft.config().ref);
    if (!self) return;
    FrequencyFrame<N> frame;
    AudioFFTResult result[N];
      fft.resultArray(result);
    for (size_t j = 0; j < N; j++) {
      frame.top_freqs[j] = result[j].frequency;
    }
    self->_recent_frames.push_back(frame);
 
    if (self->_is_recording) {
      return;
    }
 
    if (self->_recent_frames.size() > self->_max_template_len)
      self->_recent_frames.erase(self->_recent_frames.begin());
    for (size_t i = 0; i < self->_templates.size(); ++i) {
      Template& tmpl = self->_templates[i];
      if (self->_recent_frames.size() >= tmpl.frames.size()) {
        float percent = self->matchTemplate(tmpl);
        if (percent >= tmpl.threshold_percent) {
          if (self->_callback) self->_callback(tmpl.name);
        }
      }
    }
  }
 
 protected:
  Vector<Template> _templates;               
  Vector<FrequencyFrame<N>> _recent_frames;  
  Vector<T> _buffer;  
  AudioFFTBase* p_fft = nullptr;
  bool _is_recording = false;    
  size_t _frame_pos;             
  size_t _max_template_len = 0;  
  WakeWordCallback _callback = nullptr;
 
  float matchTemplate(Template& tmpl) {
    size_t matches = 0;
    size_t offset = _recent_frames.size() - tmpl.frames.size();
    for (size_t i = 0; i < tmpl.frames.size(); ++i) {
      size_t frame_matches = 0;
      for (size_t j = 0; j < N; ++j) {
        if (tmpl.frames[i].top_freqs[j] ==
            _recent_frames[offset + i].top_freqs[j])
          frame_matches++;
      }
      if (frame_matches >= (N >= 2 ? N - 1 : 1))  // at least N-1 out of N match
        matches++;
    }
    float percent = (tmpl.frames.size() > 0)
                        ? (100.0f * matches / tmpl.frames.size())
                        : 0.0f;
    tmpl.last_match_percent = percent;
    return percent;
  }
};
 
}  // namespace audio_tools