Add "Speech Recognition" condition

Allows checking for speech patterns on a given OBS audio source

CMakeLists.txt

@@ -576,7 +576,8 @@ set(_dep_license_files
     "openvr:openvr/LICENSE"
     "paho-mqtt:paho.mqtt.cpp/LICENSE"
     "tesseract:tesseract/LICENSE"
-    "websocketpp:websocketpp/COPYING")
+    "websocketpp:websocketpp/COPYING"
+    "whisper:whisper.cpp/LICENSE")
 
 if(DEB_INSTALL)
   if(NOT DATA_OUT_DIR)

data/locale/en-US.ini

@@ -840,6 +840,38 @@ AdvSceneSwitcher.condition.clipboard.condition.isImage="Clipboard contains an im
 AdvSceneSwitcher.condition.clipboard.condition.isURL="Clipboard contains an URL"
 AdvSceneSwitcher.condition.clipboard.condition.matches="Clipboard content matches"
 AdvSceneSwitcher.condition.clipboard.condition.entry="{{conditions}}{{regex}}{{urlInfo}}"
+AdvSceneSwitcher.condition.speech="Speech Recognition (beta)"
+AdvSceneSwitcher.condition.speech.condition.any="Any speech is detected for"
+AdvSceneSwitcher.condition.speech.condition.contains="contains phrase"
+AdvSceneSwitcher.condition.speech.condition.matches="matches"
+AdvSceneSwitcher.condition.speech.layout.any="{{conditions}}{{source}}"
+AdvSceneSwitcher.condition.speech.layout.contains="Transcript of{{source}}{{conditions}}:"
+AdvSceneSwitcher.condition.speech.layout.matches="Transcript of{{source}}{{conditions}}:"
+AdvSceneSwitcher.condition.speech.layout.phrase="{{phrase}}{{regex}}"
+AdvSceneSwitcher.condition.speech.layout.model="Whisper model:{{modelPath}}{{help}}"
+AdvSceneSwitcher.condition.speech.model.help="GGML model files can be downloaded from https://huggingface.co/ggerganov/whisper.cpp (e.g. ggml-base.bin).\nLarger models are more accurate but slower."
+AdvSceneSwitcher.condition.speech.layout.buffer="Audio buffer:{{bufferDuration}}{{help}}"
+AdvSceneSwitcher.condition.speech.browse="Browse..."
+AdvSceneSwitcher.condition.speech.browse.title="Select Whisper model file"
+AdvSceneSwitcher.condition.speech.browse.filter="GGML model files (*.bin);;All files (*)"
+AdvSceneSwitcher.condition.speech.buffer.help="Longer buffer durations improve accuracy but increase latency."
+AdvSceneSwitcher.condition.speech.advanced="Advanced"
+AdvSceneSwitcher.condition.speech.layout.advanced.threads="Threads:{{threads}}"
+AdvSceneSwitcher.condition.speech.layout.advanced.language="Language:{{language}}{{help}}"
+AdvSceneSwitcher.condition.speech.advanced.language.help="Whisper language code (e.g. 'en', 'de', 'fr') or 'auto' to detect automatically."
+AdvSceneSwitcher.condition.speech.layout.advanced.translate="{{translate}}{{help}}"
+AdvSceneSwitcher.condition.speech.advanced.translate="Translate to English"
+AdvSceneSwitcher.condition.speech.advanced.translate.help="Translate non-English speech to English before transcribing.\nUseful when the phrase or regex is written in English but the source speaks another language."
+AdvSceneSwitcher.condition.speech.layout.advanced.vad="VAD energy threshold:{{vad}}{{help}}"
+AdvSceneSwitcher.condition.speech.advanced.vad.help="Minimum RMS energy a buffer must have before running inference. Buffers below this level are treated as silence and skipped. Lower values are more sensitive; raise it if inference triggers on background noise."
+AdvSceneSwitcher.condition.speech.layout.advanced.suppress="{{suppress}}{{help}}"
+AdvSceneSwitcher.condition.speech.advanced.suppress="Suppress non-speech tokens"
+AdvSceneSwitcher.condition.speech.advanced.suppress.help="Remove filler tokens such as [MUSIC] or (applause) that Whisper tends to insert when it detects non-speech sounds."
+AdvSceneSwitcher.condition.speech.layout.advanced.noContext="{{noContext}}{{help}}"
+AdvSceneSwitcher.condition.speech.advanced.noContext="No context"
+AdvSceneSwitcher.condition.speech.advanced.noContext.help="Do not feed the previous transcription back as a prompt for the next buffer.\nPrevents repetition across buffer boundaries at the cost of slightly reduced coherence."
+AdvSceneSwitcher.condition.speech.advanced.listenWhenMuted="Listen when source is muted"
+AdvSceneSwitcher.condition.speech.advanced.useGpu="Use GPU"
 AdvSceneSwitcher.condition.folder="Folder watch"
 AdvSceneSwitcher.condition.folder.tooltip="This condition type will allow you to monitor the contents of a folder.\nNote that the monitoring will *not* recursively scan for changes in sub directories within directories of the selected folder!\nNote that if there are several changes during a short period of time, some of the changes might not emit this signal.\nHowever, the last change in the sequence of changes always will."
 AdvSceneSwitcher.condition.folder.condition.any="Any change happened"
@@ -2460,6 +2492,9 @@ AdvSceneSwitcher.tempVar.streaming.serviceName.description="The name of the stre
 AdvSceneSwitcher.tempVar.clipboard.text="Clipboard text"
 AdvSceneSwitcher.tempVar.clipboard.text.description="The text contained in the clipboard.\nWill be empty if the clipboard does not contain text."
 
+AdvSceneSwitcher.tempVar.speech.speech="Transcribed speech"
+AdvSceneSwitcher.tempVar.speech.speech.description="The text transcribed from the last audio buffer. Only populated when the condition matched."
+
 AdvSceneSwitcher.tempVar.file.content="File content"
 AdvSceneSwitcher.tempVar.file.date="File modification date"
 AdvSceneSwitcher.tempVar.file.basename="File basename"

plugins/CMakeLists.txt

@@ -41,6 +41,7 @@ add_plugin(stream-deck)
 add_plugin(twitch)
 add_plugin(usb)
 add_plugin(video)
+add_plugin(speech)
 
 # ---------------------------------------------------------------------------- #
 

plugins/speech/CMakeLists.txt

@@ -0,0 +1,125 @@
+cmake_minimum_required(VERSION 3.14)
+project(advanced-scene-switcher-speech)
+
+# --- Check requirements ---
+
+get_target_property(ADVSS_SOURCE_DIR advanced-scene-switcher-lib SOURCE_DIR)
+set(WHISPER_DIR "${ADVSS_SOURCE_DIR}/deps/whisper.cpp")
+if(NOT EXISTS "${WHISPER_DIR}/CMakeLists.txt")
+  message(WARNING "whisper.cpp directory \"${WHISPER_DIR}\" not found!\n"
+                  "Speech condition will be disabled!\n\n"
+                  "Clone whisper.cpp into: ${WHISPER_DIR}")
+  return()
+endif()
+
+set(WHISPER_BUILD_TESTS
+    OFF
+    CACHE BOOL "" FORCE)
+set(WHISPER_BUILD_EXAMPLES
+    OFF
+    CACHE BOOL "" FORCE)
+set(BUILD_SHARED_LIBS
+    OFF
+    CACHE BOOL "" FORCE)
+
+# Vulkan GPU backend. Auto-enabled when VULKAN_SDK is set (e.g. in CI).
+option(ADVSS_SPEECH_VULKAN "Use Vulkan GPU backend for speech recognition" OFF)
+
+if(NOT ADVSS_SPEECH_VULKAN AND DEFINED ENV{VULKAN_SDK})
+  set(ADVSS_SPEECH_VULKAN ON)
+endif()
+
+if(ADVSS_SPEECH_VULKAN)
+  set(GGML_VULKAN
+      ON
+      CACHE BOOL "" FORCE)
+endif()
+
+# Suppress warnings-as-errors on all targets in a whisper.cpp source subtree.
+function(_advss_whisper_suppress_werror dir)
+  get_property(
+    _subdirs
+    DIRECTORY "${dir}"
+    PROPERTY SUBDIRECTORIES)
+  foreach(_sub IN LISTS _subdirs)
+    _advss_whisper_suppress_werror("${_sub}")
+  endforeach()
+  get_property(
+    _targets
+    DIRECTORY "${dir}"
+    PROPERTY BUILDSYSTEM_TARGETS)
+  foreach(_target IN LISTS _targets)
+    get_target_property(_type ${_target} TYPE)
+    if(_type STREQUAL "INTERFACE_LIBRARY" OR _type STREQUAL "UTILITY")
+      continue()
+    endif()
+    get_target_property(_opts ${_target} COMPILE_OPTIONS)
+    if(_opts)
+      list(FILTER _opts EXCLUDE REGEX "^-Werror")
+      list(REMOVE_ITEM _opts /WX)
+      set_target_properties(${_target} PROPERTIES COMPILE_OPTIONS "${_opts}")
+    endif()
+    target_compile_options(
+      ${_target}
+      PRIVATE $<$<C_COMPILER_ID:GNU,Clang,AppleClang>:-Wno-error>
+              $<$<CXX_COMPILER_ID:GNU,Clang,AppleClang>:-Wno-error>
+              $<$<C_COMPILER_ID:MSVC>:/WX-> $<$<CXX_COMPILER_ID:MSVC>:/WX->)
+  endforeach()
+endfunction()
+
+set(_advss_saved_werror ${CMAKE_COMPILE_WARNING_AS_ERROR})
+set(_advss_saved_pic ${CMAKE_POSITION_INDEPENDENT_CODE})
+set(CMAKE_COMPILE_WARNING_AS_ERROR OFF)
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+if(NOT TARGET whisper)
+  add_subdirectory("${WHISPER_DIR}" "${CMAKE_BINARY_DIR}/whisper.cpp"
+                   EXCLUDE_FROM_ALL)
+  _advss_whisper_suppress_werror("${WHISPER_DIR}")
+  # ggml-metal's .m files use manual retain/release; disable ARC on that target.
+  if(TARGET ggml-metal)
+    set_target_properties(ggml-metal
+                          PROPERTIES XCODE_ATTRIBUTE_CLANG_ENABLE_OBJC_ARC NO)
+    target_compile_options(ggml-metal
+                           PRIVATE $<$<COMPILE_LANGUAGE:OBJC>:-fno-objc-arc>)
+  endif()
+endif()
+set(CMAKE_COMPILE_WARNING_AS_ERROR ${_advss_saved_werror})
+set(CMAKE_POSITION_INDEPENDENT_CODE ${_advss_saved_pic})
+
+# ggml-vulkan finds SPIRV-Headers but does not link it.
+if(ADVSS_SPEECH_VULKAN
+   AND TARGET ggml-vulkan
+   AND TARGET SPIRV-Headers::SPIRV-Headers)
+  target_link_libraries(ggml-vulkan PRIVATE SPIRV-Headers::SPIRV-Headers)
+endif()
+
+# OBS deps ship older Vulkan headers; put the SDK headers first so ggml-vulkan
+# sees the full API.
+if(ADVSS_SPEECH_VULKAN
+   AND TARGET ggml-vulkan
+   AND DEFINED ENV{VULKAN_SDK})
+  if(WIN32)
+    set(_vk_sdk_include "$ENV{VULKAN_SDK}/Include")
+  else()
+    set(_vk_sdk_include "$ENV{VULKAN_SDK}/include")
+  endif()
+  if(EXISTS "${_vk_sdk_include}")
+    target_include_directories(ggml-vulkan BEFORE PRIVATE "${_vk_sdk_include}")
+  endif()
+endif()
+
+# --- End of section ---
+
+add_library(${PROJECT_NAME} MODULE)
+
+target_sources(
+  ${PROJECT_NAME} PRIVATE macro-condition-speech.cpp macro-condition-speech.hpp
+                          speech-recognizer.cpp speech-recognizer.hpp)
+
+setup_advss_plugin(${PROJECT_NAME})
+set_target_properties(${PROJECT_NAME} PROPERTIES PREFIX "")
+
+target_include_directories(${PROJECT_NAME} PRIVATE "${WHISPER_DIR}")
+target_link_libraries(${PROJECT_NAME} PRIVATE whisper)
+
+install_advss_plugin(${PROJECT_NAME})

plugins/speech/macro-condition-speech.cpp

@@ -0,0 +1,660 @@
+#include "macro-condition-speech.hpp"
+
+#include "layout-helpers.hpp"
+#include "macro-helpers.hpp"
+#include "selection-helpers.hpp"
+
+#include <obs-module.h>
+
+#include <QHBoxLayout>
+#include <QLabel>
+#include <QVBoxLayout>
+
+namespace advss {
+
+static int defaultNThreads()
+{
+	return (int)std::min(
+		4u, std::max(1u, std::thread::hardware_concurrency() / 4));
+}
+
+static std::string getDefaultModelPath()
+{
+	return std::string(obs_get_module_data_path(obs_current_module())) +
+	       "/res/speech/ggml-tiny-q8_0.bin";
+}
+
+static QStringList getAudioSourcesList()
+{
+	auto sources = GetAudioSourceNames();
+	sources.sort();
+	return sources;
+}
+
+const std::string MacroConditionSpeech::id = "speech";
+
+bool MacroConditionSpeech::_registered = MacroConditionFactory::Register(
+	MacroConditionSpeech::id,
+	{MacroConditionSpeech::Create, MacroConditionSpeechEdit::Create,
+	 "AdvSceneSwitcher.condition.speech"});
+
+MacroConditionSpeech::MacroConditionSpeech(Macro *m)
+	: MacroCondition(m),
+	  _modelPath(getDefaultModelPath()),
+	  _nThreads(defaultNThreads()),
+	  _language("auto"),
+	  _messageBuffer(_recognizer.RegisterClient())
+{
+	_recognizer.SetNThreads(defaultNThreads());
+	_recognizer.SetLanguage("auto");
+	_recognizer.SetTranslate(false);
+	_recognizer.SetVadEnergyThreshold(1e-4f);
+	_recognizer.SetSuppressNonSpeechTokens(true);
+	_recognizer.SetNoContext(true);
+}
+
+MacroConditionSpeech::~MacroConditionSpeech()
+{
+	if (_rebuildThread.joinable()) {
+		_rebuildThread.join();
+	}
+}
+
+void MacroConditionSpeech::SetCondition(Condition c)
+{
+	_condition = c;
+	SetupTempVars();
+}
+
+void MacroConditionSpeech::SetModelPath(const std::string &path)
+{
+	_modelPath = path;
+	RebuildRecognizer();
+}
+
+void MacroConditionSpeech::SetBufferDuration(const DoubleVariable &value)
+{
+	_bufferDuration = value;
+	_recognizer.SetBufferDuration((double)_bufferDuration);
+}
+
+void MacroConditionSpeech::SetNThreads(const IntVariable &value)
+{
+	_nThreads = value;
+	_recognizer.SetNThreads((int)_nThreads);
+}
+
+void MacroConditionSpeech::SetLanguage(const std::string &lang)
+{
+	_language = lang;
+	_recognizer.SetLanguage(lang);
+}
+
+void MacroConditionSpeech::SetTranslate(bool translate)
+{
+	_translate = translate;
+	_recognizer.SetTranslate(translate);
+}
+
+void MacroConditionSpeech::SetVadEnergyThreshold(const DoubleVariable &value)
+{
+	_vadEnergyThreshold = value;
+	_recognizer.SetVadEnergyThreshold((float)(double)_vadEnergyThreshold);
+}
+
+void MacroConditionSpeech::SetSuppressNonSpeechTokens(bool suppress)
+{
+	_suppressNonSpeechTokens = suppress;
+	_recognizer.SetSuppressNonSpeechTokens(suppress);
+}
+
+void MacroConditionSpeech::SetNoContext(bool noContext)
+{
+	_noContext = noContext;
+	_recognizer.SetNoContext(noContext);
+}
+
+void MacroConditionSpeech::SetListenWhenMuted(bool listen)
+{
+	_listenWhenMuted = listen;
+	_recognizer.SetListenWhenMuted(listen);
+}
+
+void MacroConditionSpeech::SetUseGpu(bool useGpu)
+{
+	_useGpu = useGpu;
+	_recognizer.SetUseGpu(useGpu);
+	RebuildRecognizer();
+}
+
+void MacroConditionSpeech::RebuildRecognizer()
+{
+	_recognizer.StopCapture();
+
+	if (_rebuildThread.joinable()) {
+		_rebuildThread.join();
+	}
+
+	const std::string path = _modelPath;
+	if (path.empty()) {
+		return;
+	}
+
+	OBSWeakSource weakSource = _source.GetSource();
+	_messageBuffer = _recognizer.RegisterClient();
+
+	_rebuildThread = std::thread([this, path, weakSource]() {
+		if (!_recognizer.LoadModel(path)) {
+			return;
+		}
+		OBSSource source = OBSGetStrongRef(weakSource);
+		if (!source) {
+			return;
+		}
+		_recognizer.StartCapture(source);
+	});
+}
+
+bool MacroConditionSpeech::CheckCondition()
+{
+	std::string lastTranscript;
+	bool anyReceived = false;
+
+	while (!_messageBuffer->Empty()) {
+		auto msg = _messageBuffer->ConsumeMessage();
+		if (!msg) {
+			continue;
+		}
+		lastTranscript = *msg;
+		anyReceived = true;
+	}
+
+	if (anyReceived) {
+		SetTempVarValue("speech", lastTranscript);
+	}
+
+	switch (_condition) {
+	case Condition::ANY:
+		if (anyReceived) {
+			return true;
+		}
+		return false;
+
+	case Condition::CONTAINS: {
+		if (!anyReceived) {
+			return false;
+		}
+		const std::string phrase = _phrase;
+		const QRegularExpression re(
+			"\\b" +
+				QRegularExpression::escape(
+					QString::fromStdString(phrase)) +
+				"\\b",
+			QRegularExpression::CaseInsensitiveOption);
+		if (re.match(QString::fromStdString(lastTranscript)).hasMatch()) {
+			return true;
+		}
+		return false;
+	}
+
+	case Condition::MATCHES:
+		if (!anyReceived) {
+			return false;
+		}
+		if (_regex.Enabled() &&
+		    _regex.Matches(lastTranscript, _phrase)) {
+			return true;
+		}
+		return false;
+
+	default:
+		break;
+	}
+
+	return false;
+}
+
+void MacroConditionSpeech::SetupTempVars()
+{
+	MacroCondition::SetupTempVars();
+	AddTempvar(
+		"speech",
+		obs_module_text("AdvSceneSwitcher.tempVar.speech.speech"),
+		obs_module_text(
+			"AdvSceneSwitcher.tempVar.speech.speech.description"));
+}
+
+std::string MacroConditionSpeech::GetShortDesc() const
+{
+	return _source.ToString();
+}
+
+bool MacroConditionSpeech::Save(obs_data_t *obj) const
+{
+	MacroCondition::Save(obj);
+	_source.Save(obj, "source");
+	obs_data_set_int(obj, "condition", static_cast<int>(_condition));
+	_phrase.Save(obj, "phrase");
+	_regex.Save(obj);
+	_modelPath.Save(obj, "modelPath");
+	_bufferDuration.Save(obj, "bufferDuration");
+	_nThreads.Save(obj, "nThreads");
+	_language.Save(obj, "language");
+	obs_data_set_bool(obj, "translate", _translate);
+	_vadEnergyThreshold.Save(obj, "vadEnergyThreshold");
+	obs_data_set_bool(obj, "suppressNonSpeechTokens",
+			  _suppressNonSpeechTokens);
+	obs_data_set_bool(obj, "noContext", _noContext);
+	obs_data_set_bool(obj, "listenWhenMuted", _listenWhenMuted);
+	obs_data_set_bool(obj, "useGpu", _useGpu);
+	return true;
+}
+
+bool MacroConditionSpeech::Load(obs_data_t *obj)
+{
+	MacroCondition::Load(obj);
+	_source.Load(obj, "source");
+	SetCondition(
+		static_cast<Condition>(obs_data_get_int(obj, "condition")));
+	_phrase.Load(obj, "phrase");
+	_regex.Load(obj);
+	_modelPath.Load(obj, "modelPath");
+	_bufferDuration.Load(obj, "bufferDuration");
+	_nThreads.Load(obj, "nThreads");
+	_language.Load(obj, "language");
+	_translate = obs_data_get_bool(obj, "translate");
+	_vadEnergyThreshold.Load(obj, "vadEnergyThreshold");
+	_suppressNonSpeechTokens =
+		obs_data_get_bool(obj, "suppressNonSpeechTokens");
+	_noContext = obs_data_get_bool(obj, "noContext");
+	_listenWhenMuted = obs_data_get_bool(obj, "listenWhenMuted");
+	_useGpu = obs_data_get_bool(obj, "useGpu");
+	_recognizer.SetNThreads((int)_nThreads);
+	_recognizer.SetLanguage(std::string(_language));
+	_recognizer.SetTranslate(_translate);
+	_recognizer.SetVadEnergyThreshold((float)(double)_vadEnergyThreshold);
+	_recognizer.SetSuppressNonSpeechTokens(_suppressNonSpeechTokens);
+	_recognizer.SetNoContext(_noContext);
+	_recognizer.SetListenWhenMuted(_listenWhenMuted);
+	_recognizer.SetUseGpu(_useGpu);
+	RebuildRecognizer();
+	return true;
+}
+
+static void populateConditionSelection(QComboBox *list)
+{
+	static const std::map<MacroConditionSpeech::Condition, std::string>
+		conditionTypes = {
+			{MacroConditionSpeech::Condition::ANY,
+			 "AdvSceneSwitcher.condition.speech.condition.any"},
+			{MacroConditionSpeech::Condition::CONTAINS,
+			 "AdvSceneSwitcher.condition.speech.condition.contains"},
+			{MacroConditionSpeech::Condition::MATCHES,
+			 "AdvSceneSwitcher.condition.speech.condition.matches"},
+		};
+
+	for (const auto &[cond, name] : conditionTypes) {
+		list->addItem(obs_module_text(name.c_str()),
+			      static_cast<int>(cond));
+	}
+}
+
+MacroConditionSpeechEdit::MacroConditionSpeechEdit(
+	QWidget *parent, std::shared_ptr<MacroConditionSpeech> entryData)
+	: QWidget(parent),
+	  _source(new SourceSelectionWidget(this, getAudioSourcesList, true)),
+	  _conditions(new QComboBox(this)),
+	  _phrase(new VariableLineEdit(this)),
+	  _regex(new RegexConfigWidget(parent)),
+	  _modelPath(new FileSelection(
+		  FileSelection::Type::READ, this,
+		  obs_module_text(
+			  "AdvSceneSwitcher.condition.speech.browse.title"))),
+	  _modelHelp(new HelpIcon(
+		  obs_module_text(
+			  "AdvSceneSwitcher.condition.speech.model.help"),
+		  this)),
+	  _bufferDuration(new VariableDoubleSpinBox(this)),
+	  _bufferHelp(new HelpIcon(
+		  obs_module_text(
+			  "AdvSceneSwitcher.condition.speech.buffer.help"),
+		  this)),
+	  _advancedSection(new Section(300, this)),
+	  _nThreads(new VariableSpinBox(this)),
+	  _language(new VariableLineEdit(this)),
+	  _languageHelp(new HelpIcon(
+		  obs_module_text(
+			  "AdvSceneSwitcher.condition.speech.advanced.language.help"),
+		  this)),
+	  _translate(new QCheckBox(
+		  obs_module_text(
+			  "AdvSceneSwitcher.condition.speech.advanced.translate"),
+		  this)),
+	  _translateHelp(new HelpIcon(
+		  obs_module_text(
+			  "AdvSceneSwitcher.condition.speech.advanced.translate.help"),
+		  this)),
+	  _vadEnergyThreshold(new VariableDoubleSpinBox(this)),
+	  _vadHelp(new HelpIcon(
+		  obs_module_text(
+			  "AdvSceneSwitcher.condition.speech.advanced.vad.help"),
+		  this)),
+	  _suppressNonSpeechTokens(new QCheckBox(
+		  obs_module_text(
+			  "AdvSceneSwitcher.condition.speech.advanced.suppress"),
+		  this)),
+	  _suppressHelp(new HelpIcon(
+		  obs_module_text(
+			  "AdvSceneSwitcher.condition.speech.advanced.suppress.help"),
+		  this)),
+	  _noContext(new QCheckBox(
+		  obs_module_text(
+			  "AdvSceneSwitcher.condition.speech.advanced.noContext"),
+		  this)),
+	  _noContextHelp(new HelpIcon(
+		  obs_module_text(
+			  "AdvSceneSwitcher.condition.speech.advanced.noContext.help"),
+		  this)),
+	  _listenWhenMuted(new QCheckBox(
+		  obs_module_text(
+			  "AdvSceneSwitcher.condition.speech.advanced.listenWhenMuted"),
+		  this)),
+	  _useGpu(new QCheckBox(
+		  obs_module_text(
+			  "AdvSceneSwitcher.condition.speech.advanced.useGpu"),
+		  this))
+{
+	populateConditionSelection(_conditions);
+
+	_bufferDuration->setMinimum(1.0);
+	_bufferDuration->setMaximum(30.0);
+	_bufferDuration->SpinBox()->setSingleStep(0.5);
+	_bufferDuration->setSuffix(" s");
+
+	_nThreads->setMinimum(1);
+	_nThreads->setMaximum(32);
+
+	_vadEnergyThreshold->setMinimum(0.0);
+	_vadEnergyThreshold->setMaximum(1.0);
+	_vadEnergyThreshold->SpinBox()->setSingleStep(1e-5);
+	_vadEnergyThreshold->SpinBox()->setDecimals(6);
+
+	QWidget::connect(_source,
+			 SIGNAL(SourceChanged(const SourceSelection &)), this,
+			 SLOT(SourceChanged(const SourceSelection &)));
+	QWidget::connect(_conditions, SIGNAL(currentIndexChanged(int)), this,
+			 SLOT(ConditionChanged(int)));
+	QWidget::connect(_phrase, SIGNAL(editingFinished()), this,
+			 SLOT(PhraseChanged()));
+	QWidget::connect(_regex,
+			 SIGNAL(RegexConfigChanged(const RegexConfig &)), this,
+			 SLOT(RegexChanged(const RegexConfig &)));
+	QWidget::connect(_modelPath, SIGNAL(PathChanged(const QString &)), this,
+			 SLOT(ModelPathChanged(const QString &)));
+	QWidget::connect(
+		_bufferDuration,
+		SIGNAL(NumberVariableChanged(const NumberVariable<double> &)),
+		this,
+		SLOT(BufferDurationChanged(const NumberVariable<double> &)));
+	QWidget::connect(
+		_nThreads,
+		SIGNAL(NumberVariableChanged(const NumberVariable<int> &)),
+		this, SLOT(NThreadsChanged(const NumberVariable<int> &)));
+	QWidget::connect(_language, SIGNAL(editingFinished()), this,
+			 SLOT(LanguageChanged()));
+	QWidget::connect(_translate, SIGNAL(stateChanged(int)), this,
+			 SLOT(TranslateChanged(int)));
+	QWidget::connect(
+		_vadEnergyThreshold,
+		SIGNAL(NumberVariableChanged(const NumberVariable<double> &)),
+		this,
+		SLOT(VadEnergyThresholdChanged(const NumberVariable<double> &)));
+	QWidget::connect(_suppressNonSpeechTokens, SIGNAL(stateChanged(int)),
+			 this, SLOT(SuppressNonSpeechTokensChanged(int)));
+	QWidget::connect(_noContext, SIGNAL(stateChanged(int)), this,
+			 SLOT(NoContextChanged(int)));
+	QWidget::connect(_listenWhenMuted, SIGNAL(stateChanged(int)), this,
+			 SLOT(ListenWhenMutedChanged(int)));
+	QWidget::connect(_useGpu, SIGNAL(stateChanged(int)), this,
+			 SLOT(UseGpuChanged(int)));
+
+	_condSourceLayout = new QHBoxLayout;
+
+	_phraseLayout = new QHBoxLayout;
+	PlaceWidgets(obs_module_text(
+			     "AdvSceneSwitcher.condition.speech.layout.phrase"),
+		     _phraseLayout,
+		     {{"{{phrase}}", _phrase}, {"{{regex}}", _regex}}, false);
+
+	auto *modelLayout = new QHBoxLayout;
+	PlaceWidgets(obs_module_text(
+			     "AdvSceneSwitcher.condition.speech.layout.model"),
+		     modelLayout,
+		     {{"{{modelPath}}", _modelPath}, {"{{help}}", _modelHelp}},
+		     false);
+
+	auto *bufferLayout = new QHBoxLayout;
+	PlaceWidgets(obs_module_text(
+			     "AdvSceneSwitcher.condition.speech.layout.buffer"),
+		     bufferLayout,
+		     {{"{{bufferDuration}}", _bufferDuration},
+		      {"{{help}}", _bufferHelp}});
+
+	auto *threadsLayout = new QHBoxLayout;
+	PlaceWidgets(
+		obs_module_text(
+			"AdvSceneSwitcher.condition.speech.layout.advanced.threads"),
+		threadsLayout, {{"{{threads}}", _nThreads}});
+
+	auto *languageLayout = new QHBoxLayout;
+	PlaceWidgets(
+		obs_module_text(
+			"AdvSceneSwitcher.condition.speech.layout.advanced.language"),
+		languageLayout,
+		{{"{{language}}", _language}, {"{{help}}", _languageHelp}});
+
+	auto *translateLayout = new QHBoxLayout;
+	PlaceWidgets(
+		obs_module_text(
+			"AdvSceneSwitcher.condition.speech.layout.advanced.translate"),
+		translateLayout,
+		{{"{{translate}}", _translate}, {"{{help}}", _translateHelp}});
+
+	auto *vadLayout = new QHBoxLayout;
+	PlaceWidgets(
+		obs_module_text(
+			"AdvSceneSwitcher.condition.speech.layout.advanced.vad"),
+		vadLayout,
+		{{"{{vad}}", _vadEnergyThreshold}, {"{{help}}", _vadHelp}});
+
+	auto *suppressLayout = new QHBoxLayout;
+	PlaceWidgets(
+		obs_module_text(
+			"AdvSceneSwitcher.condition.speech.layout.advanced.suppress"),
+		suppressLayout,
+		{{"{{suppress}}", _suppressNonSpeechTokens},
+		 {"{{help}}", _suppressHelp}});
+
+	auto *noContextLayout = new QHBoxLayout;
+	PlaceWidgets(
+		obs_module_text(
+			"AdvSceneSwitcher.condition.speech.layout.advanced.noContext"),
+		noContextLayout,
+		{{"{{noContext}}", _noContext}, {"{{help}}", _noContextHelp}});
+
+	auto *advancedContent = new QWidget(this);
+	auto *advancedLayout = new QVBoxLayout;
+	advancedLayout->addLayout(threadsLayout);
+	advancedLayout->addLayout(languageLayout);
+	advancedLayout->addLayout(translateLayout);
+	advancedLayout->addLayout(vadLayout);
+	advancedLayout->addLayout(suppressLayout);
+	advancedLayout->addLayout(noContextLayout);
+	advancedLayout->addWidget(_listenWhenMuted);
+	advancedLayout->addWidget(_useGpu);
+	advancedContent->setLayout(advancedLayout);
+
+	_advancedSection->AddHeaderWidget(new QLabel(
+		obs_module_text("AdvSceneSwitcher.condition.speech.advanced"),
+		this));
+	_advancedSection->SetContent(advancedContent, true);
+
+	auto *mainLayout = new QVBoxLayout;
+	mainLayout->addLayout(_condSourceLayout);
+	mainLayout->addLayout(_phraseLayout);
+	mainLayout->addLayout(modelLayout);
+	mainLayout->addLayout(bufferLayout);
+	mainLayout->addWidget(_advancedSection);
+	setLayout(mainLayout);
+
+	_entryData = entryData;
+	UpdateEntryData();
+	_loading = false;
+}
+
+void MacroConditionSpeechEdit::UpdateEntryData()
+{
+	if (!_entryData) {
+		return;
+	}
+
+	_source->SetSource(_entryData->_source);
+	_conditions->setCurrentIndex(
+		static_cast<int>(_entryData->GetCondition()));
+	_phrase->setText(QString::fromStdString(_entryData->_phrase));
+	_regex->SetRegexConfig(_entryData->_regex);
+	_modelPath->SetPath(_entryData->GetModelPath());
+	_bufferDuration->SetValue(_entryData->GetBufferDuration());
+	_nThreads->SetValue(_entryData->GetNThreads());
+	_language->setText(QString::fromStdString(_entryData->GetLanguage()));
+	_translate->setChecked(_entryData->GetTranslate());
+	_vadEnergyThreshold->SetValue(_entryData->GetVadEnergyThreshold());
+	_suppressNonSpeechTokens->setChecked(
+		_entryData->GetSuppressNonSpeechTokens());
+	_noContext->setChecked(_entryData->GetNoContext());
+	_listenWhenMuted->setChecked(_entryData->GetListenWhenMuted());
+	_useGpu->setChecked(_entryData->GetUseGpu());
+	SetWidgetVisibility();
+}
+
+void MacroConditionSpeechEdit::SourceChanged(const SourceSelection &source)
+{
+	GUARD_LOADING_AND_LOCK();
+	_entryData->_source = source;
+	_entryData->RebuildRecognizer();
+	emit HeaderInfoChanged(
+		QString::fromStdString(_entryData->GetShortDesc()));
+}
+
+void MacroConditionSpeechEdit::ConditionChanged(int idx)
+{
+	{
+		GUARD_LOADING_AND_LOCK();
+		_entryData->SetCondition(
+			static_cast<MacroConditionSpeech::Condition>(
+				_conditions->itemData(idx).toInt()));
+	}
+	SetWidgetVisibility();
+}
+
+void MacroConditionSpeechEdit::PhraseChanged()
+{
+	GUARD_LOADING_AND_LOCK();
+	_entryData->_phrase = _phrase->text().toStdString();
+}
+
+void MacroConditionSpeechEdit::RegexChanged(const RegexConfig &conf)
+{
+	GUARD_LOADING_AND_LOCK();
+	_entryData->_regex = conf;
+}
+
+void MacroConditionSpeechEdit::ModelPathChanged(const QString &path)
+{
+	GUARD_LOADING_AND_LOCK();
+	_entryData->SetModelPath(path.toStdString());
+}
+
+void MacroConditionSpeechEdit::BufferDurationChanged(
+	const NumberVariable<double> &value)
+{
+	GUARD_LOADING_AND_LOCK();
+	_entryData->SetBufferDuration(value);
+}
+
+void MacroConditionSpeechEdit::NThreadsChanged(const NumberVariable<int> &value)
+{
+	GUARD_LOADING_AND_LOCK();
+	_entryData->SetNThreads(value);
+}
+
+void MacroConditionSpeechEdit::LanguageChanged()
+{
+	GUARD_LOADING_AND_LOCK();
+	_entryData->SetLanguage(_language->text().toStdString());
+}
+
+void MacroConditionSpeechEdit::TranslateChanged(int state)
+{
+	GUARD_LOADING_AND_LOCK();
+	_entryData->SetTranslate(state == Qt::Checked);
+}
+
+void MacroConditionSpeechEdit::VadEnergyThresholdChanged(
+	const NumberVariable<double> &value)
+{
+	GUARD_LOADING_AND_LOCK();
+	_entryData->SetVadEnergyThreshold(value);
+}
+
+void MacroConditionSpeechEdit::SuppressNonSpeechTokensChanged(int state)
+{
+	GUARD_LOADING_AND_LOCK();
+	_entryData->SetSuppressNonSpeechTokens(state == Qt::Checked);
+}
+
+void MacroConditionSpeechEdit::NoContextChanged(int state)
+{
+	GUARD_LOADING_AND_LOCK();
+	_entryData->SetNoContext(state == Qt::Checked);
+}
+
+void MacroConditionSpeechEdit::ListenWhenMutedChanged(int state)
+{
+	GUARD_LOADING_AND_LOCK();
+	_entryData->SetListenWhenMuted(state == Qt::Checked);
+}
+
+void MacroConditionSpeechEdit::UseGpuChanged(int state)
+{
+	GUARD_LOADING_AND_LOCK();
+	_entryData->SetUseGpu(state == Qt::Checked);
+}
+
+void MacroConditionSpeechEdit::SetWidgetVisibility()
+{
+	const auto condition = _entryData->GetCondition();
+	const bool hasPhrase = condition !=
+			       MacroConditionSpeech::Condition::ANY;
+
+	_condSourceLayout->removeWidget(_conditions);
+	_condSourceLayout->removeWidget(_source);
+	ClearLayout(_condSourceLayout);
+	const char *layoutKey = "AdvSceneSwitcher.condition.speech.layout.any";
+	if (condition == MacroConditionSpeech::Condition::CONTAINS) {
+		layoutKey = "AdvSceneSwitcher.condition.speech.layout.contains";
+	} else if (condition == MacroConditionSpeech::Condition::MATCHES) {
+		layoutKey = "AdvSceneSwitcher.condition.speech.layout.matches";
+	}
+	PlaceWidgets(obs_module_text(layoutKey), _condSourceLayout,
+		     {{"{{conditions}}", _conditions},
+		      {"{{source}}", _source}});
+
+	SetLayoutVisible(_phraseLayout, hasPhrase);
+	_regex->setVisible(condition ==
+			   MacroConditionSpeech::Condition::MATCHES);
+	adjustSize();
+	updateGeometry();
+}
+
+} // namespace advss

plugins/speech/macro-condition-speech.hpp

@@ -0,0 +1,182 @@
+#pragma once
+#include "file-selection.hpp"
+#include "help-icon.hpp"
+#include "macro-condition-edit.hpp"
+#include "regex-config.hpp"
+#include "section.hpp"
+#include "source-selection.hpp"
+#include "speech-recognizer.hpp"
+#include "variable-line-edit.hpp"
+#include "variable-number.hpp"
+#include "variable-spinbox.hpp"
+#include "variable-string.hpp"
+
+#include <QCheckBox>
+#include <QComboBox>
+#include <QHBoxLayout>
+#include <QWidget>
+
+#include <thread>
+
+namespace advss {
+
+class MacroConditionSpeech : public MacroCondition {
+public:
+	MacroConditionSpeech(Macro *m);
+	~MacroConditionSpeech();
+
+	bool CheckCondition() override;
+	bool Save(obs_data_t *obj) const override;
+	bool Load(obs_data_t *obj) override;
+	std::string GetShortDesc() const override;
+	std::string GetId() const override { return id; }
+
+	static std::shared_ptr<MacroCondition> Create(Macro *m)
+	{
+		return std::make_shared<MacroConditionSpeech>(m);
+	}
+
+	enum class Condition {
+		ANY,
+		CONTAINS, // Just a more user friendly variant of "matches"
+		MATCHES,
+	};
+
+	void SetCondition(Condition c);
+	Condition GetCondition() const { return _condition; }
+
+	void SetModelPath(const std::string &path);
+	const StringVariable &GetModelPath() const { return _modelPath; }
+
+	void SetBufferDuration(const DoubleVariable &value);
+	DoubleVariable GetBufferDuration() const { return _bufferDuration; }
+
+	void SetNThreads(const IntVariable &value);
+	IntVariable GetNThreads() const { return _nThreads; }
+
+	void SetLanguage(const std::string &lang);
+	const StringVariable &GetLanguage() const { return _language; }
+
+	void SetTranslate(bool translate);
+	bool GetTranslate() const { return _translate; }
+
+	void SetVadEnergyThreshold(const DoubleVariable &value);
+	DoubleVariable GetVadEnergyThreshold() const
+	{
+		return _vadEnergyThreshold;
+	}
+
+	void SetSuppressNonSpeechTokens(bool suppress);
+	bool GetSuppressNonSpeechTokens() const
+	{
+		return _suppressNonSpeechTokens;
+	}
+
+	void SetNoContext(bool noContext);
+	bool GetNoContext() const { return _noContext; }
+
+	void SetListenWhenMuted(bool listen);
+	bool GetListenWhenMuted() const { return _listenWhenMuted; }
+
+	void SetUseGpu(bool useGpu);
+	bool GetUseGpu() const { return _useGpu; }
+
+	SourceSelection _source;
+	StringVariable _phrase = "";
+	RegexConfig _regex;
+
+	void RebuildRecognizer();
+
+private:
+	void SetupTempVars() override;
+
+	Condition _condition = Condition::ANY;
+	StringVariable _modelPath;
+	DoubleVariable _bufferDuration = 5.0;
+	IntVariable _nThreads;
+	StringVariable _language;
+	bool _translate = false;
+	DoubleVariable _vadEnergyThreshold = 1e-4;
+	bool _suppressNonSpeechTokens = true;
+	bool _noContext = true;
+	bool _listenWhenMuted = false;
+	bool _useGpu = true;
+
+	SpeechRecognizer _recognizer;
+	std::shared_ptr<MessageBuffer<std::string>> _messageBuffer;
+	std::thread _rebuildThread;
+
+	static bool _registered;
+	static const std::string id;
+};
+
+class MacroConditionSpeechEdit : public QWidget {
+	Q_OBJECT
+
+public:
+	MacroConditionSpeechEdit(
+		QWidget *parent,
+		std::shared_ptr<MacroConditionSpeech> entryData = nullptr);
+	void UpdateEntryData();
+
+	static QWidget *Create(QWidget *parent,
+			       std::shared_ptr<MacroCondition> cond)
+	{
+		return new MacroConditionSpeechEdit(
+			parent,
+			std::dynamic_pointer_cast<MacroConditionSpeech>(cond));
+	}
+
+private slots:
+	void SourceChanged(const SourceSelection &);
+	void ConditionChanged(int);
+	void PhraseChanged();
+	void RegexChanged(const RegexConfig &);
+	void ModelPathChanged(const QString &);
+	void BufferDurationChanged(const NumberVariable<double> &);
+	void NThreadsChanged(const NumberVariable<int> &);
+	void LanguageChanged();
+	void TranslateChanged(int);
+	void VadEnergyThresholdChanged(const NumberVariable<double> &);
+	void SuppressNonSpeechTokensChanged(int);
+	void NoContextChanged(int);
+	void ListenWhenMutedChanged(int);
+	void UseGpuChanged(int);
+
+signals:
+	void HeaderInfoChanged(const QString &);
+
+private:
+	void SetWidgetVisibility();
+
+	SourceSelectionWidget *_source;
+	QComboBox *_conditions;
+	VariableLineEdit *_phrase;
+	RegexConfigWidget *_regex;
+	FileSelection *_modelPath;
+	HelpIcon *_modelHelp;
+	VariableDoubleSpinBox *_bufferDuration;
+	HelpIcon *_bufferHelp;
+	Section *_advancedSection;
+	VariableSpinBox *_nThreads;
+	VariableLineEdit *_language;
+	HelpIcon *_languageHelp;
+	QCheckBox *_translate;
+	HelpIcon *_translateHelp;
+	VariableDoubleSpinBox *_vadEnergyThreshold;
+	HelpIcon *_vadHelp;
+	QCheckBox *_suppressNonSpeechTokens;
+	HelpIcon *_suppressHelp;
+	QCheckBox *_noContext;
+	HelpIcon *_noContextHelp;
+	QCheckBox *_listenWhenMuted;
+	QCheckBox *_useGpu;
+
+	QHBoxLayout *_condSourceLayout;
+	QHBoxLayout *_phraseLayout;
+
+	std::shared_ptr<MacroConditionSpeech> _entryData;
+	bool _loading = true;
+};
+
+} // namespace advss

plugins/speech/speech-recognizer.cpp

@@ -0,0 +1,402 @@
+#include "speech-recognizer.hpp"
+
+#include "log-helper.hpp"
+#include "plugin-state-helpers.hpp"
+
+#include <obs.h>
+#include <media-io/audio-resampler.h>
+#include <util/platform.h>
+
+#include <whisper.h>
+
+namespace advss {
+
+// Ignore configured log level until after loading is complete to ensure we
+// catch the initial whisper configuration logs
+static bool ignoreLogFilter = true;
+
+static bool setup()
+{
+	AddFinishedLoadingStep([]() { ignoreLogFilter = false; });
+	return true;
+}
+const bool _ = setup();
+
+static void whisperLogCallback(ggml_log_level level, const char *text, void *)
+{
+	if (!text || *text == '\0') {
+		return;
+	}
+
+	int obsLevel = LOG_INFO;
+	if (level == GGML_LOG_LEVEL_WARN) {
+		obsLevel = LOG_WARNING;
+	} else if (level == GGML_LOG_LEVEL_ERROR) {
+		obsLevel = LOG_ERROR;
+	}
+
+	std::string msg(text);
+	if (!msg.empty() && msg.back() == '\n') {
+		msg.pop_back();
+	}
+
+	if (msg.empty()) {
+		return;
+	}
+
+	if (ignoreLogFilter) {
+		blog(obsLevel, "[speech] %s", msg.c_str());
+	} else {
+		vblog(obsLevel, "[speech] %s", msg.c_str());
+	}
+}
+
+static constexpr int whisperSampleRate = 16000;
+
+// How often to evaluate VAD and potentially trigger inference
+static constexpr double stepDurationSeconds = 1.0;
+
+// Audio kept from the previous inference run to provide word-boundary context
+// for the next run.
+// I guess repeating something is better than potentially missing stuff.
+static constexpr double keepDurationSeconds = 0.2;
+
+SpeechRecognizer::SpeechRecognizer()
+{
+	_inferenceThread = std::thread(&SpeechRecognizer::InferenceLoop, this);
+}
+
+SpeechRecognizer::~SpeechRecognizer()
+{
+	StopCapture();
+
+	{
+		std::unique_lock<std::mutex> lock(_inferenceMutex);
+		_stopThread = true;
+		_bufferReady = true;
+	}
+	_inferenceCV.notify_one();
+	if (_inferenceThread.joinable()) {
+		_inferenceThread.join();
+	}
+
+	if (_ctx) {
+		whisper_free(_ctx);
+	}
+	if (_resampler) {
+		audio_resampler_destroy(
+			static_cast<audio_resampler_t *>(_resampler));
+	}
+}
+
+bool SpeechRecognizer::LoadModel(const std::string &modelPath)
+{
+	std::lock_guard<std::mutex> lock(_ctxMutex);
+
+	if (_ctx) {
+		whisper_free(_ctx);
+		_ctx = nullptr;
+	}
+
+	whisper_log_set(whisperLogCallback, nullptr);
+
+	whisper_context_params cparams = whisper_context_default_params();
+	cparams.use_gpu = _useGpu;
+	_ctx = whisper_init_from_file_with_params(modelPath.c_str(), cparams);
+	if (!_ctx) {
+		blog(LOG_WARNING, "failed to load whisper model: %s",
+		     modelPath.c_str());
+		return false;
+	}
+	return true;
+}
+
+bool SpeechRecognizer::StartCapture(obs_source_t *source)
+{
+	StopCapture();
+
+	if (!source) {
+		return false;
+	}
+
+	const audio_t *audio = obs_get_audio();
+	if (!audio) {
+		return false;
+	}
+	const struct audio_output_info *aoi = audio_output_get_info(audio);
+	_sourceSampleRate = (int)aoi->samples_per_sec;
+	_sourceChannelCount = (int)get_audio_channels(aoi->speakers);
+
+	if (_resampler) {
+		audio_resampler_destroy(
+			static_cast<audio_resampler_t *>(_resampler));
+		_resampler = nullptr;
+	}
+
+	struct resample_info srcInfo = {};
+	srcInfo.samples_per_sec = (uint32_t)_sourceSampleRate;
+	srcInfo.format = AUDIO_FORMAT_FLOAT_PLANAR;
+	srcInfo.speakers = aoi->speakers;
+
+	struct resample_info dstInfo = {};
+	dstInfo.samples_per_sec = whisperSampleRate;
+	dstInfo.format = AUDIO_FORMAT_FLOAT;
+	dstInfo.speakers = SPEAKERS_MONO;
+
+	_resampler = audio_resampler_create(&dstInfo, &srcInfo);
+	if (!_resampler) {
+		blog(LOG_WARNING,
+		     "failed to create audio resampler for speech condition");
+		return false;
+	}
+
+	_captureSource = obs_source_get_weak_source(source);
+	obs_source_add_audio_capture_callback(source, AudioCaptureCallback,
+					      this);
+	return true;
+}
+
+void SpeechRecognizer::StopCapture()
+{
+	OBSSource source = OBSGetStrongRef(_captureSource);
+	if (source) {
+		obs_source_remove_audio_capture_callback(
+			source, AudioCaptureCallback, this);
+	}
+	_captureSource = OBSWeakSource{};
+}
+
+void SpeechRecognizer::SetBufferDuration(double seconds)
+{
+	std::lock_guard<std::mutex> lock(_audioMutex);
+	_bufferDurationSeconds = seconds;
+	_audioBuffer.clear();
+	_framesSinceLastStep = 0;
+}
+
+void SpeechRecognizer::SetNThreads(int n)
+{
+	std::lock_guard<std::mutex> lock(_inferenceMutex);
+	_nThreads = std::max(1, n);
+}
+
+void SpeechRecognizer::SetLanguage(const std::string &lang)
+{
+	std::lock_guard<std::mutex> lock(_inferenceMutex);
+	_language = lang.empty() ? "auto" : lang;
+}
+
+void SpeechRecognizer::SetTranslate(bool translate)
+{
+	std::lock_guard<std::mutex> lock(_inferenceMutex);
+	_translate = translate;
+}
+
+void SpeechRecognizer::SetVadEnergyThreshold(float threshold)
+{
+	std::lock_guard<std::mutex> lock(_audioMutex);
+	_vadEnergyThreshold = threshold;
+}
+
+void SpeechRecognizer::SetSuppressNonSpeechTokens(bool suppress)
+{
+	std::lock_guard<std::mutex> lock(_inferenceMutex);
+	_suppressNonSpeechTokens = suppress;
+}
+
+void SpeechRecognizer::SetNoContext(bool noContext)
+{
+	std::lock_guard<std::mutex> lock(_inferenceMutex);
+	_noContext = noContext;
+}
+
+void SpeechRecognizer::SetListenWhenMuted(bool listen)
+{
+	_listenWhenMuted = listen;
+}
+
+void SpeechRecognizer::SetUseGpu(bool useGpu)
+{
+	std::lock_guard<std::mutex> lock(_ctxMutex);
+	_useGpu = useGpu;
+}
+
+std::shared_ptr<MessageBuffer<std::string>> SpeechRecognizer::RegisterClient()
+{
+	return _dispatcher.RegisterClient();
+}
+
+void SpeechRecognizer::AudioCaptureCallback(void *param, obs_source_t *,
+					    const struct audio_data *audio,
+					    bool muted)
+{
+	if (!audio || !audio->data[0]) {
+		return;
+	}
+	auto *self = static_cast<SpeechRecognizer *>(param);
+	if (muted && !self->_listenWhenMuted) {
+		return;
+	}
+	self->AppendResampledAudio(audio);
+}
+
+void SpeechRecognizer::AppendResampledAudio(const struct audio_data *audio)
+{
+	if (!_resampler) {
+		return;
+	}
+
+	uint8_t *resampledData[MAX_AV_PLANES] = {};
+	uint32_t outFrames = 0;
+	uint64_t tsOffset = 0;
+
+	bool ok = audio_resampler_resample(
+		static_cast<audio_resampler_t *>(_resampler), resampledData,
+		&outFrames, &tsOffset, (const uint8_t *const *)audio->data,
+		audio->frames);
+	if (!ok || outFrames == 0 || !resampledData[0]) {
+		return;
+	}
+
+	const float *samples =
+		reinterpret_cast<const float *>(resampledData[0]);
+
+	std::unique_lock<std::mutex> lock(_audioMutex);
+	_audioBuffer.insert(_audioBuffer.end(), samples, samples + outFrames);
+	_framesSinceLastStep += outFrames;
+
+	// Keep the rolling buffer capped at the configured context window.
+	const size_t maxFrames =
+		(size_t)(_bufferDurationSeconds * whisperSampleRate);
+	if (_audioBuffer.size() > maxFrames) {
+		_audioBuffer.erase(_audioBuffer.begin(),
+				   _audioBuffer.begin() +
+					   (_audioBuffer.size() - maxFrames));
+	}
+
+	// Only consider triggering inference once per step interval.
+	const size_t stepFrames =
+		(size_t)(stepDurationSeconds * whisperSampleRate);
+	if (_framesSinceLastStep < stepFrames) {
+		return;
+	}
+	_framesSinceLastStep = 0;
+
+	// VAD: measure energy over just the most recent step window so that a
+	// short utterance at the end of a longer silent buffer is not diluted.
+	const size_t vadWindow = std::min(_audioBuffer.size(), stepFrames);
+	const size_t vadStart = _audioBuffer.size() - vadWindow;
+	float energy = 0.0f;
+	for (size_t i = vadStart; i < _audioBuffer.size(); ++i) {
+		energy += _audioBuffer[i] * _audioBuffer[i];
+	}
+	energy /= (float)vadWindow;
+	const bool tooSilent = energy < _vadEnergyThreshold;
+	if (tooSilent) {
+		return;
+	}
+
+	{
+		std::unique_lock<std::mutex> infLock(_inferenceMutex);
+		if (_bufferReady) {
+			return;
+		}
+
+		_inferenceBuffer = _audioBuffer;
+		_bufferReady = true;
+		_inferenceCV.notify_one();
+	}
+
+	// Retain a short overlap so the next inference has word-boundary context.
+	const size_t keepFrames =
+		(size_t)(keepDurationSeconds * whisperSampleRate);
+	if (_audioBuffer.size() > keepFrames) {
+		_audioBuffer.erase(_audioBuffer.begin(),
+				   _audioBuffer.begin() +
+					   (_audioBuffer.size() - keepFrames));
+	}
+}
+
+void SpeechRecognizer::InferenceLoop()
+{
+	while (true) {
+		std::vector<float> buffer;
+		int nThreads;
+		std::string language;
+		bool translate;
+		bool suppressNonSpeechTokens;
+		bool noContext;
+
+		{
+			std::unique_lock<std::mutex> lock(_inferenceMutex);
+			_inferenceCV.wait(lock,
+					  [this] { return _bufferReady; });
+			_bufferReady = false;
+			if (_stopThread) {
+				break;
+			}
+			buffer = std::move(_inferenceBuffer);
+			nThreads = _nThreads;
+			language = _language;
+			translate = _translate;
+			suppressNonSpeechTokens = _suppressNonSpeechTokens;
+			noContext = _noContext;
+		}
+
+		if (buffer.empty()) {
+			continue;
+		}
+
+		std::lock_guard<std::mutex> ctxLock(_ctxMutex);
+
+		if (!_ctx) {
+			continue;
+		}
+
+		whisper_full_params params =
+			whisper_full_default_params(WHISPER_SAMPLING_GREEDY);
+		params.print_realtime = false;
+		params.print_progress = false;
+		params.print_timestamps = false;
+		params.print_special = false;
+		params.translate = translate;
+		params.language = language.c_str();
+		params.n_threads = nThreads;
+		params.single_segment = false;
+		params.suppress_nst = suppressNonSpeechTokens;
+		params.no_context = noContext;
+
+		// Limit the encoder to the actual audio length
+		params.audio_ctx =
+			std::min(1500, (int)((float)buffer.size() /
+					     (float)whisperSampleRate * 50.0f));
+
+		int rc = whisper_full(_ctx, params, buffer.data(),
+				      (int)buffer.size());
+		if (rc != 0) {
+			blog(LOG_WARNING, "whisper_full returned %d", rc);
+			continue;
+		}
+
+		std::string transcript;
+		const int nSegments = whisper_full_n_segments(_ctx);
+		for (int i = 0; i < nSegments; ++i) {
+			const char *text =
+				whisper_full_get_segment_text(_ctx, i);
+			if (text) {
+				transcript += text;
+			}
+		}
+
+		if (!transcript.empty()) {
+			const auto begin =
+				transcript.find_first_not_of(" \t\r\n");
+			if (begin != std::string::npos) {
+				transcript = transcript.substr(begin);
+			}
+			_dispatcher.DispatchMessage(transcript);
+		}
+	}
+}
+
+} // namespace advss

plugins/speech/speech-recognizer.hpp

@@ -0,0 +1,86 @@
+#pragma once
+#include "message-buffer.hpp"
+#include "message-dispatcher.hpp"
+
+#include <obs.hpp>
+
+#include <atomic>
+#include <condition_variable>
+#include <mutex>
+#include <string>
+#include <thread>
+#include <vector>
+
+struct whisper_context;
+struct audio_data;
+
+namespace advss {
+
+// Captures audio from one OBS source, resamples to 16 kHz mono, runs
+// whisper.cpp inference on a background thread, and dispatches the resulting
+// transcript text to registered MessageBuffers.
+class SpeechRecognizer {
+public:
+	SpeechRecognizer();
+	~SpeechRecognizer();
+
+	bool LoadModel(const std::string &modelPath);
+
+	bool StartCapture(obs_source_t *source);
+	void StopCapture();
+
+	void SetBufferDuration(double seconds);
+	void SetNThreads(int n);
+	void SetLanguage(const std::string &lang);
+	void SetTranslate(bool translate);
+	void SetVadEnergyThreshold(float threshold);
+	void SetSuppressNonSpeechTokens(bool suppress);
+	void SetNoContext(bool noContext);
+	void SetListenWhenMuted(bool listen);
+	void SetUseGpu(bool useGpu);
+
+	[[nodiscard]] std::shared_ptr<MessageBuffer<std::string>>
+	RegisterClient();
+
+private:
+	static void AudioCaptureCallback(void *param, obs_source_t *source,
+					 const struct audio_data *audio,
+					 bool muted);
+	void AppendResampledAudio(const struct audio_data *audio);
+	void InferenceLoop();
+
+	// Held during whisper_full and when freeing/replacing _ctx.
+	std::mutex _ctxMutex;
+	whisper_context *_ctx = nullptr;
+	// Stored as void* to avoid pulling <media-io/audio-resampler.h> into
+	// this header. Cast to audio_resampler_t* in the .cpp.
+	void *_resampler = nullptr;
+
+	std::vector<float> _audioBuffer;
+	std::mutex _audioMutex;
+	double _bufferDurationSeconds = 5.0;
+	float _vadEnergyThreshold = 1e-4f;
+	size_t _framesSinceLastStep = 0;
+
+	std::vector<float> _inferenceBuffer;
+	std::thread _inferenceThread;
+	std::atomic_bool _stopThread{false};
+	std::condition_variable _inferenceCV;
+	std::mutex _inferenceMutex;
+	bool _bufferReady = false;
+	int _nThreads = 4;
+	std::string _language = "auto";
+	bool _translate = false;
+	bool _suppressNonSpeechTokens = true;
+	bool _noContext = true;
+	bool _listenWhenMuted = false;
+	bool _useGpu = true;
+
+	OBSWeakSource _captureSource;
+	int _sourceSampleRate = 44100;
+	int _sourceChannelCount = 2;
+
+	MessageDispatcher<std::string> _dispatcher;
+};
+
+} // namespace advss