diff --git a/CMakeLists.txt b/CMakeLists.txt index 69b3aabe..df1e47c3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -576,7 +576,8 @@ set(_dep_license_files "openvr:openvr/LICENSE" "paho-mqtt:paho.mqtt.cpp/LICENSE" "tesseract:tesseract/LICENSE" - "websocketpp:websocketpp/COPYING") + "websocketpp:websocketpp/COPYING" + "whisper:whisper.cpp/LICENSE") if(DEB_INSTALL) if(NOT DATA_OUT_DIR) diff --git a/data/locale/en-US.ini b/data/locale/en-US.ini index e97fd76a..e5b5154c 100644 --- a/data/locale/en-US.ini +++ b/data/locale/en-US.ini @@ -840,6 +840,38 @@ AdvSceneSwitcher.condition.clipboard.condition.isImage="Clipboard contains an im AdvSceneSwitcher.condition.clipboard.condition.isURL="Clipboard contains an URL" AdvSceneSwitcher.condition.clipboard.condition.matches="Clipboard content matches" AdvSceneSwitcher.condition.clipboard.condition.entry="{{conditions}}{{regex}}{{urlInfo}}" +AdvSceneSwitcher.condition.speech="Speech Recognition (beta)" +AdvSceneSwitcher.condition.speech.condition.any="Any speech is detected for" +AdvSceneSwitcher.condition.speech.condition.contains="contains phrase" +AdvSceneSwitcher.condition.speech.condition.matches="matches" +AdvSceneSwitcher.condition.speech.layout.any="{{conditions}}{{source}}" +AdvSceneSwitcher.condition.speech.layout.contains="Transcript of{{source}}{{conditions}}:" +AdvSceneSwitcher.condition.speech.layout.matches="Transcript of{{source}}{{conditions}}:" +AdvSceneSwitcher.condition.speech.layout.phrase="{{phrase}}{{regex}}" +AdvSceneSwitcher.condition.speech.layout.model="Whisper model:{{modelPath}}{{help}}" +AdvSceneSwitcher.condition.speech.model.help="GGML model files can be downloaded from https://huggingface.co/ggerganov/whisper.cpp (e.g. ggml-base.bin).\nLarger models are more accurate but slower." +AdvSceneSwitcher.condition.speech.layout.buffer="Audio buffer:{{bufferDuration}}{{help}}" +AdvSceneSwitcher.condition.speech.browse="Browse..." +AdvSceneSwitcher.condition.speech.browse.title="Select Whisper model file" +AdvSceneSwitcher.condition.speech.browse.filter="GGML model files (*.bin);;All files (*)" +AdvSceneSwitcher.condition.speech.buffer.help="Longer buffer durations improve accuracy but increase latency." +AdvSceneSwitcher.condition.speech.advanced="Advanced" +AdvSceneSwitcher.condition.speech.layout.advanced.threads="Threads:{{threads}}" +AdvSceneSwitcher.condition.speech.layout.advanced.language="Language:{{language}}{{help}}" +AdvSceneSwitcher.condition.speech.advanced.language.help="Whisper language code (e.g. 'en', 'de', 'fr') or 'auto' to detect automatically." +AdvSceneSwitcher.condition.speech.layout.advanced.translate="{{translate}}{{help}}" +AdvSceneSwitcher.condition.speech.advanced.translate="Translate to English" +AdvSceneSwitcher.condition.speech.advanced.translate.help="Translate non-English speech to English before transcribing.\nUseful when the phrase or regex is written in English but the source speaks another language." +AdvSceneSwitcher.condition.speech.layout.advanced.vad="VAD energy threshold:{{vad}}{{help}}" +AdvSceneSwitcher.condition.speech.advanced.vad.help="Minimum RMS energy a buffer must have before running inference. Buffers below this level are treated as silence and skipped. Lower values are more sensitive; raise it if inference triggers on background noise." +AdvSceneSwitcher.condition.speech.layout.advanced.suppress="{{suppress}}{{help}}" +AdvSceneSwitcher.condition.speech.advanced.suppress="Suppress non-speech tokens" +AdvSceneSwitcher.condition.speech.advanced.suppress.help="Remove filler tokens such as [MUSIC] or (applause) that Whisper tends to insert when it detects non-speech sounds." +AdvSceneSwitcher.condition.speech.layout.advanced.noContext="{{noContext}}{{help}}" +AdvSceneSwitcher.condition.speech.advanced.noContext="No context" +AdvSceneSwitcher.condition.speech.advanced.noContext.help="Do not feed the previous transcription back as a prompt for the next buffer.\nPrevents repetition across buffer boundaries at the cost of slightly reduced coherence." +AdvSceneSwitcher.condition.speech.advanced.listenWhenMuted="Listen when source is muted" +AdvSceneSwitcher.condition.speech.advanced.useGpu="Use GPU" AdvSceneSwitcher.condition.folder="Folder watch" AdvSceneSwitcher.condition.folder.tooltip="This condition type will allow you to monitor the contents of a folder.\nNote that the monitoring will *not* recursively scan for changes in sub directories within directories of the selected folder!\nNote that if there are several changes during a short period of time, some of the changes might not emit this signal.\nHowever, the last change in the sequence of changes always will." AdvSceneSwitcher.condition.folder.condition.any="Any change happened" @@ -2460,6 +2492,9 @@ AdvSceneSwitcher.tempVar.streaming.serviceName.description="The name of the stre AdvSceneSwitcher.tempVar.clipboard.text="Clipboard text" AdvSceneSwitcher.tempVar.clipboard.text.description="The text contained in the clipboard.\nWill be empty if the clipboard does not contain text." +AdvSceneSwitcher.tempVar.speech.speech="Transcribed speech" +AdvSceneSwitcher.tempVar.speech.speech.description="The text transcribed from the last audio buffer. Only populated when the condition matched." + AdvSceneSwitcher.tempVar.file.content="File content" AdvSceneSwitcher.tempVar.file.date="File modification date" AdvSceneSwitcher.tempVar.file.basename="File basename" diff --git a/data/res/speech/ggml-tiny-q8_0.bin b/data/res/speech/ggml-tiny-q8_0.bin new file mode 100644 index 00000000..16534365 Binary files /dev/null and b/data/res/speech/ggml-tiny-q8_0.bin differ diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index bebdbaf8..863171c9 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -41,6 +41,7 @@ add_plugin(stream-deck) add_plugin(twitch) add_plugin(usb) add_plugin(video) +add_plugin(speech) # ---------------------------------------------------------------------------- # diff --git a/plugins/speech/CMakeLists.txt b/plugins/speech/CMakeLists.txt new file mode 100644 index 00000000..91197a62 --- /dev/null +++ b/plugins/speech/CMakeLists.txt @@ -0,0 +1,125 @@ +cmake_minimum_required(VERSION 3.14) +project(advanced-scene-switcher-speech) + +# --- Check requirements --- + +get_target_property(ADVSS_SOURCE_DIR advanced-scene-switcher-lib SOURCE_DIR) +set(WHISPER_DIR "${ADVSS_SOURCE_DIR}/deps/whisper.cpp") +if(NOT EXISTS "${WHISPER_DIR}/CMakeLists.txt") + message(WARNING "whisper.cpp directory \"${WHISPER_DIR}\" not found!\n" + "Speech condition will be disabled!\n\n" + "Clone whisper.cpp into: ${WHISPER_DIR}") + return() +endif() + +set(WHISPER_BUILD_TESTS + OFF + CACHE BOOL "" FORCE) +set(WHISPER_BUILD_EXAMPLES + OFF + CACHE BOOL "" FORCE) +set(BUILD_SHARED_LIBS + OFF + CACHE BOOL "" FORCE) + +# Vulkan GPU backend. Auto-enabled when VULKAN_SDK is set (e.g. in CI). +option(ADVSS_SPEECH_VULKAN "Use Vulkan GPU backend for speech recognition" OFF) + +if(NOT ADVSS_SPEECH_VULKAN AND DEFINED ENV{VULKAN_SDK}) + set(ADVSS_SPEECH_VULKAN ON) +endif() + +if(ADVSS_SPEECH_VULKAN) + set(GGML_VULKAN + ON + CACHE BOOL "" FORCE) +endif() + +# Suppress warnings-as-errors on all targets in a whisper.cpp source subtree. +function(_advss_whisper_suppress_werror dir) + get_property( + _subdirs + DIRECTORY "${dir}" + PROPERTY SUBDIRECTORIES) + foreach(_sub IN LISTS _subdirs) + _advss_whisper_suppress_werror("${_sub}") + endforeach() + get_property( + _targets + DIRECTORY "${dir}" + PROPERTY BUILDSYSTEM_TARGETS) + foreach(_target IN LISTS _targets) + get_target_property(_type ${_target} TYPE) + if(_type STREQUAL "INTERFACE_LIBRARY" OR _type STREQUAL "UTILITY") + continue() + endif() + get_target_property(_opts ${_target} COMPILE_OPTIONS) + if(_opts) + list(FILTER _opts EXCLUDE REGEX "^-Werror") + list(REMOVE_ITEM _opts /WX) + set_target_properties(${_target} PROPERTIES COMPILE_OPTIONS "${_opts}") + endif() + target_compile_options( + ${_target} + PRIVATE $<$:-Wno-error> + $<$:-Wno-error> + $<$:/WX-> $<$:/WX->) + endforeach() +endfunction() + +set(_advss_saved_werror ${CMAKE_COMPILE_WARNING_AS_ERROR}) +set(_advss_saved_pic ${CMAKE_POSITION_INDEPENDENT_CODE}) +set(CMAKE_COMPILE_WARNING_AS_ERROR OFF) +set(CMAKE_POSITION_INDEPENDENT_CODE ON) +if(NOT TARGET whisper) + add_subdirectory("${WHISPER_DIR}" "${CMAKE_BINARY_DIR}/whisper.cpp" + EXCLUDE_FROM_ALL) + _advss_whisper_suppress_werror("${WHISPER_DIR}") + # ggml-metal's .m files use manual retain/release; disable ARC on that target. + if(TARGET ggml-metal) + set_target_properties(ggml-metal + PROPERTIES XCODE_ATTRIBUTE_CLANG_ENABLE_OBJC_ARC NO) + target_compile_options(ggml-metal + PRIVATE $<$:-fno-objc-arc>) + endif() +endif() +set(CMAKE_COMPILE_WARNING_AS_ERROR ${_advss_saved_werror}) +set(CMAKE_POSITION_INDEPENDENT_CODE ${_advss_saved_pic}) + +# ggml-vulkan finds SPIRV-Headers but does not link it. +if(ADVSS_SPEECH_VULKAN + AND TARGET ggml-vulkan + AND TARGET SPIRV-Headers::SPIRV-Headers) + target_link_libraries(ggml-vulkan PRIVATE SPIRV-Headers::SPIRV-Headers) +endif() + +# OBS deps ship older Vulkan headers; put the SDK headers first so ggml-vulkan +# sees the full API. +if(ADVSS_SPEECH_VULKAN + AND TARGET ggml-vulkan + AND DEFINED ENV{VULKAN_SDK}) + if(WIN32) + set(_vk_sdk_include "$ENV{VULKAN_SDK}/Include") + else() + set(_vk_sdk_include "$ENV{VULKAN_SDK}/include") + endif() + if(EXISTS "${_vk_sdk_include}") + target_include_directories(ggml-vulkan BEFORE PRIVATE "${_vk_sdk_include}") + endif() +endif() + +# --- End of section --- + +add_library(${PROJECT_NAME} MODULE) + +target_sources( + ${PROJECT_NAME} PRIVATE macro-condition-speech.cpp macro-condition-speech.hpp + speech-recognizer.cpp speech-recognizer.hpp) + +setup_advss_plugin(${PROJECT_NAME}) +set_target_properties(${PROJECT_NAME} PROPERTIES PREFIX "") + +target_include_directories(${PROJECT_NAME} PRIVATE "${WHISPER_DIR}") +target_link_libraries(${PROJECT_NAME} PRIVATE whisper) + +install_advss_plugin(${PROJECT_NAME}) diff --git a/plugins/speech/macro-condition-speech.cpp b/plugins/speech/macro-condition-speech.cpp new file mode 100644 index 00000000..f15b6832 --- /dev/null +++ b/plugins/speech/macro-condition-speech.cpp @@ -0,0 +1,660 @@ +#include "macro-condition-speech.hpp" + +#include "layout-helpers.hpp" +#include "macro-helpers.hpp" +#include "selection-helpers.hpp" + +#include + +#include +#include +#include + +namespace advss { + +static int defaultNThreads() +{ + return (int)std::min( + 4u, std::max(1u, std::thread::hardware_concurrency() / 4)); +} + +static std::string getDefaultModelPath() +{ + return std::string(obs_get_module_data_path(obs_current_module())) + + "/res/speech/ggml-tiny-q8_0.bin"; +} + +static QStringList getAudioSourcesList() +{ + auto sources = GetAudioSourceNames(); + sources.sort(); + return sources; +} + +const std::string MacroConditionSpeech::id = "speech"; + +bool MacroConditionSpeech::_registered = MacroConditionFactory::Register( + MacroConditionSpeech::id, + {MacroConditionSpeech::Create, MacroConditionSpeechEdit::Create, + "AdvSceneSwitcher.condition.speech"}); + +MacroConditionSpeech::MacroConditionSpeech(Macro *m) + : MacroCondition(m), + _modelPath(getDefaultModelPath()), + _nThreads(defaultNThreads()), + _language("auto"), + _messageBuffer(_recognizer.RegisterClient()) +{ + _recognizer.SetNThreads(defaultNThreads()); + _recognizer.SetLanguage("auto"); + _recognizer.SetTranslate(false); + _recognizer.SetVadEnergyThreshold(1e-4f); + _recognizer.SetSuppressNonSpeechTokens(true); + _recognizer.SetNoContext(true); +} + +MacroConditionSpeech::~MacroConditionSpeech() +{ + if (_rebuildThread.joinable()) { + _rebuildThread.join(); + } +} + +void MacroConditionSpeech::SetCondition(Condition c) +{ + _condition = c; + SetupTempVars(); +} + +void MacroConditionSpeech::SetModelPath(const std::string &path) +{ + _modelPath = path; + RebuildRecognizer(); +} + +void MacroConditionSpeech::SetBufferDuration(const DoubleVariable &value) +{ + _bufferDuration = value; + _recognizer.SetBufferDuration((double)_bufferDuration); +} + +void MacroConditionSpeech::SetNThreads(const IntVariable &value) +{ + _nThreads = value; + _recognizer.SetNThreads((int)_nThreads); +} + +void MacroConditionSpeech::SetLanguage(const std::string &lang) +{ + _language = lang; + _recognizer.SetLanguage(lang); +} + +void MacroConditionSpeech::SetTranslate(bool translate) +{ + _translate = translate; + _recognizer.SetTranslate(translate); +} + +void MacroConditionSpeech::SetVadEnergyThreshold(const DoubleVariable &value) +{ + _vadEnergyThreshold = value; + _recognizer.SetVadEnergyThreshold((float)(double)_vadEnergyThreshold); +} + +void MacroConditionSpeech::SetSuppressNonSpeechTokens(bool suppress) +{ + _suppressNonSpeechTokens = suppress; + _recognizer.SetSuppressNonSpeechTokens(suppress); +} + +void MacroConditionSpeech::SetNoContext(bool noContext) +{ + _noContext = noContext; + _recognizer.SetNoContext(noContext); +} + +void MacroConditionSpeech::SetListenWhenMuted(bool listen) +{ + _listenWhenMuted = listen; + _recognizer.SetListenWhenMuted(listen); +} + +void MacroConditionSpeech::SetUseGpu(bool useGpu) +{ + _useGpu = useGpu; + _recognizer.SetUseGpu(useGpu); + RebuildRecognizer(); +} + +void MacroConditionSpeech::RebuildRecognizer() +{ + _recognizer.StopCapture(); + + if (_rebuildThread.joinable()) { + _rebuildThread.join(); + } + + const std::string path = _modelPath; + if (path.empty()) { + return; + } + + OBSWeakSource weakSource = _source.GetSource(); + _messageBuffer = _recognizer.RegisterClient(); + + _rebuildThread = std::thread([this, path, weakSource]() { + if (!_recognizer.LoadModel(path)) { + return; + } + OBSSource source = OBSGetStrongRef(weakSource); + if (!source) { + return; + } + _recognizer.StartCapture(source); + }); +} + +bool MacroConditionSpeech::CheckCondition() +{ + std::string lastTranscript; + bool anyReceived = false; + + while (!_messageBuffer->Empty()) { + auto msg = _messageBuffer->ConsumeMessage(); + if (!msg) { + continue; + } + lastTranscript = *msg; + anyReceived = true; + } + + if (anyReceived) { + SetTempVarValue("speech", lastTranscript); + } + + switch (_condition) { + case Condition::ANY: + if (anyReceived) { + return true; + } + return false; + + case Condition::CONTAINS: { + if (!anyReceived) { + return false; + } + const std::string phrase = _phrase; + const QRegularExpression re( + "\\b" + + QRegularExpression::escape( + QString::fromStdString(phrase)) + + "\\b", + QRegularExpression::CaseInsensitiveOption); + if (re.match(QString::fromStdString(lastTranscript)).hasMatch()) { + return true; + } + return false; + } + + case Condition::MATCHES: + if (!anyReceived) { + return false; + } + if (_regex.Enabled() && + _regex.Matches(lastTranscript, _phrase)) { + return true; + } + return false; + + default: + break; + } + + return false; +} + +void MacroConditionSpeech::SetupTempVars() +{ + MacroCondition::SetupTempVars(); + AddTempvar( + "speech", + obs_module_text("AdvSceneSwitcher.tempVar.speech.speech"), + obs_module_text( + "AdvSceneSwitcher.tempVar.speech.speech.description")); +} + +std::string MacroConditionSpeech::GetShortDesc() const +{ + return _source.ToString(); +} + +bool MacroConditionSpeech::Save(obs_data_t *obj) const +{ + MacroCondition::Save(obj); + _source.Save(obj, "source"); + obs_data_set_int(obj, "condition", static_cast(_condition)); + _phrase.Save(obj, "phrase"); + _regex.Save(obj); + _modelPath.Save(obj, "modelPath"); + _bufferDuration.Save(obj, "bufferDuration"); + _nThreads.Save(obj, "nThreads"); + _language.Save(obj, "language"); + obs_data_set_bool(obj, "translate", _translate); + _vadEnergyThreshold.Save(obj, "vadEnergyThreshold"); + obs_data_set_bool(obj, "suppressNonSpeechTokens", + _suppressNonSpeechTokens); + obs_data_set_bool(obj, "noContext", _noContext); + obs_data_set_bool(obj, "listenWhenMuted", _listenWhenMuted); + obs_data_set_bool(obj, "useGpu", _useGpu); + return true; +} + +bool MacroConditionSpeech::Load(obs_data_t *obj) +{ + MacroCondition::Load(obj); + _source.Load(obj, "source"); + SetCondition( + static_cast(obs_data_get_int(obj, "condition"))); + _phrase.Load(obj, "phrase"); + _regex.Load(obj); + _modelPath.Load(obj, "modelPath"); + _bufferDuration.Load(obj, "bufferDuration"); + _nThreads.Load(obj, "nThreads"); + _language.Load(obj, "language"); + _translate = obs_data_get_bool(obj, "translate"); + _vadEnergyThreshold.Load(obj, "vadEnergyThreshold"); + _suppressNonSpeechTokens = + obs_data_get_bool(obj, "suppressNonSpeechTokens"); + _noContext = obs_data_get_bool(obj, "noContext"); + _listenWhenMuted = obs_data_get_bool(obj, "listenWhenMuted"); + _useGpu = obs_data_get_bool(obj, "useGpu"); + _recognizer.SetNThreads((int)_nThreads); + _recognizer.SetLanguage(std::string(_language)); + _recognizer.SetTranslate(_translate); + _recognizer.SetVadEnergyThreshold((float)(double)_vadEnergyThreshold); + _recognizer.SetSuppressNonSpeechTokens(_suppressNonSpeechTokens); + _recognizer.SetNoContext(_noContext); + _recognizer.SetListenWhenMuted(_listenWhenMuted); + _recognizer.SetUseGpu(_useGpu); + RebuildRecognizer(); + return true; +} + +static void populateConditionSelection(QComboBox *list) +{ + static const std::map + conditionTypes = { + {MacroConditionSpeech::Condition::ANY, + "AdvSceneSwitcher.condition.speech.condition.any"}, + {MacroConditionSpeech::Condition::CONTAINS, + "AdvSceneSwitcher.condition.speech.condition.contains"}, + {MacroConditionSpeech::Condition::MATCHES, + "AdvSceneSwitcher.condition.speech.condition.matches"}, + }; + + for (const auto &[cond, name] : conditionTypes) { + list->addItem(obs_module_text(name.c_str()), + static_cast(cond)); + } +} + +MacroConditionSpeechEdit::MacroConditionSpeechEdit( + QWidget *parent, std::shared_ptr entryData) + : QWidget(parent), + _source(new SourceSelectionWidget(this, getAudioSourcesList, true)), + _conditions(new QComboBox(this)), + _phrase(new VariableLineEdit(this)), + _regex(new RegexConfigWidget(parent)), + _modelPath(new FileSelection( + FileSelection::Type::READ, this, + obs_module_text( + "AdvSceneSwitcher.condition.speech.browse.title"))), + _modelHelp(new HelpIcon( + obs_module_text( + "AdvSceneSwitcher.condition.speech.model.help"), + this)), + _bufferDuration(new VariableDoubleSpinBox(this)), + _bufferHelp(new HelpIcon( + obs_module_text( + "AdvSceneSwitcher.condition.speech.buffer.help"), + this)), + _advancedSection(new Section(300, this)), + _nThreads(new VariableSpinBox(this)), + _language(new VariableLineEdit(this)), + _languageHelp(new HelpIcon( + obs_module_text( + "AdvSceneSwitcher.condition.speech.advanced.language.help"), + this)), + _translate(new QCheckBox( + obs_module_text( + "AdvSceneSwitcher.condition.speech.advanced.translate"), + this)), + _translateHelp(new HelpIcon( + obs_module_text( + "AdvSceneSwitcher.condition.speech.advanced.translate.help"), + this)), + _vadEnergyThreshold(new VariableDoubleSpinBox(this)), + _vadHelp(new HelpIcon( + obs_module_text( + "AdvSceneSwitcher.condition.speech.advanced.vad.help"), + this)), + _suppressNonSpeechTokens(new QCheckBox( + obs_module_text( + "AdvSceneSwitcher.condition.speech.advanced.suppress"), + this)), + _suppressHelp(new HelpIcon( + obs_module_text( + "AdvSceneSwitcher.condition.speech.advanced.suppress.help"), + this)), + _noContext(new QCheckBox( + obs_module_text( + "AdvSceneSwitcher.condition.speech.advanced.noContext"), + this)), + _noContextHelp(new HelpIcon( + obs_module_text( + "AdvSceneSwitcher.condition.speech.advanced.noContext.help"), + this)), + _listenWhenMuted(new QCheckBox( + obs_module_text( + "AdvSceneSwitcher.condition.speech.advanced.listenWhenMuted"), + this)), + _useGpu(new QCheckBox( + obs_module_text( + "AdvSceneSwitcher.condition.speech.advanced.useGpu"), + this)) +{ + populateConditionSelection(_conditions); + + _bufferDuration->setMinimum(1.0); + _bufferDuration->setMaximum(30.0); + _bufferDuration->SpinBox()->setSingleStep(0.5); + _bufferDuration->setSuffix(" s"); + + _nThreads->setMinimum(1); + _nThreads->setMaximum(32); + + _vadEnergyThreshold->setMinimum(0.0); + _vadEnergyThreshold->setMaximum(1.0); + _vadEnergyThreshold->SpinBox()->setSingleStep(1e-5); + _vadEnergyThreshold->SpinBox()->setDecimals(6); + + QWidget::connect(_source, + SIGNAL(SourceChanged(const SourceSelection &)), this, + SLOT(SourceChanged(const SourceSelection &))); + QWidget::connect(_conditions, SIGNAL(currentIndexChanged(int)), this, + SLOT(ConditionChanged(int))); + QWidget::connect(_phrase, SIGNAL(editingFinished()), this, + SLOT(PhraseChanged())); + QWidget::connect(_regex, + SIGNAL(RegexConfigChanged(const RegexConfig &)), this, + SLOT(RegexChanged(const RegexConfig &))); + QWidget::connect(_modelPath, SIGNAL(PathChanged(const QString &)), this, + SLOT(ModelPathChanged(const QString &))); + QWidget::connect( + _bufferDuration, + SIGNAL(NumberVariableChanged(const NumberVariable &)), + this, + SLOT(BufferDurationChanged(const NumberVariable &))); + QWidget::connect( + _nThreads, + SIGNAL(NumberVariableChanged(const NumberVariable &)), + this, SLOT(NThreadsChanged(const NumberVariable &))); + QWidget::connect(_language, SIGNAL(editingFinished()), this, + SLOT(LanguageChanged())); + QWidget::connect(_translate, SIGNAL(stateChanged(int)), this, + SLOT(TranslateChanged(int))); + QWidget::connect( + _vadEnergyThreshold, + SIGNAL(NumberVariableChanged(const NumberVariable &)), + this, + SLOT(VadEnergyThresholdChanged(const NumberVariable &))); + QWidget::connect(_suppressNonSpeechTokens, SIGNAL(stateChanged(int)), + this, SLOT(SuppressNonSpeechTokensChanged(int))); + QWidget::connect(_noContext, SIGNAL(stateChanged(int)), this, + SLOT(NoContextChanged(int))); + QWidget::connect(_listenWhenMuted, SIGNAL(stateChanged(int)), this, + SLOT(ListenWhenMutedChanged(int))); + QWidget::connect(_useGpu, SIGNAL(stateChanged(int)), this, + SLOT(UseGpuChanged(int))); + + _condSourceLayout = new QHBoxLayout; + + _phraseLayout = new QHBoxLayout; + PlaceWidgets(obs_module_text( + "AdvSceneSwitcher.condition.speech.layout.phrase"), + _phraseLayout, + {{"{{phrase}}", _phrase}, {"{{regex}}", _regex}}, false); + + auto *modelLayout = new QHBoxLayout; + PlaceWidgets(obs_module_text( + "AdvSceneSwitcher.condition.speech.layout.model"), + modelLayout, + {{"{{modelPath}}", _modelPath}, {"{{help}}", _modelHelp}}, + false); + + auto *bufferLayout = new QHBoxLayout; + PlaceWidgets(obs_module_text( + "AdvSceneSwitcher.condition.speech.layout.buffer"), + bufferLayout, + {{"{{bufferDuration}}", _bufferDuration}, + {"{{help}}", _bufferHelp}}); + + auto *threadsLayout = new QHBoxLayout; + PlaceWidgets( + obs_module_text( + "AdvSceneSwitcher.condition.speech.layout.advanced.threads"), + threadsLayout, {{"{{threads}}", _nThreads}}); + + auto *languageLayout = new QHBoxLayout; + PlaceWidgets( + obs_module_text( + "AdvSceneSwitcher.condition.speech.layout.advanced.language"), + languageLayout, + {{"{{language}}", _language}, {"{{help}}", _languageHelp}}); + + auto *translateLayout = new QHBoxLayout; + PlaceWidgets( + obs_module_text( + "AdvSceneSwitcher.condition.speech.layout.advanced.translate"), + translateLayout, + {{"{{translate}}", _translate}, {"{{help}}", _translateHelp}}); + + auto *vadLayout = new QHBoxLayout; + PlaceWidgets( + obs_module_text( + "AdvSceneSwitcher.condition.speech.layout.advanced.vad"), + vadLayout, + {{"{{vad}}", _vadEnergyThreshold}, {"{{help}}", _vadHelp}}); + + auto *suppressLayout = new QHBoxLayout; + PlaceWidgets( + obs_module_text( + "AdvSceneSwitcher.condition.speech.layout.advanced.suppress"), + suppressLayout, + {{"{{suppress}}", _suppressNonSpeechTokens}, + {"{{help}}", _suppressHelp}}); + + auto *noContextLayout = new QHBoxLayout; + PlaceWidgets( + obs_module_text( + "AdvSceneSwitcher.condition.speech.layout.advanced.noContext"), + noContextLayout, + {{"{{noContext}}", _noContext}, {"{{help}}", _noContextHelp}}); + + auto *advancedContent = new QWidget(this); + auto *advancedLayout = new QVBoxLayout; + advancedLayout->addLayout(threadsLayout); + advancedLayout->addLayout(languageLayout); + advancedLayout->addLayout(translateLayout); + advancedLayout->addLayout(vadLayout); + advancedLayout->addLayout(suppressLayout); + advancedLayout->addLayout(noContextLayout); + advancedLayout->addWidget(_listenWhenMuted); + advancedLayout->addWidget(_useGpu); + advancedContent->setLayout(advancedLayout); + + _advancedSection->AddHeaderWidget(new QLabel( + obs_module_text("AdvSceneSwitcher.condition.speech.advanced"), + this)); + _advancedSection->SetContent(advancedContent, true); + + auto *mainLayout = new QVBoxLayout; + mainLayout->addLayout(_condSourceLayout); + mainLayout->addLayout(_phraseLayout); + mainLayout->addLayout(modelLayout); + mainLayout->addLayout(bufferLayout); + mainLayout->addWidget(_advancedSection); + setLayout(mainLayout); + + _entryData = entryData; + UpdateEntryData(); + _loading = false; +} + +void MacroConditionSpeechEdit::UpdateEntryData() +{ + if (!_entryData) { + return; + } + + _source->SetSource(_entryData->_source); + _conditions->setCurrentIndex( + static_cast(_entryData->GetCondition())); + _phrase->setText(QString::fromStdString(_entryData->_phrase)); + _regex->SetRegexConfig(_entryData->_regex); + _modelPath->SetPath(_entryData->GetModelPath()); + _bufferDuration->SetValue(_entryData->GetBufferDuration()); + _nThreads->SetValue(_entryData->GetNThreads()); + _language->setText(QString::fromStdString(_entryData->GetLanguage())); + _translate->setChecked(_entryData->GetTranslate()); + _vadEnergyThreshold->SetValue(_entryData->GetVadEnergyThreshold()); + _suppressNonSpeechTokens->setChecked( + _entryData->GetSuppressNonSpeechTokens()); + _noContext->setChecked(_entryData->GetNoContext()); + _listenWhenMuted->setChecked(_entryData->GetListenWhenMuted()); + _useGpu->setChecked(_entryData->GetUseGpu()); + SetWidgetVisibility(); +} + +void MacroConditionSpeechEdit::SourceChanged(const SourceSelection &source) +{ + GUARD_LOADING_AND_LOCK(); + _entryData->_source = source; + _entryData->RebuildRecognizer(); + emit HeaderInfoChanged( + QString::fromStdString(_entryData->GetShortDesc())); +} + +void MacroConditionSpeechEdit::ConditionChanged(int idx) +{ + { + GUARD_LOADING_AND_LOCK(); + _entryData->SetCondition( + static_cast( + _conditions->itemData(idx).toInt())); + } + SetWidgetVisibility(); +} + +void MacroConditionSpeechEdit::PhraseChanged() +{ + GUARD_LOADING_AND_LOCK(); + _entryData->_phrase = _phrase->text().toStdString(); +} + +void MacroConditionSpeechEdit::RegexChanged(const RegexConfig &conf) +{ + GUARD_LOADING_AND_LOCK(); + _entryData->_regex = conf; +} + +void MacroConditionSpeechEdit::ModelPathChanged(const QString &path) +{ + GUARD_LOADING_AND_LOCK(); + _entryData->SetModelPath(path.toStdString()); +} + +void MacroConditionSpeechEdit::BufferDurationChanged( + const NumberVariable &value) +{ + GUARD_LOADING_AND_LOCK(); + _entryData->SetBufferDuration(value); +} + +void MacroConditionSpeechEdit::NThreadsChanged(const NumberVariable &value) +{ + GUARD_LOADING_AND_LOCK(); + _entryData->SetNThreads(value); +} + +void MacroConditionSpeechEdit::LanguageChanged() +{ + GUARD_LOADING_AND_LOCK(); + _entryData->SetLanguage(_language->text().toStdString()); +} + +void MacroConditionSpeechEdit::TranslateChanged(int state) +{ + GUARD_LOADING_AND_LOCK(); + _entryData->SetTranslate(state == Qt::Checked); +} + +void MacroConditionSpeechEdit::VadEnergyThresholdChanged( + const NumberVariable &value) +{ + GUARD_LOADING_AND_LOCK(); + _entryData->SetVadEnergyThreshold(value); +} + +void MacroConditionSpeechEdit::SuppressNonSpeechTokensChanged(int state) +{ + GUARD_LOADING_AND_LOCK(); + _entryData->SetSuppressNonSpeechTokens(state == Qt::Checked); +} + +void MacroConditionSpeechEdit::NoContextChanged(int state) +{ + GUARD_LOADING_AND_LOCK(); + _entryData->SetNoContext(state == Qt::Checked); +} + +void MacroConditionSpeechEdit::ListenWhenMutedChanged(int state) +{ + GUARD_LOADING_AND_LOCK(); + _entryData->SetListenWhenMuted(state == Qt::Checked); +} + +void MacroConditionSpeechEdit::UseGpuChanged(int state) +{ + GUARD_LOADING_AND_LOCK(); + _entryData->SetUseGpu(state == Qt::Checked); +} + +void MacroConditionSpeechEdit::SetWidgetVisibility() +{ + const auto condition = _entryData->GetCondition(); + const bool hasPhrase = condition != + MacroConditionSpeech::Condition::ANY; + + _condSourceLayout->removeWidget(_conditions); + _condSourceLayout->removeWidget(_source); + ClearLayout(_condSourceLayout); + const char *layoutKey = "AdvSceneSwitcher.condition.speech.layout.any"; + if (condition == MacroConditionSpeech::Condition::CONTAINS) { + layoutKey = "AdvSceneSwitcher.condition.speech.layout.contains"; + } else if (condition == MacroConditionSpeech::Condition::MATCHES) { + layoutKey = "AdvSceneSwitcher.condition.speech.layout.matches"; + } + PlaceWidgets(obs_module_text(layoutKey), _condSourceLayout, + {{"{{conditions}}", _conditions}, + {"{{source}}", _source}}); + + SetLayoutVisible(_phraseLayout, hasPhrase); + _regex->setVisible(condition == + MacroConditionSpeech::Condition::MATCHES); + adjustSize(); + updateGeometry(); +} + +} // namespace advss diff --git a/plugins/speech/macro-condition-speech.hpp b/plugins/speech/macro-condition-speech.hpp new file mode 100644 index 00000000..bffb3539 --- /dev/null +++ b/plugins/speech/macro-condition-speech.hpp @@ -0,0 +1,182 @@ +#pragma once +#include "file-selection.hpp" +#include "help-icon.hpp" +#include "macro-condition-edit.hpp" +#include "regex-config.hpp" +#include "section.hpp" +#include "source-selection.hpp" +#include "speech-recognizer.hpp" +#include "variable-line-edit.hpp" +#include "variable-number.hpp" +#include "variable-spinbox.hpp" +#include "variable-string.hpp" + +#include +#include +#include +#include + +#include + +namespace advss { + +class MacroConditionSpeech : public MacroCondition { +public: + MacroConditionSpeech(Macro *m); + ~MacroConditionSpeech(); + + bool CheckCondition() override; + bool Save(obs_data_t *obj) const override; + bool Load(obs_data_t *obj) override; + std::string GetShortDesc() const override; + std::string GetId() const override { return id; } + + static std::shared_ptr Create(Macro *m) + { + return std::make_shared(m); + } + + enum class Condition { + ANY, + CONTAINS, // Just a more user friendly variant of "matches" + MATCHES, + }; + + void SetCondition(Condition c); + Condition GetCondition() const { return _condition; } + + void SetModelPath(const std::string &path); + const StringVariable &GetModelPath() const { return _modelPath; } + + void SetBufferDuration(const DoubleVariable &value); + DoubleVariable GetBufferDuration() const { return _bufferDuration; } + + void SetNThreads(const IntVariable &value); + IntVariable GetNThreads() const { return _nThreads; } + + void SetLanguage(const std::string &lang); + const StringVariable &GetLanguage() const { return _language; } + + void SetTranslate(bool translate); + bool GetTranslate() const { return _translate; } + + void SetVadEnergyThreshold(const DoubleVariable &value); + DoubleVariable GetVadEnergyThreshold() const + { + return _vadEnergyThreshold; + } + + void SetSuppressNonSpeechTokens(bool suppress); + bool GetSuppressNonSpeechTokens() const + { + return _suppressNonSpeechTokens; + } + + void SetNoContext(bool noContext); + bool GetNoContext() const { return _noContext; } + + void SetListenWhenMuted(bool listen); + bool GetListenWhenMuted() const { return _listenWhenMuted; } + + void SetUseGpu(bool useGpu); + bool GetUseGpu() const { return _useGpu; } + + SourceSelection _source; + StringVariable _phrase = ""; + RegexConfig _regex; + + void RebuildRecognizer(); + +private: + void SetupTempVars() override; + + Condition _condition = Condition::ANY; + StringVariable _modelPath; + DoubleVariable _bufferDuration = 5.0; + IntVariable _nThreads; + StringVariable _language; + bool _translate = false; + DoubleVariable _vadEnergyThreshold = 1e-4; + bool _suppressNonSpeechTokens = true; + bool _noContext = true; + bool _listenWhenMuted = false; + bool _useGpu = true; + + SpeechRecognizer _recognizer; + std::shared_ptr> _messageBuffer; + std::thread _rebuildThread; + + static bool _registered; + static const std::string id; +}; + +class MacroConditionSpeechEdit : public QWidget { + Q_OBJECT + +public: + MacroConditionSpeechEdit( + QWidget *parent, + std::shared_ptr entryData = nullptr); + void UpdateEntryData(); + + static QWidget *Create(QWidget *parent, + std::shared_ptr cond) + { + return new MacroConditionSpeechEdit( + parent, + std::dynamic_pointer_cast(cond)); + } + +private slots: + void SourceChanged(const SourceSelection &); + void ConditionChanged(int); + void PhraseChanged(); + void RegexChanged(const RegexConfig &); + void ModelPathChanged(const QString &); + void BufferDurationChanged(const NumberVariable &); + void NThreadsChanged(const NumberVariable &); + void LanguageChanged(); + void TranslateChanged(int); + void VadEnergyThresholdChanged(const NumberVariable &); + void SuppressNonSpeechTokensChanged(int); + void NoContextChanged(int); + void ListenWhenMutedChanged(int); + void UseGpuChanged(int); + +signals: + void HeaderInfoChanged(const QString &); + +private: + void SetWidgetVisibility(); + + SourceSelectionWidget *_source; + QComboBox *_conditions; + VariableLineEdit *_phrase; + RegexConfigWidget *_regex; + FileSelection *_modelPath; + HelpIcon *_modelHelp; + VariableDoubleSpinBox *_bufferDuration; + HelpIcon *_bufferHelp; + Section *_advancedSection; + VariableSpinBox *_nThreads; + VariableLineEdit *_language; + HelpIcon *_languageHelp; + QCheckBox *_translate; + HelpIcon *_translateHelp; + VariableDoubleSpinBox *_vadEnergyThreshold; + HelpIcon *_vadHelp; + QCheckBox *_suppressNonSpeechTokens; + HelpIcon *_suppressHelp; + QCheckBox *_noContext; + HelpIcon *_noContextHelp; + QCheckBox *_listenWhenMuted; + QCheckBox *_useGpu; + + QHBoxLayout *_condSourceLayout; + QHBoxLayout *_phraseLayout; + + std::shared_ptr _entryData; + bool _loading = true; +}; + +} // namespace advss diff --git a/plugins/speech/speech-recognizer.cpp b/plugins/speech/speech-recognizer.cpp new file mode 100644 index 00000000..1b2d4355 --- /dev/null +++ b/plugins/speech/speech-recognizer.cpp @@ -0,0 +1,402 @@ +#include "speech-recognizer.hpp" + +#include "log-helper.hpp" +#include "plugin-state-helpers.hpp" + +#include +#include +#include + +#include + +namespace advss { + +// Ignore configured log level until after loading is complete to ensure we +// catch the initial whisper configuration logs +static bool ignoreLogFilter = true; + +static bool setup() +{ + AddFinishedLoadingStep([]() { ignoreLogFilter = false; }); + return true; +} +const bool _ = setup(); + +static void whisperLogCallback(ggml_log_level level, const char *text, void *) +{ + if (!text || *text == '\0') { + return; + } + + int obsLevel = LOG_INFO; + if (level == GGML_LOG_LEVEL_WARN) { + obsLevel = LOG_WARNING; + } else if (level == GGML_LOG_LEVEL_ERROR) { + obsLevel = LOG_ERROR; + } + + std::string msg(text); + if (!msg.empty() && msg.back() == '\n') { + msg.pop_back(); + } + + if (msg.empty()) { + return; + } + + if (ignoreLogFilter) { + blog(obsLevel, "[speech] %s", msg.c_str()); + } else { + vblog(obsLevel, "[speech] %s", msg.c_str()); + } +} + +static constexpr int whisperSampleRate = 16000; + +// How often to evaluate VAD and potentially trigger inference +static constexpr double stepDurationSeconds = 1.0; + +// Audio kept from the previous inference run to provide word-boundary context +// for the next run. +// I guess repeating something is better than potentially missing stuff. +static constexpr double keepDurationSeconds = 0.2; + +SpeechRecognizer::SpeechRecognizer() +{ + _inferenceThread = std::thread(&SpeechRecognizer::InferenceLoop, this); +} + +SpeechRecognizer::~SpeechRecognizer() +{ + StopCapture(); + + { + std::unique_lock lock(_inferenceMutex); + _stopThread = true; + _bufferReady = true; + } + _inferenceCV.notify_one(); + if (_inferenceThread.joinable()) { + _inferenceThread.join(); + } + + if (_ctx) { + whisper_free(_ctx); + } + if (_resampler) { + audio_resampler_destroy( + static_cast(_resampler)); + } +} + +bool SpeechRecognizer::LoadModel(const std::string &modelPath) +{ + std::lock_guard lock(_ctxMutex); + + if (_ctx) { + whisper_free(_ctx); + _ctx = nullptr; + } + + whisper_log_set(whisperLogCallback, nullptr); + + whisper_context_params cparams = whisper_context_default_params(); + cparams.use_gpu = _useGpu; + _ctx = whisper_init_from_file_with_params(modelPath.c_str(), cparams); + if (!_ctx) { + blog(LOG_WARNING, "failed to load whisper model: %s", + modelPath.c_str()); + return false; + } + return true; +} + +bool SpeechRecognizer::StartCapture(obs_source_t *source) +{ + StopCapture(); + + if (!source) { + return false; + } + + const audio_t *audio = obs_get_audio(); + if (!audio) { + return false; + } + const struct audio_output_info *aoi = audio_output_get_info(audio); + _sourceSampleRate = (int)aoi->samples_per_sec; + _sourceChannelCount = (int)get_audio_channels(aoi->speakers); + + if (_resampler) { + audio_resampler_destroy( + static_cast(_resampler)); + _resampler = nullptr; + } + + struct resample_info srcInfo = {}; + srcInfo.samples_per_sec = (uint32_t)_sourceSampleRate; + srcInfo.format = AUDIO_FORMAT_FLOAT_PLANAR; + srcInfo.speakers = aoi->speakers; + + struct resample_info dstInfo = {}; + dstInfo.samples_per_sec = whisperSampleRate; + dstInfo.format = AUDIO_FORMAT_FLOAT; + dstInfo.speakers = SPEAKERS_MONO; + + _resampler = audio_resampler_create(&dstInfo, &srcInfo); + if (!_resampler) { + blog(LOG_WARNING, + "failed to create audio resampler for speech condition"); + return false; + } + + _captureSource = obs_source_get_weak_source(source); + obs_source_add_audio_capture_callback(source, AudioCaptureCallback, + this); + return true; +} + +void SpeechRecognizer::StopCapture() +{ + OBSSource source = OBSGetStrongRef(_captureSource); + if (source) { + obs_source_remove_audio_capture_callback( + source, AudioCaptureCallback, this); + } + _captureSource = OBSWeakSource{}; +} + +void SpeechRecognizer::SetBufferDuration(double seconds) +{ + std::lock_guard lock(_audioMutex); + _bufferDurationSeconds = seconds; + _audioBuffer.clear(); + _framesSinceLastStep = 0; +} + +void SpeechRecognizer::SetNThreads(int n) +{ + std::lock_guard lock(_inferenceMutex); + _nThreads = std::max(1, n); +} + +void SpeechRecognizer::SetLanguage(const std::string &lang) +{ + std::lock_guard lock(_inferenceMutex); + _language = lang.empty() ? "auto" : lang; +} + +void SpeechRecognizer::SetTranslate(bool translate) +{ + std::lock_guard lock(_inferenceMutex); + _translate = translate; +} + +void SpeechRecognizer::SetVadEnergyThreshold(float threshold) +{ + std::lock_guard lock(_audioMutex); + _vadEnergyThreshold = threshold; +} + +void SpeechRecognizer::SetSuppressNonSpeechTokens(bool suppress) +{ + std::lock_guard lock(_inferenceMutex); + _suppressNonSpeechTokens = suppress; +} + +void SpeechRecognizer::SetNoContext(bool noContext) +{ + std::lock_guard lock(_inferenceMutex); + _noContext = noContext; +} + +void SpeechRecognizer::SetListenWhenMuted(bool listen) +{ + _listenWhenMuted = listen; +} + +void SpeechRecognizer::SetUseGpu(bool useGpu) +{ + std::lock_guard lock(_ctxMutex); + _useGpu = useGpu; +} + +std::shared_ptr> SpeechRecognizer::RegisterClient() +{ + return _dispatcher.RegisterClient(); +} + +void SpeechRecognizer::AudioCaptureCallback(void *param, obs_source_t *, + const struct audio_data *audio, + bool muted) +{ + if (!audio || !audio->data[0]) { + return; + } + auto *self = static_cast(param); + if (muted && !self->_listenWhenMuted) { + return; + } + self->AppendResampledAudio(audio); +} + +void SpeechRecognizer::AppendResampledAudio(const struct audio_data *audio) +{ + if (!_resampler) { + return; + } + + uint8_t *resampledData[MAX_AV_PLANES] = {}; + uint32_t outFrames = 0; + uint64_t tsOffset = 0; + + bool ok = audio_resampler_resample( + static_cast(_resampler), resampledData, + &outFrames, &tsOffset, (const uint8_t *const *)audio->data, + audio->frames); + if (!ok || outFrames == 0 || !resampledData[0]) { + return; + } + + const float *samples = + reinterpret_cast(resampledData[0]); + + std::unique_lock lock(_audioMutex); + _audioBuffer.insert(_audioBuffer.end(), samples, samples + outFrames); + _framesSinceLastStep += outFrames; + + // Keep the rolling buffer capped at the configured context window. + const size_t maxFrames = + (size_t)(_bufferDurationSeconds * whisperSampleRate); + if (_audioBuffer.size() > maxFrames) { + _audioBuffer.erase(_audioBuffer.begin(), + _audioBuffer.begin() + + (_audioBuffer.size() - maxFrames)); + } + + // Only consider triggering inference once per step interval. + const size_t stepFrames = + (size_t)(stepDurationSeconds * whisperSampleRate); + if (_framesSinceLastStep < stepFrames) { + return; + } + _framesSinceLastStep = 0; + + // VAD: measure energy over just the most recent step window so that a + // short utterance at the end of a longer silent buffer is not diluted. + const size_t vadWindow = std::min(_audioBuffer.size(), stepFrames); + const size_t vadStart = _audioBuffer.size() - vadWindow; + float energy = 0.0f; + for (size_t i = vadStart; i < _audioBuffer.size(); ++i) { + energy += _audioBuffer[i] * _audioBuffer[i]; + } + energy /= (float)vadWindow; + const bool tooSilent = energy < _vadEnergyThreshold; + if (tooSilent) { + return; + } + + { + std::unique_lock infLock(_inferenceMutex); + if (_bufferReady) { + return; + } + + _inferenceBuffer = _audioBuffer; + _bufferReady = true; + _inferenceCV.notify_one(); + } + + // Retain a short overlap so the next inference has word-boundary context. + const size_t keepFrames = + (size_t)(keepDurationSeconds * whisperSampleRate); + if (_audioBuffer.size() > keepFrames) { + _audioBuffer.erase(_audioBuffer.begin(), + _audioBuffer.begin() + + (_audioBuffer.size() - keepFrames)); + } +} + +void SpeechRecognizer::InferenceLoop() +{ + while (true) { + std::vector buffer; + int nThreads; + std::string language; + bool translate; + bool suppressNonSpeechTokens; + bool noContext; + + { + std::unique_lock lock(_inferenceMutex); + _inferenceCV.wait(lock, + [this] { return _bufferReady; }); + _bufferReady = false; + if (_stopThread) { + break; + } + buffer = std::move(_inferenceBuffer); + nThreads = _nThreads; + language = _language; + translate = _translate; + suppressNonSpeechTokens = _suppressNonSpeechTokens; + noContext = _noContext; + } + + if (buffer.empty()) { + continue; + } + + std::lock_guard ctxLock(_ctxMutex); + + if (!_ctx) { + continue; + } + + whisper_full_params params = + whisper_full_default_params(WHISPER_SAMPLING_GREEDY); + params.print_realtime = false; + params.print_progress = false; + params.print_timestamps = false; + params.print_special = false; + params.translate = translate; + params.language = language.c_str(); + params.n_threads = nThreads; + params.single_segment = false; + params.suppress_nst = suppressNonSpeechTokens; + params.no_context = noContext; + + // Limit the encoder to the actual audio length + params.audio_ctx = + std::min(1500, (int)((float)buffer.size() / + (float)whisperSampleRate * 50.0f)); + + int rc = whisper_full(_ctx, params, buffer.data(), + (int)buffer.size()); + if (rc != 0) { + blog(LOG_WARNING, "whisper_full returned %d", rc); + continue; + } + + std::string transcript; + const int nSegments = whisper_full_n_segments(_ctx); + for (int i = 0; i < nSegments; ++i) { + const char *text = + whisper_full_get_segment_text(_ctx, i); + if (text) { + transcript += text; + } + } + + if (!transcript.empty()) { + const auto begin = + transcript.find_first_not_of(" \t\r\n"); + if (begin != std::string::npos) { + transcript = transcript.substr(begin); + } + _dispatcher.DispatchMessage(transcript); + } + } +} + +} // namespace advss diff --git a/plugins/speech/speech-recognizer.hpp b/plugins/speech/speech-recognizer.hpp new file mode 100644 index 00000000..1d8b8e68 --- /dev/null +++ b/plugins/speech/speech-recognizer.hpp @@ -0,0 +1,86 @@ +#pragma once +#include "message-buffer.hpp" +#include "message-dispatcher.hpp" + +#include + +#include +#include +#include +#include +#include +#include + +struct whisper_context; +struct audio_data; + +namespace advss { + +// Captures audio from one OBS source, resamples to 16 kHz mono, runs +// whisper.cpp inference on a background thread, and dispatches the resulting +// transcript text to registered MessageBuffers. +class SpeechRecognizer { +public: + SpeechRecognizer(); + ~SpeechRecognizer(); + + bool LoadModel(const std::string &modelPath); + + bool StartCapture(obs_source_t *source); + void StopCapture(); + + void SetBufferDuration(double seconds); + void SetNThreads(int n); + void SetLanguage(const std::string &lang); + void SetTranslate(bool translate); + void SetVadEnergyThreshold(float threshold); + void SetSuppressNonSpeechTokens(bool suppress); + void SetNoContext(bool noContext); + void SetListenWhenMuted(bool listen); + void SetUseGpu(bool useGpu); + + [[nodiscard]] std::shared_ptr> + RegisterClient(); + +private: + static void AudioCaptureCallback(void *param, obs_source_t *source, + const struct audio_data *audio, + bool muted); + void AppendResampledAudio(const struct audio_data *audio); + void InferenceLoop(); + + // Held during whisper_full and when freeing/replacing _ctx. + std::mutex _ctxMutex; + whisper_context *_ctx = nullptr; + // Stored as void* to avoid pulling into + // this header. Cast to audio_resampler_t* in the .cpp. + void *_resampler = nullptr; + + std::vector _audioBuffer; + std::mutex _audioMutex; + double _bufferDurationSeconds = 5.0; + float _vadEnergyThreshold = 1e-4f; + size_t _framesSinceLastStep = 0; + + std::vector _inferenceBuffer; + std::thread _inferenceThread; + std::atomic_bool _stopThread{false}; + std::condition_variable _inferenceCV; + std::mutex _inferenceMutex; + bool _bufferReady = false; + int _nThreads = 4; + std::string _language = "auto"; + bool _translate = false; + bool _suppressNonSpeechTokens = true; + bool _noContext = true; + bool _listenWhenMuted = false; + bool _useGpu = true; + + OBSWeakSource _captureSource; + int _sourceSampleRate = 44100; + int _sourceChannelCount = 2; + + MessageDispatcher _dispatcher; +}; + +} // namespace advss