#include "macro-condition-speech.hpp" #include "layout-helpers.hpp" #include "macro-helpers.hpp" #include "selection-helpers.hpp" #include #include #include #include namespace advss { static int defaultNThreads() { return (int)std::min( 4u, std::max(1u, std::thread::hardware_concurrency() / 4)); } static std::string getDefaultModelPath() { return std::string(obs_get_module_data_path(obs_current_module())) + "/res/speech/ggml-tiny-q8_0.bin"; } static QStringList getAudioSourcesList() { auto sources = GetAudioSourceNames(); sources.sort(); return sources; } const std::string MacroConditionSpeech::id = "speech"; bool MacroConditionSpeech::_registered = MacroConditionFactory::Register( MacroConditionSpeech::id, {MacroConditionSpeech::Create, MacroConditionSpeechEdit::Create, "AdvSceneSwitcher.condition.speech"}); MacroConditionSpeech::MacroConditionSpeech(Macro *m) : MacroCondition(m), _modelPath(getDefaultModelPath()), _nThreads(defaultNThreads()), _language("auto"), _messageBuffer(_recognizer.RegisterClient()) { _recognizer.SetNThreads(defaultNThreads()); _recognizer.SetLanguage("auto"); _recognizer.SetTranslate(false); _recognizer.SetVadEnergyThreshold(1e-4f); _recognizer.SetSuppressNonSpeechTokens(true); _recognizer.SetNoContext(true); } MacroConditionSpeech::~MacroConditionSpeech() { if (_rebuildThread.joinable()) { _rebuildThread.join(); } } void MacroConditionSpeech::SetCondition(Condition c) { _condition = c; SetupTempVars(); } void MacroConditionSpeech::SetModelPath(const std::string &path) { _modelPath = path; RebuildRecognizer(); } void MacroConditionSpeech::SetBufferDuration(const DoubleVariable &value) { _bufferDuration = value; _recognizer.SetBufferDuration((double)_bufferDuration); } void MacroConditionSpeech::SetNThreads(const IntVariable &value) { _nThreads = value; _recognizer.SetNThreads((int)_nThreads); } void MacroConditionSpeech::SetLanguage(const std::string &lang) { _language = lang; _recognizer.SetLanguage(lang); } void MacroConditionSpeech::SetTranslate(bool translate) { _translate = translate; _recognizer.SetTranslate(translate); } void MacroConditionSpeech::SetVadEnergyThreshold(const DoubleVariable &value) { _vadEnergyThreshold = value; _recognizer.SetVadEnergyThreshold((float)(double)_vadEnergyThreshold); } void MacroConditionSpeech::SetSuppressNonSpeechTokens(bool suppress) { _suppressNonSpeechTokens = suppress; _recognizer.SetSuppressNonSpeechTokens(suppress); } void MacroConditionSpeech::SetNoContext(bool noContext) { _noContext = noContext; _recognizer.SetNoContext(noContext); } void MacroConditionSpeech::SetListenWhenMuted(bool listen) { _listenWhenMuted = listen; _recognizer.SetListenWhenMuted(listen); } void MacroConditionSpeech::SetUseGpu(bool useGpu) { _useGpu = useGpu; _recognizer.SetUseGpu(useGpu); RebuildRecognizer(); } void MacroConditionSpeech::RebuildRecognizer() { _recognizer.StopCapture(); if (_rebuildThread.joinable()) { _rebuildThread.join(); } const std::string path = _modelPath; if (path.empty()) { return; } OBSWeakSource weakSource = _source.GetSource(); _messageBuffer = _recognizer.RegisterClient(); _rebuildThread = std::thread([this, path, weakSource]() { if (!_recognizer.LoadModel(path)) { return; } OBSSource source = OBSGetStrongRef(weakSource); if (!source) { return; } _recognizer.StartCapture(source); }); } bool MacroConditionSpeech::CheckCondition() { std::string lastTranscript; bool anyReceived = false; while (!_messageBuffer->Empty()) { auto msg = _messageBuffer->ConsumeMessage(); if (!msg) { continue; } lastTranscript = *msg; anyReceived = true; } if (anyReceived) { SetTempVarValue("speech", lastTranscript); } switch (_condition) { case Condition::ANY: if (anyReceived) { return true; } return false; case Condition::CONTAINS: { if (!anyReceived) { return false; } const std::string phrase = _phrase; const QRegularExpression re( "\\b" + QRegularExpression::escape( QString::fromStdString(phrase)) + "\\b", QRegularExpression::CaseInsensitiveOption); if (re.match(QString::fromStdString(lastTranscript)).hasMatch()) { return true; } return false; } case Condition::MATCHES: if (!anyReceived) { return false; } if (_regex.Enabled() && _regex.Matches(lastTranscript, _phrase)) { return true; } return false; default: break; } return false; } void MacroConditionSpeech::SetupTempVars() { MacroCondition::SetupTempVars(); AddTempvar( "speech", obs_module_text("AdvSceneSwitcher.tempVar.speech.speech"), obs_module_text( "AdvSceneSwitcher.tempVar.speech.speech.description")); } std::string MacroConditionSpeech::GetShortDesc() const { return _source.ToString(); } bool MacroConditionSpeech::Save(obs_data_t *obj) const { MacroCondition::Save(obj); _source.Save(obj, "source"); obs_data_set_int(obj, "condition", static_cast(_condition)); _phrase.Save(obj, "phrase"); _regex.Save(obj); _modelPath.Save(obj, "modelPath"); _bufferDuration.Save(obj, "bufferDuration"); _nThreads.Save(obj, "nThreads"); _language.Save(obj, "language"); obs_data_set_bool(obj, "translate", _translate); _vadEnergyThreshold.Save(obj, "vadEnergyThreshold"); obs_data_set_bool(obj, "suppressNonSpeechTokens", _suppressNonSpeechTokens); obs_data_set_bool(obj, "noContext", _noContext); obs_data_set_bool(obj, "listenWhenMuted", _listenWhenMuted); obs_data_set_bool(obj, "useGpu", _useGpu); return true; } bool MacroConditionSpeech::Load(obs_data_t *obj) { MacroCondition::Load(obj); _source.Load(obj, "source"); SetCondition( static_cast(obs_data_get_int(obj, "condition"))); _phrase.Load(obj, "phrase"); _regex.Load(obj); _modelPath.Load(obj, "modelPath"); _bufferDuration.Load(obj, "bufferDuration"); _nThreads.Load(obj, "nThreads"); _language.Load(obj, "language"); _translate = obs_data_get_bool(obj, "translate"); _vadEnergyThreshold.Load(obj, "vadEnergyThreshold"); _suppressNonSpeechTokens = obs_data_get_bool(obj, "suppressNonSpeechTokens"); _noContext = obs_data_get_bool(obj, "noContext"); _listenWhenMuted = obs_data_get_bool(obj, "listenWhenMuted"); _useGpu = obs_data_get_bool(obj, "useGpu"); _recognizer.SetNThreads((int)_nThreads); _recognizer.SetLanguage(std::string(_language)); _recognizer.SetTranslate(_translate); _recognizer.SetVadEnergyThreshold((float)(double)_vadEnergyThreshold); _recognizer.SetSuppressNonSpeechTokens(_suppressNonSpeechTokens); _recognizer.SetNoContext(_noContext); _recognizer.SetListenWhenMuted(_listenWhenMuted); _recognizer.SetUseGpu(_useGpu); RebuildRecognizer(); return true; } static void populateConditionSelection(QComboBox *list) { static const std::map conditionTypes = { {MacroConditionSpeech::Condition::ANY, "AdvSceneSwitcher.condition.speech.condition.any"}, {MacroConditionSpeech::Condition::CONTAINS, "AdvSceneSwitcher.condition.speech.condition.contains"}, {MacroConditionSpeech::Condition::MATCHES, "AdvSceneSwitcher.condition.speech.condition.matches"}, }; for (const auto &[cond, name] : conditionTypes) { list->addItem(obs_module_text(name.c_str()), static_cast(cond)); } } MacroConditionSpeechEdit::MacroConditionSpeechEdit( QWidget *parent, std::shared_ptr entryData) : QWidget(parent), _source(new SourceSelectionWidget(this, getAudioSourcesList, true)), _conditions(new QComboBox(this)), _phrase(new VariableLineEdit(this)), _regex(new RegexConfigWidget(parent)), _modelPath(new FileSelection( FileSelection::Type::READ, this, obs_module_text( "AdvSceneSwitcher.condition.speech.browse.title"))), _modelHelp(new HelpIcon( obs_module_text( "AdvSceneSwitcher.condition.speech.model.help"), this)), _bufferDuration(new VariableDoubleSpinBox(this)), _bufferHelp(new HelpIcon( obs_module_text( "AdvSceneSwitcher.condition.speech.buffer.help"), this)), _advancedSection(new Section(300, this)), _nThreads(new VariableSpinBox(this)), _language(new VariableLineEdit(this)), _languageHelp(new HelpIcon( obs_module_text( "AdvSceneSwitcher.condition.speech.advanced.language.help"), this)), _translate(new QCheckBox( obs_module_text( "AdvSceneSwitcher.condition.speech.advanced.translate"), this)), _translateHelp(new HelpIcon( obs_module_text( "AdvSceneSwitcher.condition.speech.advanced.translate.help"), this)), _vadEnergyThreshold(new VariableDoubleSpinBox(this)), _vadHelp(new HelpIcon( obs_module_text( "AdvSceneSwitcher.condition.speech.advanced.vad.help"), this)), _suppressNonSpeechTokens(new QCheckBox( obs_module_text( "AdvSceneSwitcher.condition.speech.advanced.suppress"), this)), _suppressHelp(new HelpIcon( obs_module_text( "AdvSceneSwitcher.condition.speech.advanced.suppress.help"), this)), _noContext(new QCheckBox( obs_module_text( "AdvSceneSwitcher.condition.speech.advanced.noContext"), this)), _noContextHelp(new HelpIcon( obs_module_text( "AdvSceneSwitcher.condition.speech.advanced.noContext.help"), this)), _listenWhenMuted(new QCheckBox( obs_module_text( "AdvSceneSwitcher.condition.speech.advanced.listenWhenMuted"), this)), _useGpu(new QCheckBox( obs_module_text( "AdvSceneSwitcher.condition.speech.advanced.useGpu"), this)) { populateConditionSelection(_conditions); _bufferDuration->setMinimum(1.0); _bufferDuration->setMaximum(30.0); _bufferDuration->SpinBox()->setSingleStep(0.5); _bufferDuration->setSuffix(" s"); _nThreads->setMinimum(1); _nThreads->setMaximum(32); _vadEnergyThreshold->setMinimum(0.0); _vadEnergyThreshold->setMaximum(1.0); _vadEnergyThreshold->SpinBox()->setSingleStep(1e-5); _vadEnergyThreshold->SpinBox()->setDecimals(6); QWidget::connect(_source, SIGNAL(SourceChanged(const SourceSelection &)), this, SLOT(SourceChanged(const SourceSelection &))); QWidget::connect(_conditions, SIGNAL(currentIndexChanged(int)), this, SLOT(ConditionChanged(int))); QWidget::connect(_phrase, SIGNAL(editingFinished()), this, SLOT(PhraseChanged())); QWidget::connect(_regex, SIGNAL(RegexConfigChanged(const RegexConfig &)), this, SLOT(RegexChanged(const RegexConfig &))); QWidget::connect(_modelPath, SIGNAL(PathChanged(const QString &)), this, SLOT(ModelPathChanged(const QString &))); QWidget::connect( _bufferDuration, SIGNAL(NumberVariableChanged(const NumberVariable &)), this, SLOT(BufferDurationChanged(const NumberVariable &))); QWidget::connect( _nThreads, SIGNAL(NumberVariableChanged(const NumberVariable &)), this, SLOT(NThreadsChanged(const NumberVariable &))); QWidget::connect(_language, SIGNAL(editingFinished()), this, SLOT(LanguageChanged())); QWidget::connect(_translate, SIGNAL(stateChanged(int)), this, SLOT(TranslateChanged(int))); QWidget::connect( _vadEnergyThreshold, SIGNAL(NumberVariableChanged(const NumberVariable &)), this, SLOT(VadEnergyThresholdChanged(const NumberVariable &))); QWidget::connect(_suppressNonSpeechTokens, SIGNAL(stateChanged(int)), this, SLOT(SuppressNonSpeechTokensChanged(int))); QWidget::connect(_noContext, SIGNAL(stateChanged(int)), this, SLOT(NoContextChanged(int))); QWidget::connect(_listenWhenMuted, SIGNAL(stateChanged(int)), this, SLOT(ListenWhenMutedChanged(int))); QWidget::connect(_useGpu, SIGNAL(stateChanged(int)), this, SLOT(UseGpuChanged(int))); _condSourceLayout = new QHBoxLayout; _phraseLayout = new QHBoxLayout; PlaceWidgets(obs_module_text( "AdvSceneSwitcher.condition.speech.layout.phrase"), _phraseLayout, {{"{{phrase}}", _phrase}, {"{{regex}}", _regex}}, false); auto *modelLayout = new QHBoxLayout; PlaceWidgets(obs_module_text( "AdvSceneSwitcher.condition.speech.layout.model"), modelLayout, {{"{{modelPath}}", _modelPath}, {"{{help}}", _modelHelp}}, false); auto *bufferLayout = new QHBoxLayout; PlaceWidgets(obs_module_text( "AdvSceneSwitcher.condition.speech.layout.buffer"), bufferLayout, {{"{{bufferDuration}}", _bufferDuration}, {"{{help}}", _bufferHelp}}); auto *threadsLayout = new QHBoxLayout; PlaceWidgets( obs_module_text( "AdvSceneSwitcher.condition.speech.layout.advanced.threads"), threadsLayout, {{"{{threads}}", _nThreads}}); auto *languageLayout = new QHBoxLayout; PlaceWidgets( obs_module_text( "AdvSceneSwitcher.condition.speech.layout.advanced.language"), languageLayout, {{"{{language}}", _language}, {"{{help}}", _languageHelp}}); auto *translateLayout = new QHBoxLayout; PlaceWidgets( obs_module_text( "AdvSceneSwitcher.condition.speech.layout.advanced.translate"), translateLayout, {{"{{translate}}", _translate}, {"{{help}}", _translateHelp}}); auto *vadLayout = new QHBoxLayout; PlaceWidgets( obs_module_text( "AdvSceneSwitcher.condition.speech.layout.advanced.vad"), vadLayout, {{"{{vad}}", _vadEnergyThreshold}, {"{{help}}", _vadHelp}}); auto *suppressLayout = new QHBoxLayout; PlaceWidgets( obs_module_text( "AdvSceneSwitcher.condition.speech.layout.advanced.suppress"), suppressLayout, {{"{{suppress}}", _suppressNonSpeechTokens}, {"{{help}}", _suppressHelp}}); auto *noContextLayout = new QHBoxLayout; PlaceWidgets( obs_module_text( "AdvSceneSwitcher.condition.speech.layout.advanced.noContext"), noContextLayout, {{"{{noContext}}", _noContext}, {"{{help}}", _noContextHelp}}); auto *advancedContent = new QWidget(this); auto *advancedLayout = new QVBoxLayout; advancedLayout->addLayout(threadsLayout); advancedLayout->addLayout(languageLayout); advancedLayout->addLayout(translateLayout); advancedLayout->addLayout(vadLayout); advancedLayout->addLayout(suppressLayout); advancedLayout->addLayout(noContextLayout); advancedLayout->addWidget(_listenWhenMuted); advancedLayout->addWidget(_useGpu); advancedContent->setLayout(advancedLayout); _advancedSection->AddHeaderWidget(new QLabel( obs_module_text("AdvSceneSwitcher.condition.speech.advanced"), this)); _advancedSection->SetContent(advancedContent, true); auto *mainLayout = new QVBoxLayout; mainLayout->addLayout(_condSourceLayout); mainLayout->addLayout(_phraseLayout); mainLayout->addLayout(modelLayout); mainLayout->addLayout(bufferLayout); mainLayout->addWidget(_advancedSection); setLayout(mainLayout); _entryData = entryData; UpdateEntryData(); _loading = false; } void MacroConditionSpeechEdit::UpdateEntryData() { if (!_entryData) { return; } _source->SetSource(_entryData->_source); _conditions->setCurrentIndex( static_cast(_entryData->GetCondition())); _phrase->setText(QString::fromStdString(_entryData->_phrase)); _regex->SetRegexConfig(_entryData->_regex); _modelPath->SetPath(_entryData->GetModelPath()); _bufferDuration->SetValue(_entryData->GetBufferDuration()); _nThreads->SetValue(_entryData->GetNThreads()); _language->setText(QString::fromStdString(_entryData->GetLanguage())); _translate->setChecked(_entryData->GetTranslate()); _vadEnergyThreshold->SetValue(_entryData->GetVadEnergyThreshold()); _suppressNonSpeechTokens->setChecked( _entryData->GetSuppressNonSpeechTokens()); _noContext->setChecked(_entryData->GetNoContext()); _listenWhenMuted->setChecked(_entryData->GetListenWhenMuted()); _useGpu->setChecked(_entryData->GetUseGpu()); SetWidgetVisibility(); } void MacroConditionSpeechEdit::SourceChanged(const SourceSelection &source) { GUARD_LOADING_AND_LOCK(); _entryData->_source = source; _entryData->RebuildRecognizer(); emit HeaderInfoChanged( QString::fromStdString(_entryData->GetShortDesc())); } void MacroConditionSpeechEdit::ConditionChanged(int idx) { { GUARD_LOADING_AND_LOCK(); _entryData->SetCondition( static_cast( _conditions->itemData(idx).toInt())); } SetWidgetVisibility(); } void MacroConditionSpeechEdit::PhraseChanged() { GUARD_LOADING_AND_LOCK(); _entryData->_phrase = _phrase->text().toStdString(); } void MacroConditionSpeechEdit::RegexChanged(const RegexConfig &conf) { GUARD_LOADING_AND_LOCK(); _entryData->_regex = conf; } void MacroConditionSpeechEdit::ModelPathChanged(const QString &path) { GUARD_LOADING_AND_LOCK(); _entryData->SetModelPath(path.toStdString()); } void MacroConditionSpeechEdit::BufferDurationChanged( const NumberVariable &value) { GUARD_LOADING_AND_LOCK(); _entryData->SetBufferDuration(value); } void MacroConditionSpeechEdit::NThreadsChanged(const NumberVariable &value) { GUARD_LOADING_AND_LOCK(); _entryData->SetNThreads(value); } void MacroConditionSpeechEdit::LanguageChanged() { GUARD_LOADING_AND_LOCK(); _entryData->SetLanguage(_language->text().toStdString()); } void MacroConditionSpeechEdit::TranslateChanged(int state) { GUARD_LOADING_AND_LOCK(); _entryData->SetTranslate(state == Qt::Checked); } void MacroConditionSpeechEdit::VadEnergyThresholdChanged( const NumberVariable &value) { GUARD_LOADING_AND_LOCK(); _entryData->SetVadEnergyThreshold(value); } void MacroConditionSpeechEdit::SuppressNonSpeechTokensChanged(int state) { GUARD_LOADING_AND_LOCK(); _entryData->SetSuppressNonSpeechTokens(state == Qt::Checked); } void MacroConditionSpeechEdit::NoContextChanged(int state) { GUARD_LOADING_AND_LOCK(); _entryData->SetNoContext(state == Qt::Checked); } void MacroConditionSpeechEdit::ListenWhenMutedChanged(int state) { GUARD_LOADING_AND_LOCK(); _entryData->SetListenWhenMuted(state == Qt::Checked); } void MacroConditionSpeechEdit::UseGpuChanged(int state) { GUARD_LOADING_AND_LOCK(); _entryData->SetUseGpu(state == Qt::Checked); } void MacroConditionSpeechEdit::SetWidgetVisibility() { const auto condition = _entryData->GetCondition(); const bool hasPhrase = condition != MacroConditionSpeech::Condition::ANY; _condSourceLayout->removeWidget(_conditions); _condSourceLayout->removeWidget(_source); ClearLayout(_condSourceLayout); const char *layoutKey = "AdvSceneSwitcher.condition.speech.layout.any"; if (condition == MacroConditionSpeech::Condition::CONTAINS) { layoutKey = "AdvSceneSwitcher.condition.speech.layout.contains"; } else if (condition == MacroConditionSpeech::Condition::MATCHES) { layoutKey = "AdvSceneSwitcher.condition.speech.layout.matches"; } PlaceWidgets(obs_module_text(layoutKey), _condSourceLayout, {{"{{conditions}}", _conditions}, {"{{source}}", _source}}); SetLayoutVisible(_phraseLayout, hasPhrase); _regex->setVisible(condition == MacroConditionSpeech::Condition::MATCHES); adjustSize(); updateGeometry(); } } // namespace advss