From 3d1c613f1c58d667bae57a38216a0abcbde13e88 Mon Sep 17 00:00:00 2001 From: WarmUpTill Date: Sat, 4 Sep 2021 02:45:11 +0200 Subject: [PATCH] Add pattern match using openCV --- CMakeLists.txt | 9 +- data/locale/en-US.ini | 6 + src/headers/macro-condition-video.hpp | 27 ++++ src/macro-condition-video.cpp | 218 +++++++++++++++++++++++--- 4 files changed, 240 insertions(+), 20 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3a7a9320..47a20fea 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -164,6 +164,11 @@ if(WIN32) add_definitions(-D_WEBSOCKETPP_CPP11_STL_) endif() +# opencv +find_library(OPENCV_CORE opencv_core453d PATHS deps/opencv/build/install/x64/vc16/lib PATH_SUFFIXES lib) +find_library(OPENCV_IMGPROC opencv_imgproc453d PATHS deps/opencv/build/install/x64/vc16/lib PATH_SUFFIXES lib) +include_directories(deps/opencv/build/install/include) + # Setup QT tools set(CMAKE_AUTOMOC ON) set(CMAKE_AUTOUIC ON) @@ -352,7 +357,7 @@ add_library( # Out of tree build if(BUILD_OUT_OF_TREE) target_link_libraries( - advanced-scene-switcher ${advanced-scene-switcher_PLATFORM_LIBS} + advanced-scene-switcher ${advanced-scene-switcher_PLATFORM_LIBS} ${OPENCV_CORE} ${OPENCV_IMGPROC} ${LIBOBS_LIB} ${LIBOBS_FRONTEND_API_LIB} Qt5::Core Qt5::Widgets) # Additional commands to install the module in the correct place. Find all the @@ -383,7 +388,7 @@ if(BUILD_OUT_OF_TREE) else() # In tree build target_link_libraries( - advanced-scene-switcher ${advanced-scene-switcher_PLATFORM_LIBS} + advanced-scene-switcher ${advanced-scene-switcher_PLATFORM_LIBS} ${OPENCV_CORE} ${OPENCV_IMGPROC} obs-frontend-api Qt5::Widgets libobs) install_obs_plugin_with_data(advanced-scene-switcher data) endif() diff --git a/data/locale/en-US.ini b/data/locale/en-US.ini index 0bd1b6fd..0a7f77b3 100644 --- a/data/locale/en-US.ini +++ b/data/locale/en-US.ini @@ -113,9 +113,15 @@ AdvSceneSwitcher.condition.video.condition.differ="does not match" AdvSceneSwitcher.condition.video.condition.hasChanged="has changed" AdvSceneSwitcher.condition.video.condition.hasNotChanged="has not changed" AdvSceneSwitcher.condition.video.condition.noImage="has no output" +AdvSceneSwitcher.condition.video.condition.pattern="matches pattern" AdvSceneSwitcher.condition.video.askFileAction="Do you want to use an existing file or create a screenshot of the currently selected source?" AdvSceneSwitcher.condition.video.askFileAction.file="Use existing file" AdvSceneSwitcher.condition.video.askFileAction.screenshot="Create screenshot" +AdvSceneSwitcher.condition.video.threshold="Threshold: " +AdvSceneSwitcher.condition.video.thresholdDescription="A higher threshold value means that the pattern needs to match the video source more closely." +AdvSceneSwitcher.condition.video.showMatch="Show match" +AdvSceneSwitcher.condition.video.screenshotFail="Failed to get screenshot of source!" +AdvSceneSwitcher.condition.video.patternMatchFail="Pattern was not found!" AdvSceneSwitcher.condition.video.entry="{{videoSources}} {{condition}} {{filePath}} {{browseButton}}" AdvSceneSwitcher.condition.stream="Streaming" AdvSceneSwitcher.condition.stream.state.start="Stream running" diff --git a/src/headers/macro-condition-video.hpp b/src/headers/macro-condition-video.hpp index 2bee9bcf..c02fc676 100644 --- a/src/headers/macro-condition-video.hpp +++ b/src/headers/macro-condition-video.hpp @@ -12,6 +12,7 @@ enum class VideoCondition { HAS_NOT_CHANGED, HAS_CHANGED, NO_IMAGE, + PATTERN, }; class MacroConditionVideo : public MacroCondition { @@ -32,8 +33,10 @@ public: OBSWeakSource _videoSource; VideoCondition _condition = VideoCondition::MATCH; std::string _file = obs_module_text("AdvSceneSwitcher.enterPath"); + double _threshold = 0.8; private: + bool ScreenshotContainsPattern(); bool Compare(); std::unique_ptr _screenshotData = nullptr; @@ -42,6 +45,25 @@ private: static const std::string id; }; +class ThresholdSlider : public QWidget { + Q_OBJECT + +public: + ThresholdSlider(QWidget *parent = 0); + void SetDoubleValue(double); +public slots: + void NotifyValueChanged(int value); +signals: + void DoubleValueChanged(double value); + +private: + void SetDoubleValueText(double); + QLabel *_value; + QSlider *_slider; + double _scale = 100.0; + int _precision = 2; +}; + class MacroConditionVideoEdit : public QWidget { Q_OBJECT @@ -66,6 +88,8 @@ private slots: void ConditionChanged(int cond); void FilePathChanged(); void BrowseButtonClicked(); + void ThresholdChanged(double); + void ShowMatchClicked(); signals: void HeaderInfoChanged(const QString &); @@ -74,8 +98,11 @@ protected: QComboBox *_condition; QLineEdit *_filePath; QPushButton *_browseButton; + ThresholdSlider *_threshold; + QPushButton *_showMatch; std::shared_ptr _entryData; private: + void SetWidgetVisibility(); bool _loading = true; }; diff --git a/src/macro-condition-video.cpp b/src/macro-condition-video.cpp index 909cb517..2106e117 100644 --- a/src/macro-condition-video.cpp +++ b/src/macro-condition-video.cpp @@ -7,6 +7,7 @@ #include #include #include +#include const std::string MacroConditionVideo::id = "video"; @@ -26,11 +27,14 @@ static std::map conditionTypes = { "AdvSceneSwitcher.condition.video.condition.hasChanged"}, {VideoCondition::NO_IMAGE, "AdvSceneSwitcher.condition.video.condition.noImage"}, + {VideoCondition::PATTERN, + "AdvSceneSwitcher.condition.video.condition.pattern"}, }; bool requiresFileInput(VideoCondition t) { - return t == VideoCondition::MATCH || t == VideoCondition::DIFFER; + return t == VideoCondition::MATCH || t == VideoCondition::DIFFER || + t == VideoCondition::PATTERN; } bool MacroConditionVideo::CheckCondition() @@ -59,6 +63,7 @@ bool MacroConditionVideo::Save(obs_data_t *obj) GetWeakSourceName(_videoSource).c_str()); obs_data_set_int(obj, "condition", static_cast(_condition)); obs_data_set_string(obj, "filePath", _file.c_str()); + obs_data_set_double(obj, "threshold", _threshold); return true; } @@ -70,6 +75,7 @@ bool MacroConditionVideo::Load(obs_data_t *obj) _condition = static_cast(obs_data_get_int(obj, "condition")); _file = obs_data_get_string(obj, "filePath"); + _threshold = obs_data_get_double(obj, "threshold"); if (requiresFileInput(_condition)) { (void)LoadImageFromFile(); @@ -104,6 +110,50 @@ bool MacroConditionVideo::LoadImageFromFile() return true; } +// Assumption is that QImage uses Format_RGBX8888. +// Conversion from: https://github.com/dbzhang800/QtOpenCV +cv::Mat QImageToMat(const QImage &img) +{ + if (img.isNull()) { + return cv::Mat(); + } + return cv::Mat(img.height(), img.width(), CV_8UC(img.depth() / 8), + (uchar *)img.bits(), img.bytesPerLine()); +} + +QImage MatToQImage(const cv::Mat &mat) +{ + if (mat.empty()) { + return QImage(); + } + return QImage(mat.data, mat.cols, mat.rows, mat.step, + QImage::Format::Format_RGBX8888); +} + +void matchPattern(QImage &img, QImage &pattern, double threshold, + cv::Mat &result) +{ + if (img.isNull() || pattern.isNull()) { + return; + } + if (img.height() < pattern.height() || img.width() < pattern.width()) { + return; + } + + auto i = QImageToMat(img); + auto p = QImageToMat(pattern); + + cv::matchTemplate(i, p, result, cv::TM_CCOEFF_NORMED); + cv::threshold(result, result, threshold, 0, cv::THRESH_TOZERO); +} + +bool MacroConditionVideo::ScreenshotContainsPattern() +{ + cv::Mat result; + matchPattern(_screenshotData->image, _matchImage, _threshold, result); + return countNonZero(result) > 0; +} + bool MacroConditionVideo::Compare() { switch (_condition) { @@ -117,6 +167,8 @@ bool MacroConditionVideo::Compare() return _screenshotData->image == _matchImage; case VideoCondition::NO_IMAGE: return _screenshotData->image.isNull(); + case VideoCondition::PATTERN: + return ScreenshotContainsPattern(); default: break; } @@ -130,6 +182,53 @@ static inline void populateConditionSelection(QComboBox *list) } } +ThresholdSlider::ThresholdSlider(QWidget *parent) : QWidget(parent) +{ + _slider = new QSlider(); + _slider->setOrientation(Qt::Horizontal); + _slider->setRange(0, _scale); + _value = new QLabel(); + QString labelText = + obs_module_text("AdvSceneSwitcher.condition.video.threshold") + + QString("0."); + for (int i = 0; i < _precision; i++) { + labelText.append(QString("0")); + } + _value->setText(labelText); + connect(_slider, SIGNAL(valueChanged(int)), this, + SLOT(NotifyValueChanged(int))); + QVBoxLayout *mainLayout = new QVBoxLayout(); + QHBoxLayout *sliderLayout = new QHBoxLayout(); + sliderLayout->addWidget(_value); + sliderLayout->addWidget(_slider); + mainLayout->addLayout(sliderLayout); + mainLayout->addWidget(new QLabel(obs_module_text( + "AdvSceneSwitcher.condition.video.thresholdDescription"))); + mainLayout->setContentsMargins(0, 0, 0, 0); + setLayout(mainLayout); +} + +void ThresholdSlider::SetDoubleValue(double value) +{ + _slider->setValue(value * _scale); + SetDoubleValueText(value); +} + +void ThresholdSlider::NotifyValueChanged(int value) +{ + double doubleValue = value / _scale; + SetDoubleValueText(doubleValue); + emit DoubleValueChanged(doubleValue); +} + +void ThresholdSlider::SetDoubleValueText(double value) +{ + QString labelText = _value->text(); + labelText.chop(_precision + 2); // 2 for the part left of the "." + labelText.append(QString::number(value, 'f', _precision)); + _value->setText(labelText); +} + MacroConditionVideoEdit::MacroConditionVideoEdit( QWidget *parent, std::shared_ptr entryData) : QWidget(parent) @@ -139,9 +238,11 @@ MacroConditionVideoEdit::MacroConditionVideoEdit( _filePath = new QLineEdit(); _browseButton = new QPushButton(obs_module_text("AdvSceneSwitcher.browse")); + _threshold = new ThresholdSlider(); + _showMatch = new QPushButton( + obs_module_text("AdvSceneSwitcher.condition.video.showMatch")); _filePath->setFixedWidth(100); - _browseButton->setStyleSheet("border:1px solid gray;"); QWidget::connect(_videoSelection, @@ -153,11 +254,15 @@ MacroConditionVideoEdit::MacroConditionVideoEdit( SLOT(FilePathChanged())); QWidget::connect(_browseButton, SIGNAL(clicked()), this, SLOT(BrowseButtonClicked())); + QWidget::connect(_threshold, SIGNAL(DoubleValueChanged(double)), this, + SLOT(ThresholdChanged(double))); + QWidget::connect(_showMatch, SIGNAL(clicked()), this, + SLOT(ShowMatchClicked())); populateVideoSelection(_videoSelection); populateConditionSelection(_condition); - QHBoxLayout *mainLayout = new QHBoxLayout; + QHBoxLayout *entryLayout = new QHBoxLayout; std::unordered_map widgetPlaceholders = { {"{{videoSources}}", _videoSelection}, {"{{condition}}", _condition}, @@ -165,7 +270,14 @@ MacroConditionVideoEdit::MacroConditionVideoEdit( {"{{browseButton}}", _browseButton}, }; placeWidgets(obs_module_text("AdvSceneSwitcher.condition.video.entry"), - mainLayout, widgetPlaceholders); + entryLayout, widgetPlaceholders); + QHBoxLayout *showMatchLayout = new QHBoxLayout; + showMatchLayout->addWidget(_showMatch); + showMatchLayout->addStretch(); + QVBoxLayout *mainLayout = new QVBoxLayout; + mainLayout->addLayout(entryLayout); + mainLayout->addWidget(_threshold); + mainLayout->addLayout(showMatchLayout); setLayout(mainLayout); _entryData = entryData; @@ -217,6 +329,11 @@ void MacroConditionVideoEdit::SourceChanged(const QString &text) QString::fromStdString(_entryData->GetShortDesc())); } +bool needsThreshold(VideoCondition cond) +{ + return cond == VideoCondition::PATTERN; +} + void MacroConditionVideoEdit::ConditionChanged(int cond) { if (_loading || !_entryData) { @@ -225,14 +342,7 @@ void MacroConditionVideoEdit::ConditionChanged(int cond) std::lock_guard lock(switcher->m); _entryData->_condition = static_cast(cond); - - if (requiresFileInput(_entryData->_condition)) { - _filePath->show(); - _browseButton->show(); - } else { - _filePath->hide(); - _browseButton->hide(); - } + SetWidgetVisibility(); // Reload image data to avoid incorrect matches. // @@ -310,7 +420,8 @@ void MacroConditionVideoEdit::BrowseButtonClicked() std::this_thread::sleep_for(std::chrono::seconds(1)); } if (!screenshot->done) { - DisplayMessage("Failed to get screenshot of source!"); + DisplayMessage(obs_module_text( + "AdvSceneSwitcher.condition.video.screenshotFail")); return; } @@ -319,6 +430,80 @@ void MacroConditionVideoEdit::BrowseButtonClicked() SetFilePath(path); } +void MacroConditionVideoEdit::ThresholdChanged(double value) +{ + if (_loading || !_entryData) { + return; + } + + std::lock_guard lock(switcher->m); + _entryData->_threshold = value; +} + +QImage markPatterns(cv::Mat &matchResult, QImage &image, QImage &pattern) +{ + auto matchImg = QImageToMat(image); + for (int row = 0; row < matchResult.rows - 1; row++) { + for (int col = 0; col < matchResult.cols - 1; col++) { + if (matchResult.at(row, col) != 0.0) { + rectangle(matchImg, {col, row}, + cv::Point(col + pattern.width(), + row + pattern.height()), + cv::Scalar(255, 0, 0, 0), 2, 8, 0); + } + } + } + return MatToQImage(matchImg); +} + +void MacroConditionVideoEdit::ShowMatchClicked() +{ + auto source = obs_weak_source_get_source(_entryData->_videoSource); + auto screenshot = std::make_unique(source); + obs_source_release(source); + if (!screenshot->done) { + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + if (!screenshot->done) { + DisplayMessage(obs_module_text( + "AdvSceneSwitcher.condition.video.screenshotFail")); + return; + } + + cv::Mat result; + QImage pattern = _entryData->GetMatchImage(); + matchPattern(screenshot->image, pattern, _entryData->_threshold, + result); + + if (countNonZero(result) == 0) { + DisplayMessage(obs_module_text( + "AdvSceneSwitcher.condition.video.patternMatchFail")); + return; + } + + auto markedIamge = markPatterns(result, screenshot->image, pattern); + + QLabel *label = new QLabel; + label->setPixmap(QPixmap::fromImage(markedIamge)); + QVBoxLayout *layout = new QVBoxLayout; + layout->addWidget(label); + QDialog dialog; + dialog.setLayout(layout); + dialog.setWindowTitle("Advanced Scene Switcher"); + dialog.exec(); +} + +void MacroConditionVideoEdit::SetWidgetVisibility() +{ + bool showFileWidgets = requiresFileInput(_entryData->_condition); + _browseButton->setVisible(showFileWidgets); + _filePath->setVisible(showFileWidgets); + _threshold->setVisible(needsThreshold(_entryData->_condition)); + _showMatch->setVisible(_entryData->_condition == + VideoCondition::PATTERN); + adjustSize(); +} + void MacroConditionVideoEdit::UpdateEntryData() { if (!_entryData) { @@ -329,9 +514,6 @@ void MacroConditionVideoEdit::UpdateEntryData() GetWeakSourceName(_entryData->_videoSource).c_str()); _condition->setCurrentIndex(static_cast(_entryData->_condition)); _filePath->setText(QString::fromStdString(_entryData->_file)); - - if (!requiresFileInput(_entryData->_condition)) { - _filePath->hide(); - _browseButton->hide(); - } + _threshold->SetDoubleValue(_entryData->_threshold); + SetWidgetVisibility(); }