From a0550ca14173de3f4d63eb1fbcb09052656f6a6d Mon Sep 17 00:00:00 2001 From: WarmUpTill Date: Wed, 15 Feb 2023 22:07:19 +0100 Subject: [PATCH] Add support for TM_SQDIFF_NORMED and TM_CCOEFF_NORMED for pattern match --- data/locale/en-US.ini | 5 ++ .../video/macro-condition-video.cpp | 60 +++++++++++++++++-- .../video/macro-condition-video.hpp | 3 + src/macro-external/video/opencv-helpers.cpp | 57 ++++++++++++------ src/macro-external/video/opencv-helpers.hpp | 7 ++- .../video/paramerter-wrappers.cpp | 8 +++ .../video/paramerter-wrappers.hpp | 1 + src/macro-external/video/preview-dialog.cpp | 3 +- 8 files changed, 118 insertions(+), 26 deletions(-) diff --git a/data/locale/en-US.ini b/data/locale/en-US.ini index aedef7c3..4b36f12b 100644 --- a/data/locale/en-US.ini +++ b/data/locale/en-US.ini @@ -189,6 +189,11 @@ AdvSceneSwitcher.condition.video.usePatternForChangedCheck.tooltip="This will al AdvSceneSwitcher.condition.video.patternThreshold="Threshold: " AdvSceneSwitcher.condition.video.patternThresholdDescription="A higher threshold value means that the pattern needs to match the video source more closely." AdvSceneSwitcher.condition.video.patternThresholdUseAlphaAsMask="Use alpha channel as mask for pattern." +AdvSceneSwitcher.condition.video.patternMatchMode="Use pattern matching mode {{patternMatchingModes}}" +AdvSceneSwitcher.condition.video.patternMatchMode.crossCorrelation="Cross correlation" +AdvSceneSwitcher.condition.video.patternMatchMode.correlationCoefficient="Correlation coefficient" +AdvSceneSwitcher.condition.video.patternMatchMode.squaredDifference="Squared difference" +AdvSceneSwitcher.condition.video.patternMatchMode.tip="The best method to use depends on the specific problem and the characteristics of the images involved.\nIn practice, it's often necessary to experiment with different methods and parameters to find the best match for a particular use case." AdvSceneSwitcher.condition.video.brightnessThreshold="Average brightness is above:" AdvSceneSwitcher.condition.video.brightnessThresholdDescription="A high value is indicating a bright image and a low one a darker one." AdvSceneSwitcher.condition.video.currentBrightness="Current average brightness: %1" diff --git a/src/macro-external/video/macro-condition-video.cpp b/src/macro-external/video/macro-condition-video.cpp index fa0c923c..517bb9f6 100644 --- a/src/macro-external/video/macro-condition-video.cpp +++ b/src/macro-external/video/macro-condition-video.cpp @@ -50,6 +50,15 @@ const static std::map videoInputTypes = { "AdvSceneSwitcher.condition.video.type.scene"}, }; +const static std::map patternMatchModes = { + {cv::TemplateMatchModes::TM_CCOEFF_NORMED, + "AdvSceneSwitcher.condition.video.patternMatchMode.correlationCoefficient"}, + {cv::TemplateMatchModes::TM_CCORR_NORMED, + "AdvSceneSwitcher.condition.video.patternMatchMode.crossCorrelation"}, + {cv::TemplateMatchModes::TM_SQDIFF_NORMED, + "AdvSceneSwitcher.condition.video.patternMatchMode.squaredDifference"}, +}; + const static std::map pageSegModes = { {tesseract::PageSegMode::PSM_SINGLE_COLUMN, "AdvSceneSwitcher.condition.video.ocrMode.singleColumn"}, @@ -240,7 +249,8 @@ bool MacroConditionVideo::ScreenshotContainsPattern() cv::Mat result; matchPattern(_screenshotData.image, _patternImageData, _patternMatchParameters.threshold, result, - _patternMatchParameters.useAlphaAsMask); + _patternMatchParameters.useAlphaAsMask, + _patternMatchParameters.matchMode); return countNonZero(result) > 0; } @@ -251,7 +261,8 @@ bool MacroConditionVideo::OutputChanged() _patternImageData = createPatternData(_matchImage); matchPattern(_screenshotData.image, _patternImageData, _patternMatchParameters.threshold, result, - _patternMatchParameters.useAlphaAsMask); + _patternMatchParameters.useAlphaAsMask, + _patternMatchParameters.matchMode); return countNonZero(result) == 0; } return _screenshotData.image != _matchImage; @@ -357,6 +368,14 @@ static inline void populatePageSegModeSelection(QComboBox *list) } } +static inline void populatePatternMatchModeSelection(QComboBox *list) +{ + for (const auto &[mode, name] : patternMatchModes) { + list->addItem(obs_module_text(name.c_str()), + static_cast(mode)); + } +} + MacroConditionVideoEdit::MacroConditionVideoEdit( QWidget *parent, std::shared_ptr entryData) : QWidget(parent), @@ -378,6 +397,8 @@ MacroConditionVideoEdit::MacroConditionVideoEdit( "AdvSceneSwitcher.condition.video.patternThresholdDescription"))), _useAlphaAsMask(new QCheckBox(obs_module_text( "AdvSceneSwitcher.condition.video.patternThresholdUseAlphaAsMask"))), + _patternMatchModeLayout(new QHBoxLayout()), + _patternMatchMode(new QComboBox()), _brightnessThreshold(new SliderSpinBox( 0., 1., obs_module_text( @@ -425,6 +446,9 @@ MacroConditionVideoEdit::MacroConditionVideoEdit( _imagePath->Button()->disconnect(); _usePatternForChangedCheck->setToolTip(obs_module_text( "AdvSceneSwitcher.condition.video.usePatternForChangedCheck.tooltip")); + _patternMatchMode->setToolTip(obs_module_text( + "AdvSceneSwitcher.condition.video.patternMatchMode.tip")); + populatePatternMatchModeSelection(_patternMatchMode); _minNeighbors->setMinimum(minMinNeighbors); _minNeighbors->setMaximum(maxMinNeighbors); populatePageSegModeSelection(_pageSegMode); @@ -457,6 +481,8 @@ MacroConditionVideoEdit::MacroConditionVideoEdit( this, SLOT(PatternThresholdChanged(double))); QWidget::connect(_useAlphaAsMask, SIGNAL(stateChanged(int)), this, SLOT(UseAlphaAsMaskChanged(int))); + QWidget::connect(_patternMatchMode, SIGNAL(currentIndexChanged(int)), + this, SLOT(PatternMatchModeChanged(int))); QWidget::connect(_brightnessThreshold, SIGNAL(DoubleValueChanged(double)), this, SLOT(BrightnessThresholdChanged(double))); @@ -523,9 +549,14 @@ MacroConditionVideoEdit::MacroConditionVideoEdit( {"{{textColor}}", _textColor}, {"{{selectColor}}", _selectColor}, {"{{textType}}", _pageSegMode}, + {"{{patternMatchingModes}}", _patternMatchMode}, }; placeWidgets(obs_module_text("AdvSceneSwitcher.condition.video.entry"), entryLine1Layout, widgetPlaceholders); + placeWidgets( + obs_module_text( + "AdvSceneSwitcher.condition.video.patternMatchMode"), + _patternMatchModeLayout, widgetPlaceholders); placeWidgets( obs_module_text( "AdvSceneSwitcher.condition.video.entry.modelPath"), @@ -583,6 +614,7 @@ MacroConditionVideoEdit::MacroConditionVideoEdit( mainLayout->addWidget(_usePatternForChangedCheck); mainLayout->addWidget(_patternThreshold); mainLayout->addWidget(_useAlphaAsMask); + mainLayout->addLayout(_patternMatchModeLayout); mainLayout->addWidget(_brightnessThreshold); mainLayout->addWidget(_currentBrightness); mainLayout->addLayout(_ocrLayout); @@ -812,8 +844,7 @@ void MacroConditionVideoEdit::UsePatternForChangedCheckChanged(int value) std::lock_guard lock(GetSwitcher()->m); _entryData->_patternMatchParameters.useForChangedCheck = value; - _patternThreshold->setVisible(value); - adjustSize(); + SetWidgetVisibility(); } void MacroConditionVideoEdit::PatternThresholdChanged(double value) @@ -851,6 +882,20 @@ void MacroConditionVideoEdit::UseAlphaAsMaskChanged(int value) _entryData->_patternMatchParameters); } +void MacroConditionVideoEdit::PatternMatchModeChanged(int idx) +{ + if (_loading || !_entryData) { + return; + } + + std::lock_guard lock(GetSwitcher()->m); + _entryData->_patternMatchParameters.matchMode = + static_cast( + _patternMatchMode->itemData(idx).toInt()); + _previewDialog.PatternMatchParamtersChanged( + _entryData->_patternMatchParameters); +} + void MacroConditionVideoEdit::BrightnessThresholdChanged(double value) { if (_loading || !_entryData) { @@ -1132,6 +1177,8 @@ void MacroConditionVideoEdit::SetWidgetVisibility() _patternThreshold->setVisible(needsThreshold(_entryData->_condition)); _useAlphaAsMask->setVisible(_entryData->_condition == VideoCondition::PATTERN); + setLayoutVisible(_patternMatchModeLayout, + _entryData->_condition == VideoCondition::PATTERN); _brightnessThreshold->setVisible(_entryData->_condition == VideoCondition::BRIGHTNESS); _currentBrightness->setVisible(_entryData->_condition == @@ -1160,6 +1207,9 @@ void MacroConditionVideoEdit::SetWidgetVisibility() _entryData->_condition == VideoCondition::HAS_NOT_CHANGED) { _patternThreshold->setVisible( _entryData->_patternMatchParameters.useForChangedCheck); + setLayoutVisible( + _patternMatchModeLayout, + _entryData->_patternMatchParameters.useForChangedCheck); } adjustSize(); @@ -1197,6 +1247,8 @@ void MacroConditionVideoEdit::UpdateEntryData() _entryData->_patternMatchParameters.threshold); _useAlphaAsMask->setChecked( _entryData->_patternMatchParameters.useAlphaAsMask); + _patternMatchMode->setCurrentIndex(_patternMatchMode->findData( + _entryData->_patternMatchParameters.matchMode)); _brightnessThreshold->SetDoubleValue(_entryData->_brightnessThreshold); _modelDataPath->SetPath(_entryData->GetModelDataPath().c_str()); _objectScaleThreshold->SetDoubleValue( diff --git a/src/macro-external/video/macro-condition-video.hpp b/src/macro-external/video/macro-condition-video.hpp index 738c8b3e..cec7c30e 100644 --- a/src/macro-external/video/macro-condition-video.hpp +++ b/src/macro-external/video/macro-condition-video.hpp @@ -113,6 +113,7 @@ private slots: void UsePatternForChangedCheckChanged(int value); void PatternThresholdChanged(double); void UseAlphaAsMaskChanged(int value); + void PatternMatchModeChanged(int value); void BrightnessThresholdChanged(double); @@ -159,6 +160,8 @@ private: SliderSpinBox *_patternThreshold; QCheckBox *_useAlphaAsMask; + QHBoxLayout *_patternMatchModeLayout; + QComboBox *_patternMatchMode; SliderSpinBox *_brightnessThreshold; QLabel *_currentBrightness; diff --git a/src/macro-external/video/opencv-helpers.cpp b/src/macro-external/video/opencv-helpers.cpp index 9aac6bd5..a5f620b0 100644 --- a/src/macro-external/video/opencv-helpers.cpp +++ b/src/macro-external/video/opencv-helpers.cpp @@ -18,8 +18,19 @@ PatternImageData createPatternData(QImage &pattern) return data; } +static void invertPatternMatchResult(cv::Mat &mat) +{ + for (int r = 0; r < mat.rows; r++) { + for (int c = 0; c < mat.cols; c++) { + float value = mat.at(r, c) = + 1.0 - mat.at(r, c); + } + } +} + void matchPattern(QImage &img, const PatternImageData &patternData, - double threshold, cv::Mat &result, bool useAlphaAsMask) + double threshold, cv::Mat &result, bool useAlphaAsMask, + cv::TemplateMatchModes matchMode) { if (img.isNull() || patternData.rgbaPattern.empty()) { return; @@ -29,34 +40,44 @@ void matchPattern(QImage &img, const PatternImageData &patternData, return; } - auto i = QImageToMat(img); + auto input = QImageToMat(img); if (useAlphaAsMask) { - std::vector rgbaChannelsImage; - cv::split(i, rgbaChannelsImage); + // Remove alpha channel of input image as the alpha channel + // information is used as a stencil for the pattern instead and + // thus should not be used while matching the pattern as well + // + // Input format is Format_RGBA8888 so discard the 4th channel + std::vector inputChannels; + cv::split(input, inputChannels); std::vector rgbChanlesImage( - rgbaChannelsImage.begin(), - rgbaChannelsImage.begin() + 3); - - cv::Mat3b rgbImage; - cv::merge(rgbChanlesImage, rgbImage); - - cv::matchTemplate(rgbImage, patternData.rgbPattern, result, - cv::TM_CCORR_NORMED, patternData.mask); - cv::threshold(result, result, threshold, 0, cv::THRESH_TOZERO); + inputChannels.begin(), inputChannels.begin() + 3); + cv::Mat3b rgbInput; + cv::merge(rgbChanlesImage, rgbInput); + cv::matchTemplate(rgbInput, patternData.rgbPattern, result, + matchMode, patternData.mask); } else { - cv::matchTemplate(i, patternData.rgbaPattern, result, - cv::TM_CCOEFF_NORMED); - cv::threshold(result, result, threshold, 0, cv::THRESH_TOZERO); + cv::matchTemplate(input, patternData.rgbaPattern, result, + matchMode); } + + // A perfect match is represented as "0" for TM_SQDIFF_NORMED + // + // For TM_CCOEFF_NORMED and TM_CCORR_NORMED a perfect match is + // represented as "1" + if (matchMode == cv::TM_SQDIFF_NORMED) { + invertPatternMatchResult(result); + } + cv::threshold(result, result, threshold, 0.0, cv::THRESH_TOZERO); } void matchPattern(QImage &img, QImage &pattern, double threshold, - cv::Mat &result, bool useAlphaAsMask) + cv::Mat &result, bool useAlphaAsMask, + cv::TemplateMatchModes matchColor) { auto data = createPatternData(pattern); - matchPattern(img, data, threshold, result, useAlphaAsMask); + matchPattern(img, data, threshold, result, useAlphaAsMask, matchColor); } std::vector matchObject(QImage &img, cv::CascadeClassifier &cascade, diff --git a/src/macro-external/video/opencv-helpers.hpp b/src/macro-external/video/opencv-helpers.hpp index 2d69fce9..984134ae 100644 --- a/src/macro-external/video/opencv-helpers.hpp +++ b/src/macro-external/video/opencv-helpers.hpp @@ -47,10 +47,11 @@ struct PatternImageData { PatternImageData createPatternData(QImage &pattern); void matchPattern(QImage &img, const PatternImageData &patternData, - double threshold, cv::Mat &result, - bool useAlphaAsMask = true); + double threshold, cv::Mat &result, bool useAlphaAsMask, + cv::TemplateMatchModes matchMode); void matchPattern(QImage &img, QImage &pattern, double threshold, - cv::Mat &result, bool useAlphaAsMask); + cv::Mat &result, bool useAlphaAsMask, + cv::TemplateMatchModes matchMode); std::vector matchObject(QImage &img, cv::CascadeClassifier &cascade, double scaleFactor, int minNeighbors, cv::Size minSize, cv::Size maxSize); diff --git a/src/macro-external/video/paramerter-wrappers.cpp b/src/macro-external/video/paramerter-wrappers.cpp index 766c0399..940c23c2 100644 --- a/src/macro-external/video/paramerter-wrappers.cpp +++ b/src/macro-external/video/paramerter-wrappers.cpp @@ -6,6 +6,7 @@ bool PatternMatchParameters::Save(obs_data_t *obj) const obs_data_set_bool(data, "useForChangedCheck", useForChangedCheck); obs_data_set_double(data, "threshold", threshold); obs_data_set_bool(data, "useAlphaAsMask", useAlphaAsMask); + obs_data_set_int(data, "matchMode", matchMode); obs_data_set_obj(obj, "patternMatchData", data); obs_data_release(data); return true; @@ -25,6 +26,13 @@ bool PatternMatchParameters::Load(obs_data_t *obj) useForChangedCheck = obs_data_get_bool(data, "useForChangedCheck"); threshold = obs_data_get_double(data, "threshold"); useAlphaAsMask = obs_data_get_bool(data, "useAlphaAsMask"); + // TODO: Remove this fallback in a future version + if (!obs_data_has_user_value(obj, "matchMode")) { + matchMode = cv::TM_CCORR_NORMED; + } else { + matchMode = static_cast( + obs_data_get_int(data, "matchMode")); + } obs_data_release(data); return true; } diff --git a/src/macro-external/video/paramerter-wrappers.hpp b/src/macro-external/video/paramerter-wrappers.hpp index 31138dd0..b3c69805 100644 --- a/src/macro-external/video/paramerter-wrappers.hpp +++ b/src/macro-external/video/paramerter-wrappers.hpp @@ -54,6 +54,7 @@ public: QImage image; bool useForChangedCheck = false; bool useAlphaAsMask = false; + cv::TemplateMatchModes matchMode = cv::TM_CCORR_NORMED; double threshold = 0.8; }; diff --git a/src/macro-external/video/preview-dialog.cpp b/src/macro-external/video/preview-dialog.cpp index 8710c7d4..6e2fe92c 100644 --- a/src/macro-external/video/preview-dialog.cpp +++ b/src/macro-external/video/preview-dialog.cpp @@ -294,7 +294,8 @@ void PreviewImage::MarkMatch(QImage &screenshot, cv::Mat result; matchPattern(screenshot, patternImageData, patternMatchParams.threshold, result, - patternMatchParams.useAlphaAsMask); + patternMatchParams.useAlphaAsMask, + patternMatchParams.matchMode); if (countNonZero(result) == 0) { emit StatusUpdate(obs_module_text( "AdvSceneSwitcher.condition.video.patternMatchFail"));