Leahnaya/SceneSwitcher
1
0
Fork 0
mirror of https://github.com/WarmUpTill/SceneSwitcher.git synced 2026-03-21 17:34:57 -05:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Add support for TM_SQDIFF_NORMED and TM_CCOEFF_NORMED for pattern match

Browse Source
This commit is contained in:
WarmUpTill 2023-02-15 22:07:19 +01:00 committed by WarmUpTill
parent ac68fc016d
commit a0550ca141
8 changed files with 118 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
data/locale/en-US.ini
View File

@ -189,6 +189,11 @@ AdvSceneSwitcher.condition.video.usePatternForChangedCheck.tooltip="This will al
AdvSceneSwitcher.condition.video.patternThreshold="Threshold: "
AdvSceneSwitcher.condition.video.patternThresholdDescription="A higher threshold value means that the pattern needs to match the video source more closely."
AdvSceneSwitcher.condition.video.patternThresholdUseAlphaAsMask="Use alpha channel as mask for pattern."
AdvSceneSwitcher.condition.video.patternMatchMode="Use pattern matching mode {{patternMatchingModes}}"
AdvSceneSwitcher.condition.video.patternMatchMode.crossCorrelation="Cross correlation"
AdvSceneSwitcher.condition.video.patternMatchMode.correlationCoefficient="Correlation coefficient"
AdvSceneSwitcher.condition.video.patternMatchMode.squaredDifference="Squared difference"
AdvSceneSwitcher.condition.video.patternMatchMode.tip="The best method to use depends on the specific problem and the characteristics of the images involved.\nIn practice, it's often necessary to experiment with different methods and parameters to find the best match for a particular use case."
AdvSceneSwitcher.condition.video.brightnessThreshold="Average brightness is above:"
AdvSceneSwitcher.condition.video.brightnessThresholdDescription="A high value is indicating a bright image and a low one a darker one."
AdvSceneSwitcher.condition.video.currentBrightness="Current average brightness: %1"

60
src/macro-external/video/macro-condition-video.cpp
View File

@ -50,6 +50,15 @@ const static std::map<VideoInput::Type, std::string> videoInputTypes = {
"AdvSceneSwitcher.condition.video.type.scene"},
};
const static std::map<cv::TemplateMatchModes, std::string> patternMatchModes = {
{cv::TemplateMatchModes::TM_CCOEFF_NORMED,
"AdvSceneSwitcher.condition.video.patternMatchMode.correlationCoefficient"},
{cv::TemplateMatchModes::TM_CCORR_NORMED,
"AdvSceneSwitcher.condition.video.patternMatchMode.crossCorrelation"},
{cv::TemplateMatchModes::TM_SQDIFF_NORMED,
"AdvSceneSwitcher.condition.video.patternMatchMode.squaredDifference"},
};
const static std::map<tesseract::PageSegMode, std::string> pageSegModes = {
{tesseract::PageSegMode::PSM_SINGLE_COLUMN,
"AdvSceneSwitcher.condition.video.ocrMode.singleColumn"},
@ -240,7 +249,8 @@ bool MacroConditionVideo::ScreenshotContainsPattern()
cv::Mat result;
matchPattern(_screenshotData.image, _patternImageData,
_patternMatchParameters.threshold, result,
_patternMatchParameters.useAlphaAsMask);
_patternMatchParameters.useAlphaAsMask,
_patternMatchParameters.matchMode);
return countNonZero(result) > 0;
}
@ -251,7 +261,8 @@ bool MacroConditionVideo::OutputChanged()
_patternImageData = createPatternData(_matchImage);
matchPattern(_screenshotData.image, _patternImageData,
_patternMatchParameters.threshold, result,
_patternMatchParameters.useAlphaAsMask);
_patternMatchParameters.useAlphaAsMask,
_patternMatchParameters.matchMode);
return countNonZero(result) == 0;
}
return _screenshotData.image != _matchImage;
@ -357,6 +368,14 @@ static inline void populatePageSegModeSelection(QComboBox *list)
}
}
static inline void populatePatternMatchModeSelection(QComboBox *list)
{
for (const auto &[mode, name] : patternMatchModes) {
list->addItem(obs_module_text(name.c_str()),
static_cast<int>(mode));
}
}
MacroConditionVideoEdit::MacroConditionVideoEdit(
QWidget *parent, std::shared_ptr<MacroConditionVideo> entryData)
: QWidget(parent),
@ -378,6 +397,8 @@ MacroConditionVideoEdit::MacroConditionVideoEdit(
"AdvSceneSwitcher.condition.video.patternThresholdDescription"))),
_useAlphaAsMask(new QCheckBox(obs_module_text(
"AdvSceneSwitcher.condition.video.patternThresholdUseAlphaAsMask"))),
_patternMatchModeLayout(new QHBoxLayout()),
_patternMatchMode(new QComboBox()),
_brightnessThreshold(new SliderSpinBox(
0., 1.,
obs_module_text(
@ -425,6 +446,9 @@ MacroConditionVideoEdit::MacroConditionVideoEdit(
_imagePath->Button()->disconnect();
_usePatternForChangedCheck->setToolTip(obs_module_text(
"AdvSceneSwitcher.condition.video.usePatternForChangedCheck.tooltip"));
_patternMatchMode->setToolTip(obs_module_text(
"AdvSceneSwitcher.condition.video.patternMatchMode.tip"));
populatePatternMatchModeSelection(_patternMatchMode);
_minNeighbors->setMinimum(minMinNeighbors);
_minNeighbors->setMaximum(maxMinNeighbors);
populatePageSegModeSelection(_pageSegMode);
@ -457,6 +481,8 @@ MacroConditionVideoEdit::MacroConditionVideoEdit(
this, SLOT(PatternThresholdChanged(double)));
QWidget::connect(_useAlphaAsMask, SIGNAL(stateChanged(int)), this,
SLOT(UseAlphaAsMaskChanged(int)));
QWidget::connect(_patternMatchMode, SIGNAL(currentIndexChanged(int)),
this, SLOT(PatternMatchModeChanged(int)));
QWidget::connect(_brightnessThreshold,
SIGNAL(DoubleValueChanged(double)), this,
SLOT(BrightnessThresholdChanged(double)));
@ -523,9 +549,14 @@ MacroConditionVideoEdit::MacroConditionVideoEdit(
{"{{textColor}}", _textColor},
{"{{selectColor}}", _selectColor},
{"{{textType}}", _pageSegMode},
{"{{patternMatchingModes}}", _patternMatchMode},
};
placeWidgets(obs_module_text("AdvSceneSwitcher.condition.video.entry"),
entryLine1Layout, widgetPlaceholders);
placeWidgets(
obs_module_text(
"AdvSceneSwitcher.condition.video.patternMatchMode"),
_patternMatchModeLayout, widgetPlaceholders);
placeWidgets(
obs_module_text(
"AdvSceneSwitcher.condition.video.entry.modelPath"),
@ -583,6 +614,7 @@ MacroConditionVideoEdit::MacroConditionVideoEdit(
mainLayout->addWidget(_usePatternForChangedCheck);
mainLayout->addWidget(_patternThreshold);
mainLayout->addWidget(_useAlphaAsMask);
mainLayout->addLayout(_patternMatchModeLayout);
mainLayout->addWidget(_brightnessThreshold);
mainLayout->addWidget(_currentBrightness);
mainLayout->addLayout(_ocrLayout);
@ -812,8 +844,7 @@ void MacroConditionVideoEdit::UsePatternForChangedCheckChanged(int value)
std::lock_guard<std::mutex> lock(GetSwitcher()->m);
_entryData->_patternMatchParameters.useForChangedCheck = value;
_patternThreshold->setVisible(value);
adjustSize();
SetWidgetVisibility();
}
void MacroConditionVideoEdit::PatternThresholdChanged(double value)
@ -851,6 +882,20 @@ void MacroConditionVideoEdit::UseAlphaAsMaskChanged(int value)
_entryData->_patternMatchParameters);
}
void MacroConditionVideoEdit::PatternMatchModeChanged(int idx)
{
if (_loading || !_entryData) {
return;
}
std::lock_guard<std::mutex> lock(GetSwitcher()->m);
_entryData->_patternMatchParameters.matchMode =
static_cast<cv::TemplateMatchModes>(
_patternMatchMode->itemData(idx).toInt());
_previewDialog.PatternMatchParamtersChanged(
_entryData->_patternMatchParameters);
}
void MacroConditionVideoEdit::BrightnessThresholdChanged(double value)
{
if (_loading || !_entryData) {
@ -1132,6 +1177,8 @@ void MacroConditionVideoEdit::SetWidgetVisibility()
_patternThreshold->setVisible(needsThreshold(_entryData->_condition));
_useAlphaAsMask->setVisible(_entryData->_condition ==
VideoCondition::PATTERN);
setLayoutVisible(_patternMatchModeLayout,
_entryData->_condition == VideoCondition::PATTERN);
_brightnessThreshold->setVisible(_entryData->_condition ==
VideoCondition::BRIGHTNESS);
_currentBrightness->setVisible(_entryData->_condition ==
@ -1160,6 +1207,9 @@ void MacroConditionVideoEdit::SetWidgetVisibility()
_entryData->_condition == VideoCondition::HAS_NOT_CHANGED) {
_patternThreshold->setVisible(
_entryData->_patternMatchParameters.useForChangedCheck);
setLayoutVisible(
_patternMatchModeLayout,
_entryData->_patternMatchParameters.useForChangedCheck);
}
adjustSize();
@ -1197,6 +1247,8 @@ void MacroConditionVideoEdit::UpdateEntryData()
_entryData->_patternMatchParameters.threshold);
_useAlphaAsMask->setChecked(
_entryData->_patternMatchParameters.useAlphaAsMask);
_patternMatchMode->setCurrentIndex(_patternMatchMode->findData(
_entryData->_patternMatchParameters.matchMode));
_brightnessThreshold->SetDoubleValue(_entryData->_brightnessThreshold);
_modelDataPath->SetPath(_entryData->GetModelDataPath().c_str());
_objectScaleThreshold->SetDoubleValue(

3
src/macro-external/video/macro-condition-video.hpp
View File

@ -113,6 +113,7 @@ private slots:
void UsePatternForChangedCheckChanged(int value);
void PatternThresholdChanged(double);
void UseAlphaAsMaskChanged(int value);
void PatternMatchModeChanged(int value);
void BrightnessThresholdChanged(double);
@ -159,6 +160,8 @@ private:
SliderSpinBox *_patternThreshold;
QCheckBox *_useAlphaAsMask;
QHBoxLayout *_patternMatchModeLayout;
QComboBox *_patternMatchMode;
SliderSpinBox *_brightnessThreshold;
QLabel *_currentBrightness;

57
src/macro-external/video/opencv-helpers.cpp
View File

@ -18,8 +18,19 @@ PatternImageData createPatternData(QImage &pattern)
return data;
}
static void invertPatternMatchResult(cv::Mat &mat)
{
for (int r = 0; r < mat.rows; r++) {
for (int c = 0; c < mat.cols; c++) {
float value = mat.at<float>(r, c) =
1.0 - mat.at<float>(r, c);
}
}
}
void matchPattern(QImage &img, const PatternImageData &patternData,
double threshold, cv::Mat &result, bool useAlphaAsMask)
double threshold, cv::Mat &result, bool useAlphaAsMask,
cv::TemplateMatchModes matchMode)
{
if (img.isNull() || patternData.rgbaPattern.empty()) {
return;
@ -29,34 +40,44 @@ void matchPattern(QImage &img, const PatternImageData &patternData,
return;
}
auto i = QImageToMat(img);
auto input = QImageToMat(img);
if (useAlphaAsMask) {
std::vector<cv::Mat1b> rgbaChannelsImage;
cv::split(i, rgbaChannelsImage);
// Remove alpha channel of input image as the alpha channel
// information is used as a stencil for the pattern instead and
// thus should not be used while matching the pattern as well
//
// Input format is Format_RGBA8888 so discard the 4th channel
std::vector<cv::Mat1b> inputChannels;
cv::split(input, inputChannels);
std::vector<cv::Mat1b> rgbChanlesImage(
rgbaChannelsImage.begin(),
rgbaChannelsImage.begin() + 3);
cv::Mat3b rgbImage;
cv::merge(rgbChanlesImage, rgbImage);
cv::matchTemplate(rgbImage, patternData.rgbPattern, result,
cv::TM_CCORR_NORMED, patternData.mask);
cv::threshold(result, result, threshold, 0, cv::THRESH_TOZERO);
inputChannels.begin(), inputChannels.begin() + 3);
cv::Mat3b rgbInput;
cv::merge(rgbChanlesImage, rgbInput);
cv::matchTemplate(rgbInput, patternData.rgbPattern, result,
matchMode, patternData.mask);
} else {
cv::matchTemplate(i, patternData.rgbaPattern, result,
cv::TM_CCOEFF_NORMED);
cv::threshold(result, result, threshold, 0, cv::THRESH_TOZERO);
cv::matchTemplate(input, patternData.rgbaPattern, result,
matchMode);
}
// A perfect match is represented as "0" for TM_SQDIFF_NORMED
//
// For TM_CCOEFF_NORMED and TM_CCORR_NORMED a perfect match is
// represented as "1"
if (matchMode == cv::TM_SQDIFF_NORMED) {
invertPatternMatchResult(result);
}
cv::threshold(result, result, threshold, 0.0, cv::THRESH_TOZERO);
}
void matchPattern(QImage &img, QImage &pattern, double threshold,
cv::Mat &result, bool useAlphaAsMask)
cv::Mat &result, bool useAlphaAsMask,
cv::TemplateMatchModes matchColor)
{
auto data = createPatternData(pattern);
matchPattern(img, data, threshold, result, useAlphaAsMask);
matchPattern(img, data, threshold, result, useAlphaAsMask, matchColor);
}
std::vector<cv::Rect> matchObject(QImage &img, cv::CascadeClassifier &cascade,

7
src/macro-external/video/opencv-helpers.hpp
View File

@ -47,10 +47,11 @@ struct PatternImageData {
PatternImageData createPatternData(QImage &pattern);
void matchPattern(QImage &img, const PatternImageData &patternData,
double threshold, cv::Mat &result,
bool useAlphaAsMask = true);
double threshold, cv::Mat &result, bool useAlphaAsMask,
cv::TemplateMatchModes matchMode);
void matchPattern(QImage &img, QImage &pattern, double threshold,
cv::Mat &result, bool useAlphaAsMask);
cv::Mat &result, bool useAlphaAsMask,
cv::TemplateMatchModes matchMode);
std::vector<cv::Rect> matchObject(QImage &img, cv::CascadeClassifier &cascade,
double scaleFactor, int minNeighbors,
cv::Size minSize, cv::Size maxSize);

8
src/macro-external/video/paramerter-wrappers.cpp
View File

@ -6,6 +6,7 @@ bool PatternMatchParameters::Save(obs_data_t *obj) const
obs_data_set_bool(data, "useForChangedCheck", useForChangedCheck);
obs_data_set_double(data, "threshold", threshold);
obs_data_set_bool(data, "useAlphaAsMask", useAlphaAsMask);
obs_data_set_int(data, "matchMode", matchMode);
obs_data_set_obj(obj, "patternMatchData", data);
obs_data_release(data);
return true;
@ -25,6 +26,13 @@ bool PatternMatchParameters::Load(obs_data_t *obj)
useForChangedCheck = obs_data_get_bool(data, "useForChangedCheck");
threshold = obs_data_get_double(data, "threshold");
useAlphaAsMask = obs_data_get_bool(data, "useAlphaAsMask");
// TODO: Remove this fallback in a future version
if (!obs_data_has_user_value(obj, "matchMode")) {
matchMode = cv::TM_CCORR_NORMED;
} else {
matchMode = static_cast<cv::TemplateMatchModes>(
obs_data_get_int(data, "matchMode"));
}
obs_data_release(data);
return true;
}

1
src/macro-external/video/paramerter-wrappers.hpp
View File

@ -54,6 +54,7 @@ public:
QImage image;
bool useForChangedCheck = false;
bool useAlphaAsMask = false;
cv::TemplateMatchModes matchMode = cv::TM_CCORR_NORMED;
double threshold = 0.8;
};

3
src/macro-external/video/preview-dialog.cpp
View File

@ -294,7 +294,8 @@ void PreviewImage::MarkMatch(QImage &screenshot,
cv::Mat result;
matchPattern(screenshot, patternImageData,
patternMatchParams.threshold, result,
patternMatchParams.useAlphaAsMask);
patternMatchParams.useAlphaAsMask,
patternMatchParams.matchMode);
if (countNonZero(result) == 0) {
emit StatusUpdate(obs_module_text(
"AdvSceneSwitcher.condition.video.patternMatchFail"));