From 68c6492c3f4e5507a343bd821f7112610d32e430 Mon Sep 17 00:00:00 2001 From: WarmUpTill Date: Tue, 8 Aug 2023 21:35:36 +0200 Subject: [PATCH] Revert "Add OCL support to improve performance" This reverts commit 284c8020b26058259ec14b091bb1ac3790c3fae4. --- .../video/macro-condition-video.cpp | 9 +--- .../video/macro-condition-video.hpp | 2 +- src/macro-external/video/opencv-helpers.cpp | 53 ++++++++----------- src/macro-external/video/opencv-helpers.hpp | 13 +++-- src/macro-external/video/preview-dialog.cpp | 28 +++++----- src/macro-external/video/preview-dialog.hpp | 9 ++-- 6 files changed, 50 insertions(+), 64 deletions(-) diff --git a/src/macro-external/video/macro-condition-video.cpp b/src/macro-external/video/macro-condition-video.cpp index 745d8007..b5bb3e91 100644 --- a/src/macro-external/video/macro-condition-video.cpp +++ b/src/macro-external/video/macro-condition-video.cpp @@ -83,11 +83,6 @@ const static std::map pageSegModes = { "AdvSceneSwitcher.condition.video.ocrMode.sparseTextOSD"}, }; -MacroConditionVideo::MacroConditionVideo(Macro *m) : MacroCondition(m, true) -{ - SetupOpenCL(); -} - cv::CascadeClassifier initObjectCascade(std::string &path) { cv::CascadeClassifier cascade; @@ -266,7 +261,7 @@ bool MacroConditionVideo::SetLanguage(const std::string &language) bool MacroConditionVideo::ScreenshotContainsPattern() { - cv::UMat result; + cv::Mat result; MatchPattern(_screenshotData.image, _patternImageData, _patternMatchParameters.threshold, result, _patternMatchParameters.useAlphaAsMask, @@ -283,7 +278,7 @@ bool MacroConditionVideo::OutputChanged() return _screenshotData.image != _matchImage; } - cv::UMat result; + cv::Mat result; _patternImageData = CreatePatternData(_matchImage); MatchPattern(_screenshotData.image, _patternImageData, _patternMatchParameters.threshold, result, diff --git a/src/macro-external/video/macro-condition-video.hpp b/src/macro-external/video/macro-condition-video.hpp index 5c8aac9b..21590714 100644 --- a/src/macro-external/video/macro-condition-video.hpp +++ b/src/macro-external/video/macro-condition-video.hpp @@ -25,7 +25,7 @@ class PreviewDialog; class MacroConditionVideo : public MacroCondition { public: - MacroConditionVideo(Macro *m); + MacroConditionVideo(Macro *m) : MacroCondition(m, true){}; bool CheckCondition(); bool Save(obs_data_t *obj) const; bool Load(obs_data_t *obj); diff --git a/src/macro-external/video/opencv-helpers.cpp b/src/macro-external/video/opencv-helpers.cpp index 5dadf443..b37b5429 100644 --- a/src/macro-external/video/opencv-helpers.cpp +++ b/src/macro-external/video/opencv-helpers.cpp @@ -1,8 +1,6 @@ #include "opencv-helpers.hpp" -#include "log-helper.hpp" -#include -#include +#include namespace advss { @@ -24,19 +22,17 @@ PatternImageData CreatePatternData(const QImage &pattern) return data; } -static void invertPatternMatchResult(cv::UMat &umat) +static void invertPatternMatchResult(cv::Mat &mat) { - auto mat = umat.getMat(cv::ACCESS_RW); for (int r = 0; r < mat.rows; r++) { for (int c = 0; c < mat.cols; c++) { mat.at(r, c) = 1.0 - mat.at(r, c); } } - umat = mat.getUMat(cv::ACCESS_RW); } void MatchPattern(QImage &img, const PatternImageData &patternData, - double threshold, cv::UMat &result, bool useAlphaAsMask, + double threshold, cv::Mat &result, bool useAlphaAsMask, cv::TemplateMatchModes matchMode) { if (img.isNull() || patternData.rgbaPattern.empty()) { @@ -55,12 +51,13 @@ void MatchPattern(QImage &img, const PatternImageData &patternData, // thus should not be used while matching the pattern as well // // Input format is Format_RGBA8888 so discard the 4th channel - std::vector inputChannels; + std::vector inputChannels; cv::split(input, inputChannels); - std::vector rgbChanlesImage( + std::vector rgbChanlesImage( inputChannels.begin(), inputChannels.begin() + 3); - cv::UMat rgbInput; + cv::Mat3b rgbInput; cv::merge(rgbChanlesImage, rgbInput); + cv::matchTemplate(rgbInput, patternData.rgbPattern, result, matchMode, patternData.mask); } else { @@ -79,7 +76,7 @@ void MatchPattern(QImage &img, const PatternImageData &patternData, } void MatchPattern(QImage &img, QImage &pattern, double threshold, - cv::UMat &result, bool useAlphaAsMask, + cv::Mat &result, bool useAlphaAsMask, cv::TemplateMatchModes matchColor) { auto data = CreatePatternData(pattern); @@ -96,12 +93,16 @@ std::vector MatchObject(QImage &img, cv::CascadeClassifier &cascade, } auto image = QImageToMat(img); - cv::UMat frameGray; + cv::Mat frameGray; cv::cvtColor(image, frameGray, cv::COLOR_RGBA2GRAY); cv::equalizeHist(frameGray, frameGray); std::vector objects; - cascade.detectMultiScale(frameGray, objects, scaleFactor, minNeighbors, - 0, minSize, maxSize); + try { + cascade.detectMultiScale(frameGray, objects, scaleFactor, + minNeighbors, 0, minSize, maxSize); + } catch (const std::exception &e) { + vblog(LOG_INFO, "detectMultiScale failed: %s", e.what()); + } return objects; } @@ -111,9 +112,9 @@ uchar GetAvgBrightness(QImage &img) return 0; } - auto i = QImageToMat(img); + auto image = QImageToMat(img); cv::Mat hsvImage, rgbImage; - cv::cvtColor(i, rgbImage, cv::COLOR_RGBA2RGB); + cv::cvtColor(image, rgbImage, cv::COLOR_RGBA2RGB); cv::cvtColor(rgbImage, hsvImage, cv::COLOR_RGB2HSV); long long brightnessSum = 0; for (int i = 0; i < hsvImage.rows; ++i) { @@ -138,8 +139,7 @@ static bool colorIsSimilar(const QColor &color1, const QColor &color2, cv::Mat PreprocessForOCR(const QImage &image, const QColor &textColor, double colorDiff) { - auto umat = QImageToMat(image); - auto mat = umat.getMat(cv::ACCESS_RW); + auto mat = QImageToMat(image); // Tesseract works best when matching black text on a white background, // so everything that matches the text color will be displayed black @@ -224,14 +224,13 @@ bool ContainsPixelsInColorRange(const QImage &image, const QColor &color, // Assumption is that QImage uses Format_RGBA8888. // Conversion from: https://github.com/dbzhang800/QtOpenCV -cv::UMat QImageToMat(const QImage &img) +cv::Mat QImageToMat(const QImage &img) { if (img.isNull()) { - return cv::UMat(); + return cv::Mat(); } - auto temp = cv::Mat(img.height(), img.width(), CV_8UC(img.depth() / 8), - (uchar *)img.bits(), img.bytesPerLine()); - return temp.getUMat(cv::ACCESS_RW); + return cv::Mat(img.height(), img.width(), CV_8UC(img.depth() / 8), + (uchar *)img.bits(), img.bytesPerLine()); } QImage MatToQImage(const cv::Mat &mat) @@ -243,12 +242,4 @@ QImage MatToQImage(const cv::Mat &mat) QImage::Format::Format_RGBA8888); } -void SetupOpenCL() -{ - if (cv::ocl::haveOpenCL() && !cv::ocl::useOpenCL()) { - blog(LOG_INFO, "enabled OpenCL support for OpenCV"); - cv::ocl::setUseOpenCL(true); - } -} - } // namespace advss diff --git a/src/macro-external/video/opencv-helpers.hpp b/src/macro-external/video/opencv-helpers.hpp index 73c9f933..f973a9f3 100644 --- a/src/macro-external/video/opencv-helpers.hpp +++ b/src/macro-external/video/opencv-helpers.hpp @@ -42,17 +42,17 @@ constexpr int maxMinNeighbors = 6; constexpr double defaultScaleFactor = 1.1; struct PatternImageData { - cv::UMat rgbaPattern; - cv::UMat rgbPattern; - cv::UMat mask; + cv::Mat4b rgbaPattern; + cv::Mat3b rgbPattern; + cv::Mat1b mask; }; PatternImageData CreatePatternData(const QImage &pattern); void MatchPattern(QImage &img, const PatternImageData &patternData, - double threshold, cv::UMat &result, bool useAlphaAsMask, + double threshold, cv::Mat &result, bool useAlphaAsMask, cv::TemplateMatchModes matchMode); void MatchPattern(QImage &img, QImage &pattern, double threshold, - cv::UMat &result, bool useAlphaAsMask, + cv::Mat &result, bool useAlphaAsMask, cv::TemplateMatchModes matchMode); std::vector MatchObject(QImage &img, cv::CascadeClassifier &cascade, double scaleFactor, int minNeighbors, @@ -66,8 +66,7 @@ std::string RunOCR(tesseract::TessBaseAPI *, const QImage &, const QColor &, bool ContainsPixelsInColorRange(const QImage &image, const QColor &color, double colorDeviationThreshold, double totalPixelMatchThreshold); -cv::UMat QImageToMat(const QImage &img); +cv::Mat QImageToMat(const QImage &img); QImage MatToQImage(const cv::Mat &mat); -void SetupOpenCL(); } // namespace advss diff --git a/src/macro-external/video/preview-dialog.cpp b/src/macro-external/video/preview-dialog.cpp index e7ed92d3..c76765af 100644 --- a/src/macro-external/video/preview-dialog.cpp +++ b/src/macro-external/video/preview-dialog.cpp @@ -122,6 +122,7 @@ void PreviewDialog::PatternMatchParametersChanged( { std::unique_lock lock(_mtx); _patternMatchParams = params; + _patternImageData = CreatePatternData(_patternMatchParams.image); } void PreviewDialog::ObjDetectParametersChanged(const ObjDetectParameters ¶ms) @@ -169,8 +170,8 @@ void PreviewDialog::UpdateImage(const QPixmap &image) if (_type == PreviewType::SELECT_AREA && !_selectingArea) { DrawFrame(); } - emit NeedImage(_video, _type, _patternMatchParams, _objDetectParams, - _ocrParams, _areaParams, _condition); + emit NeedImage(_video, _type, _patternMatchParams, _patternImageData, + _objDetectParams, _ocrParams, _areaParams, _condition); } void PreviewDialog::Start() @@ -186,7 +187,7 @@ void PreviewDialog::Start() return; } - PreviewImage *worker = new PreviewImage(_mtx); + auto worker = new PreviewImage(_mtx); worker->moveToThread(&_thread); connect(&_thread, &QThread::finished, worker, &QObject::deleteLater); connect(worker, &PreviewImage::ImageReady, this, @@ -197,8 +198,8 @@ void PreviewDialog::Start() &PreviewImage::CreateImage); _thread.start(); - emit NeedImage(_video, _type, _patternMatchParams, _objDetectParams, - _ocrParams, _areaParams, _condition); + emit NeedImage(_video, _type, _patternMatchParams, _patternImageData, + _objDetectParams, _ocrParams, _areaParams, _condition); } void PreviewDialog::DrawFrame() @@ -216,14 +217,13 @@ void PreviewDialog::DrawFrame() _rubberBand->show(); } -static void markPatterns(cv::UMat &matchResult, QImage &image, - const cv::UMat &pattern) +static void markPatterns(cv::Mat &matchResult, QImage &image, + const cv::Mat &pattern) { - auto temp = matchResult.getMat(cv::ACCESS_RW); auto matchImg = QImageToMat(image); - for (int row = 0; row < temp.rows - 1; row++) { - for (int col = 0; col < temp.cols - 1; col++) { - if (temp.at(row, col) != 0.0) { + for (int row = 0; row < matchResult.rows - 1; row++) { + for (int col = 0; col < matchResult.cols - 1; col++) { + if (matchResult.at(row, col) != 0.0) { rectangle(matchImg, {col, row}, cv::Point(col + pattern.cols, row + pattern.rows), @@ -231,7 +231,6 @@ static void markPatterns(cv::UMat &matchResult, QImage &image, } } } - matchResult = temp.getUMat(cv::ACCESS_RW); } static void markObjects(QImage &image, std::vector &objects) @@ -249,6 +248,7 @@ PreviewImage::PreviewImage(std::mutex &mtx) : _mtx(mtx) {} void PreviewImage::CreateImage(const VideoInput &video, PreviewType type, const PatternMatchParameters &patternMatchParams, + const PatternImageData &patternImageData, ObjDetectParameters objDetectParams, OCRParameters ocrParams, const AreaParameters &areaParams, @@ -279,8 +279,6 @@ void PreviewImage::CreateImage(const VideoInput &video, PreviewType type, areaParams.area.x, areaParams.area.y, areaParams.area.width, areaParams.area.height); } - const auto patternImageData = - CreatePatternData(patternMatchParams.image); // Will emit status label update MarkMatch(screenshot.image, patternMatchParams, patternImageData, objDetectParams, ocrParams, @@ -299,7 +297,7 @@ void PreviewImage::MarkMatch(QImage &screenshot, VideoCondition condition) { if (condition == VideoCondition::PATTERN) { - cv::UMat result; + cv::Mat result; MatchPattern(screenshot, patternImageData, patternMatchParams.threshold, result, patternMatchParams.useAlphaAsMask, diff --git a/src/macro-external/video/preview-dialog.hpp b/src/macro-external/video/preview-dialog.hpp index f49e46f4..a55eef17 100644 --- a/src/macro-external/video/preview-dialog.hpp +++ b/src/macro-external/video/preview-dialog.hpp @@ -25,7 +25,8 @@ public: public slots: void CreateImage(const VideoInput &, PreviewType, - const PatternMatchParameters &, ObjDetectParameters, + const PatternMatchParameters &, + const PatternImageData &, ObjDetectParameters, OCRParameters, const AreaParameters &, VideoCondition); signals: void ImageReady(const QPixmap &); @@ -63,8 +64,9 @@ private slots: signals: void SelectionAreaChanged(QRect area); void NeedImage(const VideoInput &, PreviewType, - const PatternMatchParameters &, ObjDetectParameters, - OCRParameters, const AreaParameters &, VideoCondition); + const PatternMatchParameters &, const PatternImageData &, + ObjDetectParameters, OCRParameters, + const AreaParameters &, VideoCondition); private: void Start(); @@ -76,6 +78,7 @@ private: VideoInput _video; PatternMatchParameters _patternMatchParams; + PatternImageData _patternImageData; ObjDetectParameters _objDetectParams; OCRParameters _ocrParams; AreaParameters _areaParams;