From 721a786e7978da8e673131e944f4e6d094650feb Mon Sep 17 00:00:00 2001 From: WarmUpTill <19472752+WarmUpTill@users.noreply.github.com> Date: Fri, 23 May 2025 20:23:22 +0200 Subject: [PATCH] Rework preview dialog to improve performance Object detection and OCR models were constantly being re-initialized each frame due to copy in the signal / slot handling --- plugins/video/preview-dialog.cpp | 151 +++++++++++++++++++------------ plugins/video/preview-dialog.hpp | 25 +++-- 2 files changed, 109 insertions(+), 67 deletions(-) diff --git a/plugins/video/preview-dialog.cpp b/plugins/video/preview-dialog.cpp index c48b47ff..3383fa9d 100644 --- a/plugins/video/preview-dialog.cpp +++ b/plugins/video/preview-dialog.cpp @@ -134,13 +134,13 @@ void PreviewDialog::PatternMatchParametersChanged( void PreviewDialog::ObjDetectParametersChanged(const ObjDetectParameters ¶ms) { std::unique_lock lock(_mtx); - _objDetectParams = params; + _objDetectParams = std::make_shared(params); } void PreviewDialog::OCRParametersChanged(const OCRParameters ¶ms) { std::unique_lock lock(_mtx); - _ocrParams = params; + _ocrParams = std::make_shared(params); } void PreviewDialog::VideoSelectionChanged(const VideoInput &video) @@ -262,13 +262,13 @@ static void markObjects(QImage &image, std::vector &objects) PreviewImage::PreviewImage(std::mutex &mtx) : _mtx(mtx) {} -void PreviewImage::CreateImage(const VideoInput &video, PreviewType type, - const PatternMatchParameters &patternMatchParams, - const PatternImageData &patternImageData, - ObjDetectParameters objDetectParams, - OCRParameters ocrParams, - const AreaParameters &areaParams, - VideoCondition condition) +void PreviewImage::CreateImage( + const VideoInput &video, PreviewType type, + const PatternMatchParameters &patternMatchParams, + const PatternImageData &patternImageData, + std::shared_ptr objDetectParams, + std::shared_ptr ocrParams, + const AreaParameters &areaParams, VideoCondition condition) { auto source = obs_weak_source_get_source(video.GetVideo()); QRect screenshotArea; @@ -308,61 +308,94 @@ void PreviewImage::CreateImage(const VideoInput &video, PreviewType type, emit ImageReady(QPixmap::fromImage(screenshot.GetImage())); } -void PreviewImage::MarkMatch(QImage &screenshot, - const PatternMatchParameters &patternMatchParams, - const PatternImageData &patternImageData, - ObjDetectParameters &objDetectParams, - const OCRParameters &ocrParams, - VideoCondition condition) +void PreviewImage::MarkMatch( + QImage &screenshot, const PatternMatchParameters &patternMatchParams, + const PatternImageData &patternImageData, + std::shared_ptr objDetectParams, + std::shared_ptr ocrParams, VideoCondition condition) { if (condition == VideoCondition::PATTERN) { - cv::Mat result; - double matchValue = - std::numeric_limits::signaling_NaN(); - MatchPattern(screenshot, patternImageData, - patternMatchParams.threshold, result, &matchValue, - patternMatchParams.useAlphaAsMask, - patternMatchParams.matchMode); - emit ValueUpdate(matchValue); - if (result.total() == 0 || countNonZero(result) == 0) { - emit StatusUpdate(obs_module_text( - "AdvSceneSwitcher.condition.video.patternMatchFail")); - } else if (result.cols == 1 && result.rows == 1) { - emit StatusUpdate(obs_module_text( - "AdvSceneSwitcher.condition.video.patternMatchSuccessFullImage")); - } else { - emit StatusUpdate(obs_module_text( - "AdvSceneSwitcher.condition.video.patternMatchSuccess")); - markPatterns(result, screenshot, - patternImageData.rgbaPattern); - } + MarkPatternMatch(screenshot, patternMatchParams, + patternImageData); } else if (condition == VideoCondition::OBJECT) { - auto objects = MatchObject(screenshot, objDetectParams.cascade, - objDetectParams.scaleFactor, - objDetectParams.minNeighbors, - objDetectParams.minSize.CV(), - objDetectParams.maxSize.CV()); - if (objects.empty()) { - emit StatusUpdate(obs_module_text( - "AdvSceneSwitcher.condition.video.objectMatchFail")); - } else { - emit StatusUpdate(obs_module_text( - "AdvSceneSwitcher.condition.video.objectMatchSuccess")); - markObjects(screenshot, objects); - } + MarkObjectMatch(screenshot, objDetectParams); } else if (condition == VideoCondition::OCR) { - auto text = RunOCR(ocrParams.GetOCR(), screenshot, - ocrParams.color, ocrParams.colorThreshold); - - if (!text) { - emit StatusUpdate(obs_module_text( - "AdvSceneSwitcher.condition.video.ocrMatchFail")); - } - - QString status(obs_module_text( - "AdvSceneSwitcher.condition.video.ocrMatchSuccess")); - emit StatusUpdate(status.arg(QString::fromStdString(*text))); + MarkOCRMatch(screenshot, ocrParams); } } +void PreviewImage::MarkPatternMatch( + QImage &screenshot, const PatternMatchParameters &patternMatchParams, + const PatternImageData &patternImageData) +{ + cv::Mat result; + double matchValue = std::numeric_limits::signaling_NaN(); + MatchPattern(screenshot, patternImageData, patternMatchParams.threshold, + result, &matchValue, patternMatchParams.useAlphaAsMask, + patternMatchParams.matchMode); + emit ValueUpdate(matchValue); + if (result.total() == 0 || countNonZero(result) == 0) { + emit StatusUpdate(obs_module_text( + "AdvSceneSwitcher.condition.video.patternMatchFail")); + } else if (result.cols == 1 && result.rows == 1) { + emit StatusUpdate(obs_module_text( + "AdvSceneSwitcher.condition.video.patternMatchSuccessFullImage")); + } else { + emit StatusUpdate(obs_module_text( + "AdvSceneSwitcher.condition.video.patternMatchSuccess")); + markPatterns(result, screenshot, patternImageData.rgbaPattern); + } +} + +void PreviewImage::MarkObjectMatch( + QImage &screenshot, + const std::shared_ptr &objDetectParams) +{ + if (!objDetectParams) { + emit StatusUpdate(obs_module_text( + "AdvSceneSwitcher.condition.video.objectMatchFail")); + return; + } + auto model = objDetectParams->GetModel(); + if (!model) { + emit StatusUpdate(obs_module_text( + "AdvSceneSwitcher.condition.video.objectMatchFail")); + return; + } + auto objects = MatchObject(screenshot, *model, + objDetectParams->scaleFactor, + objDetectParams->minNeighbors, + objDetectParams->minSize.CV(), + objDetectParams->maxSize.CV()); + if (objects.empty()) { + emit StatusUpdate(obs_module_text( + "AdvSceneSwitcher.condition.video.objectMatchFail")); + } else { + emit StatusUpdate(obs_module_text( + "AdvSceneSwitcher.condition.video.objectMatchSuccess")); + markObjects(screenshot, objects); + } +} + +void PreviewImage::MarkOCRMatch(QImage &screenshot, + const std::shared_ptr &ocrParams) +{ + if (!ocrParams) { + emit StatusUpdate(obs_module_text( + "AdvSceneSwitcher.condition.video.ocrMatchFail")); + return; + } + auto text = RunOCR(ocrParams->GetOCR(), screenshot, ocrParams->color, + ocrParams->colorThreshold); + + if (!text) { + emit StatusUpdate(obs_module_text( + "AdvSceneSwitcher.condition.video.ocrMatchFail")); + } + + QString status(obs_module_text( + "AdvSceneSwitcher.condition.video.ocrMatchSuccess")); + emit StatusUpdate(status.arg(QString::fromStdString(*text))); +} + } // namespace advss diff --git a/plugins/video/preview-dialog.hpp b/plugins/video/preview-dialog.hpp index ee9d04fd..a32f2c96 100644 --- a/plugins/video/preview-dialog.hpp +++ b/plugins/video/preview-dialog.hpp @@ -26,8 +26,10 @@ public: public slots: void CreateImage(const VideoInput &, PreviewType, const PatternMatchParameters &, - const PatternImageData &, ObjDetectParameters, - OCRParameters, const AreaParameters &, VideoCondition); + const PatternImageData &, + std::shared_ptr, + std::shared_ptr, const AreaParameters &, + VideoCondition); signals: void ImageReady(const QPixmap &); void ValueUpdate(double); @@ -35,8 +37,14 @@ signals: private: void MarkMatch(QImage &screenshot, const PatternMatchParameters &, - const PatternImageData &, ObjDetectParameters &, - const OCRParameters &, VideoCondition); + const PatternImageData &, + std::shared_ptr, + std::shared_ptr, VideoCondition); + void MarkPatternMatch(QImage &, const PatternMatchParameters &, + const PatternImageData &); + void MarkObjectMatch(QImage &, + const std::shared_ptr &); + void MarkOCRMatch(QImage &, const std::shared_ptr &); std::mutex &_mtx; }; @@ -67,8 +75,9 @@ signals: void SelectionAreaChanged(QRect area); void NeedImage(const VideoInput &, PreviewType, const PatternMatchParameters &, const PatternImageData &, - ObjDetectParameters, OCRParameters, - const AreaParameters &, VideoCondition); + std::shared_ptr, + std::shared_ptr, const AreaParameters &, + VideoCondition); private: void Start(); @@ -81,8 +90,8 @@ private: VideoInput _video; PatternMatchParameters _patternMatchParams; PatternImageData _patternImageData; - ObjDetectParameters _objDetectParams; - OCRParameters _ocrParams; + std::shared_ptr _objDetectParams; + std::shared_ptr _ocrParams; AreaParameters _areaParams; VideoCondition _condition = VideoCondition::PATTERN;