Avoid deadlocks while opening settings window and calling frontend API

src/advanced-scene-switcher.cpp

@@ -244,6 +244,7 @@ void SwitcherData::Thread()
 		switcher->firstIntervalAfterStop = false;
 	}
 
+	mainLoopLock = nullptr;
 	blog(LOG_INFO, "stopped");
 }
 

src/macro-core/macro-action-scene-switch.cpp

@@ -106,25 +106,12 @@ bool MacroActionSwitchScene::WaitForTransition(OBSWeakSource &scene,
 		scene, transition, _duration.seconds);
 	switcher->abortMacroWait = false;
 
-	bool isInMainLoop = QThread::currentThread() == switcher->th;
-	if (isInMainLoop) {
-		if (expectedTransitionDuration < 0) {
-			waitForTransitionChange(transition, switcher->GetLock(),
-						GetMacro());
-		} else {
-			waitForTransitionChangeFixedDuration(
-				expectedTransitionDuration, switcher->GetLock(),
-				GetMacro());
-		}
+	std::unique_lock<std::mutex> lock(switcher->m);
+	if (expectedTransitionDuration < 0) {
+		waitForTransitionChange(transition, &lock, GetMacro());
 	} else {
-		std::mutex temp;
-		std::unique_lock<std::mutex> lock(temp);
-		if (expectedTransitionDuration < 0) {
-			waitForTransitionChange(transition, &lock, GetMacro());
-		} else {
-			waitForTransitionChangeFixedDuration(
-				expectedTransitionDuration, &lock, GetMacro());
-		}
+		waitForTransitionChangeFixedDuration(expectedTransitionDuration,
+						     &lock, GetMacro());
 	}
 
 	return !switcher->abortMacroWait;

src/macro-core/macro-action-wait.cpp

@@ -52,14 +52,8 @@ bool MacroActionWait::PerformAction()
 		    std::chrono::milliseconds((int)(sleepDuration * 1000));
 
 	switcher->abortMacroWait = false;
-	bool isInMainLoop = QThread::currentThread() == switcher->th;
-	if (isInMainLoop) {
-		waitHelper(switcher->GetLock(), GetMacro(), time);
-	} else {
-		std::mutex temp;
-		std::unique_lock<std::mutex> lock(temp);
-		waitHelper(&lock, GetMacro(), time);
-	}
+	std::unique_lock<std::mutex> lock(switcher->m);
+	waitHelper(&lock, GetMacro(), time);
 
 	return !switcher->abortMacroWait;
 }

src/macro-core/macro.cpp

@@ -777,7 +777,21 @@ bool SwitcherData::runMacros()
 	// as the main lock will be unlocked during this time.
 	auto runPhaseMacros = macros;
 
-	for (auto m : runPhaseMacros) {
+	// Avoid deadlocks when opening settings window and calling frontend
+	// API functions at the same time.
+	//
+	// If the timing is just right, the frontend API call will call
+	// QMetaObject::invokeMethod(...) with  Qt::BlockingQueuedConnection
+	// while holding the main switcher mutex.
+	// But this invokeMethod call itself will be blocked as it is waiting
+	// the constructor of AdvSceneSwitcher() to complete.
+	// The constructor of AdvSceneSwitcher() cannot continue however as it
+	// cannot lock the main switcher mutex.
+	if (GetLock()) {
+		GetLock()->unlock();
+	}
+
+	for (auto &m : runPhaseMacros) {
 		if (m && m->Matched()) {
 			vblog(LOG_INFO, "running macro: %s", m->Name().c_str());
 			if (!m->PerformActions()) {
@@ -786,6 +800,9 @@ bool SwitcherData::runMacros()
 			}
 		}
 	}
+	if (GetLock()) {
+		GetLock()->lock();
+	}
 	return true;
 }
 

src/switcher-data-structs.hpp

@@ -90,7 +90,7 @@ struct SwitcherData {
 
 	std::condition_variable cv;
 	std::mutex m;
-	std::unique_lock<std::mutex> *mainLoopLock;
+	std::unique_lock<std::mutex> *mainLoopLock = nullptr;
 
 	bool transitionActive = false;
 	bool waitForTransition = false;