From 7227407aa6c42510af70cb5806ab5c1526d96b5c Mon Sep 17 00:00:00 2001
From: Robin Gareus <robin@gareus.org>
Date: Wed, 18 Oct 2023 21:46:19 +0200
Subject: [PATCH] VST3: Prevent recursive calls to ::has_editor()

Some plugins call restartComponent(Vst::kParamTitlesChanged)
when their GUI is created, from the call that creates the UI.
This lead to a stack-overflow recursion in Ardour:

ProcessorBox::redisplay_processors -> VST3Plugin::has_editor
-> [plugin] -> VST3::restartComponent -> signal proc changed
-> ProcessorBox::redisplay processors
---
 libs/ardour/ardour/types.h       | 11 ++++----
 libs/ardour/ardour/vst3_plugin.h | 23 +++++++++++++++++
 libs/ardour/vst3_plugin.cc       | 44 +++++++++++++++++++++++++++++---
 3 files changed, 69 insertions(+), 9 deletions(-)

diff --git a/libs/ardour/ardour/types.h b/libs/ardour/ardour/types.h
index c013093cd8..82f0c3b897 100644
--- a/libs/ardour/ardour/types.h
+++ b/libs/ardour/ardour/types.h
@@ -725,11 +725,12 @@ enum AppleNSGLViewMode {
  */
 struct RouteProcessorChange {
 	enum Type {
-		MeterPointChange = 0x1,
-		RealTimeChange   = 0x2,
-		GeneralChange    = 0x4,
-		SendReturnChange = 0x8,
-		CustomPinChange  = 0x10
+		NoProcessorChange = 0x0,
+		MeterPointChange  = 0x1,
+		RealTimeChange    = 0x2,
+		GeneralChange     = 0x4,
+		SendReturnChange  = 0x8,
+		CustomPinChange   = 0x10
 	};
 
 	RouteProcessorChange () : type (GeneralChange), meter_visibly_changed (true)
diff --git a/libs/ardour/ardour/vst3_plugin.h b/libs/ardour/ardour/vst3_plugin.h
index f9e8c3329d..a241fa70dc 100644
--- a/libs/ardour/ardour/vst3_plugin.h
+++ b/libs/ardour/ardour/vst3_plugin.h
@@ -212,6 +212,22 @@ public:
 	bool add_slave (Vst::IEditController*, bool);
 	bool remove_slave (Vst::IEditController*);
 
+	class RouteProcessorChangeBlock
+	{
+		public:
+			RouteProcessorChangeBlock (VST3PI* p) : _impl (p)
+		{
+			_impl->block_notifications ();
+		}
+
+		~RouteProcessorChangeBlock ()
+		{
+			_impl->resume_notifications ();
+		}
+		private:
+			VST3PI* _impl;
+	};
+
 private:
 	/* prevent copy construction */
 	VST3PI (const VST3PI&);
@@ -246,6 +262,10 @@ private:
 	void psl_subscribe_to (std::shared_ptr<ARDOUR::AutomationControl>, FIDString);
 	void psl_stripable_property_changed (PBD::PropertyChange const&);
 
+	void block_notifications ();
+	void resume_notifications ();
+	void send_processors_changed (ARDOUR::RouteProcessorChange const&);
+
 	void forward_signal (Presonus::IContextInfoHandler2*, FIDString) const;
 
 	std::shared_ptr<ARDOUR::VST3PluginModule> _module;
@@ -333,6 +353,9 @@ private:
 	int _n_midi_outputs;
 	int _n_factory_presets;
 
+	mutable std::atomic<int>     _block_rpc;
+	ARDOUR::RouteProcessorChange _rpc_queue;
+
 	/* work around UADx plugin crash */
 	bool _no_kMono;
 };
diff --git a/libs/ardour/vst3_plugin.cc b/libs/ardour/vst3_plugin.cc
index a819d83d85..e6ce1beafa 100644
--- a/libs/ardour/vst3_plugin.cc
+++ b/libs/ardour/vst3_plugin.cc
@@ -341,6 +341,8 @@ VST3Plugin::possible_output () const
 bool
 VST3Plugin::has_editor () const
 {
+	VST3PI::RouteProcessorChangeBlock rpcb (_plug);
+
 	std::shared_ptr<VST3PluginInfo> nfo = std::dynamic_pointer_cast<VST3PluginInfo> (get_info ());
 	if (nfo->has_editor.has_value ()) {
 		return nfo->has_editor.value ();
@@ -1168,6 +1170,8 @@ VST3PI::VST3PI (std::shared_ptr<ARDOUR::VST3PluginModule> m, std::string unique_
 	, _owner (0)
 	, _add_to_selection (false)
 	, _n_factory_presets (0)
+	, _block_rpc (0)
+	, _rpc_queue (RouteProcessorChange::NoProcessorChange, false)
 	, _no_kMono (false)
 {
 	using namespace std;
@@ -1595,10 +1599,7 @@ VST3PI::restartComponent (int32 flags)
 				p.normal = pi.defaultNormalizedValue;
 			}
 		}
-		Route* r = dynamic_cast<Route*> (_owner);
-		if (r) {
-			r->processors_changed (RouteProcessorChange ()); /* EMIT SIGNAL */
-		}
+		send_processors_changed (RouteProcessorChange ()); /* EMIT SIGNAL */
 	}
 	if (flags & Vst::kIoChanged) {
 		warning << "VST3: Vst::kIoChanged (not implemented)" << endmsg;
@@ -3221,3 +3222,38 @@ VST3PI::resizeView (IPlugView* view, ViewRect* new_size)
 	OnResizeView (new_size->getWidth (), new_size->getHeight ()); /* EMIT SIGNAL */
 	return view->onSize (new_size);
 }
+
+void
+VST3PI::block_notifications ()
+{
+	_block_rpc.fetch_add (1);
+}
+
+void
+VST3PI::resume_notifications ()
+{
+	if (!PBD::atomic_dec_and_test (_block_rpc)) {
+		return;
+	}
+	Route* r = dynamic_cast<Route*> (_owner);
+	if (r && _rpc_queue.type != RouteProcessorChange::NoProcessorChange) {
+		r->processors_changed (_rpc_queue); /* EMIT SIGNAL */
+	}
+	_rpc_queue.type = RouteProcessorChange::NoProcessorChange;
+	_rpc_queue.meter_visibly_changed = false;
+}
+
+void
+VST3PI::send_processors_changed (RouteProcessorChange const& rpc)
+{
+	if (_block_rpc.load () != 0) {
+		_rpc_queue.type = ARDOUR::RouteProcessorChange::Type (_rpc_queue.type | rpc.type);
+		_rpc_queue.meter_visibly_changed |= rpc.meter_visibly_changed;
+		return;
+	}
+
+	Route* r = dynamic_cast<Route*> (_owner);
+	if (r) {
+		r->processors_changed (rpc); /* EMIT SIGNAL */
+	}
+}