From 7e2bb01ff1c23829dfd93063e3dcc2ab1eaadbd9 Mon Sep 17 00:00:00 2001
From: Robin Gareus <robin@gareus.org>
Date: Thu, 19 Nov 2020 23:11:01 +0100
Subject: [PATCH] VST3: ignore GUI size allocations before realizing the view

This fixes crashes with some VST3s when calling `onSize()` before
attaching thew view (e.g. Reason Rack Plug on macOS).
---
 gtk2_ardour/vst3_hwnd_plugin_ui.cc   | 3 ++-
 gtk2_ardour/vst3_nsview_plugin_ui.mm | 6 ++++--
 gtk2_ardour/vst3_plugin_ui.cc        | 2 ++
 gtk2_ardour/vst3_plugin_ui.h         | 1 +
 gtk2_ardour/vst3_x11_plugin_ui.cc    | 3 ++-
 5 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/gtk2_ardour/vst3_hwnd_plugin_ui.cc b/gtk2_ardour/vst3_hwnd_plugin_ui.cc
index 5b1a0bae83..2dd2674239 100644
--- a/gtk2_ardour/vst3_hwnd_plugin_ui.cc
+++ b/gtk2_ardour/vst3_hwnd_plugin_ui.cc
@@ -65,6 +65,7 @@ VST3HWNDPluginUI::view_realized ()
 	if (kResultOk != view->attached (reinterpret_cast<void*> (hwnd), Steinberg::kPlatformTypeHWND)) {
 		assert (0);
 	}
+	_view_realized = true;
 
 	ViewRect rect;
 	if (view->getSize (&rect) == kResultOk) {
@@ -86,7 +87,7 @@ void
 VST3HWNDPluginUI::view_size_allocate (Gtk::Allocation& allocation)
 {
 	IPlugView* view = _vst3->view ();
-	if (!view) {
+	if (!view || !_view_realized) {
 		return;
 	}
 	PBD::Unwinder<bool> uw (_resize_in_progress, true);
diff --git a/gtk2_ardour/vst3_nsview_plugin_ui.mm b/gtk2_ardour/vst3_nsview_plugin_ui.mm
index 6ce1a07a9b..d823c83733 100644
--- a/gtk2_ardour/vst3_nsview_plugin_ui.mm
+++ b/gtk2_ardour/vst3_nsview_plugin_ui.mm
@@ -106,7 +106,7 @@ void
 VST3NSViewPluginUI::view_size_allocate (Gtk::Allocation& allocation)
 {
 	IPlugView* view = _vst3->view ();
-	if (!view) {
+	if (!view || !_view_realized) {
 		return;
 	}
 
@@ -133,7 +133,8 @@ VST3NSViewPluginUI::view_size_allocate (Gtk::Allocation& allocation)
 		allocation.set_height (rect.bottom - rect.top);
 #endif
 		if (view->canResize() == kResultTrue) {
-			view->onSize (&rect);
+			printf ("canResize\n");
+			//view->onSize (&rect); // crash here
 		}
 	}
 
@@ -182,6 +183,7 @@ VST3NSViewPluginUI::view_realized ()
 
 	NSView* nsview = gdk_quartz_window_get_nsview (_gui_widget.get_window()->gobj());
 	[nsview addSubview:_ns_view];
+	_view_realized = true;
 	_gui_widget.queue_resize ();
 }
 
diff --git a/gtk2_ardour/vst3_plugin_ui.cc b/gtk2_ardour/vst3_plugin_ui.cc
index db21de7d2e..e4610c7890 100644
--- a/gtk2_ardour/vst3_plugin_ui.cc
+++ b/gtk2_ardour/vst3_plugin_ui.cc
@@ -41,6 +41,7 @@ VST3PluginUI::VST3PluginUI (boost::shared_ptr<PluginInsert> pi, boost::shared_pt
 	, _req_width (0)
 	, _req_height (0)
 	, _resize_in_progress (false)
+	, _view_realized (false)
 {
 	_ardour_buttons_box.set_spacing (6);
 	_ardour_buttons_box.set_border_width (6);
@@ -55,6 +56,7 @@ VST3PluginUI::VST3PluginUI (boost::shared_ptr<PluginInsert> pi, boost::shared_pt
 
 VST3PluginUI::~VST3PluginUI ()
 {
+	assert (_view_realized);
 	_vst3->close_view ();
 }
 
diff --git a/gtk2_ardour/vst3_plugin_ui.h b/gtk2_ardour/vst3_plugin_ui.h
index bbc92ce148..fbe45e639f 100644
--- a/gtk2_ardour/vst3_plugin_ui.h
+++ b/gtk2_ardour/vst3_plugin_ui.h
@@ -58,6 +58,7 @@ protected:
 	int _req_height;
 
 	bool _resize_in_progress;
+	bool _view_realized;
 
 private:
 	void parameter_update ();
diff --git a/gtk2_ardour/vst3_x11_plugin_ui.cc b/gtk2_ardour/vst3_x11_plugin_ui.cc
index 54612c0e92..0c4e7a7aba 100644
--- a/gtk2_ardour/vst3_x11_plugin_ui.cc
+++ b/gtk2_ardour/vst3_x11_plugin_ui.cc
@@ -203,6 +203,7 @@ VST3X11PluginUI::view_realized ()
 	if (kResultOk != view->attached (reinterpret_cast<void*> (window), Steinberg::kPlatformTypeX11EmbedWindowID)) {
 		assert (0);
 	}
+	_view_realized = true;
 #if 0
 	_gui_widget.set_sensitive (true);
 	_gui_widget.set_can_focus (true);
@@ -228,7 +229,7 @@ void
 VST3X11PluginUI::view_size_allocate (Gtk::Allocation& allocation)
 {
 	IPlugView* view = _vst3->view ();
-	if (!view) {
+	if (!view || !_view_realized) {
 		return;
 	}
 	PBD::Unwinder<bool> uw (_resize_in_progress, true);