VST3: work-around endless/recursive resize requests

Some plugins correct their own size when resized which can lead
to endlessly growing GUIs. In particular this issue exists
if Ardour's plugin-toolbar forces a plugin to grow in size to fill
the allocated space. While the plugin aims for fixed aspect-ratio.

This will need a proper solution involving VST's checkSizeConstraint,
however GTK+2 does not have an appropriate API...

gtk2_ardour/vst3_hwnd_plugin_ui.cc

@@ -22,6 +22,8 @@
 
 #include <glibmm/main.h>
 
+#include "pbd/unwind.h"
+
 #include "ardour/plugin_insert.h"
 #include "ardour/vst3_plugin.h"
 
@@ -85,6 +87,7 @@ VST3HWNDPluginUI::view_size_allocate (Gtk::Allocation& allocation)
 	if (!view) {
 		return;
 	}
+	PBD::Unwinder<bool> uw (_resize_in_progress, true);
 	ViewRect rect;
 	if (view->getSize (&rect) == kResultOk) {
 		rect.right = rect.left + allocation.get_width ();
@@ -106,18 +109,10 @@ void
 VST3HWNDPluginUI::resize_callback (int width, int height)
 {
 	//printf ("VST3HWNDPluginUI::resize_callback %d x %d\n", width, height);
-#if 0
-	HWND hwnd = gdk_win32_drawable_get_handle (_gui_widget.window);
-	WINDOWINFO windowInfo;
-	GetWindowInfo (hwnd, &windowInfo);
-	RECT clientRect {};
-	clientRect.right = newSize.width;
-	clientRect.bottom = newSize.height;
-	AdjustWindowRectEx (&clientRect, windowInfo.dwStyle, false, windowInfo.dwExStyle);
-	SetWindowPos (hwnd, HWND_TOP, 0, 0, clientRect.right - clientRect.left,
-			clientRect.bottom - clientRect.top, SWP_NOMOVE | SWP_NOCOPYBITS | SWP_NOACTIVATE);
-#else
 	IPlugView* view = _vst3->view ();
+	if (!view || _resize_in_progress) {
+		return;
+	}
 	if (view->canResize() == kResultTrue) {
 		gint xx, yy;
 		if (gtk_widget_translate_coordinates (
@@ -132,7 +127,6 @@ VST3HWNDPluginUI::resize_callback (int width, int height)
 		_req_height = height;
 		_gui_widget.queue_resize ();
 	}
-#endif
 }
 
 bool

gtk2_ardour/vst3_nsview_plugin_ui.h

@@ -35,8 +35,6 @@
 #undef NO
 #endif
 
-
-
 #include <gtkmm/widget.h>
 #include <gtkmm/eventbox.h>
 

gtk2_ardour/vst3_nsview_plugin_ui.mm

@@ -26,6 +26,7 @@
 
 #include "pbd/convert.h"
 #include "pbd/error.h"
+#include "pbd/unwind.h"
 
 #include "ardour/plugin_insert.h"
 #include "ardour/vst3_plugin.h"
@@ -116,6 +117,8 @@ VST3NSViewPluginUI::view_size_allocate (Gtk::Allocation& allocation)
 			GTK_WIDGET(_gui_widget.get_parent()->gobj()),
 			0, 0, &xx, &yy);
 
+	PBD::Unwinder<bool> uw (_resize_in_progress, true);
+
 	ViewRect rect;
 	if (view->getSize (&rect) == kResultOk) {
 		rect.left   = xx;
@@ -147,7 +150,7 @@ VST3NSViewPluginUI::resize_callback (int width, int height)
 {
 	//printf ("VST3NSViewPluginUI::resize_callback %d x %d\n", width, height);
 	IPlugView* view = _vst3->view ();
-	if (!view) {
+	if (!view || _resize_in_progress) {
 		return;
 	}
 	if (view->canResize() == kResultTrue) {

gtk2_ardour/vst3_plugin_ui.cc

@@ -35,6 +35,7 @@ VST3PluginUI::VST3PluginUI (boost::shared_ptr<PluginInsert> pi, boost::shared_pt
 	, _vst3 (vst3)
 	, _req_width (0)
 	, _req_height (0)
+	, _resize_in_progress (false)
 {
 	_ardour_buttons_box.set_spacing (6);
 	_ardour_buttons_box.set_border_width (6);

gtk2_ardour/vst3_plugin_ui.h

@@ -57,6 +57,8 @@ protected:
 	int _req_width;
 	int _req_height;
 
+	bool _resize_in_progress;
+
 private:
 	void parameter_update ();
 

gtk2_ardour/vst3_x11_plugin_ui.cc

@@ -25,6 +25,8 @@
 #include <glibmm/main.h>
 #include <gtkmm/socket.h>
 
+#include "pbd/unwind.h"
+
 #include "ardour/plugin_insert.h"
 #include "ardour/vst3_plugin.h"
 
@@ -212,6 +214,8 @@ VST3X11PluginUI::view_size_allocate (Gtk::Allocation& allocation)
 	if (!view) {
 		return;
 	}
+	PBD::Unwinder<bool> uw (_resize_in_progress, true);
+
 	ViewRect rect;
 	if (view->getSize (&rect) == kResultOk) {
 		rect.right = rect.left + allocation.get_width ();
@@ -234,7 +238,7 @@ VST3X11PluginUI::resize_callback (int width, int height)
 {
 	// printf ("VST3X11PluginUI::resize_callback %d x %d\n", width, height);
 	IPlugView* view = _vst3->view ();
-	if (!view) {
+	if (!view || _resize_in_progress) {
 		return;
 	}
 	if (view->canResize() == kResultTrue) {