From f30fbcaaf968dcb5af8b762b6e3d25f6fe313f3f Mon Sep 17 00:00:00 2001
From: Robin Gareus <robin@gareus.org>
Date: Mon, 14 Nov 2016 21:05:36 +0100
Subject: [PATCH] Fix VST state-lock SNAFU (effSetChunk and process are
 exclusive)

---
 gtk2_ardour/linux_vst_gui_support.cc | 2 ++
 libs/ardour/ardour/vst_plugin.h      | 2 --
 libs/ardour/ardour/vst_types.h       | 2 ++
 libs/ardour/vst_plugin.cc            | 7 ++++---
 libs/fst/vstwin.c                    | 2 ++
 5 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/gtk2_ardour/linux_vst_gui_support.cc b/gtk2_ardour/linux_vst_gui_support.cc
index a2f637a26c..fe85ea188e 100644
--- a/gtk2_ardour/linux_vst_gui_support.cc
+++ b/gtk2_ardour/linux_vst_gui_support.cc
@@ -320,8 +320,10 @@ maybe_set_program (VSTState* vstfx)
 	}
 
 	if (vstfx->want_chunk == 1) {
+		pthread_mutex_lock (&vstfx->state_lock);
 		vstfx->plugin->dispatcher (vstfx->plugin, 24 /* effSetChunk */, 1, vstfx->wanted_chunk_size, vstfx->wanted_chunk, 0);
 		vstfx->want_chunk = 0;
+		pthread_mutex_unlock (&vstfx->state_lock);
 	}
 }
 
diff --git a/libs/ardour/ardour/vst_plugin.h b/libs/ardour/ardour/vst_plugin.h
index 182b432692..2a8b4dc9a1 100644
--- a/libs/ardour/ardour/vst_plugin.h
+++ b/libs/ardour/ardour/vst_plugin.h
@@ -120,8 +120,6 @@ protected:
 	framepos_t _transport_frame;
 	float      _transport_speed;
 	mutable std::map <uint32_t, float> _parameter_defaults;
-
-	Glib::Threads::Mutex _state_lock;
 };
 
 }
diff --git a/libs/ardour/ardour/vst_types.h b/libs/ardour/ardour/vst_types.h
index 936cab6173..0b8bcb6273 100644
--- a/libs/ardour/ardour/vst_types.h
+++ b/libs/ardour/ardour/vst_types.h
@@ -129,6 +129,7 @@ struct LIBARDOUR_API _VSTState
 
 	struct _VSTState * next;
 	pthread_mutex_t    lock;
+	pthread_mutex_t    state_lock;
 	pthread_cond_t     window_status_change;
 	pthread_cond_t     plugin_dispatcher_called;
 	pthread_cond_t     window_created;
@@ -136,6 +137,7 @@ struct LIBARDOUR_API _VSTState
 
 	void init () {
 		pthread_mutex_init (&lock, 0);
+		pthread_mutex_init (&state_lock, 0);
 		pthread_cond_init (&window_status_change, 0);
 		pthread_cond_init (&plugin_dispatcher_called, 0);
 		pthread_cond_init (&window_created, 0);
diff --git a/libs/ardour/vst_plugin.cc b/libs/ardour/vst_plugin.cc
index 8214d0be81..1fd8d8cd55 100644
--- a/libs/ardour/vst_plugin.cc
+++ b/libs/ardour/vst_plugin.cc
@@ -172,8 +172,9 @@ VSTPlugin::set_chunk (gchar const * data, bool single)
 	int r = 0;
 	guchar* raw_data = g_base64_decode (data, &size);
 	{
-		Glib::Threads::Mutex::Lock lm (_lock);
+		pthread_mutex_lock (&_state->state_lock);
 		r = _plugin->dispatcher (_plugin, 24 /* effSetChunk */, single ? 1 : 0, size, raw_data, 0);
+		pthread_mutex_unlock (&_state->state_lock);
 	}
 	g_free (raw_data);
 	return r;
@@ -566,8 +567,7 @@ VSTPlugin::connect_and_run (BufferSet& bufs,
 {
 	Plugin::connect_and_run(bufs, start, end, speed, in_map, out_map, nframes, offset);
 
-	Glib::Threads::Mutex::Lock lm (_state_lock, Glib::Threads::TRY_LOCK);
-	if (!lm.locked()) {
+	if (pthread_mutex_trylock (&_state->state_lock)) {
 		/* by convention 'effSetChunk' should not be called while processing
 		 * http://www.reaper.fm/sdk/vst/vst_ext.php
 		 *
@@ -645,6 +645,7 @@ VSTPlugin::connect_and_run (BufferSet& bufs,
 	_plugin->processReplacing (_plugin, &ins[0], &outs[0], nframes);
 	_midi_out_buf = 0;
 
+	pthread_mutex_unlock (&_state->state_lock);
 	return 0;
 }
 
diff --git a/libs/fst/vstwin.c b/libs/fst/vstwin.c
index 7f748c6db6..1e053d1ae7 100644
--- a/libs/fst/vstwin.c
+++ b/libs/fst/vstwin.c
@@ -92,8 +92,10 @@ maybe_set_program (VSTState* fst)
 		// XXX check
 		// 24 == audioMasterGetAutomationState,
 		// 48 == audioMasterGetChunkFile
+		pthread_mutex_lock (&vstfx->state_lock);
 		fst->plugin->dispatcher (fst->plugin, 24 /* effSetChunk */, 1, fst->wanted_chunk_size, fst->wanted_chunk, 0);
 		fst->want_chunk = 0;
+		pthread_mutex_unlock (&vstfx->state_lock);
 	}
 }