diff --git a/libs/ardour/audioengine.cc b/libs/ardour/audioengine.cc
index 73e48f1e93..7fe9bebed1 100644
--- a/libs/ardour/audioengine.cc
+++ b/libs/ardour/audioengine.cc
@@ -898,6 +898,10 @@ AudioEngine::start (bool for_latency)
 		return -1;
 	}
 
+	if (_running && _backend->can_change_systemic_latency_when_running()) {
+		_started_for_latency = for_latency;
+	}
+
 	if (_running) {
 		return 0;
 	}
@@ -963,7 +967,7 @@ AudioEngine::stop (bool for_latency)
 
 	if (for_latency && _backend->can_change_systemic_latency_when_running()) {
 		stop_engine = false;
-		if (_running) {
+		if (_running && _started_for_latency) {
 			_backend->start (false); // keep running, reload latencies
 		}
 	} else {
diff --git a/libs/ardour/graph.cc b/libs/ardour/graph.cc
index ed157aeb5d..f5466d7d87 100644
--- a/libs/ardour/graph.cc
+++ b/libs/ardour/graph.cc
@@ -110,7 +110,6 @@ Graph::reset_thread_list ()
 
 	/* For now, we shouldn't be using the graph code if we only have 1 DSP thread */
 	assert (num_threads > 1);
-	assert (AudioEngine::instance ()->process_thread_count () == n_workers);
 
 	/* don't bother doing anything here if we already have the right
 	 * number of threads.