From 95693f936499a153d789bcdfc4b04b05beed5281 Mon Sep 17 00:00:00 2001
From: Robin Gareus <robin@gareus.org>
Date: Sat, 23 Apr 2022 14:53:56 +0200
Subject: [PATCH] Fix endless recursion when creating cyclic connection graphs

This is mainly relevant for Mixbus, which allows cyclic-connections
to record Master-out on a Track (which unconditionally has Mixbus send
and feeds master).

In Ardour it may also cause issues when creating loopback connections,
however latency is not usually updated with invalid graphs (old
process graph remains in use).

Otherwise it fixes a crash connecting Track 1 -> Track 2 -> Track 1.

This also optimizes Route::output_effectively_connected by caching
any prior lookup. This helps e.g. Track 1 -> Track 2 -> Master.
The connection "Track 2 -> Master" now only need to be looked up once.

See also 7958031287a4f and a556e96ed06
---
 libs/ardour/ardour/route.h |  3 +++
 libs/ardour/route.cc       | 31 ++++++++++++++++++++++++++-----
 2 files changed, 29 insertions(+), 5 deletions(-)

diff --git a/libs/ardour/ardour/route.h b/libs/ardour/ardour/route.h
index 35dd06d615..acfdf9c1f5 100644
--- a/libs/ardour/ardour/route.h
+++ b/libs/ardour/ardour/route.h
@@ -744,6 +744,9 @@ private:
 	bool input_port_count_changing (ChanCount);
 	bool output_port_count_changing (ChanCount);
 
+	bool output_effectively_connected_real () const;
+	mutable std::map<Route*, bool> _connection_cache;
+
 	int configure_processors_unlocked (ProcessorStreams*, Glib::Threads::RWLock::WriterLock*);
 	bool set_meter_point_unlocked ();
 	void apply_processor_order (const ProcessorList& new_order);
diff --git a/libs/ardour/route.cc b/libs/ardour/route.cc
index 351d2d3d39..adeb77f729 100644
--- a/libs/ardour/route.cc
+++ b/libs/ardour/route.cc
@@ -3674,6 +3674,13 @@ Route::feeds_according_to_graph (boost::shared_ptr<Route> other)
 
 bool
 Route::output_effectively_connected () const
+{
+	_connection_cache.clear ();
+	return output_effectively_connected_real ();
+}
+
+bool
+Route::output_effectively_connected_real () const
 {
 	if (!_output->connected ()) {
 		return false;
@@ -3687,15 +3694,29 @@ Route::output_effectively_connected () const
 		return true;
 	}
 
+	/* now follow connections downstream */
 	boost::shared_ptr<RouteList> routes = _session.get_routes ();
 	for (RouteList::iterator i = routes->begin(); i != routes->end(); ++i) {
-		if ((*i).get() == this) {
+		Route* rp = (*i).get();
+		if (rp == this) {
 			continue;
 		}
-		if ((*i)->input()->connected_to (_output)) {
-			if ((*i)->output_effectively_connected ()) {
-				return true;
-			}
+		if (!(*i)->input()->connected_to (_output)) {
+			continue;
+		}
+		if (_connection_cache.find (rp) != _connection_cache.end ()) {
+			return _connection_cache[rp];
+		}
+		/* First mark node a traversed to prevent endless recursion.
+		 * Othewise graph loops A -> B -> A will cause a stack overflow.
+		 */
+		_connection_cache[rp] = false;
+
+		/* recurse downstream, check connected route */
+		bool rv = (*i)->output_effectively_connected_real ();
+		_connection_cache[rp] = rv;
+		if (rv) {
+			return true;
 		}
 	}
 	return false;