Prevent concurrent loop and punch recording (backend)

This also prevents switching between punch-in/out record
and looping without transport-stop.

libs/ardour/ardour/session.h

@@ -1694,6 +1694,22 @@ private:
 	void flush_all_inserts ();
 	int  micro_locate (samplecnt_t distance);
 
+	enum PunchLoopLock {
+		NoConstraint,
+		OnlyPunch,
+		OnlyLoop,
+	};
+
+	volatile guint _punch_or_loop; // enum PunchLoopLock
+
+	bool punch_is_possible () const;
+	bool loop_is_possible () const;
+
+	bool punch_active () const;
+	void unset_punch ();
+	bool maybe_allow_only_loop (bool play_loop = false);
+	bool maybe_allow_only_punch ();
+
 	void force_locate (samplepos_t sample, LocateTransportDisposition);
 	void realtime_stop (bool abort, bool clear_state);
 	void realtime_locate (bool);

libs/ardour/session.cc

@@ -271,6 +271,7 @@ Session::Session (AudioEngine &eng,
 	, ltc_timecode_offset (0)
 	, ltc_timecode_negative_offset (false)
 	, midi_control_ui (0)
+	, _punch_or_loop (NoConstraint)
 	, _tempo_map (0)
 	, _all_route_group (new RouteGroup (*this, "all"))
 	, routes (new RouteList)
@@ -1397,6 +1398,72 @@ Session::auto_punch_start_changed (Location* location)
 	}
 }
 
+bool
+Session::punch_active () const
+{
+	if (!get_record_enabled ()) {
+		return false;
+	}
+	if (!_locations->auto_punch_location ()) {
+		return false;
+	}
+	return config.get_punch_in () || config.get_punch_out ();
+}
+
+bool
+Session::punch_is_possible () const
+{
+	return g_atomic_int_get (&_punch_or_loop) != OnlyLoop;
+}
+
+bool
+Session::loop_is_possible () const
+{
+#if 0 /* maybe prevent looping even when not rolling ? */
+	if (get_record_enabled () && punch_active ()) {
+			return false;
+		}
+	}
+#endif
+	return g_atomic_int_get(&_punch_or_loop) != OnlyPunch;
+}
+
+bool
+Session::maybe_allow_only_loop (bool play_loop) {
+	if (!(get_play_loop () || play_loop)) {
+		return false;
+	}
+	bool rv = g_atomic_int_compare_and_exchange (&_punch_or_loop, NoConstraint, OnlyLoop);
+	if (rv || loop_is_possible ()) {
+		unset_punch ();
+		return true;
+	}
+	return false;
+}
+
+bool
+Session::maybe_allow_only_punch () {
+	if (!punch_active ()) {
+		return false;
+	}
+	bool rv = g_atomic_int_compare_and_exchange (&_punch_or_loop, NoConstraint, OnlyPunch);
+	return rv || punch_is_possible ();
+}
+
+void
+Session::unset_punch ()
+{
+	/* used when enabling looping
+	 * -> _punch_or_loop = OnlyLoop;
+	 */
+	if (config.get_punch_in ()) {
+		config.set_punch_in (false);
+	}
+	if (config.get_punch_out ()) {
+		config.set_punch_out (false);
+	}
+}
+
 void
 Session::auto_punch_end_changed (Location* location)
 {
@@ -1871,6 +1938,7 @@ Session::maybe_enable_record (bool rt_context)
 	}
 
 	if (_transport_speed) {
+		maybe_allow_only_punch ();
 		if (!config.get_punch_in()) {
 			enable_record ();
 		}

libs/ardour/session_state.cc

@@ -4006,8 +4006,15 @@ Session::config_changed (std::string p, bool ours)
 
 	} else if (p == "punch-in") {
 
-		Location* location;
+		if (!punch_is_possible ()) {
+			if (config.get_punch_in ()) {
+				/* force off */
+				config.set_punch_in (false);
+				return;
+			}
+		}
 
+		Location* location;
 		if ((location = _locations->auto_punch_location()) != 0) {
 
 			if (config.get_punch_in ()) {
@@ -4019,8 +4026,15 @@ Session::config_changed (std::string p, bool ours)
 
 	} else if (p == "punch-out") {
 
-		Location* location;
+		if (!punch_is_possible ()) {
+			if (config.get_punch_out ()) {
+				/* force off */
+				config.set_punch_out (false);
+				return;
+			}
+		}
 
+		Location* location;
 		if ((location = _locations->auto_punch_location()) != 0) {
 
 			if (config.get_punch_out()) {

libs/ardour/session_transport.cc

@@ -145,6 +145,8 @@ Session::realtime_stop (bool abort, bool clear_state)
 
 	reset_slave_state ();
 
+	g_atomic_int_set (&_punch_or_loop, NoConstraint);
+
 	_transport_speed = 0;
 	_target_transport_speed = 0;
 	_engine_speed = 1.0;
@@ -604,6 +606,9 @@ Session::start_transport ()
 		break;
 	}
 
+	maybe_allow_only_loop ();
+	maybe_allow_only_punch ();
+
 	_transport_speed = _default_transport_speed;
 	_target_transport_speed = _transport_speed;
 
@@ -1575,6 +1580,10 @@ Session::set_play_loop (bool yn, bool change_transport_state)
 		return;
 	}
 
+	if (yn && !maybe_allow_only_loop (true)) {
+		return;
+	}
+
 	if (yn) {
 
 		play_loop = true;