diff --git a/libs/ardour/ardour/audioengine.h b/libs/ardour/ardour/audioengine.h
index 053a6b6750..f699e01988 100644
--- a/libs/ardour/ardour/audioengine.h
+++ b/libs/ardour/ardour/audioengine.h
@@ -307,6 +307,7 @@ class LIBARDOUR_API AudioEngine : public PortManager, public SessionHandlePtr
 	Glib::Threads::Mutex       _devicelist_update_lock;
 	gint                       _stop_hw_devicelist_processing;
 	uint32_t                   _start_cnt;
+	uint32_t                   _init_countdown;
 
 	void start_hw_event_processing();
 	void stop_hw_event_processing();
diff --git a/libs/ardour/audioengine.cc b/libs/ardour/audioengine.cc
index 65a76f745a..4ba6f6cf0a 100644
--- a/libs/ardour/audioengine.cc
+++ b/libs/ardour/audioengine.cc
@@ -100,6 +100,7 @@ AudioEngine::AudioEngine ()
 	, _hw_devicelist_update_count(0)
 	, _stop_hw_devicelist_processing(0)
 	, _start_cnt (0)
+	, _init_countdown (0)
 #ifdef SILENCE_AFTER_SECONDS
 	, _silence_countdown (0)
 	, _silence_hit_cnt (0)
@@ -248,6 +249,16 @@ AudioEngine::process_callback (pframes_t nframes)
 		thread_init_callback (NULL);
 	}
 
+	if (_session && _init_countdown > 0) {
+		--_init_countdown;
+		/* Warm up caches */
+		PortManager::cycle_start (nframes);
+		PortManager::silence (nframes);
+		_session->process (nframes);
+		PortManager::cycle_end (nframes);
+		return 0;
+	}
+
 	bool return_after_remove_check = false;
 
 	if (_measuring_latency == MeasureAudio && _mtdm) {
@@ -643,21 +654,7 @@ AudioEngine::set_session (Session *s)
 	SessionHandlePtr::set_session (s);
 
 	if (_session) {
-
-		pframes_t blocksize = samples_per_cycle ();
-
-		PortManager::cycle_start (blocksize);
-
-		_session->process (blocksize);
-		_session->process (blocksize);
-		_session->process (blocksize);
-		_session->process (blocksize);
-		_session->process (blocksize);
-		_session->process (blocksize);
-		_session->process (blocksize);
-		_session->process (blocksize);
-
-		PortManager::cycle_end (blocksize);
+		_init_countdown = 8;
 	}
 }