Implement FileArchive progress/cancel

* Use PBD::Progress API
* Allow to cancel extraction and compression
* Fix querying download size

libs/pbd/file_archive.cc

@@ -39,6 +39,7 @@
 #include "pbd/file_archive.h"
 #include "pbd/file_utils.h"
 #include "pbd/pthread_utils.h"
+#include "pbd/progress.h"
 
 using namespace PBD;
 
@@ -69,11 +70,14 @@ get_url (void* arg)
 	curl_easy_setopt (curl, CURLOPT_FOLLOWLOCATION, 1L);
 
 	/* get size */
-	if (r->mp.progress) {
+	if (r->mp.query_length) {
+		double content_length = 0;
 		curl_easy_setopt(curl, CURLOPT_NOBODY, 1L);
 		curl_easy_setopt(curl, CURLOPT_HEADER, 0L);
 		curl_easy_perform (curl);
-		curl_easy_getinfo (curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &r->mp.length);
+		if (CURLE_OK == curl_easy_getinfo (curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &content_length) && content_length > 0) {
+			r->mp.length = content_length;
+		}
 	}
 
 	curl_easy_setopt(curl, CURLOPT_NOBODY, 0L);
@@ -113,8 +117,8 @@ ar_read (struct archive* a, void* d, const void** buff)
 	p->size -= rv;
 	p->processed += rv;
 	*buff = p->buf;
-	if (p->progress) {
+	if (p->progress && p->length > 0) {
-		p->progress->progress (p->processed, p->length);
+		p->progress->set_progress ((float)p->processed / p->length);
 	}
 	p->unlock ();
 	return rv;
@@ -153,8 +157,9 @@ setup_archive ()
 	return a;
 }
 
-FileArchive::FileArchive (const std::string& url)
+FileArchive::FileArchive (const std::string& url, Progress* p)
-	: _req (url)
+	: _req (url, p)
+	, _progress (p)
 	, _current_entry (0)
 	, _archive (0)
 {
@@ -164,9 +169,9 @@ FileArchive::FileArchive (const std::string& url)
 	}
 
 	if (_req.is_remote ()) {
-		_req.mp.progress = this;
+		_req.mp.query_length = true;
 	} else {
-		_req.mp.progress = 0;
+		_req.mp.query_length = false;
 	}
 }
 
@@ -178,12 +183,6 @@ FileArchive::~FileArchive ()
 	}
 }
 
-void
-FileArchive::require_progress ()
-{
-	_req.mp.progress = this;
-}
-
 std::string
 FileArchive::fetch (const std::string & url, const std::string & destdir) const
 {
@@ -213,26 +212,6 @@ FileArchive::fetch (const std::string & url, const std::string & destdir) const
 	return Glib::build_filename (destdir, Glib::path_get_basename (url));
 }
 
-
-int
-FileArchive::make_local (const std::string& destdir)
-{
-	if (!_req.is_remote()) {
-		return 0;
-	}
-
-	std::string downloaded = fetch (_req.url, destdir);
-
-	if (downloaded.empty()) {
-		return -1;
-	}
-
-	_req.url = strdup (downloaded.c_str());
-	_req.mp.progress = 0;
-
-	return 0;
-}
-
 int
 FileArchive::inflate (const std::string& destdir)
 {
@@ -277,10 +256,9 @@ FileArchive::next_file_name ()
 	}
 
 	int r = archive_read_next_header (_archive, &_current_entry);
-	if (!_req.mp.progress) {
+	if (_progress && _req.mp.length > 0) {
-		// file i/o -- not URL
 		const uint64_t read = archive_filter_bytes (_archive, -1);
-		progress (read, _req.mp.length);
+		_progress->set_progress ((float) read / _req.mp.length);
 	}
 
 	if (r == ARCHIVE_EOF) {
@@ -384,7 +362,6 @@ FileArchive::extract_url ()
 {
 	_req.mp.reset ();
 	pthread_create (&_tid, NULL, get_url, (void*)&_req);
-
 	struct archive* a = setup_archive ();
 	archive_read_open (a, (void*)&_req.mp, NULL, ar_read, NULL);
 	int rv = do_extract (a);
@@ -400,10 +377,9 @@ FileArchive::get_contents (struct archive* a)
 	struct archive_entry* entry;
 	for (;;) {
 		int r = archive_read_next_header (a, &entry);
-		if (!_req.mp.progress) {
+		if (_progress && _req.mp.length > 0) {
-			// file i/o -- not URL
 			const uint64_t read = archive_filter_bytes (a, -1);
-			progress (read, _req.mp.length);
+			_progress->set_progress ((float) read / _req.mp.length);
 		}
 		if (r == ARCHIVE_EOF) {
 			break;
@@ -435,10 +411,13 @@ FileArchive::do_extract (struct archive* a)
 	for (;;) {
 		int r = archive_read_next_header (a, &entry);
 
-		if (_req.mp.progress) {
+		if (_progress && _req.mp.length > 0) {
-			// file i/o -- not URL
 			const uint64_t read = archive_filter_bytes (a, -1);
-			progress (read, _req.mp.length);
+			_progress->set_progress ((float)read / _req.mp.length);
+		}
+
+		if (_progress && _progress->cancelled ()) {
+			break;
 		}
 
 		if (r == ARCHIVE_EOF) {
@@ -468,6 +447,10 @@ FileArchive::do_extract (struct archive* a)
 		}
 	}
 
+	if (_progress && !_progress->cancelled ()) {
+		_progress->set_progress (1.f);
+	}
+
 	archive_read_close (a);
 	archive_read_free (a);
 	archive_write_close(ext);
@@ -503,6 +486,10 @@ FileArchive::create (const std::string& srcdir, CompressionLevel compression_lev
 int
 FileArchive::create (const std::map<std::string, std::string>& filemap, CompressionLevel compression_level)
 {
+	if (_req.is_remote ()) {
+		return -1;
+	}
+
 	struct archive *a;
 	struct archive_entry *entry;
 
@@ -521,7 +508,9 @@ FileArchive::create (const std::map<std::string, std::string>& filemap, Compress
 		return -1;
 	}
 
-	progress (0, total_bytes);
+	if (_progress) {
+		_progress->set_progress (0);
+	}
 
 	a = archive_write_new ();
 	archive_write_set_format_pax_restricted (a);
@@ -574,16 +563,32 @@ FileArchive::create (const std::map<std::string, std::string>& filemap, Compress
 		while (len > 0) {
 			read_bytes += len;
 			archive_write_data (a, buf, len);
-			progress (read_bytes, total_bytes);
+			if (_progress) {
+				_progress->set_progress ((float)read_bytes / total_bytes);
+				if (_progress->cancelled ()) {
+					break;
+				}
+			}
 			len = read (fd, buf, sizeof (buf));
 		}
 		close (fd);
+		if (_progress && _progress->cancelled ()) {
+			break;
+		}
 	}
 
 	archive_entry_free (entry);
 	archive_write_close (a);
 	archive_write_free (a);
 
+	if (_progress) {
+		if (_progress->cancelled ()) {
+			g_unlink (_req.url);
+		} else {
+			_progress->set_progress (1.f);
+		}
+	}
+
 #ifndef NDEBUG
 	const int64_t elapsed_time_us = g_get_monotonic_time() - archive_start_time;
 	std::cerr << "archived in " << std::fixed << std::setprecision (2) << elapsed_time_us / 1000000. << " sec\n";

libs/pbd/inflater.cc

@@ -27,7 +27,7 @@
 using namespace PBD;
 
 Inflater::Inflater (std::string const & ap, std::string const & dd)
-	: FileArchive (ap)
+	: FileArchive (ap, this)
 	, thread (0)
 	, _status (-1) /* means "unknown" */
 	, archive_path (ap)
@@ -51,8 +51,6 @@ Inflater::start ()
 void
 Inflater::threaded_inflate ()
 {
-	require_progress ();
-
 	try {
 		std::string pwd (Glib::get_current_dir ());
 
@@ -72,6 +70,11 @@ Inflater::threaded_inflate ()
 	 * set to be >= 0
 	 */
 
-	progress (1, 1);
+	set_progress (1);
 }
 
+void
+Inflater::set_overall_progress (float p)
+{
+	Progress (p); /* EMIT SIGNAL */
+}

libs/pbd/pbd/file_archive.h

@@ -30,18 +30,17 @@
 
 
 namespace PBD {
+class Progress;
 
 class LIBPBD_API FileArchive
 {
 	public:
-		FileArchive (const std::string& url);
+		FileArchive (const std::string& url, Progress* p = NULL);
 		~FileArchive ();
 
 		int inflate (const std::string& destdir);
 		std::vector<std::string> contents ();
 
-		int make_local (const std::string& destdir);
-
 		std::string next_file_name ();
 		int extract_current_file (const std::string& destpath);
 
@@ -57,15 +56,12 @@ class LIBPBD_API FileArchive
 		int create (const std::string& srcdir, CompressionLevel compression_level = CompressGood);
 		int create (const std::map <std::string, std::string>& filemap, CompressionLevel compression_level = CompressGood);
 
-		PBD::Signal2<void, size_t, size_t> progress; // TODO
-
-		void require_progress ();
-
 		struct MemPipe {
 			public:
-				MemPipe ()
+				MemPipe (Progress* p)
 					: data (NULL)
-					, progress (0)
+					, query_length (false)
+					, progress (p)
 				{
 					pthread_mutex_init (&_lock, NULL);
 					pthread_cond_init (&_ready, NULL);
@@ -90,7 +86,7 @@ class LIBPBD_API FileArchive
 					size = 0;
 					done = false;
 					processed = 0;
-					length = -1;
+					length = 0;
 					unlock ();
 				}
 
@@ -104,9 +100,11 @@ class LIBPBD_API FileArchive
 				size_t   size;
 				bool     done;
 
-				size_t   processed;
+				size_t processed;
-				size_t   length;
+				size_t length;
-				FileArchive* progress;
+				bool   query_length;
+
+				Progress* progress;
 
 			private:
 				pthread_mutex_t _lock;
@@ -115,7 +113,8 @@ class LIBPBD_API FileArchive
 
 		struct Request {
 			public:
-				Request (const std::string& u)
+				Request (const std::string& u, Progress* p)
+					: mp (p)
 				{
 					if (u.size () > 0) {
 						url = strdup (u.c_str());
@@ -142,7 +141,6 @@ class LIBPBD_API FileArchive
 		};
 
 	private:
-
 		int process_file ();
 		int process_url ();
 
@@ -159,13 +157,15 @@ class LIBPBD_API FileArchive
 
 		struct archive* setup_file_archive ();
 
+		std::string fetch (const std::string & url, const std::string& destdir) const;
+
 		Request   _req;
 		pthread_t _tid;
 
+		Progress* _progress;
+
 		struct archive_entry* _current_entry;
 		struct archive* _archive;
-
-		std::string fetch (const std::string & url, const std::string& destdir) const;
 };
 
 } /* namespace */

libs/pbd/pbd/inflater.h

@@ -23,6 +23,7 @@
 
 #include "pbd/file_archive.h"
 #include "pbd/libpbd_visibility.h"
+#include "pbd/progress.h"
 
 namespace PBD {
 	class Thread;
@@ -30,7 +31,7 @@ namespace PBD {
 
 namespace PBD {
 
-class LIBPBD_API Inflater : public PBD::FileArchive
+class LIBPBD_API Inflater : public PBD::FileArchive , public PBD::Progress
 {
   public:
 	Inflater (std::string const & archive_path, std::string const & destdir);
@@ -40,13 +41,17 @@ class LIBPBD_API Inflater : public PBD::FileArchive
 	bool running() const { return thread != 0; }
 	int  status() const { return _status; }
 
-  private:
+	PBD::Signal1<void, float> Progress;
+
+private:
 	PBD::Thread* thread;
 	int _status;
 	std::string archive_path;
 	std::string destdir;
 
 	void threaded_inflate ();
+
+	void set_overall_progress (float p);
 };
 
 } /* namespace */