Implement FileArchive progress/cancel
* Use PBD::Progress API * Allow to cancel extraction and compression * Fix querying download size
This commit is contained in:
parent
2cb4e8a6ca
commit
c9929698ee
GPG Key ID:
A090BCE02CF57F04
|
|
@ -39,6 +39,7 @@
|
|||
#include "pbd/file_archive.h"
|
||||
#include "pbd/file_utils.h"
|
||||
#include "pbd/pthread_utils.h"
|
||||
#include "pbd/progress.h"
|
||||
|
||||
using namespace PBD;
|
||||
|
||||
|
|
@ -69,11 +70,14 @@ get_url (void* arg)
|
|||
curl_easy_setopt (curl, CURLOPT_FOLLOWLOCATION, 1L);
|
||||
|
||||
/* get size */
|
||||
if (r->mp.progress) {
|
||||
if (r->mp.query_length) {
|
||||
double content_length = 0;
|
||||
curl_easy_setopt(curl, CURLOPT_NOBODY, 1L);
|
||||
curl_easy_setopt(curl, CURLOPT_HEADER, 0L);
|
||||
curl_easy_perform (curl);
|
||||
curl_easy_getinfo (curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &r->mp.length);
|
||||
if (CURLE_OK == curl_easy_getinfo (curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &content_length) && content_length > 0) {
|
||||
r->mp.length = content_length;
|
||||
}
|
||||
}
|
||||
|
||||
curl_easy_setopt(curl, CURLOPT_NOBODY, 0L);
|
||||
|
|
@ -113,8 +117,8 @@ ar_read (struct archive* a, void* d, const void** buff)
|
|||
p->size -= rv;
|
||||
p->processed += rv;
|
||||
*buff = p->buf;
|
||||
if (p->progress) {
|
||||
p->progress->progress (p->processed, p->length);
|
||||
if (p->progress && p->length > 0) {
|
||||
p->progress->set_progress ((float)p->processed / p->length);
|
||||
}
|
||||
p->unlock ();
|
||||
return rv;
|
||||
|
|
@ -153,8 +157,9 @@ setup_archive ()
|
|||
return a;
|
||||
}
|
||||
|
||||
FileArchive::FileArchive (const std::string& url)
|
||||
: _req (url)
|
||||
FileArchive::FileArchive (const std::string& url, Progress* p)
|
||||
: _req (url, p)
|
||||
, _progress (p)
|
||||
, _current_entry (0)
|
||||
, _archive (0)
|
||||
{
|
||||
|
|
@ -164,9 +169,9 @@ FileArchive::FileArchive (const std::string& url)
|
|||
}
|
||||
|
||||
if (_req.is_remote ()) {
|
||||
_req.mp.progress = this;
|
||||
_req.mp.query_length = true;
|
||||
} else {
|
||||
_req.mp.progress = 0;
|
||||
_req.mp.query_length = false;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -178,12 +183,6 @@ FileArchive::~FileArchive ()
|
|||
}
|
||||
}
|
||||
|
||||
void
|
||||
FileArchive::require_progress ()
|
||||
{
|
||||
_req.mp.progress = this;
|
||||
}
|
||||
|
||||
std::string
|
||||
FileArchive::fetch (const std::string & url, const std::string & destdir) const
|
||||
{
|
||||
|
|
@ -213,26 +212,6 @@ FileArchive::fetch (const std::string & url, const std::string & destdir) const
|
|||
return Glib::build_filename (destdir, Glib::path_get_basename (url));
|
||||
}
|
||||
|
||||
|
||||
int
|
||||
FileArchive::make_local (const std::string& destdir)
|
||||
{
|
||||
if (!_req.is_remote()) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
std::string downloaded = fetch (_req.url, destdir);
|
||||
|
||||
if (downloaded.empty()) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
_req.url = strdup (downloaded.c_str());
|
||||
_req.mp.progress = 0;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int
|
||||
FileArchive::inflate (const std::string& destdir)
|
||||
{
|
||||
|
|
@ -277,10 +256,9 @@ FileArchive::next_file_name ()
|
|||
}
|
||||
|
||||
int r = archive_read_next_header (_archive, &_current_entry);
|
||||
if (!_req.mp.progress) {
|
||||
// file i/o -- not URL
|
||||
if (_progress && _req.mp.length > 0) {
|
||||
const uint64_t read = archive_filter_bytes (_archive, -1);
|
||||
progress (read, _req.mp.length);
|
||||
_progress->set_progress ((float) read / _req.mp.length);
|
||||
}
|
||||
|
||||
if (r == ARCHIVE_EOF) {
|
||||
|
|
@ -384,7 +362,6 @@ FileArchive::extract_url ()
|
|||
{
|
||||
_req.mp.reset ();
|
||||
pthread_create (&_tid, NULL, get_url, (void*)&_req);
|
||||
|
||||
struct archive* a = setup_archive ();
|
||||
archive_read_open (a, (void*)&_req.mp, NULL, ar_read, NULL);
|
||||
int rv = do_extract (a);
|
||||
|
|
@ -400,10 +377,9 @@ FileArchive::get_contents (struct archive* a)
|
|||
struct archive_entry* entry;
|
||||
for (;;) {
|
||||
int r = archive_read_next_header (a, &entry);
|
||||
if (!_req.mp.progress) {
|
||||
// file i/o -- not URL
|
||||
if (_progress && _req.mp.length > 0) {
|
||||
const uint64_t read = archive_filter_bytes (a, -1);
|
||||
progress (read, _req.mp.length);
|
||||
_progress->set_progress ((float) read / _req.mp.length);
|
||||
}
|
||||
if (r == ARCHIVE_EOF) {
|
||||
break;
|
||||
|
|
@ -435,10 +411,13 @@ FileArchive::do_extract (struct archive* a)
|
|||
for (;;) {
|
||||
int r = archive_read_next_header (a, &entry);
|
||||
|
||||
if (_req.mp.progress) {
|
||||
// file i/o -- not URL
|
||||
if (_progress && _req.mp.length > 0) {
|
||||
const uint64_t read = archive_filter_bytes (a, -1);
|
||||
progress (read, _req.mp.length);
|
||||
_progress->set_progress ((float)read / _req.mp.length);
|
||||
}
|
||||
|
||||
if (_progress && _progress->cancelled ()) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (r == ARCHIVE_EOF) {
|
||||
|
|
@ -468,6 +447,10 @@ FileArchive::do_extract (struct archive* a)
|
|||
}
|
||||
}
|
||||
|
||||
if (_progress && !_progress->cancelled ()) {
|
||||
_progress->set_progress (1.f);
|
||||
}
|
||||
|
||||
archive_read_close (a);
|
||||
archive_read_free (a);
|
||||
archive_write_close(ext);
|
||||
|
|
@ -503,6 +486,10 @@ FileArchive::create (const std::string& srcdir, CompressionLevel compression_lev
|
|||
int
|
||||
FileArchive::create (const std::map<std::string, std::string>& filemap, CompressionLevel compression_level)
|
||||
{
|
||||
if (_req.is_remote ()) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
struct archive *a;
|
||||
struct archive_entry *entry;
|
||||
|
||||
|
|
@ -521,7 +508,9 @@ FileArchive::create (const std::map<std::string, std::string>& filemap, Compress
|
|||
return -1;
|
||||
}
|
||||
|
||||
progress (0, total_bytes);
|
||||
if (_progress) {
|
||||
_progress->set_progress (0);
|
||||
}
|
||||
|
||||
a = archive_write_new ();
|
||||
archive_write_set_format_pax_restricted (a);
|
||||
|
|
@ -574,16 +563,32 @@ FileArchive::create (const std::map<std::string, std::string>& filemap, Compress
|
|||
while (len > 0) {
|
||||
read_bytes += len;
|
||||
archive_write_data (a, buf, len);
|
||||
progress (read_bytes, total_bytes);
|
||||
if (_progress) {
|
||||
_progress->set_progress ((float)read_bytes / total_bytes);
|
||||
if (_progress->cancelled ()) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
len = read (fd, buf, sizeof (buf));
|
||||
}
|
||||
close (fd);
|
||||
if (_progress && _progress->cancelled ()) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
archive_entry_free (entry);
|
||||
archive_write_close (a);
|
||||
archive_write_free (a);
|
||||
|
||||
if (_progress) {
|
||||
if (_progress->cancelled ()) {
|
||||
g_unlink (_req.url);
|
||||
} else {
|
||||
_progress->set_progress (1.f);
|
||||
}
|
||||
}
|
||||
|
||||
#ifndef NDEBUG
|
||||
const int64_t elapsed_time_us = g_get_monotonic_time() - archive_start_time;
|
||||
std::cerr << "archived in " << std::fixed << std::setprecision (2) << elapsed_time_us / 1000000. << " sec\n";
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@
|
|||
using namespace PBD;
|
||||
|
||||
Inflater::Inflater (std::string const & ap, std::string const & dd)
|
||||
: FileArchive (ap)
|
||||
: FileArchive (ap, this)
|
||||
, thread (0)
|
||||
, _status (-1) /* means "unknown" */
|
||||
, archive_path (ap)
|
||||
|
|
@ -51,8 +51,6 @@ Inflater::start ()
|
|||
void
|
||||
Inflater::threaded_inflate ()
|
||||
{
|
||||
require_progress ();
|
||||
|
||||
try {
|
||||
std::string pwd (Glib::get_current_dir ());
|
||||
|
||||
|
|
@ -72,6 +70,11 @@ Inflater::threaded_inflate ()
|
|||
* set to be >= 0
|
||||
*/
|
||||
|
||||
progress (1, 1);
|
||||
set_progress (1);
|
||||
}
|
||||
|
||||
void
|
||||
Inflater::set_overall_progress (float p)
|
||||
{
|
||||
Progress (p); /* EMIT SIGNAL */
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,18 +30,17 @@
|
|||
|
||||
|
||||
namespace PBD {
|
||||
class Progress;
|
||||
|
||||
class LIBPBD_API FileArchive
|
||||
{
|
||||
public:
|
||||
FileArchive (const std::string& url);
|
||||
FileArchive (const std::string& url, Progress* p = NULL);
|
||||
~FileArchive ();
|
||||
|
||||
int inflate (const std::string& destdir);
|
||||
std::vector<std::string> contents ();
|
||||
|
||||
int make_local (const std::string& destdir);
|
||||
|
||||
std::string next_file_name ();
|
||||
int extract_current_file (const std::string& destpath);
|
||||
|
||||
|
|
@ -57,15 +56,12 @@ class LIBPBD_API FileArchive
|
|||
int create (const std::string& srcdir, CompressionLevel compression_level = CompressGood);
|
||||
int create (const std::map <std::string, std::string>& filemap, CompressionLevel compression_level = CompressGood);
|
||||
|
||||
PBD::Signal2<void, size_t, size_t> progress; // TODO
|
||||
|
||||
void require_progress ();
|
||||
|
||||
struct MemPipe {
|
||||
public:
|
||||
MemPipe ()
|
||||
MemPipe (Progress* p)
|
||||
: data (NULL)
|
||||
, progress (0)
|
||||
, query_length (false)
|
||||
, progress (p)
|
||||
{
|
||||
pthread_mutex_init (&_lock, NULL);
|
||||
pthread_cond_init (&_ready, NULL);
|
||||
|
|
@ -90,7 +86,7 @@ class LIBPBD_API FileArchive
|
|||
size = 0;
|
||||
done = false;
|
||||
processed = 0;
|
||||
length = -1;
|
||||
length = 0;
|
||||
unlock ();
|
||||
}
|
||||
|
||||
|
|
@ -104,9 +100,11 @@ class LIBPBD_API FileArchive
|
|||
size_t size;
|
||||
bool done;
|
||||
|
||||
size_t processed;
|
||||
size_t length;
|
||||
FileArchive* progress;
|
||||
size_t processed;
|
||||
size_t length;
|
||||
bool query_length;
|
||||
|
||||
Progress* progress;
|
||||
|
||||
private:
|
||||
pthread_mutex_t _lock;
|
||||
|
|
@ -115,7 +113,8 @@ class LIBPBD_API FileArchive
|
|||
|
||||
struct Request {
|
||||
public:
|
||||
Request (const std::string& u)
|
||||
Request (const std::string& u, Progress* p)
|
||||
: mp (p)
|
||||
{
|
||||
if (u.size () > 0) {
|
||||
url = strdup (u.c_str());
|
||||
|
|
@ -142,7 +141,6 @@ class LIBPBD_API FileArchive
|
|||
};
|
||||
|
||||
private:
|
||||
|
||||
int process_file ();
|
||||
int process_url ();
|
||||
|
||||
|
|
@ -159,13 +157,15 @@ class LIBPBD_API FileArchive
|
|||
|
||||
struct archive* setup_file_archive ();
|
||||
|
||||
std::string fetch (const std::string & url, const std::string& destdir) const;
|
||||
|
||||
Request _req;
|
||||
pthread_t _tid;
|
||||
|
||||
Progress* _progress;
|
||||
|
||||
struct archive_entry* _current_entry;
|
||||
struct archive* _archive;
|
||||
|
||||
std::string fetch (const std::string & url, const std::string& destdir) const;
|
||||
};
|
||||
|
||||
} /* namespace */
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@
|
|||
|
||||
#include "pbd/file_archive.h"
|
||||
#include "pbd/libpbd_visibility.h"
|
||||
#include "pbd/progress.h"
|
||||
|
||||
namespace PBD {
|
||||
class Thread;
|
||||
|
|
@ -30,7 +31,7 @@ namespace PBD {
|
|||
|
||||
namespace PBD {
|
||||
|
||||
class LIBPBD_API Inflater : public PBD::FileArchive
|
||||
class LIBPBD_API Inflater : public PBD::FileArchive , public PBD::Progress
|
||||
{
|
||||
public:
|
||||
Inflater (std::string const & archive_path, std::string const & destdir);
|
||||
|
|
@ -40,13 +41,17 @@ class LIBPBD_API Inflater : public PBD::FileArchive
|
|||
bool running() const { return thread != 0; }
|
||||
int status() const { return _status; }
|
||||
|
||||
private:
|
||||
PBD::Signal1<void, float> Progress;
|
||||
|
||||
private:
|
||||
PBD::Thread* thread;
|
||||
int _status;
|
||||
std::string archive_path;
|
||||
std::string destdir;
|
||||
|
||||
void threaded_inflate ();
|
||||
|
||||
void set_overall_progress (float p);
|
||||
};
|
||||
|
||||
} /* namespace */
|
||||
|
|
|
|||
Loading…
Reference in New Issue