Add support for built-in file/url unzip/untar
This introduces new build-dependency: libarchive (http://www.libarchive.org/)
This commit is contained in:
parent
3193aa93ce
commit
53ee3e2e72
|
|
@ -0,0 +1,292 @@
|
|||
/*
|
||||
* Copyright (C) 2016 Robin Gareus <robin@gareus.org>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License
|
||||
* as published by the Free Software Foundation; either version 2
|
||||
* of the License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <cstdio>
|
||||
|
||||
#include "pbd/gstdio_compat.h"
|
||||
#include <glibmm.h>
|
||||
|
||||
#include <archive.h>
|
||||
#include <archive_entry.h>
|
||||
#include <curl/curl.h>
|
||||
|
||||
#include "pbd/failed_constructor.h"
|
||||
#include "pbd/file_archive.h"
|
||||
|
||||
using namespace PBD;
|
||||
|
||||
static size_t
|
||||
write_callback (void* buffer, size_t size, size_t nmemb, void* d)
|
||||
{
|
||||
FileArchive::MemPipe* p = (FileArchive::MemPipe*)d;
|
||||
size_t realsize = size * nmemb;
|
||||
|
||||
p->lock ();
|
||||
p->data = (uint8_t*) realloc ((void*) p->data, p->size + realsize);
|
||||
memcpy (&p->data[p->size], buffer, realsize);
|
||||
p->size += realsize;
|
||||
p->signal ();
|
||||
p->unlock ();
|
||||
return realsize;
|
||||
}
|
||||
|
||||
static void*
|
||||
get_url (void* arg)
|
||||
{
|
||||
FileArchive::Request* r = (FileArchive::Request*) arg;
|
||||
CURL* curl;
|
||||
|
||||
curl = curl_easy_init ();
|
||||
curl_easy_setopt (curl, CURLOPT_URL, r->url);
|
||||
|
||||
curl_easy_setopt (curl, CURLOPT_WRITEFUNCTION, write_callback);
|
||||
curl_easy_setopt (curl, CURLOPT_WRITEDATA, (void*) &r->mp);
|
||||
curl_easy_setopt (curl, CURLOPT_FOLLOWLOCATION, 1L);
|
||||
|
||||
curl_easy_perform (curl);
|
||||
curl_easy_cleanup (curl);
|
||||
|
||||
r->mp.lock ();
|
||||
r->mp.done = 1;
|
||||
r->mp.signal ();
|
||||
r->mp.unlock ();
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static ssize_t
|
||||
ar_read (struct archive* a, void* d, const void** buff)
|
||||
{
|
||||
FileArchive::MemPipe* p = (FileArchive::MemPipe*)d;
|
||||
size_t rv;
|
||||
|
||||
p->lock ();
|
||||
while (p->size == 0) {
|
||||
if (p->done) {
|
||||
p->unlock ();
|
||||
return 0;
|
||||
}
|
||||
p->wait ();
|
||||
}
|
||||
|
||||
rv = p->size > 8192 ? 8192 : p->size;
|
||||
memcpy (p->buf, p->data, rv);
|
||||
if (p->size > rv) {
|
||||
memmove (p->data, &p->data[rv], p->size - rv);
|
||||
}
|
||||
p->size -= rv;
|
||||
*buff = p->buf;
|
||||
p->unlock ();
|
||||
return rv;
|
||||
}
|
||||
|
||||
static int
|
||||
ar_copy_data (struct archive *ar, struct archive *aw)
|
||||
{
|
||||
for (;;) {
|
||||
const void *buff;
|
||||
size_t size;
|
||||
int64_t offset;
|
||||
int r;
|
||||
r = archive_read_data_block (ar, &buff, &size, &offset);
|
||||
if (r == ARCHIVE_EOF) {
|
||||
return (ARCHIVE_OK);
|
||||
}
|
||||
if (r != ARCHIVE_OK) {
|
||||
return (r);
|
||||
}
|
||||
r = archive_write_data_block (aw, buff, size, offset);
|
||||
if (r != ARCHIVE_OK) {
|
||||
fprintf (stderr, "Extract/Write Archive: %s", archive_error_string(aw));
|
||||
return (r);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static struct archive*
|
||||
setup_archive ()
|
||||
{
|
||||
struct archive* a;
|
||||
a = archive_read_new ();
|
||||
archive_read_support_filter_all (a);
|
||||
archive_read_support_format_all (a);
|
||||
return a;
|
||||
}
|
||||
|
||||
|
||||
FileArchive::FileArchive (const std::string& url)
|
||||
: _req (url)
|
||||
{
|
||||
if (!_req.url) {
|
||||
fprintf (stderr, "Invalid Archive URL/filename\n");
|
||||
throw failed_constructor ();
|
||||
}
|
||||
}
|
||||
|
||||
int
|
||||
FileArchive::inflate (const std::string& destdir)
|
||||
{
|
||||
int rv = -1;
|
||||
std::string pwd (Glib::get_current_dir ());
|
||||
|
||||
if (g_chdir (destdir.c_str ())) {
|
||||
fprintf (stderr, "Archive: cannot chdir to '%s'\n", destdir.c_str ());
|
||||
return rv;
|
||||
}
|
||||
|
||||
if (_req.is_remote ()) {
|
||||
rv = extract_url ();
|
||||
} else {
|
||||
rv = extract_file ();
|
||||
}
|
||||
|
||||
g_chdir (pwd.c_str());
|
||||
return rv;
|
||||
}
|
||||
|
||||
std::vector<std::string>
|
||||
FileArchive::contents ()
|
||||
{
|
||||
if (_req.is_remote ()) {
|
||||
return contents_url ();
|
||||
} else {
|
||||
return contents_file ();
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<std::string>
|
||||
FileArchive::contents_file ()
|
||||
{
|
||||
struct archive* a = setup_archive ();
|
||||
if (ARCHIVE_OK != archive_read_open_filename (a, _req.url, 8192)) {
|
||||
fprintf (stderr, "Error opening archive: %s\n", archive_error_string(a));
|
||||
return std::vector<std::string> ();
|
||||
}
|
||||
return get_contents (a);
|
||||
}
|
||||
|
||||
std::vector<std::string>
|
||||
FileArchive::contents_url ()
|
||||
{
|
||||
_req.mp.reset ();
|
||||
pthread_create (&_tid, NULL, get_url, (void*)&_req);
|
||||
|
||||
struct archive* a = setup_archive ();
|
||||
archive_read_open (a, (void*)&_req.mp, NULL, ar_read, NULL);
|
||||
std::vector<std::string> rv (get_contents (a));
|
||||
|
||||
pthread_join (_tid, NULL);
|
||||
return rv;
|
||||
}
|
||||
|
||||
int
|
||||
FileArchive::extract_file ()
|
||||
{
|
||||
struct archive* a = setup_archive ();
|
||||
if (ARCHIVE_OK != archive_read_open_filename (a, _req.url, 8192)) {
|
||||
fprintf (stderr, "Error opening archive: %s\n", archive_error_string(a));
|
||||
return -1;
|
||||
}
|
||||
return do_extract (a);
|
||||
}
|
||||
|
||||
int
|
||||
FileArchive::extract_url ()
|
||||
{
|
||||
_req.mp.reset ();
|
||||
pthread_create (&_tid, NULL, get_url, (void*)&_req);
|
||||
|
||||
struct archive* a = setup_archive ();
|
||||
archive_read_open (a, (void*)&_req.mp, NULL, ar_read, NULL);
|
||||
int rv = do_extract (a);
|
||||
|
||||
pthread_join (_tid, NULL);
|
||||
return rv;
|
||||
}
|
||||
|
||||
std::vector<std::string>
|
||||
FileArchive::get_contents (struct archive* a)
|
||||
{
|
||||
std::vector<std::string> rv;
|
||||
struct archive_entry* entry;
|
||||
for (;;) {
|
||||
int r = archive_read_next_header (a, &entry);
|
||||
if (r == ARCHIVE_EOF) {
|
||||
break;
|
||||
}
|
||||
if (r != ARCHIVE_OK) {
|
||||
fprintf (stderr, "Error reading archive: %s\n", archive_error_string(a));
|
||||
break;
|
||||
}
|
||||
rv.push_back (archive_entry_pathname (entry));
|
||||
}
|
||||
|
||||
archive_read_close (a);
|
||||
archive_read_free (a);
|
||||
return rv;
|
||||
}
|
||||
|
||||
int
|
||||
FileArchive::do_extract (struct archive* a)
|
||||
{
|
||||
int flags = ARCHIVE_EXTRACT_TIME;
|
||||
|
||||
int rv = 0;
|
||||
struct archive_entry* entry;
|
||||
struct archive *ext;
|
||||
|
||||
ext = archive_write_disk_new();
|
||||
archive_write_disk_set_options(ext, flags);
|
||||
|
||||
for (;;) {
|
||||
int r = archive_read_next_header (a, &entry);
|
||||
if (r == ARCHIVE_EOF) {
|
||||
break;
|
||||
}
|
||||
if (r != ARCHIVE_OK) {
|
||||
fprintf (stderr, "Error reading archive: %s\n", archive_error_string(a));
|
||||
break;
|
||||
}
|
||||
|
||||
#if 0 // hacky alternative to chdir
|
||||
const std::string full_path = Glib::build_filename (destdir, archive_entry_pathname (entry));
|
||||
archive_entry_set_pathname (entry, full_path.c_str());
|
||||
#endif
|
||||
|
||||
r = archive_write_header(ext, entry);
|
||||
if (r != ARCHIVE_OK) {
|
||||
fprintf (stderr, "Extracting archive: %s\n", archive_error_string(ext));
|
||||
} else {
|
||||
ar_copy_data (a, ext);
|
||||
r = archive_write_finish_entry (ext);
|
||||
if (r != ARCHIVE_OK) {
|
||||
fprintf (stderr, "Extracting archive: %s\n", archive_error_string(ext));
|
||||
rv = -1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
archive_read_close (a);
|
||||
archive_read_free (a);
|
||||
archive_write_close(ext);
|
||||
archive_write_free(ext);
|
||||
return rv;
|
||||
}
|
||||
|
|
@ -0,0 +1,138 @@
|
|||
/*
|
||||
* Copyright (C) 2016 Robin Gareus <robin@gareus.org>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License
|
||||
* as published by the Free Software Foundation; either version 2
|
||||
* of the License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
*
|
||||
*/
|
||||
#ifndef _pbd_archive_h_
|
||||
#define _pbd_archive_h_
|
||||
|
||||
#include <pthread.h>
|
||||
|
||||
#include "pbd/signals.h"
|
||||
|
||||
#ifndef LIBPBD_API
|
||||
#include "pbd/libpbd_visibility.h"
|
||||
#endif
|
||||
|
||||
|
||||
namespace PBD {
|
||||
|
||||
class LIBPBD_API FileArchive
|
||||
{
|
||||
public:
|
||||
FileArchive (const std::string& url);
|
||||
|
||||
int inflate (const std::string& destdir);
|
||||
std::vector<std::string> contents ();
|
||||
|
||||
//PBD::Signal2<void, size_t, size_t> progress; // TODO
|
||||
|
||||
struct MemPipe {
|
||||
public:
|
||||
MemPipe ()
|
||||
: data (NULL)
|
||||
, size (0)
|
||||
, done (false)
|
||||
{
|
||||
pthread_mutex_init (&_lock, NULL);
|
||||
pthread_cond_init (&_ready, NULL);
|
||||
}
|
||||
|
||||
~MemPipe ()
|
||||
{
|
||||
lock ();
|
||||
free (data);
|
||||
unlock ();
|
||||
|
||||
pthread_mutex_destroy (&_lock);
|
||||
pthread_cond_destroy (&_ready);
|
||||
}
|
||||
|
||||
void reset ()
|
||||
{
|
||||
lock ();
|
||||
free (data);
|
||||
data = 0;
|
||||
size = 0;
|
||||
done = false;
|
||||
unlock ();
|
||||
}
|
||||
|
||||
void lock () { pthread_mutex_lock (&_lock); }
|
||||
void unlock () { pthread_mutex_unlock (&_lock); }
|
||||
void signal () { pthread_cond_signal (&_ready); }
|
||||
void wait () { pthread_cond_wait (&_ready, &_lock); }
|
||||
|
||||
uint8_t buf[8192];
|
||||
uint8_t* data;
|
||||
size_t size;
|
||||
bool done;
|
||||
|
||||
private:
|
||||
pthread_mutex_t _lock;
|
||||
pthread_cond_t _ready;
|
||||
};
|
||||
|
||||
struct Request {
|
||||
public:
|
||||
Request (const std::string& u)
|
||||
{
|
||||
if (u.size () > 0) {
|
||||
url = strdup (u.c_str());
|
||||
} else {
|
||||
url = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
~Request ()
|
||||
{
|
||||
free (url);
|
||||
}
|
||||
|
||||
bool is_remote () const
|
||||
{
|
||||
if (!strncmp (url, "https://", 8) || !strncmp (url, "http://", 7) || !strncmp (url, "ftp://", 6)) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
char* url;
|
||||
MemPipe mp;
|
||||
};
|
||||
|
||||
private:
|
||||
|
||||
int process_file ();
|
||||
int process_url ();
|
||||
|
||||
std::vector<std::string> contents_url ();
|
||||
std::vector<std::string> contents_file ();
|
||||
|
||||
int extract_url ();
|
||||
int extract_file ();
|
||||
|
||||
int do_extract (struct archive* a);
|
||||
std::vector<std::string> get_contents (struct archive *a);
|
||||
|
||||
bool is_url ();
|
||||
|
||||
Request _req;
|
||||
pthread_t _tid;
|
||||
};
|
||||
|
||||
} /* namespace */
|
||||
#endif // _reallocpool_h_
|
||||
|
|
@ -47,6 +47,7 @@ libpbd_sources = [
|
|||
'epa.cc',
|
||||
'error.cc',
|
||||
'ffs.cc',
|
||||
'file_archive.cc',
|
||||
'file_utils.cc',
|
||||
'fpu.cc',
|
||||
'id.cc',
|
||||
|
|
@ -134,7 +135,7 @@ def build(bld):
|
|||
obj.includes = ['.']
|
||||
obj.name = 'libpbd'
|
||||
obj.target = 'pbd'
|
||||
obj.uselib = 'GLIBMM SIGCPP XML UUID SNDFILE GIOMM'
|
||||
obj.uselib = 'GLIBMM SIGCPP XML UUID SNDFILE GIOMM ARCHIVE CURL'
|
||||
if sys.platform == 'darwin':
|
||||
TaskGen.task_gen.mappings['.mm'] = TaskGen.task_gen.mappings['.cc']
|
||||
if 'cocoa_open_uri.mm' not in obj.source:
|
||||
|
|
|
|||
1
wscript
1
wscript
|
|
@ -970,6 +970,7 @@ def configure(conf):
|
|||
autowaf.check_pkg(conf, 'sndfile', uselib_store='SNDFILE', atleast_version='1.0.18', mandatory=True)
|
||||
autowaf.check_pkg(conf, 'giomm-2.4', uselib_store='GIOMM', atleast_version='2.2', mandatory=True)
|
||||
autowaf.check_pkg(conf, 'libcurl', uselib_store='CURL', atleast_version='7.0.0', mandatory=True)
|
||||
autowaf.check_pkg(conf, 'libarchive', uselib_store='ARCHIVE', atleast_version='3.0.0', mandatory=True)
|
||||
autowaf.check_pkg(conf, 'liblo', uselib_store='LO', atleast_version='0.26', mandatory=True)
|
||||
autowaf.check_pkg(conf, 'taglib', uselib_store='TAGLIB', atleast_version='1.6', mandatory=True)
|
||||
autowaf.check_pkg(conf, 'vamp-sdk', uselib_store='VAMPSDK', atleast_version='2.1', mandatory=True)
|
||||
|
|
|
|||
Loading…
Reference in New Issue