13
0

library fetcher: API changes and implementation improvements including threaded download

This commit is contained in:
Paul Davis 2022-09-06 22:19:22 -06:00
parent d388a50579
commit 78d96ed0c6
2 changed files with 258 additions and 41 deletions

View File

@ -1,39 +1,89 @@
#ifndef __libardour_library_h__
#define __libardour_library_h__
#include <cstdio>
#include <cstdint>
#include <atomic>
#include <string>
#include <vector>
#include <thread>
#include <curl/curl.h>
namespace ARDOUR {
class LibraryDescription
{
public:
LibraryDescription (std::string const & n, std::string const & d, std::string const & f, std::string const & l)
: _name (n), _description (d), _file (f), _license (l) {}
LibraryDescription (std::string const & n, std::string const & d, std::string const & u, std::string const & l, std::string const & td)
: _name (n), _description (d), _url (u), _license (l), _toplevel_dir (td), _installed (false) {}
std::string const & name() const { return _name; }
std::string const & description() const { return _description; }
std::string const & file() const { return _file; }
std::string const & url() const { return _url; }
std::string const & license() const { return _license; }
std::string const & toplevel_dir() const { return _toplevel_dir; }
bool installed() const { return _installed; }
void set_installed (bool yn) { _installed = yn; }
private:
std::string _name;
std::string _description;
std::string _file;
std::string _url;
std::string _license;
std::string _toplevel_dir;
bool _installed;
};
class Downloader {
public:
Downloader (std::string const & url, std::string const & destdir);
int start ();
void cleanup ();
void cancel ();
double progress() const;
uint64_t download_size() const { return _download_size; }
uint64_t downloaded () const { return _downloaded; }
/* public so it can be called from a static C function */
size_t write (void *contents, size_t size, size_t nmemb);
int status() const { return _status; }
std::string download_path() const;
private:
std::string url;
std::string destdir;
std::string file_path;
FILE* file;
CURL* curl;
bool _cancel;
double dsize; /* temporary to match CURL API */
std::atomic<uint64_t> _download_size; /* read-only from requestor thread */
std::atomic<uint64_t> _downloaded; /* read-only from requestor thread */
std::atomic<int> _status;
std::thread thr;
void download ();
};
class LibraryFetcher {
public:
int add (std::string const & url);
LibraryFetcher() {}
int add (std::string const & path);
int get_descriptions ();
void foreach_description (boost::function<void (LibraryDescription)> f) const;
bool installed (LibraryDescription const & desc);
void foreach_description (boost::function<void (LibraryDescription)> f);
private:
std::string url;
std::thread thr;
std::vector<LibraryDescription> _descriptions;
std::string install_path_for (LibraryDescription const &);
};
} /* namespace */

View File

@ -1,5 +1,7 @@
#include <curl/curl.h>
#include <glib/gstdio.h>
#include <glibmm/fileutils.h>
#include <glibmm/miscutils.h>
#include "pbd/i18n.h"
@ -59,48 +61,51 @@ LibraryFetcher::get_descriptions ()
XMLNode const & root (*tree.root());
for (auto const & node : root.children()) {
string n, d, f, l;
if (!node->get_property (X_("name"), n) ||
!node->get_property (X_("file"), f) ||
!node->get_property (X_("license"), l)) {
string n, d, u, l, td;
if (!node->get_property (X_("name"), n)) {
std::cerr << "no name\n";
continue;
}
if (!node->get_property (X_("url"), u)) {
std::cerr << "no urln";
continue;
}
if (!node->get_property (X_("license"), l)) {
std::cerr << "no license\n";
continue;
}
if (!node->get_property (X_("toplevel"), td)) {
std::cerr << "no topevel\n";
continue;
}
d = node->content();
_descriptions.push_back (LibraryDescription (n, d, f, l));
_descriptions.push_back (LibraryDescription (n, d, u, l, td));
_descriptions.back().set_installed (installed (_descriptions.back()));
std::cerr << "got description for " << _descriptions.back().name() << std::endl;
}
return 0;
}
int
LibraryFetcher::add (string const & url)
LibraryFetcher::add (std::string const & path)
{
try {
FileArchive archive (url);
const string destdir = Config->get_clip_library_dir ();
if (archive.make_local (destdir)) {
/* failed */
return -1;
}
FileArchive archive (path);
std::vector<std::string> contents = archive.contents ();
std::set<std::string> dirs;
for (auto const & c : contents) {
std::string dir = Glib::path_get_dirname (c);
std::string gp = Glib::path_get_dirname (dir);
/* if the dirname of this dirname is the destdir, then
dirname is a top-level dir that will (should) be
created when/if we inflate the archive
*/
if (gp == destdir) {
dirs.insert (dir);
string::size_type slash = c.find (G_DIR_SEPARATOR);
if (slash != string::npos || slash == c.length() - 1) {
/* no slash or slash at end ... directory ? */
dirs.insert (c);
}
}
@ -108,26 +113,43 @@ LibraryFetcher::add (string const & url)
return -1;
}
if (archive.inflate (destdir)) {
/* cleanup ? */
return -1;
/* Unpack the archive. Likely should have a thread for this */
std::string destdir = Glib::path_get_dirname (path);
{
std::string pwd (Glib::get_current_dir ());
if (g_chdir (destdir.c_str ())) {
error << string_compose (_("cannot chdir to '%1' to unpack library archive (%2)\n"), destdir, strerror (errno)) << endmsg;
return -1;
}
if (archive.inflate (destdir)) {
/* cleanup ? */
return -1;
}
g_chdir (pwd.c_str());
}
std::string path;
std::string newpath;
for (std::set<std::string>::const_iterator d = dirs.begin(); d != dirs.end(); ++d) {
if (d != dirs.begin()) {
path += G_SEARCHPATH_SEPARATOR;
newpath += G_SEARCHPATH_SEPARATOR;
}
path += Glib::build_filename (destdir, *d);
newpath += Glib::build_filename (destdir, *d);
}
assert (!path.empty());
assert (!newpath.empty());
path += G_SEARCHPATH_SEPARATOR;
path += Config->get_sample_lib_path ();
newpath += G_SEARCHPATH_SEPARATOR;
newpath += Config->get_sample_lib_path ();
Config->set_sample_lib_path (path);
Config->set_sample_lib_path (newpath);
::g_unlink (path.c_str());
} catch (...) {
return -1;
@ -135,3 +157,148 @@ LibraryFetcher::add (string const & url)
return 0;
}
void
LibraryFetcher::foreach_description (boost::function<void (LibraryDescription)> f)
{
for (auto ld : _descriptions) {
f (ld);
}
}
std::string
LibraryFetcher::install_path_for (LibraryDescription const & desc)
{
return Glib::build_filename (Config->get_clip_library_dir(), desc.toplevel_dir());
}
bool
LibraryFetcher::installed (LibraryDescription const & desc)
{
std::string path = install_path_for (desc);
if (Glib::file_test (path, Glib::FILE_TEST_EXISTS) && Glib::file_test (path, Glib::FILE_TEST_IS_DIR)) {
return true;
}
return false;
}
static size_t
CurlFILEWrite_CallbackFunc_StdString(void *contents, size_t size, size_t nmemb, Downloader* dl)
{
return dl->write (contents, size, nmemb);
}
size_t
Downloader::write (void *ptr, size_t size, size_t nmemb)
{
if (_cancel) {
fclose (file);
::g_unlink (file_path.c_str());
_downloaded = 0;
_download_size = 0;
return 0;
}
size_t nwritten = fwrite (ptr, size, nmemb, file);
_downloaded += nwritten;
return nwritten;
}
Downloader::Downloader (string const & u, string const & dir)
: url (u)
, destdir (dir)
, _cancel (false)
, _download_size (0)
, _downloaded (0)
{
}
int
Downloader::start ()
{
file_path = Glib::build_filename (Config->get_clip_library_dir(), destdir, Glib::path_get_basename (url));
file = fopen (file_path.c_str(), "w");
if (!file) {
return -1;
}
_cancel = false;
_status = 0; /* unknown at this point */
thr = std::thread (&Downloader::download, this);
return 0;
}
void
Downloader::cleanup ()
{
thr.join ();
}
void
Downloader::cancel ()
{
_cancel = true;
cleanup ();
}
double
Downloader::progress () const
{
return (double) _downloaded / _download_size;
}
void
Downloader::download ()
{
{
curl = curl_easy_init ();
if (!curl) {
_status = -1;
return;
}
/* get size */
curl_easy_setopt(curl, CURLOPT_NOBODY, 1L);
curl_easy_setopt(curl, CURLOPT_HEADER, 0L);
curl_easy_setopt (curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_perform (curl);
curl_easy_getinfo (curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &dsize);
_download_size = dsize;
curl_easy_cleanup (curl);
}
curl = curl_easy_init ();
if (!curl) {
_status = -1;
return;
}
curl_easy_setopt (curl, CURLOPT_URL, url.c_str());
curl_easy_setopt (curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, CurlFILEWrite_CallbackFunc_StdString);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, this);
CURLcode res = curl_easy_perform (curl);
curl_easy_cleanup (curl);
if (res == CURLE_OK) {
_status = 1;
} else {
_status = -1;
}
}
std::string
Downloader::download_path() const
{
/* Can only return the download path if we completed, and completed successfully */
if (_status > 0) {
return file_path;
}
return std::string();
}