From 78d96ed0c6ca69b8d31060b46029cb5412314c4d Mon Sep 17 00:00:00 2001 From: Paul Davis Date: Tue, 6 Sep 2022 22:19:22 -0600 Subject: [PATCH] library fetcher: API changes and implementation improvements including threaded download --- libs/ardour/ardour/library.h | 64 ++++++++-- libs/ardour/library.cc | 235 ++++++++++++++++++++++++++++++----- 2 files changed, 258 insertions(+), 41 deletions(-) diff --git a/libs/ardour/ardour/library.h b/libs/ardour/ardour/library.h index 035017a6bf..93134e3266 100644 --- a/libs/ardour/ardour/library.h +++ b/libs/ardour/ardour/library.h @@ -1,39 +1,89 @@ #ifndef __libardour_library_h__ #define __libardour_library_h__ +#include +#include +#include #include #include #include +#include + namespace ARDOUR { class LibraryDescription { public: - LibraryDescription (std::string const & n, std::string const & d, std::string const & f, std::string const & l) - : _name (n), _description (d), _file (f), _license (l) {} + LibraryDescription (std::string const & n, std::string const & d, std::string const & u, std::string const & l, std::string const & td) + : _name (n), _description (d), _url (u), _license (l), _toplevel_dir (td), _installed (false) {} std::string const & name() const { return _name; } std::string const & description() const { return _description; } - std::string const & file() const { return _file; } + std::string const & url() const { return _url; } std::string const & license() const { return _license; } + std::string const & toplevel_dir() const { return _toplevel_dir; } + + bool installed() const { return _installed; } + void set_installed (bool yn) { _installed = yn; } private: std::string _name; std::string _description; - std::string _file; + std::string _url; std::string _license; + std::string _toplevel_dir; + bool _installed; +}; + +class Downloader { + public: + Downloader (std::string const & url, std::string const & destdir); + + int start (); + void cleanup (); + void cancel (); + double progress() const; + + uint64_t download_size() const { return _download_size; } + uint64_t downloaded () const { return _downloaded; } + + /* public so it can be called from a static C function */ + size_t write (void *contents, size_t size, size_t nmemb); + + int status() const { return _status; } + std::string download_path() const; + + private: + std::string url; + std::string destdir; + std::string file_path; + FILE* file; + CURL* curl; + bool _cancel; + double dsize; /* temporary to match CURL API */ + std::atomic _download_size; /* read-only from requestor thread */ + std::atomic _downloaded; /* read-only from requestor thread */ + std::atomic _status; + std::thread thr; + + void download (); }; class LibraryFetcher { public: - int add (std::string const & url); + LibraryFetcher() {} + + int add (std::string const & path); int get_descriptions (); + void foreach_description (boost::function f) const; + + bool installed (LibraryDescription const & desc); + void foreach_description (boost::function f); private: - std::string url; - std::thread thr; std::vector _descriptions; + std::string install_path_for (LibraryDescription const &); }; } /* namespace */ diff --git a/libs/ardour/library.cc b/libs/ardour/library.cc index f3b38cf79d..849c1eed51 100644 --- a/libs/ardour/library.cc +++ b/libs/ardour/library.cc @@ -1,5 +1,7 @@ #include +#include +#include #include #include "pbd/i18n.h" @@ -59,48 +61,51 @@ LibraryFetcher::get_descriptions () XMLNode const & root (*tree.root()); for (auto const & node : root.children()) { - string n, d, f, l; - if (!node->get_property (X_("name"), n) || - !node->get_property (X_("file"), f) || - !node->get_property (X_("license"), l)) { + string n, d, u, l, td; + if (!node->get_property (X_("name"), n)) { + std::cerr << "no name\n"; continue; } + if (!node->get_property (X_("url"), u)) { + std::cerr << "no urln"; + continue; + } + if (!node->get_property (X_("license"), l)) { + std::cerr << "no license\n"; + continue; + } + + if (!node->get_property (X_("toplevel"), td)) { + std::cerr << "no topevel\n"; + continue; + } + d = node->content(); - _descriptions.push_back (LibraryDescription (n, d, f, l)); + _descriptions.push_back (LibraryDescription (n, d, u, l, td)); + _descriptions.back().set_installed (installed (_descriptions.back())); + + std::cerr << "got description for " << _descriptions.back().name() << std::endl; } return 0; } int -LibraryFetcher::add (string const & url) +LibraryFetcher::add (std::string const & path) { try { - FileArchive archive (url); - const string destdir = Config->get_clip_library_dir (); - - - if (archive.make_local (destdir)) { - /* failed */ - return -1; - } + FileArchive archive (path); std::vector contents = archive.contents (); std::set dirs; for (auto const & c : contents) { - std::string dir = Glib::path_get_dirname (c); - std::string gp = Glib::path_get_dirname (dir); - - /* if the dirname of this dirname is the destdir, then - dirname is a top-level dir that will (should) be - created when/if we inflate the archive - */ - - if (gp == destdir) { - dirs.insert (dir); + string::size_type slash = c.find (G_DIR_SEPARATOR); + if (slash != string::npos || slash == c.length() - 1) { + /* no slash or slash at end ... directory ? */ + dirs.insert (c); } } @@ -108,26 +113,43 @@ LibraryFetcher::add (string const & url) return -1; } - if (archive.inflate (destdir)) { - /* cleanup ? */ - return -1; + /* Unpack the archive. Likely should have a thread for this */ + + std::string destdir = Glib::path_get_dirname (path); + + { + std::string pwd (Glib::get_current_dir ()); + + if (g_chdir (destdir.c_str ())) { + error << string_compose (_("cannot chdir to '%1' to unpack library archive (%2)\n"), destdir, strerror (errno)) << endmsg; + return -1; + } + + if (archive.inflate (destdir)) { + /* cleanup ? */ + return -1; + } + + g_chdir (pwd.c_str()); } - std::string path; + std::string newpath; for (std::set::const_iterator d = dirs.begin(); d != dirs.end(); ++d) { if (d != dirs.begin()) { - path += G_SEARCHPATH_SEPARATOR; + newpath += G_SEARCHPATH_SEPARATOR; } - path += Glib::build_filename (destdir, *d); + newpath += Glib::build_filename (destdir, *d); } - assert (!path.empty()); + assert (!newpath.empty()); - path += G_SEARCHPATH_SEPARATOR; - path += Config->get_sample_lib_path (); + newpath += G_SEARCHPATH_SEPARATOR; + newpath += Config->get_sample_lib_path (); - Config->set_sample_lib_path (path); + Config->set_sample_lib_path (newpath); + + ::g_unlink (path.c_str()); } catch (...) { return -1; @@ -135,3 +157,148 @@ LibraryFetcher::add (string const & url) return 0; } + +void +LibraryFetcher::foreach_description (boost::function f) +{ + for (auto ld : _descriptions) { + f (ld); + } +} + +std::string +LibraryFetcher::install_path_for (LibraryDescription const & desc) +{ + return Glib::build_filename (Config->get_clip_library_dir(), desc.toplevel_dir()); +} + +bool +LibraryFetcher::installed (LibraryDescription const & desc) +{ + std::string path = install_path_for (desc); + if (Glib::file_test (path, Glib::FILE_TEST_EXISTS) && Glib::file_test (path, Glib::FILE_TEST_IS_DIR)) { + return true; + } + return false; +} + +static size_t +CurlFILEWrite_CallbackFunc_StdString(void *contents, size_t size, size_t nmemb, Downloader* dl) +{ + return dl->write (contents, size, nmemb); +} + +size_t +Downloader::write (void *ptr, size_t size, size_t nmemb) +{ + if (_cancel) { + fclose (file); + ::g_unlink (file_path.c_str()); + + _downloaded = 0; + _download_size = 0; + + return 0; + } + + size_t nwritten = fwrite (ptr, size, nmemb, file); + + _downloaded += nwritten; + + return nwritten; +} + +Downloader::Downloader (string const & u, string const & dir) + : url (u) + , destdir (dir) + , _cancel (false) + , _download_size (0) + , _downloaded (0) +{ +} + +int +Downloader::start () +{ + file_path = Glib::build_filename (Config->get_clip_library_dir(), destdir, Glib::path_get_basename (url)); + file = fopen (file_path.c_str(), "w"); + + if (!file) { + return -1; + } + + _cancel = false; + _status = 0; /* unknown at this point */ + thr = std::thread (&Downloader::download, this); + return 0; +} + +void +Downloader::cleanup () +{ + thr.join (); +} + +void +Downloader::cancel () +{ + _cancel = true; + cleanup (); +} + +double +Downloader::progress () const +{ + return (double) _downloaded / _download_size; +} + +void +Downloader::download () +{ + { + curl = curl_easy_init (); + if (!curl) { + _status = -1; + return; + } + + /* get size */ + + curl_easy_setopt(curl, CURLOPT_NOBODY, 1L); + curl_easy_setopt(curl, CURLOPT_HEADER, 0L); + curl_easy_setopt (curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_perform (curl); + curl_easy_getinfo (curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &dsize); + _download_size = dsize; + curl_easy_cleanup (curl); + } + + curl = curl_easy_init (); + if (!curl) { + _status = -1; + return; + } + + curl_easy_setopt (curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt (curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, CurlFILEWrite_CallbackFunc_StdString); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, this); + CURLcode res = curl_easy_perform (curl); + curl_easy_cleanup (curl); + + if (res == CURLE_OK) { + _status = 1; + } else { + _status = -1; + } +} + +std::string +Downloader::download_path() const +{ + /* Can only return the download path if we completed, and completed successfully */ + if (_status > 0) { + return file_path; + } + return std::string(); +}