diff --git a/Modules/Server/src/api.cpp b/Modules/Server/src/api.cpp index bbcabc4..4eda569 100644 --- a/Modules/Server/src/api.cpp +++ b/Modules/Server/src/api.cpp @@ -9,8 +9,10 @@ #include #include +#include #include #include +#include #include using json = nlohmann::json; @@ -28,49 +30,82 @@ void Api::RegisterServerHandles(httplib::Server& server) server.Get("/api/files", std::bind_front(&Api::files, this)); - server.Get("/api/file/(\\d+)", std::bind_front(&Api::file, this)); + server.Get("/api/file/([A-E0-9]+)", std::bind_front(&Api::file, this)); server.Options("/(.*)", [this](httplib::Request const& rq, httplib::Response& rp) {}); server.set_pre_routing_handler([this](httplib::Request const& rq, httplib::Response& rp) { this->set_cross_headers(rp); - spdlog::info("Request {}", rq.method); + spdlog::info("Request {} {}", rq.method, rq.path); return httplib::Server::HandlerResponse::Unhandled; }); } +/* /api/add + * + * body: json + * - url + * - audio_only - default false + * - format - optional default source + * + * return: json + * - status: bool + * - file_id: hash + * - error: string + * + */ + void Api::add(const httplib::Request& rq, httplib::Response& rs) { try { auto data = json::parse(rq.body); - File f; + // Parse Request + Task task; + task.url = data["url"]; + task.audio_only = data.contains("audio_only") && data["audio_only"].get(); - f.url = data["url"]; - f.status = FileStatus::PENDING; + if (data.contains("format")) + task.format = data["format"].get(); - sql << "INSERT INTO Files (url, status) values(:url, :status)", soci::use(f.url, "url"), - soci::use(f.status, "status"); + task.timestamp = std::time(nullptr); - long long int tmp; + task.hash = fmt::format( + "{:X}", + std::hash()( + task.url + fmt::format("{:%Y-%m-%d %H:%M:%S}", *std::localtime(&task.timestamp)))); + task.status = FileStatus::PENDING; + + // Insert Sql + + sql << "INSERT INTO Tasks (hash, timestamp, url, audio_only, status, format) " + "values(:hash, :timestamp, :url, :audio_only, :status, :format) ", + soci::use(task); + + /*long long int tmp; sql.get_last_insert_id("Files", tmp); - f.id = tmp; + f.id = tmp;*/ json jr; - jr["status"] = "Ok"; - jr["id"] = f.id; + jr["status"] = "Ok"; + jr["file_id"] = task.hash; - spdlog::info("New Task: {}", f); + spdlog::info("New Task: {}", task); rs.set_content(jr.dump(), "application/json"); } catch (json::exception const& e) { spdlog::error("Api Error: {}", e.what()); + + json jr; + jr["status"] = "Err"; + jr["error"] = e.what(); + rs.status = 400; - rs.set_content(std::format("Json Error: {}", e.what()), "text/plain"); + rs.set_content(jr.dump(), "application/json"); } } @@ -80,37 +115,52 @@ void Api::files(httplib::Request const& rq, httplib::Response& rs) { json res; - soci::rowset data = (sql.prepare << "SELECT * FROM Files ORDER BY id DESC"); + // Query incomplete Tasks + + soci::rowset data = (sql.prepare << "SELECT * " + "FROM Tasks " + "WHERE status!=2 " + "ORDER BY id DESC "); for (auto const& el : data) { json current; - current["id"] = el.id; - current["status"] = magic_enum::enum_name(el.status); - current["status_id"] = static_cast(el.status); - current["source"] = el.url; - current["name"] = fmt::format("File {}", el.id); - - current["file_name"] = "Unknown"; - - if (el.status == FileStatus::COMPLETE && false) - { - std::string filename; - soci::rowset ret = - (sql.prepare << "SELECT path From Downloads WHERE id=:id", - soci::use(el.id, "id")); - - if (ret.begin() != ret.end()) - { - std::filesystem::path p(*ret.begin()); - - current["file_name"] = p.filename(); - } - } + current["hash"] = el.hash; + current["status"] = magic_enum::enum_name(el.status); + current["status_id"] = static_cast(el.status); + current["source"] = el.url; + current["name"] = fmt::format("File {}", el.id); + current["audio_only"] = el.audio_only; + current["format"] = el.format.value_or("Source"); res["queue"].push_back(current); } + // Downloaded + + soci::rowset files = + (sql.prepare + << R"(SELECT Tasks.id, Files.timestamp, Files.hash, audio_only, filename, Files.format +FROM Files, + Tasks +WHERE Tasks.hash == Files.hash +ORDER BY Files.timestamp DESC )"); + + for (auto const& el : files) + { + json current; + + std::time_t time = el.get("timestamp"); + + current["timestamp"] = fmt::format("{:%Y-%m-%d %H:%M:%S}", *std::localtime(&time)); + current["hash"] = el.get("hash"); + current["audio_only"] = el.get("audio_only"); + current["filename"] = el.get("filename"); + current["format"] = el.get("format"); + + res["files"].push_back(current); + } + rs.set_content(res.dump(), "application/json"); } catch (std::exception const& e) @@ -130,28 +180,23 @@ void Api::file(const httplib::Request& rq, httplib::Response& rs) } auto file = rq.matches[1]; - spdlog::info("File Request: {}", file); try { - Download dw; - sql << fmt::format("SELECT * FROM Downloads WHERE id={};", file), soci::into(dw); + File dw; + sql << fmt::format("SELECT * FROM Files WHERE hash='{}';", file), soci::into(dw); - std::ifstream fs(dw.path, std::ios_base::binary); + std::ifstream fs(dw.local_path, std::ios_base::binary); fs.seekg(0, std::ios_base::end); auto size = fs.tellg(); fs.seekg(0); rs.body.resize(static_cast(size)); fs.read(&rs.body[0], static_cast(size)); - rs.set_header("Content-Disposition", fmt::format("attachment; filename={};", dw.path)); - if (auto [match, type] = ctre::match(dw.path); match) - { - spdlog::debug(type); - rs.set_header("content-type", fmt::format("application/{}", type.to_view())); - } + rs.set_header("Content-Disposition", fmt::format("attachment; filename={};", dw.filename)); + rs.set_header("content-type", fmt::format("application/{}", dw.format)); } catch (std::exception const& e) { diff --git a/Modules/Server/src/main.cpp b/Modules/Server/src/main.cpp index 742966f..b191919 100644 --- a/Modules/Server/src/main.cpp +++ b/Modules/Server/src/main.cpp @@ -61,8 +61,8 @@ int main() Version v { 1, std::string { version } }; sql << "insert into Version(ID, version_info) values(:ID, :version_info)", soci::use(v); + Task::create_table(sql); File::create_table(sql); - Download::create_table(sql); } Worker worker(sql); diff --git a/Modules/Server/src/tables.hpp b/Modules/Server/src/tables.hpp index dbf2c55..bc682b2 100644 --- a/Modules/Server/src/tables.hpp +++ b/Modules/Server/src/tables.hpp @@ -66,11 +66,91 @@ namespace soci } // namespace soci +struct Task +{ + int id; + std::string hash; + std::time_t timestamp; + std::string url; + bool audio_only; + FileStatus status; + + // Optional + std::optional format; + + constexpr static std::string_view table_name = "Tasks"; + + inline static void create_table(soci::session& sql) + { + spdlog::info("Creating table {}", table_name); + auto table = sql.create_table(std::string { table_name }); + + table.column("id", soci::dt_integer)("PRIMARY KEY AUTOINCREMENT"); + table.column("hash", soci::dt_string)("NOT NULL"); + table.column("timestamp", soci::dt_date)("NOT NULL"); + table.column("url", soci::dt_string)("NOT NULL"); + table.column("audio_only", soci::dt_integer)("NOT NULL"); + table.column("status", soci::dt_integer)("NOT NULL"); + table.column("format", soci::dt_string)("NOT NULL"); + } +}; + +inline std::ostream& operator<<(std::ostream& os, Task const& file) +{ + return os << fmt::format("Task [{}][{}] url: {} ", + file.id, + magic_enum::enum_name(file.status), + file.url); +} + +namespace soci +{ + template <> + struct type_conversion + { + typedef values base_type; + + static void from_base(values const& i, indicator ind, Task& v) + { + v.id = i.get("id"); + v.timestamp = i.get("timestamp"); + v.hash = i.get("hash"); + v.url = i.get("url"); + v.audio_only = i.get("audio_only") == 1; + v.status = i.get("status"); + + auto format = i.get("format"); + if(format.empty()) + v.format = {}; + else + v.format = format ; + + } + + static void to_base(const Task& v, values& i, indicator& ind) + { + i.set("id", v.id); + i.set("timestamp", (int)v.timestamp); + i.set("hash", v.hash); + i.set("url", v.url); + i.set("audio_only", static_cast(v.audio_only)); + i.set("status", v.status); + + i.set("format", v.format ? v.format.value() : std::string{}); + ind = i_ok; + } + }; +} // namespace soci + + struct File { int id; - std::string url; - FileStatus status; + std::string hash; + std::time_t timestamp; + std::string format; + std::string filename; + std::string local_path; constexpr static std::string_view table_name = "Files"; @@ -79,21 +159,16 @@ struct File spdlog::info("Creating table {}", table_name); auto table = sql.create_table(std::string { table_name }); - table.column("id", soci::dt_integer)("PRIMARY KEY AUTOINCREMENT"); - table.column("url", soci::dt_string)("NOT NULL"); - table.column("status", soci::dt_integer)("NOT NULL"); - // table.primary_key(std::string { table_name }, "id"); + table.column("id", soci::dt_integer)("PRIMARY KEY"); + table.column("hash", soci::dt_string)("NOT NULL"); + table.column("timestamp", soci::dt_date)("NOT NULL"); + table.column("format", soci::dt_string)("NOT NULL"); + table.column("filename", soci::dt_string)("NOT NULL"); + table.column("local_path", soci::dt_string)("NOT NULL"); + table.foreign_key("FK_Downloads", "id", std::string { File::table_name }, "id"); } }; -inline std::ostream& operator<<(std::ostream& os, File const& file) -{ - return os << fmt::format("File [{}][{}] url: {} ", - file.id, - magic_enum::enum_name(file.status), - file.url); -} - namespace soci { template <> @@ -103,57 +178,22 @@ namespace soci static void from_base(values const& i, indicator ind, File& v) { - v.id = i.get("id"); - v.url = i.get("url"); - v.status = i.get("status"); + v.id = i.get("id"); + v.hash = i.get("hash"); + v.timestamp = i.get("timestamp"); + v.format = i.get("format"); + v.filename = i.get("filename"); + v.local_path = i.get("local_path"); } static void to_base(const File& v, values& i, indicator& ind) { i.set("id", v.id); - i.set("url", v.url); - i.set("status", v.status); - ind = i_ok; - } - }; -} // namespace soci - - -struct Download -{ - int id; - std::string path; - - constexpr static std::string_view table_name = "Downloads"; - - inline static void create_table(soci::session& sql) - { - spdlog::info("Creating table {}", table_name); - auto table = sql.create_table(std::string { table_name }); - - table.column("id", soci::dt_integer)("PRIMARY KEY"); - table.column("path", soci::dt_string)("NOT NULL"); - table.foreign_key("FK_Downloads", "id", std::string { File::table_name }, "id"); - } -}; - -namespace soci -{ - template <> - struct type_conversion - { - typedef values base_type; - - static void from_base(values const& i, indicator ind, Download& v) - { - v.id = i.get("id"); - v.path = i.get("path"); - } - - static void to_base(const Download& v, values& i, indicator& ind) - { - i.set("id", v.id); - i.set("path", v.path); + i.set("hash", v.hash); + i.set("timestamp", (int)v.timestamp); + i.set("format", v.format); + i.set("filename", v.filename); + i.set("local_path", v.local_path); ind = i_ok; } }; diff --git a/Modules/Server/src/worker.cpp b/Modules/Server/src/worker.cpp index 4270e54..58e5547 100644 --- a/Modules/Server/src/worker.cpp +++ b/Modules/Server/src/worker.cpp @@ -4,6 +4,7 @@ #include "worker.hpp" #include +#include #include #include @@ -21,6 +22,24 @@ void operator+=(std::vector& vec, std::string const& s) namespace bp = boost::process; +std::string san_string_path(std::string name) +{ + std::string out; + for (auto el : name) + { + if ((el >= 'a' && el <= 'z') || (el >= 'A' && el <= 'Z') || (el >= '0' && el <= '9') || + el == '_' || el == '.') + { + out += el; + } + else if(el == ' ') + { + out += '_'; + } + } + return out; +} + Worker::Worker(soci::session& sql) : thread(std::bind_front(&Worker::loop, this)), sql(sql) { spdlog::info("Worker started."); @@ -30,7 +49,7 @@ Worker::Worker(soci::session& sql) : thread(std::bind_front(&Worker::loop, this) { while (true) { - File task; + Task task; if (!getTask(task)) { std::this_thread::sleep_for(10s); @@ -42,7 +61,7 @@ Worker::Worker(soci::session& sql) : thread(std::bind_front(&Worker::loop, this) spdlog::debug("Begin Download {}", task.id); task.status = FileStatus::DOWNLOADING; - sql << "UPDATE Files SET status = :status WHERE id = :id", soci::use(task); + sql << "UPDATE Tasks SET status = :status WHERE id = :id", soci::use(task); if (download(task)) { @@ -53,20 +72,22 @@ Worker::Worker(soci::session& sql) : thread(std::bind_front(&Worker::loop, this) // Error task.status = FileStatus::DOWNLOAD_ERROR; spdlog::error("Download Error {}: ", task); - } - sql << "UPDATE Files SET status = :status WHERE id = :id", soci::use(task); + sql << "UPDATE Tasks SET status = :status WHERE id = :id", soci::use(task); std::this_thread::sleep_for(10s); } } -bool Worker::getTask(File& task) +bool Worker::getTask(Task& task) { try { - soci::rowset data = - (sql.prepare << "SELECT * FROM Files WHERE status = 0 ORDER BY id LIMIT 1;"); + soci::rowset data = (sql.prepare << "SELECT * " + "FROM Tasks " + "WHERE status=:PENDING " + "ORDER BY timestamp LIMIT 1;", + soci::use(static_cast(FileStatus::PENDING), "PENDING")); for (auto const& el : data) { @@ -76,26 +97,36 @@ bool Worker::getTask(File& task) } catch (std::exception const& e) { - spdlog::error(e.what()); - fmt::format("{}\n", e.what()); + spdlog::error("[Worker] GetTask Error: {}", e.what()); } return false; } -bool Worker::download(File& task) +bool Worker::download(Task& task) { bp::ipstream out; bp::ipstream err; + /* ** args ** */ std::vector args; args += "-o"; args += fmt::format("{}/%(title)s.%(ext)s", output_path); - args += "--extract-audio"; + if (task.audio_only) + { + args += "--extract-audio"; + } + + if (task.format) + { + args += task.audio_only ? "--audio-format" : "-f"; + args += task.format.value(); + } args += task.url; + /* ** Download ** */ spdlog::debug("youtube-dl {}", fmt::join(args, " ")); bp::child c("youtube-dl.exe", bp::args(args), bp::std_out > out, bp::std_err > err); @@ -116,6 +147,13 @@ bool Worker::download(File& task) destination_file = destination; } + if(auto [match, info, destination] = + ctre::match(line); + match) + { + destination_file = destination; + } + } else if (std::getline(err, line) && !line.empty()) { if (auto [match, type, messages] = ctre::match(line); match) @@ -132,13 +170,41 @@ bool Worker::download(File& task) c.wait(); auto exit_code = c.exit_code(); + if (exit_code != 0) + return false; + + // Process Downloaded file + namespace fs = std::filesystem; + spdlog::debug("File: {}", destination_file); - Download dwn; - dwn.id = task.id; - dwn.path = destination_file; + auto downloaded_path = std::filesystem::path(destination_file); + spdlog::info(absolute(downloaded_path).string()); + + auto san_filename = san_string_path(downloaded_path.filename().string()); + auto local_path = fs::path(output_path) / (task.hash + downloaded_path.extension().string()); + + auto current_time = std::time(nullptr); + + try{ + fs::rename(fs::path(".") / downloaded_path, local_path ); + + File file; + file.id = task.id; + file.filename = san_filename; + file.local_path = local_path.string(); + file.hash = task.hash; + file.format = downloaded_path.extension().string(); + file.timestamp = current_time; + + sql << "INSERT INTO Files (id, hash, timestamp, format, filename, local_path) " + "values(:id, :hash, :timestamp, :format, :filename, :local_path) ", soci::use(file); + }catch(std::exception const& e) + { + spdlog::error("File rename / insert error: {}", e.what()); + return false; + } - sql << "INSERT INTO Downloads (id, path) values(:id, :path)", soci::use(dwn); spdlog::debug("youtube-dl exit code {}", exit_code); return exit_code == 0; diff --git a/Modules/Server/src/worker.hpp b/Modules/Server/src/worker.hpp index d3da784..bc7f0db 100644 --- a/Modules/Server/src/worker.hpp +++ b/Modules/Server/src/worker.hpp @@ -22,7 +22,7 @@ public: [[noreturn]] void loop(); private: - bool getTask(File& task); + bool getTask(Task& task); - bool download(File& task); + bool download(Task& task); }; \ No newline at end of file