// // Created by s-har on 26.05.2021. // #include "worker.hpp" #include #include #include #include #include #include #include #include using namespace std::chrono_literals; void operator+=(std::vector& vec, std::string const& s) { vec.push_back(s); } namespace bp = boost::process; std::string san_string_path(std::string name) { std::string out; for (auto el : name) { if ((el >= 'a' && el <= 'z') || (el >= 'A' && el <= 'Z') || (el >= '0' && el <= '9') || el == '_' || el == '.') { out += el; } else if(el == ' ') { out += '_'; } } return out; } Worker::Worker(soci::session& sql) : thread(std::bind_front(&Worker::loop, this)), sql(sql) { spdlog::info("Worker started."); } [[noreturn]] void Worker::loop() { while (true) { Task task; if (!getTask(task)) { std::this_thread::sleep_for(10s); continue; } spdlog::info("Worker Task: {}", task); spdlog::debug("Begin Download {}", task.id); task.status = FileStatus::DOWNLOADING; sql << "UPDATE Tasks SET status = :status WHERE id = :id", soci::use(task); if (download(task)) { spdlog::debug("Download end {}", task.id); task.status = FileStatus::COMPLETE; } else { // Error task.status = FileStatus::DOWNLOAD_ERROR; spdlog::error("Download Error {}: ", task); } sql << "UPDATE Tasks SET status = :status WHERE id = :id", soci::use(task); std::this_thread::sleep_for(10s); } } bool Worker::getTask(Task& task) { try { soci::rowset data = (sql.prepare << "SELECT * " "FROM Tasks " "WHERE status=:PENDING " "ORDER BY timestamp LIMIT 1;", soci::use(static_cast(FileStatus::PENDING), "PENDING")); for (auto const& el : data) { task = el; return true; } } catch (std::exception const& e) { spdlog::error("[Worker] GetTask Error: {}", e.what()); } return false; } bool Worker::download(Task& task) { bp::ipstream out; bp::ipstream err; /* ** args ** */ std::vector args; args += "-o"; args += fmt::format("{}/%(title)s.%(ext)s", output_path); if (task.audio_only) { args += "--extract-audio"; } if (task.format) { args += task.audio_only ? "--audio-format" : "-f"; args += task.format.value(); } args += "--restrict-filenames"; args += task.url; /* ** Download ** */ spdlog::debug("youtube-dl {}", fmt::join(args, " ")); #ifdef __linux__ constexpr std::string_view youtube_dl = "youtube-dl"; #elif _WIN32 constexpr std::string_view youtube_dl = "youtube-dl.exe"; #else #endif bp::child c(std::string{youtube_dl}, bp::args(args), bp::std_out > out, bp::std_err > err); std::string destination_file; std::string line; while (c.running() || !out.eof() || !err.eof()) { if (std::getline(out, line) && !line.empty()) { spdlog::info("[youtube-dl] {}", line); if (auto [match, info, destination] = ctre::match(line); match) { destination_file = destination; } if(auto [match, info, destination] = ctre::match(line); match) { destination_file = destination; } } else if (std::getline(err, line) && !line.empty()) { if (auto [match, type, messages] = ctre::match(line); match) { if (type == "WARNING") spdlog::warn("[youtube-dl] {}", messages); } else { spdlog::error("[youtube-dl] {}", line); } } } c.wait(); auto exit_code = c.exit_code(); if (exit_code != 0) return false; // Process Downloaded file namespace fs = std::filesystem; spdlog::debug("File: {}", destination_file); auto downloaded_path = std::filesystem::path(destination_file); spdlog::info(absolute(downloaded_path).string()); auto san_filename = san_string_path(downloaded_path.filename().string()); auto local_path = fs::path(output_path) / (task.hash + downloaded_path.extension().string()); auto current_time = std::time(nullptr); try{ fs::rename(fs::path(".") / downloaded_path, local_path ); File file; file.id = task.id; file.filename = san_filename; file.local_path = local_path.string(); file.hash = task.hash; file.format = downloaded_path.extension().string(); file.timestamp = current_time; sql << "INSERT INTO Files (id, hash, timestamp, format, filename, local_path) " "values(:id, :hash, :timestamp, :format, :filename, :local_path) ", soci::use(file); }catch(std::exception const& e) { spdlog::error("File rename / insert error: {}", e.what()); return false; } spdlog::debug("youtube-dl exit code {}", exit_code); return exit_code == 0; }