New db layout

Modules/Server/src/worker.cpp

@@ -4,6 +4,7 @@
 #include "worker.hpp"
 
 #include <chrono>
+#include <filesystem>
 #include <functional>
 #include <vector>
 
@@ -21,6 +22,24 @@ void operator+=(std::vector<std::string>& vec, std::string const& s)
 
 namespace bp = boost::process;
 
+std::string san_string_path(std::string name)
+{
+    std::string out;
+    for (auto el : name)
+    {
+        if ((el >= 'a' && el <= 'z') || (el >= 'A' && el <= 'Z') || (el >= '0' && el <= '9') ||
+            el == '_' || el == '.')
+        {
+            out += el;
+        }
+        else if(el == ' ')
+        {
+            out += '_';
+        }
+    }
+    return out;
+}
+
 Worker::Worker(soci::session& sql) : thread(std::bind_front(&Worker::loop, this)), sql(sql)
 {
     spdlog::info("Worker started.");
@@ -30,7 +49,7 @@ Worker::Worker(soci::session& sql) : thread(std::bind_front(&Worker::loop, this)
 {
     while (true)
     {
-        File task;
+        Task task;
         if (!getTask(task))
         {
             std::this_thread::sleep_for(10s);
@@ -42,7 +61,7 @@ Worker::Worker(soci::session& sql) : thread(std::bind_front(&Worker::loop, this)
         spdlog::debug("Begin Download {}", task.id);
         task.status = FileStatus::DOWNLOADING;
 
-        sql << "UPDATE Files SET status = :status WHERE id = :id", soci::use(task);
+        sql << "UPDATE Tasks SET status = :status WHERE id = :id", soci::use(task);
 
         if (download(task))
         {
@@ -53,20 +72,22 @@ Worker::Worker(soci::session& sql) : thread(std::bind_front(&Worker::loop, this)
             // Error
             task.status = FileStatus::DOWNLOAD_ERROR;
             spdlog::error("Download Error {}: ", task);
-
         }
 
-        sql << "UPDATE Files SET status = :status WHERE id = :id", soci::use(task);
+        sql << "UPDATE Tasks SET status = :status WHERE id = :id", soci::use(task);
         std::this_thread::sleep_for(10s);
     }
 }
 
-bool Worker::getTask(File& task)
+bool Worker::getTask(Task& task)
 {
     try
     {
-        soci::rowset<File> data =
-          (sql.prepare << "SELECT * FROM Files WHERE status = 0 ORDER BY id LIMIT 1;");
+        soci::rowset<Task> data = (sql.prepare << "SELECT * "
+                                                  "FROM Tasks "
+                                                  "WHERE status=:PENDING "
+                                                  "ORDER BY timestamp LIMIT 1;",
+                                   soci::use(static_cast<int>(FileStatus::PENDING), "PENDING"));
 
         for (auto const& el : data)
         {
@@ -76,26 +97,36 @@ bool Worker::getTask(File& task)
     }
     catch (std::exception const& e)
     {
-        spdlog::error(e.what());
-        fmt::format("{}\n", e.what());
+        spdlog::error("[Worker] GetTask Error: {}", e.what());
     }
     return false;
 }
 
-bool Worker::download(File& task)
+bool Worker::download(Task& task)
 {
     bp::ipstream out;
     bp::ipstream err;
 
+    /* ** args ** */
     std::vector<std::string> args;
 
     args += "-o";
     args += fmt::format("{}/%(title)s.%(ext)s", output_path);
 
-    args += "--extract-audio";
+    if (task.audio_only)
+    {
+        args += "--extract-audio";
+    }
+
+    if (task.format)
+    {
+        args += task.audio_only ? "--audio-format" : "-f";
+        args += task.format.value();
+    }
 
     args += task.url;
 
+    /* ** Download ** */
     spdlog::debug("youtube-dl {}", fmt::join(args, " "));
 
     bp::child c("youtube-dl.exe", bp::args(args), bp::std_out > out, bp::std_err > err);
@@ -116,6 +147,13 @@ bool Worker::download(File& task)
                 destination_file = destination;
             }
 
+            if(auto [match, info, destination] =
+                ctre::match<R"regex(\[(.*)\] Merging formats into "(.*)")regex">(line);
+              match)
+            {
+                destination_file = destination;
+            }
+
         } else if (std::getline(err, line) && !line.empty())
         {
             if (auto [match, type, messages] = ctre::match<R"((.*): (.*))">(line); match)
@@ -132,13 +170,41 @@ bool Worker::download(File& task)
     c.wait();
     auto exit_code = c.exit_code();
 
+    if (exit_code != 0)
+        return false;
+
+    // Process Downloaded file
+    namespace fs = std::filesystem;
+
     spdlog::debug("File: {}", destination_file);
 
-    Download dwn;
-    dwn.id = task.id;
-    dwn.path = destination_file;
+    auto downloaded_path = std::filesystem::path(destination_file);
+    spdlog::info(absolute(downloaded_path).string());
+
+    auto san_filename = san_string_path(downloaded_path.filename().string());
+    auto local_path = fs::path(output_path) / (task.hash + downloaded_path.extension().string());
+
+    auto current_time = std::time(nullptr);
+
+    try{
+        fs::rename(fs::path(".") / downloaded_path, local_path );
+
+        File file;
+        file.id = task.id;
+        file.filename = san_filename;
+        file.local_path = local_path.string();
+        file.hash = task.hash;
+        file.format = downloaded_path.extension().string();
+        file.timestamp = current_time;
+
+        sql << "INSERT INTO Files (id, hash, timestamp, format, filename, local_path) "
+               "values(:id, :hash, :timestamp, :format, :filename, :local_path) ", soci::use(file);
+    }catch(std::exception const& e)
+    {
+        spdlog::error("File rename / insert error: {}", e.what());
+        return false;
+    }
 
-    sql << "INSERT INTO Downloads (id, path) values(:id, :path)", soci::use(dwn);
 
     spdlog::debug("youtube-dl exit code {}", exit_code);
     return exit_code == 0;