New db layout

Modules/Server/src/api.cpp

@@ -9,8 +9,10 @@
 #include <functional>
 
 #include <ctre.hpp>
+#include <fmt/chrono.h>
 #include <fmt/ostream.h>
 #include <nlohmann/json.hpp>
+#include <openssl/md5.h>
 #include <spdlog/spdlog.h>
 
 using json = nlohmann::json;
@@ -28,49 +30,82 @@ void Api::RegisterServerHandles(httplib::Server& server)
 
     server.Get("/api/files", std::bind_front(&Api::files, this));
 
-    server.Get("/api/file/(\\d+)", std::bind_front(&Api::file, this));
+    server.Get("/api/file/([A-E0-9]+)", std::bind_front(&Api::file, this));
 
     server.Options("/(.*)", [this](httplib::Request const& rq, httplib::Response& rp) {});
 
     server.set_pre_routing_handler([this](httplib::Request const& rq, httplib::Response& rp) {
         this->set_cross_headers(rp);
-        spdlog::info("Request {}", rq.method);
+        spdlog::info("Request {} {}", rq.method, rq.path);
         return httplib::Server::HandlerResponse::Unhandled;
     });
 }
 
+/* /api/add
+ *
+ * body: json
+ * - url
+ * - audio_only - default false
+ * - format - optional default source
+ *
+ * return: json
+ * - status: bool
+ * - file_id: hash
+ * - error: string
+ *
+ */
+
 void Api::add(const httplib::Request& rq, httplib::Response& rs)
 {
     try
     {
         auto data = json::parse(rq.body);
 
-        File f;
+        // Parse Request
+        Task task;
 
+        task.url        = data["url"];
+        task.audio_only = data.contains("audio_only") && data["audio_only"].get<bool>();
 
-        f.url    = data["url"];
+        if (data.contains("format"))
-        f.status = FileStatus::PENDING;
+            task.format = data["format"].get<std::string>();
 
-        sql << "INSERT INTO Files (url, status) values(:url, :status)", soci::use(f.url, "url"),
+        task.timestamp = std::time(nullptr);
-          soci::use(f.status, "status");
 
-        long long int tmp;
+        task.hash = fmt::format(
+          "{:X}",
+          std::hash<std::string>()(
+            task.url + fmt::format("{:%Y-%m-%d %H:%M:%S}", *std::localtime(&task.timestamp))));
+        task.status = FileStatus::PENDING;
+
+        // Insert Sql
+
+        sql << "INSERT INTO Tasks (hash, timestamp, url, audio_only, status, format) "
+               "values(:hash, :timestamp, :url, :audio_only, :status, :format) ",
+          soci::use(task);
+
+        /*long long int tmp;
         sql.get_last_insert_id("Files", tmp);
-        f.id = tmp;
+        f.id = tmp;*/
 
         json jr;
-        jr["status"] = "Ok";
+        jr["status"]  = "Ok";
-        jr["id"]     = f.id;
+        jr["file_id"] = task.hash;
 
-        spdlog::info("New Task: {}", f);
+        spdlog::info("New Task: {}", task);
 
         rs.set_content(jr.dump(), "application/json");
     }
     catch (json::exception const& e)
     {
         spdlog::error("Api Error: {}", e.what());
+
+        json jr;
+        jr["status"] = "Err";
+        jr["error"]  = e.what();
+
         rs.status = 400;
-        rs.set_content(std::format("Json Error: {}", e.what()), "text/plain");
+        rs.set_content(jr.dump(), "application/json");
     }
 }
 
@@ -80,37 +115,52 @@ void Api::files(httplib::Request const& rq, httplib::Response& rs)
     {
         json res;
 
-        soci::rowset<File> data = (sql.prepare << "SELECT * FROM Files ORDER BY id DESC");
+        // Query incomplete Tasks
+
+        soci::rowset<Task> data = (sql.prepare << "SELECT * "
+                                                  "FROM Tasks "
+                                                  "WHERE status!=2 "
+                                                  "ORDER BY id DESC ");
 
         for (auto const& el : data)
         {
             json current;
-            current["id"]        = el.id;
+            current["hash"]       = el.hash;
-            current["status"]    = magic_enum::enum_name(el.status);
+            current["status"]     = magic_enum::enum_name(el.status);
-            current["status_id"] = static_cast<int>(el.status);
+            current["status_id"]  = static_cast<int>(el.status);
-            current["source"]    = el.url;
+            current["source"]     = el.url;
-            current["name"]      = fmt::format("File {}", el.id);
+            current["name"]       = fmt::format("File {}", el.id);
-
+            current["audio_only"] = el.audio_only;
-            current["file_name"] = "Unknown";
+            current["format"]     = el.format.value_or("Source");
-
-            if (el.status == FileStatus::COMPLETE && false)
-            {
-                std::string filename;
-                soci::rowset<std::string> ret =
-                  (sql.prepare << "SELECT path From Downloads WHERE id=:id",
-                   soci::use(el.id, "id"));
-
-                if (ret.begin() != ret.end())
-                {
-                    std::filesystem::path p(*ret.begin());
-
-                    current["file_name"] = p.filename();
-                }
-            }
 
             res["queue"].push_back(current);
         }
 
+        // Downloaded
+
+        soci::rowset<soci::row> files =
+          (sql.prepare
+           << R"(SELECT Tasks.id, Files.timestamp, Files.hash, audio_only, filename, Files.format
+FROM Files,
+     Tasks
+WHERE Tasks.hash == Files.hash
+ORDER BY Files.timestamp DESC )");
+
+        for (auto const& el : files)
+        {
+            json current;
+
+            std::time_t time = el.get<int>("timestamp");
+
+            current["timestamp"]  = fmt::format("{:%Y-%m-%d %H:%M:%S}", *std::localtime(&time));
+            current["hash"]       = el.get<std::string>("hash");
+            current["audio_only"] = el.get<int>("audio_only");
+            current["filename"]   = el.get<std::string>("filename");
+            current["format"]     = el.get<std::string>("format");
+
+            res["files"].push_back(current);
+        }
+
         rs.set_content(res.dump(), "application/json");
     }
     catch (std::exception const& e)
@@ -130,28 +180,23 @@ void Api::file(const httplib::Request& rq, httplib::Response& rs)
     }
 
     auto file = rq.matches[1];
-
     spdlog::info("File Request: {}", file);
 
     try
     {
-        Download dw;
+        File dw;
-        sql << fmt::format("SELECT * FROM Downloads WHERE id={};", file), soci::into(dw);
+        sql << fmt::format("SELECT * FROM Files WHERE hash='{}';", file), soci::into(dw);
 
 
-        std::ifstream fs(dw.path, std::ios_base::binary);
+        std::ifstream fs(dw.local_path, std::ios_base::binary);
         fs.seekg(0, std::ios_base::end);
         auto size = fs.tellg();
         fs.seekg(0);
         rs.body.resize(static_cast<size_t>(size));
         fs.read(&rs.body[0], static_cast<std::streamsize>(size));
 
-        rs.set_header("Content-Disposition", fmt::format("attachment; filename={};", dw.path));
+        rs.set_header("Content-Disposition", fmt::format("attachment; filename={};", dw.filename));
-        if (auto [match, type] = ctre::match<R"(.*\.(.*))">(dw.path); match)
+        rs.set_header("content-type", fmt::format("application/{}", dw.format));
-        {
-            spdlog::debug(type);
-            rs.set_header("content-type", fmt::format("application/{}", type.to_view()));
-        }
     }
     catch (std::exception const& e)
     {

Modules/Server/src/main.cpp

@@ -61,8 +61,8 @@ int main()
         Version v { 1, std::string { version } };
         sql << "insert into Version(ID, version_info) values(:ID, :version_info)", soci::use(v);
 
+        Task::create_table(sql);
         File::create_table(sql);
-        Download::create_table(sql);
     }
 
     Worker worker(sql);

Modules/Server/src/tables.hpp

@@ -66,11 +66,91 @@ namespace soci
 }    // namespace soci
 
 
+struct Task
+{
+    int id;
+    std::string hash;
+    std::time_t timestamp;
+    std::string url;
+    bool audio_only;
+    FileStatus status;
+
+    // Optional
+    std::optional<std::string> format;
+
+    constexpr static std::string_view table_name = "Tasks";
+
+    inline static void create_table(soci::session& sql)
+    {
+        spdlog::info("Creating table {}", table_name);
+        auto table = sql.create_table(std::string { table_name });
+
+        table.column("id", soci::dt_integer)("PRIMARY KEY AUTOINCREMENT");
+        table.column("hash", soci::dt_string)("NOT NULL");
+        table.column("timestamp", soci::dt_date)("NOT NULL");
+        table.column("url", soci::dt_string)("NOT NULL");
+        table.column("audio_only", soci::dt_integer)("NOT NULL");
+        table.column("status", soci::dt_integer)("NOT NULL");
+        table.column("format", soci::dt_string)("NOT NULL");
+    }
+};
+
+inline std::ostream& operator<<(std::ostream& os, Task const& file)
+{
+    return os << fmt::format("Task [{}][{}] url: {} ",
+                             file.id,
+                             magic_enum::enum_name(file.status),
+                             file.url);
+}
+
+namespace soci
+{
+    template <>
+    struct type_conversion<Task>
+    {
+        typedef values base_type;
+
+        static void from_base(values const& i, indicator ind, Task& v)
+        {
+            v.id         = i.get<int>("id");
+            v.timestamp  = i.get<int>("timestamp");
+            v.hash       = i.get<std::string>("hash");
+            v.url        = i.get<std::string>("url");
+            v.audio_only = i.get<int>("audio_only") == 1;
+            v.status     = i.get<FileStatus>("status");
+
+            auto format = i.get<std::string>("format");
+            if(format.empty())
+                v.format = {};
+            else
+                v.format = format ;
+
+        }
+
+        static void to_base(const Task& v, values& i, indicator& ind)
+        {
+            i.set("id", v.id);
+            i.set("timestamp", (int)v.timestamp);
+            i.set("hash", v.hash);
+            i.set("url", v.url);
+            i.set("audio_only", static_cast<int>(v.audio_only));
+            i.set("status", v.status);
+
+            i.set("format", v.format ? v.format.value() : std::string{});
+            ind = i_ok;
+        }
+    };
+}    // namespace soci
+
+
 struct File
 {
     int id;
-    std::string url;
+    std::string hash;
-    FileStatus status;
+    std::time_t timestamp;
+    std::string format;
+    std::string filename;
+    std::string local_path;
 
     constexpr static std::string_view table_name = "Files";
 
@@ -79,21 +159,16 @@ struct File
         spdlog::info("Creating table {}", table_name);
         auto table = sql.create_table(std::string { table_name });
 
-        table.column("id", soci::dt_integer)("PRIMARY KEY AUTOINCREMENT");
+        table.column("id", soci::dt_integer)("PRIMARY KEY");
-        table.column("url", soci::dt_string)("NOT NULL");
+        table.column("hash", soci::dt_string)("NOT NULL");
-        table.column("status", soci::dt_integer)("NOT NULL");
+        table.column("timestamp", soci::dt_date)("NOT NULL");
-        // table.primary_key(std::string { table_name }, "id");
+        table.column("format", soci::dt_string)("NOT NULL");
+        table.column("filename", soci::dt_string)("NOT NULL");
+        table.column("local_path", soci::dt_string)("NOT NULL");
+        table.foreign_key("FK_Downloads", "id", std::string { File::table_name }, "id");
     }
 };
 
-inline std::ostream& operator<<(std::ostream& os, File const& file)
-{
-    return os << fmt::format("File [{}][{}] url: {} ",
-                             file.id,
-                             magic_enum::enum_name(file.status),
-                             file.url);
-}
-
 namespace soci
 {
     template <>
@@ -103,57 +178,22 @@ namespace soci
 
         static void from_base(values const& i, indicator ind, File& v)
         {
-            v.id     = i.get<int>("id");
+            v.id         = i.get<int>("id");
-            v.url    = i.get<std::string>("url");
+            v.hash       = i.get<std::string>("hash");
-            v.status = i.get<FileStatus>("status");
+            v.timestamp  = i.get<int>("timestamp");
+            v.format     = i.get<std::string>("format");
+            v.filename   = i.get<std::string>("filename");
+            v.local_path = i.get<std::string>("local_path");
         }
 
         static void to_base(const File& v, values& i, indicator& ind)
         {
             i.set("id", v.id);
-            i.set("url", v.url);
+            i.set("hash", v.hash);
-            i.set("status", v.status);
+            i.set("timestamp", (int)v.timestamp);
-            ind = i_ok;
+            i.set("format", v.format);
-        }
+            i.set("filename", v.filename);
-    };
+            i.set("local_path", v.local_path);
-}    // namespace soci
-
-
-struct Download
-{
-    int id;
-    std::string path;
-
-    constexpr static std::string_view table_name = "Downloads";
-
-    inline static void create_table(soci::session& sql)
-    {
-        spdlog::info("Creating table {}", table_name);
-        auto table = sql.create_table(std::string { table_name });
-
-        table.column("id", soci::dt_integer)("PRIMARY KEY");
-        table.column("path", soci::dt_string)("NOT NULL");
-        table.foreign_key("FK_Downloads", "id", std::string { File::table_name }, "id");
-    }
-};
-
-namespace soci
-{
-    template <>
-    struct type_conversion<Download>
-    {
-        typedef values base_type;
-
-        static void from_base(values const& i, indicator ind, Download& v)
-        {
-            v.id   = i.get<int>("id");
-            v.path = i.get<std::string>("path");
-        }
-
-        static void to_base(const Download& v, values& i, indicator& ind)
-        {
-            i.set("id", v.id);
-            i.set("path", v.path);
             ind = i_ok;
         }
     };

Modules/Server/src/worker.cpp

@@ -4,6 +4,7 @@
 #include "worker.hpp"
 
 #include <chrono>
+#include <filesystem>
 #include <functional>
 #include <vector>
 
@@ -21,6 +22,24 @@ void operator+=(std::vector<std::string>& vec, std::string const& s)
 
 namespace bp = boost::process;
 
+std::string san_string_path(std::string name)
+{
+    std::string out;
+    for (auto el : name)
+    {
+        if ((el >= 'a' && el <= 'z') || (el >= 'A' && el <= 'Z') || (el >= '0' && el <= '9') ||
+            el == '_' || el == '.')
+        {
+            out += el;
+        }
+        else if(el == ' ')
+        {
+            out += '_';
+        }
+    }
+    return out;
+}
+
 Worker::Worker(soci::session& sql) : thread(std::bind_front(&Worker::loop, this)), sql(sql)
 {
     spdlog::info("Worker started.");
@@ -30,7 +49,7 @@ Worker::Worker(soci::session& sql) : thread(std::bind_front(&Worker::loop, this)
 {
     while (true)
     {
-        File task;
+        Task task;
         if (!getTask(task))
         {
             std::this_thread::sleep_for(10s);
@@ -42,7 +61,7 @@ Worker::Worker(soci::session& sql) : thread(std::bind_front(&Worker::loop, this)
         spdlog::debug("Begin Download {}", task.id);
         task.status = FileStatus::DOWNLOADING;
 
-        sql << "UPDATE Files SET status = :status WHERE id = :id", soci::use(task);
+        sql << "UPDATE Tasks SET status = :status WHERE id = :id", soci::use(task);
 
         if (download(task))
         {
@@ -53,20 +72,22 @@ Worker::Worker(soci::session& sql) : thread(std::bind_front(&Worker::loop, this)
             // Error
             task.status = FileStatus::DOWNLOAD_ERROR;
             spdlog::error("Download Error {}: ", task);
-
         }
 
-        sql << "UPDATE Files SET status = :status WHERE id = :id", soci::use(task);
+        sql << "UPDATE Tasks SET status = :status WHERE id = :id", soci::use(task);
         std::this_thread::sleep_for(10s);
     }
 }
 
-bool Worker::getTask(File& task)
+bool Worker::getTask(Task& task)
 {
     try
     {
-        soci::rowset<File> data =
+        soci::rowset<Task> data = (sql.prepare << "SELECT * "
-          (sql.prepare << "SELECT * FROM Files WHERE status = 0 ORDER BY id LIMIT 1;");
+                                                  "FROM Tasks "
+                                                  "WHERE status=:PENDING "
+                                                  "ORDER BY timestamp LIMIT 1;",
+                                   soci::use(static_cast<int>(FileStatus::PENDING), "PENDING"));
 
         for (auto const& el : data)
         {
@@ -76,26 +97,36 @@ bool Worker::getTask(File& task)
     }
     catch (std::exception const& e)
     {
-        spdlog::error(e.what());
+        spdlog::error("[Worker] GetTask Error: {}", e.what());
-        fmt::format("{}\n", e.what());
     }
     return false;
 }
 
-bool Worker::download(File& task)
+bool Worker::download(Task& task)
 {
     bp::ipstream out;
     bp::ipstream err;
 
+    /* ** args ** */
     std::vector<std::string> args;
 
     args += "-o";
     args += fmt::format("{}/%(title)s.%(ext)s", output_path);
 
-    args += "--extract-audio";
+    if (task.audio_only)
+    {
+        args += "--extract-audio";
+    }
+
+    if (task.format)
+    {
+        args += task.audio_only ? "--audio-format" : "-f";
+        args += task.format.value();
+    }
 
     args += task.url;
 
+    /* ** Download ** */
     spdlog::debug("youtube-dl {}", fmt::join(args, " "));
 
     bp::child c("youtube-dl.exe", bp::args(args), bp::std_out > out, bp::std_err > err);
@@ -116,6 +147,13 @@ bool Worker::download(File& task)
                 destination_file = destination;
             }
 
+            if(auto [match, info, destination] =
+                ctre::match<R"regex(\[(.*)\] Merging formats into "(.*)")regex">(line);
+              match)
+            {
+                destination_file = destination;
+            }
+
         } else if (std::getline(err, line) && !line.empty())
         {
             if (auto [match, type, messages] = ctre::match<R"((.*): (.*))">(line); match)
@@ -132,13 +170,41 @@ bool Worker::download(File& task)
     c.wait();
     auto exit_code = c.exit_code();
 
+    if (exit_code != 0)
+        return false;
+
+    // Process Downloaded file
+    namespace fs = std::filesystem;
+
     spdlog::debug("File: {}", destination_file);
 
-    Download dwn;
+    auto downloaded_path = std::filesystem::path(destination_file);
-    dwn.id = task.id;
+    spdlog::info(absolute(downloaded_path).string());
-    dwn.path = destination_file;
+
+    auto san_filename = san_string_path(downloaded_path.filename().string());
+    auto local_path = fs::path(output_path) / (task.hash + downloaded_path.extension().string());
+
+    auto current_time = std::time(nullptr);
+
+    try{
+        fs::rename(fs::path(".") / downloaded_path, local_path );
+
+        File file;
+        file.id = task.id;
+        file.filename = san_filename;
+        file.local_path = local_path.string();
+        file.hash = task.hash;
+        file.format = downloaded_path.extension().string();
+        file.timestamp = current_time;
+
+        sql << "INSERT INTO Files (id, hash, timestamp, format, filename, local_path) "
+               "values(:id, :hash, :timestamp, :format, :filename, :local_path) ", soci::use(file);
+    }catch(std::exception const& e)
+    {
+        spdlog::error("File rename / insert error: {}", e.what());
+        return false;
+    }
 
-    sql << "INSERT INTO Downloads (id, path) values(:id, :path)", soci::use(dwn);
 
     spdlog::debug("youtube-dl exit code {}", exit_code);
     return exit_code == 0;

Modules/Server/src/worker.hpp

@@ -22,7 +22,7 @@ public:
     [[noreturn]] void loop();
 
 private:
-    bool getTask(File& task);
+    bool getTask(Task& task);
 
-    bool download(File& task);
+    bool download(Task& task);
 };