Init
This commit is contained in:
Simon Hardt
23
Modules/Server/CMakeLists.txt
Normal file
23
Modules/Server/CMakeLists.txt
Normal file
@@ -0,0 +1,23 @@
|
||||
|
||||
set(src_files
|
||||
src/main.cpp
|
||||
src/api.hpp
|
||||
src/api.cpp
|
||||
src/worker.hpp
|
||||
src/worker.cpp
|
||||
src/tables.hpp)
|
||||
|
||||
|
||||
add_executable(. ${src_files})
|
||||
|
||||
|
||||
target_link_libraries(.
|
||||
fmt::fmt
|
||||
spdlog::spdlog
|
||||
SOCI::soci_core
|
||||
SOCI::soci_empty
|
||||
SOCI::soci_sqlite3
|
||||
nlohmann_json::nlohmann_json
|
||||
Boost::boost
|
||||
${Boost_FILESYSTEM_LIBRARY}
|
||||
${Boost_SYSTEM_LIBRARY})
|
||||
150
Modules/Server/src/api.cpp
Normal file
150
Modules/Server/src/api.cpp
Normal file
@@ -0,0 +1,150 @@
|
||||
//
|
||||
// Created by s-Kaonnull on 25.05.2021.
|
||||
//
|
||||
#include "api.hpp"
|
||||
#include "tables.hpp"
|
||||
|
||||
#include <fstream>
|
||||
#include <functional>
|
||||
|
||||
#include <ctre.hpp>
|
||||
#include <fmt/ostream.h>
|
||||
#include <nlohmann/json.hpp>
|
||||
#include <spdlog/spdlog.h>
|
||||
|
||||
using json = nlohmann::json;
|
||||
|
||||
Api::Api(soci::session& sql, httplib::Server& server) : sql(sql)
|
||||
{
|
||||
RegisterServerHandles(server);
|
||||
}
|
||||
|
||||
void Api::RegisterServerHandles(httplib::Server& server)
|
||||
{
|
||||
spdlog::info("Register api endpoints...");
|
||||
|
||||
server.Post("/api/add", std::bind_front(&Api::add, this));
|
||||
|
||||
server.Get("/api/queue", std::bind_front(&Api::queue, this));
|
||||
|
||||
server.Get("/api/file/(\\d+)", std::bind_front(&Api::file, this));
|
||||
|
||||
server.Options("/(.*)", [this](httplib::Request const& rq, httplib::Response& rp) {
|
||||
spdlog::info("Optin");
|
||||
});
|
||||
|
||||
server.set_pre_routing_handler([this](httplib::Request const& rq, httplib::Response& rp) {
|
||||
this->set_cross_headers(rp);
|
||||
return httplib::Server::HandlerResponse::Unhandled;
|
||||
});
|
||||
}
|
||||
|
||||
void Api::add(const httplib::Request& rq, httplib::Response& rs)
|
||||
{
|
||||
try
|
||||
{
|
||||
auto data = json::parse(rq.body);
|
||||
|
||||
File f;
|
||||
|
||||
|
||||
f.url = data["url"];
|
||||
f.status = FileStatus::PENDING;
|
||||
|
||||
sql << "INSERT INTO Files (url, status) values(:url, :status)", soci::use(f.url, "url"),
|
||||
soci::use(f.status, "status");
|
||||
|
||||
long long int tmp;
|
||||
sql.get_last_insert_id("Files", tmp);
|
||||
f.id = tmp;
|
||||
|
||||
json jr;
|
||||
jr["status"] = "Ok";
|
||||
jr["id"] = f.id;
|
||||
|
||||
spdlog::info("New Task: {}", f);
|
||||
|
||||
rs.set_content(jr.dump(), "application/json");
|
||||
}
|
||||
catch (json::exception const& e)
|
||||
{
|
||||
spdlog::error("Api Error: {}", e.what());
|
||||
rs.status = 400;
|
||||
rs.set_content(std::format("Json Error: {}", e.what()), "text/plain");
|
||||
}
|
||||
}
|
||||
|
||||
void Api::queue(httplib::Request const& rq, httplib::Response& rs)
|
||||
{
|
||||
try
|
||||
{
|
||||
json res;
|
||||
|
||||
soci::rowset<File> data = (sql.prepare << "SELECT * FROM Files");
|
||||
|
||||
for (auto const& el : data)
|
||||
{
|
||||
json current;
|
||||
current["id"] = el.id;
|
||||
current["status"] = magic_enum::enum_name(el.status);
|
||||
current["status_id"] = static_cast<int>(el.status);
|
||||
current["url"] = el.url;
|
||||
|
||||
res["queue"].push_back(current);
|
||||
}
|
||||
|
||||
rs.set_content(res.dump(), "application/json");
|
||||
}
|
||||
catch (std::exception const& e)
|
||||
{
|
||||
spdlog::error(e.what());
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
void Api::file(const httplib::Request& rq, httplib::Response& rs)
|
||||
{
|
||||
if (!rq.matches[1].matched)
|
||||
{
|
||||
rs.status = 404;
|
||||
rs.set_content("Parameter error", "text/plain");
|
||||
return;
|
||||
}
|
||||
|
||||
auto file = rq.matches[1];
|
||||
|
||||
spdlog::info("File Request: {}", file);
|
||||
|
||||
try
|
||||
{
|
||||
Download dw;
|
||||
sql << fmt::format("SELECT * FROM Downloads WHERE id={};", file), soci::into(dw);
|
||||
|
||||
|
||||
std::ifstream fs(dw.path, std::ios_base::binary);
|
||||
fs.seekg(0, std::ios_base::end);
|
||||
auto size = fs.tellg();
|
||||
fs.seekg(0);
|
||||
rs.body.resize(static_cast<size_t>(size));
|
||||
fs.read(&rs.body[0], static_cast<std::streamsize>(size));
|
||||
|
||||
rs.set_header("Content-Disposition", fmt::format("attachment; filename={};", dw.path));
|
||||
if (auto [match, type] = ctre::match<R"(.*\.(.*))">(dw.path); match)
|
||||
{
|
||||
spdlog::debug(type);
|
||||
rs.set_header("content-type", fmt::format("application/{}", type.to_view()));
|
||||
}
|
||||
}
|
||||
catch (std::exception const& e)
|
||||
{
|
||||
spdlog::error(e.what());
|
||||
rs.status = 404;
|
||||
rs.set_content("File Not Found", "text/plain");
|
||||
}
|
||||
}
|
||||
void Api::set_cross_headers(httplib::Response& rs)
|
||||
{
|
||||
rs.set_header("Access-Control-Allow-Origin", "*");
|
||||
rs.set_header("Access-Control-Allow-Methods", "GET, POST, PUT");
|
||||
rs.set_header("Access-Control-Allow-Headers", "Content-Type");
|
||||
}
|
||||
23
Modules/Server/src/api.hpp
Normal file
23
Modules/Server/src/api.hpp
Normal file
@@ -0,0 +1,23 @@
|
||||
//
|
||||
// Created by Kaonnull on 25.05.2021.
|
||||
//
|
||||
#include <httplib.h>
|
||||
#include <soci/soci.h>
|
||||
|
||||
class Api
|
||||
{
|
||||
soci::session& sql;
|
||||
public:
|
||||
Api(soci::session& sql, httplib::Server& server);
|
||||
|
||||
void RegisterServerHandles(httplib::Server& server);
|
||||
|
||||
public: // Endpoints
|
||||
void add(httplib::Request const& rq, httplib::Response& rs);
|
||||
|
||||
void queue(httplib::Request const& rq, httplib::Response& rs);
|
||||
void file(httplib::Request const& rq, httplib::Response& rs);
|
||||
|
||||
void set_cross_headers(httplib::Response& rs);
|
||||
|
||||
};
|
||||
77
Modules/Server/src/main.cpp
Normal file
77
Modules/Server/src/main.cpp
Normal file
@@ -0,0 +1,77 @@
|
||||
//
|
||||
// Created by Kaonnull on 25.05.2021.
|
||||
//
|
||||
#include <string_view>
|
||||
|
||||
#include <fmt/format.h>
|
||||
#include <httplib.h>
|
||||
#include <soci/soci.h>
|
||||
#include <soci/sqlite3/soci-sqlite3.h>
|
||||
#include <spdlog/spdlog.h>
|
||||
|
||||
#include "api.hpp"
|
||||
#include "tables.hpp"
|
||||
#include "worker.hpp"
|
||||
|
||||
int main()
|
||||
{
|
||||
spdlog::set_level(spdlog::level::debug);
|
||||
|
||||
constexpr std::string_view sqlite_db_name = "db.sqlite";
|
||||
constexpr std::string_view version = "Version 1";
|
||||
|
||||
soci::register_factory_sqlite3();
|
||||
|
||||
|
||||
spdlog::info("Opening sqlite3 db: {}", sqlite_db_name);
|
||||
soci::session sql("sqlite3", fmt::format("db={}", sqlite_db_name));
|
||||
|
||||
if (!sql.is_connected())
|
||||
{
|
||||
spdlog::error("sqlite connection failed!");
|
||||
return -1;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
Version v;
|
||||
sql << "SELECT * FROM Version", soci::into(v);
|
||||
|
||||
if (v.version == version)
|
||||
{
|
||||
spdlog::info("DB Loaded");
|
||||
} else
|
||||
{
|
||||
spdlog::error("DB Version miss match. Try deleting the sqlite file.");
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
catch (soci::soci_error const& err)
|
||||
{
|
||||
spdlog::info("DB not initialised. Creating db...");
|
||||
|
||||
spdlog::info("Creating Version table");
|
||||
{
|
||||
auto version_table = sql.create_table("Version");
|
||||
version_table.column("ID", soci::dt_integer);
|
||||
version_table.column("version_info", soci::dt_string)("NOT NULL");
|
||||
version_table.primary_key("Version", "ID");
|
||||
}
|
||||
|
||||
Version v { 1, std::string { version } };
|
||||
sql << "insert into Version(ID, version_info) values(:ID, :version_info)", soci::use(v);
|
||||
|
||||
File::create_table(sql);
|
||||
Download::create_table(sql);
|
||||
}
|
||||
|
||||
Worker worker(sql);
|
||||
|
||||
httplib::Server srv;
|
||||
srv.set_mount_point("/", "downloads/");
|
||||
|
||||
Api api(sql, srv);
|
||||
|
||||
spdlog::info("Listing on 0.0.0.0 Port 80");
|
||||
srv.listen("0.0.0.0", 80);
|
||||
}
|
||||
160
Modules/Server/src/tables.hpp
Normal file
160
Modules/Server/src/tables.hpp
Normal file
@@ -0,0 +1,160 @@
|
||||
//
|
||||
// Created by s-har on 25.05.2021.
|
||||
//
|
||||
#pragma once
|
||||
#include <magic_enum.hpp>
|
||||
#include <soci/soci.h>
|
||||
#include <spdlog/spdlog.h>
|
||||
|
||||
|
||||
struct Version
|
||||
{
|
||||
int key;
|
||||
std::string version;
|
||||
};
|
||||
|
||||
namespace soci
|
||||
{
|
||||
template <>
|
||||
struct type_conversion<Version>
|
||||
{
|
||||
typedef values base_type;
|
||||
|
||||
static void from_base(values const& i, indicator ind, Version& v)
|
||||
{
|
||||
v.key = i.get<int>("ID");
|
||||
v.version = i.get<std::string>("version_info");
|
||||
}
|
||||
|
||||
static void to_base(const Version& v, values& i, indicator& ind)
|
||||
{
|
||||
i.set("ID", v.key);
|
||||
i.set("version_info", v.version);
|
||||
ind = i_ok;
|
||||
}
|
||||
};
|
||||
} // namespace soci
|
||||
|
||||
enum class FileStatus { PENDING, DOWNLOADING, COMPLETE, DOWNLOAD_ERROR, NONE };
|
||||
|
||||
namespace soci
|
||||
{
|
||||
template <>
|
||||
struct type_conversion<FileStatus>
|
||||
{
|
||||
typedef int base_type;
|
||||
|
||||
static void from_base(int i, indicator ind, FileStatus& v)
|
||||
{
|
||||
if (ind == i_null)
|
||||
{
|
||||
v = FileStatus::NONE;
|
||||
} else if (i < 0 || i >= static_cast<int>(FileStatus::NONE))
|
||||
{
|
||||
v = FileStatus::NONE;
|
||||
} else
|
||||
{
|
||||
v = static_cast<FileStatus>(i);
|
||||
}
|
||||
}
|
||||
|
||||
static void to_base(const FileStatus& v, int& i, indicator& ind)
|
||||
{
|
||||
i = static_cast<int>(v);
|
||||
}
|
||||
};
|
||||
} // namespace soci
|
||||
|
||||
|
||||
struct File
|
||||
{
|
||||
int id;
|
||||
std::string url;
|
||||
FileStatus status;
|
||||
|
||||
constexpr static std::string_view table_name = "Files";
|
||||
|
||||
inline static void create_table(soci::session& sql)
|
||||
{
|
||||
spdlog::info("Creating table {}", table_name);
|
||||
auto table = sql.create_table(std::string { table_name });
|
||||
|
||||
table.column("id", soci::dt_integer)("PRIMARY KEY AUTOINCREMENT");
|
||||
table.column("url", soci::dt_string)("NOT NULL");
|
||||
table.column("status", soci::dt_integer)("NOT NULL");
|
||||
// table.primary_key(std::string { table_name }, "id");
|
||||
}
|
||||
};
|
||||
|
||||
inline std::ostream& operator<<(std::ostream& os, File const& file)
|
||||
{
|
||||
return os << fmt::format("File [{}][{}] url: {} ",
|
||||
file.id,
|
||||
magic_enum::enum_name(file.status),
|
||||
file.url);
|
||||
}
|
||||
|
||||
namespace soci
|
||||
{
|
||||
template <>
|
||||
struct type_conversion<File>
|
||||
{
|
||||
typedef values base_type;
|
||||
|
||||
static void from_base(values const& i, indicator ind, File& v)
|
||||
{
|
||||
v.id = i.get<int>("id");
|
||||
v.url = i.get<std::string>("url");
|
||||
v.status = i.get<FileStatus>("status");
|
||||
}
|
||||
|
||||
static void to_base(const File& v, values& i, indicator& ind)
|
||||
{
|
||||
i.set("id", v.id);
|
||||
i.set("url", v.url);
|
||||
i.set("status", v.status);
|
||||
ind = i_ok;
|
||||
}
|
||||
};
|
||||
} // namespace soci
|
||||
|
||||
|
||||
struct Download
|
||||
{
|
||||
int id;
|
||||
std::string path;
|
||||
|
||||
constexpr static std::string_view table_name = "Downloads";
|
||||
|
||||
inline static void create_table(soci::session& sql)
|
||||
{
|
||||
spdlog::info("Creating table {}", table_name);
|
||||
auto table = sql.create_table(std::string { table_name });
|
||||
|
||||
table.column("id", soci::dt_integer)("PRIMARY KEY");
|
||||
table.column("path", soci::dt_string)("NOT NULL");
|
||||
table.foreign_key("FK_Downloads", "id", std::string { File::table_name }, "id");
|
||||
}
|
||||
};
|
||||
|
||||
namespace soci
|
||||
{
|
||||
template <>
|
||||
struct type_conversion<Download>
|
||||
{
|
||||
typedef values base_type;
|
||||
|
||||
static void from_base(values const& i, indicator ind, Download& v)
|
||||
{
|
||||
v.id = i.get<int>("id");
|
||||
v.path = i.get<std::string>("path");
|
||||
}
|
||||
|
||||
static void to_base(const Download& v, values& i, indicator& ind)
|
||||
{
|
||||
i.set("id", v.id);
|
||||
i.set("path", v.path);
|
||||
ind = i_ok;
|
||||
}
|
||||
};
|
||||
} // namespace soci
|
||||
145
Modules/Server/src/worker.cpp
Normal file
145
Modules/Server/src/worker.cpp
Normal file
@@ -0,0 +1,145 @@
|
||||
//
|
||||
// Created by s-har on 26.05.2021.
|
||||
//
|
||||
#include "worker.hpp"
|
||||
|
||||
#include <chrono>
|
||||
#include <functional>
|
||||
#include <vector>
|
||||
|
||||
#include <boost/process.hpp>
|
||||
#include <ctre.hpp>
|
||||
#include <fmt/ostream.h>
|
||||
#include <spdlog/spdlog.h>
|
||||
|
||||
using namespace std::chrono_literals;
|
||||
|
||||
void operator+=(std::vector<std::string>& vec, std::string const& s)
|
||||
{
|
||||
vec.push_back(s);
|
||||
}
|
||||
|
||||
namespace bp = boost::process;
|
||||
|
||||
Worker::Worker(soci::session& sql) : thread(std::bind_front(&Worker::loop, this)), sql(sql)
|
||||
{
|
||||
spdlog::info("Worker started.");
|
||||
}
|
||||
|
||||
[[noreturn]] void Worker::loop()
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
File task;
|
||||
if (!getTask(task))
|
||||
{
|
||||
std::this_thread::sleep_for(10s);
|
||||
continue;
|
||||
}
|
||||
|
||||
spdlog::info("Worker Task: {}", task);
|
||||
|
||||
spdlog::debug("Begin Download {}", task.id);
|
||||
task.status = FileStatus::DOWNLOADING;
|
||||
|
||||
sql << "UPDATE Files SET status = :status WHERE id = :id", soci::use(task);
|
||||
|
||||
if (download(task))
|
||||
{
|
||||
spdlog::debug("Download end {}", task.id);
|
||||
task.status = FileStatus::COMPLETE;
|
||||
} else
|
||||
{
|
||||
// Error
|
||||
task.status = FileStatus::DOWNLOAD_ERROR;
|
||||
spdlog::error("Download Error {}: ", task);
|
||||
|
||||
}
|
||||
|
||||
sql << "UPDATE Files SET status = :status WHERE id = :id", soci::use(task);
|
||||
std::this_thread::sleep_for(10s);
|
||||
}
|
||||
}
|
||||
|
||||
bool Worker::getTask(File& task)
|
||||
{
|
||||
try
|
||||
{
|
||||
soci::rowset<File> data =
|
||||
(sql.prepare << "SELECT * FROM Files WHERE status = 0 ORDER BY id LIMIT 1;");
|
||||
|
||||
for (auto const& el : data)
|
||||
{
|
||||
task = el;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
catch (std::exception const& e)
|
||||
{
|
||||
spdlog::error(e.what());
|
||||
fmt::format("{}\n", e.what());
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool Worker::download(File& task)
|
||||
{
|
||||
bp::ipstream out;
|
||||
bp::ipstream err;
|
||||
|
||||
std::vector<std::string> args;
|
||||
|
||||
args += "-o";
|
||||
args += fmt::format("{}/%(title)s.%(ext)s", output_path);
|
||||
|
||||
args += "--extract-audio";
|
||||
|
||||
args += task.url;
|
||||
|
||||
spdlog::debug("youtube-dl {}", fmt::join(args, " "));
|
||||
|
||||
bp::child c("youtube-dl.exe", bp::args(args), bp::std_out > out, bp::std_err > err);
|
||||
|
||||
std::string destination_file;
|
||||
|
||||
std::string line;
|
||||
while (c.running() || !out.eof() || !err.eof())
|
||||
{
|
||||
if (std::getline(out, line) && !line.empty())
|
||||
{
|
||||
spdlog::info("[youtube-dl] {}", line);
|
||||
|
||||
if (auto [match, info, destination] =
|
||||
ctre::match<R"(\[(.*)\] Destination: (.*))">(line);
|
||||
match)
|
||||
{
|
||||
destination_file = destination;
|
||||
}
|
||||
|
||||
} else if (std::getline(err, line) && !line.empty())
|
||||
{
|
||||
if (auto [match, type, messages] = ctre::match<R"((.*): (.*))">(line); match)
|
||||
{
|
||||
if (type == "WARNING")
|
||||
spdlog::warn("[youtube-dl] {}", messages);
|
||||
} else
|
||||
{
|
||||
spdlog::error("[youtube-dl] {}", line);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
c.wait();
|
||||
auto exit_code = c.exit_code();
|
||||
|
||||
spdlog::debug("File: {}", destination_file);
|
||||
|
||||
Download dwn;
|
||||
dwn.id = task.id;
|
||||
dwn.path = destination_file;
|
||||
|
||||
sql << "INSERT INTO Downloads (id, path) values(:id, :path)", soci::use(dwn);
|
||||
|
||||
spdlog::debug("youtube-dl exit code {}", exit_code);
|
||||
return exit_code == 0;
|
||||
}
|
||||
28
Modules/Server/src/worker.hpp
Normal file
28
Modules/Server/src/worker.hpp
Normal file
@@ -0,0 +1,28 @@
|
||||
//
|
||||
// Created by s-har on 25.05.2021.
|
||||
//
|
||||
#pragma once
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
|
||||
#include <soci/session.h>
|
||||
|
||||
#include "tables.hpp"
|
||||
|
||||
class Worker
|
||||
{
|
||||
std::thread thread;
|
||||
soci::session& sql;
|
||||
|
||||
std::string output_path = "./downloads";
|
||||
public:
|
||||
explicit Worker(soci::session& sql);
|
||||
|
||||
[[noreturn]] void loop();
|
||||
|
||||
private:
|
||||
bool getTask(File& task);
|
||||
|
||||
bool download(File& task);
|
||||
};
|
||||
Reference in New Issue
Block a user