Kaonnull/localTube
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

New db layout

Browse Source
This commit is contained in:
Simon Hardt
2021-05-30 02:28:50 +02:00
parent 6000613dd2
commit ad73835904
5 changed files with 275 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files

127
Modules/Server/src/api.cpp
View File

@@ -9,8 +9,10 @@
#include <functional>
#include <ctre.hpp>
#include <fmt/chrono.h>
#include <fmt/ostream.h>
#include <nlohmann/json.hpp>
#include <openssl/md5.h>
#include <spdlog/spdlog.h>
using json = nlohmann::json;
@@ -28,49 +30,82 @@ void Api::RegisterServerHandles(httplib::Server& server)
server.Get("/api/files", std::bind_front(&Api::files, this));
server.Get("/api/file/(\\d+)", std::bind_front(&Api::file, this));
server.Get("/api/file/([A-E0-9]+)", std::bind_front(&Api::file, this));
server.Options("/(.*)", [this](httplib::Request const& rq, httplib::Response& rp) {});
server.set_pre_routing_handler([this](httplib::Request const& rq, httplib::Response& rp) {
this->set_cross_headers(rp);
spdlog::info("Request {}", rq.method);
spdlog::info("Request {} {}", rq.method, rq.path);
return httplib::Server::HandlerResponse::Unhandled;
});
}
/* /api/add
*
* body: json
* - url
* - audio_only - default false
* - format - optional default source
*
* return: json
* - status: bool
* - file_id: hash
* - error: string
*
*/
void Api::add(const httplib::Request& rq, httplib::Response& rs)
{
try
{
auto data = json::parse(rq.body);
File f;
// Parse Request
Task task;
task.url = data["url"];
task.audio_only = data.contains("audio_only") && data["audio_only"].get<bool>();
f.url = data["url"];
f.status = FileStatus::PENDING;
if (data.contains("format"))
task.format = data["format"].get<std::string>();
sql << "INSERT INTO Files (url, status) values(:url, :status)", soci::use(f.url, "url"),
soci::use(f.status, "status");
task.timestamp = std::time(nullptr);
long long int tmp;
task.hash = fmt::format(
"{:X}",
std::hash<std::string>()(
task.url + fmt::format("{:%Y-%m-%d %H:%M:%S}", *std::localtime(&task.timestamp))));
task.status = FileStatus::PENDING;
// Insert Sql
sql << "INSERT INTO Tasks (hash, timestamp, url, audio_only, status, format) "
"values(:hash, :timestamp, :url, :audio_only, :status, :format) ",
soci::use(task);
/*long long int tmp;
sql.get_last_insert_id("Files", tmp);
f.id = tmp;
f.id = tmp;*/
json jr;
jr["status"] = "Ok";
jr["id"] = f.id;
jr["file_id"] = task.hash;
spdlog::info("New Task: {}", f);
spdlog::info("New Task: {}", task);
rs.set_content(jr.dump(), "application/json");
}
catch (json::exception const& e)
{
spdlog::error("Api Error: {}", e.what());
json jr;
jr["status"] = "Err";
jr["error"] = e.what();
rs.status = 400;
rs.set_content(std::format("Json Error: {}", e.what()), "text/plain");
rs.set_content(jr.dump(), "application/json");
}
}
@@ -80,37 +115,52 @@ void Api::files(httplib::Request const& rq, httplib::Response& rs)
{
json res;
soci::rowset<File> data = (sql.prepare << "SELECT * FROM Files ORDER BY id DESC");
// Query incomplete Tasks
soci::rowset<Task> data = (sql.prepare << "SELECT * "
"FROM Tasks "
"WHERE status!=2 "
"ORDER BY id DESC ");
for (auto const& el : data)
{
json current;
current["id"] = el.id;
current["hash"] = el.hash;
current["status"] = magic_enum::enum_name(el.status);
current["status_id"] = static_cast<int>(el.status);
current["source"] = el.url;
current["name"] = fmt::format("File {}", el.id);
current["file_name"] = "Unknown";
if (el.status == FileStatus::COMPLETE && false)
{
std::string filename;
soci::rowset<std::string> ret =
(sql.prepare << "SELECT path From Downloads WHERE id=:id",
soci::use(el.id, "id"));
if (ret.begin() != ret.end())
{
std::filesystem::path p(*ret.begin());
current["file_name"] = p.filename();
}
}
current["audio_only"] = el.audio_only;
current["format"] = el.format.value_or("Source");
res["queue"].push_back(current);
}
// Downloaded
soci::rowset<soci::row> files =
(sql.prepare
<< R"(SELECT Tasks.id, Files.timestamp, Files.hash, audio_only, filename, Files.format
FROM Files,
Tasks
WHERE Tasks.hash == Files.hash
ORDER BY Files.timestamp DESC )");
for (auto const& el : files)
{
json current;
std::time_t time = el.get<int>("timestamp");
current["timestamp"] = fmt::format("{:%Y-%m-%d %H:%M:%S}", *std::localtime(&time));
current["hash"] = el.get<std::string>("hash");
current["audio_only"] = el.get<int>("audio_only");
current["filename"] = el.get<std::string>("filename");
current["format"] = el.get<std::string>("format");
res["files"].push_back(current);
}
rs.set_content(res.dump(), "application/json");
}
catch (std::exception const& e)
@@ -130,28 +180,23 @@ void Api::file(const httplib::Request& rq, httplib::Response& rs)
}
auto file = rq.matches[1];
spdlog::info("File Request: {}", file);
try
{
Download dw;
sql << fmt::format("SELECT * FROM Downloads WHERE id={};", file), soci::into(dw);
File dw;
sql << fmt::format("SELECT * FROM Files WHERE hash='{}';", file), soci::into(dw);
std::ifstream fs(dw.path, std::ios_base::binary);
std::ifstream fs(dw.local_path, std::ios_base::binary);
fs.seekg(0, std::ios_base::end);
auto size = fs.tellg();
fs.seekg(0);
rs.body.resize(static_cast<size_t>(size));
fs.read(&rs.body[0], static_cast<std::streamsize>(size));
rs.set_header("Content-Disposition", fmt::format("attachment; filename={};", dw.path));
if (auto [match, type] = ctre::match<R"(.*\.(.*))">(dw.path); match)
{
spdlog::debug(type);
rs.set_header("content-type", fmt::format("application/{}", type.to_view()));
}
rs.set_header("Content-Disposition", fmt::format("attachment; filename={};", dw.filename));
rs.set_header("content-type", fmt::format("application/{}", dw.format));
}
catch (std::exception const& e)
{

2
Modules/Server/src/main.cpp
View File

@@ -61,8 +61,8 @@ int main()
Version v { 1, std::string { version } };
sql << "insert into Version(ID, version_info) values(:ID, :version_info)", soci::use(v);
Task::create_table(sql);
File::create_table(sql);
Download::create_table(sql);
}
Worker worker(sql);

158
Modules/Server/src/tables.hpp
View File

@@ -66,11 +66,91 @@ namespace soci
} // namespace soci
struct Task
{
int id;
std::string hash;
std::time_t timestamp;
std::string url;
bool audio_only;
FileStatus status;
// Optional
std::optional<std::string> format;
constexpr static std::string_view table_name = "Tasks";
inline static void create_table(soci::session& sql)
{
spdlog::info("Creating table {}", table_name);
auto table = sql.create_table(std::string { table_name });
table.column("id", soci::dt_integer)("PRIMARY KEY AUTOINCREMENT");
table.column("hash", soci::dt_string)("NOT NULL");
table.column("timestamp", soci::dt_date)("NOT NULL");
table.column("url", soci::dt_string)("NOT NULL");
table.column("audio_only", soci::dt_integer)("NOT NULL");
table.column("status", soci::dt_integer)("NOT NULL");
table.column("format", soci::dt_string)("NOT NULL");
}
};
inline std::ostream& operator<<(std::ostream& os, Task const& file)
{
return os << fmt::format("Task [{}][{}] url: {} ",
file.id,
magic_enum::enum_name(file.status),
file.url);
}
namespace soci
{
template <>
struct type_conversion<Task>
{
typedef values base_type;
static void from_base(values const& i, indicator ind, Task& v)
{
v.id = i.get<int>("id");
v.timestamp = i.get<int>("timestamp");
v.hash = i.get<std::string>("hash");
v.url = i.get<std::string>("url");
v.audio_only = i.get<int>("audio_only") == 1;
v.status = i.get<FileStatus>("status");
auto format = i.get<std::string>("format");
if(format.empty())
v.format = {};
else
v.format = format ;
}
static void to_base(const Task& v, values& i, indicator& ind)
{
i.set("id", v.id);
i.set("timestamp", (int)v.timestamp);
i.set("hash", v.hash);
i.set("url", v.url);
i.set("audio_only", static_cast<int>(v.audio_only));
i.set("status", v.status);
i.set("format", v.format ? v.format.value() : std::string{});
ind = i_ok;
}
};
} // namespace soci
struct File
{
int id;
std::string url;
FileStatus status;
std::string hash;
std::time_t timestamp;
std::string format;
std::string filename;
std::string local_path;
constexpr static std::string_view table_name = "Files";
@@ -79,21 +159,16 @@ struct File
spdlog::info("Creating table {}", table_name);
auto table = sql.create_table(std::string { table_name });
table.column("id", soci::dt_integer)("PRIMARY KEY AUTOINCREMENT");
table.column("url", soci::dt_string)("NOT NULL");
table.column("status", soci::dt_integer)("NOT NULL");
// table.primary_key(std::string { table_name }, "id");
table.column("id", soci::dt_integer)("PRIMARY KEY");
table.column("hash", soci::dt_string)("NOT NULL");
table.column("timestamp", soci::dt_date)("NOT NULL");
table.column("format", soci::dt_string)("NOT NULL");
table.column("filename", soci::dt_string)("NOT NULL");
table.column("local_path", soci::dt_string)("NOT NULL");
table.foreign_key("FK_Downloads", "id", std::string { File::table_name }, "id");
}
};
inline std::ostream& operator<<(std::ostream& os, File const& file)
{
return os << fmt::format("File [{}][{}] url: {} ",
file.id,
magic_enum::enum_name(file.status),
file.url);
}
namespace soci
{
template <>
@@ -104,56 +179,21 @@ namespace soci
static void from_base(values const& i, indicator ind, File& v)
{
v.id = i.get<int>("id");
v.url = i.get<std::string>("url");
v.status = i.get<FileStatus>("status");
v.hash = i.get<std::string>("hash");
v.timestamp = i.get<int>("timestamp");
v.format = i.get<std::string>("format");
v.filename = i.get<std::string>("filename");
v.local_path = i.get<std::string>("local_path");
}
static void to_base(const File& v, values& i, indicator& ind)
{
i.set("id", v.id);
i.set("url", v.url);
i.set("status", v.status);
ind = i_ok;
}
};
} // namespace soci
struct Download
{
int id;
std::string path;
constexpr static std::string_view table_name = "Downloads";
inline static void create_table(soci::session& sql)
{
spdlog::info("Creating table {}", table_name);
auto table = sql.create_table(std::string { table_name });
table.column("id", soci::dt_integer)("PRIMARY KEY");
table.column("path", soci::dt_string)("NOT NULL");
table.foreign_key("FK_Downloads", "id", std::string { File::table_name }, "id");
}
};
namespace soci
{
template <>
struct type_conversion<Download>
{
typedef values base_type;
static void from_base(values const& i, indicator ind, Download& v)
{
v.id = i.get<int>("id");
v.path = i.get<std::string>("path");
}
static void to_base(const Download& v, values& i, indicator& ind)
{
i.set("id", v.id);
i.set("path", v.path);
i.set("hash", v.hash);
i.set("timestamp", (int)v.timestamp);
i.set("format", v.format);
i.set("filename", v.filename);
i.set("local_path", v.local_path);
ind = i_ok;
}
};

94
Modules/Server/src/worker.cpp
View File

@@ -4,6 +4,7 @@
#include "worker.hpp"
#include <chrono>
#include <filesystem>
#include <functional>
#include <vector>
@@ -21,6 +22,24 @@ void operator+=(std::vector<std::string>& vec, std::string const& s)
namespace bp = boost::process;
std::string san_string_path(std::string name)
{
std::string out;
for (auto el : name)
{
if ((el >= 'a' && el <= 'z') || (el >= 'A' && el <= 'Z') || (el >= '0' && el <= '9') ||
el == '_' || el == '.')
{
out += el;
}
else if(el == ' ')
{
out += '_';
}
}
return out;
}
Worker::Worker(soci::session& sql) : thread(std::bind_front(&Worker::loop, this)), sql(sql)
{
spdlog::info("Worker started.");
@@ -30,7 +49,7 @@ Worker::Worker(soci::session& sql) : thread(std::bind_front(&Worker::loop, this)
{
while (true)
{
File task;
Task task;
if (!getTask(task))
{
std::this_thread::sleep_for(10s);
@@ -42,7 +61,7 @@ Worker::Worker(soci::session& sql) : thread(std::bind_front(&Worker::loop, this)
spdlog::debug("Begin Download {}", task.id);
task.status = FileStatus::DOWNLOADING;
sql << "UPDATE Files SET status = :status WHERE id = :id", soci::use(task);
sql << "UPDATE Tasks SET status = :status WHERE id = :id", soci::use(task);
if (download(task))
{
@@ -53,20 +72,22 @@ Worker::Worker(soci::session& sql) : thread(std::bind_front(&Worker::loop, this)
// Error
task.status = FileStatus::DOWNLOAD_ERROR;
spdlog::error("Download Error {}: ", task);
}
sql << "UPDATE Files SET status = :status WHERE id = :id", soci::use(task);
sql << "UPDATE Tasks SET status = :status WHERE id = :id", soci::use(task);
std::this_thread::sleep_for(10s);
}
}
bool Worker::getTask(File& task)
bool Worker::getTask(Task& task)
{
try
{
soci::rowset<File> data =
(sql.prepare << "SELECT * FROM Files WHERE status = 0 ORDER BY id LIMIT 1;");
soci::rowset<Task> data = (sql.prepare << "SELECT * "
"FROM Tasks "
"WHERE status=:PENDING "
"ORDER BY timestamp LIMIT 1;",
soci::use(static_cast<int>(FileStatus::PENDING), "PENDING"));
for (auto const& el : data)
{
@@ -76,26 +97,36 @@ bool Worker::getTask(File& task)
}
catch (std::exception const& e)
{
spdlog::error(e.what());
fmt::format("{}\n", e.what());
spdlog::error("[Worker] GetTask Error: {}", e.what());
}
return false;
}
bool Worker::download(File& task)
bool Worker::download(Task& task)
{
bp::ipstream out;
bp::ipstream err;
/* ** args ** */
std::vector<std::string> args;
args += "-o";
args += fmt::format("{}/%(title)s.%(ext)s", output_path);
if (task.audio_only)
{
args += "--extract-audio";
}
if (task.format)
{
args += task.audio_only ? "--audio-format" : "-f";
args += task.format.value();
}
args += task.url;
/* ** Download ** */
spdlog::debug("youtube-dl {}", fmt::join(args, " "));
bp::child c("youtube-dl.exe", bp::args(args), bp::std_out > out, bp::std_err > err);
@@ -116,6 +147,13 @@ bool Worker::download(File& task)
destination_file = destination;
}
if(auto [match, info, destination] =
ctre::match<R"regex(\[(.*)\] Merging formats into "(.*)")regex">(line);
match)
{
destination_file = destination;
}
} else if (std::getline(err, line) && !line.empty())
{
if (auto [match, type, messages] = ctre::match<R"((.*): (.*))">(line); match)
@@ -132,13 +170,41 @@ bool Worker::download(File& task)
c.wait();
auto exit_code = c.exit_code();
if (exit_code != 0)
return false;
// Process Downloaded file
namespace fs = std::filesystem;
spdlog::debug("File: {}", destination_file);
Download dwn;
dwn.id = task.id;
dwn.path = destination_file;
auto downloaded_path = std::filesystem::path(destination_file);
spdlog::info(absolute(downloaded_path).string());
auto san_filename = san_string_path(downloaded_path.filename().string());
auto local_path = fs::path(output_path) / (task.hash + downloaded_path.extension().string());
auto current_time = std::time(nullptr);
try{
fs::rename(fs::path(".") / downloaded_path, local_path );
File file;
file.id = task.id;
file.filename = san_filename;
file.local_path = local_path.string();
file.hash = task.hash;
file.format = downloaded_path.extension().string();
file.timestamp = current_time;
sql << "INSERT INTO Files (id, hash, timestamp, format, filename, local_path) "
"values(:id, :hash, :timestamp, :format, :filename, :local_path) ", soci::use(file);
}catch(std::exception const& e)
{
spdlog::error("File rename / insert error: {}", e.what());
return false;
}
sql << "INSERT INTO Downloads (id, path) values(:id, :path)", soci::use(dwn);
spdlog::debug("youtube-dl exit code {}", exit_code);
return exit_code == 0;

4
Modules/Server/src/worker.hpp
View File

@@ -22,7 +22,7 @@ public:
[[noreturn]] void loop();
private:
bool getTask(File& task);
bool getTask(Task& task);
bool download(File& task);
bool download(Task& task);
};