This commit is contained in:
Simon Hardt
2021-05-27 15:53:00 +02:00
parent 35d35ec087
commit 7903b0b8c5
15 changed files with 941 additions and 0 deletions

71
.clang-format Normal file
View File

@@ -0,0 +1,71 @@
Language: Cpp
TabWidth: 4
UseTab: Never
AccessModifierOffset: -4
MaxEmptyLinesToKeep: 4
ColumnLimit: 100
ContinuationIndentWidth: 2
IndentWidth: 4
BinPackArguments: false
AlignConsecutiveAssignments: true
AlignTrailingComments: true
AlignAfterOpenBracket: Align
AllowAllArgumentsOnNextLine: false
AllowAllConstructorInitializersOnNextLine: false
AllowShortBlocksOnASingleLine: false
AllowShortCaseLabelsOnASingleLine: false
AllowShortFunctionsOnASingleLine: Empty
AllowShortIfStatementsOnASingleLine: Never
AllowShortLambdasOnASingleLine: Empty
AllowShortLoopsOnASingleLine: false
AlwaysBreakTemplateDeclarations: Yes
BreakBeforeBraces: Custom
BraceWrapping:
AfterClass: true
AfterStruct: true
BeforeCatch: true
AfterControlStatement: "Always"
AfterFunction: true
AfterNamespace: true
AfterUnion: true
SplitEmptyFunction: true
SplitEmptyNamespace: true
SplitEmptyRecord: true
BreakConstructorInitializers: AfterColon
BreakInheritanceList: AfterColon
BreakStringLiterals: true
Cpp11BracedListStyle: false
FixNamespaceComments: true
IncludeBlocks: Preserve
SortIncludes: true
IndentCaseLabels: true
KeepEmptyLinesAtTheStartOfBlocks: false
NamespaceIndentation: All
PointerAlignment: Left
SortUsingDeclarations: true
SpaceAfterCStyleCast: false
SpaceBeforeAssignmentOperators: true
SpaceBeforeCpp11BracedList: true
SpaceBeforeCtorInitializerColon: true
SpaceBeforeInheritanceColon: true
SpaceBeforeRangeBasedForLoopColon: true
SpacesBeforeTrailingComments: 4
Standard: Auto

21
.gitignore vendored Normal file
View File

@@ -0,0 +1,21 @@
CMakeLists.txt.user
CMakeCache.txt
CMakeFiles
CMakeScripts
Testing
Makefile
cmake_install.cmake
install_manifest.txt
compile_commands.json
CTestTestfile.cmake
_deps
cmake-build-debug
.idea
.vs/
cmake-build-release/
CMakeSettings.json
Modules/Website/elm-stuff/

20
CMakeLists.txt Normal file
View File

@@ -0,0 +1,20 @@
cmake_minimum_required(VERSION 3.16)
if(NOT DEFINED CMAKE_TOOLCHAIN_FILE AND DEFINED ENV{VCPKG_ROOT})
set(CMAKE_TOOLCHAIN_FILE "$ENV{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake")
endif()
message("using vcpkg toolchain: ${CMAKE_TOOLCHAIN_FILE}" )
project(localTube)
set(CMAKE_CXX_STANDARD 20)
find_package(fmt CONFIG REQUIRED)
find_package(soci CONFIG REQUIRED)
find_package(spdlog CONFIG REQUIRED)
find_package(nlohmann_json CONFIG REQUIRED)
find_package(Boost COMPONENTS system filesystem REQUIRED)
add_subdirectory(Modules)

1
Modules/CMakeLists.txt Normal file
View File

@@ -0,0 +1 @@
add_subdirectory(Server)

View File

@@ -0,0 +1,23 @@
set(src_files
src/main.cpp
src/api.hpp
src/api.cpp
src/worker.hpp
src/worker.cpp
src/tables.hpp)
add_executable(. ${src_files})
target_link_libraries(.
fmt::fmt
spdlog::spdlog
SOCI::soci_core
SOCI::soci_empty
SOCI::soci_sqlite3
nlohmann_json::nlohmann_json
Boost::boost
${Boost_FILESYSTEM_LIBRARY}
${Boost_SYSTEM_LIBRARY})

150
Modules/Server/src/api.cpp Normal file
View File

@@ -0,0 +1,150 @@
//
// Created by s-Kaonnull on 25.05.2021.
//
#include "api.hpp"
#include "tables.hpp"
#include <fstream>
#include <functional>
#include <ctre.hpp>
#include <fmt/ostream.h>
#include <nlohmann/json.hpp>
#include <spdlog/spdlog.h>
using json = nlohmann::json;
Api::Api(soci::session& sql, httplib::Server& server) : sql(sql)
{
RegisterServerHandles(server);
}
void Api::RegisterServerHandles(httplib::Server& server)
{
spdlog::info("Register api endpoints...");
server.Post("/api/add", std::bind_front(&Api::add, this));
server.Get("/api/queue", std::bind_front(&Api::queue, this));
server.Get("/api/file/(\\d+)", std::bind_front(&Api::file, this));
server.Options("/(.*)", [this](httplib::Request const& rq, httplib::Response& rp) {
spdlog::info("Optin");
});
server.set_pre_routing_handler([this](httplib::Request const& rq, httplib::Response& rp) {
this->set_cross_headers(rp);
return httplib::Server::HandlerResponse::Unhandled;
});
}
void Api::add(const httplib::Request& rq, httplib::Response& rs)
{
try
{
auto data = json::parse(rq.body);
File f;
f.url = data["url"];
f.status = FileStatus::PENDING;
sql << "INSERT INTO Files (url, status) values(:url, :status)", soci::use(f.url, "url"),
soci::use(f.status, "status");
long long int tmp;
sql.get_last_insert_id("Files", tmp);
f.id = tmp;
json jr;
jr["status"] = "Ok";
jr["id"] = f.id;
spdlog::info("New Task: {}", f);
rs.set_content(jr.dump(), "application/json");
}
catch (json::exception const& e)
{
spdlog::error("Api Error: {}", e.what());
rs.status = 400;
rs.set_content(std::format("Json Error: {}", e.what()), "text/plain");
}
}
void Api::queue(httplib::Request const& rq, httplib::Response& rs)
{
try
{
json res;
soci::rowset<File> data = (sql.prepare << "SELECT * FROM Files");
for (auto const& el : data)
{
json current;
current["id"] = el.id;
current["status"] = magic_enum::enum_name(el.status);
current["status_id"] = static_cast<int>(el.status);
current["url"] = el.url;
res["queue"].push_back(current);
}
rs.set_content(res.dump(), "application/json");
}
catch (std::exception const& e)
{
spdlog::error(e.what());
throw e;
}
}
void Api::file(const httplib::Request& rq, httplib::Response& rs)
{
if (!rq.matches[1].matched)
{
rs.status = 404;
rs.set_content("Parameter error", "text/plain");
return;
}
auto file = rq.matches[1];
spdlog::info("File Request: {}", file);
try
{
Download dw;
sql << fmt::format("SELECT * FROM Downloads WHERE id={};", file), soci::into(dw);
std::ifstream fs(dw.path, std::ios_base::binary);
fs.seekg(0, std::ios_base::end);
auto size = fs.tellg();
fs.seekg(0);
rs.body.resize(static_cast<size_t>(size));
fs.read(&rs.body[0], static_cast<std::streamsize>(size));
rs.set_header("Content-Disposition", fmt::format("attachment; filename={};", dw.path));
if (auto [match, type] = ctre::match<R"(.*\.(.*))">(dw.path); match)
{
spdlog::debug(type);
rs.set_header("content-type", fmt::format("application/{}", type.to_view()));
}
}
catch (std::exception const& e)
{
spdlog::error(e.what());
rs.status = 404;
rs.set_content("File Not Found", "text/plain");
}
}
void Api::set_cross_headers(httplib::Response& rs)
{
rs.set_header("Access-Control-Allow-Origin", "*");
rs.set_header("Access-Control-Allow-Methods", "GET, POST, PUT");
rs.set_header("Access-Control-Allow-Headers", "Content-Type");
}

View File

@@ -0,0 +1,23 @@
//
// Created by Kaonnull on 25.05.2021.
//
#include <httplib.h>
#include <soci/soci.h>
class Api
{
soci::session& sql;
public:
Api(soci::session& sql, httplib::Server& server);
void RegisterServerHandles(httplib::Server& server);
public: // Endpoints
void add(httplib::Request const& rq, httplib::Response& rs);
void queue(httplib::Request const& rq, httplib::Response& rs);
void file(httplib::Request const& rq, httplib::Response& rs);
void set_cross_headers(httplib::Response& rs);
};

View File

@@ -0,0 +1,77 @@
//
// Created by Kaonnull on 25.05.2021.
//
#include <string_view>
#include <fmt/format.h>
#include <httplib.h>
#include <soci/soci.h>
#include <soci/sqlite3/soci-sqlite3.h>
#include <spdlog/spdlog.h>
#include "api.hpp"
#include "tables.hpp"
#include "worker.hpp"
int main()
{
spdlog::set_level(spdlog::level::debug);
constexpr std::string_view sqlite_db_name = "db.sqlite";
constexpr std::string_view version = "Version 1";
soci::register_factory_sqlite3();
spdlog::info("Opening sqlite3 db: {}", sqlite_db_name);
soci::session sql("sqlite3", fmt::format("db={}", sqlite_db_name));
if (!sql.is_connected())
{
spdlog::error("sqlite connection failed!");
return -1;
}
try
{
Version v;
sql << "SELECT * FROM Version", soci::into(v);
if (v.version == version)
{
spdlog::info("DB Loaded");
} else
{
spdlog::error("DB Version miss match. Try deleting the sqlite file.");
return -1;
}
}
catch (soci::soci_error const& err)
{
spdlog::info("DB not initialised. Creating db...");
spdlog::info("Creating Version table");
{
auto version_table = sql.create_table("Version");
version_table.column("ID", soci::dt_integer);
version_table.column("version_info", soci::dt_string)("NOT NULL");
version_table.primary_key("Version", "ID");
}
Version v { 1, std::string { version } };
sql << "insert into Version(ID, version_info) values(:ID, :version_info)", soci::use(v);
File::create_table(sql);
Download::create_table(sql);
}
Worker worker(sql);
httplib::Server srv;
srv.set_mount_point("/", "downloads/");
Api api(sql, srv);
spdlog::info("Listing on 0.0.0.0 Port 80");
srv.listen("0.0.0.0", 80);
}

View File

@@ -0,0 +1,160 @@
//
// Created by s-har on 25.05.2021.
//
#pragma once
#include <magic_enum.hpp>
#include <soci/soci.h>
#include <spdlog/spdlog.h>
struct Version
{
int key;
std::string version;
};
namespace soci
{
template <>
struct type_conversion<Version>
{
typedef values base_type;
static void from_base(values const& i, indicator ind, Version& v)
{
v.key = i.get<int>("ID");
v.version = i.get<std::string>("version_info");
}
static void to_base(const Version& v, values& i, indicator& ind)
{
i.set("ID", v.key);
i.set("version_info", v.version);
ind = i_ok;
}
};
} // namespace soci
enum class FileStatus { PENDING, DOWNLOADING, COMPLETE, DOWNLOAD_ERROR, NONE };
namespace soci
{
template <>
struct type_conversion<FileStatus>
{
typedef int base_type;
static void from_base(int i, indicator ind, FileStatus& v)
{
if (ind == i_null)
{
v = FileStatus::NONE;
} else if (i < 0 || i >= static_cast<int>(FileStatus::NONE))
{
v = FileStatus::NONE;
} else
{
v = static_cast<FileStatus>(i);
}
}
static void to_base(const FileStatus& v, int& i, indicator& ind)
{
i = static_cast<int>(v);
}
};
} // namespace soci
struct File
{
int id;
std::string url;
FileStatus status;
constexpr static std::string_view table_name = "Files";
inline static void create_table(soci::session& sql)
{
spdlog::info("Creating table {}", table_name);
auto table = sql.create_table(std::string { table_name });
table.column("id", soci::dt_integer)("PRIMARY KEY AUTOINCREMENT");
table.column("url", soci::dt_string)("NOT NULL");
table.column("status", soci::dt_integer)("NOT NULL");
// table.primary_key(std::string { table_name }, "id");
}
};
inline std::ostream& operator<<(std::ostream& os, File const& file)
{
return os << fmt::format("File [{}][{}] url: {} ",
file.id,
magic_enum::enum_name(file.status),
file.url);
}
namespace soci
{
template <>
struct type_conversion<File>
{
typedef values base_type;
static void from_base(values const& i, indicator ind, File& v)
{
v.id = i.get<int>("id");
v.url = i.get<std::string>("url");
v.status = i.get<FileStatus>("status");
}
static void to_base(const File& v, values& i, indicator& ind)
{
i.set("id", v.id);
i.set("url", v.url);
i.set("status", v.status);
ind = i_ok;
}
};
} // namespace soci
struct Download
{
int id;
std::string path;
constexpr static std::string_view table_name = "Downloads";
inline static void create_table(soci::session& sql)
{
spdlog::info("Creating table {}", table_name);
auto table = sql.create_table(std::string { table_name });
table.column("id", soci::dt_integer)("PRIMARY KEY");
table.column("path", soci::dt_string)("NOT NULL");
table.foreign_key("FK_Downloads", "id", std::string { File::table_name }, "id");
}
};
namespace soci
{
template <>
struct type_conversion<Download>
{
typedef values base_type;
static void from_base(values const& i, indicator ind, Download& v)
{
v.id = i.get<int>("id");
v.path = i.get<std::string>("path");
}
static void to_base(const Download& v, values& i, indicator& ind)
{
i.set("id", v.id);
i.set("path", v.path);
ind = i_ok;
}
};
} // namespace soci

View File

@@ -0,0 +1,145 @@
//
// Created by s-har on 26.05.2021.
//
#include "worker.hpp"
#include <chrono>
#include <functional>
#include <vector>
#include <boost/process.hpp>
#include <ctre.hpp>
#include <fmt/ostream.h>
#include <spdlog/spdlog.h>
using namespace std::chrono_literals;
void operator+=(std::vector<std::string>& vec, std::string const& s)
{
vec.push_back(s);
}
namespace bp = boost::process;
Worker::Worker(soci::session& sql) : thread(std::bind_front(&Worker::loop, this)), sql(sql)
{
spdlog::info("Worker started.");
}
[[noreturn]] void Worker::loop()
{
while (true)
{
File task;
if (!getTask(task))
{
std::this_thread::sleep_for(10s);
continue;
}
spdlog::info("Worker Task: {}", task);
spdlog::debug("Begin Download {}", task.id);
task.status = FileStatus::DOWNLOADING;
sql << "UPDATE Files SET status = :status WHERE id = :id", soci::use(task);
if (download(task))
{
spdlog::debug("Download end {}", task.id);
task.status = FileStatus::COMPLETE;
} else
{
// Error
task.status = FileStatus::DOWNLOAD_ERROR;
spdlog::error("Download Error {}: ", task);
}
sql << "UPDATE Files SET status = :status WHERE id = :id", soci::use(task);
std::this_thread::sleep_for(10s);
}
}
bool Worker::getTask(File& task)
{
try
{
soci::rowset<File> data =
(sql.prepare << "SELECT * FROM Files WHERE status = 0 ORDER BY id LIMIT 1;");
for (auto const& el : data)
{
task = el;
return true;
}
}
catch (std::exception const& e)
{
spdlog::error(e.what());
fmt::format("{}\n", e.what());
}
return false;
}
bool Worker::download(File& task)
{
bp::ipstream out;
bp::ipstream err;
std::vector<std::string> args;
args += "-o";
args += fmt::format("{}/%(title)s.%(ext)s", output_path);
args += "--extract-audio";
args += task.url;
spdlog::debug("youtube-dl {}", fmt::join(args, " "));
bp::child c("youtube-dl.exe", bp::args(args), bp::std_out > out, bp::std_err > err);
std::string destination_file;
std::string line;
while (c.running() || !out.eof() || !err.eof())
{
if (std::getline(out, line) && !line.empty())
{
spdlog::info("[youtube-dl] {}", line);
if (auto [match, info, destination] =
ctre::match<R"(\[(.*)\] Destination: (.*))">(line);
match)
{
destination_file = destination;
}
} else if (std::getline(err, line) && !line.empty())
{
if (auto [match, type, messages] = ctre::match<R"((.*): (.*))">(line); match)
{
if (type == "WARNING")
spdlog::warn("[youtube-dl] {}", messages);
} else
{
spdlog::error("[youtube-dl] {}", line);
}
}
}
c.wait();
auto exit_code = c.exit_code();
spdlog::debug("File: {}", destination_file);
Download dwn;
dwn.id = task.id;
dwn.path = destination_file;
sql << "INSERT INTO Downloads (id, path) values(:id, :path)", soci::use(dwn);
spdlog::debug("youtube-dl exit code {}", exit_code);
return exit_code == 0;
}

View File

@@ -0,0 +1,28 @@
//
// Created by s-har on 25.05.2021.
//
#pragma once
#include <vector>
#include <string>
#include <thread>
#include <soci/session.h>
#include "tables.hpp"
class Worker
{
std::thread thread;
soci::session& sql;
std::string output_path = "./downloads";
public:
explicit Worker(soci::session& sql);
[[noreturn]] void loop();
private:
bool getTask(File& task);
bool download(File& task);
};

29
Modules/Website/elm.json Normal file
View File

@@ -0,0 +1,29 @@
{
"type": "application",
"source-directories": [
"src"
],
"elm-version": "0.19.1",
"dependencies": {
"direct": {
"NoRedInk/elm-json-decode-pipeline": "1.0.0",
"elm/browser": "1.0.2",
"elm/core": "1.0.5",
"elm/html": "1.0.0",
"elm/http": "2.0.0",
"elm/json": "1.1.3",
"mdgriffith/elm-ui": "1.1.8"
},
"indirect": {
"elm/bytes": "1.0.8",
"elm/file": "1.0.5",
"elm/time": "1.0.0",
"elm/url": "1.0.0",
"elm/virtual-dom": "1.0.2"
}
},
"test-dependencies": {
"direct": {},
"indirect": {}
}
}

View File

@@ -0,0 +1,19 @@
module Color exposing (..)
white =
Element.rgb 1 1 1
grey =
Element.rgb 0.9 0.9 0.9
blue =
Element.rgb 0 0 0.8
red =
Element.rgb 0.8 0 0
darkBlue =
Element.rgb 0 0 0.9

View File

@@ -0,0 +1,157 @@
module Main exposing (main)
import Browser
import Html exposing (Html)
import Http
import Json.Decode exposing (Decoder, bool, int, list, string, succeed)
import Json.Decode.Pipeline exposing(optional, required)
import Json.Encode as Encode
import Element exposing (..)
import Element.Background as Background
import Element.Border as Border
import Element.Font as Font
import Element.Input as Input
import Element.Region as Region
import Html exposing (header)
white =
Element.rgb 1 1 1
grey =
Element.rgb 0.9 0.9 0.9
blue =
Element.rgb 0 0 0.8
red =
Element.rgb 0.8 0 0
darkBlue =
Element.rgb 0 0 0.9
type alias Form =
{ url: String
}
type Model
= AddPage Form
| Quere
type Msg
= UpdateForm Form
| AddUrl Form
| UrlAdded (Result Http.Error UrlAddedResponse)
initModel : Model
initModel =
AddPage
{ url = ""
}
type alias UrlAddedResponse = { id: Int, status: String }
urlAddDecoder : Decoder UrlAddedResponse
urlAddDecoder =
succeed UrlAddedResponse
|> required "id" int
|> required "status" string
urlAddEncode : Form -> Encode.Value
urlAddEncode form = Encode.object
[ ("url", Encode.string form.url)
]
-- Http.header "Access-Control-Request-Method" "POST"
-- ,
update : Msg -> Model -> (Model, Cmd Msg)
update msg model =
case (msg, model) of
(UpdateForm new_form, AddPage _) ->
(AddPage new_form, Cmd.none)
(AddUrl form, AddPage _) ->
( model
, Http.request
{ method = "POST"
, headers =
[
]
, url = "http://127.0.0.1/api/add"
, body = Http.jsonBody (urlAddEncode form)
, expect = Http.expectJson UrlAdded (urlAddDecoder)
, timeout = Nothing
, tracker = Nothing
}
)
( _, _ ) ->
Debug.todo "branch '( Decrement, _ )' not implemented"
view : Model -> Html Msg
view model =
Element.layout
[
Font.size 20
]
<| case (model) of
(AddPage form) ->
Element.column
[ width (px 800)
, height shrink
, centerX
, centerY
, spacing 36
, padding 10
]
[ el
[ Region.heading 1
, alignLeft
, Font.size 36
]
(text "localTube")
, Input.text
[ spacing 12
, below
(el
[ Font.color red
, Font.size 14
, alignRight
, moveDown 6]
(text "This one is wrong")
)
]
{ text = form.url
, placeholder = Just (Input.placeholder [] (text "http://youtube.com"))
, onChange = \new -> UpdateForm { form | url = new }
, label = Input.labelAbove [ Font.size 14 ](text "Video Url")
}
, Input.button
[ Background.color blue
, Font.color white
, Border.color darkBlue
, paddingXY 32 16
, Border.rounded 3
]
{ onPress = Just (AddUrl form)
, label = Element.text "Add Link to Query"
}
]
(_) ->
el
[ Region.heading 1
, Font.size 36](text "Non")
main: Program () Model Msg
main =
Browser.element
{ init = \flags -> (initModel, Cmd.none)
, view = view
, update = update
, subscriptions = \_ -> Sub.none
}

17
vcpkg.json Normal file
View File

@@ -0,0 +1,17 @@
{
"name": "template",
"version-string": "0.0.1",
"dependencies": [
"fmt",
"cpp-httplib",
"spdlog",
"nlohmann-json",
"magic-enum",
"boost-process",
"ctre",
{
"name": "soci",
"features": ["sqlite3"]
}
]
}