diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..8fa7edb --- /dev/null +++ b/.clang-format @@ -0,0 +1,71 @@ +Language: Cpp +TabWidth: 4 +UseTab: Never +AccessModifierOffset: -4 +MaxEmptyLinesToKeep: 4 +ColumnLimit: 100 +ContinuationIndentWidth: 2 +IndentWidth: 4 + +BinPackArguments: false + +AlignConsecutiveAssignments: true +AlignTrailingComments: true +AlignAfterOpenBracket: Align + +AllowAllArgumentsOnNextLine: false +AllowAllConstructorInitializersOnNextLine: false + +AllowShortBlocksOnASingleLine: false +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: Empty +AllowShortIfStatementsOnASingleLine: Never +AllowShortLambdasOnASingleLine: Empty +AllowShortLoopsOnASingleLine: false + + +AlwaysBreakTemplateDeclarations: Yes + +BreakBeforeBraces: Custom +BraceWrapping: + AfterClass: true + AfterStruct: true + BeforeCatch: true + AfterControlStatement: "Always" + AfterFunction: true + AfterNamespace: true + AfterUnion: true + SplitEmptyFunction: true + SplitEmptyNamespace: true + SplitEmptyRecord: true + +BreakConstructorInitializers: AfterColon +BreakInheritanceList: AfterColon +BreakStringLiterals: true + +Cpp11BracedListStyle: false + +FixNamespaceComments: true + +IncludeBlocks: Preserve +SortIncludes: true + +IndentCaseLabels: true + +KeepEmptyLinesAtTheStartOfBlocks: false +NamespaceIndentation: All + +PointerAlignment: Left + +SortUsingDeclarations: true + +SpaceAfterCStyleCast: false +SpaceBeforeAssignmentOperators: true +SpaceBeforeCpp11BracedList: true +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true +SpaceBeforeRangeBasedForLoopColon: true + +SpacesBeforeTrailingComments: 4 + +Standard: Auto \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4780ab3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,21 @@ +CMakeLists.txt.user +CMakeCache.txt +CMakeFiles +CMakeScripts +Testing +Makefile +cmake_install.cmake +install_manifest.txt +compile_commands.json +CTestTestfile.cmake +_deps + +cmake-build-debug +.idea +.vs/ + +cmake-build-release/ + +CMakeSettings.json + +Modules/Website/elm-stuff/ diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..c27ec1e --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,20 @@ +cmake_minimum_required(VERSION 3.16) + +if(NOT DEFINED CMAKE_TOOLCHAIN_FILE AND DEFINED ENV{VCPKG_ROOT}) + set(CMAKE_TOOLCHAIN_FILE "$ENV{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake") +endif() +message("using vcpkg toolchain: ${CMAKE_TOOLCHAIN_FILE}" ) + +project(localTube) + +set(CMAKE_CXX_STANDARD 20) + + +find_package(fmt CONFIG REQUIRED) +find_package(soci CONFIG REQUIRED) +find_package(spdlog CONFIG REQUIRED) +find_package(nlohmann_json CONFIG REQUIRED) +find_package(Boost COMPONENTS system filesystem REQUIRED) + + +add_subdirectory(Modules) \ No newline at end of file diff --git a/Modules/CMakeLists.txt b/Modules/CMakeLists.txt new file mode 100644 index 0000000..083d9dd --- /dev/null +++ b/Modules/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(Server) \ No newline at end of file diff --git a/Modules/Server/CMakeLists.txt b/Modules/Server/CMakeLists.txt new file mode 100644 index 0000000..29b931e --- /dev/null +++ b/Modules/Server/CMakeLists.txt @@ -0,0 +1,23 @@ + +set(src_files + src/main.cpp + src/api.hpp + src/api.cpp + src/worker.hpp + src/worker.cpp + src/tables.hpp) + + +add_executable(. ${src_files}) + + +target_link_libraries(. + fmt::fmt + spdlog::spdlog + SOCI::soci_core + SOCI::soci_empty + SOCI::soci_sqlite3 + nlohmann_json::nlohmann_json + Boost::boost + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_SYSTEM_LIBRARY}) \ No newline at end of file diff --git a/Modules/Server/src/api.cpp b/Modules/Server/src/api.cpp new file mode 100644 index 0000000..25621a6 --- /dev/null +++ b/Modules/Server/src/api.cpp @@ -0,0 +1,150 @@ +// +// Created by s-Kaonnull on 25.05.2021. +// +#include "api.hpp" +#include "tables.hpp" + +#include +#include + +#include +#include +#include +#include + +using json = nlohmann::json; + +Api::Api(soci::session& sql, httplib::Server& server) : sql(sql) +{ + RegisterServerHandles(server); +} + +void Api::RegisterServerHandles(httplib::Server& server) +{ + spdlog::info("Register api endpoints..."); + + server.Post("/api/add", std::bind_front(&Api::add, this)); + + server.Get("/api/queue", std::bind_front(&Api::queue, this)); + + server.Get("/api/file/(\\d+)", std::bind_front(&Api::file, this)); + + server.Options("/(.*)", [this](httplib::Request const& rq, httplib::Response& rp) { + spdlog::info("Optin"); + }); + + server.set_pre_routing_handler([this](httplib::Request const& rq, httplib::Response& rp) { + this->set_cross_headers(rp); + return httplib::Server::HandlerResponse::Unhandled; + }); +} + +void Api::add(const httplib::Request& rq, httplib::Response& rs) +{ + try + { + auto data = json::parse(rq.body); + + File f; + + + f.url = data["url"]; + f.status = FileStatus::PENDING; + + sql << "INSERT INTO Files (url, status) values(:url, :status)", soci::use(f.url, "url"), + soci::use(f.status, "status"); + + long long int tmp; + sql.get_last_insert_id("Files", tmp); + f.id = tmp; + + json jr; + jr["status"] = "Ok"; + jr["id"] = f.id; + + spdlog::info("New Task: {}", f); + + rs.set_content(jr.dump(), "application/json"); + } + catch (json::exception const& e) + { + spdlog::error("Api Error: {}", e.what()); + rs.status = 400; + rs.set_content(std::format("Json Error: {}", e.what()), "text/plain"); + } +} + +void Api::queue(httplib::Request const& rq, httplib::Response& rs) +{ + try + { + json res; + + soci::rowset data = (sql.prepare << "SELECT * FROM Files"); + + for (auto const& el : data) + { + json current; + current["id"] = el.id; + current["status"] = magic_enum::enum_name(el.status); + current["status_id"] = static_cast(el.status); + current["url"] = el.url; + + res["queue"].push_back(current); + } + + rs.set_content(res.dump(), "application/json"); + } + catch (std::exception const& e) + { + spdlog::error(e.what()); + throw e; + } +} + +void Api::file(const httplib::Request& rq, httplib::Response& rs) +{ + if (!rq.matches[1].matched) + { + rs.status = 404; + rs.set_content("Parameter error", "text/plain"); + return; + } + + auto file = rq.matches[1]; + + spdlog::info("File Request: {}", file); + + try + { + Download dw; + sql << fmt::format("SELECT * FROM Downloads WHERE id={};", file), soci::into(dw); + + + std::ifstream fs(dw.path, std::ios_base::binary); + fs.seekg(0, std::ios_base::end); + auto size = fs.tellg(); + fs.seekg(0); + rs.body.resize(static_cast(size)); + fs.read(&rs.body[0], static_cast(size)); + + rs.set_header("Content-Disposition", fmt::format("attachment; filename={};", dw.path)); + if (auto [match, type] = ctre::match(dw.path); match) + { + spdlog::debug(type); + rs.set_header("content-type", fmt::format("application/{}", type.to_view())); + } + } + catch (std::exception const& e) + { + spdlog::error(e.what()); + rs.status = 404; + rs.set_content("File Not Found", "text/plain"); + } +} +void Api::set_cross_headers(httplib::Response& rs) +{ + rs.set_header("Access-Control-Allow-Origin", "*"); + rs.set_header("Access-Control-Allow-Methods", "GET, POST, PUT"); + rs.set_header("Access-Control-Allow-Headers", "Content-Type"); +} diff --git a/Modules/Server/src/api.hpp b/Modules/Server/src/api.hpp new file mode 100644 index 0000000..e25feaa --- /dev/null +++ b/Modules/Server/src/api.hpp @@ -0,0 +1,23 @@ +// +// Created by Kaonnull on 25.05.2021. +// +#include +#include + +class Api +{ + soci::session& sql; +public: + Api(soci::session& sql, httplib::Server& server); + + void RegisterServerHandles(httplib::Server& server); + +public: // Endpoints + void add(httplib::Request const& rq, httplib::Response& rs); + + void queue(httplib::Request const& rq, httplib::Response& rs); + void file(httplib::Request const& rq, httplib::Response& rs); + + void set_cross_headers(httplib::Response& rs); + +}; \ No newline at end of file diff --git a/Modules/Server/src/main.cpp b/Modules/Server/src/main.cpp new file mode 100644 index 0000000..5098ad3 --- /dev/null +++ b/Modules/Server/src/main.cpp @@ -0,0 +1,77 @@ +// +// Created by Kaonnull on 25.05.2021. +// +#include + +#include +#include +#include +#include +#include + +#include "api.hpp" +#include "tables.hpp" +#include "worker.hpp" + +int main() +{ + spdlog::set_level(spdlog::level::debug); + + constexpr std::string_view sqlite_db_name = "db.sqlite"; + constexpr std::string_view version = "Version 1"; + + soci::register_factory_sqlite3(); + + + spdlog::info("Opening sqlite3 db: {}", sqlite_db_name); + soci::session sql("sqlite3", fmt::format("db={}", sqlite_db_name)); + + if (!sql.is_connected()) + { + spdlog::error("sqlite connection failed!"); + return -1; + } + + try + { + Version v; + sql << "SELECT * FROM Version", soci::into(v); + + if (v.version == version) + { + spdlog::info("DB Loaded"); + } else + { + spdlog::error("DB Version miss match. Try deleting the sqlite file."); + return -1; + } + } + catch (soci::soci_error const& err) + { + spdlog::info("DB not initialised. Creating db..."); + + spdlog::info("Creating Version table"); + { + auto version_table = sql.create_table("Version"); + version_table.column("ID", soci::dt_integer); + version_table.column("version_info", soci::dt_string)("NOT NULL"); + version_table.primary_key("Version", "ID"); + } + + Version v { 1, std::string { version } }; + sql << "insert into Version(ID, version_info) values(:ID, :version_info)", soci::use(v); + + File::create_table(sql); + Download::create_table(sql); + } + + Worker worker(sql); + + httplib::Server srv; + srv.set_mount_point("/", "downloads/"); + + Api api(sql, srv); + + spdlog::info("Listing on 0.0.0.0 Port 80"); + srv.listen("0.0.0.0", 80); +} diff --git a/Modules/Server/src/tables.hpp b/Modules/Server/src/tables.hpp new file mode 100644 index 0000000..dbf2c55 --- /dev/null +++ b/Modules/Server/src/tables.hpp @@ -0,0 +1,160 @@ +// +// Created by s-har on 25.05.2021. +// +#pragma once +#include +#include +#include + + +struct Version +{ + int key; + std::string version; +}; + +namespace soci +{ + template <> + struct type_conversion + { + typedef values base_type; + + static void from_base(values const& i, indicator ind, Version& v) + { + v.key = i.get("ID"); + v.version = i.get("version_info"); + } + + static void to_base(const Version& v, values& i, indicator& ind) + { + i.set("ID", v.key); + i.set("version_info", v.version); + ind = i_ok; + } + }; +} // namespace soci + +enum class FileStatus { PENDING, DOWNLOADING, COMPLETE, DOWNLOAD_ERROR, NONE }; + +namespace soci +{ + template <> + struct type_conversion + { + typedef int base_type; + + static void from_base(int i, indicator ind, FileStatus& v) + { + if (ind == i_null) + { + v = FileStatus::NONE; + } else if (i < 0 || i >= static_cast(FileStatus::NONE)) + { + v = FileStatus::NONE; + } else + { + v = static_cast(i); + } + } + + static void to_base(const FileStatus& v, int& i, indicator& ind) + { + i = static_cast(v); + } + }; +} // namespace soci + + +struct File +{ + int id; + std::string url; + FileStatus status; + + constexpr static std::string_view table_name = "Files"; + + inline static void create_table(soci::session& sql) + { + spdlog::info("Creating table {}", table_name); + auto table = sql.create_table(std::string { table_name }); + + table.column("id", soci::dt_integer)("PRIMARY KEY AUTOINCREMENT"); + table.column("url", soci::dt_string)("NOT NULL"); + table.column("status", soci::dt_integer)("NOT NULL"); + // table.primary_key(std::string { table_name }, "id"); + } +}; + +inline std::ostream& operator<<(std::ostream& os, File const& file) +{ + return os << fmt::format("File [{}][{}] url: {} ", + file.id, + magic_enum::enum_name(file.status), + file.url); +} + +namespace soci +{ + template <> + struct type_conversion + { + typedef values base_type; + + static void from_base(values const& i, indicator ind, File& v) + { + v.id = i.get("id"); + v.url = i.get("url"); + v.status = i.get("status"); + } + + static void to_base(const File& v, values& i, indicator& ind) + { + i.set("id", v.id); + i.set("url", v.url); + i.set("status", v.status); + ind = i_ok; + } + }; +} // namespace soci + + +struct Download +{ + int id; + std::string path; + + constexpr static std::string_view table_name = "Downloads"; + + inline static void create_table(soci::session& sql) + { + spdlog::info("Creating table {}", table_name); + auto table = sql.create_table(std::string { table_name }); + + table.column("id", soci::dt_integer)("PRIMARY KEY"); + table.column("path", soci::dt_string)("NOT NULL"); + table.foreign_key("FK_Downloads", "id", std::string { File::table_name }, "id"); + } +}; + +namespace soci +{ + template <> + struct type_conversion + { + typedef values base_type; + + static void from_base(values const& i, indicator ind, Download& v) + { + v.id = i.get("id"); + v.path = i.get("path"); + } + + static void to_base(const Download& v, values& i, indicator& ind) + { + i.set("id", v.id); + i.set("path", v.path); + ind = i_ok; + } + }; +} // namespace soci \ No newline at end of file diff --git a/Modules/Server/src/worker.cpp b/Modules/Server/src/worker.cpp new file mode 100644 index 0000000..4270e54 --- /dev/null +++ b/Modules/Server/src/worker.cpp @@ -0,0 +1,145 @@ +// +// Created by s-har on 26.05.2021. +// +#include "worker.hpp" + +#include +#include +#include + +#include +#include +#include +#include + +using namespace std::chrono_literals; + +void operator+=(std::vector& vec, std::string const& s) +{ + vec.push_back(s); +} + +namespace bp = boost::process; + +Worker::Worker(soci::session& sql) : thread(std::bind_front(&Worker::loop, this)), sql(sql) +{ + spdlog::info("Worker started."); +} + +[[noreturn]] void Worker::loop() +{ + while (true) + { + File task; + if (!getTask(task)) + { + std::this_thread::sleep_for(10s); + continue; + } + + spdlog::info("Worker Task: {}", task); + + spdlog::debug("Begin Download {}", task.id); + task.status = FileStatus::DOWNLOADING; + + sql << "UPDATE Files SET status = :status WHERE id = :id", soci::use(task); + + if (download(task)) + { + spdlog::debug("Download end {}", task.id); + task.status = FileStatus::COMPLETE; + } else + { + // Error + task.status = FileStatus::DOWNLOAD_ERROR; + spdlog::error("Download Error {}: ", task); + + } + + sql << "UPDATE Files SET status = :status WHERE id = :id", soci::use(task); + std::this_thread::sleep_for(10s); + } +} + +bool Worker::getTask(File& task) +{ + try + { + soci::rowset data = + (sql.prepare << "SELECT * FROM Files WHERE status = 0 ORDER BY id LIMIT 1;"); + + for (auto const& el : data) + { + task = el; + return true; + } + } + catch (std::exception const& e) + { + spdlog::error(e.what()); + fmt::format("{}\n", e.what()); + } + return false; +} + +bool Worker::download(File& task) +{ + bp::ipstream out; + bp::ipstream err; + + std::vector args; + + args += "-o"; + args += fmt::format("{}/%(title)s.%(ext)s", output_path); + + args += "--extract-audio"; + + args += task.url; + + spdlog::debug("youtube-dl {}", fmt::join(args, " ")); + + bp::child c("youtube-dl.exe", bp::args(args), bp::std_out > out, bp::std_err > err); + + std::string destination_file; + + std::string line; + while (c.running() || !out.eof() || !err.eof()) + { + if (std::getline(out, line) && !line.empty()) + { + spdlog::info("[youtube-dl] {}", line); + + if (auto [match, info, destination] = + ctre::match(line); + match) + { + destination_file = destination; + } + + } else if (std::getline(err, line) && !line.empty()) + { + if (auto [match, type, messages] = ctre::match(line); match) + { + if (type == "WARNING") + spdlog::warn("[youtube-dl] {}", messages); + } else + { + spdlog::error("[youtube-dl] {}", line); + } + } + } + + c.wait(); + auto exit_code = c.exit_code(); + + spdlog::debug("File: {}", destination_file); + + Download dwn; + dwn.id = task.id; + dwn.path = destination_file; + + sql << "INSERT INTO Downloads (id, path) values(:id, :path)", soci::use(dwn); + + spdlog::debug("youtube-dl exit code {}", exit_code); + return exit_code == 0; +} diff --git a/Modules/Server/src/worker.hpp b/Modules/Server/src/worker.hpp new file mode 100644 index 0000000..d3da784 --- /dev/null +++ b/Modules/Server/src/worker.hpp @@ -0,0 +1,28 @@ +// +// Created by s-har on 25.05.2021. +// +#pragma once +#include +#include +#include + +#include + +#include "tables.hpp" + +class Worker +{ + std::thread thread; + soci::session& sql; + + std::string output_path = "./downloads"; +public: + explicit Worker(soci::session& sql); + + [[noreturn]] void loop(); + +private: + bool getTask(File& task); + + bool download(File& task); +}; \ No newline at end of file diff --git a/Modules/Website/elm.json b/Modules/Website/elm.json new file mode 100644 index 0000000..e0f9707 --- /dev/null +++ b/Modules/Website/elm.json @@ -0,0 +1,29 @@ +{ + "type": "application", + "source-directories": [ + "src" + ], + "elm-version": "0.19.1", + "dependencies": { + "direct": { + "NoRedInk/elm-json-decode-pipeline": "1.0.0", + "elm/browser": "1.0.2", + "elm/core": "1.0.5", + "elm/html": "1.0.0", + "elm/http": "2.0.0", + "elm/json": "1.1.3", + "mdgriffith/elm-ui": "1.1.8" + }, + "indirect": { + "elm/bytes": "1.0.8", + "elm/file": "1.0.5", + "elm/time": "1.0.0", + "elm/url": "1.0.0", + "elm/virtual-dom": "1.0.2" + } + }, + "test-dependencies": { + "direct": {}, + "indirect": {} + } +} diff --git a/Modules/Website/src/Color.elm b/Modules/Website/src/Color.elm new file mode 100644 index 0000000..ba0bfef --- /dev/null +++ b/Modules/Website/src/Color.elm @@ -0,0 +1,19 @@ +module Color exposing (..) + +white = + Element.rgb 1 1 1 + +grey = + Element.rgb 0.9 0.9 0.9 + + +blue = + Element.rgb 0 0 0.8 + + +red = + Element.rgb 0.8 0 0 + + +darkBlue = + Element.rgb 0 0 0.9 diff --git a/Modules/Website/src/Main.elm b/Modules/Website/src/Main.elm new file mode 100644 index 0000000..a5b4684 --- /dev/null +++ b/Modules/Website/src/Main.elm @@ -0,0 +1,157 @@ +module Main exposing (main) + +import Browser +import Html exposing (Html) + +import Http +import Json.Decode exposing (Decoder, bool, int, list, string, succeed) +import Json.Decode.Pipeline exposing(optional, required) +import Json.Encode as Encode + +import Element exposing (..) +import Element.Background as Background +import Element.Border as Border +import Element.Font as Font +import Element.Input as Input +import Element.Region as Region +import Html exposing (header) + +white = + Element.rgb 1 1 1 + +grey = + Element.rgb 0.9 0.9 0.9 + + +blue = + Element.rgb 0 0 0.8 + + +red = + Element.rgb 0.8 0 0 + + +darkBlue = + Element.rgb 0 0 0.9 + +type alias Form = + { url: String + } + +type Model + = AddPage Form + | Quere + +type Msg + = UpdateForm Form + | AddUrl Form + | UrlAdded (Result Http.Error UrlAddedResponse) + +initModel : Model +initModel = + AddPage + { url = "" + } + +type alias UrlAddedResponse = { id: Int, status: String } +urlAddDecoder : Decoder UrlAddedResponse +urlAddDecoder = + succeed UrlAddedResponse + |> required "id" int + |> required "status" string + +urlAddEncode : Form -> Encode.Value +urlAddEncode form = Encode.object + [ ("url", Encode.string form.url) + ] + +-- Http.header "Access-Control-Request-Method" "POST" +-- , +update : Msg -> Model -> (Model, Cmd Msg) +update msg model = + case (msg, model) of + (UpdateForm new_form, AddPage _) -> + (AddPage new_form, Cmd.none) + + (AddUrl form, AddPage _) -> + ( model + , Http.request + { method = "POST" + , headers = + [ + ] + , url = "http://127.0.0.1/api/add" + , body = Http.jsonBody (urlAddEncode form) + , expect = Http.expectJson UrlAdded (urlAddDecoder) + , timeout = Nothing + , tracker = Nothing + } + ) + + ( _, _ ) -> + Debug.todo "branch '( Decrement, _ )' not implemented" + +view : Model -> Html Msg +view model = + Element.layout + [ + Font.size 20 + ] + <| case (model) of + (AddPage form) -> + Element.column + [ width (px 800) + , height shrink + , centerX + , centerY + , spacing 36 + , padding 10 + ] + [ el + [ Region.heading 1 + , alignLeft + , Font.size 36 + ] + (text "localTube") + , Input.text + [ spacing 12 + , below + (el + [ Font.color red + , Font.size 14 + , alignRight + , moveDown 6] + (text "This one is wrong") + ) + ] + { text = form.url + , placeholder = Just (Input.placeholder [] (text "http://youtube.com")) + , onChange = \new -> UpdateForm { form | url = new } + , label = Input.labelAbove [ Font.size 14 ](text "Video Url") + } + , Input.button + [ Background.color blue + , Font.color white + , Border.color darkBlue + , paddingXY 32 16 + , Border.rounded 3 + ] + { onPress = Just (AddUrl form) + , label = Element.text "Add Link to Query" + } + ] + + (_) -> + el + [ Region.heading 1 + , Font.size 36](text "Non") + + +main: Program () Model Msg +main = + Browser.element + { init = \flags -> (initModel, Cmd.none) + , view = view + , update = update + , subscriptions = \_ -> Sub.none + } \ No newline at end of file diff --git a/vcpkg.json b/vcpkg.json new file mode 100644 index 0000000..48dfdc7 --- /dev/null +++ b/vcpkg.json @@ -0,0 +1,17 @@ +{ + "name": "template", + "version-string": "0.0.1", + "dependencies": [ + "fmt", + "cpp-httplib", + "spdlog", + "nlohmann-json", + "magic-enum", + "boost-process", + "ctre", + { + "name": "soci", + "features": ["sqlite3"] + } + ] +} \ No newline at end of file