From e0262a7f1777496730da51b79a1ed17ea70f07e6 Mon Sep 17 00:00:00 2001 From: kjuulh Date: Sun, 17 Jul 2022 21:51:46 +0200 Subject: [PATCH 1/3] remove ticker --- .env | 7 + Cargo.lock | 584 +++++++++++++++++- src/cmd/scel/Cargo.toml | 1 + src/cmd/scel/src/main.rs | 14 +- src/lib/scel_api/Cargo.toml | 14 +- src/lib/scel_api/src/auth/mod.rs | 99 +++ src/lib/scel_api/src/graphql/mod.rs | 4 + .../scel_api/src/{ => graphql}/mutation.rs | 0 src/lib/scel_api/src/{ => graphql}/query.rs | 0 src/lib/scel_api/src/{ => graphql}/schema.rs | 2 +- .../src/{ => graphql}/subscription.rs | 0 src/lib/scel_api/src/lib.rs | 109 +++- src/lib/scel_core/src/lib.rs | 12 +- src/lib/scel_core/src/repo/users_repo.rs | 3 + 14 files changed, 807 insertions(+), 42 deletions(-) create mode 100644 .env create mode 100644 src/lib/scel_api/src/auth/mod.rs create mode 100644 src/lib/scel_api/src/graphql/mod.rs rename src/lib/scel_api/src/{ => graphql}/mutation.rs (100%) rename src/lib/scel_api/src/{ => graphql}/query.rs (100%) rename src/lib/scel_api/src/{ => graphql}/schema.rs (66%) rename src/lib/scel_api/src/{ => graphql}/subscription.rs (100%) create mode 100644 src/lib/scel_core/src/repo/users_repo.rs diff --git a/.env b/.env new file mode 100644 index 0000000..0be8e16 --- /dev/null +++ b/.env @@ -0,0 +1,7 @@ +GITEA_CLIENT_ID="cea02728-3e9a-4200-9ee2-41785a8bb175" +GITEA_CLIENT_SECRET="gto_radao6mkyg2nlat4wdoovnor32mcdqpezm3okycgj5s7ou4bjqba" +GITEA_REDIRECT_URL="http://127.0.0.1:3000/auth/authorized" +GITEA_AUTH_URL="https://git.front.kjuulh.io/login/oauth/authorize" +GITEA_TOKEN_URL="https://git.front.kjuulh.io/login/oauth/access_token" +GITEA_USER_INFO_URL="https://git.front.kjuulh.io/login/oauth/userinfo" + diff --git a/Cargo.lock b/Cargo.lock index 32c3afa..b58e11a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -36,6 +36,18 @@ version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb07d2053ccdbe10e2af2995a2f116c1330396493dc1269f6a91d0ae82e19704" +[[package]] +name = "arrayref" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" + +[[package]] +name = "arrayvec" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" + [[package]] name = "ascii_utils" version = "0.9.3" @@ -59,10 +71,12 @@ dependencies = [ "futures-util", "http", "indexmap", + "log", "mime", "multer", "num-traits", "once_cell", + "opentelemetry", "pin-project-lite", "regex", "serde", @@ -70,6 +84,8 @@ dependencies = [ "static_assertions", "tempfile", "thiserror", + "tracing", + "tracing-futures", ] [[package]] @@ -130,6 +146,36 @@ dependencies = [ "serde_json", ] +[[package]] +name = "async-lock" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e97a171d191782fba31bb902b14ad94e24a68145032b7eedf871ab0bc0d077b6" +dependencies = [ + "event-listener", +] + +[[package]] +name = "async-session" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07da4ce523b4e2ebaaf330746761df23a465b951a83d84bbce4233dabedae630" +dependencies = [ + "anyhow", + "async-lock", + "async-trait", + "base64", + "bincode", + "blake3", + "chrono", + "hmac", + "log", + "rand", + "serde", + "serde_json", + "sha2 0.9.9", +] + [[package]] name = "async-stream" version = "0.3.3" @@ -223,12 +269,45 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bitflags" version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "blake3" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b64485778c4f16a6a5a9d335e80d449ac6c70cdd6a06d2af18a6f6f775a125b3" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if 0.1.10", + "constant_time_eq", + "crypto-mac 0.8.0", + "digest 0.9.0", +] + +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array", +] + [[package]] name = "block-buffer" version = "0.10.2" @@ -238,6 +317,12 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bumpalo" +version = "3.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37ccbd214614c6783386c1af30caf03192f17891059cecc394b4fb119e363de3" + [[package]] name = "byteorder" version = "1.4.3" @@ -253,12 +338,43 @@ dependencies = [ "serde", ] +[[package]] +name = "cc" +version = "1.0.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + [[package]] name = "cfg-if" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +dependencies = [ + "libc", + "num-integer", + "num-traits", + "serde", + "winapi", +] + +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + [[package]] name = "cpufeatures" version = "0.2.2" @@ -268,6 +384,26 @@ dependencies = [ "libc", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c02a4d71819009c192cf4872265391563fd6a84c81ff2c0f2a7026ca4c1d85c" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d82ee10ce34d7bc12c2122495e7593a9c41347ecdd64185af4ecf72cb1a7f83" +dependencies = [ + "cfg-if 1.0.0", + "once_cell", +] + [[package]] name = "crypto-common" version = "0.1.5" @@ -278,6 +414,26 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-mac" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab" +dependencies = [ + "generic-array", + "subtle", +] + +[[package]] +name = "crypto-mac" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714" +dependencies = [ + "generic-array", + "subtle", +] + [[package]] name = "darling" version = "0.14.1" @@ -313,25 +469,46 @@ dependencies = [ "syn", ] +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array", +] + [[package]] name = "digest" version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506" dependencies = [ - "block-buffer", + "block-buffer 0.10.2", "crypto-common", ] +[[package]] +name = "dotenv" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" + [[package]] name = "encoding_rs" version = "0.8.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] +[[package]] +name = "event-listener" +version = "2.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77f3309417938f28bf8228fcff79a4a37103981e3e186d2ccd19c74b38f4eb71" + [[package]] name = "fast_chemail" version = "0.9.6" @@ -471,9 +648,30 @@ version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", + "js-sys", "libc", "wasi", + "wasm-bindgen", +] + +[[package]] +name = "h2" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37a82c6d637fc9515a4694bbf1cb2457b79d81ce52b3108bdeea58b07dd34a57" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", ] [[package]] @@ -516,6 +714,16 @@ dependencies = [ "libc", ] +[[package]] +name = "hmac" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b" +dependencies = [ + "crypto-mac 0.11.1", + "digest 0.9.0", +] + [[package]] name = "http" version = "0.2.8" @@ -566,6 +774,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", + "h2", "http", "http-body", "httparse", @@ -579,6 +788,19 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d87c48c02e0dc5e3b849a2041db3029fd066650f8f717c07bf8ed78ccb895cac" +dependencies = [ + "http", + "hyper", + "rustls", + "tokio", + "tokio-rustls", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -613,15 +835,30 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] +[[package]] +name = "ipnet" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b" + [[package]] name = "itoa" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d" +[[package]] +name = "js-sys" +version = "0.3.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3fac17f7123a73ca62df411b1bf727ccc805daa070338fda671c86dac1bdc27" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -650,7 +887,7 @@ version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -712,10 +949,20 @@ dependencies = [ "log", "memchr", "mime", - "spin", + "spin 0.9.4", "version_check", ] +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.15" @@ -735,12 +982,57 @@ dependencies = [ "libc", ] +[[package]] +name = "oauth2" +version = "4.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d62c436394991641b970a92e23e8eeb4eb9bca74af4f5badc53bcd568daadbd" +dependencies = [ + "base64", + "chrono", + "getrandom", + "http", + "rand", + "reqwest", + "serde", + "serde_json", + "serde_path_to_error", + "sha2 0.10.2", + "thiserror", + "url", +] + [[package]] name = "once_cell" version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18a6dbe30758c9f83eb00cbea4ac95966305f5a7772f3f42ebfc7fc7eddbd8e1" +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + +[[package]] +name = "opentelemetry" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6105e89802af13fdf48c49d7646d3b533a70e536d818aae7e78ba0433d01acb8" +dependencies = [ + "async-trait", + "crossbeam-channel", + "futures-channel", + "futures-executor", + "futures-util", + "js-sys", + "lazy_static", + "percent-encoding", + "pin-project", + "rand", + "thiserror", +] + [[package]] name = "parking_lot" version = "0.12.1" @@ -757,7 +1049,7 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "redox_syscall", "smallvec", @@ -919,6 +1211,81 @@ dependencies = [ "winapi", ] +[[package]] +name = "reqwest" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b75aa69a3f06bbcc66ede33af2af253c6f7a86b1ca0033f60c580a27074fbf92" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-rustls", + "ipnet", + "js-sys", + "lazy_static", + "log", + "mime", + "percent-encoding", + "pin-project-lite", + "rustls", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-rustls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", + "winreg", +] + +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin 0.5.2", + "untrusted", + "web-sys", + "winapi", +] + +[[package]] +name = "rustls" +version = "0.20.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aab8ee6c7097ed6057f43c187a62418d0c05a4bd5f18b3571db50ee0f9ce033" +dependencies = [ + "log", + "ring", + "sct", + "webpki", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7522c9de787ff061458fe9a829dc790a3f5b22dc571694fc5883f448b94d9a9" +dependencies = [ + "base64", +] + [[package]] name = "ryu" version = "1.0.10" @@ -930,6 +1297,7 @@ name = "scel" version = "0.1.0" dependencies = [ "anyhow", + "dotenv", "scel_api", "scel_core", "tokio", @@ -944,8 +1312,11 @@ dependencies = [ "anyhow", "async-graphql", "async-graphql-axum", + "async-session", "axum", "futures", + "oauth2", + "reqwest", "serde", "serde_json", "tokio", @@ -964,6 +1335,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "sct" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "serde" version = "1.0.139" @@ -995,6 +1376,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7868ad3b8196a8a0aea99a8220b124278ee5320a55e4fde97794b6f85b1a377" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -1013,9 +1403,33 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "cpufeatures", - "digest", + "digest 0.10.3", +] + +[[package]] +name = "sha2" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" +dependencies = [ + "block-buffer 0.9.0", + "cfg-if 1.0.0", + "cpufeatures", + "digest 0.9.0", + "opaque-debug", +] + +[[package]] +name = "sha2" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55deaec60f81eefe3cce0dc50bda92d6d8e88f2a27df7c5033b42afeb1ed2676" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures", + "digest 0.10.3", ] [[package]] @@ -1058,6 +1472,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + [[package]] name = "spin" version = "0.9.4" @@ -1076,6 +1496,12 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "subtle" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" + [[package]] name = "syn" version = "1.0.98" @@ -1099,7 +1525,7 @@ version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "fastrand", "libc", "redox_syscall", @@ -1183,6 +1609,17 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-rustls" +version = "0.23.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" +dependencies = [ + "rustls", + "tokio", + "webpki", +] + [[package]] name = "tokio-tungstenite" version = "0.17.2" @@ -1207,6 +1644,7 @@ dependencies = [ "futures-sink", "pin-project-lite", "tokio", + "tracing", ] [[package]] @@ -1251,6 +1689,7 @@ dependencies = [ "tower", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -1271,7 +1710,7 @@ version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a400e31aa60b9d44a52a8ee0343b5b18566b03a8321e0d321f695cf56e940160" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "log", "pin-project-lite", "tracing-attributes", @@ -1299,6 +1738,18 @@ dependencies = [ "valuable", ] +[[package]] +name = "tracing-futures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" +dependencies = [ + "futures", + "futures-task", + "pin-project", + "tracing", +] + [[package]] name = "tracing-log" version = "0.1.3" @@ -1386,6 +1837,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "url" version = "2.2.2" @@ -1396,6 +1853,7 @@ dependencies = [ "idna", "matches", "percent-encoding", + "serde", ] [[package]] @@ -1432,6 +1890,101 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-bindgen" +version = "0.2.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c53b543413a17a202f4be280a7e5c62a1c69345f5de525ee64f8cfdbc954994" +dependencies = [ + "cfg-if 1.0.0", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5491a68ab4500fa6b4d726bd67408630c3dbe9c4fe7bda16d5c82a1fd8c7340a" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de9a9cec1733468a8c657e57fa2413d2ae2c0129b95e87c5b72b8ace4d13f31f" +dependencies = [ + "cfg-if 1.0.0", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c441e177922bc58f1e12c022624b6216378e5febc2f0533e41ba443d505b80aa" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d94ac45fcf608c1f45ef53e748d35660f168490c10b23704c7779ab8f5c3048" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a89911bd99e5f3659ec4acf9c4d93b0a90fe4a2a11f15328472058edc5261be" + +[[package]] +name = "web-sys" +version = "0.3.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fed94beee57daf8dd7d51f2b15dc2bcde92d7a72304cdf662a4371008b71b90" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "webpki-roots" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1c760f0d366a6c24a02ed7816e23e691f5d92291f94d15e836006fd11b04daf" +dependencies = [ + "webpki", +] + [[package]] name = "winapi" version = "0.3.9" @@ -1496,3 +2049,12 @@ name = "windows_x86_64_msvc" version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" + +[[package]] +name = "winreg" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +dependencies = [ + "winapi", +] diff --git a/src/cmd/scel/Cargo.toml b/src/cmd/scel/Cargo.toml index dc26400..67215d0 100644 --- a/src/cmd/scel/Cargo.toml +++ b/src/cmd/scel/Cargo.toml @@ -10,6 +10,7 @@ tokio = { version = "1.0", features = ["full"] } tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } anyhow = { version = "1.0.58" } +dotenv = { version = "*" } scel_api = { path = "../../lib/scel_api" } scel_core = { path = "../../lib/scel_core" } diff --git a/src/cmd/scel/src/main.rs b/src/cmd/scel/src/main.rs index b3feb70..e0f07e5 100644 --- a/src/cmd/scel/src/main.rs +++ b/src/cmd/scel/src/main.rs @@ -1,10 +1,18 @@ -use tracing::{info, Level}; -use tracing_subscriber::FmtSubscriber; +use dotenv::dotenv; +use tracing::info; +use tracing_subscriber::{EnvFilter, FmtSubscriber}; #[tokio::main] async fn main() -> anyhow::Result<()> { + dotenv().ok(); + let subscriber = FmtSubscriber::builder() - .with_max_level(Level::INFO) + .with_env_filter( + EnvFilter::default() + .add_directive("tower_http=debug".parse().unwrap()) + .add_directive("scel_api=info".parse().unwrap()) + .add_directive("scel=info".parse().unwrap()), + ) .finish(); tracing::subscriber::set_global_default(subscriber)?; diff --git a/src/lib/scel_api/Cargo.toml b/src/lib/scel_api/Cargo.toml index 2b0a20d..814aed6 100644 --- a/src/lib/scel_api/Cargo.toml +++ b/src/lib/scel_api/Cargo.toml @@ -8,8 +8,12 @@ edition = "2021" [dependencies] axum = { version = "0.5.6" } futures = "0.3.21" -tower-http = {version = "0.3.3", features = ["cors"]} -async-graphql = { version = "4.0.0" } +tower-http = { version = "0.3.3", features = ["cors", "trace"] } +async-graphql = { version = "4.0.0", features = [ + 'tracing', + 'opentelemetry', + "log", +] } async-graphql-axum = { version = "4.0.0" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.68" @@ -17,3 +21,9 @@ tokio = { version = "1.0", features = ["full"] } tracing = "0.1" tracing-subscriber = { version = "0.3" } anyhow = { version = "1.0.58" } +oauth2 = { version = "*" } +async-session = { version = "*" } +reqwest = { version = "*", default-features = false, features = [ + "rustls-tls", + "json", +] } diff --git a/src/lib/scel_api/src/auth/mod.rs b/src/lib/scel_api/src/auth/mod.rs new file mode 100644 index 0000000..dfdf231 --- /dev/null +++ b/src/lib/scel_api/src/auth/mod.rs @@ -0,0 +1,99 @@ +use std::env; + +use async_session::{MemoryStore, Session, SessionStore}; +use axum::{ + extract::Query, + http::HeaderMap, + response::{IntoResponse, Redirect}, + Extension, +}; +use oauth2::{ + basic::BasicClient, reqwest::async_http_client, AuthUrl, AuthorizationCode, ClientId, + ClientSecret, CsrfToken, RedirectUrl, Scope, TokenResponse, TokenUrl, +}; +use reqwest::header::SET_COOKIE; +use serde::Deserialize; + +use crate::{User, COOKIE_NAME}; + +pub fn oauth_client() -> BasicClient { + let client_id = env::var("GITEA_CLIENT_ID").expect("Missing GITEA_CLIENT_ID"); + let client_secret = env::var("GITEA_CLIENT_SECRET").expect("Missing GITEA_CLIENT_SECRET"); + let redirect_url = env::var("GITEA_REDIRECT_URL") + .unwrap_or_else(|_| "http://127.0.0.1:3000/auth/authorized".to_string()); + + let auth_url = + env::var("GITEA_AUTH_URL").unwrap_or_else(|_| "https://git.front.kjuulh.io".to_string()); + + let token_url = + env::var("GITEA_TOKEN_URL").unwrap_or_else(|_| "https://git.front.kjuulh.io".to_string()); + + BasicClient::new( + ClientId::new(client_id), + Some(ClientSecret::new(client_secret)), + AuthUrl::new(auth_url).expect("AuthUrl was invalid"), + Some(TokenUrl::new(token_url).expect("Token url was invalid")), + ) + .set_redirect_uri(RedirectUrl::new(redirect_url).expect("RedirectUrl was invalid")) +} + +pub async fn gitea(Extension(client): Extension) -> impl IntoResponse { + let (auth_url, _crsf_token) = client.authorize_url(CsrfToken::new_random).url(); + + Redirect::to(&auth_url.to_string()) +} + +#[derive(Debug, Deserialize)] +pub struct AuthRequest { + code: String, + state: String, +} + +pub async fn authorized( + Query(query): Query, + Extension(store): Extension, + Extension(oauth_client): Extension, +) -> impl IntoResponse { + let token = oauth_client + .exchange_code(AuthorizationCode::new(query.code.clone())) + .request_async(async_http_client) + .await + .expect("failed to get http client"); + + let client = reqwest::Client::new(); + let user_data_json = client + .get(get_gitea_user_data_url()) + .bearer_auth(token.access_token().secret()) + .send() + .await + .expect("Request did not succeed"); + // .text() + // .await + // .unwrap(); + + let user_data: User = user_data_json + .json::() + .await + .expect("could not parse user"); + + let mut session = Session::new(); + session + .insert("user", &user_data) + .expect("could not insert user data"); + + let cookie = store + .store_session(session) + .await + .expect("could not insert session") + .expect("session was not valid"); + + let cookie = format!("{}={}; SameSite=Lax; Path=/", COOKIE_NAME, cookie); + + let mut headers = HeaderMap::new(); + headers.insert(SET_COOKIE, cookie.parse().expect("Cookie is not valid")); + (headers, Redirect::to("/")) +} + +fn get_gitea_user_data_url() -> String { + env::var("GITEA_USER_INFO_URL").expect("Missing GITEA_USER_INFO_URL") +} diff --git a/src/lib/scel_api/src/graphql/mod.rs b/src/lib/scel_api/src/graphql/mod.rs new file mode 100644 index 0000000..675e36a --- /dev/null +++ b/src/lib/scel_api/src/graphql/mod.rs @@ -0,0 +1,4 @@ +pub mod mutation; +pub mod query; +pub mod schema; +pub mod subscription; diff --git a/src/lib/scel_api/src/mutation.rs b/src/lib/scel_api/src/graphql/mutation.rs similarity index 100% rename from src/lib/scel_api/src/mutation.rs rename to src/lib/scel_api/src/graphql/mutation.rs diff --git a/src/lib/scel_api/src/query.rs b/src/lib/scel_api/src/graphql/query.rs similarity index 100% rename from src/lib/scel_api/src/query.rs rename to src/lib/scel_api/src/graphql/query.rs diff --git a/src/lib/scel_api/src/schema.rs b/src/lib/scel_api/src/graphql/schema.rs similarity index 66% rename from src/lib/scel_api/src/schema.rs rename to src/lib/scel_api/src/graphql/schema.rs index 24fd486..149a980 100644 --- a/src/lib/scel_api/src/schema.rs +++ b/src/lib/scel_api/src/graphql/schema.rs @@ -1,5 +1,5 @@ use async_graphql::Schema; -use crate::{mutation::MutationRoot, query::QueryRoot, subscription::SubscriptionRoot}; +use super::{mutation::MutationRoot, query::QueryRoot, subscription::SubscriptionRoot}; pub type ScelSchema = Schema; diff --git a/src/lib/scel_api/src/subscription.rs b/src/lib/scel_api/src/graphql/subscription.rs similarity index 100% rename from src/lib/scel_api/src/subscription.rs rename to src/lib/scel_api/src/graphql/subscription.rs diff --git a/src/lib/scel_api/src/lib.rs b/src/lib/scel_api/src/lib.rs index c388863..62ebc85 100644 --- a/src/lib/scel_api/src/lib.rs +++ b/src/lib/scel_api/src/lib.rs @@ -1,32 +1,43 @@ -mod mutation; -mod query; -mod schema; -mod subscription; +mod auth; +mod graphql; use std::net::SocketAddr; use async_graphql::{ + extensions::{Logger, OpenTelemetry, Tracing}, http::{playground_source, GraphQLPlaygroundConfig}, Request, Response, Schema, }; use async_graphql_axum::GraphQLSubscription; +use async_session::{async_trait, MemoryStore, SessionStore}; +use auth::{authorized, gitea}; use axum::{ - http::Method, - response::{Html, IntoResponse}, - routing, Extension, Json, Router, + extract::{rejection::TypedHeaderRejectionReason, FromRequest, RequestParts}, + headers, + http::{header, Method}, + response::{Html, IntoResponse, Redirect}, + routing, Extension, Json, Router, TypedHeader, }; -use mutation::MutationRoot; -use query::QueryRoot; -use schema::ScelSchema; -use subscription::SubscriptionRoot; -use tower_http::cors::CorsLayer; +use graphql::{ + mutation::MutationRoot, query::QueryRoot, schema::ScelSchema, subscription::SubscriptionRoot, +}; +use serde::{Deserialize, Serialize}; +use tower_http::{ + cors::CorsLayer, + trace::{DefaultMakeSpan, TraceLayer}, +}; +use tracing::Subscriber; async fn graphql_playground() -> impl IntoResponse { Html(playground_source( GraphQLPlaygroundConfig::new("/").subscription_endpoint("/ws"), )) } -async fn graphql_handler(schema: Extension, req: Json) -> Json { +async fn graphql_handler( + schema: Extension, + req: Json, + _: User, +) -> Json { schema.execute(req.0).await.into() } @@ -37,7 +48,10 @@ pub struct Server { impl Server { pub fn new() -> Server { - let schema = Schema::build(QueryRoot, MutationRoot, SubscriptionRoot).finish(); + let schema = Schema::build(QueryRoot, MutationRoot, SubscriptionRoot) + .extension(Tracing) + .extension(Logger) + .finish(); let cors = vec!["http://localhost:3000" .parse() @@ -46,13 +60,22 @@ impl Server { let app = Router::new() .route("/", routing::get(graphql_playground).post(graphql_handler)) .route("/ws", GraphQLSubscription::new(schema.clone())) + .route("/auth/gitea", routing::get(gitea)) + .route("/auth/authorized", routing::get(authorized)) .layer(Extension(schema)) + .layer(Extension(MemoryStore::new())) + .layer(Extension(auth::oauth_client())) .layer( CorsLayer::new() .allow_origin(cors) .allow_headers([axum::http::header::CONTENT_TYPE]) .allow_methods([Method::GET, Method::POST, Method::OPTIONS]), + ) + .layer( + TraceLayer::new_for_http() + .make_span_with(DefaultMakeSpan::default().include_headers(true)), ); + let addr = SocketAddr::from(([127, 0, 0, 1], 3000)); return Server { app, addr }; @@ -70,3 +93,61 @@ impl Server { } } } + +#[derive(Debug, Serialize, Deserialize)] +struct User { + #[serde(alias = "sub")] + id: String, + #[serde(alias = "picture")] + avatar: Option, + #[serde(alias = "email")] + email: String, + #[serde(alias = "preferred_username")] + username: String, +} + +struct AuthRedirect; + +impl IntoResponse for AuthRedirect { + fn into_response(self) -> axum::response::Response { + Redirect::temporary("/auth/gitea").into_response() + } +} + +const COOKIE_NAME: &str = "auth"; + +#[async_trait] +impl FromRequest for User +where + B: Send, +{ + type Rejection = AuthRedirect; + + async fn from_request(req: &mut RequestParts) -> Result { + let Extension(store) = Extension::::from_request(req) + .await + .expect("MemoryStore extension is missing"); + + let cookies = TypedHeader::::from_request(req) + .await + .map_err(|e| match *e.name() { + header::COOKIE => match e.reason() { + TypedHeaderRejectionReason::Missing => AuthRedirect, + _ => panic!("unexpected error getting Cookie header(s): {}", e), + }, + _ => panic!("unexpected error getting cookies: {}", e), + })?; + + let session_cookie = cookies.get(COOKIE_NAME).ok_or(AuthRedirect)?; + + let session = store + .load_session(session_cookie.to_string()) + .await + .expect("could not load session") + .ok_or(AuthRedirect)?; + + let user = session.get::("user").ok_or(AuthRedirect)?; + + Ok(user) + } +} diff --git a/src/lib/scel_core/src/lib.rs b/src/lib/scel_core/src/lib.rs index 78544d0..a3b76e1 100644 --- a/src/lib/scel_core/src/lib.rs +++ b/src/lib/scel_core/src/lib.rs @@ -1,12 +1,2 @@ -#[cfg(test)] -mod tests { - #[test] - fn it_works() { - let result = 2 + 2; - assert_eq!(result, 4); - } -} +pub struct App {} -pub fn something() -> String { - "".into() -} diff --git a/src/lib/scel_core/src/repo/users_repo.rs b/src/lib/scel_core/src/repo/users_repo.rs new file mode 100644 index 0000000..851555b --- /dev/null +++ b/src/lib/scel_core/src/repo/users_repo.rs @@ -0,0 +1,3 @@ +pub trait UsersRepo { + // add code here +} -- 2.45.2 From e6161797745cd6c1513e3df560cf981994f3c92f Mon Sep 17 00:00:00 2001 From: kjuulh Date: Mon, 18 Jul 2022 10:50:52 +0200 Subject: [PATCH 2/3] Added subscriptions --- Cargo.lock | 8 +++ src/cmd/scel/src/main.rs | 7 ++- src/lib/scel_api/Cargo.toml | 2 + src/lib/scel_api/src/graphql/mutation.rs | 24 ++++++- src/lib/scel_api/src/graphql/query.rs | 26 +++++++- src/lib/scel_api/src/graphql/subscription.rs | 18 +++++- src/lib/scel_api/src/lib.rs | 9 +-- src/lib/scel_core/Cargo.toml | 5 ++ src/lib/scel_core/src/lib.rs | 16 ++++- src/lib/scel_core/src/services/mod.rs | 66 ++++++++++++++++++++ 10 files changed, 170 insertions(+), 11 deletions(-) create mode 100644 src/lib/scel_core/src/services/mod.rs diff --git a/Cargo.lock b/Cargo.lock index b58e11a..2826e8f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1317,6 +1317,7 @@ dependencies = [ "futures", "oauth2", "reqwest", + "scel_core", "serde", "serde_json", "tokio", @@ -1328,6 +1329,13 @@ dependencies = [ [[package]] name = "scel_core" version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "futures", + "tokio", + "tracing", +] [[package]] name = "scopeguard" diff --git a/src/cmd/scel/src/main.rs b/src/cmd/scel/src/main.rs index e0f07e5..adadbd6 100644 --- a/src/cmd/scel/src/main.rs +++ b/src/cmd/scel/src/main.rs @@ -1,4 +1,7 @@ +use std::sync::Arc; + use dotenv::dotenv; +use scel_core::App; use tracing::info; use tracing_subscriber::{EnvFilter, FmtSubscriber}; @@ -19,5 +22,7 @@ async fn main() -> anyhow::Result<()> { info!("Starting scel"); - scel_api::Server::new().start().await + let app = Arc::new(App::new()); + + scel_api::Server::new(app.clone()).start().await } diff --git a/src/lib/scel_api/Cargo.toml b/src/lib/scel_api/Cargo.toml index 814aed6..dff5412 100644 --- a/src/lib/scel_api/Cargo.toml +++ b/src/lib/scel_api/Cargo.toml @@ -27,3 +27,5 @@ reqwest = { version = "*", default-features = false, features = [ "rustls-tls", "json", ] } + +scel_core = {path = "../scel_core"} diff --git a/src/lib/scel_api/src/graphql/mutation.rs b/src/lib/scel_api/src/graphql/mutation.rs index b49ee52..ca00e1d 100644 --- a/src/lib/scel_api/src/graphql/mutation.rs +++ b/src/lib/scel_api/src/graphql/mutation.rs @@ -1,4 +1,7 @@ +use std::sync::Arc; + use async_graphql::{Context, Object, Result, SimpleObject, ID}; +use scel_core::{services::Download, App}; pub struct MutationRoot; @@ -9,7 +12,24 @@ struct RequestDownloadResponse { #[Object] impl MutationRoot { - async fn request_download(&self, ctx: &Context<'_>) -> Result { - Err("not implemented 123".into()) + async fn request_download( + &self, + ctx: &Context<'_>, + download_link: String, + ) -> Result { + let app = ctx.data_unchecked::>(); + + let download = app + .download_service + .add_download(Download { + id: Some("some-id".to_string()), + link: download_link, + }) + .await + .unwrap(); + + Ok(RequestDownloadResponse { + id: download.id.unwrap().into(), + }) } } diff --git a/src/lib/scel_api/src/graphql/query.rs b/src/lib/scel_api/src/graphql/query.rs index b04697b..07a00f3 100644 --- a/src/lib/scel_api/src/graphql/query.rs +++ b/src/lib/scel_api/src/graphql/query.rs @@ -1,4 +1,14 @@ -use async_graphql::Object; +use std::sync::Arc; + +use async_graphql::{Context, Object, Result, SimpleObject, ID}; +use scel_core::App; + +#[derive(SimpleObject)] +struct Download { + id: ID, + link: String, + progress: i32, +} pub struct QueryRoot; @@ -7,4 +17,18 @@ impl QueryRoot { async fn hello_world(&self) -> &str { "Hello, world!" } + + async fn get_download(&self, ctx: &Context<'_>, id: ID) -> Result> { + let app = ctx.data_unchecked::>(); + + match app.download_service.get_download(id.to_string()).await { + Ok(Some(d)) => Ok(Some(Download { + id: ID::from(d.id.expect("ID could not be found")), + progress: 0, + link: d.link, + })), + Ok(None) => Ok(None), + Err(e) => Err(e.into()), + } + } } diff --git a/src/lib/scel_api/src/graphql/subscription.rs b/src/lib/scel_api/src/graphql/subscription.rs index 6686031..2573cd9 100644 --- a/src/lib/scel_api/src/graphql/subscription.rs +++ b/src/lib/scel_api/src/graphql/subscription.rs @@ -1,6 +1,9 @@ +use std::sync::Arc; + use async_graphql::{ async_stream::stream, futures_util::Stream, Context, Object, Subscription, ID, }; +use scel_core::App; pub struct SubscriptionRoot; @@ -17,9 +20,20 @@ impl DownloadChanged { #[Subscription] impl SubscriptionRoot { - async fn get_download(&self, ctx: &Context<'_>) -> impl Stream { + async fn get_download(&self, ctx: &Context<'_>, id: ID) -> impl Stream { + let app = ctx.data_unchecked::>(); + + let mut stream = app + .download_service + .subscribe_download(id.to_string()) + .await; + stream! { - yield DownloadChanged {id: "Some-id".into()} + while stream.changed().await.is_ok() { + yield DownloadChanged { + id: id.clone() + } + } } } } diff --git a/src/lib/scel_api/src/lib.rs b/src/lib/scel_api/src/lib.rs index 62ebc85..56e2760 100644 --- a/src/lib/scel_api/src/lib.rs +++ b/src/lib/scel_api/src/lib.rs @@ -1,10 +1,10 @@ mod auth; mod graphql; -use std::net::SocketAddr; +use std::{net::SocketAddr, sync::Arc}; use async_graphql::{ - extensions::{Logger, OpenTelemetry, Tracing}, + extensions::{Logger, Tracing}, http::{playground_source, GraphQLPlaygroundConfig}, Request, Response, Schema, }; @@ -21,12 +21,12 @@ use axum::{ use graphql::{ mutation::MutationRoot, query::QueryRoot, schema::ScelSchema, subscription::SubscriptionRoot, }; +use scel_core::App; use serde::{Deserialize, Serialize}; use tower_http::{ cors::CorsLayer, trace::{DefaultMakeSpan, TraceLayer}, }; -use tracing::Subscriber; async fn graphql_playground() -> impl IntoResponse { Html(playground_source( @@ -47,10 +47,11 @@ pub struct Server { } impl Server { - pub fn new() -> Server { + pub fn new(app: Arc) -> Server { let schema = Schema::build(QueryRoot, MutationRoot, SubscriptionRoot) .extension(Tracing) .extension(Logger) + .data(app) .finish(); let cors = vec!["http://localhost:3000" diff --git a/src/lib/scel_core/Cargo.toml b/src/lib/scel_core/Cargo.toml index 9cbd07d..0fb2d18 100644 --- a/src/lib/scel_core/Cargo.toml +++ b/src/lib/scel_core/Cargo.toml @@ -6,3 +6,8 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +tokio = { version = "1.0", features = ["full"] } +anyhow = { version = "*" } +async-trait = { version = "0.1.56" } +futures = "0.3.21" +tracing = "0.1" diff --git a/src/lib/scel_core/src/lib.rs b/src/lib/scel_core/src/lib.rs index a3b76e1..3c1922a 100644 --- a/src/lib/scel_core/src/lib.rs +++ b/src/lib/scel_core/src/lib.rs @@ -1,2 +1,16 @@ -pub struct App {} +use services::InMemoryDownloadService; +pub mod services; + +#[allow(dead_code)] +pub struct App { + pub download_service: InMemoryDownloadService, +} + +impl App { + pub fn new() -> Self { + Self { + download_service: InMemoryDownloadService::new(), + } + } +} diff --git a/src/lib/scel_core/src/services/mod.rs b/src/lib/scel_core/src/services/mod.rs new file mode 100644 index 0000000..c2dd46c --- /dev/null +++ b/src/lib/scel_core/src/services/mod.rs @@ -0,0 +1,66 @@ +use std::{collections::HashMap, sync::Arc, time::Duration}; +use tokio::sync::{watch, Mutex}; +use tracing::info; + +#[derive(Clone)] +pub struct Download { + pub id: Option, + pub link: String, +} + +pub struct InMemoryDownloadService { + downloads: + Mutex>, tokio::sync::watch::Receiver)>>, +} + +impl InMemoryDownloadService { + pub fn new() -> Self { + Self { + downloads: Mutex::new(HashMap::new()), + } + } + + pub async fn add_download(&self, download: Download) -> anyhow::Result { + let mut downloads = self.downloads.lock().await; + + let (tx, rx) = watch::channel(download.clone()); + + downloads.insert( + "key".to_string(), + (Arc::new(Mutex::new(download.clone())), rx.clone()), + ); + + tokio::spawn({ + let d = download.clone(); + async move { + loop { + info!("Sending event: {}", d.clone().id.unwrap()); + let _ = tx.send(d.clone()); + tokio::time::sleep(Duration::from_millis(300)).await; + } + } + }); + + Ok(download) + } + + pub async fn get_download(&self, id: String) -> anyhow::Result> { + let downloads = self.downloads.lock().await; + + if let Some(d) = downloads.get(&id) { + let download = d.0.lock().await; + + Ok(Some(download.clone())) + } else { + Ok(None) + } + } + + pub async fn subscribe_download(&self, id: String) -> tokio::sync::watch::Receiver { + let downloads = self.downloads.lock().await; + + let download = downloads.get(&id).unwrap(); + + download.1.clone() + } +} -- 2.45.2 From f4ea2d98ab48c684a1e862f3907e01271b41978a Mon Sep 17 00:00:00 2001 From: Kasper Juul Hermansen Date: Mon, 18 Jul 2022 13:16:06 +0200 Subject: [PATCH 3/3] feature/scel-base (#1) Reviewed-on: https://git.front.kjuulh.io/kjuulh/scel/pulls/1 --- .dockerignore | 4 + .drone.yml | 31 +++ Cargo.lock | 14 + Dockerfile | 21 ++ src/lib/scel_api/src/auth/mod.rs | 2 +- src/lib/scel_api/src/graphql/mutation.rs | 5 +- src/lib/scel_api/src/graphql/query.rs | 18 +- src/lib/scel_api/src/graphql/subscription.rs | 18 +- src/lib/scel_api/src/lib.rs | 2 +- src/lib/scel_core/Cargo.toml | 4 + src/lib/scel_core/src/lib.rs | 7 +- src/lib/scel_core/src/services/mod.rs | 101 ++++++-- src/lib/scel_core/src/youtube/mod.rs | 256 +++++++++++++++++++ 13 files changed, 448 insertions(+), 35 deletions(-) create mode 100644 .dockerignore create mode 100644 .drone.yml create mode 100644 Dockerfile create mode 100644 src/lib/scel_core/src/youtube/mod.rs diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..7f44911 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +target/ +.git/ +.env +data/ diff --git a/.drone.yml b/.drone.yml new file mode 100644 index 0000000..b09d09b --- /dev/null +++ b/.drone.yml @@ -0,0 +1,31 @@ +kind: pipeline +name: default +type: docker +steps: + - name: server + image: plugins/docker + environment: + DOCKER_BUILDKIT: 1 + settings: + username: kasperhermansen + password: + from_secret: + docker_secret + repo: kasperhermansen/scel + tags: latest + context: . + dockerfile: Dockerfile + cache_from: kasperhermansen/scel:latest + + - name: send telegram notification + image: appleboy/drone-telegram + settings: + token: + from_secret: telegram_token + to: 2129601481 + format: markdown + depends_on: + - server + when: + status: [failure] + diff --git a/Cargo.lock b/Cargo.lock index 2826e8f..2498522 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1333,8 +1333,12 @@ dependencies = [ "anyhow", "async-trait", "futures", + "lazy_static", + "regex", + "thiserror", "tokio", "tracing", + "uuid", ] [[package]] @@ -1870,6 +1874,16 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "uuid" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd6469f4314d5f1ffec476e05f17cc9a78bc7a27a6a857842170bdf8d6f98d2f" +dependencies = [ + "getrandom", + "rand", +] + [[package]] name = "valuable" version = "0.1.0" diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..366b1a6 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,21 @@ +FROM rust:1.60 as builder + +WORKDIR /usr/src/scel + +COPY . . + +RUN --mount=type=cache,target=/usr/src/scel/target cargo build --release +RUN --mount=type=cache,target=/usr/src/scel/target cargo install --path src/cmd/scel + +FROM debian:bullseye-slim + +# Install YTD +RUN rm -f /etc/apt/apt.conf.d/docker-clean; echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache +RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt \ + apt-get update && apt-get install -y python3 python3-pip +RUN python3 -m pip install -U yt-dlp + +# Copy binary +COPY --from=builder /usr/local/cargo/bin/scel /usr/local/bin/scel + +CMD ["scel"] diff --git a/src/lib/scel_api/src/auth/mod.rs b/src/lib/scel_api/src/auth/mod.rs index dfdf231..1282441 100644 --- a/src/lib/scel_api/src/auth/mod.rs +++ b/src/lib/scel_api/src/auth/mod.rs @@ -9,7 +9,7 @@ use axum::{ }; use oauth2::{ basic::BasicClient, reqwest::async_http_client, AuthUrl, AuthorizationCode, ClientId, - ClientSecret, CsrfToken, RedirectUrl, Scope, TokenResponse, TokenUrl, + ClientSecret, CsrfToken, RedirectUrl, TokenResponse, TokenUrl, }; use reqwest::header::SET_COOKIE; use serde::Deserialize; diff --git a/src/lib/scel_api/src/graphql/mutation.rs b/src/lib/scel_api/src/graphql/mutation.rs index ca00e1d..a781fed 100644 --- a/src/lib/scel_api/src/graphql/mutation.rs +++ b/src/lib/scel_api/src/graphql/mutation.rs @@ -21,9 +21,12 @@ impl MutationRoot { let download = app .download_service + .clone() .add_download(Download { - id: Some("some-id".to_string()), + id: None, link: download_link, + progress: None, + file_name: None, }) .await .unwrap(); diff --git a/src/lib/scel_api/src/graphql/query.rs b/src/lib/scel_api/src/graphql/query.rs index 07a00f3..4fba81c 100644 --- a/src/lib/scel_api/src/graphql/query.rs +++ b/src/lib/scel_api/src/graphql/query.rs @@ -3,29 +3,27 @@ use std::sync::Arc; use async_graphql::{Context, Object, Result, SimpleObject, ID}; use scel_core::App; -#[derive(SimpleObject)] -struct Download { - id: ID, - link: String, - progress: i32, +#[derive(SimpleObject, Clone)] +pub struct Download { + pub id: ID, + pub link: String, + pub progress: Option, + pub file_name: Option, } pub struct QueryRoot; #[Object] impl QueryRoot { - async fn hello_world(&self) -> &str { - "Hello, world!" - } - async fn get_download(&self, ctx: &Context<'_>, id: ID) -> Result> { let app = ctx.data_unchecked::>(); match app.download_service.get_download(id.to_string()).await { Ok(Some(d)) => Ok(Some(Download { id: ID::from(d.id.expect("ID could not be found")), - progress: 0, + progress: None, link: d.link, + file_name: None, })), Ok(None) => Ok(None), Err(e) => Err(e.into()), diff --git a/src/lib/scel_api/src/graphql/subscription.rs b/src/lib/scel_api/src/graphql/subscription.rs index 2573cd9..6647693 100644 --- a/src/lib/scel_api/src/graphql/subscription.rs +++ b/src/lib/scel_api/src/graphql/subscription.rs @@ -5,16 +5,18 @@ use async_graphql::{ }; use scel_core::App; +use super::query::Download; + pub struct SubscriptionRoot; struct DownloadChanged { - id: ID, + download: Download, } #[Object] impl DownloadChanged { - async fn id(&self) -> &ID { - &self.id + async fn download(&self) -> Download { + self.download.clone() } } @@ -30,8 +32,16 @@ impl SubscriptionRoot { stream! { while stream.changed().await.is_ok() { + let next_download = (*stream.borrow()).clone(); + let id = ID::from(next_download.id.unwrap()); + yield DownloadChanged { - id: id.clone() + download: Download { + id: id, + link: next_download.link, + file_name: next_download.file_name, + progress: next_download.progress, + } } } } diff --git a/src/lib/scel_api/src/lib.rs b/src/lib/scel_api/src/lib.rs index 56e2760..0da69fe 100644 --- a/src/lib/scel_api/src/lib.rs +++ b/src/lib/scel_api/src/lib.rs @@ -79,7 +79,7 @@ impl Server { let addr = SocketAddr::from(([127, 0, 0, 1], 3000)); - return Server { app, addr }; + Server { app, addr } } pub async fn start(self) -> anyhow::Result<()> { diff --git a/src/lib/scel_core/Cargo.toml b/src/lib/scel_core/Cargo.toml index 0fb2d18..6802ede 100644 --- a/src/lib/scel_core/Cargo.toml +++ b/src/lib/scel_core/Cargo.toml @@ -11,3 +11,7 @@ anyhow = { version = "*" } async-trait = { version = "0.1.56" } futures = "0.3.21" tracing = "0.1" +lazy_static = "1.4.0" +regex = { version = "1.5.5" } +thiserror = "1.0.31" +uuid = {version = "1.1.2", features = ["v4", "fast-rng"]} diff --git a/src/lib/scel_core/src/lib.rs b/src/lib/scel_core/src/lib.rs index 3c1922a..21f08ad 100644 --- a/src/lib/scel_core/src/lib.rs +++ b/src/lib/scel_core/src/lib.rs @@ -1,16 +1,19 @@ +use std::sync::Arc; + use services::InMemoryDownloadService; pub mod services; +mod youtube; #[allow(dead_code)] pub struct App { - pub download_service: InMemoryDownloadService, + pub download_service: Arc, } impl App { pub fn new() -> Self { Self { - download_service: InMemoryDownloadService::new(), + download_service: Arc::new(InMemoryDownloadService::new()), } } } diff --git a/src/lib/scel_core/src/services/mod.rs b/src/lib/scel_core/src/services/mod.rs index c2dd46c..d9497a3 100644 --- a/src/lib/scel_core/src/services/mod.rs +++ b/src/lib/scel_core/src/services/mod.rs @@ -1,16 +1,29 @@ -use std::{collections::HashMap, sync::Arc, time::Duration}; +use std::{collections::HashMap, path::PathBuf, sync::Arc}; use tokio::sync::{watch, Mutex}; -use tracing::info; +use tracing::error; +use uuid::Uuid; + +use crate::youtube::{Arg, YoutubeDL}; #[derive(Clone)] pub struct Download { pub id: Option, pub link: String, + pub progress: Option, + pub file_name: Option, } pub struct InMemoryDownloadService { - downloads: - Mutex>, tokio::sync::watch::Receiver)>>, + downloads: Mutex< + HashMap< + String, + ( + Arc>, + Arc>>, + tokio::sync::watch::Receiver, + ), + >, + >, } impl InMemoryDownloadService { @@ -20,28 +33,86 @@ impl InMemoryDownloadService { } } - pub async fn add_download(&self, download: Download) -> anyhow::Result { + pub async fn add_download(self: Arc, download: Download) -> anyhow::Result { let mut downloads = self.downloads.lock().await; let (tx, rx) = watch::channel(download.clone()); + let shared_tx = Arc::new(Mutex::new(tx)); + + let mut d = download.to_owned(); + + let id = Uuid::new_v4().to_string(); + d.id = Some(id.clone()); downloads.insert( - "key".to_string(), - (Arc::new(Mutex::new(download.clone())), rx.clone()), + id.clone(), + ( + Arc::new(Mutex::new(d.clone())), + shared_tx, + rx, + ), ); + let args = vec![ + Arg::new("--progress"), + Arg::new("--newline"), + Arg::new_with_args("--output", "%(title).90s.%(ext)s"), + ]; + let ytd = YoutubeDL::new( + &PathBuf::from("./data/downloads"), + args, + download.link.as_str(), + )?; + tokio::spawn({ - let d = download.clone(); + let download_service = self.clone(); + async move { - loop { - info!("Sending event: {}", d.clone().id.unwrap()); - let _ = tx.send(d.clone()); - tokio::time::sleep(Duration::from_millis(300)).await; + if let Err(e) = ytd + .download( + |percentage| { + let ds = download_service.clone(); + let id = id.clone(); + + async move { + let mut download = ds.get_download(id).await.unwrap().unwrap(); + download.progress = Some(percentage); + let _ = ds.update_download(download).await; + } + }, + |file_name| { + let ds = download_service.clone(); + let id = id.clone(); + + async move { + let mut download = ds.get_download(id).await.unwrap().unwrap(); + download.file_name = Some(file_name); + let _ = ds.update_download(download).await; + } + }, + ) + .await + { + error!("Download failed: {}", e); + } else { + let download = download_service.get_download(id).await.unwrap().unwrap(); + let _ = download_service.update_download(download).await; } } }); - Ok(download) + Ok(d) + } + + pub async fn update_download(self: Arc, download: Download) -> anyhow::Result<()> { + let mut downloads = self.downloads.lock().await; + if let Some(d) = downloads.get_mut(&download.clone().id.unwrap()) { + let mut d_mut = d.0.lock().await; + *d_mut = download.clone(); + let _ = d.1.lock().await.send(download); + } + + Ok(()) } pub async fn get_download(&self, id: String) -> anyhow::Result> { @@ -58,9 +129,7 @@ impl InMemoryDownloadService { pub async fn subscribe_download(&self, id: String) -> tokio::sync::watch::Receiver { let downloads = self.downloads.lock().await; - let download = downloads.get(&id).unwrap(); - - download.1.clone() + download.2.clone() } } diff --git a/src/lib/scel_core/src/youtube/mod.rs b/src/lib/scel_core/src/youtube/mod.rs new file mode 100644 index 0000000..cdcb26d --- /dev/null +++ b/src/lib/scel_core/src/youtube/mod.rs @@ -0,0 +1,256 @@ +use std::fmt::{Display, Formatter}; +use std::fs::{canonicalize, create_dir_all}; +use std::future::Future; +use std::num::ParseIntError; +use std::path::{Path, PathBuf}; +use std::process::{Output, Stdio}; + +use lazy_static::lazy_static; +use regex::Regex; +use thiserror::Error; +use tokio::io::{AsyncBufReadExt, BufReader}; +use tokio::process::Command; + +#[derive(Error, Debug)] +pub enum YoutubeDLError { + #[error("failed to execute youtube-dl")] + IOError(#[from] std::io::Error), + #[error("failed to convert path")] + UTF8Error(#[from] std::string::FromUtf8Error), + #[error("youtube-dl exited with: {0}")] + Failure(String), +} + +type Result = std::result::Result; + +const YOUTUBE_DL_COMMAND: &str = "yt-dlp"; + +#[derive(Clone, Debug)] +pub struct Arg { + arg: String, + input: Option, +} + +impl Arg { + pub fn new(argument: &str) -> Self { + Self { + arg: argument.to_string(), + input: None, + } + } + + pub fn new_with_args(argument: &str, input: &str) -> Self { + Self { + arg: argument.to_string(), + input: Option::from(input.to_string()), + } + } +} + +impl Display for Arg { + fn fmt(&self, fmt: &mut Formatter<'_>) -> std::fmt::Result { + match &self.input { + Some(input) => write!(fmt, "{} {}", self.arg, input), + None => write!(fmt, "{}", self.arg), + } + } +} + +#[derive(Clone, Debug)] +pub struct YoutubeDL { + path: PathBuf, + links: Vec, + args: Vec, +} + +#[derive(Clone, Debug)] +pub struct YoutubeDLResult { + path: PathBuf, + output: String, +} + +impl YoutubeDLResult { + fn new(path: &PathBuf) -> Self { + Self { + path: path.clone(), + output: String::new(), + } + } + + pub fn output_dir(&self) -> &PathBuf { + &self.path + } +} + +impl YoutubeDL { + pub fn new_multiple_links( + dl_path: &PathBuf, + args: Vec, + links: Vec, + ) -> Result { + let path = Path::new(dl_path); + + if !path.exists() { + create_dir_all(&path)?; + } + + if !path.is_dir() { + return Err(YoutubeDLError::IOError(std::io::Error::new( + std::io::ErrorKind::Other, + "path is not a directory", + ))); + } + + let path = canonicalize(dl_path)?; + Ok(YoutubeDL { path, links, args }) + } + + pub fn new(dl_path: &PathBuf, args: Vec, link: &str) -> Result { + YoutubeDL::new_multiple_links(dl_path, args, vec![link.to_string()]) + } + + pub async fn download( + &self, + progress_update_fn: F, + file_name_available: FAvailable, + ) -> Result + where + F: Fn(u32) -> Fut, + FAvailable: Fn(String) -> FutAvailable, + Fut: Future, + FutAvailable: Future, + { + let output = self + .spawn_youtube_dl(progress_update_fn, file_name_available) + .await?; + let mut result = YoutubeDLResult::new(&self.path); + + if !output.status.success() { + return Err(YoutubeDLError::Failure(String::from_utf8(output.stderr)?)); + } + result.output = String::from_utf8(output.stdout)?; + + Ok(result) + } + + async fn spawn_youtube_dl( + &self, + progress_update_fn: F, + file_name_available: FAvailable, + ) -> Result + where + F: Fn(u32) -> Fut, + FAvailable: Fn(String) -> FutAvailable, + Fut: Future, + FutAvailable: Future, + { + let mut cmd = Command::new(YOUTUBE_DL_COMMAND); + cmd.current_dir(&self.path) + .env("LC_ALL", "en_US.UTF-8") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + for arg in self.args.iter() { + match &arg.input { + Some(input) => cmd.arg(&arg.arg).arg(input), + None => cmd.arg(&arg.arg), + }; + } + + for link in self.links.iter() { + cmd.arg(&link); + } + + let mut pr = cmd.spawn()?; + + { + let stdout = pr.stdout.as_mut().unwrap(); + let stdout_reader = BufReader::new(stdout); + let mut stdout_lines = stdout_reader.lines(); + + let mut have_gotten_file_name = false; + while let Ok(Some(line)) = stdout_lines.next_line().await { + println!("{}", line.clone()); + + if !have_gotten_file_name { + if let Some(file_name) = parse_file_name(line.clone()) { + file_name_available(file_name).await; + have_gotten_file_name = true + } + } + + if let Some(Ok(percentage)) = parse_line(line) { + progress_update_fn(percentage).await; + } + } + } + + Ok(pr.wait_with_output().await?) + } +} + +fn parse_line(line: String) -> Option> { + lazy_static! { + static ref RE: Regex = Regex::new(r"\[download\]\s+(\d+)").unwrap(); + } + + let capture: regex::Captures = RE.captures(line.as_str())?; + if capture.len() != 2 { + return None; + } + let str = &capture[1]; + Some(str.to_string().parse::()) +} + +fn parse_file_name(line: String) -> Option { + lazy_static! { + static ref RE: Regex = Regex::new(r"^\[download\] Destination: (.+)$").unwrap(); + } + + let capture: regex::Captures = RE.captures(line.as_str())?; + if capture.len() != 2 { + return None; + } + let str = &capture[1]; + Some(str.to_string()) +} + +#[cfg(test)] +mod tests { + use crate::youtube::{parse_file_name, parse_line}; + + #[test] + fn test_parse_line() { + let percentage = parse_line( + "[download] 95.4% of ~215.85MiB at 9.61MiB/s ETA 00:01 (frag 144/151)".into(), + ); + + assert_eq!(percentage, Some(Ok(95))) + } + + #[test] + fn test_parse_line_get_nothing() { + let nothing = parse_line("[download] Got server HTTP error: The read operation timed out. Retrying (attempt 1 of 10) ...".into()); + + assert_eq!(nothing, None) + } + + #[test] + fn test_parse_file_name() { + let file_name = parse_file_name( + "[download] Destination: 10 Design Patterns Explained in 10 Minutes.mp4".into(), + ); + + assert_eq!( + file_name, + Some("10 Design Patterns Explained in 10 Minutes.mp4".into()) + ); + } + + #[test] + fn test_parse_file_name_get_nothing() { + let nothing = parse_file_name("[download] No fit: something".into()); + + assert_eq!(nothing, None) + } +} -- 2.45.2