Remove usages of surf (#873)

* docs: add info about pro version

* feat(backend): return main website url

* feat(frontend): display Ryot pro link in footer

* fix(ts-utils): remove hoisted variables

* fix(frontend): no-wrap for title

* build(backend): upgrade deps

* fix(backend): correct logic for exercise interacted

* feat(backend): add new dep

* feat(backend): new reqwest version of client

* feat(backend): add json feature

* feat(backend): switch to surf in anilist

* build(backend): add file streaming deps

* fix(backend): use reqwest for exporter

* feat(backend): use reqwest in some places

* fix(backend): don't queue notification when triggering test

* feat(frontend): show in bold

* feat(backend): remove all usages of surf

* refactor(backend): change name of old function

* build(backend): remove surf from deps

* fix(frontend): do not pass as props

* chore(frontend): do not display pro link

* build(backend): bump version

* fix(backend): add log to ABS importer

* fix(frontend): better wordings

* feat(backend): reduce interval for integration sync

* docs: add info about sync interval

* docs: add caveat for ABS

Author: IgnisDa

Cargo.lock

@@ -4,7 +4,7 @@ resolver = "2"
 
 [workspace.dependencies]
 anyhow = "=1.0.82"
-async-graphql = { version = "=7.0.5", features = [
+async-graphql = { version = "=7.0.6", features = [
     "chrono",
     "decimal",
     "log",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ryot"
-version = "6.2.2"
+version = "6.2.3"
 edition = "2021"
 repository = "https://github.com/IgnisDa/ryot"
 license = "GPL-3.0"
@@ -10,9 +10,9 @@ anyhow = { workspace = true }
 apalis = { version = "=0.5.3", features = ["cron", "limit"] }
 argon2 = "=0.6.0-pre.0"
 async-graphql = { workspace = true }
-async-graphql-axum = "=7.0.5"
+async-graphql-axum = "=7.0.6"
 async-trait = "=0.1.80"
-aws-sdk-s3 = { version = "=1.34.0", features = ["behavior-version-latest"] }
+aws-sdk-s3 = { version = "=1.36.0", features = ["behavior-version-latest"] }
 axum = { version = "=0.7.5", features = ["macros", "multipart"] }
 boilermates = "=0.3.0"
 cached = { version = "=0.51.4", features = ["disk_store"] }
@@ -30,13 +30,12 @@ derive_more = { version = "=1.0.0-beta.6", features = [
     "add_assign",
 ], default-features = false }
 dotenvy = "=0.15.7"
-enum_meta = "=0.6.0"
+enum_meta = "=0.7.0"
 flate2 = "=1.0.30"
 futures = "=0.3.30"
 graphql_client = "=0.14.0"
 hashbag = "=0.1.12"
 http = "=1.1.0"
-http-types = "=2.12.0"
 isolang = { version = "=2.4.0", features = ["list_languages"] }
 itertools = "=0.13.0"
 jsonwebtoken = { version = "=9.3.0", default-features = false }
@@ -51,7 +50,12 @@ nanoid = { workspace = true }
 openidconnect = "=3.5.0"
 paginate = "=1.1.11"
 rand = "=0.9.0-alpha.1"
-regex = "=1.10.4"
+regex = "=1.10.5"
+# FIXME: Upgrade once https://github.com/seanmonstar/reqwest/pull/1620 is merged
+reqwest = { git = "https://github.com/thomasqueirozb/reqwest", branch = "base_url", features = [
+    "json",
+    "stream",
+] }
 rs-utils = { path = "../../libs/rs-utils" }
 rust_decimal = "=1.35.0"
 rust_decimal_macros = "=1.34.2"
@@ -68,10 +72,8 @@ serde-xml-rs = "=0.6.0"
 slug = "=0.1.5"
 strum = { workspace = true }
 struson = { version = "=0.5.0", features = ["serde"] }
-surf = { version = "=2.3.2", features = [
-    "h1-client-rustls",
-], default-features = false }
 tokio = { version = "=1.38.0", features = ["full"] }
+tokio-util = { version = "=0.7.11", features = ["codec"] }
 tower = { version = "=0.4.13", features = ["buffer"] }
 tower-http = { version = "=0.5.2", features = ["catch-panic", "cors", "trace"] }
 tracing = { workspace = true }

apps/backend/Cargo.toml

@@ -1,13 +1,16 @@
-use std::{collections::HashMap, fs::File, path::PathBuf, sync::Arc};
+use std::{collections::HashMap, path::PathBuf, sync::Arc};
 
 use apalis::prelude::MessageQueue;
 use async_graphql::{Context, Error, Object, Result, SimpleObject};
 use chrono::{DateTime, Utc};
 use nanoid::nanoid;
+use reqwest::{Body, Client};
 use rs_utils::IsFeatureEnabled;
 use sea_orm::prelude::DateTimeUtc;
 use serde::{Deserialize, Serialize};
 use struson::writer::{JsonStreamWriter, JsonWriter};
+use tokio::fs::File;
+use tokio_util::codec::{BytesCodec, FramedRead};
 
 use crate::{
     background::ApplicationJob, file_storage::FileStorageService,
@@ -99,7 +102,7 @@ impl ExporterService {
         }
         let started_at = Utc::now();
         let export_path = PathBuf::from(TEMP_DIR).join(format!("ryot-export-{}.json", nanoid!()));
-        let file = File::create(&export_path).unwrap();
+        let file = std::fs::File::create(&export_path).unwrap();
         let mut writer = JsonStreamWriter::new(file);
         writer.begin_object().unwrap();
         for export in to_export.iter() {
@@ -158,16 +161,20 @@ impl ExporterService {
                 ])),
             )
             .await;
-        surf::put(url)
+        let file = File::open(&export_path).await.unwrap();
+        let stream = FramedRead::new(file, BytesCodec::new());
+        let body = Body::wrap_stream(stream);
+        let client = Client::new();
+        client
+            .put(url)
             .header("x-amz-meta-started_at", started_at.to_rfc2822())
             .header("x-amz-meta-ended_at", ended_at.to_rfc2822())
             .header(
                 "x-amz-meta-exported",
                 serde_json::to_string(&to_export).unwrap(),
             )
-            .body_file(&export_path)
-            .await
-            .unwrap()
+            .body(body)
+            .send()
             .await
             .unwrap();
         Ok(true)

apps/backend/src/exporter.rs

@@ -149,6 +149,7 @@ impl UserWorkoutInput {
                         created_on: ActiveValue::Set(
                             first_set_of_exercise_confirmed_at.unwrap_or(end_time),
                         ),
+                        exercise_num_times_interacted: ActiveValue::Set(Some(1)),
                         ..Default::default()
                     };
                     user_to_ex.insert(db).await.unwrap()

apps/backend/src/fitness/logic.rs

@@ -331,11 +331,10 @@ impl ExerciseService {
     }
 
     async fn get_all_exercises_from_dataset(&self) -> Result<Vec<GithubExercise>> {
-        let data: Vec<GithubExercise> = surf::get(JSON_URL)
-            .send()
+        let data = reqwest::get(JSON_URL)
             .await
             .unwrap()
-            .body_json()
+            .json::<Vec<GithubExercise>>()
             .await
             .unwrap();
         Ok(data

apps/backend/src/fitness/resolver.rs

@@ -4,10 +4,13 @@ use anyhow::anyhow;
 use async_graphql::Result;
 use data_encoding::BASE64;
 use database::{ImportSource, MediaLot, MediaSource};
+use reqwest::{
+    header::{HeaderValue, AUTHORIZATION},
+    Client,
+};
 use sea_orm::DatabaseConnection;
 use serde::{Deserialize, Serialize};
 use serde_json::json;
-use surf::{http::headers::AUTHORIZATION, Client};
 
 use crate::{
     importer::{ImportFailStep, ImportFailedItem, ImportResult},
@@ -48,13 +51,17 @@ where
     let mut failed_items = vec![];
     let client = get_base_http_client(
         &format!("{}/api/", input.api_url),
-        vec![(AUTHORIZATION, format!("Bearer {}", input.api_key))],
+        Some(vec![(
+            AUTHORIZATION,
+            HeaderValue::from_str(&format!("Bearer {}", input.api_key)).unwrap(),
+        )]),
     );
-    let libraries_resp: LibrariesListResponse = client
+    let libraries_resp = client
         .get("libraries")
+        .send()
         .await
         .map_err(|e| anyhow!(e))?
-        .body_json()
+        .json::<LibrariesListResponse>()
         .await
         .unwrap();
     for library in libraries_resp.libraries {
@@ -63,18 +70,20 @@ where
         if let Some(audiobookshelf_models::MediaType::Book) = library.media_type {
             query["filter"] = json!(format!("progress.{}", BASE64.encode(b"finished")));
         }
-        let finished_items: ListResponse = client
+        let finished_items = client
             .get(&format!("libraries/{}/items", library.id))
             .query(&query)
-            .unwrap()
+            .send()
             .await
             .map_err(|e| anyhow!(e))?
-            .body_json()
+            .json::<ListResponse>()
             .await
             .unwrap();
-        for item in finished_items.results {
+        let len = finished_items.results.len();
+        for (idx, item) in finished_items.results.into_iter().enumerate() {
             let metadata = item.media.clone().unwrap().metadata;
             let title = metadata.title.clone();
+            tracing::trace!("Importing item {:?} ({}/{})", title, idx + 1, len);
             let (identifier, lot, source, episodes) = if Some("epub".to_string())
                 == item.media.as_ref().unwrap().ebook_format
             {
@@ -111,6 +120,7 @@ where
                         let source = MediaSource::Itunes;
                         let mut to_return = vec![];
                         for episode in episodes {
+                            tracing::trace!("Importing episode {:?}", episode.title);
                             let episode_details =
                                 get_item_details(&client, &item.id, Some(episode.id.unwrap()))
                                     .await?;
@@ -192,12 +202,13 @@ async fn get_item_details(
     if let Some(episode) = episode {
         query["episode"] = json!(episode);
     }
-    let item: audiobookshelf_models::Item = client
+    let item = client
         .get(&format!("items/{}", id))
-        .query(&query)?
+        .query(&query)
+        .send()
         .await
         .map_err(|e| anyhow!(e))?
-        .body_json()
+        .json::<audiobookshelf_models::Item>()
         .await?;
     Ok(item)
 }

apps/backend/src/importer/audiobookshelf.rs

@@ -1,13 +1,13 @@
 use async_graphql::Result;
 use database::{MediaLot, MediaSource};
 use enum_meta::HashMap;
+use reqwest::{
+    header::{HeaderMap, HeaderValue, ACCEPT, AUTHORIZATION, USER_AGENT},
+    Client, ClientBuilder,
+};
 use sea_orm::prelude::DateTimeUtc;
 use serde::{Deserialize, Serialize};
 use serde_json::json;
-use surf::{
-    http::headers::{ACCEPT, AUTHORIZATION, USER_AGENT},
-    Client, Config, Url,
-};
 
 use crate::{
     importer::{
@@ -16,7 +16,7 @@ use crate::{
     models::media::{
         ImportOrExportItemIdentifier, ImportOrExportMediaItem, ImportOrExportMediaItemSeen,
     },
-    utils::USER_AGENT_STR,
+    utils::{JSON, USER_AGENT_STR},
 };
 
 static EMBY_HEADER_VALUE: &str =
@@ -74,26 +74,30 @@ struct AuthenticateResponse {
 
 pub async fn import(input: DeployUrlAndKeyAndUsernameImportInput) -> Result<ImportResult> {
     let uri = format!("{}/Users/AuthenticateByName", input.api_url);
-    let authenticate: AuthenticateResponse = surf::post(uri)
+    let client = Client::new();
+    let authenticate = client
+        .post(uri)
         .header(AUTHORIZATION, EMBY_HEADER_VALUE)
-        .body_json(&serde_json::json!({ "Username": input.username, "Pw": input.password }))
-        .unwrap()
+        .json(&serde_json::json!({ "Username": input.username, "Pw": input.password }))
+        .send()
         .await
         .unwrap()
-        .body_json()
+        .json::<AuthenticateResponse>()
         .await
         .unwrap();
     tracing::debug!("Authenticated with token: {}", authenticate.access_token);
 
-    let client: Client = Config::new()
-        .add_header(USER_AGENT, USER_AGENT_STR)
-        .unwrap()
-        .add_header(ACCEPT, "application/json")
-        .unwrap()
-        .add_header("X-Emby-Token", authenticate.access_token)
-        .unwrap()
-        .set_base_url(Url::parse(&input.api_url).unwrap().join("/").unwrap())
-        .try_into()
+    let mut headers = HeaderMap::new();
+    headers.insert(USER_AGENT, HeaderValue::from_static(USER_AGENT_STR));
+    headers.insert(ACCEPT, JSON.clone());
+    headers.insert(
+        "X-Emby-Token",
+        HeaderValue::from_str(&authenticate.access_token).unwrap(),
+    );
+    let client: Client = ClientBuilder::new()
+        .base_url(input.api_url + "/")
+        .default_headers(headers)
+        .build()
         .unwrap();
     let user_id = authenticate.user.id;
     tracing::debug!("Authenticated as user id: {}", user_id);
@@ -102,13 +106,13 @@ pub async fn import(input: DeployUrlAndKeyAndUsernameImportInput) -> Result<Impo
     let mut failed_items = vec![];
 
     let query = json!({ "recursive": true, "IsPlayed": true, "fields": "ProviderIds" });
-    let library_data: ItemsResponse = client
+    let library_data = client
         .get(&format!("Users/{}/Items", user_id))
         .query(&query)
-        .unwrap()
+        .send()
         .await
         .unwrap()
-        .body_json()
+        .json::<ItemsResponse>()
         .await
         .unwrap();
 
@@ -123,11 +127,12 @@ pub async fn import(input: DeployUrlAndKeyAndUsernameImportInput) -> Result<Impo
                 if let Some(series_id) = item.series_id {
                     let mut tmdb_id = series_id_to_tmdb_id.get(&series_id).cloned().flatten();
                     if tmdb_id.is_none() {
-                        let details: ItemResponse = client
+                        let details = client
                             .get(&format!("Items/{}", series_id))
+                            .send()
                             .await
                             .unwrap()
-                            .body_json()
+                            .json::<ItemResponse>()
                             .await
                             .unwrap();
                         let insert_id = details.provider_ids.unwrap().tmdb;