From 4c9ad2318c554bc6cf2b761de3737a8626a96d3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E1=B4=80=E1=B4=8D=E1=B4=9B=E1=B4=8F=E1=B4=80=E1=B4=87?= =?UTF-8?q?=CA=80?= Date: Wed, 3 Jul 2024 18:57:12 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=A4=A7=E8=8C=83=E5=9B=B4=E9=87=8D?= =?UTF-8?q?=E6=9E=84=EF=BC=8C=E6=94=AF=E6=8C=81=E8=A7=86=E9=A2=91=E5=90=88?= =?UTF-8?q?=E9=9B=86=E4=B8=8B=E8=BD=BD=20(#97)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 27 +- Cargo.toml | 8 +- Justfile | 13 +- crates/bili_sync/Cargo.toml | 2 +- crates/bili_sync/src/adapter/collection.rs | 241 ++++++++++++++++ crates/bili_sync/src/adapter/favorite.rs | 199 +++++++++++++ crates/bili_sync/src/adapter/mod.rs | 89 ++++++ crates/bili_sync/src/bilibili/collection.rs | 265 ++++++++++++++++++ .../bili_sync/src/bilibili/favorite_list.rs | 22 +- crates/bili_sync/src/bilibili/mod.rs | 59 +++- crates/bili_sync/src/bilibili/video.rs | 22 +- crates/bili_sync/src/config.rs | 102 ++++++- crates/bili_sync/src/core/mod.rs | 3 - crates/bili_sync/src/database.rs | 17 +- crates/bili_sync/src/main.rs | 34 ++- crates/bili_sync/src/utils/convert.rs | 131 +++++++++ crates/bili_sync/src/utils/mod.rs | 23 ++ crates/bili_sync/src/utils/model.rs | 95 +++++++ .../src/{core/utils.rs => utils/nfo.rs} | 263 +---------------- .../bili_sync/src/{core => utils}/status.rs | 0 .../src/{core/command.rs => workflow.rs} | 157 +++-------- crates/bili_sync_entity/Cargo.toml | 1 - .../src/entities/collection.rs | 21 ++ crates/bili_sync_entity/src/entities/mod.rs | 1 + crates/bili_sync_entity/src/entities/video.rs | 3 +- crates/bili_sync_migration/src/lib.rs | 6 +- .../src/m20240505_130850_add_collection.rs | 187 ++++++++++++ 27 files changed, 1545 insertions(+), 446 deletions(-) create mode 100644 crates/bili_sync/src/adapter/collection.rs create mode 100644 crates/bili_sync/src/adapter/favorite.rs create mode 100644 crates/bili_sync/src/adapter/mod.rs create mode 100644 crates/bili_sync/src/bilibili/collection.rs delete mode 100644 crates/bili_sync/src/core/mod.rs create mode 100644 crates/bili_sync/src/utils/convert.rs create mode 100644 crates/bili_sync/src/utils/mod.rs create mode 100644 crates/bili_sync/src/utils/model.rs rename crates/bili_sync/src/{core/utils.rs => utils/nfo.rs} (60%) rename crates/bili_sync/src/{core => utils}/status.rs (100%) rename crates/bili_sync/src/{core/command.rs => workflow.rs} (78%) create mode 100644 crates/bili_sync_entity/src/entities/collection.rs create mode 100644 crates/bili_sync_migration/src/m20240505_130850_add_collection.rs diff --git a/Cargo.lock b/Cargo.lock index 7428b25..515c4a4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -342,9 +342,9 @@ checksum = "fbb36e985947064623dbd357f727af08ffd077f93d696782f3c56365fa2e2799" [[package]] name = "async-trait" -version = "0.1.79" +version = "0.1.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507401cad91ec6a857ed5513a2073c82a9b9048762b885bb98655b306964681" +checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", @@ -423,6 +423,7 @@ dependencies = [ "anyhow", "arc-swap", "async-stream", + "async-trait", "bili_sync_entity", "bili_sync_migration", "chrono", @@ -434,7 +435,6 @@ dependencies = [ "futures", "handlebars", "hex", - "log", "memchr", "once_cell", "prost", @@ -459,7 +459,6 @@ name = "bili_sync_entity" version = "2.0.3" dependencies = [ "sea-orm", - "serde", "serde_json", ] @@ -622,9 +621,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.7" +version = "4.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5db83dced34638ad474f39f250d7fea9598bdd239eaced1bdf45d597da0f433f" +checksum = "84b3edb18336f4df585bc9aa31dd99c036dfa5dc5e9a2939a722a188f3a8970d" dependencies = [ "clap_builder", "clap_derive", @@ -632,9 +631,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.7" +version = "4.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7e204572485eb3fbf28f871612191521df159bc3e15a9f5064c66dba3a8c05f" +checksum = "c1c09dd5ada6c6c78075d6fd0da3f90d8080651e2d6cc8eb2f1aaa4034ced708" dependencies = [ "anstream", "anstyle", @@ -644,9 +643,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.5" +version = "4.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c780290ccf4fb26629baa7a1081e68ced113f1d3ec302fa5948f1c381ebf06c6" +checksum = "2bac35c6dafb060fd4d275d9a4ffae97917c13a6327903a8be2153cd964f7085" dependencies = [ "heck 0.5.0", "proc-macro2", @@ -2119,9 +2118,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.34.0" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f24d770aeca0eacb81ac29dfbc55ebcc09312fdd1f8bbecdc7e4a84e000e3b4" +checksum = "86e446ed58cef1bbfe847bc2fda0e2e4ea9f0e57b90c507d4781292590d72a4e" dependencies = [ "memchr", "tokio", @@ -2747,9 +2746,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.118" +version = "1.0.120" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d947f6b3163d8857ea16c4fa0dd4840d52f3041039a85decd46867eb1abef2e4" +checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" dependencies = [ "itoa", "ryu", diff --git a/Cargo.toml b/Cargo.toml index 5980623..af14090 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,8 +19,9 @@ anyhow = { version = "1.0.86", features = ["backtrace"] } arc-swap = { version = "1.7.1", features = ["serde"] } async-std = { version = "1.12.0", features = ["attributes", "tokio1"] } async-stream = "0.3.5" +async-trait = "0.1.80" chrono = { version = "0.4.38", features = ["serde"] } -clap = { version = "4.5.7", features = ["env"] } +clap = { version = "4.5.8", features = ["env"] } cookie = "0.18.1" dirs = "5.0.1" filenamify = "0.1.0" @@ -28,11 +29,10 @@ float-ord = "0.3.2" futures = "0.3.30" handlebars = "5.1.2" hex = "0.4.3" -log = "0.4.22" memchr = "2.7.4" once_cell = "1.19.0" prost = "0.12.6" -quick-xml = { version = "0.34.0", features = ["async-tokio"] } +quick-xml = { version = "0.35.0", features = ["async-tokio"] } rand = "0.8.5" regex = "1.10.5" reqwest = { version = "0.12.5", features = [ @@ -52,7 +52,7 @@ sea-orm = { version = "0.12.15", features = [ ] } sea-orm-migration = { version = "0.12.15", features = [] } serde = { version = "1.0.203", features = ["derive"] } -serde_json = "1.0.118" +serde_json = "1.0.120" strum = { version = "0.26.3", features = ["derive"] } thiserror = "1.0.61" tokio = { version = "1.38.0", features = ["full"] } diff --git a/Justfile b/Justfile index 570a24e..37d4d9d 100644 --- a/Justfile +++ b/Justfile @@ -7,4 +7,15 @@ build: build-docker: build cp target/x86_64-unknown-linux-musl/release/bili-sync-rs ./Linux-x86_64-bili-sync-rs docker build . -t bili-sync-rs-local --build-arg="TARGETPLATFORM=linux/amd64" - just clean \ No newline at end of file + just clean + +copy-config: + rm -rf ~/.config/bili-sync + cp -r ~/.config/nas/bili-sync-rs ~/.config/bili-sync + sed -i -e 's/\/Bilibilis/\/Test_Bilibilis/g' -e 's/.config\/nas/.config\/test_nas/g' ~/.config/bili-sync/config.toml + +run: + cargo run + +debug: copy-config + just run \ No newline at end of file diff --git a/crates/bili_sync/Cargo.toml b/crates/bili_sync/Cargo.toml index dc1529d..78f44fe 100644 --- a/crates/bili_sync/Cargo.toml +++ b/crates/bili_sync/Cargo.toml @@ -12,6 +12,7 @@ readme = "../../README.md" anyhow = { workspace = true } arc-swap = { workspace = true } async-stream = { workspace = true } +async-trait = { workspace = true } bili_sync_entity = { workspace = true } bili_sync_migration = { workspace = true } chrono = { workspace = true } @@ -23,7 +24,6 @@ float-ord = { workspace = true } futures = { workspace = true } handlebars = { workspace = true } hex = { workspace = true } -log = { workspace = true } memchr = { workspace = true } once_cell = { workspace = true } prost = { workspace = true } diff --git a/crates/bili_sync/src/adapter/collection.rs b/crates/bili_sync/src/adapter/collection.rs new file mode 100644 index 0000000..8ded88b --- /dev/null +++ b/crates/bili_sync/src/adapter/collection.rs @@ -0,0 +1,241 @@ +use std::collections::HashSet; +use std::path::Path; +use std::pin::Pin; + +use anyhow::Result; +use bili_sync_entity::*; +use bili_sync_migration::OnConflict; +use filenamify::filenamify; +use futures::Stream; +use sea_orm::entity::prelude::*; +use sea_orm::ActiveValue::Set; +use sea_orm::{DatabaseConnection, QuerySelect, TransactionTrait}; + +use super::VideoListModel; +use crate::bilibili::{BiliClient, BiliError, Collection, CollectionItem, CollectionType, Video, VideoInfo}; +use crate::config::TEMPLATE; +use crate::utils::id_time_key; +use crate::utils::model::create_video_pages; +use crate::utils::status::Status; + +pub async fn collection_from<'a>( + collection_item: &'a CollectionItem, + path: &Path, + bili_client: &'a BiliClient, + connection: &DatabaseConnection, +) -> Result<(Box, Pin + 'a>>)> { + let collection = Collection::new(bili_client, collection_item); + let collection_info = collection.get_info().await?; + collection::Entity::insert(collection::ActiveModel { + s_id: Set(collection_info.sid), + m_id: Set(collection_info.mid), + r#type: Set(collection_info.collection_type.into()), + name: Set(collection_info.name.clone()), + path: Set(path.to_string_lossy().to_string()), + ..Default::default() + }) + .on_conflict( + OnConflict::columns([ + collection::Column::SId, + collection::Column::MId, + collection::Column::Type, + ]) + .update_columns([collection::Column::Name, collection::Column::Path]) + .to_owned(), + ) + .exec(connection) + .await?; + Ok(( + Box::new( + collection::Entity::find() + .filter( + collection::Column::SId + .eq(collection_item.sid.clone()) + .and(collection::Column::MId.eq(collection_item.mid.clone())) + .and(collection::Column::Type.eq(Into::::into(collection_item.collection_type.clone()))), + ) + .one(connection) + .await? + .unwrap(), + ), + Box::pin(collection.into_simple_video_stream()), + )) +} +use async_trait::async_trait; + +#[async_trait] +impl VideoListModel for collection::Model { + async fn video_count(&self, connection: &DatabaseConnection) -> Result { + Ok(video::Entity::find() + .filter(video::Column::CollectionId.eq(self.id)) + .count(connection) + .await?) + } + + async fn unfilled_videos(&self, connection: &DatabaseConnection) -> Result> { + Ok(video::Entity::find() + .filter( + video::Column::CollectionId + .eq(self.id) + .and(video::Column::Valid.eq(true)) + .and(video::Column::DownloadStatus.eq(0)) + .and(video::Column::Category.eq(2)) + .and(video::Column::SinglePage.is_null()), + ) + .all(connection) + .await?) + } + + async fn unhandled_video_pages( + &self, + connection: &DatabaseConnection, + ) -> Result)>> { + Ok(video::Entity::find() + .filter( + video::Column::CollectionId + .eq(self.id) + .and(video::Column::Valid.eq(true)) + .and(video::Column::DownloadStatus.lt(Status::handled())) + .and(video::Column::Category.eq(2)) + .and(video::Column::SinglePage.is_not_null()), + ) + .find_with_related(page::Entity) + .all(connection) + .await?) + } + + async fn exist_labels( + &self, + videos_info: &[VideoInfo], + connection: &DatabaseConnection, + ) -> Result> { + let bvids = videos_info.iter().map(|v| v.bvid().to_string()).collect::>(); + Ok(video::Entity::find() + .filter( + video::Column::CollectionId + .eq(self.id) + .and(video::Column::Bvid.is_in(bvids)), + ) + .select_only() + .columns([video::Column::Bvid, video::Column::Favtime]) + .into_tuple() + .all(connection) + .await? + .into_iter() + .map(|(bvid, time)| id_time_key(&bvid, &time)) + .collect::>()) + } + + fn video_model_by_info(&self, video_info: &VideoInfo, base_model: Option) -> video::ActiveModel { + let mut video_model = video_info.to_model(base_model); + video_model.collection_id = Set(Some(self.id)); + if let Some(fmt_args) = &video_info.to_fmt_args() { + video_model.path = Set(Path::new(&self.path) + .join(filenamify( + TEMPLATE + .render("video", fmt_args) + .unwrap_or_else(|_| video_info.bvid().to_string()), + )) + .to_string_lossy() + .to_string()); + } + video_model + } + + async fn fetch_videos_detail( + &self, + bili_clent: &BiliClient, + videos_model: Vec, + connection: &DatabaseConnection, + ) -> Result<()> { + for video_model in videos_model { + let video = Video::new(bili_clent, video_model.bvid.clone()); + let info: Result<_> = async { Ok((video.get_tags().await?, video.get_view_info().await?)) }.await; + match info { + Ok((tags, view_info)) => { + let VideoInfo::View { pages, .. } = &view_info else { + unreachable!("view_info must be VideoInfo::View") + }; + let txn = connection.begin().await?; + // 将分页信息写入数据库 + create_video_pages(pages, &video_model, &txn).await?; + // 将页标记和 tag 写入数据库 + let mut video_active_model = self.video_model_by_info(&view_info, Some(video_model)); + video_active_model.single_page = Set(Some(pages.len() == 1)); + video_active_model.tags = Set(Some(serde_json::to_value(tags).unwrap())); + video_active_model.save(&txn).await?; + txn.commit().await?; + } + Err(e) => { + error!( + "获取视频 {} - {} 的详细信息失败，错误为：{}", + &video_model.bvid, &video_model.name, e + ); + if let Some(BiliError::RequestFailed(-404, _)) = e.downcast_ref::() { + let mut video_active_model: video::ActiveModel = video_model.into(); + video_active_model.valid = Set(false); + video_active_model.save(connection).await?; + } + continue; + } + }; + } + Ok(()) + } + + fn log_fetch_video_start(&self) { + info!( + "开始获取{} {} - {} 的视频与分页信息...", + CollectionType::from(self.r#type), + self.s_id, + self.name + ); + } + + fn log_fetch_video_end(&self) { + info!( + "获取{} {} - {} 的视频与分页信息完成", + CollectionType::from(self.r#type), + self.s_id, + self.name + ); + } + + fn log_download_video_start(&self) { + info!( + "开始下载{}: {} - {} 中所有未处理过的视频...", + CollectionType::from(self.r#type), + self.s_id, + self.name + ); + } + + fn log_download_video_end(&self) { + info!( + "下载{}: {} - {} 中未处理过的视频完成", + CollectionType::from(self.r#type), + self.s_id, + self.name + ); + } + + fn log_refresh_video_start(&self) { + info!( + "开始扫描{}: {} - {} 的新视频...", + CollectionType::from(self.r#type), + self.s_id, + self.name + ); + } + + fn log_refresh_video_end(&self, got_count: usize, new_count: u64) { + info!( + "扫描{}: {} - {} 的新视频完成，获取了 {} 条新视频，其中有 {} 条新视频", + CollectionType::from(self.r#type), + self.s_id, + self.name, + got_count, + new_count, + ); + } +} diff --git a/crates/bili_sync/src/adapter/favorite.rs b/crates/bili_sync/src/adapter/favorite.rs new file mode 100644 index 0000000..6ebc55b --- /dev/null +++ b/crates/bili_sync/src/adapter/favorite.rs @@ -0,0 +1,199 @@ +use std::collections::HashSet; +use std::path::Path; +use std::pin::Pin; + +use anyhow::Result; +use bili_sync_entity::*; +use bili_sync_migration::OnConflict; +use filenamify::filenamify; +use futures::Stream; +use sea_orm::entity::prelude::*; +use sea_orm::ActiveValue::Set; +use sea_orm::{DatabaseConnection, QuerySelect, TransactionTrait}; + +use super::VideoListModel; +use crate::bilibili::{BiliClient, BiliError, FavoriteList, Video, VideoInfo}; +use crate::config::TEMPLATE; +use crate::utils::id_time_key; +use crate::utils::model::create_video_pages; +use crate::utils::status::Status; + +pub async fn favorite_from<'a>( + fid: &str, + path: &Path, + bili_client: &'a BiliClient, + connection: &DatabaseConnection, +) -> Result<(Box, Pin + 'a>>)> { + let favorite = FavoriteList::new(bili_client, fid.to_owned()); + let favorite_info = favorite.get_info().await?; + favorite::Entity::insert(favorite::ActiveModel { + f_id: Set(favorite_info.id), + name: Set(favorite_info.title.clone()), + path: Set(path.to_string_lossy().to_string()), + ..Default::default() + }) + .on_conflict( + OnConflict::column(favorite::Column::FId) + .update_columns([favorite::Column::Name, favorite::Column::Path]) + .to_owned(), + ) + .exec(connection) + .await?; + Ok(( + Box::new( + favorite::Entity::find() + .filter(favorite::Column::FId.eq(favorite_info.id)) + .one(connection) + .await? + .unwrap(), + ), + Box::pin(favorite.into_video_stream()), + )) +} + +use async_trait::async_trait; + +#[async_trait] +impl VideoListModel for favorite::Model { + async fn video_count(&self, connection: &DatabaseConnection) -> Result { + Ok(video::Entity::find() + .filter(video::Column::FavoriteId.eq(self.id)) + .count(connection) + .await?) + } + + async fn unfilled_videos(&self, connection: &DatabaseConnection) -> Result> { + Ok(video::Entity::find() + .filter( + video::Column::FavoriteId + .eq(self.id) + .and(video::Column::Valid.eq(true)) + .and(video::Column::DownloadStatus.eq(0)) + .and(video::Column::Category.eq(2)) + .and(video::Column::SinglePage.is_null()), + ) + .all(connection) + .await?) + } + + async fn unhandled_video_pages( + &self, + connection: &DatabaseConnection, + ) -> Result)>> { + Ok(video::Entity::find() + .filter( + video::Column::FavoriteId + .eq(self.id) + .and(video::Column::Valid.eq(true)) + .and(video::Column::DownloadStatus.lt(Status::handled())) + .and(video::Column::Category.eq(2)) + .and(video::Column::SinglePage.is_not_null()), + ) + .find_with_related(page::Entity) + .all(connection) + .await?) + } + + async fn exist_labels( + &self, + videos_info: &[VideoInfo], + connection: &DatabaseConnection, + ) -> Result> { + let bvids = videos_info.iter().map(|v| v.bvid().to_string()).collect::>(); + Ok(video::Entity::find() + .filter( + video::Column::FavoriteId + .eq(self.id) + .and(video::Column::Bvid.is_in(bvids)), + ) + .select_only() + .columns([video::Column::Bvid, video::Column::Favtime]) + .into_tuple() + .all(connection) + .await? + .into_iter() + .map(|(bvid, time)| id_time_key(&bvid, &time)) + .collect::>()) + } + + fn video_model_by_info(&self, video_info: &VideoInfo, base_model: Option) -> video::ActiveModel { + let mut video_model = video_info.to_model(base_model); + video_model.favorite_id = Set(Some(self.id)); + if let Some(fmt_args) = &video_info.to_fmt_args() { + video_model.path = Set(Path::new(&self.path) + .join(filenamify( + TEMPLATE + .render("video", fmt_args) + .unwrap_or_else(|_| video_info.bvid().to_string()), + )) + .to_string_lossy() + .to_string()); + } + video_model + } + + async fn fetch_videos_detail( + &self, + bili_clent: &BiliClient, + videos_model: Vec, + connection: &DatabaseConnection, + ) -> Result<()> { + for video_model in videos_model { + let video = Video::new(bili_clent, video_model.bvid.clone()); + let info: Result<_> = async { Ok((video.get_tags().await?, video.get_pages().await?)) }.await; + match info { + Ok((tags, pages_info)) => { + let txn = connection.begin().await?; + // 将分页信息写入数据库 + create_video_pages(&pages_info, &video_model, &txn).await?; + // 将页标记和 tag 写入数据库 + let mut video_active_model: video::ActiveModel = video_model.into(); + video_active_model.single_page = Set(Some(pages_info.len() == 1)); + video_active_model.tags = Set(Some(serde_json::to_value(tags).unwrap())); + video_active_model.save(&txn).await?; + txn.commit().await?; + } + Err(e) => { + error!( + "获取视频 {} - {} 的详细信息失败，错误为：{}", + &video_model.bvid, &video_model.name, e + ); + if let Some(BiliError::RequestFailed(-404, _)) = e.downcast_ref::() { + let mut video_active_model: video::ActiveModel = video_model.into(); + video_active_model.valid = Set(false); + video_active_model.save(connection).await?; + } + continue; + } + }; + } + Ok(()) + } + + fn log_fetch_video_start(&self) { + info!("开始获取收藏夹 {} - {} 的视频与分页信息...", self.f_id, self.name); + } + + fn log_fetch_video_end(&self) { + info!("获取收藏夹 {} - {} 的视频与分页信息完成", self.f_id, self.name); + } + + fn log_download_video_start(&self) { + info!("开始下载收藏夹: {} - {} 中所有未处理过的视频...", self.f_id, self.name); + } + + fn log_download_video_end(&self) { + info!("下载收藏夹: {} - {} 中未处理过的视频完成", self.f_id, self.name); + } + + fn log_refresh_video_start(&self) { + info!("开始扫描收藏夹: {} - {} 的新视频...", self.f_id, self.name); + } + + fn log_refresh_video_end(&self, got_count: usize, new_count: u64) { + info!( + "扫描收藏夹: {} - {} 的新视频完成，获取了 {} 条新视频，其中有 {} 条新视频", + self.f_id, self.name, got_count, new_count + ); + } +} diff --git a/crates/bili_sync/src/adapter/mod.rs b/crates/bili_sync/src/adapter/mod.rs new file mode 100644 index 0000000..29250b6 --- /dev/null +++ b/crates/bili_sync/src/adapter/mod.rs @@ -0,0 +1,89 @@ +mod collection; +mod favorite; + +use std::collections::HashSet; +use std::path::Path; +use std::pin::Pin; + +use anyhow::Result; +use async_trait::async_trait; +use bili_sync_migration::IntoIden; +pub use collection::collection_from; +pub use favorite::favorite_from; +use futures::Stream; +use sea_orm::DatabaseConnection; + +use crate::bilibili::{BiliClient, CollectionItem, VideoInfo}; + +pub enum Args<'a> { + Favorite { fid: &'a str }, + Collection { collection_item: &'a CollectionItem }, +} + +pub async fn video_list_from<'a>( + args: Args<'a>, + path: &Path, + bili_client: &'a BiliClient, + connection: &DatabaseConnection, +) -> Result<(Box, Pin + 'a>>)> { + match args { + Args::Favorite { fid } => favorite_from(fid, path, bili_client, connection).await, + Args::Collection { collection_item } => collection_from(collection_item, path, bili_client, connection).await, + } +} + +pub const fn unique_video_columns() -> impl IntoIterator { + [ + bili_sync_entity::video::Column::CollectionId, + bili_sync_entity::video::Column::FavoriteId, + bili_sync_entity::video::Column::Bvid, + ] +} + +#[async_trait] +pub trait VideoListModel { + /* 逻辑相关 */ + + async fn video_count(&self, connection: &DatabaseConnection) -> Result; + + /// 获取未填充的视频 + async fn unfilled_videos(&self, connection: &DatabaseConnection) -> Result>; + + /// 获取未处理的视频和分页 + async fn unhandled_video_pages( + &self, + connection: &DatabaseConnection, + ) -> Result)>>; + + /// 获取该批次视频的存在标记 + async fn exist_labels(&self, videos_info: &[VideoInfo], connection: &DatabaseConnection) + -> Result>; + + /// 获取视频信息对应的视频 model + fn video_model_by_info( + &self, + video_info: &VideoInfo, + base_model: Option, + ) -> bili_sync_entity::video::ActiveModel; + + /// 获取视频 model 中缺失的信息 + async fn fetch_videos_detail( + &self, + bili_client: &BiliClient, + videos_model: Vec, + connection: &DatabaseConnection, + ) -> Result<()>; + + /* 日志相关 */ + fn log_fetch_video_start(&self); + + fn log_fetch_video_end(&self); + + fn log_download_video_start(&self); + + fn log_download_video_end(&self); + + fn log_refresh_video_start(&self); + + fn log_refresh_video_end(&self, got_count: usize, new_count: u64); +} diff --git a/crates/bili_sync/src/bilibili/collection.rs b/crates/bili_sync/src/bilibili/collection.rs new file mode 100644 index 0000000..3360e90 --- /dev/null +++ b/crates/bili_sync/src/bilibili/collection.rs @@ -0,0 +1,265 @@ +#![allow(dead_code)] + +use std::fmt::{Display, Formatter}; + +use anyhow::Result; +use async_stream::stream; +use futures::Stream; +use reqwest::Method; +use serde::Deserialize; +use serde_json::Value; + +use crate::bilibili::{BiliClient, Validate, VideoInfo}; + +#[derive(PartialEq, Eq, Hash, Clone, Debug)] +pub enum CollectionType { + Series, + Season, +} + +impl From for i32 { + fn from(v: CollectionType) -> Self { + match v { + CollectionType::Series => 1, + CollectionType::Season => 2, + } + } +} + +impl From for CollectionType { + fn from(v: i32) -> Self { + match v { + 1 => CollectionType::Series, + 2 => CollectionType::Season, + _ => panic!("invalid collection type"), + } + } +} + +impl Display for CollectionType { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let s = match self { + CollectionType::Series => "视频列表", + CollectionType::Season => "视频合集", + }; + write!(f, "{}", s) + } +} + +#[derive(PartialEq, Eq, Hash, Debug)] +pub struct CollectionItem { + pub mid: String, + pub sid: String, + pub collection_type: CollectionType, +} + +pub struct Collection<'a> { + client: &'a BiliClient, + collection: &'a CollectionItem, +} + +#[derive(Debug, PartialEq)] +pub struct CollectionInfo { + pub name: String, + pub mid: i64, + pub sid: i64, + pub collection_type: CollectionType, +} + +impl<'de> Deserialize<'de> for CollectionInfo { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + #[derive(Deserialize)] + struct CollectionInfoRaw { + mid: i64, + name: String, + season_id: Option, + series_id: Option, + } + let raw = CollectionInfoRaw::deserialize(deserializer)?; + let (sid, collection_type) = match (raw.season_id, raw.series_id) { + (Some(sid), None) => (sid, CollectionType::Season), + (None, Some(sid)) => (sid, CollectionType::Series), + _ => return Err(serde::de::Error::custom("invalid collection info")), + }; + Ok(CollectionInfo { + mid: raw.mid, + name: raw.name, + sid, + collection_type, + }) + } +} + +impl<'a> Collection<'a> { + pub fn new(client: &'a BiliClient, collection: &'a CollectionItem) -> Self { + Self { client, collection } + } + + pub async fn get_info(&self) -> Result { + let meta = match self.collection.collection_type { + // 没有找到专门获取 Season 信息的接口，所以直接获取第一页，从里面取 meta 信息 + CollectionType::Season => self.get_videos(1).await?["data"]["meta"].take(), + CollectionType::Series => self.get_series_info().await?["data"]["meta"].take(), + }; + Ok(serde_json::from_value(meta)?) + } + + async fn get_series_info(&self) -> Result { + assert!( + self.collection.collection_type == CollectionType::Series, + "collection type is not series" + ); + self.client + .request(Method::GET, "https://api.bilibili.com/x/series/series") + .query(&[("series_id", self.collection.sid.as_str())]) + .send() + .await? + .error_for_status()? + .json::() + .await? + .validate() + } + + async fn get_videos(&self, page: i32) -> Result { + let page = page.to_string(); + let (url, query) = match self.collection.collection_type { + CollectionType::Series => ( + "https://api.bilibili.com/x/series/archives", + vec![ + ("mid", self.collection.mid.as_str()), + ("series_id", self.collection.sid.as_str()), + ("only_normal", "true"), + ("sort", "desc"), + ("pn", page.as_str()), + ("ps", "30"), + ], + ), + CollectionType::Season => ( + "https://api.bilibili.com/x/polymer/web-space/seasons_archives_list", + vec![ + ("mid", self.collection.mid.as_str()), + ("season_id", self.collection.sid.as_str()), + ("sort_reverse", "true"), + ("page_num", page.as_str()), + ("page_size", "30"), + ], + ), + }; + self.client + .request(Method::GET, url) + .query(&query) + .send() + .await? + .error_for_status()? + .json::() + .await? + .validate() + } + + pub fn into_simple_video_stream(self) -> impl Stream + 'a { + stream! { + let mut page = 1; + loop { + let mut videos = match self.get_videos(page).await { + Ok(v) => v, + Err(e) => { + error!("failed to get videos of collection {:?} page {}: {}", self.collection, page, e); + break; + }, + }; + if !videos["data"]["archives"].is_array() { + warn!("no videos found in collection {:?} page {}", self.collection, page); + break; + } + let videos_info = match serde_json::from_value::>(videos["data"]["archives"].take()) { + Ok(v) => v, + Err(e) => { + error!("failed to parse videos of collection {:?} page {}: {}", self.collection, page, e); + break; + }, + }; + for video_info in videos_info.into_iter(){ + yield video_info; + } + let fields = match self.collection.collection_type{ + CollectionType::Series => ["num", "size", "total"], + CollectionType::Season => ["page_num", "page_size", "total"], + }; + let fields = fields.into_iter().map(|f| videos["data"]["page"][f].as_i64()).collect::>>().map(|v| (v[0], v[1], v[2])); + let Some((num, size, total)) = fields else { + error!("failed to get pages of collection {:?} page {}: {:?}", self.collection, page, fields); + break; + }; + if num * size >= total { + break; + } + page += 1; + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_collection_info_parse() { + let testcases = vec![ + ( + r#" + { + "category": 0, + "cover": "https://archive.biliimg.com/bfs/archive/a6fbf7a7b9f4af09d9cf40482268634df387ef68.jpg", + "description": "", + "mid": 521722088, + "name": "合集·【命运方舟全剧情解说】", + "ptime": 1714701600, + "season_id": 1987140, + "total": 10 + } + "#, + CollectionInfo { + mid: 521722088, + name: "合集·【命运方舟全剧情解说】".to_owned(), + sid: 1987140, + collection_type: CollectionType::Season, + }, + ), + ( + r#" + { + "series_id": 387212, + "mid": 521722088, + "name": "提瓦特冒险记", + "description": "原神沙雕般的游戏体验", + "keywords": [ + "" + ], + "creator": "", + "state": 2, + "last_update_ts": 1633167320, + "total": 3, + "ctime": 1633167320, + "mtime": 1633167320, + "raw_keywords": "", + "category": 1 + } + "#, + CollectionInfo { + mid: 521722088, + name: "提瓦特冒险记".to_owned(), + sid: 387212, + collection_type: CollectionType::Series, + }, + ), + ]; + for (json, expect) in testcases { + let info: CollectionInfo = serde_json::from_str(json).unwrap(); + assert_eq!(info, expect); + } + } +} diff --git a/crates/bili_sync/src/bilibili/favorite_list.rs b/crates/bili_sync/src/bilibili/favorite_list.rs index 55e9c33..211eb65 100644 --- a/crates/bili_sync/src/bilibili/favorite_list.rs +++ b/crates/bili_sync/src/bilibili/favorite_list.rs @@ -1,11 +1,9 @@ use anyhow::Result; use async_stream::stream; -use chrono::serde::ts_seconds; -use chrono::{DateTime, Utc}; use futures::Stream; use serde_json::Value; -use crate::bilibili::{BiliClient, Validate}; +use crate::bilibili::{BiliClient, Validate, VideoInfo}; pub struct FavoriteList<'a> { client: &'a BiliClient, fid: String, @@ -17,24 +15,6 @@ pub struct FavoriteListInfo { pub title: String, } -#[derive(Debug, serde::Deserialize)] -pub struct VideoInfo { - pub title: String, - #[serde(rename = "type")] - pub vtype: i32, - pub bvid: String, - pub intro: String, - pub cover: String, - pub upper: Upper, - #[serde(with = "ts_seconds")] - pub ctime: DateTime, - #[serde(with = "ts_seconds")] - pub fav_time: DateTime, - #[serde(with = "ts_seconds")] - pub pubtime: DateTime, - pub attr: i32, -} - #[derive(Debug, serde::Deserialize)] pub struct Upper { pub mid: i64, diff --git a/crates/bili_sync/src/bilibili/mod.rs b/crates/bili_sync/src/bilibili/mod.rs index 12d771c..45567a7 100644 --- a/crates/bili_sync/src/bilibili/mod.rs +++ b/crates/bili_sync/src/bilibili/mod.rs @@ -1,14 +1,19 @@ pub use analyzer::{BestStream, FilterOption}; use anyhow::{bail, Result}; +use chrono::serde::ts_seconds; +use chrono::{DateTime, Utc}; pub use client::{BiliClient, Client}; +pub use collection::{Collection, CollectionItem, CollectionType}; pub use credential::Credential; pub use danmaku::DanmakuOption; pub use error::BiliError; -pub use favorite_list::{FavoriteList, FavoriteListInfo, VideoInfo}; +pub use favorite_list::FavoriteList; +use favorite_list::Upper; pub use video::{Dimension, PageInfo, Video}; mod analyzer; mod client; +mod collection; mod credential; mod danmaku; mod error; @@ -35,3 +40,55 @@ impl Validate for serde_json::Value { Ok(self) } } + +#[derive(Debug, serde::Deserialize)] +#[serde(untagged)] +/// 注意此处的顺序是有要求的，因为对于 untagged 的 enum 来说，serde 会按照顺序匹配 +/// > There is no explicit tag identifying which variant the data contains. +/// > Serde will try to match the data against each variant in order and the first one that deserializes successfully is the one returned. +pub enum VideoInfo { + /// 从视频详情接口获取的视频信息 + View { + title: String, + bvid: String, + #[serde(rename = "desc")] + intro: String, + #[serde(rename = "pic")] + cover: String, + #[serde(rename = "owner")] + upper: Upper, + #[serde(with = "ts_seconds")] + ctime: DateTime, + #[serde(rename = "pubdate", with = "ts_seconds")] + pubtime: DateTime, + pages: Vec, + state: i32, + }, + /// 从收藏夹中获取的视频信息 + Detail { + title: String, + #[serde(rename = "type")] + vtype: i32, + bvid: String, + intro: String, + cover: String, + upper: Upper, + #[serde(with = "ts_seconds")] + ctime: DateTime, + #[serde(with = "ts_seconds")] + fav_time: DateTime, + #[serde(with = "ts_seconds")] + pubtime: DateTime, + attr: i32, + }, + /// 从视频列表中获取的视频信息 + Simple { + bvid: String, + #[serde(rename = "pic")] + cover: String, + #[serde(with = "ts_seconds")] + ctime: DateTime, + #[serde(rename = "pubdate", with = "ts_seconds")] + pubtime: DateTime, + }, +} diff --git a/crates/bili_sync/src/bilibili/video.rs b/crates/bili_sync/src/bilibili/video.rs index d2798db..3ac460b 100644 --- a/crates/bili_sync/src/bilibili/video.rs +++ b/crates/bili_sync/src/bilibili/video.rs @@ -7,7 +7,7 @@ use reqwest::Method; use crate::bilibili::analyzer::PageAnalyzer; use crate::bilibili::client::BiliClient; use crate::bilibili::danmaku::{DanmakuElem, DanmakuWriter, DmSegMobileReply}; -use crate::bilibili::Validate; +use crate::bilibili::{Validate, VideoInfo}; static MASK_CODE: u64 = 2251799813685247; static XOR_CODE: u64 = 23442827791579; @@ -61,6 +61,22 @@ impl<'a> Video<'a> { Self { client, aid, bvid } } + #[allow(dead_code)] + /// 直接调用视频信息接口获取详细的视频信息 + pub async fn get_view_info(&self) -> Result { + let mut res = self + .client + .request(Method::GET, "https://api.bilibili.com/x/web-interface/view") + .query(&[("aid", &self.aid), ("bvid", &self.bvid)]) + .send() + .await? + .error_for_status()? + .json::() + .await? + .validate()?; + Ok(serde_json::from_value(res["data"].take())?) + } + pub async fn get_pages(&self) -> Result> { let mut res = self .client @@ -158,8 +174,8 @@ fn bvid_to_aid(bvid: &str) -> u64 { mod tests { use super::*; - #[tokio::test] - async fn test_bvid_to_aid() { + #[test] + fn test_bvid_to_aid() { assert_eq!(bvid_to_aid("BV1Tr421n746"), 1401752220u64); assert_eq!(bvid_to_aid("BV1sH4y1s7fe"), 1051892992u64); } diff --git a/crates/bili_sync/src/config.rs b/crates/bili_sync/src/config.rs index 22bc4b3..63801bf 100644 --- a/crates/bili_sync/src/config.rs +++ b/crates/bili_sync/src/config.rs @@ -5,10 +5,30 @@ use std::sync::Arc; use anyhow::Result; use arc_swap::ArcSwapOption; +use handlebars::handlebars_helper; use once_cell::sync::Lazy; +use serde::de::{Deserializer, MapAccess, Visitor}; +use serde::ser::SerializeMap; use serde::{Deserialize, Serialize}; -use crate::bilibili::{Credential, DanmakuOption, FilterOption}; +use crate::bilibili::{CollectionItem, CollectionType, Credential, DanmakuOption, FilterOption}; + +pub static TEMPLATE: Lazy = Lazy::new(|| { + let mut handlebars = handlebars::Handlebars::new(); + handlebars_helper!(truncate: |s: String, len: usize| { + if s.chars().count() > len { + s.chars().take(len).collect::() + } else { + s.to_string() + } + }); + handlebars.register_helper("truncate", Box::new(truncate)); + handlebars + .register_template_string("video", &CONFIG.video_name) + .unwrap(); + handlebars.register_template_string("page", &CONFIG.page_name).unwrap(); + handlebars +}); pub static CONFIG: Lazy = Lazy::new(|| { let config = Config::load().unwrap_or_else(|err| { @@ -42,6 +62,12 @@ pub struct Config { #[serde(default)] pub danmaku_option: DanmakuOption, pub favorite_list: HashMap, + #[serde( + default, + serialize_with = "serialize_collection_list", + deserialize_with = "deserialize_collection_list" + )] + pub collection_list: HashMap, pub video_name: Cow<'static, str>, pub page_name: Cow<'static, str>, pub interval: u64, @@ -65,6 +91,7 @@ impl Default for Config { filter_option: FilterOption::default(), danmaku_option: DanmakuOption::default(), favorite_list: HashMap::new(), + collection_list: HashMap::new(), video_name: Cow::Borrowed("{{title}}"), page_name: Cow::Borrowed("{{bvid}}"), interval: 1200, @@ -78,9 +105,9 @@ impl Config { /// 简单的预检查 pub fn check(&self) { let mut ok = true; - if self.favorite_list.is_empty() { + if self.favorite_list.is_empty() && self.collection_list.is_empty() { ok = false; - error!("未设置需监听的收藏夹，程序空转没有意义"); + error!("未设置需监听的收藏夹或视频合集，程序空转没有意义"); } for path in self.favorite_list.values() { if !path.is_absolute() { @@ -141,6 +168,75 @@ impl Config { } } +fn serialize_collection_list( + collection_list: &HashMap, + serializer: S, +) -> Result +where + S: serde::Serializer, +{ + let mut map = serializer.serialize_map(Some(collection_list.len()))?; + for (k, v) in collection_list { + let prefix = match k.collection_type { + CollectionType::Series => "series", + CollectionType::Season => "season", + }; + map.serialize_entry(&[prefix, &k.mid, &k.sid].join(":"), v)?; + } + map.end() +} + +fn deserialize_collection_list<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + struct CollectionListVisitor; + + impl<'de> Visitor<'de> for CollectionListVisitor { + type Value = HashMap; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a map of collection list") + } + + fn visit_map(self, mut map: A) -> Result + where + A: MapAccess<'de>, + { + let mut collection_list = HashMap::new(); + while let Some((key, value)) = map.next_entry::()? { + let collection_item = match key.split(':').collect::>().as_slice() { + [prefix, mid, sid] => { + let collection_type = match *prefix { + "series" => CollectionType::Series, + "season" => CollectionType::Season, + _ => { + return Err(serde::de::Error::custom( + "invalid collection type, should be series or season", + )) + } + }; + CollectionItem { + mid: mid.to_string(), + sid: sid.to_string(), + collection_type, + } + } + _ => { + return Err(serde::de::Error::custom( + "invalid collection key, should be series:mid:sid or season:mid:sid", + )) + } + }; + collection_list.insert(collection_item, value); + } + Ok(collection_list) + } + } + + deserializer.deserialize_map(CollectionListVisitor) +} + use clap::Parser; #[derive(Parser)] diff --git a/crates/bili_sync/src/core/mod.rs b/crates/bili_sync/src/core/mod.rs deleted file mode 100644 index 6806150..0000000 --- a/crates/bili_sync/src/core/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod command; -pub mod status; -pub mod utils; diff --git a/crates/bili_sync/src/database.rs b/crates/bili_sync/src/database.rs index 4aecc73..c611dd7 100644 --- a/crates/bili_sync/src/database.rs +++ b/crates/bili_sync/src/database.rs @@ -1,13 +1,15 @@ use anyhow::Result; use bili_sync_migration::{Migrator, MigratorTrait}; use sea_orm::{ConnectOptions, Database, DatabaseConnection}; -use tokio::fs; use crate::config::CONFIG_DIR; + +fn database_url() -> String { + format!("sqlite://{}?mode=rwc", CONFIG_DIR.join("data.sqlite").to_string_lossy()) +} + pub async fn database_connection() -> Result { - let target = CONFIG_DIR.join("data.sqlite"); - fs::create_dir_all(&*CONFIG_DIR).await?; - let mut option = ConnectOptions::new(format!("sqlite://{}?mode=rwc", target.to_str().unwrap())); + let mut option = ConnectOptions::new(database_url()); option .max_connections(100) .min_connections(5) @@ -15,6 +17,9 @@ pub async fn database_connection() -> Result { Ok(Database::connect(option).await?) } -pub async fn migrate_database(connection: &DatabaseConnection) -> Result<()> { - Ok(Migrator::up(connection, None).await?) +pub async fn migrate_database() -> Result<()> { + // 注意此处使用内部构造的 DatabaseConnection，而不是通过 database_connection() 获取 + // 这是因为使用多个连接的 Connection 会导致奇怪的迁移顺序问题，而使用默认的连接选项不会 + let connection = Database::connect(database_url()).await?; + Ok(Migrator::up(&connection, None).await?) } diff --git a/crates/bili_sync/src/main.rs b/crates/bili_sync/src/main.rs index 035d49d..ea5d430 100644 --- a/crates/bili_sync/src/main.rs +++ b/crates/bili_sync/src/main.rs @@ -1,34 +1,34 @@ #[macro_use] extern crate tracing; +mod adapter; mod bilibili; mod config; -mod core; mod database; mod downloader; mod error; - +mod utils; +mod workflow; use std::time::Duration; -use config::ARGS; use once_cell::sync::Lazy; use tokio::time; +use crate::adapter::Args; use crate::bilibili::BiliClient; -use crate::config::CONFIG; -use crate::core::command::process_favorite_list; -use crate::core::utils::init_logger; +use crate::config::{ARGS, CONFIG}; use crate::database::{database_connection, migrate_database}; +use crate::utils::init_logger; +use crate::workflow::process_video_list; #[tokio::main] async fn main() { - Lazy::force(&ARGS); init_logger(&ARGS.log_level); Lazy::force(&CONFIG); + migrate_database().await.expect("数据库迁移失败"); + let connection = database_connection().await.expect("获取数据库连接失败"); let mut anchor = chrono::Local::now().date_naive(); let bili_client = BiliClient::new(); - let connection = database_connection().await.unwrap(); - migrate_database(&connection).await.unwrap(); loop { if let Err(e) = bili_client.is_login().await { error!("检查登录状态时遇到错误：{e}，等待下一轮执行"); @@ -44,12 +44,20 @@ async fn main() { anchor = chrono::Local::now().date_naive(); } for (fid, path) in &CONFIG.favorite_list { - if let Err(e) = process_favorite_list(&bili_client, fid, path, &connection).await { - // 可预期的错误都被内部处理了，这里漏出来应该是大问题 + if let Err(e) = process_video_list(Args::Favorite { fid }, &bili_client, path, &connection).await { error!("处理收藏夹 {fid} 时遇到非预期的错误：{e}"); } } - info!("所有收藏夹处理完毕，等待下一轮执行"); - time::sleep(Duration::from_secs(CONFIG.interval)).await; + info!("所有收藏夹处理完毕"); + for (collection_item, path) in &CONFIG.collection_list { + if let Err(e) = + process_video_list(Args::Collection { collection_item }, &bili_client, path, &connection).await + { + error!("处理合集 {collection_item:?} 时遇到非预期的错误：{e}"); + } + } + info!("所有合集处理完毕"); + info!("本轮任务执行完毕，等待下一轮执行"); + tokio::time::sleep(std::time::Duration::from_secs(CONFIG.interval)).await; } } diff --git a/crates/bili_sync/src/utils/convert.rs b/crates/bili_sync/src/utils/convert.rs new file mode 100644 index 0000000..98810ba --- /dev/null +++ b/crates/bili_sync/src/utils/convert.rs @@ -0,0 +1,131 @@ +use sea_orm::ActiveValue::NotSet; +use sea_orm::{IntoActiveModel, Set}; +use serde_json::json; + +use crate::bilibili::VideoInfo; +use crate::utils::id_time_key; + +impl VideoInfo { + /// 将 VideoInfo 转换为 ActiveModel + pub fn to_model(&self, base_model: Option) -> bili_sync_entity::video::ActiveModel { + let base_model = match base_model { + Some(base_model) => base_model.into_active_model(), + None => { + let mut tmp_model = bili_sync_entity::video::Model::default().into_active_model(); + // 注意此处要把 id 设置成 NotSet，否则 id 会是 Unchanged(0) + tmp_model.id = NotSet; + tmp_model + } + }; + match self { + VideoInfo::Simple { + bvid, + cover, + ctime, + pubtime, + } => bili_sync_entity::video::ActiveModel { + bvid: Set(bvid.clone()), + cover: Set(cover.clone()), + ctime: Set(ctime.naive_utc()), + pubtime: Set(pubtime.naive_utc()), + category: Set(2), // 视频合集里的内容类型肯定是视频 + valid: Set(true), + ..base_model + }, + VideoInfo::Detail { + title, + vtype, + bvid, + intro, + cover, + upper, + ctime, + fav_time, + pubtime, + attr, + } => bili_sync_entity::video::ActiveModel { + bvid: Set(bvid.clone()), + name: Set(title.clone()), + category: Set(*vtype), + intro: Set(intro.clone()), + cover: Set(cover.clone()), + ctime: Set(ctime.naive_utc()), + pubtime: Set(pubtime.naive_utc()), + favtime: Set(fav_time.naive_utc()), + download_status: Set(0), + valid: Set(*attr == 0), + tags: Set(None), + single_page: Set(None), + upper_id: Set(upper.mid), + upper_name: Set(upper.name.clone()), + upper_face: Set(upper.face.clone()), + ..base_model + }, + VideoInfo::View { + title, + bvid, + intro, + cover, + upper, + ctime, + pubtime, + state, + .. + } => bili_sync_entity::video::ActiveModel { + bvid: Set(bvid.clone()), + name: Set(title.clone()), + category: Set(2), // 视频合集里的内容类型肯定是视频 + intro: Set(intro.clone()), + cover: Set(cover.clone()), + ctime: Set(ctime.naive_utc()), + pubtime: Set(pubtime.naive_utc()), + favtime: Set(pubtime.naive_utc()), // 合集不包括 fav_time，使用发布时间代替 + download_status: Set(0), + valid: Set(*state == 0), + tags: Set(None), + single_page: Set(None), + upper_id: Set(upper.mid), + upper_name: Set(upper.name.clone()), + upper_face: Set(upper.face.clone()), + ..base_model + }, + } + } + + pub fn to_fmt_args(&self) -> Option { + match self { + VideoInfo::Simple { .. } => None, // 不能从简单的视频信息中构造格式化参数 + VideoInfo::Detail { title, bvid, upper, .. } => Some(json!({ + "bvid": &bvid, + "title": &title, + "upper_name": &upper.name, + "upper_mid": &upper.mid, + })), + VideoInfo::View { title, bvid, upper, .. } => Some(json!({ + "bvid": &bvid, + "title": &title, + "upper_name": &upper.name, + "upper_mid": &upper.mid, + })), + } + } + + pub fn video_key(&self) -> String { + match self { + // 对于合集没有 fav_time，只能用 pubtime 代替 + VideoInfo::Simple { bvid, pubtime, .. } => id_time_key(bvid, pubtime), + VideoInfo::Detail { bvid, fav_time, .. } => id_time_key(bvid, fav_time), + // 详情接口返回的数据仅用于填充详情，不会被作为 video_key + _ => unreachable!(), + } + } + + pub fn bvid(&self) -> &str { + match self { + VideoInfo::Simple { bvid, .. } => bvid, + VideoInfo::Detail { bvid, .. } => bvid, + // 同上 + _ => unreachable!(), + } + } +} diff --git a/crates/bili_sync/src/utils/mod.rs b/crates/bili_sync/src/utils/mod.rs new file mode 100644 index 0000000..7f8c230 --- /dev/null +++ b/crates/bili_sync/src/utils/mod.rs @@ -0,0 +1,23 @@ +pub mod convert; +pub mod model; +pub mod nfo; +pub mod status; + +use chrono::{DateTime, Utc}; +use tracing_subscriber::util::SubscriberInitExt; + +pub fn init_logger(log_level: &str) { + tracing_subscriber::fmt::Subscriber::builder() + .with_env_filter(tracing_subscriber::EnvFilter::builder().parse_lossy(log_level)) + .with_timer(tracing_subscriber::fmt::time::ChronoLocal::new( + "%Y-%m-%d %H:%M:%S%.3f".to_owned(), + )) + .finish() + .try_init() + .expect("初始化日志失败"); +} + +/// 生成视频的唯一标记，均由 bvid 和时间戳构成 +pub fn id_time_key(bvid: &String, time: &DateTime) -> String { + format!("{}-{}", bvid, time.timestamp()) +} diff --git a/crates/bili_sync/src/utils/model.rs b/crates/bili_sync/src/utils/model.rs new file mode 100644 index 0000000..cccf01c --- /dev/null +++ b/crates/bili_sync/src/utils/model.rs @@ -0,0 +1,95 @@ +use anyhow::Result; +use bili_sync_entity::*; +use bili_sync_migration::OnConflict; +use sea_orm::entity::prelude::*; +use sea_orm::ActiveValue::Set; + +use crate::adapter::{unique_video_columns, VideoListModel}; +use crate::bilibili::{PageInfo, VideoInfo}; + +/// 尝试创建 Video Model，如果发生冲突则忽略 +pub async fn create_videos( + videos_info: &[VideoInfo], + video_list_model: &dyn VideoListModel, + connection: &DatabaseConnection, +) -> Result<()> { + let video_models = videos_info + .iter() + .map(|v| video_list_model.video_model_by_info(v, None)) + .collect::>(); + video::Entity::insert_many(video_models) + .on_conflict(OnConflict::columns(unique_video_columns()).do_nothing().to_owned()) + .do_nothing() + .exec(connection) + .await?; + Ok(()) +} + +/// 创建视频的所有分 P +pub async fn create_video_pages( + pages_info: &[PageInfo], + video_model: &video::Model, + connection: &impl ConnectionTrait, +) -> Result<()> { + let page_models = pages_info + .iter() + .map(move |p| { + let (width, height) = match &p.dimension { + Some(d) => { + if d.rotate == 0 { + (Some(d.width), Some(d.height)) + } else { + (Some(d.height), Some(d.width)) + } + } + None => (None, None), + }; + page::ActiveModel { + video_id: Set(video_model.id), + cid: Set(p.cid), + pid: Set(p.page), + name: Set(p.name.clone()), + width: Set(width), + height: Set(height), + duration: Set(p.duration), + image: Set(p.first_frame.clone()), + download_status: Set(0), + ..Default::default() + } + }) + .collect::>(); + page::Entity::insert_many(page_models) + .on_conflict( + OnConflict::columns([page::Column::VideoId, page::Column::Pid]) + .do_nothing() + .to_owned(), + ) + .do_nothing() + .exec(connection) + .await?; + Ok(()) +} + +/// 更新视频 model 的下载状态 +pub async fn update_videos_model(videos: Vec, connection: &DatabaseConnection) -> Result<()> { + video::Entity::insert_many(videos) + .on_conflict( + OnConflict::column(video::Column::Id) + .update_column(video::Column::DownloadStatus) + .to_owned(), + ) + .exec(connection) + .await?; + Ok(()) +} + +/// 更新视频页 model 的下载状态 +pub async fn update_pages_model(pages: Vec, connection: &DatabaseConnection) -> Result<()> { + let query = page::Entity::insert_many(pages).on_conflict( + OnConflict::column(page::Column::Id) + .update_columns([page::Column::DownloadStatus, page::Column::Path]) + .to_owned(), + ); + query.exec(connection).await?; + Ok(()) +} diff --git a/crates/bili_sync/src/core/utils.rs b/crates/bili_sync/src/utils/nfo.rs similarity index 60% rename from crates/bili_sync/src/core/utils.rs rename to crates/bili_sync/src/utils/nfo.rs index f4a099f..8b4e03e 100644 --- a/crates/bili_sync/src/core/utils.rs +++ b/crates/bili_sync/src/utils/nfo.rs @@ -1,42 +1,11 @@ -use std::collections::HashSet; -use std::path::Path; - use anyhow::Result; use bili_sync_entity::*; -use bili_sync_migration::OnConflict; -use filenamify::filenamify; -use handlebars::handlebars_helper; -use once_cell::sync::Lazy; use quick_xml::events::{BytesCData, BytesText}; use quick_xml::writer::Writer; use quick_xml::Error; -use sea_orm::entity::prelude::*; -use sea_orm::ActiveValue::Set; -use sea_orm::QuerySelect; -use serde_json::json; use tokio::io::AsyncWriteExt; -use tracing_subscriber::util::SubscriberInitExt; - -use crate::bilibili::{FavoriteListInfo, PageInfo, VideoInfo}; -use crate::config::{NFOTimeType, CONFIG}; -use crate::core::status::Status; -pub static TEMPLATE: Lazy = Lazy::new(|| { - let mut handlebars = handlebars::Handlebars::new(); - handlebars_helper!(truncate: |s: String, len: usize| { - if s.chars().count() > len { - s.chars().take(len).collect::() - } else { - s.to_string() - } - }); - handlebars.register_helper("truncate", Box::new(truncate)); - handlebars - .register_template_string("video", &CONFIG.video_name) - .unwrap(); - handlebars.register_template_string("page", &CONFIG.page_name).unwrap(); - handlebars -}); +use crate::config::NFOTimeType; #[allow(clippy::upper_case_acronyms)] pub enum NFOMode { @@ -53,225 +22,6 @@ pub enum ModelWrapper<'a> { pub struct NFOSerializer<'a>(pub ModelWrapper<'a>, pub NFOMode); -/// 根据获得的收藏夹信息，插入或更新数据库中的收藏夹，并返回收藏夹对象 -pub async fn handle_favorite_info( - info: &FavoriteListInfo, - path: &Path, - connection: &DatabaseConnection, -) -> Result { - favorite::Entity::insert(favorite::ActiveModel { - f_id: Set(info.id), - name: Set(info.title.clone()), - path: Set(path.to_string_lossy().to_string()), - ..Default::default() - }) - .on_conflict( - OnConflict::column(favorite::Column::FId) - .update_columns([favorite::Column::Name, favorite::Column::Path]) - .to_owned(), - ) - .exec(connection) - .await?; - Ok(favorite::Entity::find() - .filter(favorite::Column::FId.eq(info.id)) - .one(connection) - .await? - .unwrap()) -} - -/// 获取数据库中存在的与该视频 favorite_id 和 bvid 重合的视频中的 bvid 和 favtime -/// 如果 bvid 和 favtime 均相同，说明到达了上次处理到的位置 -pub async fn exist_labels( - videos_info: &[VideoInfo], - favorite_model: &favorite::Model, - connection: &DatabaseConnection, -) -> Result> { - let bvids = videos_info.iter().map(|v| v.bvid.clone()).collect::>(); - let exist_labels = video::Entity::find() - .filter( - video::Column::FavoriteId - .eq(favorite_model.id) - .and(video::Column::Bvid.is_in(bvids)), - ) - .select_only() - .columns([video::Column::Bvid, video::Column::Favtime]) - .into_tuple() - .all(connection) - .await? - .into_iter() - .collect::>(); - Ok(exist_labels) -} - -/// 尝试创建 Video Model，如果发生冲突则忽略 -pub async fn create_videos( - videos_info: &[VideoInfo], - favorite: &favorite::Model, - connection: &DatabaseConnection, -) -> Result<()> { - let video_models = videos_info - .iter() - .map(move |v| video::ActiveModel { - favorite_id: Set(favorite.id), - bvid: Set(v.bvid.clone()), - name: Set(v.title.clone()), - path: Set(Path::new(&favorite.path) - .join(filenamify( - TEMPLATE - .render( - "video", - &json!({ - "bvid": &v.bvid, - "title": &v.title, - "upper_name": &v.upper.name, - "upper_mid": &v.upper.mid, - }), - ) - .unwrap_or_else(|_| v.bvid.clone()), - )) - .to_str() - .unwrap() - .to_owned()), - category: Set(v.vtype), - intro: Set(v.intro.clone()), - cover: Set(v.cover.clone()), - ctime: Set(v.ctime.naive_utc()), - pubtime: Set(v.pubtime.naive_utc()), - favtime: Set(v.fav_time.naive_utc()), - download_status: Set(0), - valid: Set(v.attr == 0), - tags: Set(None), - single_page: Set(None), - upper_id: Set(v.upper.mid), - upper_name: Set(v.upper.name.clone()), - upper_face: Set(v.upper.face.clone()), - ..Default::default() - }) - .collect::>(); - video::Entity::insert_many(video_models) - .on_conflict( - OnConflict::columns([video::Column::FavoriteId, video::Column::Bvid]) - .do_nothing() - .to_owned(), - ) - .do_nothing() - .exec(connection) - .await?; - Ok(()) -} - -pub async fn total_video_count(favorite_model: &favorite::Model, connection: &DatabaseConnection) -> Result { - Ok(video::Entity::find() - .filter(video::Column::FavoriteId.eq(favorite_model.id)) - .count(connection) - .await?) -} - -/// 筛选所有未 -pub async fn filter_unfilled_videos( - favorite_model: &favorite::Model, - connection: &DatabaseConnection, -) -> Result> { - Ok(video::Entity::find() - .filter( - video::Column::FavoriteId - .eq(favorite_model.id) - .and(video::Column::Valid.eq(true)) - .and(video::Column::DownloadStatus.eq(0)) - .and(video::Column::Category.eq(2)) - .and(video::Column::SinglePage.is_null()), - ) - .all(connection) - .await?) -} - -/// 创建视频的所有分 P -pub async fn create_video_pages( - pages_info: &[PageInfo], - video_model: &video::Model, - connection: &impl ConnectionTrait, -) -> Result<()> { - let page_models = pages_info - .iter() - .map(move |p| { - let (width, height) = match &p.dimension { - Some(d) => { - if d.rotate == 0 { - (Some(d.width), Some(d.height)) - } else { - (Some(d.height), Some(d.width)) - } - } - None => (None, None), - }; - page::ActiveModel { - video_id: Set(video_model.id), - cid: Set(p.cid), - pid: Set(p.page), - name: Set(p.name.clone()), - width: Set(width), - height: Set(height), - duration: Set(p.duration), - image: Set(p.first_frame.clone()), - download_status: Set(0), - ..Default::default() - } - }) - .collect::>(); - page::Entity::insert_many(page_models) - .on_conflict( - OnConflict::columns([page::Column::VideoId, page::Column::Pid]) - .do_nothing() - .to_owned(), - ) - .do_nothing() - .exec(connection) - .await?; - Ok(()) -} - -/// 获取所有未处理的视频和页 -pub async fn unhandled_videos_pages( - favorite_model: &favorite::Model, - connection: &DatabaseConnection, -) -> Result)>> { - Ok(video::Entity::find() - .filter( - video::Column::FavoriteId - .eq(favorite_model.id) - .and(video::Column::Valid.eq(true)) - .and(video::Column::DownloadStatus.lt(Status::handled())) - .and(video::Column::Category.eq(2)) - .and(video::Column::SinglePage.is_not_null()), - ) - .find_with_related(page::Entity) - .all(connection) - .await?) -} -/// 更新视频 model 的下载状态 -pub async fn update_videos_model(videos: Vec, connection: &DatabaseConnection) -> Result<()> { - video::Entity::insert_many(videos) - .on_conflict( - OnConflict::column(video::Column::Id) - .update_column(video::Column::DownloadStatus) - .to_owned(), - ) - .exec(connection) - .await?; - Ok(()) -} - -/// 更新视频页 model 的下载状态 -pub async fn update_pages_model(pages: Vec, connection: &DatabaseConnection) -> Result<()> { - let query = page::Entity::insert_many(pages).on_conflict( - OnConflict::column(page::Column::Id) - .update_columns([page::Column::DownloadStatus, page::Column::Path]) - .to_owned(), - ); - query.exec(connection).await?; - Ok(()) -} - /// serde xml 似乎不太好用，先这么裸着写 /// （真是又臭又长啊 impl<'a> NFOSerializer<'a> { @@ -483,17 +233,6 @@ impl<'a> NFOSerializer<'a> { } } -pub fn init_logger(log_level: &str) { - tracing_subscriber::fmt::Subscriber::builder() - .with_env_filter(tracing_subscriber::EnvFilter::builder().parse_lossy(log_level)) - .with_timer(tracing_subscriber::fmt::time::ChronoLocal::new( - "%Y-%m-%d %H:%M:%S%.3f".to_owned(), - )) - .finish() - .try_init() - .expect("初始化日志失败"); -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/bili_sync/src/core/status.rs b/crates/bili_sync/src/utils/status.rs similarity index 100% rename from crates/bili_sync/src/core/status.rs rename to crates/bili_sync/src/utils/status.rs diff --git a/crates/bili_sync/src/core/command.rs b/crates/bili_sync/src/workflow.rs similarity index 78% rename from crates/bili_sync/src/core/command.rs rename to crates/bili_sync/src/workflow.rs index 878f4fc..ea1bb25 100644 --- a/crates/bili_sync/src/core/command.rs +++ b/crates/bili_sync/src/workflow.rs @@ -1,158 +1,96 @@ +#![allow(dead_code, unused_variables)] + use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::pin::Pin; use anyhow::{bail, Result}; -use bili_sync_entity::{favorite, page, video}; +use bili_sync_entity::{page, video}; use filenamify::filenamify; use futures::stream::{FuturesOrdered, FuturesUnordered}; -use futures::{pin_mut, Future, StreamExt}; +use futures::{Future, Stream, StreamExt}; use sea_orm::entity::prelude::*; use sea_orm::ActiveValue::Set; -use sea_orm::TransactionTrait; use serde_json::json; use tokio::fs; use tokio::sync::{Mutex, Semaphore}; -use crate::bilibili::{BestStream, BiliClient, BiliError, Dimension, FavoriteList, PageInfo, Video}; -use crate::config::{ARGS, CONFIG}; -use crate::core::status::{PageStatus, VideoStatus}; -use crate::core::utils::{ - create_video_pages, create_videos, exist_labels, filter_unfilled_videos, handle_favorite_info, total_video_count, - unhandled_videos_pages, update_pages_model, update_videos_model, ModelWrapper, NFOMode, NFOSerializer, TEMPLATE, -}; +use crate::adapter::{video_list_from, Args, VideoListModel}; +use crate::bilibili::{BestStream, BiliClient, BiliError, Dimension, PageInfo, Video, VideoInfo}; +use crate::config::{ARGS, CONFIG, TEMPLATE}; use crate::downloader::Downloader; use crate::error::{DownloadAbortError, ProcessPageError}; +use crate::utils::model::{create_videos, update_pages_model, update_videos_model}; +use crate::utils::nfo::{ModelWrapper, NFOMode, NFOSerializer}; +use crate::utils::status::{PageStatus, VideoStatus}; -/// 处理某个收藏夹，首先刷新收藏夹信息，然后下载收藏夹中未下载成功的视频 -pub async fn process_favorite_list( +pub async fn process_video_list( + args: Args<'_>, bili_client: &BiliClient, - fid: &str, path: &Path, connection: &DatabaseConnection, ) -> Result<()> { - let favorite_model = refresh_favorite_list(bili_client, fid, path, connection).await?; - let favorite_model = fetch_video_details(bili_client, favorite_model, connection).await?; + let (video_list_model, video_streams) = video_list_from(args, path, bili_client, connection).await?; + let video_list_model = refresh_video_list(bili_client, video_list_model, video_streams, connection).await?; + let video_list_model = fetch_video_details(bili_client, video_list_model, connection).await?; if ARGS.scan_only { warn!("已开启仅扫描模式，跳过视频下载..."); return Ok(()); } - download_unprocessed_videos(bili_client, favorite_model, connection).await + download_unprocessed_videos(bili_client, video_list_model, connection).await } -/// 获取收藏夹 Model，从收藏夹列表中获取所有新添加的视频，将其写入数据库 -pub async fn refresh_favorite_list( - bili_client: &BiliClient, - fid: &str, - path: &Path, +/// 请求接口，获取视频列表中所有新添加的视频信息，将其写入数据库 +pub async fn refresh_video_list<'a>( + bili_client: &'a BiliClient, + video_list_model: Box, + video_streams: Pin + 'a>>, connection: &DatabaseConnection, -) -> Result { - let bili_favorite_list = FavoriteList::new(bili_client, fid.to_owned()); - let favorite_list_info = bili_favorite_list.get_info().await?; - let favorite_model = handle_favorite_info(&favorite_list_info, path, connection).await?; - info!("开始扫描收藏夹: {} - {}...", favorite_model.f_id, favorite_model.name); - // 每十个视频一组，避免太多的数据库操作 - let video_stream = bili_favorite_list.into_video_stream().chunks(10); - pin_mut!(video_stream); +) -> Result> { + video_list_model.log_refresh_video_start(); + let mut video_streams = video_streams.chunks(10); let mut got_count = 0; - let total_count = total_video_count(&favorite_model, connection).await?; - while let Some(videos_info) = video_stream.next().await { + let mut new_count = video_list_model.video_count(connection).await?; + while let Some(videos_info) = video_streams.next().await { got_count += videos_info.len(); - let exist_labels = exist_labels(&videos_info, &favorite_model, connection).await?; + let exist_labels = video_list_model.exist_labels(&videos_info, connection).await?; // 如果发现有视频的收藏时间和 bvid 和数据库中重合，说明到达了上次处理到的地方，可以直接退出 - let should_break = videos_info - .iter() - .any(|v| exist_labels.contains(&(v.bvid.clone(), v.fav_time.naive_utc()))); + let should_break = videos_info.iter().any(|v| exist_labels.contains(&v.video_key())); // 将视频信息写入数据库 - create_videos(&videos_info, &favorite_model, connection).await?; + create_videos(&videos_info, video_list_model.as_ref(), connection).await?; if should_break { info!("到达上一次处理的位置，提前中止"); break; } } - let total_count = total_video_count(&favorite_model, connection).await? - total_count; - info!( - "扫描收藏夹: {} - {} 完成, 获取了 {} 条视频, 其中有 {} 条新视频", - favorite_model.f_id, favorite_model.name, got_count, total_count - ); - Ok(favorite_model) + new_count = video_list_model.video_count(connection).await? - new_count; + video_list_model.log_refresh_video_end(got_count, new_count); + Ok(video_list_model) } -/// 筛选出所有没有获取到分页信息和 tag 的视频，请求分页信息和 tag 并写入数据库 +/// 筛选出所有未获取到全部信息的视频，尝试补充其详细信息 pub async fn fetch_video_details( bili_client: &BiliClient, - favorite_model: favorite::Model, + video_list_model: Box, connection: &DatabaseConnection, -) -> Result { - info!( - "开始获取收藏夹 {} - {} 的视频与分页信息...", - favorite_model.f_id, favorite_model.name - ); - let videos_model = filter_unfilled_videos(&favorite_model, connection).await?; - for video_model in videos_model { - let bili_video = Video::new(bili_client, video_model.bvid.clone()); - let tags = match bili_video.get_tags().await { - Ok(tags) => tags, - Err(e) => { - error!( - "获取视频 {} - {} 的标签失败，错误为：{}", - &video_model.bvid, &video_model.name, e - ); - if let Some(BiliError::RequestFailed(code, _)) = e.downcast_ref::() { - if *code == -404 { - let mut video_active_model: video::ActiveModel = video_model.into(); - video_active_model.valid = Set(false); - video_active_model.save(connection).await?; - } - } - continue; - } - }; - let pages_info = match bili_video.get_pages().await { - Ok(pages) => pages, - Err(e) => { - error!( - "获取视频 {} - {} 的分页信息失败，错误为：{}", - &video_model.bvid, &video_model.name, e - ); - if let Some(BiliError::RequestFailed(code, _)) = e.downcast_ref::() { - if *code == -404 { - let mut video_active_model: video::ActiveModel = video_model.into(); - video_active_model.valid = Set(false); - video_active_model.save(connection).await?; - } - } - continue; - } - }; - let txn = connection.begin().await?; - // 将分页信息写入数据库 - create_video_pages(&pages_info, &video_model, &txn).await?; - // 将页标记和 tag 写入数据库 - let mut video_active_model: video::ActiveModel = video_model.into(); - video_active_model.single_page = Set(Some(pages_info.len() == 1)); - video_active_model.tags = Set(Some(serde_json::to_value(tags).unwrap())); - video_active_model.save(&txn).await?; - txn.commit().await?; - } - info!( - "获取收藏夹 {} - {} 的视频与分页信息完成", - favorite_model.f_id, favorite_model.name - ); - Ok(favorite_model) +) -> Result> { + video_list_model.log_fetch_video_start(); + let videos_model = video_list_model.unfilled_videos(connection).await?; + video_list_model + .fetch_videos_detail(bili_client, videos_model, connection) + .await?; + video_list_model.log_fetch_video_end(); + Ok(video_list_model) } /// 下载所有未处理成功的视频 pub async fn download_unprocessed_videos( bili_client: &BiliClient, - favorite_model: favorite::Model, + video_list_model: Box, connection: &DatabaseConnection, ) -> Result<()> { - info!( - "开始下载收藏夹: {} - {} 中所有未处理过的视频...", - favorite_model.f_id, favorite_model.name - ); - let unhandled_videos_pages = unhandled_videos_pages(&favorite_model, connection).await?; + video_list_model.log_download_video_start(); + let unhandled_videos_pages = video_list_model.unhandled_video_pages(connection).await?; // 对于视频，允许三个同时下载（视频内还有分页、不同分页还有多种下载任务） let semaphore = Semaphore::new(3); let downloader = Downloader::new(bili_client.client.clone()); @@ -197,10 +135,7 @@ pub async fn download_unprocessed_videos( if !models.is_empty() { update_videos_model(models, connection).await?; } - info!( - "下载收藏夹: {} - {} 中未处理过的视频完成", - favorite_model.f_id, favorite_model.name - ); + video_list_model.log_download_video_end(); Ok(()) } diff --git a/crates/bili_sync_entity/Cargo.toml b/crates/bili_sync_entity/Cargo.toml index 45a4688..f16459f 100644 --- a/crates/bili_sync_entity/Cargo.toml +++ b/crates/bili_sync_entity/Cargo.toml @@ -6,5 +6,4 @@ publish = { workspace = true } [dependencies] sea-orm = { workspace = true } -serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/bili_sync_entity/src/entities/collection.rs b/crates/bili_sync_entity/src/entities/collection.rs new file mode 100644 index 0000000..620d8d3 --- /dev/null +++ b/crates/bili_sync_entity/src/entities/collection.rs @@ -0,0 +1,21 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.15 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "collection")] +pub struct Model { + #[sea_orm(primary_key)] + pub id: i32, + pub s_id: i64, + pub m_id: i64, + pub name: String, + pub r#type: i32, + pub path: String, + pub created_at: String, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/crates/bili_sync_entity/src/entities/mod.rs b/crates/bili_sync_entity/src/entities/mod.rs index 03759bb..0050fbd 100644 --- a/crates/bili_sync_entity/src/entities/mod.rs +++ b/crates/bili_sync_entity/src/entities/mod.rs @@ -2,6 +2,7 @@ pub mod prelude; +pub mod collection; pub mod favorite; pub mod page; pub mod video; diff --git a/crates/bili_sync_entity/src/entities/video.rs b/crates/bili_sync_entity/src/entities/video.rs index 47ca29f..29a0c1f 100644 --- a/crates/bili_sync_entity/src/entities/video.rs +++ b/crates/bili_sync_entity/src/entities/video.rs @@ -7,7 +7,8 @@ use sea_orm::entity::prelude::*; pub struct Model { #[sea_orm(primary_key)] pub id: i32, - pub favorite_id: i32, + pub collection_id: Option, + pub favorite_id: Option, pub upper_id: i64, pub upper_name: String, pub upper_face: String, diff --git a/crates/bili_sync_migration/src/lib.rs b/crates/bili_sync_migration/src/lib.rs index de6c550..d349cc1 100644 --- a/crates/bili_sync_migration/src/lib.rs +++ b/crates/bili_sync_migration/src/lib.rs @@ -1,12 +1,16 @@ pub use sea_orm_migration::prelude::*; mod m20240322_000001_create_table; +mod m20240505_130850_add_collection; pub struct Migrator; #[async_trait::async_trait] impl MigratorTrait for Migrator { fn migrations() -> Vec> { - vec![Box::new(m20240322_000001_create_table::Migration)] + vec![ + Box::new(m20240322_000001_create_table::Migration), + Box::new(m20240505_130850_add_collection::Migration), + ] } } diff --git a/crates/bili_sync_migration/src/m20240505_130850_add_collection.rs b/crates/bili_sync_migration/src/m20240505_130850_add_collection.rs new file mode 100644 index 0000000..c23646b --- /dev/null +++ b/crates/bili_sync_migration/src/m20240505_130850_add_collection.rs @@ -0,0 +1,187 @@ +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + let db = manager.get_connection(); + manager + .create_table( + Table::create() + .table(Collection::Table) + .if_not_exists() + .col( + ColumnDef::new(Collection::Id) + .unsigned() + .not_null() + .auto_increment() + .primary_key(), + ) + .col(ColumnDef::new(Collection::SId).unsigned().not_null()) + .col(ColumnDef::new(Collection::MId).unsigned().not_null()) + .col(ColumnDef::new(Collection::Name).string().not_null()) + .col(ColumnDef::new(Collection::Type).small_unsigned().not_null()) + .col(ColumnDef::new(Collection::Path).string().not_null()) + .col( + ColumnDef::new(Collection::CreatedAt) + .timestamp() + .default(Expr::current_timestamp()) + .not_null(), + ) + .to_owned(), + ) + .await?; + manager + .create_index( + Index::create() + .table(Collection::Table) + .name("idx_collection_sid_mid_type") + .col(Collection::SId) + .col(Collection::MId) + .col(Collection::Type) + .unique() + .to_owned(), + ) + .await?; + manager + .drop_index( + Index::drop() + .table(Video::Table) + .name("idx_video_favorite_id_bvid") + .to_owned(), + ) + .await?; + manager + .alter_table( + Table::alter() + .table(Video::Table) + .add_column(ColumnDef::new(Video::CollectionId).unsigned().null()) + .to_owned(), + ) + .await?; + manager + .alter_table( + Table::alter() + .table(Video::Table) + .add_column(ColumnDef::new(Video::TempFavoriteId).unsigned().null()) + .to_owned(), + ) + .await?; + db.execute_unprepared("UPDATE video SET temp_favorite_id = favorite_id") + .await?; + manager + .alter_table( + Table::alter() + .table(Video::Table) + .drop_column(Video::FavoriteId) + .to_owned(), + ) + .await?; + manager + .alter_table( + Table::alter() + .table(Video::Table) + .rename_column(Video::TempFavoriteId, Video::FavoriteId) + .to_owned(), + ) + .await?; + manager + .create_index( + Index::create() + .table(Video::Table) + .name("idx_video_cid_fid_bvid") + .col(Video::CollectionId) + .col(Video::FavoriteId) + .col(Video::Bvid) + .unique() + .to_owned(), + ) + .await + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + let db = manager.get_connection(); + manager + .drop_index( + Index::drop() + .table(Video::Table) + .name("idx_video_cid_fid_bvid") + .to_owned(), + ) + .await?; + db.execute_unprepared("DELETE FROM video WHERE favorite_id IS NULL") + .await?; + manager + .alter_table( + Table::alter() + .table(Video::Table) + // 向存在记录的表中添加非空列时，必须提供默认值 + .add_column(ColumnDef::new(Video::TempFavoriteId).unsigned().not_null().default(0)) + .to_owned(), + ) + .await?; + db.execute_unprepared("UPDATE video SET temp_favorite_id = favorite_id") + .await?; + manager + .alter_table( + Table::alter() + .table(Video::Table) + .drop_column(Video::FavoriteId) + .to_owned(), + ) + .await?; + manager + .alter_table( + Table::alter() + .table(Video::Table) + .rename_column(Video::TempFavoriteId, Video::FavoriteId) + .to_owned(), + ) + .await?; + manager + .alter_table( + Table::alter() + .table(Video::Table) + .drop_column(Video::CollectionId) + .to_owned(), + ) + .await?; + manager + .create_index( + Index::create() + .table(Video::Table) + .name("idx_video_favorite_id_bvid") + .col(Video::FavoriteId) + .col(Video::Bvid) + .unique() + .to_owned(), + ) + .await?; + manager + .drop_table(Table::drop().table(Collection::Table).to_owned()) + .await + } +} + +#[derive(DeriveIden)] +enum Collection { + Table, + Id, + SId, + MId, + Name, + Type, + Path, + CreatedAt, +} + +#[derive(DeriveIden)] +enum Video { + Table, + FavoriteId, + TempFavoriteId, + CollectionId, + Bvid, +}