feat: 拆分 refresh job

config.py

@@ -21,13 +21,18 @@ class RedisConfigModel(BaseModel):
     password: Optional[str] = None
     database: int = 0
 
+class JobInterval(BaseModel):
+    interval_curseforge_refresh: int = 60 * 60 * 2 # 2 hours
+    interval_modrinth_refresh: int = 60 * 60 * 2 # 2 hours
+    interval_sync_curseforge: int = 60 * 5 # 5 minutes
+    interval_sync_modrinth: int = 60 * 5 # 5 minutes
+    interval_global_statistics: int = 60 * 60 * 24 # 24 hours
+
 class ConfigModel(BaseModel):
     debug: bool = False
     mongodb: MongodbConfigModel = MongodbConfigModel()
     redis: RedisConfigModel = RedisConfigModel()
-    interval_refresh: int = 60 * 60 * 2 # 2 hours
-    interval_sync_curseforge: int = 60 * 5 # 5 minutes
-    interval_sync_modrinth: int = 60 * 5 # 5 minutes
+    interval: JobInterval = JobInterval()
     max_workers: int = 8
     sync_curseforge: bool = True
     sync_modrinth: bool = True

start.py

@@ -7,9 +7,11 @@
 from utils.loger import log
 from config import Config
 from sync.tasks import (
-    refresh_with_modify_date,
+    refresh_curseforge_with_modify_date,
+    refresh_modrinth_with_modify_date,
     sync_curseforge_queue,
     sync_modrinth_queue,
+    send_statistics_to_telegram
 )
 
 config = Config.load()
@@ -28,26 +30,40 @@ def main():
     scheduler = BackgroundScheduler()
 
     # 添加定时刷新任务，每小时执行一次
-    refresh_job = scheduler.add_job(
-        refresh_with_modify_date,
-        trigger=IntervalTrigger(seconds=config.interval_refresh),
+    curseforge_refresh_job = scheduler.add_job(
+        refresh_curseforge_with_modify_date,
+        trigger=IntervalTrigger(seconds=config.interval.interval_curseforge_refresh),
         next_run_time=datetime.datetime.now(),  # 立即执行一次任务
-        name="mcim_refresh",
+        name="mcim_curseforge_refresh",
+    )
+
+    modrinth_refresh_job = scheduler.add_job(
+        refresh_modrinth_with_modify_date,
+        trigger=IntervalTrigger(seconds=config.interval.interval_modrinth_refresh),
+        next_run_time=datetime.datetime.now(),  # 立即执行一次任务
+        name="mcim_modrinth_refresh",
     )
 
     # 添加定时同步任务，用于检查 api 未找到的请求数据
     curseforge_sync_job = scheduler.add_job(
         sync_curseforge_queue,
-        trigger=IntervalTrigger(seconds=config.interval_sync_curseforge),
+        trigger=IntervalTrigger(seconds=config.interval.interval_sync_curseforge),
         name="mcim_sync_curseforge",
     )
 
     modrinth_sync_job = scheduler.add_job(
         sync_modrinth_queue,
-        trigger=IntervalTrigger(seconds=config.interval_sync_modrinth),
+        trigger=IntervalTrigger(seconds=config.interval.interval_sync_modrinth),
         name="mcim_sync_modrinth",
     )
 
+    # 单独发布统计信息
+    statistics_job = scheduler.add_job(
+        send_statistics_to_telegram,
+        trigger=IntervalTrigger(seconds=config.interval.interval_global_statistics),
+        name="mcim_statistics",
+    )
+
     # 启动调度器
     scheduler.start()
     log.info(f"Scheduler started")

sync/tasks.py

@@ -3,7 +3,7 @@
 from concurrent.futures import ThreadPoolExecutor, as_completed
 
 from utils.loger import log
-from utils.telegram import RefreshNotification, SyncNotification, ProjectDetail
+from utils.telegram import SyncNotification, CurseforgeRefreshNotification, ModrinthRefreshNotification, StatisticsNotification
 from utils import SyncMode
 from config import Config
 from sync.curseforge import sync_mod, sync_mod_all_files
@@ -87,63 +87,60 @@ def create_tasks_pool(sync_function, data, max_workers, thread_name_prefix):
     return thread_pool, futures
 
 
-def refresh_with_modify_date():
-    log.info("Start fetching expired data.")
-    notification = RefreshNotification()
+def refresh_curseforge_with_modify_date():
+    log.info("Start fetching expired CurseForge data.")
+    notification = CurseforgeRefreshNotification()
 
-    # fetch all expired data
-    curseforge_expired_data = []
-    modrinth_expired_data = []
     if SYNC_CURSEFORGE:
         curseforge_expired_data = fetch_expired_curseforge_data()
-        notification.curseforge_refreshed_count = len(curseforge_expired_data)
-        log.info(
-            f"Curseforge expired data totally fetched: {notification.curseforge_refreshed_count}"
-        )
-    if SYNC_MODRINTH:
-        modrinth_expired_data = fetch_expired_modrinth_data()
-        notification.modrinth_refreshed_count = len(modrinth_expired_data)
-        log.info(
-            f"Modrinth expired data totally fetched: {notification.modrinth_refreshed_count}"
+        notification.refreshed_count = len(curseforge_expired_data)
+        log.info(f"Curseforge expired data fetched: {notification.refreshed_count}")
+        log.info(f"Start syncing CurseForge expired data...")
+
+        curseforge_pause_event.set()
+        curseforge_pool, curseforge_futures = create_tasks_pool(
+            sync_mod_all_files,
+            curseforge_expired_data,
+            MAX_WORKERS,
+            "curseforge",
         )
 
-    notification.sync_mode = SyncMode.MODIFY_DATE
-    log.info(
-        f"All expired data fetched \
-            curseforge: {notification.curseforge_refreshed_count}, \
-            modrinth: {notification.modrinth_refreshed_count}, \
-            start syncing..."
-    )
+        for future in as_completed(curseforge_futures):
+            pass
+        else:
+            curseforge_pool.shutdown()
 
-    # 允许请求
-    curseforge_pause_event.set()
-    modrinth_pause_event.set()
+        if config.telegram_bot:
+            notification.send_to_telegram()
+            log.info("CurseForge refresh message sent to telegram.")
 
-    curseforge_pool, curseforge_futures = create_tasks_pool(
-        sync_mod_all_files, curseforge_expired_data, MAX_WORKERS, "curseforge"
-    )
-    modrinth_pool, modrinth_futures = create_tasks_pool(
-        sync_project_all_version, modrinth_expired_data, MAX_WORKERS, "modrinth"
-    )
 
-    log.info(
-        f"All {len(curseforge_futures) + len(modrinth_futures)} tasks submitted, waiting for completion..."
-    )
+def refresh_modrinth_with_modify_date():
+    log.info("Start fetching expired Modrinth data.")
+    notification = ModrinthRefreshNotification()
 
-    for future in as_completed(curseforge_futures + modrinth_futures):
-        # 不需要返回值
-        pass
-    else:
-        curseforge_pool.shutdown()
-        modrinth_pool.shutdown()
+    if SYNC_MODRINTH:
+        modrinth_expired_data = fetch_expired_modrinth_data()
+        notification.refreshed_count = len(modrinth_expired_data)
+        log.info(f"Modrinth expired data fetched: {notification.refreshed_count}")
+        log.info(f"Start syncing Modrinth expired data...")
 
-    # log.info(
-    #     f"All expired data sync finished, total: {total_expired_count}. Next run at: {sync_job.next_run_time.strftime('%Y-%m-%d %H:%M:%S %Z')}"
-    # )
+        modrinth_pause_event.set()
+        modrinth_pool, modrinth_futures = create_tasks_pool(
+            sync_project_all_version,
+            modrinth_expired_data,
+            MAX_WORKERS,
+            "modrinth",
+        )
+
+        for future in as_completed(modrinth_futures):
+            pass
+        else:
+            modrinth_pool.shutdown()
 
-    if config.telegram_bot:
-        notification.send_to_telegram()
-        log.info("All Message sent to telegram.")
+        if config.telegram_bot:
+            notification.send_to_telegram()
+            log.info("Modrinth refresh message sent to telegram.")
 
 
 def sync_modrinth_full():
@@ -406,3 +403,10 @@ def sync_modrinth_queue():
             notice.send_to_telegram()
 
             log.info("All Message sent to telegram.")
+
+
+def send_statistics_to_telegram():
+    log.info("Start fetching statistics to telegram.")
+    message = StatisticsNotification.send_to_telegram()
+    log.info("Statistics message sent to telegram.")
+    log.info(f"Statistics message: {message}")

utils/telegram/__init__.py

@@ -34,44 +34,11 @@ class Notification(ABC):
     def send_to_telegram(self):
         pass
 
-
-class RefreshNotification(Notification):
-    def __init__(self, sync_mode: SyncMode = SyncMode.MODIFY_DATE):
-        self.modrinth_refreshed_count: int = 0
-        self.curseforge_refreshed_count: int = 0
-        self.sync_mode: SyncMode = sync_mode
-
+class StatisticsNotification(Notification):
+    @classmethod
     def send_to_telegram(self):
-        sync_message = (
-            f"本次同步为{self.sync_mode.value}同步\n"
-            f"CurseForge: {self.curseforge_refreshed_count} 个 Mod 的数据已更新\n"
-            f"Modrinth: {self.modrinth_refreshed_count} 个 Mod 的数据已更新"
-        )
-        """
-        https://mod.mcimirror.top/statistics
-        {
-            "curseforge": {
-                "mod": 75613,
-                "file": 1265312,
-                "fingerprint": 1264259
-            },
-            "modrinth": {
-                "project": 42832,
-                "version": 415467,
-                "file": 458877
-            },
-            "file_cdn": {
-                "file": 924573
-            }
-        }
-        """
         mcim_stats = request_sync("https://mod.mcimirror.top/statistics").json()
-        """
-        MCIM 已缓存：
-        Curseforge 模组 75613 个，文件 1265312 个，指纹 1264259 个
-        Modrinth 项目 42832 个，版本 415467 个，文件 458877 个
-        CDN 文件 924573 个
-        """
+        files_stats = request_sync("https://files.mcimirror.top/api/stats/center").json()
         mcim_message = (
             "MCIM API 已缓存：\n"
             f"Curseforge 模组 {mcim_stats['curseforge']['mod']} 个，文件 {mcim_stats['curseforge']['file']} 个，指纹 {mcim_stats['curseforge']['fingerprint']} 个\n"
@@ -114,8 +81,29 @@ def send_to_telegram(self):
             f"总文件数：{files_stats['totalFiles']} 个\n"
             f"总文件大小：{files_stats['totalSize'] / 1024 / 1024 / 1024/ 1024:.2f} TB\n"
         )
-        final_message = f"{sync_message}\n\n{mcim_message}\n\n{files_message}"
+        final_message = f"{mcim_message}\n\n{files_message}"
         send_message_sync(final_message)
+        return final_message
+
+class CurseforgeRefreshNotification(Notification):
+    def __init__(self, count: int):
+        self.refreshed_count: int = count
+
+    def send_to_telegram(self):
+        sync_message = (
+            f"Curseforge 缓存刷新完成，共刷新 {self.refreshed_count} 个模组"
+        )
+        send_message_sync(sync_message)
+
+class ModrinthRefreshNotification(Notification):
+    def __init__(self, count: int):
+        self.refreshed_count: int = count
+
+    def send_to_telegram(self):
+        sync_message = (
+            f"Modrinth 缓存刷新完成，共刷新 {self.refreshed_count} 个模组"
+        )
+        send_message_sync(sync_message)
 
 
 class ProjectDetail(BaseModel):
@@ -132,10 +120,11 @@ def __init__(self, platform: str, total_catached_count: int, projects_detail_inf
 
     def send_to_telegram(self):
         message = f"本次从 API 请求中总共捕捉到 {self.total_catached_count} 个 {self.platform} 的模组数据" + \
-                    f'有 {len(self.projects_detail_info)} 个模组是新捕获到的'
+                    f'有 {len(self.projects_detail_info)} 个模组是新捕获到的' + \
+                    f'以下格式为 模组名(模组ID): 版本数量'
         for project in self.projects_detail_info:
             if len(message) > 4000: # Telegram 限制消息长度 4096 字符
                 break
-            message += f"\n{project.name} (ID: {project.id}) 共有 {project.version_count} 个版本"
+            message += f"\n{project.name}({project.id}): {project.version_count}"
 
         send_message_sync(message)