Code refactoring

- better async handling
- split youtube module
- rename config to settings

Justfile

@@ -1,5 +1,5 @@
 run: 
-    poetry run -- dotenv run -- fastapi dev yourss/main.py
+    poetry run -- dotenv run -- fastapi dev --host 0.0.0.0 yourss/main.py
 
 run-redis:
     docker run --rm -ti -p 6379:6379 redis 
@@ -19,4 +19,4 @@ release bump="patch":
 publish:
     git log -1 --pretty="%B" | grep '^🔖 New release: '
     git push
-    git push --tags
+    git push --tags

tests/conftest.py

@@ -3,12 +3,12 @@
 
 from yourss.cache import NoCache
 from yourss.main import app
-from yourss.youtube import YoutubeWebClient
+from yourss.youtube.client import YoutubeClient
 
 
 @pytest_asyncio.fixture
 async def yt_client():
-    yield YoutubeWebClient(cache=NoCache())
+    yield YoutubeClient(cache=NoCache())
 
 
 @pytest_asyncio.fixture

tests/test_youtube.py

@@ -1,6 +1,7 @@
 import pytest
 
-from yourss.youtube import YoutubeScrapper, YoutubeUrl
+from yourss.youtube.client import YoutubeMetadata
+from yourss.youtube.url import YoutubeUrl
 
 USER = "DAN1ELmadison"
 USER_HOME = YoutubeUrl.user_home(USER)
@@ -23,7 +24,7 @@ async def test_channel_rssfeed(yt_client):
 @pytest.mark.asyncio
 async def test_channel_metadata(yt_client):
     response = await yt_client.get_html(CHANNEL_ID_HOME)
-    metadata = YoutubeScrapper.fromresponse(response)
+    metadata = YoutubeMetadata.fromresponse(response)
     assert metadata.title == "Jonny Giger"
     assert (
         metadata.homepage_url
@@ -41,7 +42,7 @@ async def test_user_rssfeed(yt_client):
 @pytest.mark.asyncio
 async def test_user_metadata(yt_client):
     response = await yt_client.get_html(USER_HOME)
-    metadata = YoutubeScrapper.fromresponse(response)
+    metadata = YoutubeMetadata.fromresponse(response)
     assert metadata.title == "Daniel Madison"
     assert (
         metadata.homepage_url
@@ -53,7 +54,7 @@ async def test_user_metadata(yt_client):
 @pytest.mark.asyncio
 async def test_slug_metadata(yt_client):
     response = await yt_client.get_html(SLUG_HOME)
-    metadata = YoutubeScrapper.fromresponse(response)
+    metadata = YoutubeMetadata.fromresponse(response)
     assert metadata.title == "Jonny Giger"
     assert (
         metadata.homepage_url

yourss/async_utils.py

@@ -0,0 +1,20 @@
+import asyncio
+from typing import Iterable
+
+from loguru import logger
+
+from .rss import Feed
+from .youtube.client import YoutubeClient
+
+
+async def get_feeds(client: YoutubeClient, channels: Iterable[str]) -> list[Feed]:
+    feeds = await asyncio.gather(
+        *[client.get_rss_feed(channel) for channel in channels],
+        return_exceptions=True,
+    )
+    for name, feed in zip(channels, feeds):
+        if isinstance(feed, Feed):
+            logger.debug("Get feed: {} -> {}", name, feed.get_url())
+        else:
+            logger.warning("Error with feed: {} -> {}", name, feed)
+    return [feed for feed in feeds if isinstance(feed, Feed)]

yourss/main.py

@@ -3,8 +3,8 @@
 
 from . import __name__ as app_name
 from . import __version__ as app_version
-from .config import static_folder
 from .routers import api, www
+from .settings import static_folder
 
 app = FastAPI(name=app_name, version=app_version)
 app.include_router(www.router)

yourss/routers/api.py

@@ -6,7 +6,7 @@
 
 from .. import __name__ as app_name
 from .. import __version__ as app_version
-from ..youtube import YoutubeWebClient
+from ..youtube.client import YoutubeClient
 from .utils import force_https, get_youtube_client
 
 router = APIRouter()
@@ -19,15 +19,15 @@ async def version():
 
 @router.get("/rss/{name}", response_class=RedirectResponse)
 async def rss_feed(
-    name: str, yt_client: Annotated[YoutubeWebClient, Depends(get_youtube_client)]
+    name: str, yt_client: Annotated[YoutubeClient, Depends(get_youtube_client)]
 ):
     feed = await yt_client.get_rss_feed(name)
     return RedirectResponse(force_https(str(feed.get_url())))
 
 
 @router.get("/avatar/{name}", response_class=RedirectResponse)
 async def avatar(
-    name: str, yt_client: Annotated[YoutubeWebClient, Depends(get_youtube_client)]
+    name: str, yt_client: Annotated[YoutubeClient, Depends(get_youtube_client)]
 ):
     url = await yt_client.get_avatar_url(name)
     if url is None:

yourss/routers/utils.py

@@ -3,8 +3,8 @@
 from starlette.templating import Jinja2Templates, _TemplateResponse
 
 from ..cache import create_cache
-from ..config import current_config
-from ..youtube import YoutubeWebClient
+from ..settings import current_config
+from ..youtube.client import YoutubeClient
 
 
 def force_https(url: str) -> str:
@@ -16,8 +16,8 @@ def force_https(url: str) -> str:
 
 async def get_youtube_client(
     refresh: bool = False,
-) -> AsyncGenerator[YoutubeWebClient, None]:
-    yield YoutubeWebClient(
+) -> AsyncGenerator[YoutubeClient, None]:
+    yield YoutubeClient(
         cache=await create_cache(
             redis_url=current_config.redis_url, force_renew=refresh
         )

yourss/routers/www.py

@@ -6,16 +6,16 @@
 from fastapi.responses import RedirectResponse
 from fastapi.templating import Jinja2Templates
 from jinja2 import Environment, FileSystemLoader
-from loguru import logger
 from starlette.responses import HTMLResponse
 from starlette.status import HTTP_404_NOT_FOUND
 
 import yourss
+from yourss.async_utils import get_feeds
 
-from ..config import current_config, templates_folder
 from ..schema import Theme, User
 from ..security import get_auth_user
-from ..youtube import YoutubeWebClient
+from ..settings import current_config, templates_folder
+from ..youtube.client import YoutubeClient
 from .utils import custom_template_response, get_youtube_client, parse_channel_names
 
 
@@ -61,20 +61,12 @@ async def watch(video: str = Query(alias="v", min_length=11, max_length=11)):
 @router.get("/u/{username}", response_class=HTMLResponse)
 async def get_user(
     request: Request,
-    yt_client: Annotated[YoutubeWebClient, Depends(get_youtube_client)],
+    yt_client: Annotated[YoutubeClient, Depends(get_youtube_client)],
     theme: Theme | None = None,
     user: User = Depends(get_auth_user),
 ):
-    feeds = []
-    for name in user.channels:
-        try:
-            feeds.append(await yt_client.get_rss_feed(name))
-        except BaseException as error:
-            logger.exception("Cannot get rss feed for {}: {}", name, error)
-
-    if len(feeds) == 0:
+    if len(feeds := await get_feeds(yt_client, user.channels)) == 0:
         raise HTTPException(status_code=HTTP_404_NOT_FOUND, detail="No channels found")
-
     return ViewTemplateResponse(
         request=request,
         title=f"/u/{user.name}",
@@ -87,19 +79,11 @@ async def get_user(
 async def view_channels(
     request: Request,
     channels: str,
-    yt_client: Annotated[YoutubeWebClient, Depends(get_youtube_client)],
+    yt_client: Annotated[YoutubeClient, Depends(get_youtube_client)],
     theme: Theme | None = None,
 ):
-    feeds = []
-    for name in parse_channel_names(channels):
-        try:
-            feeds.append(await yt_client.get_rss_feed(name))
-        except BaseException as error:
-            logger.exception("Cannot get rss feed for {}: {}", name, error)
-
-    if len(feeds) == 0:
+    if len(feeds := await get_feeds(yt_client, parse_channel_names(channels))) == 0:
         raise HTTPException(status_code=HTTP_404_NOT_FOUND, detail="No channels found")
-
     return ViewTemplateResponse(
         request=request,
         title=", ".join(sorted(map(lambda f: f.title, feeds))),

yourss/security.py

@@ -30,7 +30,7 @@ def verify_password(user: User, value: str) -> bool:
 
 
 def find_user(username: str) -> Optional[User]:
-    from .config import current_config
+    from .settings import current_config
 
     if current_config.users_file is not None and current_config.users_file.exists():
         try: