Fixed video pagination. New exceptions for error handling. Improved m…

…odel relation links. Resource pagination (wip).

setup.py

@@ -39,6 +39,6 @@
     test_suite='tests',
     tests_require=test_requirements,
     url='https://github.com/PetterKraabol/Twitch-Python',
-    version='0.0.10',
+    version='0.0.11',
     zip_safe=True,
 )

twitch/__init__.py

@@ -2,7 +2,6 @@
 from .helix import Helix
 from .v5 import V5
 
-
 name = "twitch"
 
 __all__ = [Helix, V5, Chat]

twitch/cache.py

@@ -10,9 +10,8 @@ def __init__(self, duration: timedelta = None):
         self._duration: timedelta = duration or timedelta(minutes=30)
 
     def get(self, key: str, ignore_expiration: bool = False) -> Optional[dict]:
-        if self.has(key):
-            if ignore_expiration or self.expired(key):
-                return self._store[key]['value']
+        if self.has(key) and (ignore_expiration or self.expired(key)):
+            return self._store[key]['value']
 
     def set(self, key: str, value: dict, duration: timedelta = None) -> datetime:
         expiration: datetime = datetime.now() + (duration or self._duration)

twitch/helix/__init__.py

@@ -5,9 +5,9 @@
 from .models import Stream, User, Video, Game
 
 # Resources
-from .streams import Streams
+from .streams import Streams, StreamNotFound
 from .users import Users
 from .videos import Videos
 from .games import Games
 
-__all__ = [Helix, Stream, User, Video, Streams, Users, Videos]
+__all__ = [Helix, Stream, User, Video, Streams, Users, Videos, StreamNotFound]

twitch/helix/games.py

@@ -15,4 +15,5 @@ def __init__(self, api: API, **kwargs: Optional):
                           self._api.get(self._path, params=kwargs)['data']]
 
     def top(self, **kwargs) -> List['helix.Game']:
-        return [helix.Game(api=self._api, data=game) for game in self._api.get('games/top', params=kwargs)['data']]
+        return [helix.Game(api=self._api, data=game) for game in
+                self._api.get(f'{self._path}/top', params=kwargs)['data']]

twitch/helix/helix.py

@@ -1,5 +1,5 @@
 from datetime import timedelta
-from typing import List, Union
+from typing import List, Union, Optional
 
 import twitch.helix as helix
 from twitch.api import API
@@ -9,11 +9,21 @@ class Helix:
     BASE_URL: str = 'https://api.twitch.tv/helix/'
 
     def __init__(self, client_id: str, client_secret: str = None, use_cache: bool = False,
-                 cache_duration: timedelta = timedelta(minutes=30), rate_limit: int = 30):
+                 cache_duration: Optional[timedelta] = None, rate_limit: int = 30):
+        """
+        Helix API (New Twitch API)
+        https://dev.twitch.tv/docs/api/
+
+        :param client_id: Twitch client ID
+        :param client_secret: Twitch client secret
+        :param use_cache: Cache API requests (recommended)
+        :param cache_duration: Cache duration
+        :param rate_limit: API rate limit
+        """
         self.client_id: str = client_id
         self.client_secret: str = client_secret
         self.use_cache: bool = use_cache
-        self.cache_duration: timedelta = cache_duration
+        self.cache_duration: Optional[timedelta] = cache_duration
         self.rate_limit: int = rate_limit
 
     def api(self) -> API:
@@ -25,9 +35,9 @@ def users(self, *args) -> 'helix.Users':
     def user(self, user: Union[str, int]) -> 'helix.User':
         return self.users(user)[0]
 
-    def videos(self, video_ids: Union[str, int, List[Union[str, int]]], **kwargs) -> 'helix.Videos':
-        if type(video_ids) != list:
-            video_ids = [video_ids]
+    def videos(self, video_ids: Union[str, int, List[Union[str, int]]] = None, **kwargs) -> 'helix.Videos':
+        if video_ids and type(video_ids) != list:
+            video_ids = [int(video_ids)]
         return helix.Videos(self.api(), video_ids=video_ids, **kwargs)
 
     def video(self, video_id: Union[str, int] = None, **kwargs) -> 'helix.Video':

twitch/helix/models/game.py

@@ -1,3 +1,4 @@
+import twitch.helix as helix
 from twitch.api import API
 
 
@@ -19,3 +20,6 @@ def __init__(self, api: API, data: dict):
 
     def __str__(self):
         return self.name
+
+    def videos(self, **kwargs) -> 'helix.Videos':
+        return helix.Videos(self._api, game_id=self.id, **kwargs)

twitch/helix/models/stream.py

@@ -28,7 +28,7 @@ def __init__(self, api: API, data: dict):
             self.__dict__[key] = value
 
     def user(self) -> 'helix.User':
-        return helix.Users(self._api, self.user_id)[0]
+        return helix.Users(self._api, int(self.user_id))[0]
 
     def __str__(self):
         return self.title

twitch/helix/models/user.py

@@ -29,7 +29,7 @@ def __str__(self):
         return self.login
 
     def videos(self, **kwargs) -> 'helix.Videos':
-        return helix.Videos(api=self._api, user_id=self.id, **kwargs)
+        return helix.Videos(api=self._api, user_id=int(self.id), **kwargs)
 
     def stream(self) -> 'helix.Stream':
-        return helix.Streams(api=self._api, user_id=self.id)[0]
+        return helix.Streams(api=self._api, user_id=int(self.id))[0]

twitch/helix/models/video.py

@@ -34,8 +34,8 @@ def comments(self) -> 'v5.Comments':
                      use_cache=self._api.use_cache,
                      cache_duration=self._api.cache_duration).comments(self.id)
 
-    def user(self) -> 'helix.Users':
-        return helix.Users(self._api, self.user_id)
+    def user(self) -> 'helix.User':
+        return helix.Users(self._api, int(self.user_id))[0]
 
     def __str__(self):
         return self.title

twitch/helix/streams.py

@@ -5,13 +5,22 @@
 from twitch.resource import Resource
 
 
+class StreamNotFound(Exception):
+    pass
+
+
 class Streams(Resource['Stream']):
 
     def __init__(self, api: API, **kwargs):
         super().__init__(api=api, path='streams')
 
-        self._data = [helix.Stream(api=self._api, data=video) for video in
-                      self._api.get(self._path, params=kwargs)['data']]
+        response: dict = self._api.get(self._path, params=kwargs)
+
+        if response['data']:
+            self._data = [helix.Stream(api=self._api, data=video) for video in
+                          self._api.get(self._path, params=kwargs)['data']]
+        else:
+            raise StreamNotFound('No stream was found')
 
     def users(self) -> Generator[Tuple['helix.Stream', 'helix.User'], None, None]:
         for stream in self:

twitch/helix/videos.py

@@ -6,51 +6,73 @@
 from twitch.resource import Resource
 
 
+class VideosAPIException(Exception):
+    pass
+
+
 class Videos(Resource[helix.Video]):
     DEFAULT_FIRST: int = 20
-    MAX_FIRST: int = 100
+    FIRST_API_LIMIT: int = 100
+    ID_API_LIMIT: int = 100
+    CACHE_PREFIX: str = 'helix.video.'
 
     def __init__(self,
                  api: API,
-                 video_ids: Union[str, int, List[Union[str, int]]] = None,
+                 video_ids: Union[int, List[int]] = None,
                  **kwargs):
         super().__init__(api=api, path='videos')
 
+        # Store kwargs as class property for __iter__
         self._kwargs = kwargs
 
-        if video_ids:
-            self._kwargs['id'] = list(set(video_ids))
+        # 'id' parameter can be a singular or a list
+        # Create list of video ids by combining video_ids and kwargs['id']
+
+        # Convert singular string to list
+        if 'id' in self._kwargs and type(self._kwargs['id']) == str:
+            self._kwargs['id'] = [self._kwargs['id']]
+
+        self._kwargs['id'] = list(self._kwargs['id']) if 'id' in self._kwargs.keys() else []
+        self._kwargs['id'] = self._kwargs['id'] + list(video_ids) if video_ids else self._kwargs['id']
+
+        # Convert to integers
+        self._kwargs['id'] = [int(x) for x in self._kwargs['id']]
+
+        # Remove duplicates
+        self._kwargs['id'] = list(set(self._kwargs['id'])) if self._kwargs['id'] else []
+
+        # Download video ids
+        if len(self._kwargs['id']) > 0:
             self._download_video_ids(**self._kwargs)
-        else:
-            # Limit videos if first parameter is specified.
-            # If not limit is specified, the pagination cursor
-            # is used to continuously fetch videos.
 
-            # The Helix API has a maximum of 100 videos per API call.
-            # To fetch >100, we split the request in chunks of =<100 videos
-            first: int = kwargs['first'] if 'first' in kwargs else 0
+        # Download first n videos from a user or game
+        elif 'first' in self._kwargs.keys():
+            self._download_videos(**self._kwargs)
 
-            while first > 0:
-                kwargs['first'] = min(Videos.MAX_FIRST, first)
-                kwargs['after'] = self._cursor or None
-                self._download_by_login(**kwargs)
-                first -= kwargs['first']
+        # If neither statements are True, videos will be fetched continuously in __iter__
 
     def __iter__(self) -> Generator['helix.Video', None, None]:
-        # if first or ids are specified, yield data
-        # that was downloader in the constructor
-        if 'first' in self._kwargs or 'id' in self._kwargs:
+        # If videos were downloaded in the constructor, yield only those
+        if self._data:
             for video in self._data:
                 yield video
+            return
+
+        # Yield a continuous stream of videos from a user id or game id
+        if len([key for key in self._kwargs.keys() if key in ['user_id', 'game_id']]) != 1:
+            raise VideosAPIException('A user_id or a game_id must be specified.')
 
-        else:
-            # Fetch the maximum amount of videos from
-            # API to minimize the number of requests
-            self._kwargs['first'] = Videos.MAX_FIRST
+        # Fetch the maximum amount of videos from
+        # API to minimize the number of requests
+        self._kwargs['first'] = Videos.FIRST_API_LIMIT
+
+        while True:
+            for video in self._videos_fragment(**self._kwargs):
+                yield video
 
-            while True:
-                for video in self._videos_fragment(**self._kwargs):
-                    yield video
+            # Break if no cursor
+            if not self._cursor:
+                break
 
     def __getitem__(self, item: int) -> 'helix.Video':
         for index, value in enumerate(self):
@@ -67,35 +89,63 @@ def _videos_fragment(self, ignore_cache: bool = False, **kwargs) -> List['helix.
         # Set pagination cursor
         self._cursor = response.get('pagination', {}).get('cursor', None)
 
+        # Download videos
+        videos: List['helix.Video'] = [helix.Video(api=self._api, data=video) for video in response['data']]
+
+        # Cache individual videos
+        if self._api.use_cache:
+            for video in videos:
+                API.SHARED_CACHE.set(f'{Videos.CACHE_PREFIX}{video.id}', video.data)
+
         # Return video data
-        return [helix.Video(api=self._api, data=video) for video in response['data']]
+        return videos
 
     def _download_video_ids(self, **kwargs):
-        # Custom video caching
+        # Custom cache lookup
         if self._api.use_cache:
             cache_hits: list = []
             for video_id in list(kwargs['id']):
-                cache_key: str = f'helix.video.{video_id}'
-                cache_data: dict = API.SHARED_CACHE.get(cache_key)
+                cache_data: dict = API.SHARED_CACHE.get(f'{Videos.CACHE_PREFIX}{video_id}')
                 if cache_data:
                     self._data.append(helix.Video(api=self._api, data=cache_data))
                     cache_hits.append(video_id)
 
+            # Removed cached ids from kwargs
             kwargs['id'] = [n for n in kwargs['id'] if n not in cache_hits]
 
-        # Download from API
-        if len(kwargs['id']):
-            # Video data
-            # Ignore cache, as we want to cache individual videos and not a collection of videos.
-            for video in self._videos_fragment(ignore_cache=True, **kwargs):
-                self._data.append(video)
+        # Download uncached videos from API
+        if len(kwargs['id']) > 0:
+
+            # When the number of IDs exceeds API limitations, divide into multiple requests
+            remaining_video_ids: list = kwargs['id']
+
+            while remaining_video_ids:
+                kwargs['id'] = remaining_video_ids[:Videos.ID_API_LIMIT]
+
+                # Ignore default caching method, as we want to cache individual videos and not a collection of videos.
+                videos: List['helix.Video'] = self._videos_fragment(ignore_cache=True, **kwargs)
+                self._data.extend(videos)
+
+                # Update remaining video ids
+                remaining_video_ids = [] if len(videos) < len(remaining_video_ids) else remaining_video_ids[
+                                                                                        Videos.ID_API_LIMIT:]
+
+    def _download_videos(self, **kwargs):
+        # user_id or game_id must be provided
+        if len([key for key in kwargs.keys() if key in ['user_id', 'game_id']]) != 1:
+            raise VideosAPIException('A user_id or a game_id must be specified.')
+
+        remaining_videos: int = kwargs['first']
+
+        while remaining_videos:
+            kwargs['first'] = min(Videos.FIRST_API_LIMIT, remaining_videos)
 
-                # Save individual videos to cache
-                if self._api.use_cache:
-                    API.SHARED_CACHE.set(f'helix.videos.{video.id}', video.data)
+            # Use custom and API cache to cache individual and collections of videos
+            videos: List['helix.Video'] = self._videos_fragment(ignore_cache=False, **kwargs)
+            self._data.extend(videos)
 
-    def _download_by_login(self, **kwargs):
-        self._data.extend(self._videos_fragment(**kwargs))
+            # Update remaining videos
+            remaining_videos = 0 if len(videos) < remaining_videos else remaining_videos - kwargs['first']
 
     def comments(self) -> Generator[Tuple[helix.Video, v5.Comments], None, None]:
         for video in self:

twitch/resource.py

@@ -1,22 +1,42 @@
-from typing import TypeVar, Generic, Generator, List, Any
+from typing import TypeVar, Generic, Generator, List
 
 from twitch.api import API
 
 T = TypeVar('T')
 
 
 class Resource(Generic[T]):
+    FIRST_API_LIMIT: int = 100
 
     def __init__(self, path: str, api: API, data: List[T] = None):
         self._path: str = path
         self._api: API = api
         self._data: List[T] = data or []
         self._cursor: str = None
-        self._kwargs: Any = None
+        self._kwargs: dict = {}
 
     def __iter__(self) -> Generator[T, None, None]:
-        for entry in self._data:
-            yield entry
+        # Yield available data
+        if self._data:
+            for entry in self._data:
+                yield entry
+            return
+
+        # Stream data from API
+
+        # Set start cursor
+        self._cursor = self._kwargs or self._cursor or '0'
+
+        # Set 'first' to limit
+        self._kwargs['first'] = Resource.FIRST_API_LIMIT
+
+        # Paginate
+        while self._cursor:
+            # API Response
+            response: dict = self._api.get(self._path, params=self._kwargs)
+
+            # Set pagination cursor
+            self._cursor = response.get('pagination', {}).get('cursor', None)
 
     def __getitem__(self, item: int) -> T:
         return self._data[item]

twitch/v5/models/comment.py

@@ -135,4 +135,4 @@ def __init__(self, api: API, data: dict):
                 self.__dict__[key] = value
 
     def user(self) -> 'helix.User':
-        return helix.Helix(client_id=self._api.client_id).user(self.commenter.id)
+        return helix.Helix(client_id=self._api.client_id).user(int(self.commenter._id))