Make list methods of clients iterable

Author: Pijukatel

docs/02_concepts/08_pagination.mdx

@@ -12,10 +12,10 @@ import ApiLink from '@site/src/components/ApiLink';
 
 import PaginationAsyncExample from '!!raw-loader!./code/08_pagination_async.py';
 import PaginationSyncExample from '!!raw-loader!./code/08_pagination_sync.py';
-
 import IterateItemsAsyncExample from '!!raw-loader!./code/08_iterate_items_async.py';
 import IterateItemsSyncExample from '!!raw-loader!./code/08_iterate_items_sync.py';
 
+
 Most methods named `list` or `list_something` in the Apify client return a <ApiLink to="class/ListPage">`ListPage`</ApiLink>  object. This object provides a consistent interface for working with paginated data and includes the following properties:
 
 - `items` - The main results you're looking for.
@@ -45,7 +45,7 @@ The <ApiLink to="class/ListPage">`ListPage`</ApiLink> interface offers several k
 
 ## Generator-based iteration
 
-For most use cases, `iterate_items()` is the recommended way to process all items in a dataset. It handles pagination automatically using a Python generator, fetching items in batches behind the scenes so you don't need to manage offsets or limits yourself.
+You can also use the `list` methods directly in iteration. It handles pagination automatically, fetching items in batches behind the scenes so you don't need to manage offsets or limits yourself.
 
 <Tabs>
     <TabItem value="AsyncExample" label="Async client" default>
@@ -60,6 +60,4 @@ For most use cases, `iterate_items()` is the recommended way to process all item
     </TabItem>
 </Tabs>
 
-`iterate_items()` accepts the same filtering parameters as `list_items()` (`clean`, `fields`, `omit`, `unwind`, `skip_empty`, `skip_hidden`), so you can combine automatic pagination with data filtering.
-
-Similarly, `KeyValueStoreClient` provides an `iterate_keys()` method for iterating over all keys in a key-value store without manual pagination.
+Similarly, you can iterate over the return value of `KeyValueStoreClient.list_keys()` to go through all keys in a key-value store without manual pagination. The older `iterate_keys()` method is deprecated.

docs/02_concepts/code/08_iterate_items_async.py

@@ -7,6 +7,11 @@ async def main() -> None:
     apify_client = ApifyClientAsync(TOKEN)
     dataset_client = apify_client.dataset('dataset-id')
 
-    # Iterate through all items automatically.
-    async for item in dataset_client.iterate_items():
-        print(item)
+    # Define the pagination parameters
+    limit = 1500  # Number of items in total
+    offset = 100  # Starting offset
+
+    # Iterate through items automatically, lazily sending as many API calls
+    # as needed and receiving items in chunks.
+    async for item in dataset_client.list_items(limit=limit, offset=offset):
+        print(item)  # Process the item as needed

docs/02_concepts/code/08_iterate_items_sync.py

@@ -7,9 +7,14 @@ def main() -> None:
     apify_client = ApifyClient(TOKEN)
     dataset_client = apify_client.dataset('dataset-id')
 
-    # Iterate through all items automatically.
-    for item in dataset_client.iterate_items():
-        print(item)
+    # Define the pagination parameters
+    limit = 1500  # Number of items in total
+    offset = 100  # Starting offset
+
+    # Iterate through items automatically, lazily sending as many API calls
+    # as needed and receiving items in chunks.
+    for item in dataset_client.list_items(limit=limit, offset=offset):
+        print(item)  # Process the item as needed
 
 
 if __name__ == '__main__':

docs/02_concepts/code/08_pagination_async.py

@@ -10,26 +10,15 @@ async def main() -> None:
     dataset_client = apify_client.dataset('dataset-id')
 
     # Define the pagination parameters
-    limit = 1000  # Number of items per page
+    limit = 1000  # Number items to request from API
     offset = 0  # Starting offset
-    all_items = []  # List to store all fetched items
 
-    while True:
-        # Fetch a page of items
-        response = await dataset_client.list_items(limit=limit, offset=offset)
-        items = response.items
-        total = response.total
+    # Send single API call to fetch paginated items.
+    # (number of items per single call can be limited by API)
+    paginated_items = await dataset_client.list_items(limit=limit, offset=offset)
 
-        print(f'Fetched {len(items)} items')
+    # Inspect pagination metadata returned by API
+    print(paginated_items.total)
 
-        # Add the fetched items to the complete list
-        all_items.extend(items)
-
-        # Exit the loop if there are no more items to fetch
-        if offset + limit >= total:
-            break
-
-        # Increment the offset for the next page
-        offset += limit
-
-    print(f'Overall fetched {len(all_items)} items')
+    for item in paginated_items.items:
+        print(item)  # Process the item as needed

docs/02_concepts/code/08_pagination_sync.py

@@ -10,26 +10,15 @@ def main() -> None:
     dataset_client = apify_client.dataset('dataset-id')
 
     # Define the pagination parameters
-    limit = 1000  # Number of items per page
+    limit = 1000  # Number items to request from API
     offset = 0  # Starting offset
-    all_items = []  # List to store all fetched items
 
-    while True:
-        # Fetch a page of items
-        response = dataset_client.list_items(limit=limit, offset=offset)
-        items = response.items
-        total = response.total
+    # Send single API call to fetch paginated items.
+    # (number of items per single call can be limited by API)
+    paginated_items = dataset_client.list_items(limit=limit, offset=offset)
 
-        print(f'Fetched {len(items)} items')
+    # Inspect pagination metadata returned by API
+    print(paginated_items.total)
 
-        # Add the fetched items to the complete list
-        all_items.extend(items)
-
-        # Exit the loop if there are no more items to fetch
-        if offset + limit >= total:
-            break
-
-        # Increment the offset for the next page
-        offset += limit
-
-    print(f'Overall fetched {len(all_items)} items')
+    for item in paginated_items.items:
+        print(item)  # Process the item as needed

scripts/_utils.py

@@ -27,6 +27,7 @@
     (re.compile(r'\bSynchronous\b'), 'Asynchronous'),
     (re.compile(r'Retry a function'), 'Retry an async function'),
     (re.compile(r'Function to retry'), 'Async function to retry'),
+    (re.compile(r'returned page also supports iteration: `for'), 'returned page also supports iteration: `async for'),
 ]
 """Patterns for converting sync docstrings to async docstrings."""
 

src/apify_client/_resource_clients/actor_collection.py

@@ -10,15 +10,25 @@
     CreateActorRequest,
     DefaultRunOptions,
     ExampleRunInput,
-    ListOfActors,
     ListOfActorsResponse,
 )
+from apify_client._pagination import (
+    _LazyTask,
+    build_get_iterator,
+    build_get_iterator_async,
+)
+from apify_client._pagination_classes import (
+    IterablePageOfActors,
+    IterablePageOfActorsAsync,
+    PageOfItems,
+)
 from apify_client._resource_clients._resource_client import ResourceClient, ResourceClientAsync
 from apify_client._utils import to_seconds
 
 if TYPE_CHECKING:
     from datetime import timedelta
 
+    from apify_client._models_generated import ActorShort
     from apify_client._types import Timeout
 
 _SORT_BY_TO_API: dict[str, str] = {
@@ -55,9 +65,12 @@ def list(
         desc: bool | None = None,
         sort_by: Literal['created_at', 'last_run_started_at'] | None = 'created_at',
         timeout: Timeout = 'medium',
-    ) -> ListOfActors:
+    ) -> IterablePageOfActors:
         """List the Actors the user has created or used.
 
+        The returned page also supports iteration: `for item in client.list(...)` yields individual Actors
+        and transparently fetches further pages from the API.
+
         https://docs.apify.com/api/v2#/reference/actors/actor-collection/get-list-of-actors
 
         Args:
@@ -72,8 +85,31 @@ def list(
             The list of available Actors matching the specified filters.
         """
         api_sort_by = _SORT_BY_TO_API[sort_by] if sort_by is not None else None
-        result = self._list(timeout=timeout, my=my, limit=limit, offset=offset, desc=desc, sortBy=api_sort_by)
-        return ListOfActorsResponse.model_validate(result).data
+
+        def _callback(**kwargs: Any) -> PageOfItems[ActorShort]:
+            result = self._list(timeout=timeout, my=my, sortBy=api_sort_by, **kwargs)
+            data = ListOfActorsResponse.model_validate(result).data
+            return PageOfItems(
+                items=data.items,
+                count=data.count,
+                limit=data.limit,
+                total=data.total,
+                offset=data.offset,
+                desc=data.desc,
+            )
+
+        first_page = _callback(limit=limit, offset=offset, desc=desc)
+        get_iterator = build_get_iterator(_callback, first_page, limit=limit, offset=offset, desc=desc)
+
+        return IterablePageOfActors(
+            _get_iterator=get_iterator,
+            items=first_page.items,
+            count=first_page.count,
+            limit=first_page.limit,
+            total=first_page.total,
+            offset=first_page.offset,
+            desc=first_page.desc,
+        )
 
     def create(
         self,
@@ -192,7 +228,7 @@ def __init__(
             **kwargs,
         )
 
-    async def list(
+    def list(
         self,
         *,
         my: bool | None = None,
@@ -201,9 +237,12 @@ async def list(
         desc: bool | None = None,
         sort_by: Literal['created_at', 'last_run_started_at'] | None = 'created_at',
         timeout: Timeout = 'medium',
-    ) -> ListOfActors:
+    ) -> IterablePageOfActorsAsync:
         """List the Actors the user has created or used.
 
+        The returned page also supports iteration: `async for item in client.list(...)` yields individual Actors
+        and transparently fetches further pages from the API.
+
         https://docs.apify.com/api/v2#/reference/actors/actor-collection/get-list-of-actors
 
         Args:
@@ -218,8 +257,28 @@ async def list(
             The list of available Actors matching the specified filters.
         """
         api_sort_by = _SORT_BY_TO_API[sort_by] if sort_by is not None else None
-        result = await self._list(timeout=timeout, my=my, limit=limit, offset=offset, desc=desc, sortBy=api_sort_by)
-        return ListOfActorsResponse.model_validate(result).data
+
+        async def _callback(**kwargs: Any) -> PageOfItems[ActorShort]:
+            result = await self._list(timeout=timeout, my=my, sortBy=api_sort_by, **kwargs)
+            data = ListOfActorsResponse.model_validate(result).data
+            return PageOfItems(
+                items=data.items,
+                count=data.count,
+                limit=data.limit,
+                total=data.total,
+                offset=data.offset,
+                desc=data.desc,
+            )
+
+        fetch_first_page = _LazyTask(_callback(limit=limit, offset=offset, desc=desc))
+        get_async_iterator = build_get_iterator_async(
+            _callback, fetch_first_page, limit=limit, offset=offset, desc=desc
+        )
+
+        return IterablePageOfActorsAsync(
+            _awaitable_first_page=fetch_first_page,
+            _get_async_iterator=get_async_iterator,
+        )
 
     async def create(
         self,

src/apify_client/_resource_clients/actor_env_var_collection.py

@@ -3,7 +3,17 @@
 from typing import TYPE_CHECKING, Any
 
 from apify_client._docs import docs_group
-from apify_client._models_generated import EnvVar, EnvVarResponse, ListOfEnvVars, ListOfEnvVarsResponse
+from apify_client._models_generated import EnvVar, EnvVarResponse, ListOfEnvVarsResponse
+from apify_client._pagination import (
+    _LazyTask,
+    build_get_iterator,
+    build_get_iterator_async,
+)
+from apify_client._pagination_classes import (
+    IterablePageOfEnvVars,
+    IterablePageOfEnvVarsAsync,
+    PageOfItemsOnlyTotal,
+)
 from apify_client._resource_clients._resource_client import ResourceClient, ResourceClientAsync
 
 if TYPE_CHECKING:
@@ -29,9 +39,12 @@ def __init__(
             **kwargs,
         )
 
-    def list(self, *, timeout: Timeout = 'short') -> ListOfEnvVars:
+    def list(self, *, timeout: Timeout = 'short') -> IterablePageOfEnvVars:
         """List the available Actor environment variables.
 
+        The returned page also supports iteration: `for item in client.list()` yields individual environment
+        variables.
+
         https://docs.apify.com/api/v2#/reference/actors/environment-variable-collection/get-list-of-environment-variables
 
         Args:
@@ -40,8 +53,20 @@ def list(self, *, timeout: Timeout = 'short') -> ListOfEnvVars:
         Returns:
             The list of available Actor environment variables.
         """
-        result = self._list(timeout=timeout)
-        return ListOfEnvVarsResponse.model_validate(result).data
+
+        def _callback(**kwargs: Any) -> PageOfItemsOnlyTotal[EnvVar]:
+            result = self._list(timeout=timeout, **kwargs)
+            data = ListOfEnvVarsResponse.model_validate(result).data
+            return PageOfItemsOnlyTotal(items=data.items, total=data.total)
+
+        first_page = _callback()
+        get_iterator = build_get_iterator(_callback, first_page)
+
+        return IterablePageOfEnvVars(
+            _get_iterator=get_iterator,
+            items=first_page.items,
+            total=first_page.total,
+        )
 
     def create(
         self,
@@ -90,9 +115,12 @@ def __init__(
             **kwargs,
         )
 
-    async def list(self, *, timeout: Timeout = 'short') -> ListOfEnvVars:
+    def list(self, *, timeout: Timeout = 'short') -> IterablePageOfEnvVarsAsync:
         """List the available Actor environment variables.
 
+        The returned page also supports iteration: `async for item in client.list()` yields individual environment
+        variables.
+
         https://docs.apify.com/api/v2#/reference/actors/environment-variable-collection/get-list-of-environment-variables
 
         Args:
@@ -101,8 +129,19 @@ async def list(self, *, timeout: Timeout = 'short') -> ListOfEnvVars:
         Returns:
             The list of available Actor environment variables.
         """
-        result = await self._list(timeout=timeout)
-        return ListOfEnvVarsResponse.model_validate(result).data
+
+        async def _callback(**kwargs: Any) -> PageOfItemsOnlyTotal[EnvVar]:
+            result = await self._list(timeout=timeout, **kwargs)
+            data = ListOfEnvVarsResponse.model_validate(result).data
+            return PageOfItemsOnlyTotal(items=data.items, total=data.total)
+
+        fetch_first_page = _LazyTask(_callback())
+        get_async_iterator = build_get_iterator_async(_callback, fetch_first_page)
+
+        return IterablePageOfEnvVarsAsync(
+            _awaitable_first_page=fetch_first_page,
+            _get_async_iterator=get_async_iterator,
+        )
 
     async def create(
         self,