#764 - Support cursor-based pagination from NetBox 4.6

docs/advanced.md

@@ -108,6 +108,54 @@ nb = pynetbox.api(
 The executor is constructed once per threaded query and shut down when that query's pages
 have been fetched, so pass the class (or a factory), not an already-instantiated pool.
 
+## Cursor-Based Pagination
+
+Starting with NetBox 4.6, the REST API supports cursor-based pagination as an
+alternative to the default offset-based pagination. Instead of skipping a
+growing number of rows with an `offset`, the server pages forward using the
+primary key (`id`) as a cursor. This avoids the cost of scanning the table up to
+the offset position, so it performs significantly better on very large result
+sets.
+
+### Enabling Cursor Pagination
+
+Pass `pagination="cursor"` when constructing the API client:
+
+```python
+import pynetbox
+
+nb = pynetbox.api(
+    'http://localhost:8000',
+    token='your-token',
+    pagination="cursor",
+)
+
+# .all() and .filter() now page using the server's `start` cursor
+devices = nb.dcim.devices.all()
+```
+
+pynetbox probes the NetBox version once on the first query. If the server is
+older than 4.6 (and therefore does not support cursor pagination), it
+transparently falls back to offset-based pagination, so it is safe to leave this
+option enabled across mixed environments.
+
+### Trade-offs
+
+!!! note "Things to know about cursor pagination"
+    - **No total count up front.** NetBox omits the `count` in cursor mode for
+      performance. pynetbox still supports `len(record_set)`, but doing so
+      triggers a separate count request.
+    - **Fixed ordering.** Results are always ordered by `id`; an explicit
+      `ordering` filter cannot be combined with cursor pagination.
+    - **Not combined with threading.** Cursor pagination is inherently
+      sequential (each page's cursor depends on the previous page), so it cannot
+      be parallelized. When `pagination="cursor"` is set, queries page
+      sequentially even if `threading=True`. For parallel fetching of large
+      offset-paginated result sets, use threading instead (see above).
+    - **Explicit `offset` still works.** Passing an explicit `offset` to
+      `.all()`/`.filter()` requests a single offset-based page as before, since
+      `start` and `offset` are mutually exclusive on the server.
+
 ## Filter Validation
 
 NetBox does not validate filter parameters passed to list endpoints. An unrecognized parameter is silently ignored, which means a typo in a `.filter()` or `.get()` call can quietly return the entire table.

pynetbox/core/api.py

@@ -15,11 +15,14 @@
 """
 
 import contextlib
+import warnings
 
 import requests
+from packaging import version
+from packaging.version import InvalidVersion
 
 from pynetbox.core.app import App, PluginsApp
-from pynetbox.core.query import Request, TOKEN_PREFIX
+from pynetbox.core.query import Request, RequestError, TOKEN_PREFIX
 from pynetbox.core.response import Record
 from pynetbox.models.mapper import CONTENT_TYPE_MAPPER
 
@@ -89,6 +92,7 @@ def __init__(
         threading=False,
         strict_filters=False,
         extensions=None,
+        pagination="offset",
         thread_pool_executor=None,
         max_workers=4,
     ):
@@ -100,9 +104,14 @@ def __init__(
             threading (bool, optional): Set to True to use threading in `.all()` and `.filter()` requests, defaults to False.
             strict_filters (bool, optional): Set to True to check GET call filters against OpenAPI specifications (intentionally not done in NetBox API), defaults to False.
             extensions (list, optional): A list of `Extension` classes or instances that register custom `Record` subclasses and content-type mappings for NetBox plugins. See `pynetbox.core.extension`.
+            pagination (str, optional): Pagination strategy for `.all()` and `.filter()`, either `"offset"` (default) or `"cursor"`. Cursor pagination (NetBox 4.6+) offers better performance on very large result sets but omits the total count and cannot be combined with threading or `ordering`. On NetBox versions older than 4.6 it transparently falls back to offset pagination.
             thread_pool_executor (callable, optional): A `concurrent.futures.ThreadPoolExecutor` class, or any callable matching its `(max_workers=...)` signature and context-manager protocol, used to build the pool for threaded requests. Defaults to `concurrent.futures.ThreadPoolExecutor`.
             max_workers (int, optional): Maximum number of worker threads used for threaded requests, defaults to 4.
         """
+        if pagination not in ("offset", "cursor"):
+            raise ValueError(
+                "pagination must be 'offset' or 'cursor', got {!r}".format(pagination)
+            )
         if max_workers <= 0:
             raise ValueError("max_workers must be a positive integer")
 
@@ -119,6 +128,8 @@ def __init__(
         self.thread_pool_executor = thread_pool_executor
         self.max_workers = max_workers
         self.strict_filters = strict_filters
+        self.pagination = pagination
+        self._cursor_supported = None
 
         self._register_extensions(extensions or [])
 
@@ -215,6 +226,42 @@ def version(self):
         ).get_version()
         return version
 
+    def _effective_pagination(self):
+        """Resolve the pagination strategy to use for list requests.
+
+        Returns ``"cursor"`` only when cursor pagination was requested *and*
+        the connected NetBox supports it (4.6+). Otherwise returns
+        ``"offset"``. The server version is probed once and cached, so only
+        the first `.all()`/`.filter()` on a cursor-mode `Api` pays the cost;
+        offset-mode instances never make the extra request.
+        """
+        if self.pagination != "cursor":
+            return "offset"
+        if self._cursor_supported is None:
+            try:
+                self._cursor_supported = version.parse(self.version) >= version.parse(
+                    "4.6"
+                )
+            except (RequestError, InvalidVersion, requests.exceptions.RequestException):
+                # RequestError covers a non-ok HTTP response from the version
+                # probe; requests.exceptions.RequestException covers transport
+                # failures (ConnectionError, Timeout, ...) raised before a
+                # response exists. In every case fall back to offset, as the
+                # docstring promises, and let the real list request surface any
+                # underlying connectivity error.
+                self._cursor_supported = False
+            if self._cursor_supported and self.threading:
+                # Cursor pagination follows next links sequentially and cannot
+                # be parallelised; the cursor path ignores self.threading.
+                # Warn so the no-op threading configuration is not a silent
+                # performance surprise.
+                warnings.warn(
+                    "threading=True has no effect with cursor pagination; "
+                    "cursor pages are fetched sequentially.",
+                    stacklevel=2,
+                )
+        return "cursor" if self._cursor_supported else "offset"
+
     def openapi(self):
         """Returns the OpenAPI spec.
 

pynetbox/core/endpoint.py

@@ -190,6 +190,7 @@ def all(self, limit=0, offset=None):
             max_workers=self.api.max_workers,
             limit=limit,
             offset=offset,
+            pagination=self.api._effective_pagination(),
         )
 
         return RecordSet(self, req)
@@ -367,6 +368,7 @@ def filter(self, *args, **kwargs):
             max_workers=self.api.max_workers,
             limit=limit,
             offset=offset,
+            pagination=self.api._effective_pagination(),
         )
 
         return RecordSet(self, req)

pynetbox/core/query.py

@@ -226,6 +226,7 @@ def __init__(
         thread_pool_executor=None,
         max_workers=4,
         expect_json=True,
+        pagination="offset",
     ):
         """Instantiates a new Request object.
 
@@ -240,6 +241,11 @@ def __init__(
         * **expect_json** (bool, optional): If True, expects JSON response
             and sets appropriate Accept header. If False, expects raw content
             (e.g., SVG, XML) and returns text. Defaults to True.
+        * **pagination** (str, optional): Pagination strategy for list
+            views, either ``"offset"`` (default) or ``"cursor"``. Cursor
+            pagination (NetBox 4.6+) pages with the ``start`` parameter and
+            follows ``next`` links sequentially; it omits the total count
+            and is mutually exclusive with threading.
 
         ## Note
 
@@ -264,6 +270,7 @@ def __init__(
         self.limit = limit
         self.offset = offset
         self.expect_json = expect_json
+        self.pagination = pagination
 
     def get_openapi(self):
         """Gets the OpenAPI Spec."""
@@ -446,15 +453,39 @@ def get(self, add_params=None):
         * ContentError if response is not json.
         """
 
+        # Cursor-based pagination (NetBox 4.6+) is used only for full
+        # iteration of a list view. An explicit offset (single requested
+        # page) or a detail route (add_params already set) falls back to
+        # the default offset behavior, since 'start' and 'offset' are
+        # mutually exclusive on the server.
+        use_cursor = (
+            self.pagination == "cursor"
+            and self.offset is None
+            and add_params is None
+        )
+
         if not add_params and self.limit is not None:
             add_params = {"limit": self.limit}
-            if self.limit and self.offset is not None:
+            if use_cursor:
+                # 'start' is the numeric id of the first object to return;
+                # begin at the start and follow the server's next links.
+                add_params["start"] = 0
+            elif self.limit and self.offset is not None:
                 # if non-zero limit and some offset -> add offset
                 add_params["offset"] = self.offset
         req = self._make_call(add_params=add_params)
         if isinstance(req, dict) and req.get("results") is not None:
+            # In cursor mode NetBox omits the count (returns null); it is
+            # fetched lazily by get_count() if len() is requested.
             self.count = req["count"]
-            if self.offset is not None:
+            if use_cursor:
+                # Sequentially follow next links; each link carries the
+                # next 'start' cursor (last pk + 1) computed by the server.
+                yield from req["results"]
+                while req.get("next"):
+                    req = self._make_call(url_override=req["next"])
+                    yield from req["results"]
+            elif self.offset is not None:
                 # only yield requested page results if paginating
                 for i in req["results"]:
                     yield i
@@ -603,6 +634,10 @@ def get_count(self, *args, **kwargs):
         * ContentError if response is not json.
         """
 
-        if not hasattr(self, "count"):
+        # ``count`` may be unset, or set to None when cursor pagination was
+        # used (NetBox omits the count in that mode). In both cases fetch it
+        # explicitly. This uses an offset-style request (no 'start' param),
+        # which always returns the total count.
+        if getattr(self, "count", None) is None:
             self.count = self._make_call(add_params={"limit": 1, "brief": 1})["count"]
         return self.count

pynetbox/core/response.py

@@ -136,13 +136,18 @@ def __next__(self):
 
     def __len__(self):
         try:
-            return self.request.count
+            count = self.request.count
         except AttributeError:
             try:
                 self._response_cache.append(next(self.response))
             except StopIteration:
                 return 0
-            return self.request.count
+            count = self.request.count
+        if count is None:
+            # Cursor-based pagination omits the total count; fetch it
+            # explicitly with a separate (offset-based) count request.
+            count = self.request.get_count()
+        return count
 
     def update(self, **kwargs):
         """Updates kwargs onto all Records in the RecordSet and saves these.

tests/integration/test_pagination.py

@@ -0,0 +1,56 @@
+import pytest
+from packaging import version
+
+import pynetbox
+
+
+@pytest.fixture(scope="module")
+def cursor_api(docker_netbox_service):
+    """An Api instance configured for cursor pagination (NetBox 4.6+)."""
+    nb = pynetbox.api(docker_netbox_service["url"], pagination="cursor")
+    nb.create_token("admin", "admin")
+    return nb
+
+
+@pytest.fixture(scope="module")
+def prefixes(api):
+    """Create enough prefixes to span more than one cursor page."""
+    created = [
+        api.ipam.prefixes.create(prefix="198.51.100.{}/32".format(i))
+        for i in range(5)
+    ]
+    yield created
+    for prefix in created:
+        prefix.delete()
+
+
+class TestCursorPagination:
+    def test_cursor_pagination_returns_all_objects(
+        self, cursor_api, nb_version, prefixes
+    ):
+        """Cursor pagination walks every page and yields all objects.
+
+        Gated to NetBox 4.6+, where cursor pagination is available. On older
+        versions the client transparently falls back to offset pagination, so
+        this end-to-end check of the ``start`` semantics and ``next`` link
+        following is only meaningful from 4.6 onward.
+        """
+        if nb_version < version.parse("4.6"):
+            pytest.skip("cursor pagination requires NetBox 4.6+")
+
+        assert cursor_api._effective_pagination() == "cursor"
+
+        # Force a small page size so the result set spans multiple cursor
+        # pages and the next-link following is actually exercised.
+        results = list(cursor_api.ipam.prefixes.filter(limit=2))
+        returned = {str(p.prefix) for p in results}
+        for prefix in prefixes:
+            assert str(prefix.prefix) in returned
+
+    def test_cursor_pagination_count(self, cursor_api, nb_version, prefixes):
+        """len() on a cursor RecordSet refetches the count NetBox omits."""
+        if nb_version < version.parse("4.6"):
+            pytest.skip("cursor pagination requires NetBox 4.6+")
+
+        record_set = cursor_api.ipam.prefixes.filter(limit=2)
+        assert len(record_set) >= len(prefixes)