#764 - Support cursor-based pagination from NetBox 4.6

.github/workflows/py3.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       matrix:
         python: ["3.12", "3.13", "3.14"]
-        netbox: ["4.3", "4.4", "4.5"]
+        netbox: ["4.4", "4.5", "4.6"]
 
     steps:
       - name: Checkout pynetbox

docs/advanced.md

@@ -108,6 +108,54 @@ nb = pynetbox.api(
 The executor is constructed once per threaded query and shut down when that query's pages
 have been fetched, so pass the class (or a factory), not an already-instantiated pool.
 
+## Cursor-Based Pagination
+
+Starting with NetBox 4.6, the REST API supports cursor-based pagination as an
+alternative to the default offset-based pagination. Instead of skipping a
+growing number of rows with an `offset`, the server pages forward using the
+primary key (`id`) as a cursor. This avoids the cost of scanning the table up to
+the offset position, so it performs significantly better on very large result
+sets.
+
+### Enabling Cursor Pagination
+
+Pass `pagination="cursor"` when constructing the API client:
+
+```python
+import pynetbox
+
+nb = pynetbox.api(
+    'http://localhost:8000',
+    token='your-token',
+    pagination="cursor",
+)
+
+# .all() and .filter() now page using the server's `start` cursor
+devices = nb.dcim.devices.all()
+```
+
+pynetbox probes the NetBox version once on the first query. If the server is
+older than 4.6 (and therefore does not support cursor pagination), it
+transparently falls back to offset-based pagination, so it is safe to leave this
+option enabled across mixed environments.
+
+### Trade-offs
+
+!!! note "Things to know about cursor pagination"
+    - **No total count up front.** NetBox omits the `count` in cursor mode for
+      performance. pynetbox still supports `len(record_set)`, but doing so
+      triggers a separate count request.
+    - **Fixed ordering.** Results are always ordered by `id`; an explicit
+      `ordering` filter cannot be combined with cursor pagination.
+    - **Not combined with threading.** Cursor pagination is inherently
+      sequential (each page's cursor depends on the previous page), so it cannot
+      be parallelized. When `pagination="cursor"` is set, queries page
+      sequentially even if `threading=True`. For parallel fetching of large
+      offset-paginated result sets, use threading instead (see above).
+    - **Explicit `offset` still works.** Passing an explicit `offset` to
+      `.all()`/`.filter()` requests a single offset-based page as before, since
+      `start` and `offset` are mutually exclusive on the server.
+
 ## Filter Validation
 
 NetBox does not validate filter parameters passed to list endpoints. An unrecognized parameter is silently ignored, which means a typo in a `.filter()` or `.get()` call can quietly return the entire table.

pynetbox/core/api.py

@@ -15,11 +15,14 @@
 """
 
 import contextlib
+import warnings
 
 import requests
+from packaging import version
+from packaging.version import InvalidVersion
 
 from pynetbox.core.app import App, PluginsApp
-from pynetbox.core.query import Request, TOKEN_PREFIX
+from pynetbox.core.query import Request, RequestError, TOKEN_PREFIX
 from pynetbox.core.response import Record
 from pynetbox.models.mapper import CONTENT_TYPE_MAPPER
 
@@ -89,6 +92,7 @@ def __init__(
         threading=False,
         strict_filters=False,
         extensions=None,
+        pagination="offset",
         thread_pool_executor=None,
         max_workers=4,
     ):
@@ -100,9 +104,14 @@ def __init__(
             threading (bool, optional): Set to True to use threading in `.all()` and `.filter()` requests, defaults to False.
             strict_filters (bool, optional): Set to True to check GET call filters against OpenAPI specifications (intentionally not done in NetBox API), defaults to False.
             extensions (list, optional): A list of `Extension` classes or instances that register custom `Record` subclasses and content-type mappings for NetBox plugins. See `pynetbox.core.extension`.
+            pagination (str, optional): Pagination strategy for `.all()` and `.filter()`, either `"offset"` (default) or `"cursor"`. Cursor pagination (NetBox 4.6+) offers better performance on very large result sets but omits the total count and cannot be combined with threading or `ordering`. On NetBox versions older than 4.6 it transparently falls back to offset pagination.
             thread_pool_executor (callable, optional): A `concurrent.futures.ThreadPoolExecutor` class, or any callable matching its `(max_workers=...)` signature and context-manager protocol, used to build the pool for threaded requests. Defaults to `concurrent.futures.ThreadPoolExecutor`.
             max_workers (int, optional): Maximum number of worker threads used for threaded requests, defaults to 4.
         """
+        if pagination not in ("offset", "cursor"):
+            raise ValueError(
+                "pagination must be 'offset' or 'cursor', got {!r}".format(pagination)
+            )
         if max_workers <= 0:
             raise ValueError("max_workers must be a positive integer")
 
@@ -119,6 +128,8 @@ def __init__(
         self.thread_pool_executor = thread_pool_executor
         self.max_workers = max_workers
         self.strict_filters = strict_filters
+        self.pagination = pagination
+        self._cursor_supported = None
 
         self._register_extensions(extensions or [])
 
@@ -215,6 +226,48 @@ def version(self):
         ).get_version()
         return version
 
+    def _effective_pagination(self):
+        """Resolve the pagination strategy to use for list requests.
+
+        Returns ``"cursor"`` only when cursor pagination was requested *and*
+        the connected NetBox supports it (4.6+). Otherwise returns
+        ``"offset"``. The server version is probed once and cached, so only
+        the first `.all()`/`.filter()` on a cursor-mode `Api` pays the cost;
+        offset-mode instances never make the extra request.
+        """
+        if self.pagination != "cursor":
+            return "offset"
+        if self._cursor_supported is None:
+            try:
+                self._cursor_supported = version.parse(self.version) >= version.parse(
+                    "4.6"
+                )
+            except (RequestError, InvalidVersion, requests.exceptions.RequestException):
+                # RequestError covers a non-ok HTTP response from the version
+                # probe; requests.exceptions.RequestException covers transport
+                # failures (ConnectionError, Timeout, ...) raised before a
+                # response exists. In every case fall back to offset, as the
+                # docstring promises, and let the real list request surface any
+                # underlying connectivity error.
+                self._cursor_supported = False
+            if self._cursor_supported and self.threading:
+                # Cursor pagination follows next links sequentially and cannot
+                # be parallelised; the cursor path ignores self.threading.
+                # Warn so the no-op threading configuration is not a silent
+                # performance surprise.
+                # stacklevel=5 attributes the warning to the caller's list
+                # request rather than to pynetbox internals. The version probe
+                # is resolved lazily on the first page fetch, so the fixed
+                # frame chain at this point is:
+                #   _effective_pagination -> Request._resolve_pagination
+                #   -> Request.get -> RecordSet.__next__/__len__ -> caller.
+                warnings.warn(
+                    "threading=True has no effect with cursor pagination; "
+                    "cursor pages are fetched sequentially.",
+                    stacklevel=5,
+                )
+        return "cursor" if self._cursor_supported else "offset"
+
     def openapi(self):
         """Returns the OpenAPI spec.
 

pynetbox/core/endpoint.py

@@ -190,6 +190,10 @@ def all(self, limit=0, offset=None):
             max_workers=self.api.max_workers,
             limit=limit,
             offset=offset,
+            # Passed uncalled so the version probe behind
+            # _effective_pagination() is deferred until the request actually
+            # runs, rather than firing when this lazy RecordSet is built.
+            pagination=self.api._effective_pagination,
         )
 
         return RecordSet(self, req)
@@ -367,6 +371,10 @@ def filter(self, *args, **kwargs):
             max_workers=self.api.max_workers,
             limit=limit,
             offset=offset,
+            # Passed uncalled so the version probe behind
+            # _effective_pagination() is deferred until the request actually
+            # runs, rather than firing when this lazy RecordSet is built.
+            pagination=self.api._effective_pagination,
         )
 
         return RecordSet(self, req)

pynetbox/core/query.py

@@ -18,6 +18,7 @@
 import io
 import os
 import json
+import warnings
 
 from packaging import version
 
@@ -226,6 +227,7 @@ def __init__(
         thread_pool_executor=None,
         max_workers=4,
         expect_json=True,
+        pagination="offset",
     ):
         """Instantiates a new Request object.
 
@@ -240,6 +242,14 @@ def __init__(
         * **expect_json** (bool, optional): If True, expects JSON response
             and sets appropriate Accept header. If False, expects raw content
             (e.g., SVG, XML) and returns text. Defaults to True.
+        * **pagination** (str or callable, optional): Pagination strategy for
+            list views, either ``"offset"`` (default) or ``"cursor"``. May
+            also be a zero-argument callable returning one of those strings,
+            which is resolved lazily on the first list request so any work it
+            performs (e.g. a server version probe) is deferred until needed.
+            Cursor pagination (NetBox 4.6+) pages with the ``start`` parameter
+            and follows ``next`` links sequentially; it omits the total count
+            and is mutually exclusive with threading.
 
         ## Note
 
@@ -264,6 +274,7 @@ def __init__(
         self.limit = limit
         self.offset = offset
         self.expect_json = expect_json
+        self.pagination = pagination
 
     def get_openapi(self):
         """Gets the OpenAPI Spec."""
@@ -432,6 +443,18 @@ def concurrent_get(self, ret, page_size, page_offsets):
                 result = future.result()
                 ret.extend(result["results"])
 
+    def _resolve_pagination(self):
+        """Resolve the configured pagination strategy.
+
+        ``pagination`` may be either a string (``"offset"``/``"cursor"``) or a
+        zero-argument callable returning one. Deferring the call to here keeps
+        any work it does (such as the NetBox version probe behind
+        ``Api._effective_pagination``) out of ``Request`` construction, so a
+        lazily-built ``RecordSet`` that is never iterated makes no extra call.
+        """
+        pagination = self.pagination
+        return pagination() if callable(pagination) else pagination
+
     def get(self, add_params=None):
         """Makes a GET request.
 
@@ -446,15 +469,52 @@ def get(self, add_params=None):
         * ContentError if response is not json.
         """
 
+        # Cursor-based pagination (NetBox 4.6+) is used only for full
+        # iteration of a list view. An explicit offset (single requested
+        # page) or a detail route (add_params already set) falls back to
+        # the default offset behavior, since 'start' and 'offset' are
+        # mutually exclusive on the server.
+        use_cursor = (
+            self._resolve_pagination() == "cursor"
+            and self.offset is None
+            and add_params is None
+        )
+
+        if use_cursor and self.filters and "ordering" in self.filters:
+            # Cursor pagination derives the page boundaries from a fixed
+            # server-side ordering, so a caller-supplied 'ordering' filter is
+            # silently ignored by NetBox. Warn rather than let the result come
+            # back in an unexpected order.
+            # stacklevel=3: get() (generator body, resumed on the first
+            # next()) -> RecordSet.__next__/__len__ -> caller.
+            warnings.warn(
+                "ordering has no effect with cursor pagination; results are "
+                "returned in NetBox's fixed cursor order.",
+                stacklevel=3,
+            )
+
         if not add_params and self.limit is not None:
             add_params = {"limit": self.limit}
-            if self.limit and self.offset is not None:
+            if use_cursor:
+                # 'start' is the numeric id of the first object to return;
+                # begin at the start and follow the server's next links.
+                add_params["start"] = 0
+            elif self.limit and self.offset is not None:
                 # if non-zero limit and some offset -> add offset
                 add_params["offset"] = self.offset
         req = self._make_call(add_params=add_params)
         if isinstance(req, dict) and req.get("results") is not None:
+            # In cursor mode NetBox omits the count (returns null); it is
+            # fetched lazily by get_count() if len() is requested.
             self.count = req["count"]
-            if self.offset is not None:
+            if use_cursor:
+                # Sequentially follow next links; each link carries the
+                # next 'start' cursor (last pk + 1) computed by the server.
+                yield from req["results"]
+                while req.get("next"):
+                    req = self._make_call(url_override=req["next"])
+                    yield from req["results"]
+            elif self.offset is not None:
                 # only yield requested page results if paginating
                 for i in req["results"]:
                     yield i
@@ -603,6 +663,10 @@ def get_count(self, *args, **kwargs):
         * ContentError if response is not json.
         """
 
-        if not hasattr(self, "count"):
+        # ``count`` may be unset, or set to None when cursor pagination was
+        # used (NetBox omits the count in that mode). In both cases fetch it
+        # explicitly. This uses an offset-style request (no 'start' param),
+        # which always returns the total count.
+        if getattr(self, "count", None) is None:
             self.count = self._make_call(add_params={"limit": 1, "brief": 1})["count"]
         return self.count

pynetbox/core/response.py

@@ -136,13 +136,18 @@ def __next__(self):
 
     def __len__(self):
         try:
-            return self.request.count
+            count = self.request.count
         except AttributeError:
             try:
                 self._response_cache.append(next(self.response))
             except StopIteration:
                 return 0
-            return self.request.count
+            count = self.request.count
+        if count is None:
+            # Cursor-based pagination omits the total count; fetch it
+            # explicitly with a separate (offset-based) count request.
+            count = self.request.get_count()
+        return count
 
     def update(self, **kwargs):
         """Updates kwargs onto all Records in the RecordSet and saves these.

tests/integration/conftest.py

@@ -34,6 +34,8 @@ def get_netbox_docker_version_tag(netbox_version):
         tag = "3.4.2"
     elif (major, minor) == (4, 5):
         tag = "4.0.2"
+    elif (major, minor) == (4, 6):
+        tag = "5.0.1"
     else:
         raise NotImplementedError(
             "Version %s is not currently supported" % netbox_version