feat: retire idle keep-alive connections after 2s to avoid stale-socket ReadError (OPS-4976)

CHANGELOG.md

@@ -1,5 +1,9 @@
 # CHANGELOG
 
+## [3.8.7] - 2026-06-26
+
+- [Changed] The HTTP clients now retire idle keep-alive connections after 2s (`httpx.Limits(keepalive_expiry=2.0)`, down from httpx's 5.0 default), tunable via the `PYGQLC_KEEPALIVE_EXPIRY` env var (a non-numeric or negative value falls back to the default with a warning). The expiry applies on both the default transport and the `ipv4_only` custom transport — httpx ignores the client-level `limits` when a custom transport is supplied, so the limits are threaded into the transport too. This stops the client from reusing a socket the server/LB has already closed during an idle gap — the stale keep-alive that surfaces as `ReadError('')`. Hot connections under load are reused within milliseconds, so pooling is preserved; only genuinely idle sockets are dropped. Pairs with the 3.8.4 fresh-connection retry: fewer stale sockets to begin with, so the retry (which is unsafe to apply blindly to non-idempotent mutations) fires far less often.
+
 ## [3.8.6] - 2026-06-26
 
 - [Fixed] `_get_async_client` reuses the shared client instead of recreating it every call. Its probe awaited a non-existent `get_timeout()`, so every call closed + rebuilt the client — under concurrency that closed it mid-use elsewhere (`Cannot send a request, as the client has been closed.`). Now rebuilt only when missing or `is_closed`, and that error is retryable.

pygqlc/GraphQLClient.py

@@ -8,6 +8,7 @@
 """
 
 import asyncio
+import os
 import traceback
 import time
 import threading
@@ -50,6 +51,44 @@
     httpx.ConnectTimeout,
 )
 
+# Drop an idle pooled keep-alive connection after this many seconds so we never
+# reuse a socket the server has already closed (the stale keep-alive that
+# surfaces as ReadError('')). httpx defaults to 5.0, which is long enough for a
+# server/LB with a shorter idle timeout to close the socket first. 2.0 sits below
+# any reasonable server idle timeout while still pooling hot connections under
+# load (those are reused within milliseconds). Tunable via PYGQLC_KEEPALIVE_EXPIRY.
+DEFAULT_KEEPALIVE_EXPIRY = 2.0
+
+
+def _resolve_keepalive_expiry() -> float:
+    """Idle keep-alive expiry (seconds) from the PYGQLC_KEEPALIVE_EXPIRY env var.
+
+    Falls back to DEFAULT_KEEPALIVE_EXPIRY (with a WARNING) on a non-numeric or
+    negative value, so a misconfigured env var degrades to the safe default
+    instead of crashing GraphQLClient construction (and the worker) at startup.
+    """
+    raw = os.environ.get("PYGQLC_KEEPALIVE_EXPIRY")
+    if raw is None:
+        return DEFAULT_KEEPALIVE_EXPIRY
+    try:
+        value = float(raw)
+    except ValueError:
+        log(
+            LogLevel.WARNING,
+            f"Invalid PYGQLC_KEEPALIVE_EXPIRY={raw!r} (not a number); "
+            f"using default {DEFAULT_KEEPALIVE_EXPIRY}s",
+        )
+        return DEFAULT_KEEPALIVE_EXPIRY
+    if value < 0:
+        log(
+            LogLevel.WARNING,
+            f"Invalid PYGQLC_KEEPALIVE_EXPIRY={raw!r} (negative); "
+            f"using default {DEFAULT_KEEPALIVE_EXPIRY}s",
+        )
+        return DEFAULT_KEEPALIVE_EXPIRY
+    return value
+
+
 # * Custom Exception class for GraphQL responses
 
 
@@ -286,9 +325,20 @@ def __init__(self):
         self.pingIntervalTime = 15
         self.pingTimer = time.time()
 
-        # Setup common client parameters
-        self.client_params = {"http2": True}
-        self.async_client_params = {"http2": True}
+        # Setup common client parameters. Retire idle keep-alive connections
+        # quickly (see DEFAULT_KEEPALIVE_EXPIRY) so a socket the server has
+        # already closed is never reused. Stashed on self so _update_client_params
+        # can thread the same limits into a custom transport (ipv4_only), where
+        # httpx ignores the Client-level `limits` kwarg.
+        # Keep httpx's standard pool sizing (passing only keepalive_expiry would
+        # reset both maxes to None / unlimited); just shorten the idle expiry.
+        self._limits = httpx.Limits(
+            max_connections=100,
+            max_keepalive_connections=20,
+            keepalive_expiry=_resolve_keepalive_expiry(),
+        )
+        self.client_params = {"http2": True, "limits": self._limits}
+        self.async_client_params = {"http2": True, "limits": self._limits}
 
         # Reuse HTTP client for better performance
         self._http_client = None
@@ -963,11 +1013,17 @@ def _update_client_params(self, ipv4_only):
             # Binding the local egress address to 0.0.0.0 forces httpx to use an
             # IPv4 source socket (the standard idiom for IPv4-only egress). This is
             # not a listening socket, so the bind-all-interfaces concern does not apply.
+            # httpx applies the Client-level `limits` only to its *default*
+            # transport; with a custom transport the limits kwarg is ignored, so
+            # pass our limits (keepalive_expiry) into the transport itself or the
+            # mitigation is silently dropped on this path.
             self.client_params["transport"] = httpx.HTTPTransport(
-                local_address="0.0.0.0"  # nosec B104
+                local_address="0.0.0.0",  # nosec B104
+                limits=self._limits,
             )
             self.async_client_params["transport"] = httpx.AsyncHTTPTransport(
-                local_address="0.0.0.0"  # nosec B104
+                local_address="0.0.0.0",  # nosec B104
+                limits=self._limits,
             )
         else:
             # Remove transport if it exists

pygqlc/__version__.py

@@ -1 +1 @@
-__version__ = "3.8.6"
+__version__ = "3.8.7"

tests/pygqlc/gql_client/test_keepalive_expiry.py

@@ -0,0 +1,91 @@
+"""Idle keep-alive connections are retired quickly via httpx ``keepalive_expiry``
+so a socket the server already closed is never reused — the stale keep-alive that
+surfaces as ``ReadError('')``. Default 2s, tunable via ``PYGQLC_KEEPALIVE_EXPIRY``.
+"""
+
+import httpx
+import pytest
+
+from pygqlc import GraphQLClient
+from pygqlc.GraphQLClient import DEFAULT_KEEPALIVE_EXPIRY
+from pygqlc.helper_modules.Singleton import Singleton
+
+
+@pytest.fixture
+def fresh_client():
+    """Build GraphQLClient instances whose ``__init__`` re-reads the env, without
+    leaking state into the process-wide singleton other tests share."""
+    saved = Singleton._instances.pop(GraphQLClient, None)
+
+    def _build():
+        Singleton._instances.pop(GraphQLClient, None)
+        return GraphQLClient()
+
+    yield _build
+
+    Singleton._instances.pop(GraphQLClient, None)
+    if saved is not None:
+        Singleton._instances[GraphQLClient] = saved
+
+
+def test_default_keepalive_expiry_on_both_clients(fresh_client, monkeypatch):
+    monkeypatch.delenv("PYGQLC_KEEPALIVE_EXPIRY", raising=False)
+    gql = fresh_client()
+    for params in (gql.client_params, gql.async_client_params):
+        limits = params["limits"]
+        assert isinstance(limits, httpx.Limits)
+        assert limits.keepalive_expiry == DEFAULT_KEEPALIVE_EXPIRY
+        # httpx's standard pool caps must be preserved, not reset to None.
+        assert limits.max_connections == 100
+        assert limits.max_keepalive_connections == 20
+
+
+def test_keepalive_expiry_env_override(fresh_client, monkeypatch):
+    monkeypatch.setenv("PYGQLC_KEEPALIVE_EXPIRY", "0.5")
+    gql = fresh_client()
+    assert gql.client_params["limits"].keepalive_expiry == 0.5
+    assert gql.async_client_params["limits"].keepalive_expiry == 0.5
+
+
+def test_default_below_httpx_default():
+    # The point of the mitigation: shorter than httpx's own default so a
+    # server/LB with a shorter idle timeout can't close the socket first.
+    # Compare against the live httpx default so this doesn't go stale on upgrades.
+    assert DEFAULT_KEEPALIVE_EXPIRY < httpx.Limits().keepalive_expiry
+
+
+def test_invalid_env_falls_back_to_default(fresh_client, monkeypatch):
+    monkeypatch.setenv("PYGQLC_KEEPALIVE_EXPIRY", "not-a-number")
+    gql = fresh_client()
+    assert gql.client_params["limits"].keepalive_expiry == DEFAULT_KEEPALIVE_EXPIRY
+
+
+def test_negative_env_falls_back_to_default(fresh_client, monkeypatch):
+    monkeypatch.setenv("PYGQLC_KEEPALIVE_EXPIRY", "-5")
+    gql = fresh_client()
+    assert gql.client_params["limits"].keepalive_expiry == DEFAULT_KEEPALIVE_EXPIRY
+
+
+def test_ipv4_only_threads_limits_into_transports(fresh_client, monkeypatch):
+    """In ipv4_only mode a custom transport is installed; httpx ignores the
+    Client-level limits there, so the limits must be passed into the transport."""
+    gql = fresh_client()
+    captured = {}
+
+    def fake_http_transport(**kwargs):
+        captured["sync"] = kwargs
+        return object()
+
+    def fake_async_transport(**kwargs):
+        captured["async"] = kwargs
+        return object()
+
+    monkeypatch.setattr(httpx, "HTTPTransport", fake_http_transport)
+    monkeypatch.setattr(httpx, "AsyncHTTPTransport", fake_async_transport)
+
+    gql._update_client_params(ipv4_only=True)
+
+    assert captured["sync"]["limits"] is gql._limits
+    assert captured["async"]["limits"] is gql._limits
+    assert captured["sync"]["local_address"] == "0.0.0.0"
+    assert captured["async"]["local_address"] == "0.0.0.0"