fix: retry once on stale pooled connection before returning fallback

lib/split/sockets/pool.ex

@@ -58,6 +58,25 @@ defmodule Split.Sockets.Pool do
                {:ok, conn, resp} <- Conn.send_message(conn, message) do
             {{:ok, resp}, update_if_open(conn, state, caller)}
           else
+            {:error, conn, _error} when state == :reused ->
+              # Connection was reused but failed mid-request (stale socket after
+              # splitd restart). Disconnect the dead socket and retry once with
+              # a fresh connection within the same checkout window.
+              conn = Conn.disconnect(conn)
+
+              with {:ok, conn} <- Conn.connect(conn),
+                   {:ok, conn, resp} <- Conn.send_message(conn, message) do
+                NimblePool.update(caller, conn)
+                {{:ok, resp}, {:ok, conn}}
+              else
+                {:error, conn, retry_error} ->
+                  Logger.error(
+                    "Retry after stale connection also failed: #{inspect(retry_error)}"
+                  )
+
+                  {{:error, retry_error}, update_if_open(conn, :new, caller)}
+              end
+
             {:error, conn, error} ->
               {{:error, error}, update_if_open(conn, state, caller)}
           end

test/sockets/pool_test.exs

@@ -107,4 +107,74 @@ defmodule Split.Sockets.PoolTest do
 
   # Wait 5 milliseconds, enough time for the connection to be returned back to the pool.
   defp wait_connection_checkin(), do: Process.sleep(5)
+
+  describe "stale connection retry" do
+    setup context do
+      # Dedicated pool with pool_size: 1 so we can deterministically
+      # hit the same connection after restarting the mock server.
+      test_id = :erlang.phash2(context.test)
+      address = "/tmp/test-splitd-retry-#{test_id}.sock"
+      server_name = :"retry-server-#{test_id}"
+      pool_name = :"retry-pool-#{test_id}"
+
+      start_supervised!(
+        {Split.Test.MockSplitdServer, address: address, name: server_name},
+        id: server_name
+      )
+
+      Split.Test.MockSplitdServer.wait_until_listening(address)
+
+      start_supervised!(
+        {Split, address: address, pool_name: pool_name, pool_size: 1},
+        id: pool_name
+      )
+
+      {:ok, address: address, server_name: server_name, pool_name: pool_name}
+    end
+
+    test "retries transparently when splitd restarts mid-pool-lifecycle", %{
+      address: address,
+      server_name: server_name,
+      pool_name: pool_name
+    } do
+      message = Message.get_treatment(key: "user-id", feature_name: "feature")
+
+      # Establish a pooled connection
+      assert {:ok, _resp} = Pool.send_message(message, pool_name: pool_name)
+      wait_connection_checkin()
+
+      # Kill the mock splitd server (simulates splitd sidecar restart).
+      # The pooled socket is now stale.
+      stop_supervised!(server_name)
+
+      # Restart the mock server on the same address (splitd comes back up)
+      start_supervised!(
+        {Split.Test.MockSplitdServer, address: address, name: server_name},
+        id: server_name
+      )
+
+      Split.Test.MockSplitdServer.wait_until_listening(address)
+
+      # This should: hit the stale connection, get :closed, retry with
+      # a fresh connection to the new server, and succeed.
+      assert {:ok, _resp} = Pool.send_message(message, pool_name: pool_name)
+    end
+
+    test "returns error when splitd is completely down (no retry helps)", %{
+      server_name: server_name,
+      pool_name: pool_name
+    } do
+      message = Message.get_treatment(key: "user-id", feature_name: "feature")
+
+      # Establish a pooled connection
+      assert {:ok, _resp} = Pool.send_message(message, pool_name: pool_name)
+      wait_connection_checkin()
+
+      # Kill splitd and don't restart it
+      stop_supervised!(server_name)
+
+      # Should fail: stale connection fails, retry also fails (no server)
+      assert {:error, _reason} = Pool.send_message(message, pool_name: pool_name)
+    end
+  end
 end