From f7eb55cbcf9ce03d77eb7ee4e92a1f4da6091287 Mon Sep 17 00:00:00 2001 From: Marlies Mayerhofer <74332854+marliessophie@users.noreply.github.com> Date: Wed, 18 Feb 2026 12:55:28 +0100 Subject: [PATCH 1/6] chore(experiment-runner): ensure backwards compatibility for old server versions --- langfuse/_client/client.py | 51 +++++++++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/langfuse/_client/client.py b/langfuse/_client/client.py index c8782c638..257629eee 100644 --- a/langfuse/_client/client.py +++ b/langfuse/_client/client.py @@ -130,6 +130,27 @@ from langfuse.types import MaskFunction, ScoreDataType, SpanLevel, TraceContext +def _is_unrecognized_field_error(e: Exception, field_name: str) -> bool: + """Check if error contains an unrecognized field. + + Used for backward compatibility when newer SDK sends fields not supported by older servers. + + Args: + e: Exception from API call + field_name: Name of field to check for in error + + Returns: + True if error indicates field_name is unrecognized by the server + """ + try: + return any( + field_name in error.get("keys", []) + for error in getattr(e, "body", {}).get("error", []) + ) + except (AttributeError, TypeError): + return False + + class Langfuse: """Main client for Langfuse tracing and platform features. @@ -2937,7 +2958,35 @@ async def _process_experiment_item( dataset_run_id = dataset_run_item.dataset_run_id except Exception as e: - langfuse_logger.error(f"Failed to create dataset run item: {e}") + # Only retry if the error is specifically about datasetVersion being unrecognized + # This handles backward compatibility with older server versions + if _is_unrecognized_field_error(e, "datasetVersion"): + langfuse_logger.warning( + "Server doesn't support datasetVersion field, retrying without it" + ) + try: + dataset_run_item = await asyncio.to_thread( + self.api.dataset_run_items.create, + request=CreateDatasetRunItemRequest( + runName=experiment_run_name, + runDescription=experiment_description, + metadata=experiment_metadata, + datasetItemId=item.id, # type: ignore + traceId=trace_id, + observationId=span.id, + # Note: datasetVersion omitted for backward compatibility + ), + ) + dataset_run_id = dataset_run_item.dataset_run_id + except Exception as retry_error: + error_msg = ( + f"Failed to create dataset run item: {retry_error}" + ) + langfuse_logger.error(error_msg) + else: + # Different error, just log it + error_msg = f"Failed to create dataset run item: {e}" + langfuse_logger.error(error_msg) if ( not isinstance(item, dict) From 77c5e76882588e081c3b5dc38abc9948c4b89527 Mon Sep 17 00:00:00 2001 From: Marlies Mayerhofer <74332854+marliessophie@users.noreply.github.com> Date: Wed, 18 Feb 2026 17:13:24 +0100 Subject: [PATCH 2/6] feat(http): add automatic retry for unknown field errors in API requests --- langfuse/_client/http_retry_patch.py | 132 +++++++++++++ langfuse/_client/resource_manager.py | 14 ++ tests/test_unknown_field_retry.py | 268 +++++++++++++++++++++++++++ 3 files changed, 414 insertions(+) create mode 100644 langfuse/_client/http_retry_patch.py create mode 100644 tests/test_unknown_field_retry.py diff --git a/langfuse/_client/http_retry_patch.py b/langfuse/_client/http_retry_patch.py new file mode 100644 index 000000000..22381e385 --- /dev/null +++ b/langfuse/_client/http_retry_patch.py @@ -0,0 +1,132 @@ +"""HTTP client patch for handling unknown field errors. + +This module provides functionality to automatically retry API requests when the server +rejects unknown or unexpected fields. This enables backward compatibility when using +newer SDK versions with older server versions. +""" + +import logging +from typing import Any, Set + +import httpx + +logger = logging.getLogger(__name__) + + +def patch_client_for_unknown_field_retry(client: httpx.Client) -> None: + """Patch an httpx.Client instance to automatically retry on unknown field errors. + + When the server returns a 400/422 error with unrecognized_keys, this automatically + retries the request with those fields removed. + + Args: + client: The httpx.Client instance to patch + + Example: + >>> client = httpx.Client() + >>> patch_client_for_unknown_field_retry(client) + >>> # Now all requests through this client will handle unknown field errors + """ + original_request = client.request + + def request_with_retry( + method: str, url: str | httpx.URL, **kwargs: Any + ) -> httpx.Response: + """Wrapped request that handles unknown field errors with retry.""" + response = original_request(method, url, **kwargs) + + # Retry if server rejected unrecognized keys + if response.status_code in [400, 422] and "json" in kwargs: + try: + unknown_keys = _extract_unknown_keys(response) + if unknown_keys: + logger.warning( + "Server rejected unrecognized keys %s for %s %s. Retrying without these fields.", + unknown_keys, + method, + url, + ) + kwargs["json"] = _remove_fields(kwargs["json"], unknown_keys) + response = original_request(method, url, **kwargs) + except Exception as e: + logger.debug("Failed to parse unknown field error: %s", e) + + return response + + client.request = request_with_retry # type: ignore[method-assign] + + +def patch_async_client_for_unknown_field_retry(client: httpx.AsyncClient) -> None: + """Patch an httpx.AsyncClient instance to automatically retry on unknown field errors. + + Async version of patch_client_for_unknown_field_retry. + + Args: + client: The httpx.AsyncClient instance to patch + """ + original_request = client.request + + async def request_with_retry( + method: str, url: str | httpx.URL, **kwargs: Any + ) -> httpx.Response: + """Wrapped async request that handles unknown field errors with retry.""" + response = await original_request(method, url, **kwargs) + + # Retry if server rejected unrecognized keys + if response.status_code in [400, 422] and "json" in kwargs: + try: + unknown_keys = _extract_unknown_keys(response) + if unknown_keys: + logger.warning( + "Server rejected unrecognized keys %s for %s %s. Retrying without these fields.", + unknown_keys, + method, + url, + ) + kwargs["json"] = _remove_fields(kwargs["json"], unknown_keys) + response = await original_request(method, url, **kwargs) + except Exception as e: + logger.debug("Failed to parse unknown field error: %s", e) + + return response + + client.request = request_with_retry # type: ignore[method-assign] + + +def _extract_unknown_keys(response: httpx.Response) -> Set[str]: + """Extract unknown keys from server error response. + + Args: + response: The HTTP response from the server + + Returns: + Set of field names that were rejected as unrecognized + """ + body = response.json() + if isinstance(body, dict) and "error" in body: + unknown_keys = set() + for error in body.get("error", []): + if isinstance(error, dict) and error.get("code") == "unrecognized_keys": + unknown_keys.update(error.get("keys", [])) + return unknown_keys + return set() + + +def _remove_fields(data: Any, fields: Set[str]) -> Any: + """Remove specified fields from nested dict/list structures. + + Args: + data: The data structure to filter (dict, list, or primitive) + fields: Set of field names to remove + + Returns: + Filtered data structure with specified fields removed + """ + if isinstance(data, dict): + return { + k: _remove_fields(v, fields) for k, v in data.items() if k not in fields + } + elif isinstance(data, list): + return [_remove_fields(item, fields) for item in data] + else: + return data diff --git a/langfuse/_client/resource_manager.py b/langfuse/_client/resource_manager.py index 08c008234..5dd9580a0 100644 --- a/langfuse/_client/resource_manager.py +++ b/langfuse/_client/resource_manager.py @@ -35,6 +35,10 @@ LANGFUSE_RELEASE, LANGFUSE_TRACING_ENVIRONMENT, ) +from langfuse._client.http_retry_patch import ( + patch_async_client_for_unknown_field_retry, + patch_client_for_unknown_field_retry, +) from langfuse._client.span_processor import LangfuseSpanProcessor from langfuse._task_manager.media_manager import MediaManager from langfuse._task_manager.media_upload_consumer import MediaUploadConsumer @@ -213,6 +217,10 @@ def _initialize_instance( client_headers = additional_headers if additional_headers else {} self.httpx_client = httpx.Client(timeout=timeout, headers=client_headers) + # Patch the httpx client to handle unknown field errors automatically + # This enables backward compatibility when newer SDK versions interact with older servers + patch_client_for_unknown_field_retry(self.httpx_client) + self.api = FernLangfuse( base_url=base_url, username=self.public_key, @@ -223,6 +231,11 @@ def _initialize_instance( httpx_client=self.httpx_client, timeout=timeout, ) + + # Create and patch async client for async API operations + async_httpx_client = httpx.AsyncClient(timeout=timeout) + patch_async_client_for_unknown_field_retry(async_httpx_client) + self.async_api = AsyncFernLangfuse( base_url=base_url, username=self.public_key, @@ -230,6 +243,7 @@ def _initialize_instance( x_langfuse_sdk_name="python", x_langfuse_sdk_version=langfuse_version, x_langfuse_public_key=self.public_key, + httpx_client=async_httpx_client, timeout=timeout, ) score_ingestion_client = LangfuseClient( diff --git a/tests/test_unknown_field_retry.py b/tests/test_unknown_field_retry.py new file mode 100644 index 000000000..b602f7d3c --- /dev/null +++ b/tests/test_unknown_field_retry.py @@ -0,0 +1,268 @@ +"""Tests for unknown field retry functionality. + +This module tests the automatic retry behavior when the server rejects unknown fields, +enabling backward compatibility between newer SDK versions and older server versions. +""" + +import httpx +import pytest +from datetime import datetime + +from langfuse import Langfuse +from langfuse._client.http_retry_patch import _remove_fields +from langfuse._client.resource_manager import LangfuseResourceManager +from langfuse.api.core import ApiError + + +class TestUnknownFieldRetry: + """Test that SDK handles unknown field errors gracefully.""" + + @pytest.fixture(autouse=True) + def reset_resource_manager(self): + """Reset LangfuseResourceManager singleton before each test.""" + LangfuseResourceManager.reset() + yield + LangfuseResourceManager.reset() + + @pytest.fixture + def complete_dataset_response(self): + """Provide a complete dataset response with all required fields.""" + return { + "id": "ds-1", + "name": "test", + "projectId": "proj-1", + "metadata": {}, + "createdAt": datetime.now().isoformat(), + "updatedAt": datetime.now().isoformat(), + } + + def test_datasets_route_retries(self, monkeypatch, complete_dataset_response): + """Test retry works for datasets route.""" + calls = [] + + def mock_request(client_self, method, url, **kwargs): + calls.append({"method": method, "url": str(url)}) + + if "datasets" in str(url) and len(calls) == 1: + # First call to datasets returns unrecognized_keys error + return httpx.Response( + 400, + json={ + "message": "Invalid request data", + "error": [{"code": "unrecognized_keys", "keys": ["badField"]}], + }, + ) + elif "datasets" in str(url): + # Retry succeeds + return httpx.Response(200, json=complete_dataset_response) + else: + # Health check + return httpx.Response(200, json={"status": "OK"}) + + # Patch BEFORE creating the client + monkeypatch.setattr("httpx.Client.request", mock_request) + + langfuse = Langfuse( + public_key="pk-test", secret_key="sk-test", host="http://localhost:3000" + ) + dataset = langfuse.create_dataset(name="test") + + assert dataset.id == "ds-1" + dataset_calls = [c for c in calls if "datasets" in c["url"]] + assert len(dataset_calls) == 2, "Should retry once on unrecognized_keys error" + + def test_prompts_route_retries(self, monkeypatch): + """Test retry works for prompts route.""" + calls = [] + + def mock_request(client_self, method, url, **kwargs): + url_str = str(url) + calls.append({"method": method, "url": url_str}) + + # Count how many prompts calls we've had so far (before this one) + prompts_count = len([c for c in calls[:-1] if "prompts" in c["url"]]) + + print(f"Mock request: {url_str}, prompts_count: {prompts_count}") + + if "prompts" in url_str: + if prompts_count == 0: + # First call to prompts returns error + print("Returning 400 error for prompts") + return httpx.Response( + 400, + json={ + "message": "Invalid request data", + "error": [ + {"code": "unrecognized_keys", "keys": ["extraField"]} + ], + }, + ) + else: + # Retry succeeds + print("Returning 200 success for prompts") + return httpx.Response( + 200, + json={ + "id": "prompt-1", + "name": "test", + "version": 1, + "type": "text", + "prompt": "hello", + "config": {}, + "labels": [], + "tags": [], + }, + ) + else: + # Health check or other calls + print(f"Returning health check for: {url_str}") + return httpx.Response(200, json={"status": "OK"}) + + monkeypatch.setattr("httpx.Client.request", mock_request) + + langfuse = Langfuse( + public_key="pk-test", secret_key="sk-test", host="http://localhost:3000" + ) + print("Created Langfuse client, now calling create_prompt") + prompt = langfuse.create_prompt(name="test", prompt="hello", type="text") + + assert prompt.name == "test" + assert prompt.version == 1 + prompt_calls = [c for c in calls if "prompts" in c["url"]] + print(f"All calls: {calls}") + assert len(prompt_calls) == 2, "Should retry once on unrecognized_keys error" + + def test_multiple_unrecognized_keys(self, monkeypatch, complete_dataset_response): + """Test handling multiple keys in one error.""" + calls = [] + + def mock_request(client_self, method, url, **kwargs): + calls.append({"method": method, "url": str(url)}) + + if "datasets" in str(url) and len(calls) == 1: + return httpx.Response( + 400, + json={ + "message": "Invalid request data", + "error": [ + { + "code": "unrecognized_keys", + "keys": ["field1", "field2", "field3"], + } + ], + }, + ) + elif "datasets" in str(url): + return httpx.Response(200, json=complete_dataset_response) + else: + return httpx.Response(200, json={"status": "OK"}) + + monkeypatch.setattr("httpx.Client.request", mock_request) + + langfuse = Langfuse( + public_key="pk-test", secret_key="sk-test", host="http://localhost:3000" + ) + dataset = langfuse.create_dataset(name="test") + + assert dataset.id == "ds-1" + + def test_422_status_code_also_retries(self, monkeypatch, complete_dataset_response): + """Test 422 validation errors trigger retry.""" + calls = [] + + def mock_request(client_self, method, url, **kwargs): + calls.append({"method": method, "url": str(url)}) + + if "datasets" in str(url) and len(calls) == 1: + return httpx.Response( + 422, + json={ + "message": "Invalid request data", + "error": [{"code": "unrecognized_keys", "keys": ["badField"]}], + }, + ) + elif "datasets" in str(url): + return httpx.Response(200, json=complete_dataset_response) + else: + return httpx.Response(200, json={"status": "OK"}) + + monkeypatch.setattr("httpx.Client.request", mock_request) + + langfuse = Langfuse( + public_key="pk-test", secret_key="sk-test", host="http://localhost:3000" + ) + dataset = langfuse.create_dataset(name="test") + + assert dataset.id == "ds-1" + + def test_other_errors_not_retried(self, monkeypatch): + """Test non-unrecognized_keys errors don't retry.""" + calls = [] + + def mock_request(client_self, method, url, **kwargs): + url_str = str(url) + calls.append({"method": method, "url": url_str}) + print(f"Mock request for other_errors test: {url_str}") + + if "datasets" in url_str: + print("Returning 401 for datasets") + return httpx.Response(401, json={"message": "Invalid authentication"}) + else: + print("Returning OK for health check") + return httpx.Response(200, json={"status": "OK"}) + + monkeypatch.setattr("httpx.Client.request", mock_request) + + langfuse = Langfuse( + public_key="pk-test", secret_key="sk-test", host="http://localhost:3000" + ) + print("About to call create_dataset expecting error") + + with pytest.raises(ApiError): + result = langfuse.create_dataset(name="test") + print(f"create_dataset returned: {result}") + + dataset_calls = [c for c in calls if "datasets" in c["url"]] + print(f"Dataset calls: {dataset_calls}") + assert len(dataset_calls) == 1, "Should not retry on auth errors" + + def test_non_json_response_handled_gracefully(self, monkeypatch): + """Test plain text errors don't crash.""" + + def mock_request(client_self, method, url, **kwargs): + if "datasets" in str(url): + return httpx.Response(400, text="Internal error") + else: + return httpx.Response(200, json={"status": "OK"}) + + monkeypatch.setattr("httpx.Client.request", mock_request) + + langfuse = Langfuse( + public_key="pk-test", secret_key="sk-test", host="http://localhost:3000" + ) + + with pytest.raises(ApiError): + langfuse.create_dataset(name="test") + + +def test_remove_fields_helper(): + """Test the field removal utility function.""" + # Top-level removal + assert _remove_fields({"keep": "yes", "remove": "no"}, {"remove"}) == { + "keep": "yes" + } + + # Nested removal + assert _remove_fields({"a": {"b": 1, "c": 2}}, {"c"}) == {"a": {"b": 1}} + + # List of dicts + assert _remove_fields({"items": [{"id": 1, "bad": "x"}]}, {"bad"}) == { + "items": [{"id": 1}] + } + + # Primitives unchanged + assert _remove_fields("string", {"field"}) == "string" + assert _remove_fields(123, {"field"}) == 123 + + # Multiple fields + assert _remove_fields({"a": 1, "b": 2, "c": 3}, {"a", "c"}) == {"b": 2} From f692b5012eb83855be8ac2544e0bcf67e00150ab Mon Sep 17 00:00:00 2001 From: Marlies Mayerhofer <74332854+marliessophie@users.noreply.github.com> Date: Wed, 18 Feb 2026 17:49:33 +0100 Subject: [PATCH 3/6] chore: push --- langfuse/_client/http_retry_patch.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/langfuse/_client/http_retry_patch.py b/langfuse/_client/http_retry_patch.py index 22381e385..cc4c52626 100644 --- a/langfuse/_client/http_retry_patch.py +++ b/langfuse/_client/http_retry_patch.py @@ -41,7 +41,7 @@ def request_with_retry( unknown_keys = _extract_unknown_keys(response) if unknown_keys: logger.warning( - "Server rejected unrecognized keys %s for %s %s. Retrying without these fields.", + "Server rejected unrecognized keys %s for %s %s. If this is unexpected, please check if your server version supports these fields. Retrying without these fields.", unknown_keys, method, url, @@ -78,7 +78,7 @@ async def request_with_retry( unknown_keys = _extract_unknown_keys(response) if unknown_keys: logger.warning( - "Server rejected unrecognized keys %s for %s %s. Retrying without these fields.", + "Server rejected unrecognized keys %s for %s %s. If this is unexpected, please check if your server version supports these fields. Retrying without these fields.", unknown_keys, method, url, From 16bccdb7639839f1a31d709eca3fcc558aef69f9 Mon Sep 17 00:00:00 2001 From: Marlies Mayerhofer <74332854+marliessophie@users.noreply.github.com> Date: Wed, 18 Feb 2026 18:18:21 +0100 Subject: [PATCH 4/6] Revert "chore(experiment-runner): ensure backwards compatibility for old server versions" This reverts commit f7eb55cbcf9ce03d77eb7ee4e92a1f4da6091287. --- langfuse/_client/client.py | 51 +------------------------------------- 1 file changed, 1 insertion(+), 50 deletions(-) diff --git a/langfuse/_client/client.py b/langfuse/_client/client.py index 257629eee..c8782c638 100644 --- a/langfuse/_client/client.py +++ b/langfuse/_client/client.py @@ -130,27 +130,6 @@ from langfuse.types import MaskFunction, ScoreDataType, SpanLevel, TraceContext -def _is_unrecognized_field_error(e: Exception, field_name: str) -> bool: - """Check if error contains an unrecognized field. - - Used for backward compatibility when newer SDK sends fields not supported by older servers. - - Args: - e: Exception from API call - field_name: Name of field to check for in error - - Returns: - True if error indicates field_name is unrecognized by the server - """ - try: - return any( - field_name in error.get("keys", []) - for error in getattr(e, "body", {}).get("error", []) - ) - except (AttributeError, TypeError): - return False - - class Langfuse: """Main client for Langfuse tracing and platform features. @@ -2958,35 +2937,7 @@ async def _process_experiment_item( dataset_run_id = dataset_run_item.dataset_run_id except Exception as e: - # Only retry if the error is specifically about datasetVersion being unrecognized - # This handles backward compatibility with older server versions - if _is_unrecognized_field_error(e, "datasetVersion"): - langfuse_logger.warning( - "Server doesn't support datasetVersion field, retrying without it" - ) - try: - dataset_run_item = await asyncio.to_thread( - self.api.dataset_run_items.create, - request=CreateDatasetRunItemRequest( - runName=experiment_run_name, - runDescription=experiment_description, - metadata=experiment_metadata, - datasetItemId=item.id, # type: ignore - traceId=trace_id, - observationId=span.id, - # Note: datasetVersion omitted for backward compatibility - ), - ) - dataset_run_id = dataset_run_item.dataset_run_id - except Exception as retry_error: - error_msg = ( - f"Failed to create dataset run item: {retry_error}" - ) - langfuse_logger.error(error_msg) - else: - # Different error, just log it - error_msg = f"Failed to create dataset run item: {e}" - langfuse_logger.error(error_msg) + langfuse_logger.error(f"Failed to create dataset run item: {e}") if ( not isinstance(item, dict) From 2573496157c74d2b4a05f775ea36fb87915f9f81 Mon Sep 17 00:00:00 2001 From: Marlies Mayerhofer <74332854+marliessophie@users.noreply.github.com> Date: Thu, 19 Feb 2026 09:36:37 +0100 Subject: [PATCH 5/6] fix(run-experiment): pass version param only if it's present --- langfuse/_client/client.py | 32 ++++++++++++----- tests/test_datasets.py | 72 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 9 deletions(-) diff --git a/langfuse/_client/client.py b/langfuse/_client/client.py index c8782c638..4214bc44b 100644 --- a/langfuse/_client/client.py +++ b/langfuse/_client/client.py @@ -2921,17 +2921,31 @@ async def _process_experiment_item( try: # Use sync API to avoid event loop issues when run_async_safely # creates multiple event loops across different threads + + # Create request, only include datasetVersion if not None + if dataset_version is not None: + dataset_run_item_request = CreateDatasetRunItemRequest( + run_name=experiment_run_name, + run_description=experiment_description, + metadata=experiment_metadata, + dataset_item_id=item.id, # type: ignore + trace_id=trace_id, + observation_id=span.id, + dataset_version=dataset_version, + ) + else: + dataset_run_item_request = CreateDatasetRunItemRequest( + run_name=experiment_run_name, + run_description=experiment_description, + metadata=experiment_metadata, + dataset_item_id=item.id, # type: ignore + trace_id=trace_id, + observation_id=span.id, + ) + dataset_run_item = await asyncio.to_thread( self.api.dataset_run_items.create, - request=CreateDatasetRunItemRequest( - runName=experiment_run_name, - runDescription=experiment_description, - metadata=experiment_metadata, - datasetItemId=item.id, # type: ignore - traceId=trace_id, - observationId=span.id, - datasetVersion=dataset_version, - ), + request=dataset_run_item_request, ) dataset_run_id = dataset_run_item.dataset_run_id diff --git a/tests/test_datasets.py b/tests/test_datasets.py index 1a0cda63a..0c6e03d61 100644 --- a/tests/test_datasets.py +++ b/tests/test_datasets.py @@ -640,3 +640,75 @@ def simple_task(*, item, **kwargs): assert result.name == "Versioned Dataset Test" assert len(result.item_results) == 1 # Only one item in versioned dataset assert result.item_results[0].output == 4 + + +def test_run_experiment_without_dataset_version(monkeypatch): + langfuse = Langfuse(debug=False) + + # Create dataset + name = create_uuid() + langfuse.create_dataset(name=name) + + # Create item + langfuse.create_dataset_item( + dataset_name=name, input={"question": "What is 2+2?"}, expected_output=4 + ) + langfuse.flush() + time.sleep(3) + + # Get dataset without version timestamp - this should have version=None + dataset = langfuse.get_dataset(name) + assert dataset.version is None, "Dataset should have no version" + + # Capture HTTP requests to dataset-run-items endpoint to inspect the request body + captured_requests = [] + original_request = langfuse.api._client_wrapper.httpx_client.request + + def capture_request(method, url, **kwargs): + # Capture requests to the dataset-run-items endpoint + if "dataset-run-items" in str(url) and method.upper() == "POST": + # Capture the JSON body being sent + captured_requests.append(kwargs.get("json")) + return original_request(method, url, **kwargs) + + monkeypatch.setattr( + langfuse.api._client_wrapper.httpx_client, "request", capture_request + ) + + # Run a simple experiment on the dataset + def simple_task(*, item, **kwargs): + # Just return a static answer + return item.expected_output + + result = dataset.run_experiment( + name="Dataset without version Test", + description="Testing experiment with dataset without version", + task=simple_task, + ) + + # Verify experiment ran successfully + assert result.name == "Dataset without version Test" + assert len(result.item_results) == 1 # Only one item in dataset + assert result.item_results[0].output == 4 + assert result.dataset_run_id is not None, "Should have created a dataset run" + + # Verify that HTTP requests were captured + assert len(captured_requests) > 0, ( + "Should have captured dataset run item creation requests" + ) + + # Verify that datasetVersion was NOT included in any request body + for request_body in captured_requests: + assert request_body is not None, "Request body should not be None" + # Check if request_body is a dict or has a dict method + if hasattr(request_body, "dict"): + body_dict = request_body.dict() + elif isinstance(request_body, dict): + body_dict = request_body + else: + body_dict = json.loads(json.dumps(request_body)) + + assert "datasetVersion" not in body_dict, ( + f"datasetVersion should not be in request body when dataset has no version. " + f"Found in body: {body_dict}" + ) From f9fe1907c5175cc671c5fdd923b4691eaded5936 Mon Sep 17 00:00:00 2001 From: Marlies Mayerhofer <74332854+marliessophie@users.noreply.github.com> Date: Thu, 19 Feb 2026 09:42:54 +0100 Subject: [PATCH 6/6] Revert "feat(http): add automatic retry for unknown field errors in API requests" This reverts commit 77c5e76882588e081c3b5dc38abc9948c4b89527. --- langfuse/_client/http_retry_patch.py | 132 ------------- langfuse/_client/resource_manager.py | 14 -- tests/test_unknown_field_retry.py | 268 --------------------------- 3 files changed, 414 deletions(-) delete mode 100644 langfuse/_client/http_retry_patch.py delete mode 100644 tests/test_unknown_field_retry.py diff --git a/langfuse/_client/http_retry_patch.py b/langfuse/_client/http_retry_patch.py deleted file mode 100644 index cc4c52626..000000000 --- a/langfuse/_client/http_retry_patch.py +++ /dev/null @@ -1,132 +0,0 @@ -"""HTTP client patch for handling unknown field errors. - -This module provides functionality to automatically retry API requests when the server -rejects unknown or unexpected fields. This enables backward compatibility when using -newer SDK versions with older server versions. -""" - -import logging -from typing import Any, Set - -import httpx - -logger = logging.getLogger(__name__) - - -def patch_client_for_unknown_field_retry(client: httpx.Client) -> None: - """Patch an httpx.Client instance to automatically retry on unknown field errors. - - When the server returns a 400/422 error with unrecognized_keys, this automatically - retries the request with those fields removed. - - Args: - client: The httpx.Client instance to patch - - Example: - >>> client = httpx.Client() - >>> patch_client_for_unknown_field_retry(client) - >>> # Now all requests through this client will handle unknown field errors - """ - original_request = client.request - - def request_with_retry( - method: str, url: str | httpx.URL, **kwargs: Any - ) -> httpx.Response: - """Wrapped request that handles unknown field errors with retry.""" - response = original_request(method, url, **kwargs) - - # Retry if server rejected unrecognized keys - if response.status_code in [400, 422] and "json" in kwargs: - try: - unknown_keys = _extract_unknown_keys(response) - if unknown_keys: - logger.warning( - "Server rejected unrecognized keys %s for %s %s. If this is unexpected, please check if your server version supports these fields. Retrying without these fields.", - unknown_keys, - method, - url, - ) - kwargs["json"] = _remove_fields(kwargs["json"], unknown_keys) - response = original_request(method, url, **kwargs) - except Exception as e: - logger.debug("Failed to parse unknown field error: %s", e) - - return response - - client.request = request_with_retry # type: ignore[method-assign] - - -def patch_async_client_for_unknown_field_retry(client: httpx.AsyncClient) -> None: - """Patch an httpx.AsyncClient instance to automatically retry on unknown field errors. - - Async version of patch_client_for_unknown_field_retry. - - Args: - client: The httpx.AsyncClient instance to patch - """ - original_request = client.request - - async def request_with_retry( - method: str, url: str | httpx.URL, **kwargs: Any - ) -> httpx.Response: - """Wrapped async request that handles unknown field errors with retry.""" - response = await original_request(method, url, **kwargs) - - # Retry if server rejected unrecognized keys - if response.status_code in [400, 422] and "json" in kwargs: - try: - unknown_keys = _extract_unknown_keys(response) - if unknown_keys: - logger.warning( - "Server rejected unrecognized keys %s for %s %s. If this is unexpected, please check if your server version supports these fields. Retrying without these fields.", - unknown_keys, - method, - url, - ) - kwargs["json"] = _remove_fields(kwargs["json"], unknown_keys) - response = await original_request(method, url, **kwargs) - except Exception as e: - logger.debug("Failed to parse unknown field error: %s", e) - - return response - - client.request = request_with_retry # type: ignore[method-assign] - - -def _extract_unknown_keys(response: httpx.Response) -> Set[str]: - """Extract unknown keys from server error response. - - Args: - response: The HTTP response from the server - - Returns: - Set of field names that were rejected as unrecognized - """ - body = response.json() - if isinstance(body, dict) and "error" in body: - unknown_keys = set() - for error in body.get("error", []): - if isinstance(error, dict) and error.get("code") == "unrecognized_keys": - unknown_keys.update(error.get("keys", [])) - return unknown_keys - return set() - - -def _remove_fields(data: Any, fields: Set[str]) -> Any: - """Remove specified fields from nested dict/list structures. - - Args: - data: The data structure to filter (dict, list, or primitive) - fields: Set of field names to remove - - Returns: - Filtered data structure with specified fields removed - """ - if isinstance(data, dict): - return { - k: _remove_fields(v, fields) for k, v in data.items() if k not in fields - } - elif isinstance(data, list): - return [_remove_fields(item, fields) for item in data] - else: - return data diff --git a/langfuse/_client/resource_manager.py b/langfuse/_client/resource_manager.py index 5dd9580a0..08c008234 100644 --- a/langfuse/_client/resource_manager.py +++ b/langfuse/_client/resource_manager.py @@ -35,10 +35,6 @@ LANGFUSE_RELEASE, LANGFUSE_TRACING_ENVIRONMENT, ) -from langfuse._client.http_retry_patch import ( - patch_async_client_for_unknown_field_retry, - patch_client_for_unknown_field_retry, -) from langfuse._client.span_processor import LangfuseSpanProcessor from langfuse._task_manager.media_manager import MediaManager from langfuse._task_manager.media_upload_consumer import MediaUploadConsumer @@ -217,10 +213,6 @@ def _initialize_instance( client_headers = additional_headers if additional_headers else {} self.httpx_client = httpx.Client(timeout=timeout, headers=client_headers) - # Patch the httpx client to handle unknown field errors automatically - # This enables backward compatibility when newer SDK versions interact with older servers - patch_client_for_unknown_field_retry(self.httpx_client) - self.api = FernLangfuse( base_url=base_url, username=self.public_key, @@ -231,11 +223,6 @@ def _initialize_instance( httpx_client=self.httpx_client, timeout=timeout, ) - - # Create and patch async client for async API operations - async_httpx_client = httpx.AsyncClient(timeout=timeout) - patch_async_client_for_unknown_field_retry(async_httpx_client) - self.async_api = AsyncFernLangfuse( base_url=base_url, username=self.public_key, @@ -243,7 +230,6 @@ def _initialize_instance( x_langfuse_sdk_name="python", x_langfuse_sdk_version=langfuse_version, x_langfuse_public_key=self.public_key, - httpx_client=async_httpx_client, timeout=timeout, ) score_ingestion_client = LangfuseClient( diff --git a/tests/test_unknown_field_retry.py b/tests/test_unknown_field_retry.py deleted file mode 100644 index b602f7d3c..000000000 --- a/tests/test_unknown_field_retry.py +++ /dev/null @@ -1,268 +0,0 @@ -"""Tests for unknown field retry functionality. - -This module tests the automatic retry behavior when the server rejects unknown fields, -enabling backward compatibility between newer SDK versions and older server versions. -""" - -import httpx -import pytest -from datetime import datetime - -from langfuse import Langfuse -from langfuse._client.http_retry_patch import _remove_fields -from langfuse._client.resource_manager import LangfuseResourceManager -from langfuse.api.core import ApiError - - -class TestUnknownFieldRetry: - """Test that SDK handles unknown field errors gracefully.""" - - @pytest.fixture(autouse=True) - def reset_resource_manager(self): - """Reset LangfuseResourceManager singleton before each test.""" - LangfuseResourceManager.reset() - yield - LangfuseResourceManager.reset() - - @pytest.fixture - def complete_dataset_response(self): - """Provide a complete dataset response with all required fields.""" - return { - "id": "ds-1", - "name": "test", - "projectId": "proj-1", - "metadata": {}, - "createdAt": datetime.now().isoformat(), - "updatedAt": datetime.now().isoformat(), - } - - def test_datasets_route_retries(self, monkeypatch, complete_dataset_response): - """Test retry works for datasets route.""" - calls = [] - - def mock_request(client_self, method, url, **kwargs): - calls.append({"method": method, "url": str(url)}) - - if "datasets" in str(url) and len(calls) == 1: - # First call to datasets returns unrecognized_keys error - return httpx.Response( - 400, - json={ - "message": "Invalid request data", - "error": [{"code": "unrecognized_keys", "keys": ["badField"]}], - }, - ) - elif "datasets" in str(url): - # Retry succeeds - return httpx.Response(200, json=complete_dataset_response) - else: - # Health check - return httpx.Response(200, json={"status": "OK"}) - - # Patch BEFORE creating the client - monkeypatch.setattr("httpx.Client.request", mock_request) - - langfuse = Langfuse( - public_key="pk-test", secret_key="sk-test", host="http://localhost:3000" - ) - dataset = langfuse.create_dataset(name="test") - - assert dataset.id == "ds-1" - dataset_calls = [c for c in calls if "datasets" in c["url"]] - assert len(dataset_calls) == 2, "Should retry once on unrecognized_keys error" - - def test_prompts_route_retries(self, monkeypatch): - """Test retry works for prompts route.""" - calls = [] - - def mock_request(client_self, method, url, **kwargs): - url_str = str(url) - calls.append({"method": method, "url": url_str}) - - # Count how many prompts calls we've had so far (before this one) - prompts_count = len([c for c in calls[:-1] if "prompts" in c["url"]]) - - print(f"Mock request: {url_str}, prompts_count: {prompts_count}") - - if "prompts" in url_str: - if prompts_count == 0: - # First call to prompts returns error - print("Returning 400 error for prompts") - return httpx.Response( - 400, - json={ - "message": "Invalid request data", - "error": [ - {"code": "unrecognized_keys", "keys": ["extraField"]} - ], - }, - ) - else: - # Retry succeeds - print("Returning 200 success for prompts") - return httpx.Response( - 200, - json={ - "id": "prompt-1", - "name": "test", - "version": 1, - "type": "text", - "prompt": "hello", - "config": {}, - "labels": [], - "tags": [], - }, - ) - else: - # Health check or other calls - print(f"Returning health check for: {url_str}") - return httpx.Response(200, json={"status": "OK"}) - - monkeypatch.setattr("httpx.Client.request", mock_request) - - langfuse = Langfuse( - public_key="pk-test", secret_key="sk-test", host="http://localhost:3000" - ) - print("Created Langfuse client, now calling create_prompt") - prompt = langfuse.create_prompt(name="test", prompt="hello", type="text") - - assert prompt.name == "test" - assert prompt.version == 1 - prompt_calls = [c for c in calls if "prompts" in c["url"]] - print(f"All calls: {calls}") - assert len(prompt_calls) == 2, "Should retry once on unrecognized_keys error" - - def test_multiple_unrecognized_keys(self, monkeypatch, complete_dataset_response): - """Test handling multiple keys in one error.""" - calls = [] - - def mock_request(client_self, method, url, **kwargs): - calls.append({"method": method, "url": str(url)}) - - if "datasets" in str(url) and len(calls) == 1: - return httpx.Response( - 400, - json={ - "message": "Invalid request data", - "error": [ - { - "code": "unrecognized_keys", - "keys": ["field1", "field2", "field3"], - } - ], - }, - ) - elif "datasets" in str(url): - return httpx.Response(200, json=complete_dataset_response) - else: - return httpx.Response(200, json={"status": "OK"}) - - monkeypatch.setattr("httpx.Client.request", mock_request) - - langfuse = Langfuse( - public_key="pk-test", secret_key="sk-test", host="http://localhost:3000" - ) - dataset = langfuse.create_dataset(name="test") - - assert dataset.id == "ds-1" - - def test_422_status_code_also_retries(self, monkeypatch, complete_dataset_response): - """Test 422 validation errors trigger retry.""" - calls = [] - - def mock_request(client_self, method, url, **kwargs): - calls.append({"method": method, "url": str(url)}) - - if "datasets" in str(url) and len(calls) == 1: - return httpx.Response( - 422, - json={ - "message": "Invalid request data", - "error": [{"code": "unrecognized_keys", "keys": ["badField"]}], - }, - ) - elif "datasets" in str(url): - return httpx.Response(200, json=complete_dataset_response) - else: - return httpx.Response(200, json={"status": "OK"}) - - monkeypatch.setattr("httpx.Client.request", mock_request) - - langfuse = Langfuse( - public_key="pk-test", secret_key="sk-test", host="http://localhost:3000" - ) - dataset = langfuse.create_dataset(name="test") - - assert dataset.id == "ds-1" - - def test_other_errors_not_retried(self, monkeypatch): - """Test non-unrecognized_keys errors don't retry.""" - calls = [] - - def mock_request(client_self, method, url, **kwargs): - url_str = str(url) - calls.append({"method": method, "url": url_str}) - print(f"Mock request for other_errors test: {url_str}") - - if "datasets" in url_str: - print("Returning 401 for datasets") - return httpx.Response(401, json={"message": "Invalid authentication"}) - else: - print("Returning OK for health check") - return httpx.Response(200, json={"status": "OK"}) - - monkeypatch.setattr("httpx.Client.request", mock_request) - - langfuse = Langfuse( - public_key="pk-test", secret_key="sk-test", host="http://localhost:3000" - ) - print("About to call create_dataset expecting error") - - with pytest.raises(ApiError): - result = langfuse.create_dataset(name="test") - print(f"create_dataset returned: {result}") - - dataset_calls = [c for c in calls if "datasets" in c["url"]] - print(f"Dataset calls: {dataset_calls}") - assert len(dataset_calls) == 1, "Should not retry on auth errors" - - def test_non_json_response_handled_gracefully(self, monkeypatch): - """Test plain text errors don't crash.""" - - def mock_request(client_self, method, url, **kwargs): - if "datasets" in str(url): - return httpx.Response(400, text="Internal error") - else: - return httpx.Response(200, json={"status": "OK"}) - - monkeypatch.setattr("httpx.Client.request", mock_request) - - langfuse = Langfuse( - public_key="pk-test", secret_key="sk-test", host="http://localhost:3000" - ) - - with pytest.raises(ApiError): - langfuse.create_dataset(name="test") - - -def test_remove_fields_helper(): - """Test the field removal utility function.""" - # Top-level removal - assert _remove_fields({"keep": "yes", "remove": "no"}, {"remove"}) == { - "keep": "yes" - } - - # Nested removal - assert _remove_fields({"a": {"b": 1, "c": 2}}, {"c"}) == {"a": {"b": 1}} - - # List of dicts - assert _remove_fields({"items": [{"id": 1, "bad": "x"}]}, {"bad"}) == { - "items": [{"id": 1}] - } - - # Primitives unchanged - assert _remove_fields("string", {"field"}) == "string" - assert _remove_fields(123, {"field"}) == 123 - - # Multiple fields - assert _remove_fields({"a": 1, "b": 2, "c": 3}, {"a", "c"}) == {"b": 2}