diff --git a/hindsight-api-slim/hindsight_api/engine/retain/fact_extraction.py b/hindsight-api-slim/hindsight_api/engine/retain/fact_extraction.py index a27d46c80..ca789886b 100644 --- a/hindsight-api-slim/hindsight_api/engine/retain/fact_extraction.py +++ b/hindsight-api-slim/hindsight_api/engine/retain/fact_extraction.py @@ -193,7 +193,7 @@ class ExtractedFact(BaseModel): occurred_start: str | None = Field(default=None, description="ISO timestamp for events") occurred_end: str | None = Field(default=None, description="ISO timestamp for event end") fact_type: Literal["world", "assistant"] = Field( - description="'world' = objective/external facts. 'assistant' = first-person actions, experiences, or observations by the speaker." + description="'world' = objective/external facts, including user preferences, rules, corrections, and constraints even when stated during a conversation. 'assistant' = actions, experiences, or observations the assistant/agent actually performed." ) entities: list[Entity] | None = Field(default=None, description="People, places, concepts") causal_relations: list[FactCausalRelation] | None = Field( @@ -296,7 +296,7 @@ class ExtractedFactVerbose(BaseModel): ) fact_type: Literal["world", "assistant"] = Field( - description="'world' = objective/external facts about other people, events, general knowledge. 'assistant' = first-person actions, experiences, or observations by the speaker (e.g., 'I changed X', 'I discovered Y')." + description="'world' = objective/external facts about the user, other people, events, general knowledge, preferences, rules, corrections, or constraints. 'assistant' = actions, experiences, or observations the assistant/agent actually performed (e.g., 'I changed X', 'I discovered Y')." ) entities: list[Entity] | None = Field( @@ -346,7 +346,7 @@ class ExtractedFactNoCausal(BaseModel): occurred_start: str | None = Field(default=None, description="WHEN the event happened (ISO timestamp).") occurred_end: str | None = Field(default=None, description="WHEN the event ended (ISO timestamp).") fact_type: Literal["world", "assistant"] = Field( - description="'world' = about the user/others. 'assistant' = experience with assistant." + description="'world' = about the user/others, including user preferences, rules, corrections, and constraints. 'assistant' = actions or experiences the assistant/agent actually performed." ) entities: list[Entity] | None = Field( default=None, @@ -663,8 +663,8 @@ def _flush() -> None: - "conversation": Ongoing state, preference, trait (no dates) fact_type: -- "world": About other people, external events, general knowledge, objective facts -- "assistant": First-person actions, experiences, or observations by the speaker/author (e.g., "I changed X", "I discovered Y", "I debugged Z"). Also includes interactions with the user (requests, recommendations). If the narrator describes something they did, tried, learned, or decided — use "assistant". +- "world": Objective/external facts, including the user's preferences, rules, corrections, constraints, plans, traits, or context. These stay "world" even when the user states them during an assistant interaction (e.g., "User prefers browser_navigate over web_search", "User corrected the project deadline"). +- "assistant": Actions, experiences, or observations the assistant/agent actually performed (e.g., "I changed X", "I discovered Y", "I debugged Z"). Use this for the assistant/agent doing, trying, learning, deciding, recommending, or responding — not merely for user facts mentioned in conversation. ══════════════════════════════════════════════════════════════════════════ TEMPORAL HANDLING @@ -766,7 +766,7 @@ def _flush() -> None: - Extract all entities (people, places, organizations, objects, concepts). - Extract temporal information (occurred_start, occurred_end, fact_kind, when). - Extract location (where) and people (who). -- fact_type: use "world" unless the content is clearly an interaction with the assistant.""" +- fact_type: use "world" for user preferences, rules, corrections, constraints, traits, and other objective facts, even when stated during an assistant interaction. Use "assistant" only for actions or experiences the assistant/agent actually performed.""" VERBATIM_FACT_EXTRACTION_PROMPT = _BASE_FACT_EXTRACTION_PROMPT.format( retain_mission_section="{retain_mission_section}", @@ -867,8 +867,8 @@ def _flush() -> None: FACT TYPE ══════════════════════════════════════════════════════════════════════════ -- **world**: User's life, other people, events (would exist without this conversation) -- **assistant**: Interactions with assistant (requests, recommendations, help) +- **world**: User's life, preferences, rules, corrections, constraints, other people, and events (facts that would exist without this conversation) +- **assistant**: Actions or experiences the assistant/agent actually performed while helping the user (requests, recommendations, help) ⚠️ CRITICAL for assistant facts: ALWAYS capture the user's request/question in the fact! Include: what the user asked, what problem they wanted solved, what context they provided diff --git a/hindsight-api-slim/tests/test_chunking.py b/hindsight-api-slim/tests/test_chunking.py index ea69a4656..e6e40f610 100644 --- a/hindsight-api-slim/tests/test_chunking.py +++ b/hindsight-api-slim/tests/test_chunking.py @@ -455,4 +455,3 @@ def test_merged_json_array_routes_to_conversation_chunking(): assert isinstance(parsed, list), f"Chunk must be a JSON array: {chunk[:60]}" assert all(isinstance(e, dict) for e in parsed), f"Every element must be a dict: {chunk[:60]}" assert all("role" in e for e in parsed), f"Every element must have a role key: {chunk[:60]}" - diff --git a/hindsight-api-slim/tests/test_fact_extraction_fact_type_prompt.py b/hindsight-api-slim/tests/test_fact_extraction_fact_type_prompt.py new file mode 100644 index 000000000..8fe115477 --- /dev/null +++ b/hindsight-api-slim/tests/test_fact_extraction_fact_type_prompt.py @@ -0,0 +1,41 @@ +from unittest.mock import MagicMock + +from hindsight_api.engine.retain.fact_extraction import ( + ExtractedFact, + ExtractedFactNoCausal, + ExtractedFactVerbose, + _build_extraction_prompt_and_schema, +) + + +def _baseline_config() -> MagicMock: + config = MagicMock() + config.entity_labels = None + config.entities_allow_free_form = True + config.retain_extraction_mode = "concise" + config.retain_extract_causal_links = False + config.retain_mission = None + config.retain_custom_instructions = None + config.llm_output_language = None + return config + + +def test_concise_prompt_keeps_user_preferences_rules_and_corrections_world(): + prompt, _ = _build_extraction_prompt_and_schema(_baseline_config()) + + assert '"world": Objective/external facts' in prompt + assert "user's preferences, rules, corrections, constraints" in prompt + assert 'These stay "world" even when the user states them during an assistant interaction' in prompt + assert "Use this for the assistant/agent doing" in prompt + assert "not merely for user facts mentioned in conversation" in prompt + + +def test_fact_type_schema_descriptions_distinguish_user_facts_from_agent_actions(): + for model in (ExtractedFact, ExtractedFactVerbose, ExtractedFactNoCausal): + description = model.model_fields["fact_type"].description + + assert description is not None + assert "preferences" in description + assert "rules" in description + assert "corrections" in description + assert "assistant/agent actually performed" in description diff --git a/hindsight-api-slim/tests/test_retain_append_mode.py b/hindsight-api-slim/tests/test_retain_append_mode.py index d02b65b35..44060a2ee 100644 --- a/hindsight-api-slim/tests/test_retain_append_mode.py +++ b/hindsight-api-slim/tests/test_retain_append_mode.py @@ -254,10 +254,12 @@ async def test_append_mode_conversation_arrays_produce_valid_json(memory, reques try: # First retain - JSON conversation array - turn1 = json.dumps([ - {"role": "user", "content": "Hello"}, - {"role": "assistant", "content": "Hi there"}, - ]) + turn1 = json.dumps( + [ + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi there"}, + ] + ) await memory.retain_batch_async( bank_id=bank_id, contents=[ @@ -271,10 +273,12 @@ async def test_append_mode_conversation_arrays_produce_valid_json(memory, reques ) # Second retain - append more turns - turn2 = json.dumps([ - {"role": "user", "content": "How are you"}, - {"role": "assistant", "content": "Doing well"}, - ]) + turn2 = json.dumps( + [ + {"role": "user", "content": "How are you"}, + {"role": "assistant", "content": "Doing well"}, + ] + ) await memory.retain_batch_async( bank_id=bank_id, contents=[ @@ -300,10 +304,12 @@ async def test_append_mode_conversation_arrays_produce_valid_json(memory, reques assert len(parsed) == 4, "Should contain all 4 messages from both retains" # Third retain - append again, verify no degradation - turn3 = json.dumps([ - {"role": "user", "content": "What is new"}, - {"role": "assistant", "content": "Not much"}, - ]) + turn3 = json.dumps( + [ + {"role": "user", "content": "What is new"}, + {"role": "assistant", "content": "Not much"}, + ] + ) await memory.retain_batch_async( bank_id=bank_id, contents=[ @@ -329,4 +335,3 @@ async def test_append_mode_conversation_arrays_produce_valid_json(memory, reques finally: await memory.delete_bank(bank_id, request_context=request_context) -