Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion ohmo/gateway/bridge.py
Original file line number Diff line number Diff line change
Expand Up @@ -257,9 +257,13 @@ async def _process_message(self, message, session_key: str) -> None:
inbound_meta["message_id"] = message.metadata["message_id"]
try:
reply = ""
final_media: list[str] = []
final_metadata: dict[str, object] = {}
async for update in self._runtime_pool.stream_message(message, session_key):
if update.kind == "final":
reply = update.text
final_media = list(getattr(update, "media", None) or (update.metadata or {}).get("_media") or [])
final_metadata = dict(update.metadata or {})
continue
if not update.text:
continue
Expand Down Expand Up @@ -319,7 +323,8 @@ async def _process_message(self, message, session_key: str) -> None:
channel=message.channel,
chat_id=message.chat_id,
content=reply,
metadata={**inbound_meta, "_session_key": session_key},
media=final_media,
metadata={**inbound_meta, **final_metadata, "_session_key": session_key},
)
)

Expand Down
55 changes: 54 additions & 1 deletion ohmo/gateway/runtime.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
from pathlib import Path
import json
import os
import re
import string

from openharness.channels.bus.events import InboundMessage
Expand Down Expand Up @@ -62,6 +63,10 @@
_TEXT_PREVIEW_BYTES = 4096
_TEXT_PREVIEW_CHARS = 900
_BINARY_HEAD_BYTES = 32
_FINAL_REPLY_IMAGE_PATH_RE = re.compile(
r"(?P<path>(?:[A-Za-z]:[\\/]|/)[^\r\n`\"'<>|?*\x00]+?\.(?:png|jpe?g|webp|gif|bmp))",
re.IGNORECASE,
)
_IMAGE_FALLBACK_NOTE = (
"[Image attachment omitted because the active model does not support image input. "
"Use the attachment paths and summaries above if needed.]"
Expand Down Expand Up @@ -405,6 +410,7 @@ async def _stream_engine_message(
):
bundle.engine.set_system_prompt(self._runtime_system_prompt(bundle, user_prompt))
reply_parts: list[str] = []
emitted_media: set[str] = set()
yield GatewayStreamUpdate(
kind="progress",
text=_format_channel_progress(
Expand Down Expand Up @@ -450,6 +456,7 @@ async def _stream_engine_message(
content=user_prompt,
reply_parts=reply_parts,
):
_remember_update_media(emitted_media, update)
yield update
break
async for update in self._convert_stream_event(
Expand All @@ -460,6 +467,7 @@ async def _stream_engine_message(
content=user_prompt,
reply_parts=reply_parts,
):
_remember_update_media(emitted_media, update)
yield update
except MaxTurnsExceeded as exc:
yield GatewayStreamUpdate(
Expand All @@ -483,10 +491,15 @@ async def _stream_engine_message(
bundle.session_id,
_content_snippet(reply),
)
final_media = _extract_final_reply_media(reply, emitted_media)
metadata: dict[str, object] = {"_session_key": session_key}
if final_media:
metadata.update({"_media": final_media, "_final_media_fallback": True})
yield GatewayStreamUpdate(
kind="final",
text=reply,
metadata={"_session_key": session_key},
metadata=metadata,
media=final_media or None,
)

async def _convert_stream_event(
Expand Down Expand Up @@ -820,6 +833,46 @@ def _extract_tool_media(event: ToolExecutionCompleted) -> list[str]:
return media


def _remember_update_media(seen: set[str], update: GatewayStreamUpdate) -> None:
"""Track media already emitted during this gateway turn."""
raw_media = update.media or (update.metadata or {}).get("_media") or []
if isinstance(raw_media, str):
candidates = [raw_media]
elif isinstance(raw_media, list):
candidates = [str(item) for item in raw_media if isinstance(item, str) and item.strip()]
else:
candidates = []
for raw in candidates:
try:
path = Path(raw).expanduser()
if not path.is_absolute():
path = path.resolve()
seen.add(str(path))
except Exception:
continue


def _extract_final_reply_media(reply: str, emitted_media: set[str]) -> list[str]:
"""Return local image paths mentioned in final text that were not already emitted."""
media: list[str] = []
seen = set(emitted_media)
for match in _FINAL_REPLY_IMAGE_PATH_RE.finditer(reply or ""):
raw = match.group("path").strip(" \t\r\n\"'.,;:,。;:、)]}")
if not raw:
continue
path = Path(raw).expanduser()
if not path.is_absolute():
continue
if not path.is_file():
continue
resolved = str(path)
if resolved in seen:
continue
seen.add(resolved)
media.append(resolved)
return media


def _format_tool_media_caption(event: ToolExecutionCompleted, media: list[str]) -> str:
"""Return a short caption for media generated by tools."""
if event.tool_name == "image_generation":
Expand Down
119 changes: 119 additions & 0 deletions tests/test_ohmo/test_gateway.py
Original file line number Diff line number Diff line change
Expand Up @@ -770,6 +770,95 @@ async def fake_start_runtime(bundle):
assert "已生成图片 via codex" in media_updates[0].text


@pytest.mark.asyncio
async def test_runtime_pool_attaches_final_reply_image_path_as_media(tmp_path, monkeypatch):
workspace = tmp_path / ".ohmo-home"
initialize_workspace(workspace)
image_path = tmp_path / "generated.png"
image_path.write_bytes(b"png")

async def fake_build_runtime(**kwargs):
class FakeEngine:
messages = []
total_usage = UsageSnapshot()

def set_system_prompt(self, prompt):
return None

async def submit_message(self, content):
yield AssistantTextDelta(text=f"已生成图片:\n```text\n{image_path}\n```")

return SimpleNamespace(
engine=FakeEngine(),
cwd=str(tmp_path),
session_id="sess123",
current_settings=lambda: SimpleNamespace(model="gpt-5.4"),
commands=SimpleNamespace(lookup=lambda raw: None),
)

async def fake_start_runtime(bundle):
return None

monkeypatch.setattr("ohmo.gateway.runtime.build_runtime", fake_build_runtime)
monkeypatch.setattr("ohmo.gateway.runtime.start_runtime", fake_start_runtime)

pool = OhmoSessionRuntimePool(cwd=tmp_path, workspace=workspace, provider_profile="codex")
message = InboundMessage(channel="feishu", sender_id="u1", chat_id="c1", content="draw")
updates = [u async for u in pool.stream_message(message, "feishu:c1")]

assert updates[-1].kind == "final"
assert updates[-1].media == [str(image_path)]
assert updates[-1].metadata["_media"] == [str(image_path)]
assert updates[-1].metadata["_final_media_fallback"] is True


@pytest.mark.asyncio
async def test_runtime_pool_does_not_duplicate_final_reply_image_media(tmp_path, monkeypatch):
workspace = tmp_path / ".ohmo-home"
initialize_workspace(workspace)
image_path = tmp_path / "generated.png"
image_path.write_bytes(b"png")

async def fake_build_runtime(**kwargs):
class FakeEngine:
messages = []
total_usage = UsageSnapshot()

def set_system_prompt(self, prompt):
return None

async def submit_message(self, content):
yield ToolExecutionCompleted(
tool_name="image_generation",
output=f"Wrote {image_path}",
metadata={"paths": [str(image_path)], "provider": "codex"},
)
yield AssistantTextDelta(text=f"已生成图片:\n```text\n{image_path}\n```")

return SimpleNamespace(
engine=FakeEngine(),
cwd=str(tmp_path),
session_id="sess123",
current_settings=lambda: SimpleNamespace(model="gpt-5.4"),
commands=SimpleNamespace(lookup=lambda raw: None),
)

async def fake_start_runtime(bundle):
return None

monkeypatch.setattr("ohmo.gateway.runtime.build_runtime", fake_build_runtime)
monkeypatch.setattr("ohmo.gateway.runtime.start_runtime", fake_start_runtime)

pool = OhmoSessionRuntimePool(cwd=tmp_path, workspace=workspace, provider_profile="codex")
message = InboundMessage(channel="feishu", sender_id="u1", chat_id="c1", content="draw")
updates = [u async for u in pool.stream_message(message, "feishu:c1")]

assert [u.kind for u in updates].count("media") == 1
assert updates[-1].kind == "final"
assert updates[-1].media is None
assert "_final_media_fallback" not in updates[-1].metadata


@pytest.mark.asyncio
async def test_runtime_pool_stream_message_formats_auto_compact_status_for_feishu(tmp_path, monkeypatch):
workspace = tmp_path / ".ohmo-home"
Expand Down Expand Up @@ -1696,6 +1785,36 @@ async def stream_message(self, message, session_key):
assert outbound.media == ["/tmp/generated.png"]


@pytest.mark.asyncio
async def test_gateway_bridge_publishes_final_media_updates():
bus = MessageBus()

class FakeRuntimePool:
async def stream_message(self, message, session_key):
yield SimpleNamespace(
kind="final",
text="已生成图片:generated.png",
media=["/tmp/generated.png"],
metadata={"_session_key": session_key, "_media": ["/tmp/generated.png"]},
)

bridge = OhmoGatewayBridge(bus=bus, runtime_pool=FakeRuntimePool())
task = asyncio.create_task(bridge.run())
try:
await bus.publish_inbound(InboundMessage(channel="feishu", sender_id="u1", chat_id="c1", content="draw"))
outbound = await asyncio.wait_for(bus.consume_outbound(), timeout=1.0)
finally:
bridge.stop()
task.cancel()
try:
await task
except asyncio.CancelledError:
pass

assert outbound.content == "已生成图片:generated.png"
assert outbound.media == ["/tmp/generated.png"]


@pytest.mark.asyncio
async def test_gateway_bridge_publishes_progress_updates():
bus = MessageBus()
Expand Down
Loading