From 59695d19b01f9eaa1dc8588fe7cf7f70b8592fda Mon Sep 17 00:00:00 2001 From: yuhanzhu <2219179228@qq.com> Date: Thu, 23 Jul 2026 12:45:01 +0800 Subject: [PATCH] Add Hy3 API quickstart guide and 6 examples - quickstart.md: Base info, curl + Python SDK minimal examples, full parameter reference (temperature/top_p/max_tokens/stop/tools/ reasoning_effort), rate limits, and troubleshooting guide - Example 1: Basic chat (single-turn and multi-turn conversations) - Example 2: Streaming (SSE chunk parsing, reasoning content in stream) - Example 3: Latency comparison (TTFT, total time, multi-run benchmark) - Example 4: Tool calling (single call, multi-turn agent loop, parallel) - Example 5: Reasoning mode (fast vs deep thinking side-by-side comparison) - Example 6: Error handling (retry with exponential backoff + jitter, rate-limit handling, timeout config, production client wrapper) Each example includes a .md walkthrough and a standalone .py script. --- quickstart/examples/01-basic-chat.md | 210 +++++++++ quickstart/examples/01-basic-chat.py | 141 ++++++ quickstart/examples/02-streaming.md | 205 +++++++++ quickstart/examples/02-streaming.py | 199 ++++++++ quickstart/examples/03-latency-comparison.md | 213 +++++++++ quickstart/examples/03-latency-comparison.py | 214 +++++++++ quickstart/examples/04-tool-calling.md | 317 +++++++++++++ quickstart/examples/04-tool-calling.py | 309 +++++++++++++ quickstart/examples/05-reasoning-mode.md | 248 ++++++++++ quickstart/examples/05-reasoning-mode.py | 206 +++++++++ quickstart/examples/06-error-handling.md | 350 ++++++++++++++ quickstart/examples/06-error-handling.py | 422 +++++++++++++++++ quickstart/quickstart.md | 461 +++++++++++++++++++ 13 files changed, 3495 insertions(+) create mode 100644 quickstart/examples/01-basic-chat.md create mode 100644 quickstart/examples/01-basic-chat.py create mode 100644 quickstart/examples/02-streaming.md create mode 100644 quickstart/examples/02-streaming.py create mode 100644 quickstart/examples/03-latency-comparison.md create mode 100644 quickstart/examples/03-latency-comparison.py create mode 100644 quickstart/examples/04-tool-calling.md create mode 100644 quickstart/examples/04-tool-calling.py create mode 100644 quickstart/examples/05-reasoning-mode.md create mode 100644 quickstart/examples/05-reasoning-mode.py create mode 100644 quickstart/examples/06-error-handling.md create mode 100644 quickstart/examples/06-error-handling.py create mode 100644 quickstart/quickstart.md diff --git a/quickstart/examples/01-basic-chat.md b/quickstart/examples/01-basic-chat.md new file mode 100644 index 00000000..06ab13c4 --- /dev/null +++ b/quickstart/examples/01-basic-chat.md @@ -0,0 +1,210 @@ +# Example 1: Basic Chat + +Single-turn and multi-turn conversations with Hy3. + +## What You'll Learn + +- Send a single-turn chat request +- Build a multi-turn conversation with history +- Parse the response and extract content +- Understand `finish_reason` + +--- + +## Single-Turn Chat + +### Request + +```python +from openai import OpenAI + +client = OpenAI( + base_url="http://127.0.0.1:8000/v1", + api_key="EMPTY", +) + +response = client.chat.completions.create( + model="hy3", + messages=[ + {"role": "user", "content": "Explain quantum computing in one paragraph."}, + ], + temperature=0.9, + top_p=1.0, + max_tokens=256, +) +``` + +### Response Parsing + +```python +# Extract the assistant's reply +message = response.choices[0].message +content = message.content +finish_reason = response.choices[0].finish_reason + +print(f"Role: {message.role}") +print(f"Content: {content}") +print(f"Finish reason: {finish_reason}") +print(f"Tokens used: {response.usage.total_tokens}") + +# Check if the response was truncated +if finish_reason == "length": + print("⚠️ Response was truncated — increase max_tokens.") +``` + +### Full Response Object (reference) + +```json +{ + "id": "chatcmpl-abc123", + "object": "chat.completion", + "created": 1719000000, + "model": "hy3", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Quantum computing leverages the principles of quantum mechanics—superposition, entanglement, and interference—to process information in ways that classical computers cannot. Unlike classical bits that are either 0 or 1, quantum bits (qubits) can exist in multiple states simultaneously, enabling quantum computers to explore vast solution spaces in parallel. This makes them potentially transformative for problems like factoring large numbers, simulating molecular interactions for drug discovery, optimizing complex logistics, and enhancing machine learning. However, current quantum computers are still in the noisy intermediate-scale quantum (NISQ) era, with limited qubit counts and high error rates, requiring significant advances in error correction and hardware stability before they can achieve practical, large-scale quantum advantage.", + "tool_calls": null + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 13, + "completion_tokens": 129, + "total_tokens": 142 + } +} +``` + +### Sample Output + +``` +Role: assistant +Content: Quantum computing leverages the principles of quantum mechanics... +Finish reason: stop +Tokens used: 142 +``` + +--- + +## Multi-Turn Chat + +Maintain conversation context by appending each response to the message history. + +### Request + +```python +from openai import OpenAI + +client = OpenAI( + base_url="http://127.0.0.1:8000/v1", + api_key="EMPTY", +) + +# Initialize conversation with a system message (optional) +messages = [ + { + "role": "system", + "content": "You are a helpful physics tutor. Keep explanations concise.", + }, +] + +# Turn 1 +messages.append({"role": "user", "content": "What is quantum entanglement?"}) + +response = client.chat.completions.create( + model="hy3", + messages=messages, + temperature=0.9, + top_p=1.0, + max_tokens=256, +) + +reply_1 = response.choices[0].message.content +print(f"Assistant: {reply_1}\n") + +# Add the assistant's response to history +messages.append({"role": "assistant", "content": reply_1}) + +# Turn 2 — model remembers context from Turn 1 +messages.append({"role": "user", "content": "Can you give a real-world analogy for that?"}) + +response = client.chat.completions.create( + model="hy3", + messages=messages, + temperature=0.9, + top_p=1.0, + max_tokens=256, +) + +reply_2 = response.choices[0].message.content +print(f"Assistant: {reply_2}") +``` + +### Response Parsing + +```python +def chat_turn(messages, user_input): + """Send a message and return the assistant's reply, updating history.""" + messages.append({"role": "user", "content": user_input}) + + response = client.chat.completions.create( + model="hy3", + messages=messages, + temperature=0.9, + top_p=1.0, + max_tokens=512, + ) + + reply = response.choices[0].message.content + messages.append({"role": "assistant", "content": reply}) + return reply + + +# Usage +messages = [] +print("Turn 1:", chat_turn(messages, "What is a black hole?")) +print("Turn 2:", chat_turn(messages, "How big can they get?")) +print("Turn 3:", chat_turn(messages, "What happens if you fall into one?")) +``` + +### Sample Output + +``` +Turn 1: A black hole is a region of spacetime where gravity is so intense +that nothing, not even light, can escape. It forms when a massive star +collapses under its own gravity at the end of its life cycle... + +Turn 2: Black holes vary enormously in size. Stellar-mass black holes are +typically 3 to tens of solar masses. Supermassive black holes at galactic +centers can reach millions or billions of solar masses—Sagittarius A* in +our Milky Way is about 4 million solar masses, while TON 618 is estimated +at 66 billion solar masses... + +Turn 3: If you fell into a black hole, you'd experience "spaghettification" +— the tidal forces would stretch you vertically and compress you +horizontally as you approach the singularity. To an outside observer, you'd +appear to slow down and redshift, never quite crossing the event horizon, +due to extreme time dilation... +``` + +--- + +## Key Takeaways + +1. **Append each response** to `messages` to maintain conversation context. +2. **Monitor `finish_reason`**: `"stop"` = natural end; `"length"` = truncated; `"tool_calls"` = model wants to call a function. +3. **System messages** are optional but recommended for controlling tone and behavior. +4. Hy3 supports **256K context**, so very long multi-turn conversations are possible. + +--- + +## Run the Script + +```bash +pip install openai +python 01-basic-chat.py +``` diff --git a/quickstart/examples/01-basic-chat.py b/quickstart/examples/01-basic-chat.py new file mode 100644 index 00000000..e71de046 --- /dev/null +++ b/quickstart/examples/01-basic-chat.py @@ -0,0 +1,141 @@ +""" +Example 1: Basic Chat — single-turn and multi-turn conversations with Hy3. + +Usage: + python 01-basic-chat.py + +Prerequisites: + - Hy3 server running at http://127.0.0.1:8000 (vLLM or SGLang) + - pip install openai +""" + +from openai import OpenAI + +# ────────────────────────────────────────────────────────────────────── +# Configuration +# ────────────────────────────────────────────────────────────────────── +BASE_URL = "http://127.0.0.1:8000/v1" +API_KEY = "EMPTY" +MODEL = "hy3" + +client = OpenAI(base_url=BASE_URL, api_key=API_KEY) + + +# ────────────────────────────────────────────────────────────────────── +# 1. Single-Turn Chat +# ────────────────────────────────────────────────────────────────────── +def single_turn(): + print("=" * 60) + print("1. SINGLE-TURN CHAT") + print("=" * 60) + + response = client.chat.completions.create( + model=MODEL, + messages=[ + {"role": "user", "content": "Explain quantum computing in one paragraph."}, + ], + temperature=0.9, + top_p=1.0, + max_tokens=256, + ) + + message = response.choices[0].message + finish_reason = response.choices[0].finish_reason + + print(f"\nRole: {message.role}") + print(f"Content:\n{message.content}") + print(f"\nFinish reason: {finish_reason}") + print(f"Prompt tokens: {response.usage.prompt_tokens}") + print(f"Completion tokens: {response.usage.completion_tokens}") + print(f"Total tokens: {response.usage.total_tokens}") + + if finish_reason == "length": + print("\n⚠️ Response was truncated — consider increasing max_tokens.") + + +# ────────────────────────────────────────────────────────────────────── +# 2. Multi-Turn Chat +# ────────────────────────────────────────────────────────────────────── +def multi_turn(): + print("\n" + "=" * 60) + print("2. MULTI-TURN CHAT") + print("=" * 60) + + messages = [ + { + "role": "system", + "content": "You are a helpful physics tutor. Keep explanations concise but thorough.", + }, + ] + + questions = [ + "What is quantum entanglement?", + "Can you give a real-world analogy for that?", + "How is this used in quantum computing?", + ] + + for i, question in enumerate(questions, 1): + messages.append({"role": "user", "content": question}) + + response = client.chat.completions.create( + model=MODEL, + messages=messages, + temperature=0.9, + top_p=1.0, + max_tokens=300, + ) + + reply = response.choices[0].message.content + messages.append({"role": "assistant", "content": reply}) + + print(f"\n--- Turn {i} ---") + print(f"User: {question}") + print(f"Assistant: {reply[:200]}{'...' if len(reply) > 200 else ''}") + print(f"(Tokens this turn: {response.usage.total_tokens})") + + +# ────────────────────────────────────────────────────────────────────── +# 3. Helper Function for Reusable Chat +# ────────────────────────────────────────────────────────────────────── +def chat_turn(messages: list, user_input: str) -> str: + """Send a user message and return the assistant's reply, updating history in place.""" + messages.append({"role": "user", "content": user_input}) + + response = client.chat.completions.create( + model=MODEL, + messages=messages, + temperature=0.9, + top_p=1.0, + max_tokens=512, + ) + + reply = response.choices[0].message.content + messages.append({"role": "assistant", "content": reply}) + return reply + + +def reusable_chat(): + print("\n" + "=" * 60) + print("3. REUSABLE CHAT HELPER") + print("=" * 60) + + messages = [] + turns = [ + "What is a black hole?", + "How big can they get?", + ] + + for i, question in enumerate(turns, 1): + reply = chat_turn(messages, question) + print(f"\n--- Turn {i} ---") + print(f"Q: {question}") + print(f"A: {reply[:250]}{'...' if len(reply) > 250 else ''}") + + +# ────────────────────────────────────────────────────────────────────── +# Main +# ────────────────────────────────────────────────────────────────────── +if __name__ == "__main__": + single_turn() + multi_turn() + reusable_chat() diff --git a/quickstart/examples/02-streaming.md b/quickstart/examples/02-streaming.md new file mode 100644 index 00000000..15f8e2d4 --- /dev/null +++ b/quickstart/examples/02-streaming.md @@ -0,0 +1,205 @@ +# Example 2: Streaming + +Stream responses token-by-token for real-time display, similar to ChatGPT's typing effect. + +## What You'll Learn + +- Enable streaming with `stream=True` +- Iterate over SSE chunks +- Distinguish `delta.content` from `delta.reasoning_content` +- Handle the `[DONE]` signal and final usage stats + +--- + +## Streaming Request + +```python +from openai import OpenAI + +client = OpenAI( + base_url="http://127.0.0.1:8000/v1", + api_key="EMPTY", +) + +stream = client.chat.completions.create( + model="hy3", + messages=[ + {"role": "user", "content": "Write a haiku about programming."}, + ], + temperature=0.9, + top_p=1.0, + max_tokens=256, + stream=True, # ← Enable streaming +) +``` + +### Chunk-by-Chunk Parsing + +```python +for chunk in stream: + if not chunk.choices: + continue + + delta = chunk.choices[0].delta + + # Regular content + if delta.content: + print(delta.content, end="", flush=True) + + # Finish reason (signals end of stream) + if chunk.choices[0].finish_reason: + print(f"\n\n[Finished: {chunk.choices[0].finish_reason}]") + + # Usage stats (sent in the final chunk) + if hasattr(chunk, "usage") and chunk.usage: + print(f"[Tokens: {chunk.usage.total_tokens}]") +``` + +### Raw Chunk Structure + +Each chunk is a JSON object sent as an SSE event: + +``` +data: {"id":"chatcmpl-...","object":"chat.completion.chunk","created":...,"model":"hy3","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null}]} + +data: {"id":"chatcmpl-...","object":"chat.completion.chunk","created":...,"model":"hy3","choices":[{"index":0,"delta":{"content":"Bugs"},"finish_reason":null}]} + +data: {"id":"chatcmpl-...","object":"chat.completion.chunk","created":...,"model":"hy3","choices":[{"index":0,"delta":{"content":" hide"},"finish_reason":null}]} + +data: {"id":"chatcmpl-...","object":"chat.completion.chunk","created":...,"model":"hy3","choices":[{"index":0,"delta":{"content":" in"},"finish_reason":null}]} + +data: {"id":"chatcmpl-...","object":"chat.completion.chunk","created":...,"model":"hy3","choices":[{"index":0,"delta":{"content":" light"},"finish_reason":null}]} + +... + +data: {"id":"chatcmpl-...","object":"chat.completion.chunk","created":...,"model":"hy3","choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":14,"completion_tokens":18,"total_tokens":32}} + +data: [DONE] +``` + +### Sample Output + +``` +Bugs hide in the code, +Compile errors light the screen— +Debugger drinks coffee. + +[Finished: stop] +[Tokens: 32] +``` + +--- + +## Streaming with Reasoning Content + +When `reasoning_effort="high"` is enabled, the stream includes both thinking tokens and final output: + +```python +stream = client.chat.completions.create( + model="hy3", + messages=[{"role": "user", "content": "What is 15 * 37?"}], + stream=True, + extra_body={"chat_template_kwargs": {"reasoning_effort": "high"}}, +) + +thinking = [] +answer = [] + +for chunk in stream: + if not chunk.choices: + continue + delta = chunk.choices[0].delta + + # Thinking tokens (model's internal reasoning) + if getattr(delta, "reasoning_content", None): + thinking.append(delta.reasoning_content) + print(f"\x1b[90m{delta.reasoning_content}\x1b[0m", end="", flush=True) + + # Final answer tokens + if delta.content: + answer.append(delta.content) + print(delta.content, end="", flush=True) + +print(f"\n\nThinking tokens: {len(''.join(thinking).split())}") +print(f"Answer tokens: {len(''.join(answer).split())}") +``` + +### Sample Output (Reasoning Mode) + +``` +Let me break this down: +15 * 37 = 15 * (40 - 3) = 15*40 - 15*3 = 600 - 45 = 555 +So 15 * 37 = 555. + +Thinking tokens: 32 +Answer tokens: 11 +``` + +> 💡 **Tip**: In UI applications, show `reasoning_content` in a collapsed "thinking" panel and `content` as the main output. + +--- + +## Collecting Full Response from Stream + +If you need the complete text (e.g., for logging or downstream processing): + +```python +def stream_and_collect(client, messages, **kwargs): + """Stream the response and return the full text + usage stats.""" + stream = client.chat.completions.create( + model="hy3", + messages=messages, + stream=True, + **kwargs, + ) + + content_parts = [] + reasoning_parts = [] + finish_reason = None + usage = None + + for chunk in stream: + if not chunk.choices: + continue + delta = chunk.choices[0].delta + + if getattr(delta, "reasoning_content", None): + reasoning_parts.append(delta.reasoning_content) + + if delta.content: + content_parts.append(delta.content) + print(delta.content, end="", flush=True) + + if chunk.choices[0].finish_reason: + finish_reason = chunk.choices[0].finish_reason + + if hasattr(chunk, "usage") and chunk.usage: + usage = chunk.usage + + return { + "content": "".join(content_parts), + "reasoning": "".join(reasoning_parts), + "finish_reason": finish_reason, + "usage": usage, + } +``` + +--- + +## Key Takeaways + +1. **Set `stream=True`** in `client.chat.completions.create()`. +2. **Iterate over chunks** — each chunk contains a `delta` with partial content. +3. **The first chunk** carries `delta.role: "assistant"` with no content. +4. **The last meaningful chunk** contains `finish_reason` and `usage`. +5. **`[DONE]`** signals the stream end; the OpenAI SDK handles this automatically. +6. **Reasoning content** arrives in `delta.reasoning_content` before `delta.content`. + +--- + +## Run the Script + +```bash +pip install openai +python 02-streaming.py +``` diff --git a/quickstart/examples/02-streaming.py b/quickstart/examples/02-streaming.py new file mode 100644 index 00000000..9c96c80f --- /dev/null +++ b/quickstart/examples/02-streaming.py @@ -0,0 +1,199 @@ +""" +Example 2: Streaming — token-by-token response streaming with Hy3. + +Usage: + python 02-streaming.py + +Prerequisites: + - Hy3 server running at http://127.0.0.1:8000 (vLLM or SGLang) + - pip install openai +""" + +import time +from openai import OpenAI + +# ────────────────────────────────────────────────────────────────────── +# Configuration +# ────────────────────────────────────────────────────────────────────── +BASE_URL = "http://127.0.0.1:8000/v1" +API_KEY = "EMPTY" +MODEL = "hy3" + +client = OpenAI(base_url=BASE_URL, api_key=API_KEY) + + +# ────────────────────────────────────────────────────────────────────── +# 1. Basic Streaming +# ────────────────────────────────────────────────────────────────────── +def basic_streaming(): + print("=" * 60) + print("1. BASIC STREAMING") + print("=" * 60) + + stream = client.chat.completions.create( + model=MODEL, + messages=[ + {"role": "user", "content": "Write a haiku about programming."}, + ], + temperature=0.9, + top_p=1.0, + max_tokens=256, + stream=True, + ) + + print("\nStreaming output:\n") + for chunk in stream: + if not chunk.choices: + continue + delta = chunk.choices[0].delta + + # Print content token by token + if delta.content: + print(delta.content, end="", flush=True) + + # Finish reason + if chunk.choices[0].finish_reason: + print(f"\n\n[Finished: {chunk.choices[0].finish_reason}]") + + # Usage stats (final chunk only) + if hasattr(chunk, "usage") and chunk.usage: + print(f"[Prompt: {chunk.usage.prompt_tokens} | " + f"Completion: {chunk.usage.completion_tokens} | " + f"Total: {chunk.usage.total_tokens}]") + + +# ────────────────────────────────────────────────────────────────────── +# 2. Streaming with Reasoning Content +# ────────────────────────────────────────────────────────────────────── +def streaming_with_reasoning(): + print("\n" + "=" * 60) + print("2. STREAMING WITH REASONING MODE") + print("=" * 60) + + stream = client.chat.completions.create( + model=MODEL, + messages=[ + {"role": "user", "content": "A bat and a ball cost $1.10 in total. " + "The bat costs $1.00 more than the ball. " + "How much does the ball cost?"}, + ], + temperature=0.9, + top_p=1.0, + max_tokens=512, + stream=True, + extra_body={"chat_template_kwargs": {"reasoning_effort": "high"}}, + ) + + thinking_parts = [] + answer_parts = [] + + print("\nThinking (dim):") + for chunk in stream: + if not chunk.choices: + continue + delta = chunk.choices[0].delta + + # Reasoning content (thinking process) + if getattr(delta, "reasoning_content", None): + thinking_parts.append(delta.reasoning_content) + print(f"\x1b[90m{delta.reasoning_content}\x1b[0m", end="", flush=True) + + # Final answer content + if delta.content: + answer_parts.append(delta.content) + print(delta.content, end="", flush=True) + + if chunk.choices[0].finish_reason: + print(f"\n\n[Finish reason: {chunk.choices[0].finish_reason}]") + + if thinking_parts: + print(f"\n[Thinking: {len(''.join(thinking_parts))} chars]") + if answer_parts: + print(f"[Answer: {len(''.join(answer_parts))} chars]") + + +# ────────────────────────────────────────────────────────────────────── +# 3. Stream and Collect Full Response +# ────────────────────────────────────────────────────────────────────── +def stream_and_collect(messages, **kwargs): + """Stream the response in real-time and return the full text + metadata.""" + stream = client.chat.completions.create( + model=MODEL, + messages=messages, + stream=True, + **kwargs, + ) + + content_parts = [] + reasoning_parts = [] + finish_reason = None + usage = None + first_token_time = None + start_time = time.time() + + for chunk in stream: + if not chunk.choices: + continue + delta = chunk.choices[0].delta + + # Track time to first token + if first_token_time is None and ( + delta.content or getattr(delta, "reasoning_content", None) + ): + first_token_time = time.time() + + if getattr(delta, "reasoning_content", None): + reasoning_parts.append(delta.reasoning_content) + + if delta.content: + content_parts.append(delta.content) + print(delta.content, end="", flush=True) + + if chunk.choices[0].finish_reason: + finish_reason = chunk.choices[0].finish_reason + + if hasattr(chunk, "usage") and chunk.usage: + usage = chunk.usage + + elapsed = time.time() - start_time + ttft = (first_token_time - start_time) if first_token_time else None + + return { + "content": "".join(content_parts), + "reasoning": "".join(reasoning_parts), + "finish_reason": finish_reason, + "usage": usage, + "elapsed_s": elapsed, + "ttft_s": ttft, + } + + +def streaming_with_timing(): + print("\n" + "=" * 60) + print("3. STREAMING WITH TIMING") + print("=" * 60) + + result = stream_and_collect( + messages=[{"role": "user", "content": "Explain how SSL/TLS works in 3 paragraphs."}], + temperature=0.9, + top_p=1.0, + max_tokens=512, + ) + + print(f"\n\n--- Timing ---") + print(f"Time to first token (TTFT): {result['ttft_s']:.2f}s" if result['ttft_s'] else "TTFT: N/A") + print(f"Total elapsed: {result['elapsed_s']:.2f}s") + print(f"Finish reason: {result['finish_reason']}") + if result["usage"]: + print(f"Tokens: prompt={result['usage'].prompt_tokens}, " + f"completion={result['usage'].completion_tokens}, " + f"total={result['usage'].total_tokens}") + + +# ────────────────────────────────────────────────────────────────────── +# Main +# ────────────────────────────────────────────────────────────────────── +if __name__ == "__main__": + basic_streaming() + streaming_with_reasoning() + streaming_with_timing() diff --git a/quickstart/examples/03-latency-comparison.md b/quickstart/examples/03-latency-comparison.md new file mode 100644 index 00000000..7aa3d27d --- /dev/null +++ b/quickstart/examples/03-latency-comparison.md @@ -0,0 +1,213 @@ +# Example 3: Non-Streaming vs Streaming — Latency Comparison + +Compare time-to-first-token (TTFT) and total elapsed time between non-streaming and streaming modes. + +## What You'll Learn + +- Measure TTFT (time to first token) for both modes +- Measure total response time +- Understand the UX tradeoffs +- Choose the right mode for your use case + +--- + +## Comparison Setup + +Same prompt, same parameters — only `stream` flag changes. + +```python +import time +from openai import OpenAI + +client = OpenAI( + base_url="http://127.0.0.1:8000/v1", + api_key="EMPTY", +) + +PROMPT = "Explain the differences between TCP and UDP in detail." +``` + +### Non-Streaming + +```python +def non_streaming_call(): + start = time.time() + + response = client.chat.completions.create( + model="hy3", + messages=[{"role": "user", "content": PROMPT}], + temperature=0.9, + top_p=1.0, + max_tokens=512, + stream=False, + ) + + elapsed = time.time() - start + content = response.choices[0].message.content + + return { + "mode": "non-streaming", + "ttft_s": elapsed, # No TTFT — entire response arrives at once + "total_s": elapsed, + "content_length": len(content), + "tokens": response.usage.completion_tokens, + } +``` + +### Streaming + +```python +def streaming_call(): + start = time.time() + first_token_time = None + + stream = client.chat.completions.create( + model="hy3", + messages=[{"role": "user", "content": PROMPT}], + temperature=0.9, + top_p=1.0, + max_tokens=512, + stream=True, + ) + + content_parts = [] + for chunk in stream: + if not chunk.choices: + continue + delta = chunk.choices[0].delta + + if first_token_time is None and delta.content: + first_token_time = time.time() + + if delta.content: + content_parts.append(delta.content) + + elapsed = time.time() - start + ttft = (first_token_time - start) if first_token_time else None + + return { + "mode": "streaming", + "ttft_s": ttft, + "total_s": elapsed, + "content_length": len("".join(content_parts)), + } +``` + +### Results Comparison + +```python +def compare(): + results = [] + for func in [non_streaming_call, streaming_call]: + result = func() + results.append(result) + print(f"Mode: {result['mode']}") + print(f" TTFT: {result['ttft_s']:.2f}s") + print(f" Total: {result['total_s']:.2f}s") + print(f" Length: {result['content_length']} chars") + print() +``` + +### Expected Output + +``` +Mode: non-streaming + TTFT: 4.83s ← User sees NOTHING for 4.83s + Total: 4.83s + Length: 1024 chars + +Mode: streaming + TTFT: 0.31s ← User sees first token in 0.31s! + Total: 5.12s ← Slightly more overhead overall + Length: 1024 chars +``` + +--- + +## Analysis + +| Metric | Non-Streaming | Streaming | Winner | +|:---|:---|:---|:---| +| **Time to first visible output** | = total time (~4.8s) | ~0.3s | ✅ Streaming | +| **Total response time** | ~4.8s | ~5.1s | ✅ Non-streaming (slightly) | +| **Perceived responsiveness** | Poor (long blank wait) | Excellent (instant feedback) | ✅ Streaming | +| **Implementation complexity** | Simple | Moderate | ✅ Non-streaming | +| **Best for** | Backend/batch processing | User-facing chat UIs | — | + +### When to Use Each Mode + +**Use Non-Streaming for:** +- Backend processing (no user waiting) +- Batch jobs where you need the full response to proceed +- Tool calling — you need the complete `tool_calls` before executing +- Simple scripts and automation + +**Use Streaming for:** +- Chat interfaces — users see responses as they're generated +- Long-form content generation — avoids the "spinning wheel" UX +- Real-time applications (voice assistants, live captioning) +- When reasoning mode is on — users can see the thinking process + +--- + +## Multi-Run Benchmark + +For reliable numbers, run multiple iterations: + +```python +import statistics + +def benchmark(func, runs=5): + times = [] + for _ in range(runs): + result = func() + times.append(result["total_s"]) + return { + "min": min(times), + "max": max(times), + "mean": statistics.mean(times), + "stdev": statistics.stdev(times) if len(times) > 1 else 0, + } + +ns_stats = benchmark(non_streaming_call, runs=5) +s_stats = benchmark(streaming_call, runs=5) + +print("Non-streaming (5 runs):") +print(f" Mean: {ns_stats['mean']:.2f}s ± {ns_stats['stdev']:.2f}s") +print(f" Range: {ns_stats['min']:.2f}s – {ns_stats['max']:.2f}s") + +print("\nStreaming (5 runs):") +print(f" Mean: {s_stats['mean']:.2f}s ± {s_stats['stdev']:.2f}s") +print(f" Range: {s_stats['min']:.2f}s – {s_stats['max']:.2f}s") +``` + +### Sample Output (5 runs) + +``` +Non-streaming (5 runs): + Mean: 4.91s ± 0.34s + Range: 4.53s – 5.41s + +Streaming (5 runs): + Mean: 5.18s ± 0.41s + Range: 4.72s – 5.83s +``` + +--- + +## Key Takeaways + +1. **Streaming delivers ~10-20× faster perceived first response** (TTFT of ~0.3s vs ~5s total). +2. **Streaming has ~5-10% overhead** in total time due to SSE framing. +3. **User experience trumps raw speed** — always use streaming for interactive UIs. +4. **Network conditions matter** — streaming performs better on high-latency connections because the user sees output immediately. +5. **MTP speculative decoding** (enabled by default in the vLLM recipe) improves both modes significantly. + +--- + +## Run the Script + +```bash +pip install openai +python 03-latency-comparison.py +``` diff --git a/quickstart/examples/03-latency-comparison.py b/quickstart/examples/03-latency-comparison.py new file mode 100644 index 00000000..900af597 --- /dev/null +++ b/quickstart/examples/03-latency-comparison.py @@ -0,0 +1,214 @@ +""" +Example 3: Non-Streaming vs Streaming — Latency Comparison. + +Compares time-to-first-token (TTFT) and total elapsed time between +non-streaming and streaming modes. Use this to decide which mode +fits your use case. + +Usage: + python 03-latency-comparison.py + +Prerequisites: + - Hy3 server running at http://127.0.0.1:8000 (vLLM or SGLang) + - pip install openai +""" + +import time +import statistics +from openai import OpenAI + +# ────────────────────────────────────────────────────────────────────── +# Configuration +# ────────────────────────────────────────────────────────────────────── +BASE_URL = "http://127.0.0.1:8000/v1" +API_KEY = "EMPTY" +MODEL = "hy3" +BENCHMARK_RUNS = 3 # Number of runs per mode for averaging + +PROMPT = ( + "Explain the key differences between TCP and UDP protocols. " + "Cover reliability, ordering, congestion control, use cases, and " + "the tradeoffs involved in choosing one over the other." +) + +client = OpenAI(base_url=BASE_URL, api_key=API_KEY) + + +# ────────────────────────────────────────────────────────────────────── +# 1. Non-Streaming Call +# ────────────────────────────────────────────────────────────────────── +def non_streaming_call(): + """Make a non-streaming request and measure timing.""" + start = time.perf_counter() + + response = client.chat.completions.create( + model=MODEL, + messages=[{"role": "user", "content": PROMPT}], + temperature=0.9, + top_p=1.0, + max_tokens=512, + stream=False, + ) + + elapsed = time.perf_counter() - start + content = response.choices[0].message.content + + return { + "mode": "non-streaming", + "ttft_s": elapsed, # Same as total — user sees everything at once + "total_s": elapsed, + "content_chars": len(content), + "completion_tokens": response.usage.completion_tokens, + "total_tokens": response.usage.total_tokens, + } + + +# ────────────────────────────────────────────────────────────────────── +# 2. Streaming Call +# ────────────────────────────────────────────────────────────────────── +def streaming_call(): + """Make a streaming request and measure TTFT + total time.""" + start = time.perf_counter() + first_token_time = None + + stream = client.chat.completions.create( + model=MODEL, + messages=[{"role": "user", "content": PROMPT}], + temperature=0.9, + top_p=1.0, + max_tokens=512, + stream=True, + ) + + content_parts = [] + usage = None + + for chunk in stream: + if not chunk.choices: + continue + delta = chunk.choices[0].delta + + # Record time to first content token + if first_token_time is None and delta.content: + first_token_time = time.perf_counter() + + if delta.content: + content_parts.append(delta.content) + + if hasattr(chunk, "usage") and chunk.usage: + usage = chunk.usage + + elapsed = time.perf_counter() - start + ttft = (first_token_time - start) if first_token_time else None + + return { + "mode": "streaming", + "ttft_s": ttft, + "total_s": elapsed, + "content_chars": len("".join(content_parts)), + "completion_tokens": usage.completion_tokens if usage else None, + "total_tokens": usage.total_tokens if usage else None, + } + + +# ────────────────────────────────────────────────────────────────────── +# 3. Benchmark Runner +# ────────────────────────────────────────────────────────────────────── +def benchmark(func, runs=BENCHMARK_RUNS): + """Run a call function multiple times and return timing statistics.""" + results = [] + for i in range(runs): + print(f" Run {i + 1}/{runs}...", end=" ", flush=True) + result = func() + results.append(result) + print(f"→ {result['total_s']:.2f}s total, " + f"{result.get('ttft_s') or result['total_s']:.2f}s TTFT") + + times = [r["total_s"] for r in results] + ttfts = [r.get("ttft_s") or r["total_s"] for r in results] + + return { + "mode": results[0]["mode"], + "runs": runs, + "total_mean": statistics.mean(times), + "total_stdev": statistics.stdev(times) if len(times) > 1 else 0, + "total_min": min(times), + "total_max": max(times), + "ttft_mean": statistics.mean(ttfts), + "ttft_stdev": statistics.stdev(ttfts) if len(ttfts) > 1 else 0, + "avg_content_chars": statistics.mean(r["content_chars"] for r in results), + "avg_completion_tokens": statistics.mean( + r["completion_tokens"] for r in results if r["completion_tokens"] + ), + } + + +# ────────────────────────────────────────────────────────────────────── +# 4. Single-Run Comparison (Detailed) +# ────────────────────────────────────────────────────────────────────── +def detailed_comparison(): + """Run both modes once and show detailed comparison.""" + print("=" * 60) + print("DETAILED COMPARISON (Single Run)") + print("=" * 60) + + print("\nRunning non-streaming...", end=" ", flush=True) + ns = non_streaming_call() + print(f"done ({ns['total_s']:.2f}s)") + + print("Running streaming...", end=" ", flush=True) + s = streaming_call() + print(f"done ({s['total_s']:.2f}s)") + + # ── Results Table ── + print(f"\n{'Metric':<35} {'Non-Streaming':<20} {'Streaming':<20}") + print("-" * 75) + print(f"{'TTFT (time to first token)':<35} {f'{ns['ttft_s']:.2f}s':<20} {f'{s['ttft_s']:.2f}s':<20}") + print(f"{'Total elapsed':<35} {f'{ns['total_s']:.2f}s':<20} {f'{s['total_s']:.2f}s':<20}") + print(f"{'Content length (chars)':<35} {ns['content_chars']:<20} {s['content_chars']:<20}") + print(f"{'Completion tokens':<35} {ns['completion_tokens']:<20} {s['completion_tokens']:<20}") + print(f"{'Total tokens':<35} {ns['total_tokens']:<20} {s['total_tokens']:<20}") + + # UX analysis + speedup = ns['ttft_s'] / s['ttft_s'] if s['ttft_s'] and s['ttft_s'] > 0 else 0 + print(f"\n📊 Streaming delivers first visible output {speedup:.0f}× faster " + f"({ns['ttft_s']:.2f}s → {s['ttft_s']:.2f}s)") + + overhead_pct = ((s['total_s'] - ns['total_s']) / ns['total_s']) * 100 + print(f"📊 Streaming overhead: {overhead_pct:+.1f}% in total response time") + + +# ────────────────────────────────────────────────────────────────────── +# 5. Multi-Run Benchmark +# ────────────────────────────────────────────────────────────────────── +def multi_run_benchmark(): + """Run multiple iterations for statistically meaningful results.""" + print("\n" + "=" * 60) + print(f"MULTI-RUN BENCHMARK ({BENCHMARK_RUNS} runs each)") + print("=" * 60) + + print("\nNon-streaming:") + ns_stats = benchmark(non_streaming_call) + + print("\nStreaming:") + s_stats = benchmark(streaming_call) + + print("\n" + "=" * 60) + print("SUMMARY") + print("=" * 60) + + for stats in [ns_stats, s_stats]: + print(f"\n{stats['mode'].upper()}:") + print(f" Total time: {stats['total_mean']:.2f}s ± {stats['total_stdev']:.2f}s " + f"(range: {stats['total_min']:.2f}s – {stats['total_max']:.2f}s)") + print(f" TTFT: {stats['ttft_mean']:.2f}s ± {stats['ttft_stdev']:.2f}s") + print(f" Avg chars: {stats['avg_content_chars']:.0f}") + print(f" Avg tokens: {stats['avg_completion_tokens']:.0f}") + + +# ────────────────────────────────────────────────────────────────────── +# Main +# ────────────────────────────────────────────────────────────────────── +if __name__ == "__main__": + detailed_comparison() + multi_run_benchmark() diff --git a/quickstart/examples/04-tool-calling.md b/quickstart/examples/04-tool-calling.md new file mode 100644 index 00000000..0c4f77f7 --- /dev/null +++ b/quickstart/examples/04-tool-calling.md @@ -0,0 +1,317 @@ +# Example 4: Tool Calling + +Use Hy3's function-calling capability to build agentic workflows — let the model decide when and how to call external tools. + +## What You'll Learn + +- Define tools with JSON Schema +- Make a single tool call and parse the response +- Implement a multi-turn tool loop (execute → feed result → continue) +- Handle parallel tool calls + +--- + +## Tool Definition + +Tools are defined using the OpenAI function-calling format. Here are two example tools: + +```python +tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get current weather for a city. Returns temperature, conditions, and humidity.", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "City name and optional country code, e.g. 'Beijing, CN'", + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "Temperature unit. Default: celsius.", + }, + }, + "required": ["location"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "search_web", + "description": "Search the web for up-to-date information.", + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "The search query.", + }, + "num_results": { + "type": "integer", + "description": "Number of results to return (1-10). Default: 5.", + "minimum": 1, + "maximum": 10, + }, + }, + "required": ["query"], + }, + }, + }, +] +``` + +--- + +## Single Tool Call + +### Request + +```python +from openai import OpenAI + +client = OpenAI( + base_url="http://127.0.0.1:8000/v1", + api_key="EMPTY", +) + +response = client.chat.completions.create( + model="hy3", + messages=[ + {"role": "user", "content": "What's the weather in Tokyo?"}, + ], + tools=tools, + tool_choice="auto", # Let the model decide + temperature=0.9, + top_p=1.0, +) +``` + +### Response Parsing + +```python +message = response.choices[0].message + +# Check if the model wants to call a tool +if message.tool_calls: + for tool_call in message.tool_calls: + print(f"Function called: {tool_call.function.name}") + print(f"Arguments: {tool_call.function.arguments}") + print(f"Tool call ID: {tool_call.id}") +elif message.content: + print(f"Direct response: {message.content}") +``` + +### Sample Response + +```json +{ + "id": "chatcmpl-xyz789", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_abc123", + "type": "function", + "function": { + "name": "get_weather", + "arguments": "{\"location\": \"Tokyo, JP\", \"unit\": \"celsius\"}" + } + } + ] + }, + "finish_reason": "tool_calls" + } + ], + "usage": { + "prompt_tokens": 120, + "completion_tokens": 25, + "total_tokens": 145 + } +} +``` + +### Sample Output + +``` +Function called: get_weather +Arguments: {"location": "Tokyo, JP", "unit": "celsius"} +Tool call ID: call_abc123 +``` + +--- + +## Multi-Turn Tool Loop + +The canonical agentic pattern: call → execute → feed result → iterate. + +```python +import json + +def execute_tool(name: str, arguments: dict) -> str: + """Mock tool executor — replace with real API calls.""" + if name == "get_weather": + location = arguments.get("location", "unknown") + return json.dumps({ + "location": location, + "temperature": 22, + "conditions": "partly cloudy", + "humidity": "65%", + "unit": arguments.get("unit", "celsius"), + }) + elif name == "search_web": + query = arguments.get("query", "") + return json.dumps({ + "query": query, + "results": [ + {"title": f"Result 1 for '{query}'", "snippet": "..."}, + {"title": f"Result 2 for '{query}'", "snippet": "..."}, + ], + }) + return json.dumps({"error": f"Unknown tool: {name}"}) + + +def run_agent(user_query: str, max_turns: int = 5): + """Run the agent loop: call tools until the model gives a final answer.""" + messages = [{"role": "user", "content": user_query}] + + for turn in range(max_turns): + print(f"\n--- Turn {turn + 1} ---") + + response = client.chat.completions.create( + model="hy3", + messages=messages, + tools=tools, + tool_choice="auto", + temperature=0.9, + top_p=1.0, + ) + + message = response.choices[0].message + + # Final answer — no tool calls + if not message.tool_calls: + print(f"✅ Final answer:\n{message.content}") + return message.content + + # Execute tool calls + messages.append({ + "role": "assistant", + "content": message.content, + "tool_calls": [ + { + "id": tc.id, + "type": "function", + "function": { + "name": tc.function.name, + "arguments": tc.function.arguments, + }, + } + for tc in message.tool_calls + ], + }) + + for tc in message.tool_calls: + func_name = tc.function.name + func_args = json.loads(tc.function.arguments) + print(f"🔧 Calling: {func_name}({func_args})") + + result = execute_tool(func_name, func_args) + + messages.append({ + "role": "tool", + "tool_call_id": tc.id, + "content": result, + }) + print(f"📥 Result: {result[:100]}...") + + print("⚠️ Max turns reached without final answer.") + return None + + +# Run the agent +run_agent("What's the weather in Tokyo and search for best ramen shops there?") +``` + +### Sample Output + +``` +--- Turn 1 --- +🔧 Calling: get_weather({'location': 'Tokyo, JP', 'unit': 'celsius'}) +📥 Result: {"location": "Tokyo, JP", "temperature": 22, "conditions": "partly cloudy", "humidity": "65%", "unit": "celsius"} + +🔧 Calling: search_web({'query': 'best ramen shops Tokyo 2025'}) +📥 Result: {"query": "best ramen shops Tokyo 2025", "results": [...]} + +--- Turn 2 --- +✅ Final answer: +Here's what I found for Tokyo: + +🌤️ **Weather**: Currently 22°C, partly cloudy with 65% humidity. + +🍜 **Top Ramen Shops** (from web search): +1. Ichiran Shibuya — Famous for solo booth dining and tonkotsu ramen +2. Nakiryu — Michelin-starred tantanmen +3. Fuunji — Known for tsukemen (dipping ramen) +4. Afuri — Yuzu-infused light broth, very refreshing for this weather + +Given the mild weather, it's a perfect day to explore these spots on foot! +``` + +--- + +## Parallel Tool Calls + +Hy3 can call multiple tools simultaneously when they're independent: + +```python +response = client.chat.completions.create( + model="hy3", + messages=[ + {"role": "user", "content": "Compare the weather in Beijing, Shanghai, and Guangzhou."}, + ], + tools=tools, + tool_choice="auto", +) + +for tc in response.choices[0].message.tool_calls: + print(f"→ {tc.function.name}({tc.function.arguments})") +``` + +### Sample Output (Parallel) + +``` +→ get_weather({"location": "Beijing, CN"}) +→ get_weather({"location": "Shanghai, CN"}) +→ get_weather({"location": "Guangzhou, CN"}) +``` + +> 💡 **Tip**: Execute independent tool calls **in parallel** (e.g., with `asyncio.gather` or `concurrent.futures`) to minimize latency. + +--- + +## Key Takeaways + +1. **Define tools** with clear names, descriptions, and JSON Schema parameters — the model uses these to decide when and how to call. +2. **`tool_choice: "auto"`** lets the model decide; use `"required"` to force a tool call. +3. **Always append the tool result** to `messages` with `role: "tool"` and the matching `tool_call_id`. +4. **Loop until `finish_reason != "tool_calls"`** or the model returns a `content` response. +5. **Execute independent calls in parallel** to minimize total latency. +6. **Hy3's tool-calling stability** has been specifically optimized — expect production-grade reliability. + +--- + +## Run the Script + +```bash +pip install openai +python 04-tool-calling.py +``` diff --git a/quickstart/examples/04-tool-calling.py b/quickstart/examples/04-tool-calling.py new file mode 100644 index 00000000..99aedf8c --- /dev/null +++ b/quickstart/examples/04-tool-calling.py @@ -0,0 +1,309 @@ +""" +Example 4: Tool Calling — function calling with single and multi-turn tool loops. + +Usage: + python 04-tool-calling.py + +Prerequisites: + - Hy3 server running at http://127.0.0.1:8000 (vLLM or SGLang) + - vLLM must be launched with --tool-call-parser hy_v3 --enable-auto-tool-choice + or SGLang with --tool-call-parser hunyuan + - pip install openai +""" + +import json +from openai import OpenAI + +# ────────────────────────────────────────────────────────────────────── +# Configuration +# ────────────────────────────────────────────────────────────────────── +BASE_URL = "http://127.0.0.1:8000/v1" +API_KEY = "EMPTY" +MODEL = "hy3" + +client = OpenAI(base_url=BASE_URL, api_key=API_KEY) + +# ────────────────────────────────────────────────────────────────────── +# Tool Definitions +# ────────────────────────────────────────────────────────────────────── +TOOLS = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get current weather for a city. Returns temperature, conditions, and humidity.", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "City name and optional country code, e.g. 'Beijing, CN'", + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "Temperature unit. Default: celsius.", + }, + }, + "required": ["location"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "search_web", + "description": "Search the web for up-to-date information. Returns a list of relevant results.", + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "The search query.", + }, + "num_results": { + "type": "integer", + "description": "Number of results to return (1-10). Default: 5.", + "minimum": 1, + "maximum": 10, + }, + }, + "required": ["query"], + }, + }, + }, +] + + +# ────────────────────────────────────────────────────────────────────── +# Mock Tool Executor (replace with real API calls in production) +# ────────────────────────────────────────────────────────────────────── +def execute_tool(name: str, arguments: dict) -> str: + """Execute a tool and return the result as a JSON string.""" + if name == "get_weather": + location = arguments.get("location", "unknown") + # Simulated weather data + weather_db = { + "tokyo": (22, "partly cloudy", "65%"), + "beijing": (30, "sunny", "40%"), + "shanghai": (28, "light rain", "75%"), + "guangzhou": (33, "thunderstorm", "85%"), + "london": (18, "overcast", "70%"), + "new york": (25, "clear", "55%"), + } + key = location.split(",")[0].strip().lower() + temp, conditions, humidity = weather_db.get(key, (20, "unknown", "50%")) + return json.dumps({ + "location": location, + "temperature": temp, + "conditions": conditions, + "humidity": humidity, + "unit": arguments.get("unit", "celsius"), + }) + + elif name == "search_web": + query = arguments.get("query", "") + num = min(arguments.get("num_results", 5), 10) + return json.dumps({ + "query": query, + "results": [ + {"title": f"Result {i} for '{query}'", "url": f"https://example.com/{i}", "snippet": f"Relevant information about {query}..."} + for i in range(1, num + 1) + ], + }) + + return json.dumps({"error": f"Unknown tool: {name}"}) + + +# ────────────────────────────────────────────────────────────────────── +# 1. Single Tool Call +# ────────────────────────────────────────────────────────────────────── +def single_tool_call(): + print("=" * 60) + print("1. SINGLE TOOL CALL") + print("=" * 60) + + response = client.chat.completions.create( + model=MODEL, + messages=[{"role": "user", "content": "What's the weather in Tokyo?"}], + tools=TOOLS, + tool_choice="auto", + temperature=0.9, + top_p=1.0, + ) + + message = response.choices[0].message + + if message.tool_calls: + for tc in message.tool_calls: + print(f"\n🔧 Function called: {tc.function.name}") + print(f" Arguments: {tc.function.arguments}") + print(f" Tool call ID: {tc.id}") + print(f" Finish reason: {response.choices[0].finish_reason}") + else: + print(f"\n💬 Direct response (no tool call):\n{message.content}") + + +# ────────────────────────────────────────────────────────────────────── +# 2. Multi-Turn Tool Loop (Agentic Pattern) +# ────────────────────────────────────────────────────────────────────── +def run_agent(user_query: str, max_turns: int = 5, verbose: bool = True): + """ + Run the agent loop: call tools until the model gives a final answer. + + Args: + user_query: The user's question or command. + max_turns: Maximum number of tool-calling rounds before giving up. + verbose: Print each step. + + Returns: + The final answer string, or None if max_turns exceeded. + """ + messages = [{"role": "user", "content": user_query}] + + for turn in range(max_turns): + if verbose: + print(f"\n--- Turn {turn + 1} ---") + + response = client.chat.completions.create( + model=MODEL, + messages=messages, + tools=TOOLS, + tool_choice="auto", + temperature=0.9, + top_p=1.0, + ) + + message = response.choices[0].message + + # Final answer — no more tool calls needed + if not message.tool_calls: + if verbose: + print(f"✅ Final answer:\n{message.content}") + return message.content + + # Add assistant message with tool calls to history + messages.append({ + "role": "assistant", + "content": message.content, + "tool_calls": [ + { + "id": tc.id, + "type": "function", + "function": { + "name": tc.function.name, + "arguments": tc.function.arguments, + }, + } + for tc in message.tool_calls + ], + }) + + # Execute each tool call and append results + for tc in message.tool_calls: + func_name = tc.function.name + func_args = json.loads(tc.function.arguments) + + if verbose: + print(f"🔧 Calling: {func_name}({json.dumps(func_args)})") + + result = execute_tool(func_name, func_args) + + messages.append({ + "role": "tool", + "tool_call_id": tc.id, + "content": result, + }) + + if verbose: + print(f"📥 Result: {result[:120]}{'...' if len(result) > 120 else ''}") + + print("⚠️ Max turns reached without final answer.") + return None + + +def multi_turn_example(): + print("\n" + "=" * 60) + print("2. MULTI-TURN TOOL LOOP") + print("=" * 60) + + run_agent("What's the weather in Tokyo and Shanghai? Which city is warmer?") + + +# ────────────────────────────────────────────────────────────────────── +# 3. Parallel Tool Calls +# ────────────────────────────────────────────────────────────────────── +def parallel_tool_calls(): + print("\n" + "=" * 60) + print("3. PARALLEL TOOL CALLS") + print("=" * 60) + + response = client.chat.completions.create( + model=MODEL, + messages=[ + {"role": "user", "content": "Compare the weather in Beijing, Shanghai, and Guangzhou."}, + ], + tools=TOOLS, + tool_choice="auto", + temperature=0.9, + top_p=1.0, + ) + + message = response.choices[0].message + + if message.tool_calls: + print(f"\nModel requested {len(message.tool_calls)} parallel tool calls:\n") + for i, tc in enumerate(message.tool_calls, 1): + args = json.loads(tc.function.arguments) + print(f" {i}. {tc.function.name}({json.dumps(args)})") + print("\n💡 Execute independent calls in parallel to minimize latency.") + else: + print(f"\nDirect response: {message.content}") + + +# ────────────────────────────────────────────────────────────────────── +# 4. Tool Choice Modes +# ────────────────────────────────────────────────────────────────────── +def tool_choice_modes(): + print("\n" + "=" * 60) + print("4. TOOL CHOICE MODES") + print("=" * 60) + + test_cases = [ + ("auto (no tool needed)", "auto", "Say hello in Chinese."), + ("required (force tool)", "required", "Say hello in Chinese."), + ("none (no tools)", "none", "What's the weather in Beijing?"), + ] + + for label, tool_choice, prompt in test_cases: + print(f"\n--- {label} ---") + print(f"Prompt: {prompt}") + + try: + response = client.chat.completions.create( + model=MODEL, + messages=[{"role": "user", "content": prompt}], + tools=TOOLS, + tool_choice=tool_choice if tool_choice != "required" else "required", + temperature=0.9, + top_p=1.0, + ) + + msg = response.choices[0].message + if msg.tool_calls: + for tc in msg.tool_calls: + print(f" → Tool: {tc.function.name}({tc.function.arguments})") + else: + print(f" → Content: {msg.content[:100]}...") + except Exception as e: + print(f" → Error: {e}") + + +# ────────────────────────────────────────────────────────────────────── +# Main +# ────────────────────────────────────────────────────────────────────── +if __name__ == "__main__": + single_tool_call() + multi_turn_example() + parallel_tool_calls() + tool_choice_modes() diff --git a/quickstart/examples/05-reasoning-mode.md b/quickstart/examples/05-reasoning-mode.md new file mode 100644 index 00000000..528d6bfe --- /dev/null +++ b/quickstart/examples/05-reasoning-mode.md @@ -0,0 +1,248 @@ +# Example 5: Reasoning Mode + +Hy3 supports **fast thinking** (direct response) and **slow thinking** (chain-of-thought reasoning). This example shows how to control reasoning mode and compare outputs. + +## What You'll Learn + +- Enable/disable reasoning mode with `reasoning_effort` +- Compare fast vs. deep thinking on the same prompt +- Access reasoning content in the response +- Know when to use each mode + +--- + +## Reasoning Modes + +| Mode | `reasoning_effort` | Mechanism | Best For | Approx. Tokens | +|:---|:---|:---|:---|:---| +| **Fast** | `"no_think"` | Direct generation | Chat, translation, summarization, simple Q&A | Baseline | +| **Light** | `"low"` | Brief chain-of-thought | Planning, analysis, moderate reasoning | +30-50% | +| **Deep** | `"high"` | Extensive chain-of-thought | Math proofs, complex coding, multi-step logic | +100-300% | + +--- + +## Fast Thinking (Default) + +```python +from openai import OpenAI + +client = OpenAI( + base_url="http://127.0.0.1:8000/v1", + api_key="EMPTY", +) + +# Fast thinking — "no_think" is the default +response = client.chat.completions.create( + model="hy3", + messages=[{"role": "user", "content": "What is 156 * 234?"}], + temperature=0.9, + top_p=1.0, + extra_body={"chat_template_kwargs": {"reasoning_effort": "no_think"}}, +) + +print(response.choices[0].message.content) +``` + +### Sample Output (Fast) + +``` +156 × 234 = 36,504. +``` + +--- + +## Deep Reasoning + +```python +# Deep reasoning for complex tasks +response = client.chat.completions.create( + model="hy3", + messages=[{"role": "user", "content": "Prove that the square root of 2 is irrational."}], + temperature=0.9, + top_p=1.0, + max_tokens=1024, + extra_body={"chat_template_kwargs": {"reasoning_effort": "high"}}, +) + +message = response.choices[0].message + +# The thinking process +if hasattr(message, "reasoning_content") and message.reasoning_content: + print("=== THINKING ===") + print(message.reasoning_content) + print() + +print("=== ANSWER ===") +print(message.content) +``` + +### Sample Output (Deep Reasoning) + +``` +=== THINKING === +We need to prove that √2 is irrational using proof by contradiction. + +Assume √2 = a/b where a, b are coprime integers, b ≠ 0. +Then 2 = a²/b² +→ a² = 2b² +So a² is even, which means a is even. +Let a = 2k for some integer k. +Then (2k)² = 2b² +→ 4k² = 2b² +→ 2k² = b² +So b² is even, which means b is also even. +But this contradicts our assumption that a and b are coprime (both even means they share factor 2). +Therefore, our assumption is false, and √2 cannot be expressed as a ratio of integers. +Thus √2 is irrational. + +=== ANSWER === +Proof that √2 is irrational (by contradiction): + +1. Assume √2 is rational: √2 = a/b where a, b are integers with no common factors, b ≠ 0. +2. Square both sides: 2 = a²/b² → a² = 2b². +3. a² is even → a is even. Write a = 2k. +4. Substitute: (2k)² = 2b² → 4k² = 2b² → 2k² = b². +5. b² is even → b is even. +6. Both a and b are even, contradicting that they have no common factors. +7. Therefore, the assumption is false. √2 is irrational. ∎ +``` + +--- + +## Comparing Fast vs. Deep on the Same Prompt + +```python +PROMPTS = [ + ("Math", "Solve: If 3x + 7 = 22, what is x?"), + ("Logic", "All cats are mammals. All mammals are animals. Is a cat an animal? Explain."), + ("Code", "Write a Python function to find the longest palindrome substring."), + ("Creative", "Write a haiku about machine learning."), +] + +modes = [ + ("no_think", "Fast"), + ("high", "Deep"), +] + +for category, prompt in PROMPTS: + print(f"\n{'='*60}") + print(f"CATEGORY: {category}") + print(f"PROMPT: {prompt}") + + for effort, label in modes: + response = client.chat.completions.create( + model="hy3", + messages=[{"role": "user", "content": prompt}], + temperature=0.9, + top_p=1.0, + max_tokens=512, + extra_body={"chat_template_kwargs": {"reasoning_effort": effort}}, + ) + + msg = response.choices[0].message + thinking = getattr(msg, "reasoning_content", "") or "" + content = msg.content or "" + + print(f"\n--- {label} Thinking ({effort}) ---") + print(f"Tokens: {response.usage.total_tokens} " + f"(prompt: {response.usage.prompt_tokens}, " + f"completion: {response.usage.completion_tokens})") + if thinking: + print(f"Thinking chars: {len(thinking)}") + print(f"Answer: {content[:200]}...") +``` + +### Sample Comparison Output + +``` +============================================================ +CATEGORY: Math +PROMPT: Solve: If 3x + 7 = 22, what is x? + +--- Fast Thinking (no_think) --- +Tokens: 28 (prompt: 18, completion: 10) +Answer: x = 5. + +--- Deep Thinking (high) --- +Tokens: 72 (prompt: 18, completion: 54) +Thinking chars: 85 +Answer: Let me solve this step by step: 3x + 7 = 22 → subtract 7 from both sides: 3x = 15 → divide by 3: x = 5. + +============================================================ +CATEGORY: Logic +PROMPT: All cats are mammals. All mammals are animals. Is a cat an animal? Explain. + +--- Fast Thinking (no_think) --- +Tokens: 45 (prompt: 27, completion: 18) +Answer: Yes, a cat is an animal. By transitive property: cats → mammals → animals. + +--- Deep Thinking (high) --- +Tokens: 150 (prompt: 27, completion: 123) +Thinking chars: 215 +Answer: Yes. This is a syllogism. The transitive property of set membership states that if A ⊆ B and B ⊆ C, then A ⊆ C. Here: Cats ⊆ Mammals ⊆ Animals, therefore Cats ⊆ Animals. So every cat is an animal. +``` + +--- + +## When to Use Each Mode + +### Use Fast Thinking (`"no_think"`) For: +- Casual conversation and chat +- Translation tasks +- Text summarization +- Simple factual queries +- When latency matters more than depth +- When token cost is a concern + +### Use Deep Reasoning (`"high"`) For: +- Mathematical proofs and calculations +- Complex coding problems +- Multi-step logical reasoning +- Planning and strategy +- Analysis requiring structured thinking +- When accuracy > speed + +### Use Light Reasoning (`"low"`) For: +- Moderate analysis tasks +- When you want some reasoning but don't need full depth +- Balancing speed and quality + +--- + +## Accessing Reasoning Content in the Response + +```python +message = response.choices[0].message + +# The final answer +final_answer = message.content + +# The thinking process (only with reasoning_effort="low" or "high") +thinking_process = getattr(message, "reasoning_content", None) + +if thinking_process: + print(f"Model thought for {len(thinking_process)} characters before answering.") + print(f"Thinking: {thinking_process[:300]}...") +print(f"Answer: {final_answer}") +``` + +In streaming mode, reasoning content arrives in `delta.reasoning_content` before the final answer in `delta.content`. See [Example 2: Streaming](02-streaming.md) for details. + +--- + +## Key Takeaways + +1. **`reasoning_effort` is passed via `extra_body`**, not as a top-level parameter. +2. **Default is `"no_think"`** — the model answers directly without visible reasoning. +3. **Deep reasoning increases token usage** but dramatically improves accuracy on complex tasks. +4. **`reasoning_content`** contains the model's internal thinking — access it for transparency or debugging. +5. **Match the mode to the task** — don't use deep reasoning for simple chat; don't use fast thinking for math proofs. + +--- + +## Run the Script + +```bash +pip install openai +python 05-reasoning-mode.py +``` diff --git a/quickstart/examples/05-reasoning-mode.py b/quickstart/examples/05-reasoning-mode.py new file mode 100644 index 00000000..c7528e65 --- /dev/null +++ b/quickstart/examples/05-reasoning-mode.py @@ -0,0 +1,206 @@ +""" +Example 5: Reasoning Mode — compare fast vs. deep thinking on the same prompts. + +Usage: + python 05-reasoning-mode.py + +Prerequisites: + - Hy3 server running at http://127.0.0.1:8000 (vLLM or SGLang) + - vLLM must be launched with --reasoning-parser hy_v3 + or SGLang with --reasoning-parser hunyuan + - pip install openai +""" + +import time +from openai import OpenAI + +# ────────────────────────────────────────────────────────────────────── +# Configuration +# ────────────────────────────────────────────────────────────────────── +BASE_URL = "http://127.0.0.1:8000/v1" +API_KEY = "EMPTY" +MODEL = "hy3" + +client = OpenAI(base_url=BASE_URL, api_key=API_KEY) + + +# ────────────────────────────────────────────────────────────────────── +# 1. Fast Thinking (no_think) — Default Mode +# ────────────────────────────────────────────────────────────────────── +def fast_thinking(): + print("=" * 60) + print("1. FAST THINKING (no_think)") + print("=" * 60) + + prompts = [ + "What is the capital of France?", + "Translate 'Good morning, how are you?' to Japanese.", + "Summarize the theory of evolution in two sentences.", + ] + + for prompt in prompts: + start = time.perf_counter() + response = client.chat.completions.create( + model=MODEL, + messages=[{"role": "user", "content": prompt}], + temperature=0.9, + top_p=1.0, + max_tokens=256, + extra_body={"chat_template_kwargs": {"reasoning_effort": "no_think"}}, + ) + elapsed = time.perf_counter() - start + + msg = response.choices[0].message + thinking = getattr(msg, "reasoning_content", None) + has_thinking = bool(thinking) + + print(f"\n📝 {prompt}") + print(f" Answer: {msg.content[:120]}{'...' if len(msg.content or '') > 120 else ''}") + print(f" Time: {elapsed:.2f}s") + print(f" Tokens: {response.usage.total_tokens}") + print(f" Thinking: {'Yes (' + str(len(thinking)) + ' chars)' if has_thinking else 'No (direct)'}") + + +# ────────────────────────────────────────────────────────────────────── +# 2. Deep Reasoning (high) — Chain-of-Thought +# ────────────────────────────────────────────────────────────────────── +def deep_reasoning(): + print("\n" + "=" * 60) + print("2. DEEP REASONING (high)") + print("=" * 60) + + prompts = [ + "If a train leaves Boston at 60 mph and another leaves New York at 80 mph, " + "and the cities are 215 miles apart, when and where do they meet?", + "Prove that there are infinitely many prime numbers.", + ] + + for prompt in prompts: + start = time.perf_counter() + response = client.chat.completions.create( + model=MODEL, + messages=[{"role": "user", "content": prompt}], + temperature=0.9, + top_p=1.0, + max_tokens=1024, + extra_body={"chat_template_kwargs": {"reasoning_effort": "high"}}, + ) + elapsed = time.perf_counter() - start + + msg = response.choices[0].message + thinking = getattr(msg, "reasoning_content", None) + content = msg.content or "" + + print(f"\n📝 {prompt[:100]}...") + if thinking: + print(f" 🧠 Thinking: {len(thinking)} chars") + print(f" 💬 Answer: {len(content)} chars") + print(f" ⏱️ Time: {elapsed:.2f}s") + print(f" 🔢 Tokens: {response.usage.total_tokens}") + print(f"\n --- Thinking preview ---") + print(f" {thinking[:300]}{'...' if len(thinking) > 300 else ''}") + print(f"\n --- Answer preview ---") + print(f" {content[:300]}{'...' if len(content) > 300 else ''}") + else: + print(f" Answer: {content[:200]}...") + print(f" Time: {elapsed:.2f}s") + print(f" Tokens: {response.usage.total_tokens}") + + +# ────────────────────────────────────────────────────────────────────── +# 3. Side-by-Side Comparison +# ────────────────────────────────────────────────────────────────────── +def compare_modes(): + print("\n" + "=" * 60) + print("3. FAST vs DEEP — SIDE-BY-SIDE COMPARISON") + print("=" * 60) + + test_cases = [ + ("Math", "Solve: A rectangle has perimeter 30 and area 54. Find its dimensions."), + ("Logic", "If all A are B, and some C are not B, can some C be A? Explain."), + ("Code", "Write a function to check if a string is a valid palindrome, ignoring case and non-alphanumeric chars."), + ] + + modes = [ + ("no_think", "Fast"), + ("high", "Deep"), + ] + + for category, prompt in test_cases: + print(f"\n{'─' * 60}") + print(f"📂 Category: {category}") + print(f"📝 Prompt: {prompt[:100]}{'...' if len(prompt) > 100 else ''}") + + for effort, label in modes: + start = time.perf_counter() + response = client.chat.completions.create( + model=MODEL, + messages=[{"role": "user", "content": prompt}], + temperature=0.9, + top_p=1.0, + max_tokens=512, + extra_body={"chat_template_kwargs": {"reasoning_effort": effort}}, + ) + elapsed = time.perf_counter() - start + + msg = response.choices[0].message + thinking = getattr(msg, "reasoning_content", None) + content = msg.content or "" + + print(f"\n [{label}] ⏱️ {elapsed:.2f}s 🔢 {response.usage.total_tokens} tokens " + f"🧠 {'Yes' if thinking else 'No'} thinking") + print(f" [{label}] {content[:150]}{'...' if len(content) > 150 else ''}") + + +# ────────────────────────────────────────────────────────────────────── +# 4. Streaming with Reasoning +# ────────────────────────────────────────────────────────────────────── +def streaming_reasoning(): + print("\n" + "=" * 60) + print("4. STREAMING WITH REASONING") + print("=" * 60) + + prompt = "Explain step by step: why does ice float on water?" + print(f"\n📝 {prompt}\n") + + stream = client.chat.completions.create( + model=MODEL, + messages=[{"role": "user", "content": prompt}], + temperature=0.9, + top_p=1.0, + max_tokens=512, + stream=True, + extra_body={"chat_template_kwargs": {"reasoning_effort": "high"}}, + ) + + in_thinking = False + for chunk in stream: + if not chunk.choices: + continue + delta = chunk.choices[0].delta + + # Reasoning content + if getattr(delta, "reasoning_content", None): + if not in_thinking: + print("\n🧠 THINKING: ", end="") + in_thinking = True + print(f"\x1b[90m{delta.reasoning_content}\x1b[0m", end="", flush=True) + + # Final answer content + if delta.content: + if in_thinking: + print("\n\n💬 ANSWER: ", end="") + in_thinking = False + print(delta.content, end="", flush=True) + + print("\n") + + +# ────────────────────────────────────────────────────────────────────── +# Main +# ────────────────────────────────────────────────────────────────────── +if __name__ == "__main__": + fast_thinking() + deep_reasoning() + compare_modes() + streaming_reasoning() diff --git a/quickstart/examples/06-error-handling.md b/quickstart/examples/06-error-handling.md new file mode 100644 index 00000000..4076bc68 --- /dev/null +++ b/quickstart/examples/06-error-handling.md @@ -0,0 +1,350 @@ +# Example 6: Error Handling & Retry + +Production-grade error handling for the Hy3 API — handle timeouts, rate limits, network errors, and server errors with automatic retry and exponential backoff. + +## What You'll Learn + +- Identify and classify common API errors +- Implement retry with exponential backoff +- Handle rate limiting gracefully +- Set timeouts and handle network errors +- Build a production-ready API wrapper + +--- + +## Common Error Types + +| Error | HTTP Code | Typical Cause | Retryable? | +|:---|:---|:---|:---| +| `BadRequestError` | 400 | Invalid parameters, bad tool schema | ❌ No | +| `AuthenticationError` | 401 | Wrong API key | ❌ No | +| `RateLimitError` | 429 | Too many requests | ✅ Yes | +| `InternalServerError` | 500 | Server-side error | ✅ Yes | +| `APITimeoutError` | — | Request timed out | ✅ Yes | +| `APIConnectionError` | — | Network issue | ✅ Yes | +| `APIError` | other | Other API errors | ⚠️ Maybe | + +--- + +## Basic Error Handling + +```python +from openai import ( + OpenAI, + BadRequestError, + AuthenticationError, + RateLimitError, + InternalServerError, + APITimeoutError, + APIConnectionError, + APIError, +) + +client = OpenAI( + base_url="http://127.0.0.1:8000/v1", + api_key="EMPTY", + timeout=60.0, # 60-second timeout +) + +def safe_chat(messages, **kwargs): + """Make a chat completion request with basic error handling.""" + try: + response = client.chat.completions.create( + model="hy3", + messages=messages, + **kwargs, + ) + return response.choices[0].message.content + + except BadRequestError as e: + print(f"❌ Bad request (400): {e}") + print(" Check your parameters, tool schema, or message format.") + raise + + except AuthenticationError as e: + print(f"❌ Authentication failed (401): {e}") + print(" Check your API key.") + raise + + except RateLimitError as e: + print(f"⏳ Rate limited (429): {e}") + print(" Slow down your requests or increase your quota.") + raise + + except APITimeoutError as e: + print(f"⏰ Request timed out: {e}") + print(" Try increasing timeout or reducing max_tokens.") + raise + + except APIConnectionError as e: + print(f"🔌 Connection error: {e}") + print(" Check if the server is running and reachable.") + raise + + except InternalServerError as e: + print(f"💥 Server error (500): {e}") + print(" The server encountered an error. Retry may help.") + raise + + except APIError as e: + print(f"⚠️ API error ({e.status_code}): {e}") + raise + + except Exception as e: + print(f"🔥 Unexpected error: {type(e).__name__}: {e}") + raise +``` + +--- + +## Retry with Exponential Backoff + +The most reliable approach: retry on transient errors with increasing delays. + +```python +import time +import random + +RETRYABLE_ERRORS = ( + RateLimitError, + InternalServerError, + APITimeoutError, + APIConnectionError, +) + + +def chat_with_retry( + messages, + max_retries=3, + base_delay=1.0, + max_delay=60.0, + **kwargs, +): + """ + Make a chat completion request with automatic retry + exponential backoff. + + Args: + messages: List of message dicts. + max_retries: Maximum number of retry attempts. + base_delay: Initial delay in seconds (doubles each retry). + max_delay: Maximum delay cap in seconds. + + Returns: + The assistant's reply text. + """ + last_error = None + + for attempt in range(max_retries + 1): + try: + response = client.chat.completions.create( + model="hy3", + messages=messages, + **kwargs, + ) + return response.choices[0].message.content + + except RETRYABLE_ERRORS as e: + last_error = e + + if attempt == max_retries: + print(f"❌ All {max_retries} retries exhausted. Last error: {e}") + raise + + # Exponential backoff with jitter + delay = min(base_delay * (2 ** attempt), max_delay) + jitter = random.uniform(0, delay * 0.5) + wait = delay + jitter + + print(f"🔄 Retry {attempt + 1}/{max_retries} after {wait:.1f}s " + f"({type(e).__name__})") + + time.sleep(wait) + + except (BadRequestError, AuthenticationError) as e: + # Non-retryable — fail immediately + print(f"❌ Non-retryable error: {type(e).__name__}: {e}") + raise + + raise last_error +``` + +### Retry Timeline + +``` +Attempt 0: fails with 429 RateLimitError + → wait ~1.2s (base 1s + jitter) + +Attempt 1: fails with 500 InternalServerError + → wait ~2.3s (base 2s + jitter) + +Attempt 2: fails with APITimeoutError + → wait ~4.6s (base 4s + jitter) + +Attempt 3: succeeds ✅ +``` + +--- + +## Handling Rate Limits (429) + +When the server returns 429 (Too Many Requests), check for `Retry-After` header: + +```python +def chat_with_rate_limit_handling(messages, **kwargs): + """Handle rate limiting with Retry-After header support.""" + while True: + try: + response = client.chat.completions.create( + model="hy3", + messages=messages, + **kwargs, + ) + return response.choices[0].message.content + + except RateLimitError as e: + # Check for Retry-After header + retry_after = getattr(e, "response", None) + if retry_after is not None: + retry_after = retry_after.headers.get("Retry-After") + retry_after = retry_after if retry_after else None + + if retry_after: + try: + wait = float(retry_after) + except ValueError: + wait = 5.0 + print(f"⏳ Rate limited. Waiting {wait:.0f}s (per Retry-After header)...") + else: + wait = 5.0 + print(f"⏳ Rate limited. Waiting {wait:.0f}s...") + + time.sleep(wait) + + except APITimeoutError: + print("⏰ Timeout — retrying with shorter max_tokens...") + kwargs["max_tokens"] = max(64, kwargs.get("max_tokens", 256) // 2) +``` + +--- + +## Timeout Configuration + +```python +# Global timeout (applies to all requests) +client = OpenAI( + base_url="http://127.0.0.1:8000/v1", + api_key="EMPTY", + timeout=120.0, # 120 seconds +) + +# Per-request timeout override +response = client.chat.completions.create( + model="hy3", + messages=[{"role": "user", "content": "..."}], + timeout=30.0, # 30 seconds for this specific request +) + +# Separate connect vs. read timeout +from httpx import Timeout + +client = OpenAI( + base_url="http://127.0.0.1:8000/v1", + api_key="EMPTY", + timeout=Timeout(connect=10.0, read=120.0, write=10.0, pool=10.0), +) +``` + +--- + +## Full Production Wrapper + +Combining everything into a reusable class: + +```python +class Hy3Client: + """Production-grade Hy3 API client with retry, timeout, and error handling.""" + + def __init__( + self, + base_url="http://127.0.0.1:8000/v1", + api_key="EMPTY", + timeout=120.0, + max_retries=3, + base_delay=1.0, + max_delay=60.0, + ): + self.client = OpenAI(base_url=base_url, api_key=api_key, timeout=timeout) + self.max_retries = max_retries + self.base_delay = base_delay + self.max_delay = max_delay + + def chat(self, messages, **kwargs): + """Send a chat request with automatic retry on transient errors.""" + last_error = None + + for attempt in range(self.max_retries + 1): + try: + response = self.client.chat.completions.create( + model="hy3", + messages=messages, + **kwargs, + ) + return response + + except (RateLimitError, InternalServerError, + APITimeoutError, APIConnectionError) as e: + last_error = e + if attempt == self.max_retries: + raise + + delay = min(self.base_delay * (2 ** attempt), self.max_delay) + jitter = random.uniform(0, delay * 0.5) + time.sleep(delay + jitter) + + except (BadRequestError, AuthenticationError) as e: + raise # Non-retryable + + raise last_error + + def chat_content(self, messages, **kwargs): + """Convenience: return content string directly.""" + return self.chat(messages, **kwargs).choices[0].message.content +``` + +### Usage + +```python +hy3 = Hy3Client(max_retries=3, timeout=60.0) + +# Safe, retryable chat +try: + reply = hy3.chat_content([ + {"role": "user", "content": "Explain quantum computing."}, + ]) + print(reply) +except BadRequestError: + print("Invalid request — check your parameters.") +except Exception as e: + print(f"Failed after all retries: {e}") +``` + +--- + +## Key Takeaways + +1. **Always set a timeout** — `timeout=120.0` prevents hanging requests. +2. **Retry transient errors** — 429, 500, timeouts, and connection errors are retryable. +3. **Use exponential backoff with jitter** — prevents thundering herd on recovery. +4. **Never retry 400 (Bad Request)** — fix the request instead. +5. **Check `Retry-After` headers** for rate limits to avoid wasting retries. +6. **Cap max_delay** at 60s to avoid excessive waits. +7. **Log every retry** — it helps with debugging and monitoring. + +--- + +## Run the Script + +```bash +pip install openai +python 06-error-handling.py +``` diff --git a/quickstart/examples/06-error-handling.py b/quickstart/examples/06-error-handling.py new file mode 100644 index 00000000..4489683f --- /dev/null +++ b/quickstart/examples/06-error-handling.py @@ -0,0 +1,422 @@ +""" +Example 6: Error Handling & Retry — production-grade error handling for Hy3 API. + +Covers: + - Error type identification and classification + - Retry with exponential backoff + jitter + - Rate limit handling with Retry-After header + - Timeout configuration + - Production-ready client wrapper + +Usage: + python 06-error-handling.py + +Prerequisites: + - Hy3 server running at http://127.0.0.1:8000 (vLLM or SGLang) + - pip install openai +""" + +import time +import random +from openai import ( + OpenAI, + BadRequestError, + AuthenticationError, + RateLimitError, + InternalServerError, + APITimeoutError, + APIConnectionError, + APIError, +) + +# ────────────────────────────────────────────────────────────────────── +# Configuration +# ────────────────────────────────────────────────────────────────────── +BASE_URL = "http://127.0.0.1:8000/v1" +API_KEY = "EMPTY" +MODEL = "hy3" + +RETRYABLE_ERRORS = ( + RateLimitError, + InternalServerError, + APITimeoutError, + APIConnectionError, +) + +NON_RETRYABLE_ERRORS = ( + BadRequestError, + AuthenticationError, +) + +client = OpenAI(base_url=BASE_URL, api_key=API_KEY, timeout=120.0) + + +# ────────────────────────────────────────────────────────────────────── +# 1. Basic Error Classification +# ────────────────────────────────────────────────────────────────────── +def demonstrate_error_types(): + """Show how to catch and classify different OpenAI API errors.""" + print("=" * 60) + print("1. ERROR TYPE CLASSIFICATION") + print("=" * 60) + + test_cases = [ + # (description, kwargs, expected_error) + ("Valid request", { + "messages": [{"role": "user", "content": "Hello!"}], + "max_tokens": 32, + }, None), + ("Invalid model name", { + "model": "nonexistent-model", + "messages": [{"role": "user", "content": "Hello!"}], + }, "error"), + ("Missing messages (should 400)", { + # messages intentionally omitted — this will raise a TypeError + # from the SDK before hitting the API, so we handle it separately + }, "TypeError"), + ] + + # Test 1: Normal request — should succeed + print("\n--- Test: Valid request ---") + try: + response = client.chat.completions.create( + model=MODEL, + messages=[{"role": "user", "content": "Say 'hello' in one word."}], + temperature=0.9, + top_p=1.0, + max_tokens=16, + ) + print(f"✅ Success: {response.choices[0].message.content}") + except Exception as e: + print(f"❌ Error: {type(e).__name__}: {e}") + + # Test 2: Bad model name — should get 400 + print("\n--- Test: Invalid model name ---") + try: + response = client.chat.completions.create( + model="nonexistent-model", + messages=[{"role": "user", "content": "Hello!"}], + ) + print(f"✅ Success (unexpected): {response.choices[0].message.content}") + except BadRequestError as e: + print(f"❌ BadRequestError (400) — as expected: {e}") + print(" → This is NOT retryable. Fix the model name.") + except Exception as e: + print(f"❌ Other error: {type(e).__name__}: {e}") + + +# ────────────────────────────────────────────────────────────────────── +# 2. Retry with Exponential Backoff +# ────────────────────────────────────────────────────────────────────── +def chat_with_retry( + messages, + max_retries=3, + base_delay=1.0, + max_delay=60.0, + **kwargs, +): + """ + Chat completion with automatic retry + exponential backoff + jitter. + + Retryable errors: RateLimitError, InternalServerError, + APITimeoutError, APIConnectionError + Non-retryable: BadRequestError, AuthenticationError + + Args: + messages: List of message dicts. + max_retries: Max retry attempts (default: 3). + base_delay: Initial delay in seconds, doubles each retry. + max_delay: Maximum delay cap in seconds. + + Returns: + The full ChatCompletion response object. + """ + last_error = None + + for attempt in range(max_retries + 1): + try: + response = client.chat.completions.create( + model=MODEL, + messages=messages, + **kwargs, + ) + if attempt > 0: + print(f" ✅ Succeeded on attempt {attempt + 1}") + return response + + except RETRYABLE_ERRORS as e: + last_error = e + + if attempt == max_retries: + print(f" ❌ All {max_retries} retries exhausted.") + raise + + # Exponential backoff: 1s → 2s → 4s → ... capped at max_delay + delay = min(base_delay * (2 ** attempt), max_delay) + # Add jitter: ±25% random variation to avoid thundering herd + jitter = random.uniform(-delay * 0.25, delay * 0.25) + wait = max(0.1, delay + jitter) + + print(f" 🔄 Retry {attempt + 1}/{max_retries} after {wait:.1f}s " + f"({type(e).__name__}: {str(e)[:80]})") + + time.sleep(wait) + + except NON_RETRYABLE_ERRORS as e: + print(f" ❌ Non-retryable error: {type(e).__name__}: {e}") + raise + + raise last_error + + +def demonstrate_retry_logic(): + """Show how the retry wrapper works with a normal request.""" + print("\n" + "=" * 60) + print("2. RETRY WITH EXPONENTIAL BACKOFF") + print("=" * 60) + print(" (Making a normal request — retries only trigger on errors)\n") + + try: + response = chat_with_retry( + messages=[{"role": "user", "content": "What is 2+2? Answer in one word."}], + max_retries=3, + temperature=0.9, + top_p=1.0, + max_tokens=32, + ) + print(f"\n 📝 Response: {response.choices[0].message.content}") + print(f" 🔢 Tokens: {response.usage.total_tokens}") + except Exception as e: + print(f"\n ❌ Failed: {type(e).__name__}: {e}") + + +# ────────────────────────────────────────────────────────────────────── +# 3. Rate Limit Handling with Retry-After +# ────────────────────────────────────────────────────────────────────── +def chat_with_rate_limit_handling(messages, **kwargs): + """ + Handle rate limits with Retry-After header support. + + When the server returns 429, it may include a Retry-After header + indicating how long to wait. This function respects that header. + """ + while True: + try: + response = client.chat.completions.create( + model=MODEL, + messages=messages, + **kwargs, + ) + return response + + except RateLimitError as e: + # Try to extract Retry-After header + retry_after = None + try: + if hasattr(e, "response") and e.response: + retry_after = e.response.headers.get("Retry-After") + except Exception: + pass + + if retry_after: + try: + wait = float(retry_after) + except ValueError: + wait = 5.0 + print(f" ⏳ Rate limited. Respecting Retry-After: {wait:.0f}s") + else: + wait = 5.0 + print(f" ⏳ Rate limited. No Retry-After header. Waiting {wait:.0f}s...") + + time.sleep(wait) + + except RETRYABLE_ERRORS as e: + print(f" 🔄 Transient error: {type(e).__name__}. Retrying in 2s...") + time.sleep(2) + + +# ────────────────────────────────────────────────────────────────────── +# 4. Timeout Configuration +# ────────────────────────────────────────────────────────────────────── +def demonstrate_timeouts(): + """Show different timeout configurations.""" + print("\n" + "=" * 60) + print("3. TIMEOUT CONFIGURATION") + print("=" * 60) + + # Per-request timeout + print("\n--- Per-request timeout (30s) ---") + try: + start = time.perf_counter() + response = client.chat.completions.create( + model=MODEL, + messages=[{"role": "user", "content": "Say hello."}], + max_tokens=16, + timeout=30.0, + ) + elapsed = time.perf_counter() - start + print(f" ✅ Completed in {elapsed:.2f}s") + print(f" 📝 {response.choices[0].message.content}") + except APITimeoutError: + print(" ⏰ Request timed out after 30s") + except Exception as e: + print(f" ❌ {type(e).__name__}: {e}") + + # Short timeout that may trigger (for demonstration) + print("\n--- Aggressive timeout (0.001s) — will likely fire ---") + try: + response = client.chat.completions.create( + model=MODEL, + messages=[{"role": "user", "content": "Write a very long essay about the history of Rome."}], + max_tokens=2048, + timeout=0.001, # Impossible timeout — will fire immediately + ) + print(f" ✅ Unexpected success") + except APITimeoutError: + print(" ⏰ APITimeoutError caught — this is expected with 0.001s timeout") + except Exception as e: + print(f" ❌ {type(e).__name__}: {e}") + + +# ────────────────────────────────────────────────────────────────────── +# 5. Production-Grade Client Wrapper +# ────────────────────────────────────────────────────────────────────── +class Hy3Client: + """ + Production-grade Hy3 API client with built-in retry, timeout, + and comprehensive error handling. + + Usage: + hy3 = Hy3Client(max_retries=3) + reply = hy3.chat_content([{"role": "user", "content": "Hello"}]) + """ + + def __init__( + self, + base_url=BASE_URL, + api_key=API_KEY, + timeout=120.0, + max_retries=3, + base_delay=1.0, + max_delay=60.0, + ): + self.client = OpenAI(base_url=base_url, api_key=api_key, timeout=timeout) + self.max_retries = max_retries + self.base_delay = base_delay + self.max_delay = max_delay + + def chat(self, messages, **kwargs): + """Send a chat request with retry. Returns full ChatCompletion object.""" + last_error = None + + for attempt in range(self.max_retries + 1): + try: + response = self.client.chat.completions.create( + model=MODEL, + messages=messages, + **kwargs, + ) + return response + + except RETRYABLE_ERRORS as e: + last_error = e + if attempt == self.max_retries: + raise + + delay = min(self.base_delay * (2 ** attempt), self.max_delay) + jitter = random.uniform(-delay * 0.25, delay * 0.25) + wait = max(0.1, delay + jitter) + + print(f" 🔄 [{type(e).__name__}] Retry {attempt + 1}/{self.max_retries} " + f"in {wait:.1f}s") + + time.sleep(wait) + + except NON_RETRYABLE_ERRORS as e: + raise + + raise last_error + + def chat_content(self, messages, **kwargs): + """Convenience method: return content string directly.""" + return self.chat(messages, **kwargs).choices[0].message.content + + def chat_stream(self, messages, **kwargs): + """Stream a chat response, yielding content deltas.""" + kwargs["stream"] = True + response = self.chat(messages, **kwargs) + for chunk in response: + if chunk.choices and chunk.choices[0].delta.content: + yield chunk.choices[0].delta.content + + +def demonstrate_client_wrapper(): + """Show the production client wrapper in action.""" + print("\n" + "=" * 60) + print("4. PRODUCTION CLIENT WRAPPER") + print("=" * 60) + + hy3 = Hy3Client(max_retries=3, timeout=60.0) + + # Normal chat + print("\n--- Normal chat ---") + try: + reply = hy3.chat_content( + [{"role": "user", "content": "Explain APIs in one sentence."}], + temperature=0.9, + max_tokens=128, + ) + print(f" 📝 {reply}") + except Exception as e: + print(f" ❌ {type(e).__name__}: {e}") + + # Streaming via wrapper + print("\n--- Streaming via wrapper ---") + try: + for token in hy3.chat_stream( + [{"role": "user", "content": "Count from 1 to 5."}], + temperature=0.9, + max_tokens=64, + ): + print(token, end="", flush=True) + print() + except Exception as e: + print(f"\n ❌ {type(e).__name__}: {e}") + + +# ────────────────────────────────────────────────────────────────────── +# 6. Error Recovery Strategy Summary +# ────────────────────────────────────────────────────────────────────── +def print_strategy_summary(): + print("\n" + "=" * 60) + print("5. RECOVERY STRATEGY SUMMARY") + print("=" * 60) + + print(""" + ┌──────────────────────┬──────────┬──────────────────────────────┐ + │ Error │ Retry? │ Strategy │ + ├──────────────────────┼──────────┼──────────────────────────────┤ + │ BadRequestError(400) │ NO │ Fix parameters, tool schema │ + │ AuthError(401) │ NO │ Check API key │ + │ RateLimitError(429) │ YES │ Retry-After + backoff │ + │ InternalError(500) │ YES │ Exponential backoff + jitter │ + │ APITimeoutError │ YES │ Increase timeout, backoff │ + │ APIConnectionError │ YES │ Check server, backoff │ + │ Other APIError │ MAYBE │ Depends on status code │ + └──────────────────────┴──────────┴──────────────────────────────┘ + + Backoff formula: delay = min(base_delay * 2^attempt, max_delay) + Jitter: ±25% random variation to avoid thundering herd + Non-retryable: fail immediately, surface error to caller + """) + + +# ────────────────────────────────────────────────────────────────────── +# Main +# ────────────────────────────────────────────────────────────────────── +if __name__ == "__main__": + demonstrate_error_types() + demonstrate_retry_logic() + demonstrate_timeouts() + demonstrate_client_wrapper() + print_strategy_summary() diff --git a/quickstart/quickstart.md b/quickstart/quickstart.md new file mode 100644 index 00000000..764e2aeb --- /dev/null +++ b/quickstart/quickstart.md @@ -0,0 +1,461 @@ +# Hy3 API Quickstart + +Get your first Hy3 API call running in 5 minutes, and master the full feature set in 30 minutes. + +--- + +## Table of Contents + +- [Prerequisites](#prerequisites) +- [5-Minute Quickstart](#5-minute-quickstart) + - [1. Deploy Hy3](#1-deploy-hy3) + - [2. Your First Call (curl)](#2-your-first-call-curl) + - [3. Your First Call (Python SDK)](#3-your-first-call-python-sdk) +- [API Reference](#api-reference) + - [Base Information](#base-information) + - [Request Parameters](#request-parameters) + - [Response Structure](#response-structure) +- [Reasoning Mode](#reasoning-mode) +- [Tool Calling](#tool-calling) +- [Rate Limits & Concurrency](#rate-limits--concurrency) +- [Troubleshooting](#troubleshooting) +- [Examples](#examples) + +--- + +## Prerequisites + +- **Deployed Hy3 server** via vLLM or SGLang (see [Deployment](#1-deploy-hy3) below). +- **Python 3.10+** with `openai` SDK installed: + ```bash + pip install openai + ``` +- The API is **OpenAI-compatible** — you can use any OpenAI SDK or HTTP client. + +--- + +## 5-Minute Quickstart + +### 1. Deploy Hy3 + +Choose one of the deployment options below to start a local API server. + +
+Option A: vLLM (recommended) + +```bash +# Build vLLM from source +uv venv --python 3.12 --seed --managed-python +source .venv/bin/activate +git clone https://github.com/vllm-project/vllm.git +cd vllm +uv pip install --editable . --torch-backend=auto + +# Start the server with MTP speculative decoding +export VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm + +vllm serve tencent/Hy3 \ + --tensor-parallel-size 8 \ + --speculative-config.method mtp \ + --speculative-config.num_speculative_tokens 2 \ + --tool-call-parser hy_v3 \ + --reasoning-parser hy_v3 \ + --enable-auto-tool-choice \ + --port 8000 \ + --served-model-name hy3 +``` + +> **Hardware**: 8×H20-3e (or GPUs with ≥80GB VRAM) for full BF16 deployment. + +
+ +
+Option B: SGLang + +```bash +git clone https://github.com/sgl-project/sglang +cd sglang +pip3 install pip --upgrade +pip3 install "transformers>=5.6.0" +pip3 install -e "python" + +python3 -m sglang.launch_server \ + --model tencent/Hy3 \ + --tp-size 8 \ + --tool-call-parser hunyuan \ + --reasoning-parser hunyuan \ + --speculative-num-steps 2 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 3 \ + --speculative-algorithm EAGLE \ + --port 8000 \ + --served-model-name hy3 +``` + +
+ +
+Option C: Third-party API endpoint + +If you are using a hosted Hy3 endpoint (e.g., from Tencent Cloud or a partner platform), replace the base URL and API key with the ones provided by your platform. The API interface remains the same. + +
+ +Once the server is running, verify it's reachable: + +```bash +curl http://127.0.0.1:8000/v1/models +``` + +### 2. Your First Call (curl) + +```bash +curl http://127.0.0.1:8000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer EMPTY" \ + -d '{ + "model": "hy3", + "messages": [ + {"role": "user", "content": "Hello! Can you briefly introduce yourself?"} + ], + "temperature": 0.9, + "top_p": 1.0, + "max_tokens": 256 + }' +``` + +**Expected response:** + +```json +{ + "id": "chatcmpl-xxxxxxxx", + "object": "chat.completion", + "created": 1719000000, + "model": "hy3", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Hello! I'm Hy3, a 295B-parameter Mixture-of-Experts large language model developed by Tencent's Hunyuan team..." + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 15, + "completion_tokens": 44, + "total_tokens": 59 + } +} +``` + +### 3. Your First Call (Python SDK) + +```python +from openai import OpenAI + +# Initialize client — base_url points to your local server +client = OpenAI( + base_url="http://127.0.0.1:8000/v1", + api_key="EMPTY", # No authentication needed for local deployment +) + +# Make your first call +response = client.chat.completions.create( + model="hy3", + messages=[ + {"role": "user", "content": "Hello! Can you briefly introduce yourself?"}, + ], + temperature=0.9, + top_p=1.0, + max_tokens=256, +) + +# Print the response +print(response.choices[0].message.content) +``` + +> ✅ **You're done!** You've made your first Hy3 API call. Read on for the full API reference, or jump to the [Examples](#examples) section. + +--- + +## API Reference + +### Base Information + +| Item | Value | +|:---|:---| +| **Base URL** | `http://127.0.0.1:8000/v1` (local default) | +| **API Key** | `EMPTY` (local deployment) or your platform key | +| **Model Name** | `hy3` | +| **API Protocol** | OpenAI-compatible `/v1/chat/completions` | +| **Context Length** | 256K tokens | +| **Supported Methods** | Chat completions (sync + streaming), tool calling, reasoning mode | + +### Request Parameters + +#### Required + +| Parameter | Type | Description | +|:---|:---|:---| +| `model` | `string` | Model name. Use `"hy3"`. | +| `messages` | `array` | List of message objects. See [Message Format](#message-format). | + +#### Optional + +| Parameter | Type | Default | Description | +|:---|:---|:---|:---| +| `temperature` | `float` | `0.9` | Sampling temperature (0–2.0). Higher = more creative. **Recommended: 0.9**. | +| `top_p` | `float` | `1.0` | Nucleus sampling threshold (0–1.0). **Recommended: 1.0**. | +| `max_tokens` | `int` | model max | Maximum tokens to generate. If omitted, the model generates until a stop condition. | +| `stop` | `string` / `array` | — | Stop sequence(s). Generation stops when any sequence is encountered. | +| `stream` | `bool` | `false` | If `true`, the response is streamed as server-sent events (SSE). | +| `tools` | `array` | — | List of tool definitions for function calling. See [Tool Calling](#tool-calling). | +| `tool_choice` | `string` / `object` | `"auto"` | Tool selection strategy: `"auto"`, `"none"`, `"required"`, or a specific tool. | +| `seed` | `int` | — | Random seed for reproducible outputs. | +| `extra_body` | `object` | — | Extra parameters passed to the backend. Used for reasoning mode. | + +#### Reasoning Mode (via `extra_body`) + +| Parameter | Type | Default | Description | +|:---|:---|:---|:---| +| `extra_body.chat_template_kwargs.reasoning_effort` | `string` | `"no_think"` | `"no_think"` — direct response; `"low"` — brief reasoning; `"high"` — deep chain-of-thought. | + +### Message Format + +```json +{ + "role": "system | user | assistant | tool", + "content": "message text or multimodal content", + "tool_calls": [{"id": "...", "type": "function", "function": {"name": "...", "arguments": "..."}}], + "tool_call_id": "..." // only for role: "tool" +} +``` + +### Response Structure + +**Non-streaming:** + +```json +{ + "id": "chatcmpl-xxxxxxxx", + "object": "chat.completion", + "created": 1719000000, + "model": "hy3", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "response text...", + "tool_calls": [] + }, + "finish_reason": "stop | length | tool_calls" + } + ], + "usage": { + "prompt_tokens": 15, + "completion_tokens": 44, + "total_tokens": 59 + } +} +``` + +**Streaming (SSE):** + +``` +data: {"id":"...","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null}]} + +data: {"id":"...","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"content":"Hello"},"finish_reason":null}]} + +data: {"id":"...","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"content":"!"},"finish_reason":null}]} + +data: {"id":"...","object":"chat.completion.chunk","choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":{...}} + +data: [DONE] +``` + +--- + +## Reasoning Mode + +Hy3 supports both **fast thinking** (direct response) and **slow thinking** (chain-of-thought reasoning): + +| Mode | `reasoning_effort` | Best for | Latency | +|:---|:---|:---|:---| +| **Fast** | `"no_think"` (default) | Simple Q&A, chat, translation, summarization | Lowest | +| **Reasoning** | `"low"` | Moderate reasoning, planning, analysis | Medium | +| **Deep reasoning** | `"high"` | Math proofs, complex coding, multi-step logic | Highest | + +```python +# Fast thinking (default) +response = client.chat.completions.create( + model="hy3", + messages=[{"role": "user", "content": "What is 2+2?"}], + extra_body={"chat_template_kwargs": {"reasoning_effort": "no_think"}}, +) + +# Deep reasoning +response = client.chat.completions.create( + model="hy3", + messages=[{"role": "user", "content": "Prove the Pythagorean theorem."}], + extra_body={"chat_template_kwargs": {"reasoning_effort": "high"}}, +) +``` + +When reasoning mode is on, the model's thinking process is returned in `reasoning_content` and the final answer in `content`. In streaming mode, watch for `delta.reasoning_content` vs `delta.content`. + +> ⚠️ **Note**: `reasoning_effort` is passed via `extra_body` (not as a top-level parameter). This is because the reasoning mode is implemented at the chat-template level in vLLM/SGLang. + +--- + +## Tool Calling + +Hy3 supports OpenAI-compatible function calling. Define tools in the request, and the model returns structured `tool_calls`: + +```python +tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get current weather for a location", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string", "description": "City name"} + }, + "required": ["location"], + }, + }, + } +] + +response = client.chat.completions.create( + model="hy3", + messages=[{"role": "user", "content": "What's the weather in Beijing?"}], + tools=tools, + tool_choice="auto", +) + +# Check for tool calls +if response.choices[0].message.tool_calls: + for tc in response.choices[0].message.tool_calls: + print(f"Function: {tc.function.name}") + print(f"Arguments: {tc.function.arguments}") +``` + +See [Example 4: Tool Calling](examples/04-tool-calling.md) for a complete multi-turn tool loop. + +--- + +## Rate Limits & Concurrency + +| Aspect | Local Deployment | Hosted Platform | +|:---|:---|:---| +| **Rate limit** | Limited only by GPU throughput | Check your platform's tier | +| **Concurrency** | Tune via `--max-num-seqs` in vLLM | Per API key quota | +| **Context length** | 256K tokens | 256K tokens | + +For local vLLM deployments, control concurrency with: + +```bash +vllm serve tencent/Hy3 --max-num-seqs 256 ... +``` + +--- + +## Troubleshooting + +### Common Errors + +
+Connection refused: http://127.0.0.1:8000 + +**Cause**: The server is not running or not listening on port 8000. + +**Fix**: +1. Check if the server process is running. +2. Verify the port with `curl http://127.0.0.1:8000/v1/models`. +3. Check server logs for startup errors (common: insufficient GPU memory). +
+ +
+404 Not Found on /v1/chat/completions + +**Cause**: Wrong URL path or the server doesn't serve the chat completions endpoint. + +**Fix**: Ensure the full URL is `http://127.0.0.1:8000/v1/chat/completions` (note the `/v1` prefix). +
+ +
+model "hy3" not found or 400 Bad Request + +**Cause**: The model name doesn't match what the server expects. + +**Fix**: Check `--served-model-name` in your server launch command. It should be `hy3`. Verify with `GET /v1/models`. +
+ +
+CUDA out of memory + +**Cause**: Insufficient GPU memory. + +**Fix**: +1. Reduce `--max-model-len` (e.g., `--max-model-len 32768` for 32K instead of 256K). +2. Use FP8 quantized weights (`tencent/Hy3-FP8`). +3. Increase tensor parallelism (`--tensor-parallel-size`). +4. Enable CPU offloading if supported. +
+ +
+Streaming: chunks arrive slowly or in large bursts + +**Cause**: Server-side batching or MTP speculative decoding settings. + +**Fix**: +1. Reduce `--speculative-config.num_speculative_tokens` for vLLM. +2. Check client-side buffering — use `stream=True` and iterate chunks without accumulation. +
+ +
+Reasoning mode has no effect + +**Cause**: The `reasoning_effort` must be passed via `extra_body.chat_template_kwargs`, not as a top-level parameter. + +**Fix**: +```python +# Correct ✅ +extra_body={"chat_template_kwargs": {"reasoning_effort": "high"}} + +# Wrong ❌ +reasoning_effort="high" +``` +
+ +
+Tool calls not returned + +**Cause**: Server not configured with tool-call parser, or tool schema is invalid. + +**Fix**: +1. Ensure `--tool-call-parser hy_v3` (vLLM) or `--tool-call-parser hunyuan` (SGLang) is set. +2. Ensure `--enable-auto-tool-choice` (vLLM) is set for automatic tool selection. +3. Verify your tool schema follows the OpenAI function-calling format. +
+ +--- + +## Examples + +Each example below includes a standalone Python script and a detailed walkthrough. Start with Example 1 if you're new to the API. + +| # | Example | What You'll Learn | Files | +|:---|:---|:---|:---| +| 1 | **Basic Chat** | Single-turn and multi-turn conversations | [.md](examples/01-basic-chat.md) / [.py](examples/01-basic-chat.py) | +| 2 | **Streaming** | Streaming requests with chunk-by-chunk parsing | [.md](examples/02-streaming.md) / [.py](examples/02-streaming.py) | +| 3 | **Latency Comparison** | Non-streaming vs streaming: TTFT, total time | [.md](examples/03-latency-comparison.md) / [.py](examples/03-latency-comparison.py) | +| 4 | **Tool Calling** | Function definitions, tool loops, multi-turn execution | [.md](examples/04-tool-calling.md) / [.py](examples/04-tool-calling.py) | +| 5 | **Reasoning Mode** | Fast vs. deep thinking — when and how to use | [.md](examples/05-reasoning-mode.md) / [.py](examples/05-reasoning-mode.py) | +| 6 | **Error Handling** | Retries, backoff, timeout, rate-limit handling | [.md](examples/06-error-handling.md) / [.py](examples/06-error-handling.py) |