diff --git a/Android/src/app/src/main/java/com/google/ai/edge/gallery/customtasks/agentchat/AgentChatTaskModule.kt b/Android/src/app/src/main/java/com/google/ai/edge/gallery/customtasks/agentchat/AgentChatTaskModule.kt index 6edb24b79..7f951c76e 100644 --- a/Android/src/app/src/main/java/com/google/ai/edge/gallery/customtasks/agentchat/AgentChatTaskModule.kt +++ b/Android/src/app/src/main/java/com/google/ai/edge/gallery/customtasks/agentchat/AgentChatTaskModule.kt @@ -49,7 +49,7 @@ import kotlinx.coroutines.launch private const val TAG = "AGAgentChatTask" // The default system prompt for the agent chat task with both skills and MCP tools. -internal const val DEFAULT_SYSTEM_PROMPT = +const val DEFAULT_SYSTEM_PROMPT = """ You are an AI assistant that helps users by answering questions and completing tasks using skills and tools. For EVERY new task, request, or question, you MUST execute the following steps in exact order. You MUST NOT skip any steps. @@ -94,7 +94,7 @@ internal const val DEFAULT_SYSTEM_PROMPT = private val DEFAULT_SYSTEM_PROMPT_TRIMMED = DEFAULT_SYSTEM_PROMPT.trimIndent() // The default system prompt for the agent chat task with only skills. -internal const val DEFAULT_SYSTEM_PROMPT_SKILLS_ONLY = +const val DEFAULT_SYSTEM_PROMPT_SKILLS_ONLY = """ You are an AI assistant that helps users by answering questions and completes tasks using skills. For EVERY new task or request or question, you MUST execute the following steps in exact order. You MUST NOT skip any steps. @@ -117,7 +117,7 @@ private val DEFAULT_SYSTEM_PROMPT_SKILLS_ONLY_TRIMMED = DEFAULT_SYSTEM_PROMPT_SKILLS_ONLY.trimIndent() class AgentChatTask @Inject constructor() : CustomTask { - private val agentTools = AgentTools() + private val agentTools: AgentTools = AgentToolsImpl() override val task: Task = Task( @@ -242,13 +242,13 @@ fun injectSkillsAndMcpTools( } // Check whether the system prompt is the default one. -internal fun isDefaultSystemPrompt(prompt: String): Boolean { +fun isDefaultSystemPrompt(prompt: String): Boolean { return prompt == DEFAULT_SYSTEM_PROMPT_TRIMMED || prompt == DEFAULT_SYSTEM_PROMPT_SKILLS_ONLY_TRIMMED } // Returns the effective default system prompt depending on whether MCP tools are enabled. -internal fun getEffectiveBaseSystemPrompt(currentPrompt: String, hasMcpTools: Boolean): String { +fun getEffectiveBaseSystemPrompt(currentPrompt: String, hasMcpTools: Boolean): String { return if (isDefaultSystemPrompt(currentPrompt)) { if (hasMcpTools) DEFAULT_SYSTEM_PROMPT_TRIMMED else DEFAULT_SYSTEM_PROMPT_SKILLS_ONLY_TRIMMED } else { diff --git a/Android/src/app/src/main/java/com/google/ai/edge/gallery/customtasks/agentchat/AgentTools.kt b/Android/src/app/src/main/java/com/google/ai/edge/gallery/customtasks/agentchat/AgentTools.kt index 49c5f66dd..40ec4ba7e 100644 --- a/Android/src/app/src/main/java/com/google/ai/edge/gallery/customtasks/agentchat/AgentTools.kt +++ b/Android/src/app/src/main/java/com/google/ai/edge/gallery/customtasks/agentchat/AgentTools.kt @@ -36,6 +36,7 @@ import com.google.ai.edge.gallery.common.RequestPermissionAgentAction import com.google.ai.edge.gallery.common.SkillProgressAgentAction import com.google.ai.edge.gallery.common.convertStringToJsonObject import com.google.ai.edge.gallery.firebaseAnalytics +import com.google.ai.edge.gallery.proto.Skill import com.google.ai.edge.litertlm.Tool import com.google.ai.edge.litertlm.ToolParam import com.google.ai.edge.litertlm.ToolSet @@ -49,21 +50,72 @@ import kotlinx.coroutines.channels.ReceiveChannel import kotlinx.coroutines.runBlocking private const val TAG = "AGAgentTools" +const val SKILL_INSTRUCTIONS_TEMPLATE = "---\nname: %s\ndescription: %s\n---\n\n%s" -open class AgentTools() : ToolSet { - lateinit var context: Context - lateinit var skillManagerViewModel: SkillManagerViewModel - lateinit var mcpManagerViewModel: McpManagerViewModel - lateinit var taskId: String +fun Skill.getSkillContent(): String { + return SKILL_INSTRUCTIONS_TEMPLATE.format(name, description, instructions) +} + +interface AgentTools : ToolSet { + var context: Context + var skillManagerViewModel: SkillManagerViewModel + var mcpManagerViewModel: McpManagerViewModel + var taskId: String + val actionChannel: ReceiveChannel + var resultImageToShow: CallJsSkillResultImage? + var resultWebviewToShow: CallJsSkillResultWebview? + + @Tool(description = "Loads a skill.") + fun loadSkill( + @ToolParam(description = "The name of the skill to load.") skillName: String + ): Map + + @Tool(description = "Run a MCP tool") + fun runMcpTool( + @ToolParam(description = "The name of the tool to run.") toolName: String, + @ToolParam(description = "The parameters passed to tool as input") input: String, + ): Map + + @Tool(description = "Runs JS script") + fun runJs( + @ToolParam(description = "The name of skill") skillName: String, + @ToolParam(description = "The script name to run. Use 'index.html' if not provided by user") + scriptName: String, + @ToolParam( + description = "The data to pass to the script. Use empty string if not provided by user" + ) + data: String, + ): Map + + @Tool( + description = + "Run an Android intent. It is used to interact with the app to perform certain actions." + ) + fun runIntent( + @ToolParam(description = "The intent to run.") intent: String, + @ToolParam( + description = "A JSON string containing the parameter values required for the intent." + ) + parameters: String, + ): Map + + fun sendAgentAction(action: AgentAction) +} + +open class AgentToolsImpl : AgentTools { + override lateinit var context: Context + override lateinit var skillManagerViewModel: SkillManagerViewModel + override lateinit var mcpManagerViewModel: McpManagerViewModel + override lateinit var taskId: String private val _actionChannel = Channel(Channel.UNLIMITED) - val actionChannel: ReceiveChannel = _actionChannel - var resultImageToShow: CallJsSkillResultImage? = null - var resultWebviewToShow: CallJsSkillResultWebview? = null + override val actionChannel: ReceiveChannel = _actionChannel + override var resultImageToShow: CallJsSkillResultImage? = null + override var resultWebviewToShow: CallJsSkillResultWebview? = null /** Loads skill. */ @Tool(description = "Loads a skill.") - fun loadSkill( + override fun loadSkill( @ToolParam(description = "The name of the skill to load.") skillName: String ): Map { return runBlocking(Dispatchers.Default) { @@ -71,7 +123,7 @@ open class AgentTools() : ToolSet { val skill = skills.find { it.name == skillName.trim() } val skillContent = if (skill != null) { - "---\nname: ${skill.name}\ndescription: ${skill.description}\n---\n\n${skill.instructions}" + skill.getSkillContent() } else { "Skill not found" } @@ -100,7 +152,7 @@ open class AgentTools() : ToolSet { } @Tool(description = "Run a MCP tool") - fun runMcpTool( + override fun runMcpTool( @ToolParam(description = "The name of the tool to run.") toolName: String, @ToolParam(description = "The parameters passed to tool as input") input: String, ): Map { @@ -223,7 +275,7 @@ open class AgentTools() : ToolSet { /** Call JS skill */ @Tool(description = "Runs JS script") - fun runJs( + override fun runJs( @ToolParam(description = "The name of skill") skillName: String, @ToolParam(description = "The script name to run. Use 'index.html' if not provided by user") scriptName: String, @@ -359,7 +411,7 @@ open class AgentTools() : ToolSet { description = "Run an Android intent. It is used to interact with the app to perform certain actions." ) - fun runIntent( + override fun runIntent( @ToolParam(description = "The intent to run.") intent: String, @ToolParam( description = "A JSON string containing the parameter values required for the intent." @@ -403,7 +455,7 @@ open class AgentTools() : ToolSet { return mapOf("error" to error, "status" to "failed") } - fun sendAgentAction(action: AgentAction) { + override fun sendAgentAction(action: AgentAction) { runBlocking(Dispatchers.Default) { _actionChannel.send(action) } } diff --git a/Android/src/app/src/main/java/com/google/ai/edge/gallery/customtasks/agentchat/SkillManagerViewModel.kt b/Android/src/app/src/main/java/com/google/ai/edge/gallery/customtasks/agentchat/SkillManagerViewModel.kt index d237c4cf3..4cc134513 100644 --- a/Android/src/app/src/main/java/com/google/ai/edge/gallery/customtasks/agentchat/SkillManagerViewModel.kt +++ b/Android/src/app/src/main/java/com/google/ai/edge/gallery/customtasks/agentchat/SkillManagerViewModel.kt @@ -175,50 +175,7 @@ constructor( Log.d(TAG, "data store built-in skills selection map: $builtInSelectionMap") // 3. Read and parse SKILL.md files from assets/skills directories. - val builtInSkills = mutableListOf() - try { - val skillAssetDirs = context.assets.list("skills") ?: emptyArray() - for (dirName in skillAssetDirs) { - val skillMdPath = "skills/$dirName/SKILL.md" - try { - context.assets.open(skillMdPath).use { inputStream -> - val mdContent = inputStream.bufferedReader().use { it.readText() } - val (skillProto, errors) = - convertSkillMdToProto( - mdContent, - builtIn = true, - // Selection state will be reconciled with DataStore later - selected = true, - importDir = "assets/skills/$dirName", - ) - if (errors.isNotEmpty()) { - Log.w(TAG, "Error parsing asset skill $dirName: ${errors.joinToString(", ")}") - } else { - skillProto?.let { - // Apply the previous selection state if the user explicitly modified it, - // otherwise use the default selection state. - val defaultSelected = it.name !in DEFAULT_DISABLED_SKILLS - val (persistedSelected, userModified) = - builtInSelectionMap[it.name] ?: Pair(defaultSelected, false) - val selectedState = if (userModified) persistedSelected else defaultSelected - builtInSkills.add( - it - .toBuilder() - .setSelected(selectedState) - .setUserModifiedSelection(userModified) - .build() - ) - Log.d(TAG, "Added built-in skill: ${it.name}") - } - } - } - } catch (e: Exception) { - Log.w(TAG, "SKILL.md not found or error reading for asset skill $dirName", e) - } - } - } catch (e: Exception) { - Log.e(TAG, "Error listing assets/skills", e) - } + val builtInSkills = loadBuiltInSkills(context, builtInSelectionMap, DEFAULT_DISABLED_SKILLS) Log.d( TAG, "Final built-in skills:\n${builtInSkills.joinToString(separator = "\n") { "${it.name}(${it.selected})" }}", @@ -783,104 +740,6 @@ constructor( } } - /** - * Converts the content of a skill.md file to a [Skill] proto. - * - * The expected format is: - * ``` - * --- - * name: name-of-the-skill - * description: description of the skill - * metadata: - * key: value - * --- - * - * other instructions text - * ``` - * - * @return A [Pair] containing the parsed [Skill] proto (or null if errors occurred) and a list of - * error messages. - */ - fun convertSkillMdToProto( - mdContent: String, - builtIn: Boolean, - selected: Boolean, - skillUrl: String = "", - importDir: String = "", - ): Pair> { - val parts = mdContent.split("---") - val errors = mutableListOf() - - if (parts.size < 3) { - errors.add("Invalid format: Expected at least two '---' sections.") - return Pair(null, errors) - } - - // Part 1: Header (index 1) - val header = parts[1].trim() - var name: String? = null - var description: String? = null - var requireSecret = false - var requireSecretDescription = "" - var homepage: String? = null - - var startMetadata = false - for (line in header.lines()) { - val trimmedLine = line.trim() - if (trimmedLine == "metadata:") { - startMetadata = true - continue - } - if (!startMetadata) { - when { - trimmedLine.startsWith("name:") -> name = trimmedLine.substringAfter("name:").trim() - trimmedLine.startsWith("description:") -> - description = trimmedLine.substringAfter("description:").trim() - } - } else { - when { - trimmedLine.startsWith("require-secret:") -> - requireSecret = trimmedLine.substringAfter("require-secret:").trim().toBoolean() - trimmedLine.startsWith("require-secret-description:") -> - requireSecretDescription = - trimmedLine.substringAfter("require-secret-description:").trim() - trimmedLine.startsWith("homepage:") -> - homepage = trimmedLine.substringAfter("homepage:").trim() - } - } - } - - if (name.isNullOrEmpty()) { - errors.add("Missing or empty 'name' in the header.") - } - if (description.isNullOrEmpty()) { - errors.add("Missing or empty 'description' in the header.") - } - - // Part 2: Instructions (index 2 onwards) - val instructions = parts.drop(2).joinToString("---").trim() - - if (errors.isNotEmpty()) { - return Pair(null, errors) - } - - val skill = - Skill.newBuilder() - .setName(name!!) - .setDescription(description!!) - .setInstructions(instructions) - .setBuiltIn(builtIn) - .setSelected(selected) - .setSkillUrl(skillUrl) - .setRequireSecret(requireSecret) - .setRequireSecretDescription(requireSecretDescription) - .setHomepage(homepage ?: "") - .setImportDirName(importDir) - .build() - - return Pair(skill, emptyList()) - } - /** Saves or updates a custom skill. */ fun saveSkillEdit( index: Int, @@ -1216,8 +1075,167 @@ constructor( } companion object { - private val DEFAULT_DISABLED_SKILLS = + val DEFAULT_DISABLED_SKILLS = setOf("calculate-hash", "kitchen-adventure", "text-spinner", "send-email") + + /** + * Converts the content of a skill.md file to a [Skill] proto. + * + * The expected format is: + * ``` + * --- + * name: name-of-the-skill + * description: description of the skill + * metadata: + * key: value + * --- + * + * other instructions text + * ``` + * + * @return A [Pair] containing the parsed [Skill] proto (or null if errors occurred) and a list + * of error messages. + */ + fun convertSkillMdToProto( + mdContent: String, + builtIn: Boolean, + selected: Boolean, + skillUrl: String = "", + importDir: String = "", + ): Pair> { + val parts = mdContent.split("---") + val errors = mutableListOf() + + if (parts.size < 3) { + errors.add("Invalid format: Expected at least two '---' sections.") + return Pair(null, errors) + } + + // Part 1: Header (index 1) + val header = parts[1].trim() + var name: String? = null + var description: String? = null + var requireSecret = false + var requireSecretDescription = "" + var homepage: String? = null + + var startMetadata = false + for (line in header.lines()) { + val trimmedLine = line.trim() + if (trimmedLine == "metadata:") { + startMetadata = true + continue + } + if (!startMetadata) { + when { + trimmedLine.startsWith("name:") -> name = trimmedLine.substringAfter("name:").trim() + trimmedLine.startsWith("description:") -> + description = trimmedLine.substringAfter("description:").trim() + } + } else { + when { + trimmedLine.startsWith("require-secret:") -> + requireSecret = trimmedLine.substringAfter("require-secret:").trim().toBoolean() + trimmedLine.startsWith("require-secret-description:") -> + requireSecretDescription = + trimmedLine.substringAfter("require-secret-description:").trim() + trimmedLine.startsWith("homepage:") -> + homepage = trimmedLine.substringAfter("homepage:").trim() + } + } + } + + if (name.isNullOrEmpty()) { + errors.add("Missing or empty 'name' in the header.") + } + if (description.isNullOrEmpty()) { + errors.add("Missing or empty 'description' in the header.") + } + + // Part 2: Instructions (index 2 onwards) + val instructions = parts.drop(2).joinToString("---").trim() + + if (errors.isNotEmpty()) { + return Pair(null, errors) + } + + val skill = + Skill.newBuilder() + .setName(name!!) + .setDescription(description!!) + .setInstructions(instructions) + .setBuiltIn(builtIn) + .setSelected(selected) + .setSkillUrl(skillUrl) + .setRequireSecret(requireSecret) + .setRequireSecretDescription(requireSecretDescription) + .setHomepage(homepage ?: "") + .setImportDirName(importDir) + .build() + + return Pair(skill, emptyList()) + } + + /** + * Reads and parses SKILL.md files from assets/skills directories to load all built-in skills. + * + * @param context The application context. + * @param builtInSelectionMap A map of skill names to their selection state and whether they + * were user-modified. + * @return A list of [Skill] protos representing the built-in skills. + */ + fun loadBuiltInSkills( + context: Context, + builtInSelectionMap: Map> = + emptyMap(), + defaultDisabledSkills: Set = emptySet(), + ): List { + val builtInSkills = mutableListOf() + try { + val skillAssetDirs = context.assets.list("skills") ?: emptyArray() + for (dirName in skillAssetDirs) { + val skillMdPath = "skills/$dirName/SKILL.md" + try { + context.assets.open(skillMdPath).use { inputStream -> + val mdContent = inputStream.bufferedReader().use { it.readText() } + val (skillProto, errors) = + convertSkillMdToProto( + mdContent, + builtIn = true, + // Selection state will be reconciled with DataStore later + selected = true, + importDir = "assets/skills/$dirName", + ) + if (errors.isNotEmpty()) { + Log.w(TAG, "Error parsing asset skill $dirName: ${errors.joinToString(", ")}") + } else { + skillProto?.let { + // Apply the previous selection state if the user explicitly modified it, + // otherwise use the default selection state. + val defaultSelected = it.name !in defaultDisabledSkills + val (persistedSelected, userModified) = + builtInSelectionMap[it.name] ?: Pair(defaultSelected, false) + val selectedState = if (userModified) persistedSelected else defaultSelected + builtInSkills.add( + it + .toBuilder() + .setSelected(selectedState) + .setUserModifiedSelection(userModified) + .build() + ) + Log.d(TAG, "Added built-in skill: ${it.name}") + } + } + } + } catch (e: Exception) { + Log.w(TAG, "SKILL.md not found or error reading for asset skill $dirName", e) + } + } + } catch (e: Exception) { + Log.e(TAG, "Error listing assets/skills", e) + } + return builtInSkills + } } }