Feature/mcp support (#79)

* feat: Implement MCP server integration with navigation and tool execution capabilities

- Added mcp_server.py to define basic math tools and server file retrieval.
- Introduced mcp_servers.json for server configuration.
- Updated poetry.lock to include MCP and related dependencies.
- Enhanced test feature to validate MCP server connection and calculations.
- Implemented MCP Navigation Agent for executing tools and managing resources.
- Developed MCP tools for server communication and tool execution, including connection management and error handling.
- Configured environment variables for MCP integration in BaseConfigManager.

* working mcp client

* bug fix

* deps update

* removed unwanted file

* update gitignore

* feat: Enhance MCP support with detailed documentation and configuration

- Added MCP integration section in README.md, including quick start guide and example usage.
- Introduced MCP Integration details in environment_variables.md, outlining configuration options.
- Created MCP_Usage.md to provide comprehensive guidance on connecting Hercules to MCP servers and executing tools.
- Updated simple_hercules.py to ensure MCP agents are properly set up.
- Refactored mcp_tools.py for improved organization and clarity in MCP server communication.

* mcp working code but not clean

* updated mcp agent workflow

Author: Shahalt1

testzeus_hercules/core/agents/base_nav_agent.py

@@ -15,7 +15,6 @@
 class BaseNavAgent:
     agent_name: str = "base_nav_agent"
     prompt = "Base Agent"
-    prompt = "Base Agent"
 
     def __init__(self, model_config_list, llm_config_params: dict[str, Any], system_prompt: str | None, nav_executor: autogen.UserProxyAgent, agent_name: str = None, agent_prompt: str | None = None):  # type: ignore
         """
@@ -91,6 +90,12 @@ def register_tools(self) -> None:
         # Register the tools that were dynamically discovered
         return None
 
+    def shutdown(self) -> None:
+        """Shutdown the agent."""
+        pass
+
+    # MCP shutdown is managed centrally in the runner; no agent-level scheduling here.
+
     def load_tools(self, additional_tool_dirs: str = os.getenv("ADDITIONAL_TOOL_DIRS", "")) -> None:
         """
         Dynamically load additional tools from directories or specific Python files

testzeus_hercules/core/agents/mcp_nav_agent.py

@@ -1,77 +1,114 @@
 """MCP Navigation Agent for Model Context Protocol server interactions."""
 
+import asyncio
+
 from testzeus_hercules.core.agents.base_nav_agent import BaseNavAgent
+from testzeus_hercules.utils.mcp_helper import MCPHelper, set_mcp_agents
+from testzeus_hercules.utils.logger import logger
 
 class McpNavAgent(BaseNavAgent):
     """MCP Navigation Agent for executing MCP server tools and managing resources."""
 
     agent_name: str = "mcp_nav_agent"
-    prompt = """# MCP Navigation Agent
+    prompt = """### MCP Navigation Agent
+
+You are an MCP (Model Context Protocol) Navigation Agent that assists the Testing Agent by discovering MCP servers, cataloging their exposed tools/resources, and executing the right tool calls to complete the task. Always begin by scanning all configured servers before taking any action.
+
+---
+
+#### 1. Core Functions
+
+- **Discover servers**: Enumerate configured MCP servers and their connection status.
+- **Catalog capabilities**: List tools and resource namespaces for each connected server.
+- **Execute tools**: Call tools with correct parameters and handle responses.
+- **Retrieve resources**: Read resources by URI when required.
+- **Summarize results**: Capture server, tool, arguments, outputs; include timings/status if available.
+- **Stay on task**: Only perform actions required by the "TASK FOR HELPER"; use extra info cautiously.
+
+---
+
+#### 2. Operational Rules
+
+1. **Previous Step Validation**
+   - MANDATORY: Before any new action, explicitly review the previous step and its outcome.
+   - Do not proceed if the prior critical step failed; address it first.
+
+2. **Server Scan First**
+   - Call `get_configured_mcp_servers`.
+   - For each server, call `check_mcp_server_status`.
+   - For connected servers, call `list_mcp_tools` to build a capability map.
+   - If no server is connected, report INCOMPLETE and wait for initialization rather than guessing.
 
-You are an MCP (Model Context Protocol) Navigation Agent responsible for executing tools and accessing resources from MCP servers. Your primary focus is on communicating with MCP servers and executing their available tools.
+3. **Sequential Execution**
+   - Execute only one function/tool at a time and await its result before the next call.
 
-## Core Responsibilities
+4. **Accurate Parameters**
+   - Include all required parameters for every call; if unsure, pass explicit defaults.
+   - Use server names exactly as discovered; do not invent names or tools.
 
-- **Execute MCP Tools**: Call tools available on connected MCP servers
-- **Manage Resources**: Access and retrieve resources from MCP servers  
-- **Server Communication**: Maintain connections and handle server interactions
-- **Result Processing**: Process tool execution results and provide clear summaries
+5. **Tool Selection**
+   - Choose the minimal set of tools to satisfy the task.
+   - If ambiguous, request clarification once with the discovered server/tool information.
 
-## Available Functions
+6. **Validation & Idempotency**
+   - Verify each result is sufficient before moving on.
+   - Avoid redundant or duplicate calls to the same tool with the same arguments.
 
-1. **get_configured_mcp_servers()**: Get list of configured MCP servers and their status
-2. **initialize_mcp_connections()**: Initialize connections to configured MCP servers
-3. **check_mcp_server_status(server_name)**: Check if a specific server is connected
-4. **execute_mcp_tool(server_name, tool_name, arguments)**: Execute a specific tool on an MCP server
-5. **list_mcp_tools(server_name)**: List all available tools from a server
-6. **get_mcp_resource(server_name, resource_uri)**: Retrieve a resource from a server
+7. **Error Handling**
+   - If a call fails due to validation or parameter mismatch, correct and retry at most once.
+   - Provide precise errors; do not retry blindly or loop.
 
-## Core Rules
+8. **Data Usage**
+   - Use provided test data and prior results to build inputs.
+   - Use `data_only` vs `all_fields` appropriately when extracting structured content.
 
-1. **Connection First**: Always check configured servers and initialize connections before using MCP tools
-2. **Use Correct Server Names**: Use server names from configuration
-3. **Sequential Execution**: Execute one function at a time, wait for response
-4. **Proper Parameters**: Always include all required parameters in function calls
-5. **Error Handling**: Handle connection errors and tool failures gracefully
-6. **Result Verification**: Verify each result before proceeding to next operation
+---
 
-## Response Formats
+#### 3. Server and Tool Workflow (Default)
+1) `get_configured_mcp_servers`
+2) `check_mcp_server_status` for each server
+3) `list_mcp_tools` for each connected server
+4) Select and `execute_mcp_tool` in order; `get_mcp_resource` when required
+5) Verify outcomes and summarize
 
-**Success:**
-```
-previous_step: [previous step summary]
-current_output: [DETAILED tool execution result with server response]
+---
+
+#### 4. Response Formats
+
+- **Success:**
+previous_step: [MANDATORY: Explicit review of previous step outcome and its impact on task progression]
+previous_step_status: [MANDATORY: COMPLETED_SUCCESSFULLY or FAILED or INCOMPLETE]
+current_output: [DETAILED EXPANDED COMPLETE LOSS LESS output with server, tool, arguments, and results]
+Task_Completion_Validation: [MANDATORY before terminating: Explicit confirmation that ALL task requirements have been met]
 ##FLAG::SAVE_IN_MEM##
 ##TERMINATE TASK##
-```
 
-**Error:**
-```
-previous_step: [previous step summary]
-current_output: [Error description and attempted resolution]
+- **Information Request:**
+previous_step: [MANDATORY: Explicit review of previous step outcome and its impact on task progression]
+previous_step_status: [MANDATORY: COMPLETED_SUCCESSFULLY or FAILED or INCOMPLETE]
+current_output: [Describe missing info or ambiguity; summarize discovered servers and tools]
 ##TERMINATE TASK##
-```
 
-## Error Handling
+- **Error:**
+previous_step: [MANDATORY: Explicit review of previous step outcome and its impact on task progression]
+previous_step_status: [MANDATORY: COMPLETED_SUCCESSFULLY or FAILED or INCOMPLETE]
+current_output: [Issue description; server/tool involved; parameters; single corrective attempt summary]
+##TERMINATE TASK##
 
-- Check server connectivity before tool execution
-- Provide clear error messages for failed operations
-- Suggest alternative approaches when tools fail
-- Document all server interactions for debugging
+**CRITICAL RULE: DO NOT TERMINATE if previous_step_status is FAILED or INCOMPLETE unless you have addressed and resolved the issue first.**
 
 Available Test Data: $basic_test_information
 """
 
-    def __init__(self, model_config_list, llm_config_params, system_prompt, nav_executor, agent_name=None, agent_prompt=None):
-        """Initialize MCP Navigation Agent and connect to configured MCP servers."""
-        super().__init__(model_config_list, llm_config_params, system_prompt, nav_executor, agent_name, agent_prompt)
-
-        
-
     def register_tools(self) -> None:
         """Register MCP-specific tools and MCP server toolkits for the agent."""
         self.load_tools()
-
-
-                        
-
+        try:
+            self._mcp_init_task = asyncio.create_task(set_mcp_agents(self, self.nav_executor))
+        except Exception as e:
+            logger.error("Failed to schedule MCP initialization: %s", e)
+
+    async def shutdown(self) -> None:
+        """Shutdown the agent."""
+        await MCPHelper.instance().destroy()
+        await super().shutdown()

testzeus_hercules/core/runner.py

@@ -92,7 +92,7 @@ async def initialize(self) -> None:
     async def clean_up(self) -> None:
         """Clean up resources."""
         if self.simple_hercules:
-            await self.simple_hercules.clean_up_plan()
+            await self.simple_hercules.shutdown()
         if self.browser_manager:
             await self.browser_manager.stop_playwright()
 
@@ -204,9 +204,11 @@ async def shutdown(self) -> None:
         Shuts down the components gracefully.
         """
         logger.info("Shutting down...")
+        # Then shut down browser components
         if self.browser_manager:
             await self.browser_manager.stop_playwright()
         PlaywrightManager.close_all_instances()
+        
         self.shutdown_event.set()
 
     async def wait_for_exit(self) -> None:

testzeus_hercules/core/simple_hercules.py

@@ -48,7 +48,6 @@
 )
 from testzeus_hercules.utils.timestamp_helper import get_timestamp_str
 from testzeus_hercules.utils.ui_messagetype import MessageType
-from testzeus_hercules.core.tools.mcp_tools import set_mcp_agents
 
 nest_asyncio.apply()  # type: ignore
 from autogen import oai
@@ -422,6 +421,13 @@ def get_chat_logs_dir(self) -> str | None:
         # Get paths from config with timestamp
         return get_global_conf().get_source_log_folder_path(self.stake_id)
 
+    async def shutdown(self) -> None:
+        """Shutdown the SimpleHercules instance."""
+        await self.clean_up_plan()
+        for agent in self.agents_map.values():
+            if isinstance(agent, autogen.ConversableAgent):
+                await agent.shutdown()
+
     def set_chat_logs_dir(self, chat_logs_dir: str) -> None:
         """
         Set the directory for saving chat logs.
@@ -499,14 +505,6 @@ async def __initialize_agents(self) -> dict[str, autogen.ConversableAgent]:
         agents_map["time_keeper_nav_agent"] = self.__create_time_keeper_nav_agent(agents_map["time_keeper_nav_executor"])
         agents_map["mcp_nav_executor"] = self.__create_mcp_nav_executor_agent()
         agents_map["mcp_nav_agent"] = self.__create_mcp_nav_agent(agents_map["mcp_nav_executor"])
-        # Provide MCP agents to the MCP toolkit layer for native registration
-        try:
-
-            set_mcp_agents(agents_map["mcp_nav_agent"], agents_map["mcp_nav_executor"])
-        except Exception as e:
-            logger.error(
-                f"Failed to set MCP agents for toolkit registration: {e}"
-            )
         agents_map["planner_agent"] = self.__create_planner_agent(agents_map["user"])
         return agents_map