Ryan Malloy
64c1505a00
Integrate Espressif's QEMU fork for virtual ESP device management: - QemuManager component with 5 MCP tools (start/stop/list/status/flash) - Config auto-detects QEMU binaries from ~/.espressif/tools/ - Supports esp32, esp32s2, esp32s3, esp32c3 chip emulation - Virtual serial over TCP (socket://localhost:PORT) transparent to esptool - Scan integration: QEMU instances appear in esp_scan_ports results - Blank flash images initialized to 0xFF (erased NOR flash state) - 38 unit tests covering lifecycle, port allocation, flash writes
553 lines
21 KiB
Markdown
553 lines
21 KiB
Markdown
# 🚀 ESP-IDF Middleware Integration with Host Applications
|
|
|
|
## Overview
|
|
|
|
ESP-IDF middleware integration creates a comprehensive development environment that combines the power of ESP-IDF's professional framework with MCP's AI-powered workflows. The inclusion of ESP-IDF Host Applications enables rapid prototyping and debugging without physical hardware.
|
|
|
|
## 🎯 ESP-IDF Integration Architecture
|
|
|
|
### Core `idf.py` Middleware
|
|
|
|
```python
|
|
# middleware/idf_middleware.py
|
|
from .logger_interceptor import LoggerInterceptor
|
|
from typing import Dict, List, Optional, Any
|
|
import re
|
|
import asyncio
|
|
|
|
class IDFMiddleware(LoggerInterceptor):
|
|
"""ESP-IDF development framework middleware integration"""
|
|
|
|
def __init__(self, context: Context, operation_id: str):
|
|
super().__init__(context, operation_id)
|
|
self.project_path = None
|
|
self.target_chip = None
|
|
self.build_config = {}
|
|
self.host_mode = False
|
|
|
|
def get_logging_interface(self) -> Dict[str, Callable]:
|
|
"""Map ESP-IDF logging to MCP"""
|
|
return {
|
|
'info': self._handle_info,
|
|
'warning': self._handle_warning,
|
|
'error': self._handle_error,
|
|
'debug': self._handle_debug,
|
|
'verbose': self._handle_verbose,
|
|
'success': self._handle_success
|
|
}
|
|
|
|
def get_interaction_points(self) -> List[str]:
|
|
"""Operations requiring user interaction"""
|
|
return [
|
|
'menuconfig', # Interactive configuration
|
|
'set-target', # Target chip selection
|
|
'erase-flash', # Destructive flash operations
|
|
'erase-otadata', # OTA data erase
|
|
'fullclean', # Complete project clean
|
|
'monitor', # Serial monitoring
|
|
'gdb', # GDB debugging session
|
|
'openocd', # OpenOCD debugging
|
|
'partition-table', # Partition operations
|
|
'efuse-burn' # eFuse programming
|
|
]
|
|
|
|
def get_progress_interface(self) -> Optional[Callable]:
|
|
"""Progress tracking for build operations"""
|
|
return self._parse_idf_progress
|
|
|
|
async def _parse_idf_progress(self, output_line: str) -> Optional[Dict]:
|
|
"""Parse ESP-IDF build output for progress information"""
|
|
|
|
# Build progress patterns
|
|
patterns = {
|
|
'ninja_progress': r'\[(\d+)/(\d+)\]\s+(.+)',
|
|
'cmake_progress': r'(\d+)%\]\s+(.+)',
|
|
'component_build': r'Building (\w+)',
|
|
'linking': r'Linking (.+)',
|
|
'generating': r'Generating (.+)'
|
|
}
|
|
|
|
for pattern_name, pattern in patterns.items():
|
|
match = re.search(pattern, output_line)
|
|
if match:
|
|
return await self._format_progress_update(pattern_name, match, output_line)
|
|
|
|
return None
|
|
|
|
async def _format_progress_update(self, pattern_type: str, match: re.Match, line: str) -> Dict:
|
|
"""Format progress update based on pattern type"""
|
|
|
|
if pattern_type == 'ninja_progress':
|
|
current, total, task = match.groups()
|
|
return {
|
|
'progress': (int(current) / int(total)) * 100,
|
|
'current': int(current),
|
|
'total': int(total),
|
|
'message': f"Building: {task}",
|
|
'stage': 'build'
|
|
}
|
|
elif pattern_type == 'cmake_progress':
|
|
percentage, task = match.groups()
|
|
return {
|
|
'progress': float(percentage),
|
|
'message': f"Configuring: {task}",
|
|
'stage': 'configure'
|
|
}
|
|
|
|
return {'message': line.strip(), 'stage': 'unknown'}
|
|
|
|
async def _handle_menuconfig_interaction(self, stage_message: str) -> bool:
|
|
"""Handle menuconfig interactive sessions"""
|
|
if not self.capabilities['elicitation']:
|
|
return True
|
|
|
|
# Determine what configuration might be needed
|
|
config_suggestions = await self._analyze_project_config()
|
|
|
|
if config_suggestions:
|
|
response = await self.context.request_user_input(
|
|
prompt="🔧 Configuration options detected. Would you like me to suggest optimal settings?",
|
|
input_type="confirmation",
|
|
additional_data={
|
|
"suggestions": config_suggestions,
|
|
"action": "configure_project"
|
|
}
|
|
)
|
|
|
|
if response.get('confirmed', False):
|
|
await self._apply_suggested_config(config_suggestions)
|
|
|
|
return True
|
|
|
|
async def _analyze_project_config(self) -> List[Dict]:
|
|
"""Analyze project and suggest configuration options"""
|
|
suggestions = []
|
|
|
|
# Analyze based on detected components/libraries
|
|
if self._project_uses_wifi():
|
|
suggestions.append({
|
|
"category": "WiFi Configuration",
|
|
"options": [
|
|
"Enable WiFi power saving",
|
|
"Set optimal WiFi buffer sizes",
|
|
"Configure WiFi security settings"
|
|
]
|
|
})
|
|
|
|
if self._project_uses_bluetooth():
|
|
suggestions.append({
|
|
"category": "Bluetooth Configuration",
|
|
"options": [
|
|
"Enable BLE optimization",
|
|
"Configure BT/WiFi coexistence",
|
|
"Set BLE advertising parameters"
|
|
]
|
|
})
|
|
|
|
if self.host_mode:
|
|
suggestions.append({
|
|
"category": "Host Application Mode",
|
|
"options": [
|
|
"Enable Linux simulator",
|
|
"Configure host-specific logging",
|
|
"Enable Valgrind compatibility"
|
|
]
|
|
})
|
|
|
|
return suggestions
|
|
|
|
def _project_uses_wifi(self) -> bool:
|
|
"""Check if project uses WiFi components"""
|
|
# Implementation to scan project files/components
|
|
return True # Placeholder
|
|
|
|
def _project_uses_bluetooth(self) -> bool:
|
|
"""Check if project uses Bluetooth components"""
|
|
# Implementation to scan project files/components
|
|
return False # Placeholder
|
|
```
|
|
|
|
## 🔬 Host Applications Integration
|
|
|
|
### Rapid Prototyping Workflow
|
|
|
|
```python
|
|
# components/idf_host_applications.py
|
|
class IDFHostApplications:
|
|
"""ESP-IDF Host Applications for rapid prototyping"""
|
|
|
|
def __init__(self, app: FastMCP, config):
|
|
self.app = app
|
|
self.config = config
|
|
self.register_tools()
|
|
|
|
def register_tools(self):
|
|
"""Register host application tools"""
|
|
|
|
@self.app.tool("idf_create_host_project")
|
|
async def create_host_project(
|
|
context: Context,
|
|
project_name: str,
|
|
template: str = "basic",
|
|
enable_mocking: bool = True
|
|
) -> str:
|
|
"""
|
|
Create ESP-IDF project optimized for host development
|
|
|
|
Args:
|
|
project_name: Name of the project
|
|
template: Project template (basic, wifi, bluetooth, etc.)
|
|
enable_mocking: Enable CMock for hardware mocking
|
|
|
|
Returns:
|
|
Project creation status and next steps
|
|
"""
|
|
with IDFMiddleware(context, f"create_host_{project_name}") as middleware:
|
|
try:
|
|
# Create project structure
|
|
await self._create_project_structure(project_name, template)
|
|
|
|
# Configure for host target
|
|
await self._configure_host_target(project_name)
|
|
|
|
# Set up mocking if requested
|
|
if enable_mocking:
|
|
await self._setup_cmock_environment(project_name)
|
|
|
|
# Initialize development environment
|
|
await self._initialize_dev_environment(project_name)
|
|
|
|
return f"""✅ Host project '{project_name}' created successfully!
|
|
|
|
🔄 Next steps:
|
|
1. `cd {project_name}`
|
|
2. Configure: `idf.py menuconfig`
|
|
3. Build: `idf.py build`
|
|
4. Run: `idf.py monitor`
|
|
|
|
🏃♂️ Ready for rapid prototyping without hardware!"""
|
|
|
|
except Exception as e:
|
|
return f"❌ Failed to create host project: {e}"
|
|
|
|
@self.app.tool("idf_host_build_and_run")
|
|
async def host_build_and_run(
|
|
context: Context,
|
|
project_path: str = ".",
|
|
debug_mode: bool = False,
|
|
valgrind_check: bool = False
|
|
) -> str:
|
|
"""
|
|
Build and run ESP-IDF application on host
|
|
|
|
Args:
|
|
project_path: Path to ESP-IDF project
|
|
debug_mode: Enable debug symbols and logging
|
|
valgrind_check: Run with Valgrind memory checking
|
|
|
|
Returns:
|
|
Build and execution results
|
|
"""
|
|
with IDFMiddleware(context, "host_build_run") as middleware:
|
|
try:
|
|
# Ensure target is set to linux
|
|
await self._ensure_linux_target(project_path)
|
|
|
|
# Build with host-specific optimizations
|
|
build_result = await self._build_for_host(
|
|
project_path, debug_mode
|
|
)
|
|
|
|
if not build_result['success']:
|
|
return f"❌ Build failed: {build_result['error']}"
|
|
|
|
# Run application
|
|
if valgrind_check:
|
|
return await self._run_with_valgrind(project_path)
|
|
else:
|
|
return await self._run_host_application(project_path, debug_mode)
|
|
|
|
except Exception as e:
|
|
return f"❌ Host execution failed: {e}"
|
|
|
|
@self.app.tool("idf_host_debug_interactive")
|
|
async def host_debug_interactive(
|
|
context: Context,
|
|
project_path: str = ".",
|
|
debugger: str = "gdb"
|
|
) -> str:
|
|
"""
|
|
Start interactive debugging session for host application
|
|
|
|
Args:
|
|
project_path: Path to ESP-IDF project
|
|
debugger: Debugger to use (gdb, lldb, valgrind)
|
|
|
|
Returns:
|
|
Debug session status and instructions
|
|
"""
|
|
with IDFMiddleware(context, "host_debug") as middleware:
|
|
try:
|
|
# Build with debug symbols
|
|
await self._build_debug_version(project_path)
|
|
|
|
# Start debugging session
|
|
if debugger == "gdb":
|
|
return await self._start_gdb_session(project_path)
|
|
elif debugger == "valgrind":
|
|
return await self._start_valgrind_session(project_path)
|
|
elif debugger == "lldb":
|
|
return await self._start_lldb_session(project_path)
|
|
else:
|
|
return f"❌ Unsupported debugger: {debugger}"
|
|
|
|
except Exception as e:
|
|
return f"❌ Debug session failed: {e}"
|
|
|
|
@self.app.tool("idf_host_test_automation")
|
|
async def host_test_automation(
|
|
context: Context,
|
|
project_path: str = ".",
|
|
test_framework: str = "unity",
|
|
coverage: bool = True
|
|
) -> str:
|
|
"""
|
|
Run automated tests on host application
|
|
|
|
Args:
|
|
project_path: Path to ESP-IDF project
|
|
test_framework: Testing framework (unity, googletest)
|
|
coverage: Generate code coverage report
|
|
|
|
Returns:
|
|
Test results and coverage information
|
|
"""
|
|
with IDFMiddleware(context, "host_testing") as middleware:
|
|
try:
|
|
# Configure test environment
|
|
await self._configure_test_environment(project_path, test_framework)
|
|
|
|
# Run tests
|
|
test_results = await self._run_host_tests(project_path)
|
|
|
|
# Generate coverage if requested
|
|
if coverage:
|
|
coverage_results = await self._generate_coverage_report(project_path)
|
|
test_results['coverage'] = coverage_results
|
|
|
|
return await self._format_test_results(test_results)
|
|
|
|
except Exception as e:
|
|
return f"❌ Testing failed: {e}"
|
|
|
|
async def _ensure_linux_target(self, project_path: str) -> None:
|
|
"""Ensure project is configured for Linux target"""
|
|
cmd = ["idf.py", "--preview", "set-target", "linux"]
|
|
# Execute command with middleware integration
|
|
pass
|
|
|
|
async def _build_for_host(self, project_path: str, debug: bool) -> Dict:
|
|
"""Build project for host execution"""
|
|
cmd = ["idf.py", "build"]
|
|
if debug:
|
|
cmd.extend(["-DCMAKE_BUILD_TYPE=Debug"])
|
|
|
|
# Execute with progress tracking
|
|
return await self._execute_with_progress(cmd, project_path)
|
|
|
|
async def _run_with_valgrind(self, project_path: str) -> str:
|
|
"""Run application with Valgrind memory checking"""
|
|
binary_path = f"{project_path}/build/main"
|
|
cmd = [
|
|
"valgrind",
|
|
"--leak-check=full",
|
|
"--show-leak-kinds=all",
|
|
"--track-origins=yes",
|
|
"--verbose",
|
|
binary_path
|
|
]
|
|
|
|
result = await self._execute_command(cmd)
|
|
return self._format_valgrind_output(result)
|
|
|
|
async def _start_gdb_session(self, project_path: str) -> str:
|
|
"""Start GDB debugging session"""
|
|
binary_path = f"{project_path}/build/main"
|
|
|
|
# Create GDB command file for better UX
|
|
gdb_commands = [
|
|
"set confirm off",
|
|
"set height 0",
|
|
"file " + binary_path,
|
|
"break main",
|
|
"run",
|
|
"backtrace"
|
|
]
|
|
|
|
gdb_script = "\n".join(gdb_commands)
|
|
|
|
return f"""🐛 GDB Debug Session Ready
|
|
|
|
Binary: {binary_path}
|
|
|
|
🔧 Quick Commands:
|
|
- `break function_name` - Set breakpoint
|
|
- `run` - Start execution
|
|
- `continue` - Continue execution
|
|
- `step` - Step into function
|
|
- `next` - Step over function
|
|
- `print variable` - Print variable value
|
|
- `backtrace` - Show call stack
|
|
|
|
Starting GDB with automatic setup..."""
|
|
```
|
|
|
|
## 🎨 Natural Language Workflows
|
|
|
|
### Rapid Prototyping Workflow
|
|
|
|
```
|
|
User: "I want to quickly prototype an ESP32 WiFi scanner without hardware"
|
|
|
|
Claude: I'll set up a host-based ESP-IDF project for rapid WiFi scanner prototyping.
|
|
|
|
🔄 Creating host application project...
|
|
→ idf_create_host_project("wifi_scanner", template="wifi", enable_mocking=True)
|
|
✓ Project structure created with WiFi template
|
|
|
|
🔧 Configuring for host development...
|
|
→ idf.py --preview set-target linux
|
|
→ idf.py menuconfig (auto-configured for WiFi scanning)
|
|
✓ Host target configured with WiFi simulation
|
|
|
|
🏗️ Building host application...
|
|
→ idf.py build
|
|
Progress: ████████████████████████████████ 100%
|
|
✓ Built successfully with host-specific optimizations
|
|
|
|
🚀 Running prototype...
|
|
→ ./build/main
|
|
[INFO] WiFi Scanner Prototype Starting...
|
|
[INFO] Simulated WiFi networks detected:
|
|
- "TestNetwork_1" (RSSI: -45 dBm)
|
|
- "TestNetwork_2" (RSSI: -62 dBm)
|
|
✓ Prototype running successfully!
|
|
|
|
💡 Next steps:
|
|
- Modify wifi_scan.c to test different scenarios
|
|
- Use `idf_host_debug_interactive` for debugging
|
|
- Run `idf_host_test_automation` for automated testing
|
|
- Deploy to real hardware with `esp_flash_firmware`
|
|
```
|
|
|
|
### Development → Production Pipeline
|
|
|
|
```
|
|
User: "I've finished prototyping on host, now deploy to real ESP32"
|
|
|
|
Claude: I'll transition your host prototype to real hardware deployment.
|
|
|
|
🔍 Analyzing host application...
|
|
→ Project uses WiFi scanning and JSON output
|
|
→ Detected optimizations for host environment
|
|
→ Found test mocks that need hardware equivalents
|
|
|
|
🔄 Preparing for hardware deployment...
|
|
→ idf.py set-target esp32
|
|
✓ Target changed from linux to ESP32
|
|
|
|
🔧 Hardware-specific configuration...
|
|
→ Disabling host-only features
|
|
→ Enabling ESP32 WiFi drivers
|
|
→ Configuring partition table for OTA
|
|
✓ Hardware configuration optimized
|
|
|
|
🏗️ Building for ESP32...
|
|
→ idf.py build
|
|
Progress: ████████████████████████████████ 100%
|
|
✓ ESP32 firmware built successfully
|
|
|
|
📡 Deploying to hardware...
|
|
→ esp_detect_chip("/dev/ttyUSB0")
|
|
✓ ESP32 detected and ready
|
|
|
|
→ esp_flash_firmware(firmware="build/wifi_scanner.bin")
|
|
Progress: ████████████████████████████████ 100%
|
|
✓ Firmware flashed successfully
|
|
|
|
🔍 Monitoring real hardware...
|
|
→ idf.py monitor
|
|
[INFO] WiFi Scanner starting on ESP32...
|
|
[INFO] Real networks detected:
|
|
- "MyHomeWiFi" (RSSI: -35 dBm)
|
|
- "NeighborNetwork" (RSSI: -78 dBm)
|
|
|
|
✅ Successfully transitioned from host prototype to hardware!
|
|
```
|
|
|
|
### Advanced Debugging Workflow
|
|
|
|
```
|
|
User: "Debug memory issues in my ESP32 application using host tools"
|
|
|
|
Claude: I'll help debug memory issues using ESP-IDF host applications with advanced tooling.
|
|
|
|
🔧 Setting up debug environment...
|
|
→ idf_host_build_and_run(debug_mode=True, valgrind_check=True)
|
|
|
|
🔍 Building with debug symbols...
|
|
→ idf.py build -DCMAKE_BUILD_TYPE=Debug
|
|
✓ Debug build completed
|
|
|
|
🧪 Running Valgrind analysis...
|
|
→ valgrind --leak-check=full --show-leak-kinds=all ./build/main
|
|
|
|
⚠️ Memory issues detected:
|
|
==12345== 48 bytes in 1 blocks are definitely lost
|
|
==12345== at malloc (vg_replace_malloc.c:307)
|
|
==12345== at wifi_scanner_init (wifi_scanner.c:45)
|
|
|
|
🐛 Starting interactive GDB session...
|
|
→ idf_host_debug_interactive(debugger="gdb")
|
|
|
|
(gdb) break wifi_scanner_init
|
|
(gdb) run
|
|
(gdb) print scan_buffer
|
|
$1 = (void *) 0x0
|
|
|
|
💡 Issue found: Uninitialized buffer in wifi_scanner_init()
|
|
|
|
🔧 Suggested fix: Add proper buffer allocation check
|
|
Would you like me to suggest code improvements?
|
|
```
|
|
|
|
## 🚀 Advanced Integration Benefits
|
|
|
|
### Development Acceleration
|
|
|
|
1. **No Hardware Required**: Develop and test without physical ESP32
|
|
2. **Faster Iteration**: Host execution is much faster than chip debugging
|
|
3. **Rich Tooling**: Access to Valgrind, GDB, profilers
|
|
4. **Automated Testing**: Easy CI/CD integration with host tests
|
|
5. **Memory Analysis**: Detect leaks and issues before hardware deployment
|
|
|
|
### Seamless Hardware Transition
|
|
|
|
1. **Same Codebase**: Code runs on both host and hardware
|
|
2. **Configuration Management**: Easy target switching
|
|
3. **Progressive Testing**: Host → Hardware validation pipeline
|
|
4. **Production Deployment**: Smooth transition to final hardware
|
|
|
|
`★ Insight ─────────────────────────────────────`
|
|
**Revolutionary Development Model**: ESP-IDF Host Applications combined with MCP middleware creates a revolutionary development model where AI can rapidly prototype, test, and deploy ESP32 applications without physical hardware limitations.
|
|
|
|
**Debugging Superpower**: The ability to use host-based debugging tools (Valgrind, GDB) while maintaining ESP-IDF compatibility provides unprecedented debugging capabilities for embedded development.
|
|
|
|
**Continuous Integration Paradise**: Host applications enable true CI/CD for embedded projects - automated testing, memory analysis, and performance profiling without hardware in the loop.
|
|
`─────────────────────────────────────────────────`
|
|
|
|
This integration transforms ESP32 development from a hardware-dependent process into a modern, AI-assisted workflow that rivals traditional software development in speed and tooling while maintaining embedded-specific capabilities.
|
|
|
|
<function_calls>
|
|
<invoke name="TodoWrite">
|
|
<parameter name="todos">[{"content": "Document middleware architecture pattern", "status": "completed", "activeForm": "Documenting middleware architecture pattern"}, {"content": "Create reusable middleware design patterns", "status": "completed", "activeForm": "Creating reusable middleware design patterns"}, {"content": "Document broader applications beyond esptool", "status": "completed", "activeForm": "Documenting broader applications beyond esptool"}, {"content": "Design idf.py middleware integration", "status": "completed", "activeForm": "Designing idf.py middleware integration"}] |