Ryan Malloy
bea4a2e5d3
Community-driven testing excellence for the MCP ecosystem MCPTesta is a comprehensive testing framework for FastMCP servers that brings scientific rigor and enterprise-grade capabilities to MCP protocol testing. 🎯 Core Features: • Comprehensive FastMCP server testing with advanced protocol support • Parallel execution with intelligent dependency resolution • Flexible CLI and YAML configuration system • Rich reporting: console, HTML, JSON, and JUnit formats • Advanced MCP protocol features: notifications, cancellation, progress tracking • Production-ready Docker environment with caddy-docker-proxy integration 🧪 Advanced Testing Capabilities: • Multi-transport support (stdio, SSE, WebSocket) • Authentication testing (Bearer tokens, OAuth flows) • Stress testing and performance validation • Memory profiling and leak detection • CI/CD integration with comprehensive reporting 🎨 Professional Assets: • Complete logo package with lab experiment theme • Comprehensive documentation with Diátaxis framework • Community-focused branding and messaging • Multi-platform favicon and social media assets 📚 Documentation: • Getting started tutorials and comprehensive guides • Complete CLI and YAML reference documentation • Architecture explanations and testing strategies • Team collaboration and security compliance guides 🚀 Ready for: • Community contributions and external development • Enterprise deployment and production use • Integration with existing FastMCP workflows • Extension and customization for specific needs Built with modern Python practices using uv, FastMCP, and Starlight documentation. Designed for developers who demand scientific precision in their testing tools. Repository: https://git.supported.systems/mcp/mcptesta Documentation: https://mcptesta.l.supported.systems
276 lines
7.9 KiB
YAML
276 lines
7.9 KiB
YAML
# MCPTesta Intermediate Configuration Template
|
|
#
|
|
# This template demonstrates intermediate features including:
|
|
# - Multiple test suites with dependencies
|
|
# - Basic MCP protocol features (notifications, progress)
|
|
# - Error handling and validation
|
|
# - HTML reporting and output management
|
|
# - Environment variable usage
|
|
|
|
# Global configuration
|
|
config:
|
|
parallel_workers: 4
|
|
output_directory: "./test_results"
|
|
output_format: "html" # Generate HTML reports
|
|
global_timeout: 180
|
|
max_concurrent_operations: 8
|
|
|
|
# Enable advanced features
|
|
features:
|
|
test_notifications: true
|
|
test_progress: true
|
|
test_cancellation: false # Enable when ready
|
|
test_sampling: false
|
|
|
|
# Retry policy for flaky tests
|
|
retry_policy:
|
|
max_retries: 2
|
|
backoff_factor: 1.5
|
|
retry_on_errors: ["ConnectionError", "TimeoutError"]
|
|
|
|
# Multiple server configurations
|
|
servers:
|
|
- name: "primary_server"
|
|
command: "${SERVER_COMMAND:python -m my_fastmcp_server}"
|
|
transport: "stdio"
|
|
timeout: 30
|
|
enabled: true
|
|
env_vars:
|
|
DEBUG: "${DEBUG_MODE:0}"
|
|
LOG_LEVEL: "${LOG_LEVEL:INFO}"
|
|
|
|
- name: "backup_server"
|
|
command: "${BACKUP_SERVER_COMMAND:python -m my_fastmcp_server --port 8081}"
|
|
transport: "stdio"
|
|
timeout: 30
|
|
enabled: false # Enable when needed
|
|
|
|
# Test suites with progressive complexity
|
|
test_suites:
|
|
- name: "Prerequisites"
|
|
description: "Essential setup and connectivity tests"
|
|
enabled: true
|
|
tags: ["setup", "prerequisite"]
|
|
parallel: false # Run sequentially for setup
|
|
timeout: 60
|
|
|
|
tests:
|
|
- name: "server_startup"
|
|
description: "Verify server starts and responds"
|
|
test_type: "ping"
|
|
target: ""
|
|
timeout: 10
|
|
tags: ["startup"]
|
|
|
|
- name: "capability_discovery"
|
|
description: "Discover all server capabilities"
|
|
test_type: "tool_call"
|
|
target: "list_tools"
|
|
timeout: 15
|
|
tags: ["discovery"]
|
|
depends_on: ["server_startup"]
|
|
|
|
- name: "Core Tool Testing"
|
|
description: "Comprehensive tool testing with validation"
|
|
enabled: true
|
|
tags: ["tools", "core"]
|
|
parallel: true
|
|
timeout: 120
|
|
|
|
setup:
|
|
validate_connection: true
|
|
discover_capabilities: true
|
|
|
|
tests:
|
|
- name: "echo_simple"
|
|
description: "Basic echo functionality"
|
|
test_type: "tool_call"
|
|
target: "echo"
|
|
parameters:
|
|
message: "${TEST_MESSAGE:Hello, World!}"
|
|
expected:
|
|
message: "${TEST_MESSAGE:Hello, World!}"
|
|
timeout: 10
|
|
tags: ["echo", "basic"]
|
|
depends_on: ["capability_discovery"]
|
|
|
|
- name: "echo_with_progress"
|
|
description: "Echo with progress monitoring"
|
|
test_type: "tool_call"
|
|
target: "echo"
|
|
parameters:
|
|
message: "Testing progress reporting"
|
|
simulate_work: true
|
|
enable_progress: true
|
|
timeout: 20
|
|
tags: ["echo", "progress"]
|
|
depends_on: ["echo_simple"]
|
|
|
|
- name: "parameterized_tool"
|
|
description: "Tool with complex parameters"
|
|
test_type: "tool_call"
|
|
target: "process_data" # Replace with actual tool
|
|
parameters:
|
|
data:
|
|
items: [1, 2, 3, 4, 5]
|
|
options:
|
|
format: "json"
|
|
validate: true
|
|
metadata:
|
|
source: "mcptesta"
|
|
timestamp: "2024-01-01T00:00:00Z"
|
|
expected:
|
|
success: true
|
|
processed_count: 5
|
|
timeout: 25
|
|
tags: ["complex", "data"]
|
|
retry_count: 1
|
|
|
|
- name: "Resource Management"
|
|
description: "Test resource reading and management"
|
|
enabled: true
|
|
tags: ["resources"]
|
|
parallel: true
|
|
timeout: 90
|
|
|
|
tests:
|
|
- name: "read_configuration"
|
|
description: "Read server configuration"
|
|
test_type: "resource_read"
|
|
target: "config://server.json"
|
|
timeout: 15
|
|
tags: ["config"]
|
|
expected:
|
|
content_type: "application/json"
|
|
|
|
- name: "read_file_resource"
|
|
description: "Read file system resource"
|
|
test_type: "resource_read"
|
|
target: "file://${CONFIG_FILE:./config.yml}"
|
|
timeout: 15
|
|
tags: ["filesystem"]
|
|
|
|
- name: "resource_with_parameters"
|
|
description: "Parameterized resource reading"
|
|
test_type: "resource_read"
|
|
target: "data://query"
|
|
parameters:
|
|
query: "SELECT * FROM items LIMIT 5"
|
|
format: "json"
|
|
timeout: 20
|
|
tags: ["database", "query"]
|
|
|
|
- name: "Prompt Testing"
|
|
description: "Test prompt generation and templating"
|
|
enabled: true
|
|
tags: ["prompts"]
|
|
parallel: true
|
|
timeout: 60
|
|
|
|
tests:
|
|
- name: "simple_prompt"
|
|
description: "Basic prompt generation"
|
|
test_type: "prompt_get"
|
|
target: "greeting"
|
|
parameters:
|
|
name: "${USER_NAME:MCPTesta User}"
|
|
context: "testing"
|
|
expected:
|
|
messages_count: ">0"
|
|
timeout: 15
|
|
tags: ["greeting"]
|
|
|
|
- name: "template_prompt"
|
|
description: "Complex template with variables"
|
|
test_type: "prompt_get"
|
|
target: "analysis"
|
|
parameters:
|
|
subject: "FastMCP server performance"
|
|
data_points: ["latency", "throughput", "error_rate"]
|
|
analysis_type: "comprehensive"
|
|
timeout: 20
|
|
tags: ["analysis", "template"]
|
|
|
|
- name: "Notification Testing"
|
|
description: "Test notification subscription and handling"
|
|
enabled: true
|
|
tags: ["notifications", "advanced"]
|
|
parallel: false # Sequential for proper notification testing
|
|
timeout: 90
|
|
|
|
tests:
|
|
- name: "subscribe_notifications"
|
|
description: "Subscribe to resource change notifications"
|
|
test_type: "notification"
|
|
target: "resources_list_changed"
|
|
timeout: 30
|
|
tags: ["subscription"]
|
|
|
|
- name: "trigger_notification"
|
|
description: "Trigger a notification event"
|
|
test_type: "tool_call"
|
|
target: "update_resource" # Tool that triggers notifications
|
|
parameters:
|
|
resource_id: "test_resource"
|
|
action: "update"
|
|
timeout: 15
|
|
tags: ["trigger"]
|
|
depends_on: ["subscribe_notifications"]
|
|
|
|
- name: "Error Handling"
|
|
description: "Test error conditions and edge cases"
|
|
enabled: true
|
|
tags: ["errors", "validation"]
|
|
parallel: true
|
|
timeout: 60
|
|
|
|
tests:
|
|
- name: "invalid_tool"
|
|
description: "Test non-existent tool error"
|
|
test_type: "tool_call"
|
|
target: "non_existent_tool"
|
|
expected_error: "Tool not found"
|
|
timeout: 10
|
|
tags: ["invalid"]
|
|
|
|
- name: "malformed_parameters"
|
|
description: "Test parameter validation"
|
|
test_type: "tool_call"
|
|
target: "echo"
|
|
parameters:
|
|
invalid_param: "should_fail"
|
|
expected_error: "Invalid parameters"
|
|
timeout: 10
|
|
tags: ["validation"]
|
|
|
|
- name: "timeout_handling"
|
|
description: "Test timeout behavior"
|
|
test_type: "tool_call"
|
|
target: "slow_operation"
|
|
parameters:
|
|
delay: 20
|
|
timeout: 5 # Will timeout
|
|
expected_error: "timeout"
|
|
tags: ["timeout"]
|
|
|
|
# Variables for customization and environment-specific values
|
|
variables:
|
|
SERVER_COMMAND: "python -m my_fastmcp_server"
|
|
BACKUP_SERVER_COMMAND: "python -m my_fastmcp_server --backup"
|
|
TEST_MESSAGE: "Intermediate testing with MCPTesta"
|
|
USER_NAME: "MCPTesta User"
|
|
CONFIG_FILE: "./server_config.yml"
|
|
DEBUG_MODE: "1"
|
|
LOG_LEVEL: "DEBUG"
|
|
|
|
# Configuration Tips:
|
|
# 1. Use ${VARIABLE:default_value} syntax for flexible configurations
|
|
# 2. Set enabled: false for tests you're not ready to run
|
|
# 3. Use depends_on to create test execution order
|
|
# 4. Tags help organize and filter tests
|
|
# 5. HTML reports provide better visualization: output_format: "html"
|
|
#
|
|
# Run specific test suites:
|
|
# mcptesta yaml config.yaml --tag core
|
|
# mcptesta yaml config.yaml --exclude-tag advanced
|