Ryan Malloy
bea4a2e5d3
Community-driven testing excellence for the MCP ecosystem MCPTesta is a comprehensive testing framework for FastMCP servers that brings scientific rigor and enterprise-grade capabilities to MCP protocol testing. 🎯 Core Features: • Comprehensive FastMCP server testing with advanced protocol support • Parallel execution with intelligent dependency resolution • Flexible CLI and YAML configuration system • Rich reporting: console, HTML, JSON, and JUnit formats • Advanced MCP protocol features: notifications, cancellation, progress tracking • Production-ready Docker environment with caddy-docker-proxy integration 🧪 Advanced Testing Capabilities: • Multi-transport support (stdio, SSE, WebSocket) • Authentication testing (Bearer tokens, OAuth flows) • Stress testing and performance validation • Memory profiling and leak detection • CI/CD integration with comprehensive reporting 🎨 Professional Assets: • Complete logo package with lab experiment theme • Comprehensive documentation with Diátaxis framework • Community-focused branding and messaging • Multi-platform favicon and social media assets 📚 Documentation: • Getting started tutorials and comprehensive guides • Complete CLI and YAML reference documentation • Architecture explanations and testing strategies • Team collaboration and security compliance guides 🚀 Ready for: • Community contributions and external development • Enterprise deployment and production use • Integration with existing FastMCP workflows • Extension and customization for specific needs Built with modern Python practices using uv, FastMCP, and Starlight documentation. Designed for developers who demand scientific precision in their testing tools. Repository: https://git.supported.systems/mcp/mcptesta Documentation: https://mcptesta.l.supported.systems
504 lines
15 KiB
YAML
504 lines
15 KiB
YAML
# MCPTesta Advanced Configuration Template
|
|
#
|
|
# This template demonstrates advanced MCPTesta capabilities including:
|
|
# - Full MCP protocol feature testing
|
|
# - Complex dependency management
|
|
# - Performance and stress testing
|
|
# - Advanced error handling and recovery
|
|
# - Comprehensive monitoring and profiling
|
|
# - Multi-server coordination
|
|
|
|
# Global configuration with all features enabled
|
|
config:
|
|
parallel_workers: 6
|
|
output_directory: "./advanced_test_results"
|
|
output_format: "all" # Generate all report types
|
|
global_timeout: 300
|
|
max_concurrent_operations: 15
|
|
|
|
# Enable all advanced features
|
|
enable_stress_testing: true
|
|
enable_memory_profiling: true
|
|
enable_performance_profiling: true
|
|
|
|
# Complete feature set
|
|
features:
|
|
test_notifications: true
|
|
test_cancellation: true
|
|
test_progress: true
|
|
test_sampling: true
|
|
test_auth: false # Enable if authentication is configured
|
|
|
|
# Advanced retry policy
|
|
retry_policy:
|
|
max_retries: 3
|
|
backoff_factor: 2.0
|
|
retry_on_errors: ["ConnectionError", "TimeoutError", "ServerError"]
|
|
exponential_backoff: true
|
|
|
|
# Comprehensive notification configuration
|
|
notifications:
|
|
enable_resource_changes: true
|
|
enable_tool_changes: true
|
|
enable_prompt_changes: true
|
|
notification_timeout: 45
|
|
buffer_size: 1000
|
|
|
|
# Multi-server configuration with load balancing
|
|
servers:
|
|
- name: "primary_server"
|
|
command: "${PRIMARY_SERVER_CMD:python -m my_fastmcp_server}"
|
|
transport: "stdio"
|
|
timeout: 30
|
|
enabled: true
|
|
env_vars:
|
|
DEBUG: "${DEBUG:1}"
|
|
LOG_LEVEL: "${LOG_LEVEL:DEBUG}"
|
|
MAX_CONNECTIONS: "100"
|
|
CACHE_SIZE: "1000"
|
|
working_directory: "${SERVER_DIR:.}"
|
|
|
|
- name: "secondary_server"
|
|
command: "${SECONDARY_SERVER_CMD:python -m my_fastmcp_server --instance 2}"
|
|
transport: "stdio"
|
|
timeout: 30
|
|
enabled: true
|
|
env_vars:
|
|
DEBUG: "${DEBUG:1}"
|
|
INSTANCE_ID: "2"
|
|
|
|
- name: "sse_server"
|
|
command: "${SSE_SERVER_URL:http://localhost:8080/sse}"
|
|
transport: "sse"
|
|
timeout: 30
|
|
enabled: false
|
|
headers:
|
|
"User-Agent": "MCPTesta-Advanced/1.0"
|
|
"Accept": "text/event-stream"
|
|
|
|
- name: "websocket_server"
|
|
command: "${WS_SERVER_URL:ws://localhost:8081/mcp}"
|
|
transport: "ws"
|
|
timeout: 30
|
|
enabled: false
|
|
|
|
# Comprehensive test suites covering all scenarios
|
|
test_suites:
|
|
- name: "Infrastructure Validation"
|
|
description: "Comprehensive server and infrastructure testing"
|
|
enabled: true
|
|
tags: ["infrastructure", "setup"]
|
|
parallel: false
|
|
timeout: 120
|
|
|
|
setup:
|
|
validate_connection: true
|
|
discover_capabilities: true
|
|
warm_up_cache: true
|
|
|
|
tests:
|
|
- name: "multi_server_connectivity"
|
|
description: "Test connectivity to all configured servers"
|
|
test_type: "ping"
|
|
target: ""
|
|
timeout: 15
|
|
tags: ["connectivity", "multi"]
|
|
|
|
- name: "capability_matrix"
|
|
description: "Build comprehensive capability matrix"
|
|
test_type: "tool_call"
|
|
target: "list_tools"
|
|
timeout: 20
|
|
tags: ["discovery"]
|
|
depends_on: ["multi_server_connectivity"]
|
|
|
|
- name: "performance_baseline"
|
|
description: "Establish performance baseline"
|
|
test_type: "tool_call"
|
|
target: "benchmark"
|
|
parameters:
|
|
operations: 100
|
|
complexity: "medium"
|
|
timeout: 60
|
|
tags: ["baseline", "performance"]
|
|
depends_on: ["capability_matrix"]
|
|
|
|
- name: "Advanced Protocol Features"
|
|
description: "Test cutting-edge MCP protocol capabilities"
|
|
enabled: true
|
|
tags: ["protocol", "advanced"]
|
|
parallel: false
|
|
timeout: 180
|
|
|
|
tests:
|
|
- name: "notification_orchestration"
|
|
description: "Complex notification subscription and handling"
|
|
test_type: "notification"
|
|
target: "all_changes"
|
|
parameters:
|
|
filter: ["resources", "tools", "prompts"]
|
|
batch_size: 50
|
|
timeout: 45
|
|
tags: ["notifications", "orchestration"]
|
|
|
|
- name: "progress_monitoring_complex"
|
|
description: "Multi-stage progress monitoring"
|
|
test_type: "tool_call"
|
|
target: "multi_stage_process"
|
|
parameters:
|
|
stages: ["initialize", "process", "validate", "finalize"]
|
|
stage_duration: 5
|
|
enable_progress: true
|
|
timeout: 30
|
|
tags: ["progress", "multi_stage"]
|
|
|
|
- name: "cancellation_recovery"
|
|
description: "Test cancellation and recovery mechanisms"
|
|
test_type: "tool_call"
|
|
target: "long_running_task"
|
|
parameters:
|
|
duration: 60
|
|
checkpoints: 5
|
|
enable_cancellation: true
|
|
enable_progress: true
|
|
timeout: 15 # Will trigger cancellation
|
|
tags: ["cancellation", "recovery"]
|
|
|
|
- name: "sampling_strategies"
|
|
description: "Test various sampling strategies"
|
|
test_type: "tool_call"
|
|
target: "echo"
|
|
parameters:
|
|
message: "Sampling test ${ITERATION}"
|
|
enable_sampling: true
|
|
sampling_rate: 0.3
|
|
retry_count: 50
|
|
timeout: 30
|
|
tags: ["sampling", "strategies"]
|
|
|
|
- name: "batch_operations"
|
|
description: "Test batch operation handling"
|
|
test_type: "tool_call"
|
|
target: "batch_process"
|
|
parameters:
|
|
batch_size: 100
|
|
operations:
|
|
- type: "transform"
|
|
data: "${BATCH_DATA}"
|
|
- type: "validate"
|
|
- type: "store"
|
|
enable_progress: true
|
|
timeout: 45
|
|
tags: ["batch", "bulk"]
|
|
|
|
- name: "Complex Tool Interactions"
|
|
description: "Sophisticated tool testing with complex scenarios"
|
|
enabled: true
|
|
tags: ["tools", "complex"]
|
|
parallel: true
|
|
timeout: 240
|
|
|
|
tests:
|
|
- name: "tool_composition"
|
|
description: "Compose multiple tools in sequence"
|
|
test_type: "tool_call"
|
|
target: "pipeline"
|
|
parameters:
|
|
steps:
|
|
- tool: "data_extract"
|
|
params: {source: "database", query: "SELECT * FROM items"}
|
|
- tool: "data_transform"
|
|
params: {format: "json", validate: true}
|
|
- tool: "data_store"
|
|
params: {destination: "cache", ttl: 3600}
|
|
enable_progress: true
|
|
timeout: 60
|
|
tags: ["composition", "pipeline"]
|
|
|
|
- name: "conditional_execution"
|
|
description: "Conditional tool execution based on results"
|
|
test_type: "tool_call"
|
|
target: "conditional_processor"
|
|
parameters:
|
|
conditions:
|
|
- if: "data_valid"
|
|
then: "process_data"
|
|
else: "handle_error"
|
|
data: "${TEST_DATA}"
|
|
timeout: 30
|
|
tags: ["conditional", "logic"]
|
|
|
|
- name: "resource_tool_integration"
|
|
description: "Integration between resources and tools"
|
|
test_type: "tool_call"
|
|
target: "process_resource"
|
|
parameters:
|
|
resource_uri: "file://./test_data.json"
|
|
processor: "json_analyzer"
|
|
output_format: "summary"
|
|
timeout: 25
|
|
tags: ["integration", "resources"]
|
|
|
|
- name: "Advanced Resource Management"
|
|
description: "Complex resource operations and management"
|
|
enabled: true
|
|
tags: ["resources", "advanced"]
|
|
parallel: true
|
|
timeout: 150
|
|
|
|
tests:
|
|
- name: "dynamic_resource_discovery"
|
|
description: "Dynamically discover and test resources"
|
|
test_type: "resource_read"
|
|
target: "discovery://auto"
|
|
parameters:
|
|
patterns: ["config://*", "file://*.json", "data://*"]
|
|
max_depth: 3
|
|
timeout: 30
|
|
tags: ["discovery", "dynamic"]
|
|
|
|
- name: "resource_streaming"
|
|
description: "Test large resource streaming"
|
|
test_type: "resource_read"
|
|
target: "stream://large_dataset"
|
|
parameters:
|
|
chunk_size: 8192
|
|
max_chunks: 1000
|
|
timeout: 45
|
|
tags: ["streaming", "large"]
|
|
|
|
- name: "resource_caching"
|
|
description: "Test resource caching mechanisms"
|
|
test_type: "resource_read"
|
|
target: "cached://expensive_computation"
|
|
parameters:
|
|
cache_ttl: 300
|
|
force_refresh: false
|
|
timeout: 60
|
|
tags: ["caching", "optimization"]
|
|
|
|
- name: "Prompt Engineering Suite"
|
|
description: "Advanced prompt testing and optimization"
|
|
enabled: true
|
|
tags: ["prompts", "engineering"]
|
|
parallel: true
|
|
timeout: 120
|
|
|
|
tests:
|
|
- name: "prompt_chain"
|
|
description: "Chain multiple prompts for complex reasoning"
|
|
test_type: "prompt_get"
|
|
target: "reasoning_chain"
|
|
parameters:
|
|
problem: "Analyze server performance bottlenecks"
|
|
steps: ["data_collection", "analysis", "recommendations"]
|
|
context: "${PERFORMANCE_DATA}"
|
|
timeout: 45
|
|
tags: ["chain", "reasoning"]
|
|
|
|
- name: "adaptive_prompts"
|
|
description: "Test adaptive prompt generation"
|
|
test_type: "prompt_get"
|
|
target: "adaptive"
|
|
parameters:
|
|
user_level: "expert"
|
|
domain: "system_administration"
|
|
task_complexity: "high"
|
|
timeout: 20
|
|
tags: ["adaptive", "personalization"]
|
|
|
|
- name: "prompt_optimization"
|
|
description: "Test prompt performance optimization"
|
|
test_type: "prompt_get"
|
|
target: "optimized"
|
|
parameters:
|
|
optimization_level: "maximum"
|
|
cache_hints: true
|
|
parallel_generation: true
|
|
enable_progress: true
|
|
timeout: 30
|
|
tags: ["optimization", "performance"]
|
|
|
|
- name: "Stress Testing Suite"
|
|
description: "Comprehensive stress testing scenarios"
|
|
enabled: true
|
|
tags: ["stress", "performance"]
|
|
parallel: true
|
|
timeout: 600
|
|
|
|
tests:
|
|
- name: "high_concurrency_test"
|
|
description: "Test high concurrency scenarios"
|
|
test_type: "tool_call"
|
|
target: "echo"
|
|
parameters:
|
|
message: "Concurrency test ${WORKER_ID}"
|
|
retry_count: 500
|
|
timeout: 120
|
|
tags: ["concurrency", "load"]
|
|
|
|
- name: "memory_pressure_test"
|
|
description: "Test under memory pressure"
|
|
test_type: "tool_call"
|
|
target: "memory_intensive"
|
|
parameters:
|
|
data_size: "50MB"
|
|
operations: 100
|
|
timeout: 180
|
|
tags: ["memory", "pressure"]
|
|
|
|
- name: "sustained_load_test"
|
|
description: "Sustained load over extended period"
|
|
test_type: "tool_call"
|
|
target: "sustained_operation"
|
|
parameters:
|
|
duration: 300 # 5 minutes
|
|
rate: 10 # 10 operations per second
|
|
enable_progress: true
|
|
timeout: 360
|
|
tags: ["sustained", "endurance"]
|
|
|
|
- name: "failure_recovery_test"
|
|
description: "Test recovery from various failure modes"
|
|
test_type: "tool_call"
|
|
target: "failure_simulator"
|
|
parameters:
|
|
failure_modes: ["connection_drop", "timeout", "memory_error"]
|
|
recovery_strategy: "exponential_backoff"
|
|
retry_count: 10
|
|
timeout: 120
|
|
tags: ["failure", "recovery"]
|
|
|
|
- name: "Edge Cases and Boundaries"
|
|
description: "Test edge cases and boundary conditions"
|
|
enabled: true
|
|
tags: ["edge_cases", "boundaries"]
|
|
parallel: true
|
|
timeout: 180
|
|
|
|
tests:
|
|
- name: "maximum_payload_test"
|
|
description: "Test maximum payload handling"
|
|
test_type: "tool_call"
|
|
target: "echo"
|
|
parameters:
|
|
large_data: "${GENERATE_LARGE_PAYLOAD:10MB}"
|
|
timeout: 60
|
|
tags: ["payload", "limits"]
|
|
|
|
- name: "unicode_handling"
|
|
description: "Test Unicode and special character handling"
|
|
test_type: "tool_call"
|
|
target: "echo"
|
|
parameters:
|
|
message: "Testing 🧪 Unicode: 测试 العربية русский 日本語"
|
|
encoding: "utf-8"
|
|
timeout: 15
|
|
tags: ["unicode", "encoding"]
|
|
|
|
- name: "nested_data_structures"
|
|
description: "Test deeply nested data structures"
|
|
test_type: "tool_call"
|
|
target: "deep_processor"
|
|
parameters:
|
|
data:
|
|
level1:
|
|
level2:
|
|
level3:
|
|
level4:
|
|
level5:
|
|
deep_value: "Found at level 5"
|
|
array: [1, 2, 3, [4, 5, [6, 7, 8]]]
|
|
timeout: 20
|
|
tags: ["nested", "structures"]
|
|
|
|
- name: "Security and Validation"
|
|
description: "Security testing and input validation"
|
|
enabled: true
|
|
tags: ["security", "validation"]
|
|
parallel: true
|
|
timeout: 120
|
|
|
|
tests:
|
|
- name: "input_sanitization"
|
|
description: "Test input sanitization"
|
|
test_type: "tool_call"
|
|
target: "sanitize_input"
|
|
parameters:
|
|
potentially_malicious: "<script>alert('xss')</script>"
|
|
sql_injection: "'; DROP TABLE users; --"
|
|
expected:
|
|
sanitized: true
|
|
threats_detected: 2
|
|
timeout: 15
|
|
tags: ["sanitization", "xss"]
|
|
|
|
- name: "rate_limiting"
|
|
description: "Test rate limiting mechanisms"
|
|
test_type: "tool_call"
|
|
target: "echo"
|
|
parameters:
|
|
message: "Rate limit test"
|
|
retry_count: 1000 # Should trigger rate limiting
|
|
timeout: 60
|
|
expected_error: "rate limit exceeded"
|
|
tags: ["rate_limiting", "throttling"]
|
|
|
|
- name: "authentication_validation"
|
|
description: "Test authentication mechanisms"
|
|
test_type: "tool_call"
|
|
target: "protected_resource"
|
|
parameters:
|
|
auth_token: "${INVALID_TOKEN:invalid_token_123}"
|
|
expected_error: "authentication failed"
|
|
timeout: 10
|
|
tags: ["auth", "security"]
|
|
|
|
# Comprehensive variables for advanced configuration
|
|
variables:
|
|
PRIMARY_SERVER_CMD: "python -m my_fastmcp_server --advanced"
|
|
SECONDARY_SERVER_CMD: "python -m my_fastmcp_server --replica"
|
|
SSE_SERVER_URL: "http://localhost:8080/sse"
|
|
WS_SERVER_URL: "ws://localhost:8081/mcp"
|
|
|
|
DEBUG: "1"
|
|
LOG_LEVEL: "DEBUG"
|
|
SERVER_DIR: "/path/to/server"
|
|
|
|
TEST_DATA: '{"items": [1,2,3,4,5], "metadata": {"source": "test"}}'
|
|
PERFORMANCE_DATA: '{"cpu": 45, "memory": 2048, "latency": 12.5}'
|
|
BATCH_DATA: '[{"id": 1, "value": "test1"}, {"id": 2, "value": "test2"}]'
|
|
|
|
GENERATE_LARGE_PAYLOAD: "generate_data_mb"
|
|
INVALID_TOKEN: "deliberately_invalid_token_for_testing"
|
|
|
|
ITERATION: "0"
|
|
WORKER_ID: "worker_1"
|
|
|
|
# Advanced Usage Notes:
|
|
#
|
|
# Performance Monitoring:
|
|
# - Enable profiling: enable_memory_profiling, enable_performance_profiling
|
|
# - Use HTML reports for detailed visualization
|
|
# - Monitor resource usage during stress tests
|
|
#
|
|
# Parallel Execution:
|
|
# - Dependency management ensures correct execution order
|
|
# - Use parallel: false for tests that must run sequentially
|
|
# - Balance parallel_workers with system resources
|
|
#
|
|
# Error Handling:
|
|
# - expected_error tests validate error conditions
|
|
# - Retry policies handle transient failures
|
|
# - Cancellation tests verify graceful shutdown
|
|
#
|
|
# Customization:
|
|
# - Variables support environment-specific values
|
|
# - Tags enable selective test execution
|
|
# - Conditional execution based on server capabilities
|
|
#
|
|
# Run Examples:
|
|
# mcptesta yaml advanced_config.yaml --parallel 8 --output ./results
|
|
# mcptesta yaml advanced_config.yaml --tag performance --format html
|
|
# mcptesta yaml advanced_config.yaml --exclude-tag stress --dry-run
|