Ryan Malloy
1abce7f26d
New tools for easier document navigation: - get_document_outline: Structured view of headings with chapter detection - check_style_consistency: Find formatting issues and missing chapters - search_document: Search with context and chapter location All tools tested with 200+ page manuscript. Detects issues like Chapter 3 being styled as "normal" instead of "Heading 1".
395 lines
15 KiB
Python
395 lines
15 KiB
Python
"""Comprehensive testing strategy for mixin-based FastMCP architecture.
|
|
|
|
This test suite demonstrates the recommended patterns for testing FastMCP servers
|
|
that use the mixin composition pattern. It covers:
|
|
|
|
1. Individual mixin functionality testing
|
|
2. Tool registration verification
|
|
3. Integration testing of the composed server
|
|
4. Mocking strategies for file operations
|
|
5. Tool functionality testing (not just registration)
|
|
"""
|
|
|
|
import pytest
|
|
import tempfile
|
|
import os
|
|
from pathlib import Path
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
from typing import Dict, Any
|
|
|
|
from fastmcp import FastMCP
|
|
# FastMCP testing - using direct tool access
|
|
|
|
from mcp_office_tools.mixins import UniversalMixin, WordMixin, ExcelMixin, PowerPointMixin
|
|
from mcp_office_tools.utils import OfficeFileError
|
|
|
|
|
|
class TestMixinArchitecture:
|
|
"""Test the mixin architecture and tool registration."""
|
|
|
|
def test_mixin_initialization(self):
|
|
"""Test that mixins initialize correctly with FastMCP app."""
|
|
app = FastMCP("Test Office Tools")
|
|
|
|
# Test each mixin initializes and registers without errors
|
|
universal = UniversalMixin()
|
|
word = WordMixin()
|
|
excel = ExcelMixin()
|
|
powerpoint = PowerPointMixin()
|
|
|
|
# Register all mixins with the app
|
|
universal.register_all(app)
|
|
word.register_all(app)
|
|
excel.register_all(app)
|
|
powerpoint.register_all(app)
|
|
|
|
# Mixins should be created successfully
|
|
assert universal is not None
|
|
assert word is not None
|
|
assert excel is not None
|
|
assert powerpoint is not None
|
|
|
|
def test_tool_registration_count(self):
|
|
"""Test that all expected tools are registered."""
|
|
app = FastMCP("Test Office Tools")
|
|
|
|
# Count tools before and after each mixin
|
|
initial_tool_count = len(app._tool_manager._tools)
|
|
|
|
universal = UniversalMixin()
|
|
universal.register_all(app)
|
|
universal_tools = len(app._tool_manager._tools) - initial_tool_count
|
|
assert universal_tools == 6 # 6 universal tools
|
|
|
|
word = WordMixin()
|
|
word.register_all(app)
|
|
word_tools = len(app._tool_manager._tools) - initial_tool_count - universal_tools
|
|
assert word_tools == 6 # convert_to_markdown, extract_word_tables, analyze_word_structure, get_document_outline, check_style_consistency, search_document
|
|
|
|
excel = ExcelMixin()
|
|
excel.register_all(app)
|
|
excel_tools = len(app._tool_manager._tools) - initial_tool_count - universal_tools - word_tools
|
|
assert excel_tools == 3 # analyze_excel_data, extract_excel_formulas, create_excel_chart_data
|
|
|
|
powerpoint = PowerPointMixin()
|
|
powerpoint.register_all(app)
|
|
powerpoint_tools = len(app._tool_manager._tools) - initial_tool_count - universal_tools - word_tools - excel_tools
|
|
assert powerpoint_tools == 0 # Placeholder - no tools yet
|
|
|
|
def test_tool_names_registration(self):
|
|
"""Test that specific tool names are registered correctly."""
|
|
app = FastMCP("Test Office Tools")
|
|
|
|
# Register all mixins
|
|
UniversalMixin().register_all(app)
|
|
WordMixin().register_all(app)
|
|
ExcelMixin().register_all(app)
|
|
PowerPointMixin().register_all(app)
|
|
|
|
# Check expected tool names
|
|
tool_names = set(app._tool_manager._tools.keys())
|
|
expected_universal_tools = {
|
|
"extract_text",
|
|
"extract_images",
|
|
"extract_metadata",
|
|
"detect_office_format",
|
|
"analyze_document_health",
|
|
"get_supported_formats"
|
|
}
|
|
expected_word_tools = {"convert_to_markdown", "extract_word_tables", "analyze_word_structure"}
|
|
expected_excel_tools = {"analyze_excel_data", "extract_excel_formulas", "create_excel_chart_data"}
|
|
|
|
assert expected_universal_tools.issubset(tool_names)
|
|
assert expected_word_tools.issubset(tool_names)
|
|
assert expected_excel_tools.issubset(tool_names)
|
|
|
|
|
|
class TestUniversalMixinUnit:
|
|
"""Unit tests for UniversalMixin tools."""
|
|
|
|
@pytest.fixture
|
|
def universal_mixin(self):
|
|
"""Create a UniversalMixin instance for testing."""
|
|
app = FastMCP("Test Universal")
|
|
mixin = UniversalMixin()
|
|
mixin.register_all(app)
|
|
return mixin
|
|
|
|
@pytest.fixture
|
|
def mock_csv_file(self):
|
|
"""Create a mock CSV file for testing."""
|
|
temp_file = tempfile.NamedTemporaryFile(suffix='.csv', delete=False, mode='w')
|
|
temp_file.write("Name,Age,City\nJohn,30,New York\nJane,25,Boston\n")
|
|
temp_file.close()
|
|
yield temp_file.name
|
|
os.unlink(temp_file.name)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_extract_text_error_handling(self, universal_mixin):
|
|
"""Test extract_text error handling for invalid files."""
|
|
with pytest.raises(OfficeFileError):
|
|
await universal_mixin.extract_text("/nonexistent/file.docx")
|
|
|
|
@pytest.mark.asyncio
|
|
@patch('mcp_office_tools.mixins.universal.validate_office_file')
|
|
@patch('mcp_office_tools.mixins.universal.detect_format')
|
|
@patch('mcp_office_tools.mixins.universal.resolve_office_file_path')
|
|
async def test_extract_text_csv_success(self, mock_resolve, mock_detect, mock_validate, universal_mixin, mock_csv_file):
|
|
"""Test successful CSV text extraction with proper mocking."""
|
|
# Setup mocks
|
|
mock_resolve.return_value = mock_csv_file
|
|
mock_validate.return_value = {"is_valid": True, "errors": []}
|
|
mock_detect.return_value = {
|
|
"category": "data",
|
|
"extension": ".csv",
|
|
"format_name": "CSV"
|
|
}
|
|
|
|
# Mock the internal method
|
|
with patch.object(universal_mixin, '_extract_text_by_category') as mock_extract:
|
|
mock_extract.return_value = {
|
|
"text": "Name,Age,City\nJohn,30,New York\nJane,25,Boston",
|
|
"method_used": "pandas",
|
|
"methods_tried": ["pandas"]
|
|
}
|
|
|
|
with patch.object(universal_mixin, '_extract_basic_metadata') as mock_metadata:
|
|
mock_metadata.return_value = {"file_size": 1024}
|
|
|
|
result = await universal_mixin.extract_text(mock_csv_file)
|
|
|
|
assert "text" in result
|
|
assert "metadata" in result
|
|
assert result["metadata"]["extraction_method"] == "pandas"
|
|
assert "John" in result["text"]
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_supported_formats(self, universal_mixin):
|
|
"""Test get_supported_formats returns expected structure."""
|
|
result = await universal_mixin.get_supported_formats()
|
|
|
|
assert isinstance(result, dict)
|
|
assert "supported_extensions" in result
|
|
assert "format_details" in result
|
|
assert "categories" in result
|
|
assert "total_formats" in result
|
|
|
|
# Check that common formats are supported
|
|
extensions = result["supported_extensions"]
|
|
assert ".docx" in extensions
|
|
assert ".xlsx" in extensions
|
|
assert ".pptx" in extensions
|
|
assert ".csv" in extensions
|
|
|
|
|
|
class TestWordMixinUnit:
|
|
"""Unit tests for WordMixin tools."""
|
|
|
|
@pytest.fixture
|
|
def word_mixin(self):
|
|
"""Create a WordMixin instance for testing."""
|
|
app = FastMCP("Test Word")
|
|
mixin = WordMixin()
|
|
mixin.register_all(app)
|
|
return mixin
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_convert_to_markdown_error_handling(self, word_mixin):
|
|
"""Test convert_to_markdown error handling for invalid files."""
|
|
with pytest.raises(OfficeFileError):
|
|
await word_mixin.convert_to_markdown("/nonexistent/file.docx")
|
|
|
|
@pytest.mark.asyncio
|
|
@patch('mcp_office_tools.mixins.word.validate_office_file')
|
|
@patch('mcp_office_tools.mixins.word.detect_format')
|
|
@patch('mcp_office_tools.mixins.word.resolve_office_file_path')
|
|
async def test_convert_to_markdown_non_word_document(self, mock_resolve, mock_detect, mock_validate, word_mixin):
|
|
"""Test that non-Word documents are rejected for markdown conversion."""
|
|
# Setup mocks for a non-Word document
|
|
mock_resolve.return_value = "/test/file.xlsx"
|
|
mock_validate.return_value = {"is_valid": True, "errors": []}
|
|
mock_detect.return_value = {
|
|
"category": "excel",
|
|
"extension": ".xlsx",
|
|
"format_name": "Excel"
|
|
}
|
|
|
|
with pytest.raises(OfficeFileError, match="Markdown conversion currently only supports Word documents"):
|
|
await word_mixin.convert_to_markdown("/test/file.xlsx")
|
|
|
|
|
|
class TestComposedServerIntegration:
|
|
"""Integration tests for the fully composed server."""
|
|
|
|
@pytest.fixture
|
|
def composed_app(self):
|
|
"""Create a fully composed FastMCP app with all mixins."""
|
|
app = FastMCP("MCP Office Tools Test")
|
|
|
|
# Initialize and register all mixins
|
|
UniversalMixin().register_all(app)
|
|
WordMixin().register_all(app)
|
|
ExcelMixin().register_all(app)
|
|
PowerPointMixin().register_all(app)
|
|
|
|
return app
|
|
|
|
def test_all_tools_registered(self, composed_app):
|
|
"""Test that all tools are registered in the composed server."""
|
|
tool_names = set(composed_app._tool_manager._tools.keys())
|
|
|
|
# Expected tools from all mixins
|
|
expected_tools = {
|
|
# Universal tools
|
|
"extract_text",
|
|
"extract_images",
|
|
"extract_metadata",
|
|
"detect_office_format",
|
|
"analyze_document_health",
|
|
"get_supported_formats",
|
|
# Word tools
|
|
"convert_to_markdown",
|
|
"extract_word_tables",
|
|
"analyze_word_structure",
|
|
# Excel tools
|
|
"analyze_excel_data",
|
|
"extract_excel_formulas",
|
|
"create_excel_chart_data"
|
|
}
|
|
|
|
assert expected_tools.issubset(tool_names)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_tool_execution_direct(self, composed_app):
|
|
"""Test tool execution through direct tool access."""
|
|
# Test get_supported_formats through direct access
|
|
get_supported_formats_tool = composed_app._tool_manager._tools["get_supported_formats"]
|
|
result = await get_supported_formats_tool.fn()
|
|
|
|
assert "supported_extensions" in result
|
|
assert "format_details" in result
|
|
|
|
|
|
class TestMockingStrategies:
|
|
"""Demonstrate effective mocking strategies for FastMCP tools."""
|
|
|
|
@pytest.fixture
|
|
def mock_office_file(self):
|
|
"""Create a realistic mock Office file context."""
|
|
return {
|
|
"path": "/test/document.docx",
|
|
"content": "Mock document content",
|
|
"metadata": {
|
|
"title": "Test Document",
|
|
"author": "Test Author",
|
|
"created": "2024-01-01T00:00:00Z"
|
|
}
|
|
}
|
|
|
|
@pytest.mark.asyncio
|
|
@patch('mcp_office_tools.mixins.universal.resolve_office_file_path')
|
|
@patch('mcp_office_tools.mixins.universal.validate_office_file')
|
|
@patch('mcp_office_tools.mixins.universal.detect_format')
|
|
async def test_comprehensive_mocking_pattern(self, mock_detect, mock_validate, mock_resolve, mock_office_file):
|
|
"""Demonstrate comprehensive mocking pattern for tool testing."""
|
|
app = FastMCP("Test App")
|
|
universal = UniversalMixin()
|
|
universal.register_all(app)
|
|
|
|
# Setup comprehensive mocks
|
|
mock_resolve.return_value = mock_office_file["path"]
|
|
mock_validate.return_value = {"is_valid": True, "errors": []}
|
|
mock_detect.return_value = {
|
|
"category": "word",
|
|
"extension": ".docx",
|
|
"format_name": "Word Document"
|
|
}
|
|
|
|
# Mock the internal processing methods
|
|
with patch.object(universal, '_extract_text_by_category') as mock_extract_text:
|
|
mock_extract_text.return_value = {
|
|
"text": mock_office_file["content"],
|
|
"method_used": "python-docx",
|
|
"methods_tried": ["python-docx"]
|
|
}
|
|
|
|
with patch.object(universal, '_extract_basic_metadata') as mock_extract_metadata:
|
|
mock_extract_metadata.return_value = mock_office_file["metadata"]
|
|
|
|
result = await universal.extract_text(mock_office_file["path"])
|
|
|
|
# Verify comprehensive result structure
|
|
assert result["text"] == mock_office_file["content"]
|
|
assert result["metadata"]["extraction_method"] == "python-docx"
|
|
assert result["document_metadata"] == mock_office_file["metadata"]
|
|
|
|
# Verify mocks were called correctly
|
|
mock_resolve.assert_called_once_with(mock_office_file["path"])
|
|
mock_validate.assert_called_once_with(mock_office_file["path"])
|
|
mock_detect.assert_called_once_with(mock_office_file["path"])
|
|
|
|
|
|
class TestFileOperationMocking:
|
|
"""Advanced file operation mocking strategies."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_temporary_file_creation(self):
|
|
"""Test using temporary files for realistic testing."""
|
|
# Create a temporary CSV file
|
|
with tempfile.NamedTemporaryFile(mode='w', suffix='.csv', delete=False) as tmp:
|
|
tmp.write("Name,Value\nTest,123\n")
|
|
tmp_path = tmp.name
|
|
|
|
try:
|
|
# Test with real file
|
|
app = FastMCP("Test App")
|
|
universal = UniversalMixin()
|
|
universal.register_all(app)
|
|
|
|
# Mock only the validation/detection layers
|
|
with patch('mcp_office_tools.utils.validation.validate_office_file') as mock_validate:
|
|
with patch('mcp_office_tools.utils.file_detection.detect_format') as mock_detect:
|
|
mock_validate.return_value = {"is_valid": True, "errors": []}
|
|
mock_detect.return_value = {
|
|
"category": "data",
|
|
"extension": ".csv",
|
|
"format_name": "CSV"
|
|
}
|
|
|
|
# Test would work with real CSV processing
|
|
# (This demonstrates the pattern without running the full pipeline)
|
|
assert os.path.exists(tmp_path)
|
|
|
|
finally:
|
|
os.unlink(tmp_path)
|
|
|
|
|
|
class TestAsyncPatterns:
|
|
"""Test async patterns specific to FastMCP."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_async_tool_execution(self):
|
|
"""Test async tool execution patterns."""
|
|
app = FastMCP("Async Test")
|
|
universal = UniversalMixin()
|
|
universal.register_all(app)
|
|
|
|
# Mock all async dependencies
|
|
with patch('mcp_office_tools.mixins.universal.resolve_office_file_path') as mock_resolve:
|
|
with patch('mcp_office_tools.mixins.universal.validate_office_file') as mock_validate:
|
|
with patch('mcp_office_tools.mixins.universal.detect_format') as mock_detect:
|
|
# Make mocks properly async
|
|
mock_resolve.return_value = "/test.csv"
|
|
mock_validate.return_value = {"is_valid": True, "errors": []}
|
|
mock_detect.return_value = {"category": "data", "extension": ".csv", "format_name": "CSV"}
|
|
|
|
with patch.object(universal, '_extract_text_by_category') as mock_extract:
|
|
mock_extract.return_value = {"text": "test", "method_used": "pandas"}
|
|
|
|
# This should complete quickly
|
|
result = await universal.extract_text("/test.csv")
|
|
assert "text" in result
|
|
|
|
|
|
if __name__ == "__main__":
|
|
pytest.main([__file__, "-v"]) |