7634f9fc32
- Comprehensive test suite (700+ lines) for JS execution in high-level API - Test coverage analysis and validation infrastructure - Enhancement proposal and implementation strategy - Mock HTTP server with realistic JavaScript scenarios - Parallel implementation strategy using expert agents and git worktrees Ready for test-driven implementation of JavaScript enhancements.
173 lines
5.8 KiB
Python
173 lines
5.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Simple test runner to validate our JavaScript API tests without external dependencies."""
|
|
|
|
import sys
|
|
import os
|
|
import asyncio
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
# Add src to path
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src'))
|
|
|
|
# Mock playwright before any imports
|
|
sys.modules['playwright'] = MagicMock()
|
|
sys.modules['playwright.async_api'] = MagicMock()
|
|
|
|
async def test_mock_server():
|
|
"""Test our mock HTTP server functionality."""
|
|
print("🧪 Testing Mock HTTP Server...")
|
|
|
|
# Import our mock server
|
|
from tests.test_javascript_api import MockHTTPServer
|
|
|
|
server = MockHTTPServer()
|
|
base_url = await server.start()
|
|
print(f"✅ Mock server started at {base_url}")
|
|
|
|
# Test the server endpoints
|
|
import aiohttp
|
|
|
|
async with aiohttp.ClientSession() as session:
|
|
# Test index page
|
|
async with session.get(f"{base_url}/") as resp:
|
|
text = await resp.text()
|
|
assert "Test Page" in text
|
|
print("✅ Index page works")
|
|
|
|
# Test dynamic price page
|
|
async with session.get(f"{base_url}/dynamic-price") as resp:
|
|
text = await resp.text()
|
|
assert "Amazing Product" in text
|
|
assert "price-container" in text
|
|
print("✅ Dynamic price page works")
|
|
|
|
# Test infinite scroll page
|
|
async with session.get(f"{base_url}/infinite-scroll") as resp:
|
|
text = await resp.text()
|
|
assert "Infinite Content" in text
|
|
assert "loadMore" in text
|
|
print("✅ Infinite scroll page works")
|
|
|
|
await server.stop()
|
|
print("✅ Mock server stopped cleanly")
|
|
|
|
def test_webcontent_enhancements():
|
|
"""Test WebContent with JavaScript fields."""
|
|
print("🧪 Testing WebContent JavaScript enhancements...")
|
|
|
|
# We need to mock the WebContent class since we can't import it
|
|
# But we can test the concept
|
|
|
|
class MockWebContent:
|
|
def __init__(self, url, title, text, markdown, html, script_result=None, script_error=None):
|
|
self.url = url
|
|
self.title = title
|
|
self.text = text
|
|
self.markdown = markdown
|
|
self.html = html
|
|
self.script_result = script_result
|
|
self.script_error = script_error
|
|
|
|
# Test with script result
|
|
content = MockWebContent(
|
|
url="https://example.com",
|
|
title="Test",
|
|
text="Content",
|
|
markdown="# Test",
|
|
html="<html></html>",
|
|
script_result={"data": "value"}
|
|
)
|
|
|
|
assert content.script_result == {"data": "value"}
|
|
assert content.script_error is None
|
|
print("✅ WebContent with script_result works")
|
|
|
|
# Test with script error
|
|
content_error = MockWebContent(
|
|
url="https://example.com",
|
|
title="Test",
|
|
text="Content",
|
|
markdown="# Test",
|
|
html="<html></html>",
|
|
script_error="ReferenceError: x is not defined"
|
|
)
|
|
|
|
assert content_error.script_result is None
|
|
assert "ReferenceError" in content_error.script_error
|
|
print("✅ WebContent with script_error works")
|
|
|
|
def test_api_signatures():
|
|
"""Test that our proposed API signatures make sense."""
|
|
print("🧪 Testing proposed API signatures...")
|
|
|
|
# Test function signature compatibility
|
|
def mock_get(url, *, wait_for=None, script=None, script_before=None,
|
|
script_after=None, timeout=30, clean=True,
|
|
extract_links=True, extract_metadata=True):
|
|
return {
|
|
'url': url,
|
|
'wait_for': wait_for,
|
|
'script': script,
|
|
'script_before': script_before,
|
|
'script_after': script_after,
|
|
'timeout': timeout
|
|
}
|
|
|
|
# Test basic call
|
|
result = mock_get("https://example.com")
|
|
assert result['url'] == "https://example.com"
|
|
assert result['script'] is None
|
|
print("✅ Basic get() signature works")
|
|
|
|
# Test with script
|
|
result = mock_get("https://example.com", script="return document.title")
|
|
assert result['script'] == "return document.title"
|
|
print("✅ get() with script parameter works")
|
|
|
|
# Test with script_before/after
|
|
result = mock_get("https://example.com",
|
|
script_before="window.scrollTo(0, document.body.scrollHeight)",
|
|
script_after="return window.scrollY")
|
|
assert result['script_before'] is not None
|
|
assert result['script_after'] is not None
|
|
print("✅ get() with script_before/script_after works")
|
|
|
|
async def main():
|
|
"""Run all our validation tests."""
|
|
print("🚀 Starting JavaScript API Enhancement Tests\n")
|
|
|
|
try:
|
|
# Test mock server
|
|
await test_mock_server()
|
|
print()
|
|
|
|
# Test WebContent enhancements
|
|
test_webcontent_enhancements()
|
|
print()
|
|
|
|
# Test API signatures
|
|
test_api_signatures()
|
|
print()
|
|
|
|
print("🎉 All validation tests passed!")
|
|
print("\n📋 Test Summary:")
|
|
print(" ✅ Mock HTTP server with JavaScript scenarios")
|
|
print(" ✅ WebContent enhancements for script results")
|
|
print(" ✅ Proposed API signatures are valid")
|
|
print(" ✅ Error handling patterns work")
|
|
|
|
print("\n🔍 Next Steps:")
|
|
print(" 1. Install Playwright browsers: crawailer setup")
|
|
print(" 2. Implement JavaScript execution in api.py")
|
|
print(" 3. Update Browser.fetch_page() for script execution")
|
|
print(" 4. Add script_result/script_error to WebContent")
|
|
print(" 5. Run full test suite: pytest tests/test_javascript_api.py")
|
|
|
|
except Exception as e:
|
|
print(f"❌ Test failed: {e}")
|
|
return 1
|
|
|
|
return 0
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(asyncio.run(main())) |