"""
Comprehensive tests for JavaScript execution API enhancements.
Tests the proposed JavaScript execution features in get(), get_many(),
and discover() functions using a mock HTTP server.
"""
import asyncio
import json
import pytest
from aiohttp import web
from aiohttp.test_utils import TestServer
from unittest.mock import AsyncMock, MagicMock, patch
from typing import Dict, Any, List
# These imports assume the enhanced API is implemented
# For now, we'll test against the proposed interface
from crawailer import Browser, BrowserConfig
from crawailer.content import WebContent
from crawailer.api import get, get_many, discover
class MockHTTPServer:
"""Mock HTTP server that serves test pages with JavaScript."""
def __init__(self):
self.app = web.Application()
self.setup_routes()
self.server = None
self.port = None
def setup_routes(self):
"""Set up test routes with various JavaScript scenarios."""
self.app.router.add_get('/', self.index_page)
self.app.router.add_get('/dynamic-price', self.dynamic_price_page)
self.app.router.add_get('/infinite-scroll', self.infinite_scroll_page)
self.app.router.add_get('/load-more', self.load_more_page)
self.app.router.add_get('/spa-content', self.spa_page)
self.app.router.add_get('/search', self.search_results_page)
self.app.router.add_get('/article/{id}', self.article_page)
self.app.router.add_get('/api/data', self.api_endpoint)
async def start(self):
"""Start the mock server."""
self.server = TestServer(self.app, port=0)
await self.server.start_server()
self.port = self.server.port
return f"http://localhost:{self.port}"
async def stop(self):
"""Stop the mock server."""
if self.server:
await self.server.close()
async def index_page(self, request):
"""Simple index page."""
html = """
Test Page
Test Page
Initial content
"""
return web.Response(text=html, content_type='text/html')
async def dynamic_price_page(self, request):
"""E-commerce page with dynamically loaded price."""
html = """
Product Page
Amazing Product
$99.99
"""
return web.Response(text=html, content_type='text/html')
async def infinite_scroll_page(self, request):
"""Page with infinite scroll functionality."""
html = """
Infinite Scroll
Infinite Content
Item 1
Item 2
Item 3
Loading...
No more content
"""
return web.Response(text=html, content_type='text/html')
async def load_more_page(self, request):
"""Page with 'Load More' button."""
html = """
Load More Content
Articles
Article 1
Preview text...
Full article content here...
Article 2
Preview text...
Full article content here...
"""
return web.Response(text=html, content_type='text/html')
async def spa_page(self, request):
"""Single Page Application with React-like behavior."""
html = """
SPA Demo