"""
Basic tests for Crawailer functionality.
Simple tests to verify the core components work together.
"""
import pytest
import asyncio
from unittest.mock import Mock, AsyncMock
from crawailer.content import WebContent, ContentExtractor
from crawailer.utils import clean_text, detect_content_type, calculate_reading_time
from crawailer.config import BrowserConfig, CrawlConfig
def test_web_content_creation():
"""Test WebContent dataclass creation and properties."""
content = WebContent(
url="https://example.com",
title="Test Article",
markdown="# Test\n\nThis is a test article.",
text="Test\n\nThis is a test article.",
html="
Test
This is a test article.
"
)
assert content.url == "https://example.com"
assert content.title == "Test Article"
assert content.word_count == 6 # "This is a test article."
assert content.reading_time == "1 min read"
assert content.content_hash != ""
def test_clean_text():
"""Test text cleaning utility."""
dirty_text = " Hello world \n\n with spaces "
clean = clean_text(dirty_text)
assert clean == "Hello world with spaces"
# Test aggressive cleaning
dirty_with_boilerplate = "Read our Cookie Policy and Privacy Policy. Hello world."
clean_aggressive = clean_text(dirty_with_boilerplate, aggressive=True)
assert "Cookie Policy" not in clean_aggressive
assert "Hello world" in clean_aggressive
def test_detect_content_type():
"""Test content type detection."""
# Product page
product_html = '