7634f9fc32
- Comprehensive test suite (700+ lines) for JS execution in high-level API - Test coverage analysis and validation infrastructure - Enhancement proposal and implementation strategy - Mock HTTP server with realistic JavaScript scenarios - Parallel implementation strategy using expert agents and git worktrees Ready for test-driven implementation of JavaScript enhancements.
101 lines
2.3 KiB
TOML
101 lines
2.3 KiB
TOML
[build-system]
|
|
requires = ["hatchling"]
|
|
build-backend = "hatchling.build"
|
|
|
|
[project]
|
|
name = "crawailer"
|
|
dynamic = ["version"]
|
|
description = "Browser control for robots - delightful web automation and content extraction"
|
|
readme = "README.md"
|
|
license = "MIT"
|
|
requires-python = ">=3.11"
|
|
authors = [
|
|
{name = "rpm & Claude", email = "hello@crawailer.dev"},
|
|
]
|
|
classifiers = [
|
|
"Development Status :: 4 - Beta",
|
|
"Intended Audience :: Developers",
|
|
"License :: OSI Approved :: MIT License",
|
|
"Programming Language :: Python :: 3",
|
|
"Programming Language :: Python :: 3.11",
|
|
"Programming Language :: Python :: 3.12",
|
|
"Topic :: Internet :: WWW/HTTP",
|
|
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
"Topic :: Text Processing :: Markup :: HTML",
|
|
]
|
|
keywords = ["web-automation", "browser-control", "content-extraction", "ai", "crawling", "robots"]
|
|
|
|
dependencies = [
|
|
# Browser automation
|
|
"playwright>=1.40.0",
|
|
# Fast HTML processing (5-10x faster than BeautifulSoup)
|
|
"selectolax>=0.3.17",
|
|
"markdownify>=0.11.6",
|
|
# Content intelligence
|
|
"justext>=3.0.0",
|
|
# Async & HTTP
|
|
"httpx>=0.25.0",
|
|
"anyio>=4.0.0",
|
|
# Storage & utilities
|
|
"msgpack>=1.0.0",
|
|
"pydantic>=2.0.0",
|
|
"rich>=13.0.0",
|
|
# Optional fast libraries
|
|
"xxhash>=3.4.0",
|
|
]
|
|
|
|
[project.optional-dependencies]
|
|
dev = [
|
|
"pytest>=7.0.0",
|
|
"pytest-asyncio>=0.21.0",
|
|
"pytest-httpserver>=1.0.0",
|
|
"aiohttp>=3.9.0",
|
|
"black>=23.0.0",
|
|
"ruff>=0.1.0",
|
|
"mypy>=1.5.0",
|
|
]
|
|
ai = [
|
|
"sentence-transformers>=2.2.0",
|
|
"spacy>=3.7.0",
|
|
]
|
|
mcp = [
|
|
"mcp>=0.5.0",
|
|
]
|
|
all = [
|
|
"crawailer[dev,ai,mcp]",
|
|
]
|
|
|
|
[project.urls]
|
|
Homepage = "https://github.com/rpm/crawailer"
|
|
Repository = "https://github.com/rpm/crawailer"
|
|
Documentation = "https://crawailer.dev"
|
|
Issues = "https://github.com/rpm/crawailer/issues"
|
|
|
|
[project.scripts]
|
|
crawailer = "crawailer.cli:main"
|
|
|
|
[tool.hatch.version]
|
|
path = "src/crawailer/__init__.py"
|
|
|
|
[tool.hatch.build.targets.wheel]
|
|
packages = ["src/crawailer"]
|
|
|
|
[tool.black]
|
|
line-length = 88
|
|
target-version = ['py311']
|
|
|
|
[tool.ruff]
|
|
target-version = "py311"
|
|
line-length = 88
|
|
select = ["E", "F", "I", "N", "UP", "RUF"]
|
|
|
|
[tool.mypy]
|
|
python_version = "3.11"
|
|
strict = true
|
|
warn_return_any = true
|
|
warn_unused_configs = true
|
|
|
|
[tool.pytest.ini_options]
|
|
testpaths = ["tests"]
|
|
asyncio_mode = "auto"
|