d35dcbb494
- Enhanced get() function with script, script_before, script_after parameters - Enhanced get_many() function with script parameter (str or List[str]) - Enhanced discover() function with script and content_script parameters - Updated ContentExtractor to populate script fields from page_data - Maintained 100% backward compatibility - Added comprehensive parameter validation and error handling - Implemented script parameter alias support (script -> script_before) - Added smart script distribution for multi-URL operations - Enabled two-stage JavaScript execution for discovery workflow All API functions now support JavaScript execution while preserving existing functionality. The enhancement provides intuitive, optional JavaScript capabilities that integrate seamlessly with the browser automation layer.
108 lines
2.4 KiB
TOML
108 lines
2.4 KiB
TOML
[build-system]
|
|
requires = ["hatchling"]
|
|
build-backend = "hatchling.build"
|
|
|
|
[project]
|
|
name = "crawailer"
|
|
dynamic = ["version"]
|
|
description = "Browser control for robots - delightful web automation and content extraction"
|
|
readme = "README.md"
|
|
license = "MIT"
|
|
requires-python = ">=3.11"
|
|
authors = [
|
|
{name = "rpm & Claude", email = "hello@crawailer.dev"},
|
|
]
|
|
classifiers = [
|
|
"Development Status :: 4 - Beta",
|
|
"Intended Audience :: Developers",
|
|
"License :: OSI Approved :: MIT License",
|
|
"Programming Language :: Python :: 3",
|
|
"Programming Language :: Python :: 3.11",
|
|
"Programming Language :: Python :: 3.12",
|
|
"Topic :: Internet :: WWW/HTTP",
|
|
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
"Topic :: Text Processing :: Markup :: HTML",
|
|
]
|
|
keywords = ["web-automation", "browser-control", "content-extraction", "ai", "crawling", "robots"]
|
|
|
|
dependencies = [
|
|
# Browser automation
|
|
"playwright>=1.40.0",
|
|
# Fast HTML processing (5-10x faster than BeautifulSoup)
|
|
"selectolax>=0.3.17",
|
|
"markdownify>=0.11.6",
|
|
# Content intelligence
|
|
"justext>=3.0.0",
|
|
# Async & HTTP
|
|
"httpx>=0.25.0",
|
|
"anyio>=4.0.0",
|
|
# Storage & utilities
|
|
"msgpack>=1.0.0",
|
|
"pydantic>=2.0.0",
|
|
"rich>=13.0.0",
|
|
# Optional fast libraries
|
|
"xxhash>=3.4.0",
|
|
]
|
|
|
|
[project.optional-dependencies]
|
|
dev = [
|
|
"pytest>=7.0.0",
|
|
"pytest-asyncio>=0.21.0",
|
|
"pytest-httpserver>=1.0.0",
|
|
"aiohttp>=3.9.0",
|
|
"black>=23.0.0",
|
|
"ruff>=0.1.0",
|
|
"mypy>=1.5.0",
|
|
]
|
|
ai = [
|
|
"sentence-transformers>=2.2.0",
|
|
"spacy>=3.7.0",
|
|
]
|
|
mcp = [
|
|
"mcp>=0.5.0",
|
|
]
|
|
all = [
|
|
"crawailer[dev,ai,mcp]",
|
|
]
|
|
|
|
[project.urls]
|
|
Homepage = "https://github.com/rpm/crawailer"
|
|
Repository = "https://github.com/rpm/crawailer"
|
|
Documentation = "https://crawailer.dev"
|
|
Issues = "https://github.com/rpm/crawailer/issues"
|
|
|
|
[project.scripts]
|
|
crawailer = "crawailer.cli:main"
|
|
|
|
[tool.hatch.version]
|
|
path = "src/crawailer/__init__.py"
|
|
|
|
[tool.hatch.build.targets.wheel]
|
|
packages = ["src/crawailer"]
|
|
|
|
[tool.black]
|
|
line-length = 88
|
|
target-version = ['py311']
|
|
|
|
[tool.ruff]
|
|
target-version = "py311"
|
|
line-length = 88
|
|
select = ["E", "F", "I", "N", "UP", "RUF"]
|
|
|
|
[tool.mypy]
|
|
python_version = "3.11"
|
|
strict = true
|
|
warn_return_any = true
|
|
warn_unused_configs = true
|
|
|
|
[tool.pytest.ini_options]
|
|
testpaths = ["tests"]
|
|
asyncio_mode = "auto"
|
|
|
|
[dependency-groups]
|
|
dev = [
|
|
"aiohttp>=3.12.15",
|
|
"pytest>=8.4.2",
|
|
"pytest-asyncio>=1.2.0",
|
|
]
|