From 6b14ff6da5ec45fbe585ae0beea4303a69b6c892 Mon Sep 17 00:00:00 2001 From: longhao Date: Tue, 27 May 2025 18:02:45 +0800 Subject: [PATCH] feat: add advanced dependency resolution and package download tools - Add DependencyParser for parsing and categorizing package dependencies - Add DependencyResolver for recursive dependency tree analysis - Add PackageDownloader for downloading packages with dependencies - Add resolve_dependencies MCP tool for comprehensive dependency analysis - Add download_package MCP tool for package collection - Support Python version filtering and extra dependencies - Include comprehensive test coverage for new functionality - Add demonstration script for new features - Update README with new capabilities and usage examples Signed-off-by: Hal --- README.md | 21 +- examples/dependency_analysis_demo.py | 157 ++++++++++ pypi_query_mcp/core/dependency_parser.py | 179 +++++++++++ pypi_query_mcp/server.py | 139 +++++++++ pypi_query_mcp/tools/__init__.py | 4 + pypi_query_mcp/tools/dependency_resolver.py | 244 +++++++++++++++ pypi_query_mcp/tools/package_downloader.py | 329 ++++++++++++++++++++ tests/test_dependency_resolver.py | 188 +++++++++++ tests/test_package_downloader.py | 280 +++++++++++++++++ 9 files changed, 1540 insertions(+), 1 deletion(-) create mode 100644 examples/dependency_analysis_demo.py create mode 100644 pypi_query_mcp/core/dependency_parser.py create mode 100644 pypi_query_mcp/tools/dependency_resolver.py create mode 100644 pypi_query_mcp/tools/package_downloader.py create mode 100644 tests/test_dependency_resolver.py create mode 100644 tests/test_package_downloader.py diff --git a/README.md b/README.md index f6717f6..be705ed 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,8 @@ A Model Context Protocol (MCP) server for querying PyPI package information, dep - šŸ“¦ Query PyPI package information (name, version, description, dependencies) - šŸ Python version compatibility checking -- šŸ” Dependency analysis and resolution +- šŸ” **Advanced dependency analysis and recursive resolution** +- šŸ“„ **Package download with dependency collection** - šŸ¢ Private PyPI repository support - āš” Fast async operations with caching - šŸ› ļø Easy integration with MCP clients @@ -185,22 +186,40 @@ export PYPI_PRIVATE_PYPI_PASSWORD="your_password" The server provides the following MCP tools: +### Core Package Information 1. **get_package_info** - Get comprehensive package information 2. **get_package_versions** - List all available versions for a package 3. **get_package_dependencies** - Analyze package dependencies + +### Python Compatibility 4. **check_package_python_compatibility** - Check Python version compatibility 5. **get_package_compatible_python_versions** - Get all compatible Python versions +### Advanced Dependency Analysis +6. **resolve_dependencies** - Recursively resolve all package dependencies with detailed analysis +7. **download_package** - Download package and all dependencies to local directory + ## Usage Examples Once configured in your MCP client (Claude Desktop, Cline, Cursor, Windsurf), you can ask questions like: +### Basic Package Queries - "What are the dependencies of Django 4.2?" - "Is FastAPI compatible with Python 3.9?" - "Show me all versions of requests package" - "What Python versions does numpy support?" - "Get detailed information about the pandas package" +### Advanced Dependency Analysis +- "Please help me analyze the complete dependency tree for PySide2 with Python 3.10" +- "Resolve all dependencies for Django including development dependencies" +- "What are all the transitive dependencies of FastAPI?" + +### Package Download +- "Please help me download PySide2 and all its dependencies for Python 3.10 to my local machine" +- "Download the requests package with all dependencies to ./downloads folder" +- "Collect all packages needed for Django development" + ### Example Conversations **User**: "Check if Django 4.2 is compatible with Python 3.9" diff --git a/examples/dependency_analysis_demo.py b/examples/dependency_analysis_demo.py new file mode 100644 index 0000000..6d20158 --- /dev/null +++ b/examples/dependency_analysis_demo.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +""" +Demonstration of PyPI Query MCP Server dependency analysis and download features. + +This script shows how to use the new dependency resolution and package download +capabilities for analyzing Python packages like PySide2. +""" + +import asyncio +import json +from pathlib import Path + +from pypi_query_mcp.tools.dependency_resolver import resolve_package_dependencies +from pypi_query_mcp.tools.package_downloader import download_package_with_dependencies + + +async def analyze_pyside2_dependencies(): + """Analyze PySide2 dependencies for Python 3.10.""" + print("šŸ” Analyzing PySide2 dependencies for Python 3.10...") + + try: + result = await resolve_package_dependencies( + package_name="PySide2", + python_version="3.10", + include_extras=[], + include_dev=False, + max_depth=3 + ) + + print(f"āœ… Successfully resolved dependencies for {result['package_name']}") + print(f"šŸ“Š Summary:") + summary = result['summary'] + print(f" - Total packages: {summary['total_packages']}") + print(f" - Runtime dependencies: {summary['total_runtime_dependencies']}") + print(f" - Max depth: {summary['max_depth']}") + + print(f"\nšŸ“¦ Package list:") + for i, pkg in enumerate(summary['package_list'][:10], 1): # Show first 10 + print(f" {i}. {pkg}") + + if len(summary['package_list']) > 10: + print(f" ... and {len(summary['package_list']) - 10} more packages") + + return result + + except Exception as e: + print(f"āŒ Error analyzing dependencies: {e}") + return None + + +async def download_pyside2_packages(): + """Download PySide2 and its dependencies.""" + print("\nšŸ“„ Downloading PySide2 and dependencies...") + + download_dir = Path("./pyside2_downloads") + + try: + result = await download_package_with_dependencies( + package_name="PySide2", + download_dir=str(download_dir), + python_version="3.10", + include_extras=[], + include_dev=False, + prefer_wheel=True, + verify_checksums=True, + max_depth=2 # Limit depth for demo + ) + + print(f"āœ… Download completed!") + print(f"šŸ“Š Download Summary:") + summary = result['summary'] + print(f" - Total packages: {summary['total_packages']}") + print(f" - Successful downloads: {summary['successful_downloads']}") + print(f" - Failed downloads: {summary['failed_downloads']}") + print(f" - Total size: {summary['total_downloaded_size']:,} bytes") + print(f" - Success rate: {summary['success_rate']:.1f}%") + print(f" - Download directory: {summary['download_directory']}") + + if result['failed_downloads']: + print(f"\nāš ļø Failed downloads:") + for failure in result['failed_downloads']: + print(f" - {failure['package']}: {failure['error']}") + + return result + + except Exception as e: + print(f"āŒ Error downloading packages: {e}") + return None + + +async def analyze_small_package(): + """Analyze a smaller package for demonstration.""" + print("\nšŸ” Analyzing 'click' package dependencies...") + + try: + result = await resolve_package_dependencies( + package_name="click", + python_version="3.10", + include_extras=[], + include_dev=False, + max_depth=5 + ) + + print(f"āœ… Successfully resolved dependencies for {result['package_name']}") + + # Show detailed dependency tree + print(f"\nšŸŒ³ Dependency Tree:") + dependency_tree = result['dependency_tree'] + + for pkg_name, pkg_info in dependency_tree.items(): + indent = " " * pkg_info['depth'] + print(f"{indent}- {pkg_info['name']} ({pkg_info['version']})") + + runtime_deps = pkg_info['dependencies']['runtime'] + if runtime_deps: + for dep in runtime_deps[:3]: # Show first 3 dependencies + print(f"{indent} ā””ā”€ {dep}") + if len(runtime_deps) > 3: + print(f"{indent} ā””ā”€ ... and {len(runtime_deps) - 3} more") + + return result + + except Exception as e: + print(f"āŒ Error analyzing dependencies: {e}") + return None + + +async def main(): + """Main demonstration function.""" + print("šŸš€ PyPI Query MCP Server - Dependency Analysis Demo") + print("=" * 60) + + # Analyze a small package first + click_result = await analyze_small_package() + + # Analyze PySide2 dependencies + pyside2_result = await analyze_pyside2_dependencies() + + # Optionally download packages (commented out to avoid large downloads in demo) + # download_result = await download_pyside2_packages() + + print("\n" + "=" * 60) + print("āœØ Demo completed!") + + if click_result: + print(f"šŸ“ Click analysis saved to: click_dependencies.json") + with open("click_dependencies.json", "w") as f: + json.dump(click_result, f, indent=2) + + if pyside2_result: + print(f"šŸ“ PySide2 analysis saved to: pyside2_dependencies.json") + with open("pyside2_dependencies.json", "w") as f: + json.dump(pyside2_result, f, indent=2) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/pypi_query_mcp/core/dependency_parser.py b/pypi_query_mcp/core/dependency_parser.py new file mode 100644 index 0000000..0f3b4d6 --- /dev/null +++ b/pypi_query_mcp/core/dependency_parser.py @@ -0,0 +1,179 @@ +"""Dependency parsing utilities for PyPI packages.""" + +import re +from typing import Any, Dict, List, Optional, Set, Tuple +from packaging.requirements import Requirement +from packaging.specifiers import SpecifierSet +from packaging.version import Version +import logging + +logger = logging.getLogger(__name__) + + +class DependencyParser: + """Parser for Python package dependencies.""" + + def __init__(self): + self.parsed_cache: Dict[str, List[Requirement]] = {} + + def parse_requirements(self, requires_dist: List[str]) -> List[Requirement]: + """Parse requirements from requires_dist list. + + Args: + requires_dist: List of requirement strings from PyPI metadata + + Returns: + List of parsed Requirement objects + """ + requirements = [] + + for req_str in requires_dist or []: + if not req_str or not req_str.strip(): + continue + + try: + req = Requirement(req_str) + requirements.append(req) + except Exception as e: + logger.warning(f"Failed to parse requirement '{req_str}': {e}") + continue + + return requirements + + def filter_requirements_by_python_version( + self, + requirements: List[Requirement], + python_version: str + ) -> List[Requirement]: + """Filter requirements based on Python version. + + Args: + requirements: List of Requirement objects + python_version: Target Python version (e.g., "3.10") + + Returns: + Filtered list of requirements applicable to the Python version + """ + filtered = [] + + try: + target_version = Version(python_version) + except Exception as e: + logger.warning(f"Invalid Python version '{python_version}': {e}") + return requirements + + for req in requirements: + if self._is_requirement_applicable(req, target_version): + filtered.append(req) + + return filtered + + def _is_requirement_applicable(self, req: Requirement, python_version: Version) -> bool: + """Check if a requirement is applicable for the given Python version. + + Args: + req: Requirement object + python_version: Target Python version + + Returns: + True if requirement applies to the Python version + """ + if not req.marker: + return True + + # Create environment for marker evaluation + env = { + 'python_version': str(python_version), + 'python_full_version': str(python_version), + 'platform_system': 'Linux', # Default assumption + 'platform_machine': 'x86_64', # Default assumption + 'implementation_name': 'cpython', + 'implementation_version': str(python_version), + } + + try: + return req.marker.evaluate(env) + except Exception as e: + logger.warning(f"Failed to evaluate marker for {req}: {e}") + return True # Include by default if evaluation fails + + def categorize_dependencies( + self, + requirements: List[Requirement] + ) -> Dict[str, List[Requirement]]: + """Categorize dependencies into runtime, development, and optional groups. + + Args: + requirements: List of Requirement objects + + Returns: + Dictionary with categorized dependencies + """ + categories = { + 'runtime': [], + 'development': [], + 'optional': {}, + 'extras': {} + } + + for req in requirements: + if not req.marker: + # No marker means it's a runtime dependency + categories['runtime'].append(req) + continue + + marker_str = str(req.marker) + + # Check for extra dependencies + if 'extra ==' in marker_str: + extra_match = re.search(r'extra\s*==\s*["\']([^"\']+)["\']', marker_str) + if extra_match: + extra_name = extra_match.group(1) + if extra_name not in categories['extras']: + categories['extras'][extra_name] = [] + categories['extras'][extra_name].append(req) + continue + + # Check for development dependencies + if any(keyword in marker_str.lower() for keyword in ['dev', 'test', 'lint', 'doc']): + categories['development'].append(req) + else: + categories['runtime'].append(req) + + return categories + + def extract_package_names(self, requirements: List[Requirement]) -> Set[str]: + """Extract package names from requirements. + + Args: + requirements: List of Requirement objects + + Returns: + Set of package names + """ + return {req.name.lower() for req in requirements} + + def get_version_constraints(self, req: Requirement) -> Dict[str, Any]: + """Get version constraints from a requirement. + + Args: + req: Requirement object + + Returns: + Dictionary with version constraint information + """ + if not req.specifier: + return {'constraints': [], 'allows_any': True} + + constraints = [] + for spec in req.specifier: + constraints.append({ + 'operator': spec.operator, + 'version': str(spec.version) + }) + + return { + 'constraints': constraints, + 'allows_any': len(constraints) == 0, + 'specifier_str': str(req.specifier) + } diff --git a/pypi_query_mcp/server.py b/pypi_query_mcp/server.py index 898e396..793f6e9 100644 --- a/pypi_query_mcp/server.py +++ b/pypi_query_mcp/server.py @@ -9,10 +9,12 @@ from fastmcp import FastMCP from .core.exceptions import InvalidPackageNameError, NetworkError, PackageNotFoundError from .tools import ( check_python_compatibility, + download_package_with_dependencies, get_compatible_python_versions, query_package_dependencies, query_package_info, query_package_versions, + resolve_package_dependencies, ) # Configure logging @@ -268,6 +270,143 @@ async def get_package_compatible_python_versions( } +@mcp.tool() +async def resolve_dependencies( + package_name: str, + python_version: str | None = None, + include_extras: list[str] | None = None, + include_dev: bool = False, + max_depth: int = 5 +) -> dict[str, Any]: + """Resolve all dependencies for a PyPI package recursively. + + This tool performs comprehensive dependency resolution for a Python package, + analyzing the complete dependency tree including transitive dependencies. + + Args: + package_name: The name of the PyPI package to analyze (e.g., 'pyside2', 'django') + python_version: Target Python version for dependency filtering (e.g., '3.10', '3.11') + include_extras: List of extra dependency groups to include (e.g., ['dev', 'test']) + include_dev: Whether to include development dependencies (default: False) + max_depth: Maximum recursion depth for dependency resolution (default: 5) + + Returns: + Dictionary containing comprehensive dependency analysis including: + - Complete dependency tree with all transitive dependencies + - Dependency categorization (runtime, development, extras) + - Package metadata for each dependency + - Summary statistics and analysis + + Raises: + InvalidPackageNameError: If package name is empty or invalid + PackageNotFoundError: If package is not found on PyPI + NetworkError: For network-related errors + """ + try: + logger.info( + f"MCP tool: Resolving dependencies for {package_name} " + f"(Python {python_version}, extras: {include_extras})" + ) + result = await resolve_package_dependencies( + package_name=package_name, + python_version=python_version, + include_extras=include_extras, + include_dev=include_dev, + max_depth=max_depth + ) + logger.info(f"Successfully resolved dependencies for package: {package_name}") + return result + except (InvalidPackageNameError, PackageNotFoundError, NetworkError) as e: + logger.error(f"Error resolving dependencies for {package_name}: {e}") + return { + "error": str(e), + "error_type": type(e).__name__, + "package_name": package_name, + "python_version": python_version, + } + except Exception as e: + logger.error(f"Unexpected error resolving dependencies for {package_name}: {e}") + return { + "error": f"Unexpected error: {e}", + "error_type": "UnexpectedError", + "package_name": package_name, + "python_version": python_version, + } + + +@mcp.tool() +async def download_package( + package_name: str, + download_dir: str = "./downloads", + python_version: str | None = None, + include_extras: list[str] | None = None, + include_dev: bool = False, + prefer_wheel: bool = True, + verify_checksums: bool = True, + max_depth: int = 5 +) -> dict[str, Any]: + """Download a PyPI package and all its dependencies to local directory. + + This tool downloads a Python package and all its dependencies, providing + comprehensive package collection for offline installation or analysis. + + Args: + package_name: The name of the PyPI package to download (e.g., 'pyside2', 'requests') + download_dir: Local directory to download packages to (default: './downloads') + python_version: Target Python version for compatibility (e.g., '3.10', '3.11') + include_extras: List of extra dependency groups to include (e.g., ['dev', 'test']) + include_dev: Whether to include development dependencies (default: False) + prefer_wheel: Whether to prefer wheel files over source distributions (default: True) + verify_checksums: Whether to verify downloaded file checksums (default: True) + max_depth: Maximum dependency resolution depth (default: 5) + + Returns: + Dictionary containing download results including: + - Download statistics and file information + - Dependency resolution results + - File verification results + - Success/failure summary for each package + + Raises: + InvalidPackageNameError: If package name is empty or invalid + PackageNotFoundError: If package is not found on PyPI + NetworkError: For network-related errors + """ + try: + logger.info( + f"MCP tool: Downloading {package_name} and dependencies to {download_dir} " + f"(Python {python_version})" + ) + result = await download_package_with_dependencies( + package_name=package_name, + download_dir=download_dir, + python_version=python_version, + include_extras=include_extras, + include_dev=include_dev, + prefer_wheel=prefer_wheel, + verify_checksums=verify_checksums, + max_depth=max_depth + ) + logger.info(f"Successfully downloaded {package_name} and dependencies") + return result + except (InvalidPackageNameError, PackageNotFoundError, NetworkError) as e: + logger.error(f"Error downloading {package_name}: {e}") + return { + "error": str(e), + "error_type": type(e).__name__, + "package_name": package_name, + "download_dir": download_dir, + } + except Exception as e: + logger.error(f"Unexpected error downloading {package_name}: {e}") + return { + "error": f"Unexpected error: {e}", + "error_type": "UnexpectedError", + "package_name": package_name, + "download_dir": download_dir, + } + + @click.command() @click.option( "--log-level", diff --git a/pypi_query_mcp/tools/__init__.py b/pypi_query_mcp/tools/__init__.py index 9dd64d7..2d39272 100644 --- a/pypi_query_mcp/tools/__init__.py +++ b/pypi_query_mcp/tools/__init__.py @@ -9,6 +9,8 @@ from .compatibility_check import ( get_compatible_python_versions, suggest_python_version_for_packages, ) +from .dependency_resolver import resolve_package_dependencies +from .package_downloader import download_package_with_dependencies from .package_query import ( query_package_dependencies, query_package_info, @@ -22,4 +24,6 @@ __all__ = [ "check_python_compatibility", "get_compatible_python_versions", "suggest_python_version_for_packages", + "resolve_package_dependencies", + "download_package_with_dependencies", ] diff --git a/pypi_query_mcp/tools/dependency_resolver.py b/pypi_query_mcp/tools/dependency_resolver.py new file mode 100644 index 0000000..4f180cf --- /dev/null +++ b/pypi_query_mcp/tools/dependency_resolver.py @@ -0,0 +1,244 @@ +"""Dependency resolution tools for PyPI packages.""" + +import asyncio +import logging +from typing import Any, Dict, List, Optional, Set +from packaging.requirements import Requirement + +from ..core import PyPIClient, PyPIError +from ..core.dependency_parser import DependencyParser +from ..core.exceptions import InvalidPackageNameError, NetworkError, PackageNotFoundError + +logger = logging.getLogger(__name__) + + +class DependencyResolver: + """Resolves package dependencies recursively.""" + + def __init__(self, max_depth: int = 10): + self.max_depth = max_depth + self.parser = DependencyParser() + self.resolved_cache: Dict[str, Dict[str, Any]] = {} + + async def resolve_dependencies( + self, + package_name: str, + python_version: Optional[str] = None, + include_extras: Optional[List[str]] = None, + include_dev: bool = False, + max_depth: Optional[int] = None + ) -> Dict[str, Any]: + """Resolve all dependencies for a package recursively. + + Args: + package_name: Name of the package to resolve + python_version: Target Python version (e.g., "3.10") + include_extras: List of extra dependencies to include + include_dev: Whether to include development dependencies + max_depth: Maximum recursion depth (overrides instance default) + + Returns: + Dictionary containing resolved dependency tree + """ + if not package_name or not package_name.strip(): + raise InvalidPackageNameError(package_name) + + max_depth = max_depth or self.max_depth + include_extras = include_extras or [] + + logger.info(f"Resolving dependencies for {package_name} (Python {python_version})") + + # Track visited packages to avoid circular dependencies + visited: Set[str] = set() + dependency_tree = {} + + try: + await self._resolve_recursive( + package_name=package_name, + python_version=python_version, + include_extras=include_extras, + include_dev=include_dev, + visited=visited, + dependency_tree=dependency_tree, + current_depth=0, + max_depth=max_depth + ) + + # Check if main package was resolved + normalized_name = package_name.lower().replace("_", "-") + if normalized_name not in dependency_tree: + raise PackageNotFoundError(f"Package '{package_name}' not found on PyPI") + + # Generate summary + summary = self._generate_dependency_summary(dependency_tree) + + return { + "package_name": package_name, + "python_version": python_version, + "include_extras": include_extras, + "include_dev": include_dev, + "dependency_tree": dependency_tree, + "summary": summary + } + + except PyPIError: + raise + except Exception as e: + logger.error(f"Unexpected error resolving dependencies for {package_name}: {e}") + raise NetworkError(f"Failed to resolve dependencies: {e}", e) from e + + async def _resolve_recursive( + self, + package_name: str, + python_version: Optional[str], + include_extras: List[str], + include_dev: bool, + visited: Set[str], + dependency_tree: Dict[str, Any], + current_depth: int, + max_depth: int + ) -> None: + """Recursively resolve dependencies.""" + + # Normalize package name + normalized_name = package_name.lower().replace("_", "-") + + # Check if already visited or max depth reached + if normalized_name in visited or current_depth >= max_depth: + return + + visited.add(normalized_name) + + try: + # Get package information + async with PyPIClient() as client: + package_data = await client.get_package_info(package_name) + + info = package_data.get("info", {}) + requires_dist = info.get("requires_dist", []) or [] + + # Parse requirements + requirements = self.parser.parse_requirements(requires_dist) + + # Filter by Python version if specified + if python_version: + requirements = self.parser.filter_requirements_by_python_version( + requirements, python_version + ) + + # Categorize dependencies + categorized = self.parser.categorize_dependencies(requirements) + + # Build dependency info for this package + package_info = { + "name": info.get("name", package_name), + "version": info.get("version", "unknown"), + "requires_python": info.get("requires_python", ""), + "dependencies": { + "runtime": [str(req) for req in categorized["runtime"]], + "development": [str(req) for req in categorized["development"]] if include_dev else [], + "extras": {} + }, + "depth": current_depth, + "children": {} + } + + # Add requested extras + for extra in include_extras: + if extra in categorized["extras"]: + package_info["dependencies"]["extras"][extra] = [ + str(req) for req in categorized["extras"][extra] + ] + + dependency_tree[normalized_name] = package_info + + # Collect all dependencies to resolve + deps_to_resolve = [] + deps_to_resolve.extend(categorized["runtime"]) + + if include_dev: + deps_to_resolve.extend(categorized["development"]) + + for extra in include_extras: + if extra in categorized["extras"]: + deps_to_resolve.extend(categorized["extras"][extra]) + + # Resolve child dependencies + for dep_req in deps_to_resolve: + dep_name = dep_req.name + if dep_name.lower() not in visited: + await self._resolve_recursive( + package_name=dep_name, + python_version=python_version, + include_extras=[], # Don't propagate extras to children + include_dev=False, # Don't propagate dev deps to children + visited=visited, + dependency_tree=dependency_tree, + current_depth=current_depth + 1, + max_depth=max_depth + ) + + # Add to children if resolved + if dep_name.lower() in dependency_tree: + package_info["children"][dep_name.lower()] = dependency_tree[dep_name.lower()] + + except PackageNotFoundError: + logger.warning(f"Package {package_name} not found, skipping") + except Exception as e: + logger.error(f"Error resolving {package_name}: {e}") + # Continue with other dependencies + + def _generate_dependency_summary(self, dependency_tree: Dict[str, Any]) -> Dict[str, Any]: + """Generate summary statistics for the dependency tree.""" + + total_packages = len(dependency_tree) + total_runtime_deps = 0 + total_dev_deps = 0 + total_extra_deps = 0 + max_depth = 0 + + for package_info in dependency_tree.values(): + total_runtime_deps += len(package_info["dependencies"]["runtime"]) + total_dev_deps += len(package_info["dependencies"]["development"]) + + for extra_deps in package_info["dependencies"]["extras"].values(): + total_extra_deps += len(extra_deps) + + max_depth = max(max_depth, package_info["depth"]) + + return { + "total_packages": total_packages, + "total_runtime_dependencies": total_runtime_deps, + "total_development_dependencies": total_dev_deps, + "total_extra_dependencies": total_extra_deps, + "max_depth": max_depth, + "package_list": list(dependency_tree.keys()) + } + + +async def resolve_package_dependencies( + package_name: str, + python_version: Optional[str] = None, + include_extras: Optional[List[str]] = None, + include_dev: bool = False, + max_depth: int = 5 +) -> Dict[str, Any]: + """Resolve package dependencies with comprehensive analysis. + + Args: + package_name: Name of the package to resolve + python_version: Target Python version (e.g., "3.10") + include_extras: List of extra dependencies to include + include_dev: Whether to include development dependencies + max_depth: Maximum recursion depth + + Returns: + Comprehensive dependency resolution results + """ + resolver = DependencyResolver(max_depth=max_depth) + return await resolver.resolve_dependencies( + package_name=package_name, + python_version=python_version, + include_extras=include_extras, + include_dev=include_dev + ) diff --git a/pypi_query_mcp/tools/package_downloader.py b/pypi_query_mcp/tools/package_downloader.py new file mode 100644 index 0000000..970c71d --- /dev/null +++ b/pypi_query_mcp/tools/package_downloader.py @@ -0,0 +1,329 @@ +"""Package download tools for PyPI packages.""" + +import asyncio +import hashlib +import logging +import os +from pathlib import Path +from typing import Any, Dict, List, Optional, Set +from urllib.parse import urlparse + +import httpx + +from ..core import PyPIClient, PyPIError +from ..core.exceptions import InvalidPackageNameError, NetworkError, PackageNotFoundError +from .dependency_resolver import DependencyResolver + +logger = logging.getLogger(__name__) + + +class PackageDownloader: + """Downloads PyPI packages and their dependencies.""" + + def __init__(self, download_dir: str = "./downloads"): + self.download_dir = Path(download_dir) + self.download_dir.mkdir(parents=True, exist_ok=True) + self.resolver = DependencyResolver() + + async def download_package_with_dependencies( + self, + package_name: str, + python_version: Optional[str] = None, + include_extras: Optional[List[str]] = None, + include_dev: bool = False, + prefer_wheel: bool = True, + verify_checksums: bool = True, + max_depth: int = 5 + ) -> Dict[str, Any]: + """Download a package and all its dependencies. + + Args: + package_name: Name of the package to download + python_version: Target Python version (e.g., "3.10") + include_extras: List of extra dependencies to include + include_dev: Whether to include development dependencies + prefer_wheel: Whether to prefer wheel files over source distributions + verify_checksums: Whether to verify file checksums + max_depth: Maximum dependency resolution depth + + Returns: + Dictionary containing download results and statistics + """ + if not package_name or not package_name.strip(): + raise InvalidPackageNameError(package_name) + + logger.info(f"Starting download of {package_name} and dependencies") + + try: + # First resolve all dependencies + resolution_result = await self.resolver.resolve_dependencies( + package_name=package_name, + python_version=python_version, + include_extras=include_extras, + include_dev=include_dev, + max_depth=max_depth + ) + + dependency_tree = resolution_result["dependency_tree"] + + # Download all packages + download_results = {} + failed_downloads = [] + + for pkg_name, pkg_info in dependency_tree.items(): + try: + result = await self._download_single_package( + package_name=pkg_info["name"], + version=pkg_info["version"], + python_version=python_version, + prefer_wheel=prefer_wheel, + verify_checksums=verify_checksums + ) + download_results[pkg_name] = result + + except Exception as e: + logger.error(f"Failed to download {pkg_name}: {e}") + failed_downloads.append({ + "package": pkg_name, + "error": str(e) + }) + + # Generate summary + summary = self._generate_download_summary(download_results, failed_downloads) + + return { + "package_name": package_name, + "python_version": python_version, + "download_directory": str(self.download_dir), + "resolution_result": resolution_result, + "download_results": download_results, + "failed_downloads": failed_downloads, + "summary": summary + } + + except PyPIError: + raise + except Exception as e: + logger.error(f"Unexpected error downloading {package_name}: {e}") + raise NetworkError(f"Failed to download package: {e}", e) from e + + async def _download_single_package( + self, + package_name: str, + version: Optional[str] = None, + python_version: Optional[str] = None, + prefer_wheel: bool = True, + verify_checksums: bool = True + ) -> Dict[str, Any]: + """Download a single package.""" + + logger.info(f"Downloading {package_name} version {version or 'latest'}") + + async with PyPIClient() as client: + package_data = await client.get_package_info(package_name) + + info = package_data.get("info", {}) + releases = package_data.get("releases", {}) + + # Determine version to download + target_version = version or info.get("version") + if not target_version or target_version not in releases: + raise PackageNotFoundError(f"Version {target_version} not found for {package_name}") + + # Get release files + release_files = releases[target_version] + if not release_files: + raise PackageNotFoundError(f"No files found for {package_name} {target_version}") + + # Select best file to download + selected_file = self._select_best_file( + release_files, python_version, prefer_wheel + ) + + if not selected_file: + raise PackageNotFoundError(f"No suitable file found for {package_name} {target_version}") + + # Download the file + download_result = await self._download_file( + selected_file, verify_checksums + ) + + return { + "package_name": package_name, + "version": target_version, + "file_info": selected_file, + "download_result": download_result + } + + def _select_best_file( + self, + release_files: List[Dict[str, Any]], + python_version: Optional[str] = None, + prefer_wheel: bool = True + ) -> Optional[Dict[str, Any]]: + """Select the best file to download from available release files.""" + + # Separate wheels and source distributions + wheels = [f for f in release_files if f.get("packagetype") == "bdist_wheel"] + sdists = [f for f in release_files if f.get("packagetype") == "sdist"] + + # If prefer wheel and wheels available + if prefer_wheel and wheels: + # Try to find compatible wheel + if python_version: + compatible_wheels = self._filter_compatible_wheels(wheels, python_version) + if compatible_wheels: + return compatible_wheels[0] + + # Return any wheel if no specific version or no compatible found + return wheels[0] + + # Fall back to source distribution + if sdists: + return sdists[0] + + # Last resort: any file + return release_files[0] if release_files else None + + def _filter_compatible_wheels( + self, + wheels: List[Dict[str, Any]], + python_version: str + ) -> List[Dict[str, Any]]: + """Filter wheels compatible with the specified Python version.""" + + # Simple compatibility check based on filename + # This is a basic implementation - could be enhanced with proper wheel tag parsing + compatible = [] + + major_minor = ".".join(python_version.split(".")[:2]) + major_minor_nodot = major_minor.replace(".", "") + + for wheel in wheels: + filename = wheel.get("filename", "") + + # Check for Python version in filename + if (f"py{major_minor_nodot}" in filename or + f"cp{major_minor_nodot}" in filename or + "py3" in filename or + "py2.py3" in filename): + compatible.append(wheel) + + return compatible + + async def _download_file( + self, + file_info: Dict[str, Any], + verify_checksums: bool = True + ) -> Dict[str, Any]: + """Download a single file.""" + + url = file_info.get("url") + filename = file_info.get("filename") + expected_md5 = file_info.get("md5_digest") + expected_size = file_info.get("size") + + if not url or not filename: + raise ValueError("Invalid file info: missing URL or filename") + + # Create package-specific directory + file_path = self.download_dir / filename + + logger.info(f"Downloading {filename} from {url}") + + async with httpx.AsyncClient() as client: + async with client.stream("GET", url) as response: + response.raise_for_status() + + # Download with progress tracking + downloaded_size = 0 + md5_hash = hashlib.md5() + + with open(file_path, "wb") as f: + async for chunk in response.aiter_bytes(chunk_size=8192): + f.write(chunk) + downloaded_size += len(chunk) + if verify_checksums: + md5_hash.update(chunk) + + # Verify download + verification_result = {} + if verify_checksums and expected_md5: + actual_md5 = md5_hash.hexdigest() + verification_result["md5_match"] = actual_md5 == expected_md5 + verification_result["expected_md5"] = expected_md5 + verification_result["actual_md5"] = actual_md5 + + if expected_size: + verification_result["size_match"] = downloaded_size == expected_size + verification_result["expected_size"] = expected_size + verification_result["actual_size"] = downloaded_size + + return { + "filename": filename, + "file_path": str(file_path), + "downloaded_size": downloaded_size, + "verification": verification_result, + "success": True + } + + def _generate_download_summary( + self, + download_results: Dict[str, Any], + failed_downloads: List[Dict[str, Any]] + ) -> Dict[str, Any]: + """Generate download summary statistics.""" + + successful_downloads = len(download_results) + failed_count = len(failed_downloads) + total_size = sum( + result["download_result"]["downloaded_size"] + for result in download_results.values() + ) + + return { + "total_packages": successful_downloads + failed_count, + "successful_downloads": successful_downloads, + "failed_downloads": failed_count, + "total_downloaded_size": total_size, + "download_directory": str(self.download_dir), + "success_rate": successful_downloads / (successful_downloads + failed_count) * 100 + if (successful_downloads + failed_count) > 0 else 0 + } + + +async def download_package_with_dependencies( + package_name: str, + download_dir: str = "./downloads", + python_version: Optional[str] = None, + include_extras: Optional[List[str]] = None, + include_dev: bool = False, + prefer_wheel: bool = True, + verify_checksums: bool = True, + max_depth: int = 5 +) -> Dict[str, Any]: + """Download a package and its dependencies to local directory. + + Args: + package_name: Name of the package to download + download_dir: Directory to download packages to + python_version: Target Python version (e.g., "3.10") + include_extras: List of extra dependencies to include + include_dev: Whether to include development dependencies + prefer_wheel: Whether to prefer wheel files over source distributions + verify_checksums: Whether to verify file checksums + max_depth: Maximum dependency resolution depth + + Returns: + Comprehensive download results + """ + downloader = PackageDownloader(download_dir) + return await downloader.download_package_with_dependencies( + package_name=package_name, + python_version=python_version, + include_extras=include_extras, + include_dev=include_dev, + prefer_wheel=prefer_wheel, + verify_checksums=verify_checksums, + max_depth=max_depth + ) diff --git a/tests/test_dependency_resolver.py b/tests/test_dependency_resolver.py new file mode 100644 index 0000000..6ac47f3 --- /dev/null +++ b/tests/test_dependency_resolver.py @@ -0,0 +1,188 @@ +"""Tests for dependency resolver functionality.""" + +import pytest +from unittest.mock import AsyncMock, patch + +from pypi_query_mcp.tools.dependency_resolver import DependencyResolver, resolve_package_dependencies +from pypi_query_mcp.core.exceptions import InvalidPackageNameError, PackageNotFoundError + + +class TestDependencyResolver: + """Test cases for DependencyResolver class.""" + + @pytest.fixture + def resolver(self): + """Create a DependencyResolver instance for testing.""" + return DependencyResolver(max_depth=3) + + @pytest.mark.asyncio + async def test_resolve_dependencies_invalid_package_name(self, resolver): + """Test that invalid package names raise appropriate errors.""" + with pytest.raises(InvalidPackageNameError): + await resolver.resolve_dependencies("") + + with pytest.raises(InvalidPackageNameError): + await resolver.resolve_dependencies(" ") + + @pytest.mark.asyncio + async def test_resolve_dependencies_basic(self, resolver): + """Test basic dependency resolution.""" + mock_package_data = { + "info": { + "name": "test-package", + "version": "1.0.0", + "requires_python": ">=3.8", + "requires_dist": [ + "requests>=2.25.0", + "click>=8.0.0" + ] + } + } + + with patch('pypi_query_mcp.core.PyPIClient') as mock_client_class: + mock_client = AsyncMock() + mock_client_class.return_value.__aenter__.return_value = mock_client + mock_client.get_package_info.return_value = mock_package_data + + result = await resolver.resolve_dependencies("test-package") + + assert result["package_name"] == "test-package" + assert "dependency_tree" in result + assert "summary" in result + + @pytest.mark.asyncio + async def test_resolve_dependencies_with_python_version(self, resolver): + """Test dependency resolution with Python version filtering.""" + mock_package_data = { + "info": { + "name": "test-package", + "version": "1.0.0", + "requires_python": ">=3.8", + "requires_dist": [ + "requests>=2.25.0", + "typing-extensions>=4.0.0; python_version<'3.10'" + ] + } + } + + with patch('pypi_query_mcp.core.PyPIClient') as mock_client_class: + mock_client = AsyncMock() + mock_client_class.return_value.__aenter__.return_value = mock_client + mock_client.get_package_info.return_value = mock_package_data + + result = await resolver.resolve_dependencies( + "test-package", + python_version="3.11" + ) + + assert result["python_version"] == "3.11" + assert "dependency_tree" in result + + @pytest.mark.asyncio + async def test_resolve_dependencies_with_extras(self, resolver): + """Test dependency resolution with extra dependencies.""" + mock_package_data = { + "info": { + "name": "test-package", + "version": "1.0.0", + "requires_python": ">=3.8", + "requires_dist": [ + "requests>=2.25.0", + "pytest>=6.0.0; extra=='test'" + ] + } + } + + with patch('pypi_query_mcp.core.PyPIClient') as mock_client_class: + mock_client = AsyncMock() + mock_client_class.return_value.__aenter__.return_value = mock_client + mock_client.get_package_info.return_value = mock_package_data + + result = await resolver.resolve_dependencies( + "test-package", + include_extras=["test"] + ) + + assert result["include_extras"] == ["test"] + assert "dependency_tree" in result + + @pytest.mark.asyncio + async def test_resolve_dependencies_max_depth(self, resolver): + """Test that max depth is respected.""" + mock_package_data = { + "info": { + "name": "test-package", + "version": "1.0.0", + "requires_python": ">=3.8", + "requires_dist": ["requests>=2.25.0"] + } + } + + with patch('pypi_query_mcp.core.PyPIClient') as mock_client_class: + mock_client = AsyncMock() + mock_client_class.return_value.__aenter__.return_value = mock_client + mock_client.get_package_info.return_value = mock_package_data + + result = await resolver.resolve_dependencies( + "test-package", + max_depth=1 + ) + + assert result["summary"]["max_depth"] <= 1 + + @pytest.mark.asyncio + async def test_resolve_package_dependencies_function(self): + """Test the standalone resolve_package_dependencies function.""" + mock_package_data = { + "info": { + "name": "test-package", + "version": "1.0.0", + "requires_python": ">=3.8", + "requires_dist": ["requests>=2.25.0"] + } + } + + with patch('pypi_query_mcp.core.PyPIClient') as mock_client_class: + mock_client = AsyncMock() + mock_client_class.return_value.__aenter__.return_value = mock_client + mock_client.get_package_info.return_value = mock_package_data + + result = await resolve_package_dependencies("test-package") + + assert result["package_name"] == "test-package" + assert "dependency_tree" in result + assert "summary" in result + + @pytest.mark.asyncio + async def test_circular_dependency_handling(self, resolver): + """Test that circular dependencies are handled properly.""" + # This is a simplified test - in reality, circular dependencies + # are prevented by the visited set + mock_package_data = { + "info": { + "name": "test-package", + "version": "1.0.0", + "requires_python": ">=3.8", + "requires_dist": ["test-package>=1.0.0"] # Self-dependency + } + } + + with patch('pypi_query_mcp.core.PyPIClient') as mock_client_class: + mock_client = AsyncMock() + mock_client_class.return_value.__aenter__.return_value = mock_client + mock_client.get_package_info.return_value = mock_package_data + + # Should not hang or crash + result = await resolver.resolve_dependencies("test-package") + assert "dependency_tree" in result + + @pytest.mark.asyncio + async def test_package_not_found_handling(self, resolver): + """Test handling of packages that are not found.""" + with patch('pypi_query_mcp.core.PyPIClient') as mock_client_class: + mock_client = AsyncMock() + mock_client_class.return_value.__aenter__.return_value = mock_client + mock_client.get_package_info.side_effect = PackageNotFoundError("Package not found") + + with pytest.raises(PackageNotFoundError): + await resolver.resolve_dependencies("nonexistent-package") diff --git a/tests/test_package_downloader.py b/tests/test_package_downloader.py new file mode 100644 index 0000000..c59391a --- /dev/null +++ b/tests/test_package_downloader.py @@ -0,0 +1,280 @@ +"""Tests for package downloader functionality.""" + +import pytest +from pathlib import Path +from unittest.mock import AsyncMock, patch, mock_open +import tempfile +import shutil + +from pypi_query_mcp.tools.package_downloader import PackageDownloader, download_package_with_dependencies +from pypi_query_mcp.core.exceptions import InvalidPackageNameError, PackageNotFoundError + + +class TestPackageDownloader: + """Test cases for PackageDownloader class.""" + + @pytest.fixture + def temp_download_dir(self): + """Create a temporary download directory for testing.""" + temp_dir = tempfile.mkdtemp() + yield temp_dir + shutil.rmtree(temp_dir) + + @pytest.fixture + def downloader(self, temp_download_dir): + """Create a PackageDownloader instance for testing.""" + return PackageDownloader(download_dir=temp_download_dir) + + @pytest.mark.asyncio + async def test_download_package_invalid_name(self, downloader): + """Test that invalid package names raise appropriate errors.""" + with pytest.raises(InvalidPackageNameError): + await downloader.download_package_with_dependencies("") + + with pytest.raises(InvalidPackageNameError): + await downloader.download_package_with_dependencies(" ") + + @pytest.mark.asyncio + async def test_download_package_basic(self, downloader): + """Test basic package download functionality.""" + mock_package_data = { + "info": { + "name": "test-package", + "version": "1.0.0", + "requires_python": ">=3.8", + "requires_dist": [] + }, + "releases": { + "1.0.0": [ + { + "filename": "test_package-1.0.0-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/test_package-1.0.0-py3-none-any.whl", + "packagetype": "bdist_wheel", + "md5_digest": "abc123", + "size": 1024 + } + ] + } + } + + mock_resolution_result = { + "package_name": "test-package", + "dependency_tree": { + "test-package": { + "name": "test-package", + "version": "1.0.0", + "dependencies": {"runtime": [], "development": [], "extras": {}}, + "depth": 0, + "children": {} + } + }, + "summary": {"total_packages": 1} + } + + with patch.object(downloader.resolver, 'resolve_dependencies') as mock_resolve: + mock_resolve.return_value = mock_resolution_result + + # Mock the _download_single_package method directly + with patch.object(downloader, '_download_single_package') as mock_download_single: + mock_download_single.return_value = { + "package_name": "test-package", + "version": "1.0.0", + "file_info": mock_package_data["releases"]["1.0.0"][0], + "download_result": { + "filename": "test_package-1.0.0-py3-none-any.whl", + "file_path": "/tmp/test_package-1.0.0-py3-none-any.whl", + "downloaded_size": 1024, + "verification": {}, + "success": True + } + } + + result = await downloader.download_package_with_dependencies("test-package") + + assert result["package_name"] == "test-package" + assert "download_results" in result + assert "summary" in result + mock_download_single.assert_called() + + @pytest.mark.asyncio + async def test_select_best_file_prefer_wheel(self, downloader): + """Test file selection with wheel preference.""" + release_files = [ + { + "filename": "test_package-1.0.0.tar.gz", + "packagetype": "sdist", + "url": "https://example.com/test_package-1.0.0.tar.gz" + }, + { + "filename": "test_package-1.0.0-py3-none-any.whl", + "packagetype": "bdist_wheel", + "url": "https://example.com/test_package-1.0.0-py3-none-any.whl" + } + ] + + selected = downloader._select_best_file(release_files, prefer_wheel=True) + assert selected["packagetype"] == "bdist_wheel" + + @pytest.mark.asyncio + async def test_select_best_file_prefer_source(self, downloader): + """Test file selection with source preference.""" + release_files = [ + { + "filename": "test_package-1.0.0.tar.gz", + "packagetype": "sdist", + "url": "https://example.com/test_package-1.0.0.tar.gz" + }, + { + "filename": "test_package-1.0.0-py3-none-any.whl", + "packagetype": "bdist_wheel", + "url": "https://example.com/test_package-1.0.0-py3-none-any.whl" + } + ] + + selected = downloader._select_best_file(release_files, prefer_wheel=False) + assert selected["packagetype"] == "sdist" + + @pytest.mark.asyncio + async def test_filter_compatible_wheels(self, downloader): + """Test filtering wheels by Python version compatibility.""" + wheels = [ + {"filename": "test_package-1.0.0-py38-none-any.whl"}, + {"filename": "test_package-1.0.0-py310-none-any.whl"}, + {"filename": "test_package-1.0.0-py3-none-any.whl"}, + {"filename": "test_package-1.0.0-cp39-cp39-linux_x86_64.whl"} + ] + + compatible = downloader._filter_compatible_wheels(wheels, "3.10") + + # Should include py310 and py3 wheels + assert len(compatible) >= 2 + filenames = [w["filename"] for w in compatible] + assert any("py310" in f for f in filenames) + assert any("py3" in f for f in filenames) + + @pytest.mark.asyncio + async def test_download_with_python_version(self, downloader): + """Test download with specific Python version.""" + mock_package_data = { + "info": { + "name": "test-package", + "version": "1.0.0", + "requires_python": ">=3.8", + "requires_dist": [] + }, + "releases": { + "1.0.0": [ + { + "filename": "test_package-1.0.0-py310-none-any.whl", + "url": "https://files.pythonhosted.org/packages/test_package-1.0.0-py310-none-any.whl", + "packagetype": "bdist_wheel", + "md5_digest": "abc123", + "size": 1024 + } + ] + } + } + + mock_resolution_result = { + "package_name": "test-package", + "dependency_tree": { + "test-package": { + "name": "test-package", + "version": "1.0.0", + "dependencies": {"runtime": [], "development": [], "extras": {}}, + "depth": 0, + "children": {} + } + }, + "summary": {"total_packages": 1} + } + + with patch('pypi_query_mcp.core.PyPIClient') as mock_client_class, \ + patch('httpx.AsyncClient') as mock_httpx_class, \ + patch.object(downloader.resolver, 'resolve_dependencies') as mock_resolve: + + mock_client = AsyncMock() + mock_client_class.return_value.__aenter__.return_value = mock_client + mock_client.get_package_info.return_value = mock_package_data + + mock_resolve.return_value = mock_resolution_result + + mock_httpx_client = AsyncMock() + mock_httpx_class.return_value.__aenter__.return_value = mock_httpx_client + + mock_response = AsyncMock() + mock_response.raise_for_status.return_value = None + mock_response.aiter_bytes.return_value = [b"test content"] + mock_httpx_client.stream.return_value.__aenter__.return_value = mock_response + + with patch("builtins.open", mock_open()): + result = await downloader.download_package_with_dependencies( + "test-package", + python_version="3.10" + ) + + assert result["python_version"] == "3.10" + + @pytest.mark.asyncio + async def test_download_package_with_dependencies_function(self, temp_download_dir): + """Test the standalone download_package_with_dependencies function.""" + mock_package_data = { + "info": { + "name": "test-package", + "version": "1.0.0", + "requires_python": ">=3.8", + "requires_dist": [] + }, + "releases": { + "1.0.0": [ + { + "filename": "test_package-1.0.0-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/test_package-1.0.0-py3-none-any.whl", + "packagetype": "bdist_wheel", + "md5_digest": "abc123", + "size": 1024 + } + ] + } + } + + with patch('pypi_query_mcp.tools.package_downloader.PackageDownloader') as mock_downloader_class: + # Setup downloader mock + mock_downloader = AsyncMock() + mock_downloader_class.return_value = mock_downloader + mock_downloader.download_package_with_dependencies.return_value = { + "package_name": "test-package", + "python_version": None, + "download_directory": temp_download_dir, + "resolution_result": { + "package_name": "test-package", + "dependency_tree": { + "test-package": { + "name": "test-package", + "version": "1.0.0", + "dependencies": {"runtime": [], "development": [], "extras": {}}, + "depth": 0, + "children": {} + } + }, + "summary": {"total_packages": 1} + }, + "download_results": {}, + "failed_downloads": [], + "summary": { + "total_packages": 1, + "successful_downloads": 1, + "failed_downloads": 0, + "total_downloaded_size": 1024, + "download_directory": temp_download_dir, + "success_rate": 100.0 + } + } + + result = await download_package_with_dependencies( + "test-package", + download_dir=temp_download_dir + ) + + assert result["package_name"] == "test-package" + assert result["download_directory"] == temp_download_dir