diff --git a/EXTRAS_FIX_SUMMARY.md b/EXTRAS_FIX_SUMMARY.md new file mode 100644 index 0000000..57ca873 --- /dev/null +++ b/EXTRAS_FIX_SUMMARY.md @@ -0,0 +1,151 @@ +# Fix for include_extras Parameter Validation Issue + +## Summary + +Fixed a critical issue where the `include_extras` parameter in the `resolve_dependencies` tool was not working correctly due to incorrect Python version filtering that was removing extra dependencies from consideration. + +## Root Cause Analysis + +The issue was in the `_is_requirement_applicable` method in `pypi_query_mcp/core/dependency_parser.py`. When filtering requirements by Python version, the method was evaluating markers like `extra == "socks"` in an environment that didn't include the `extra` variable, causing these requirements to be filtered out incorrectly. + +### The Problem + +1. **Python version filtering was too aggressive**: The `filter_requirements_by_python_version` method was filtering out ALL requirements with markers that couldn't be evaluated, including extra requirements. + +2. **Marker evaluation environment was incomplete**: When evaluating markers like `extra == "socks"`, the environment didn't include the `extra` variable, so the evaluation always returned `False`. + +3. **Incorrect filtering logic**: Extra dependencies should not be filtered by Python version at all - they should be handled separately based on user selection. + +### Before the Fix + +```python +# In _is_requirement_applicable method +def _is_requirement_applicable(self, req: Requirement, python_version: Version) -> bool: + if not req.marker: + return True + + env = { + "python_version": str(python_version), + # ... other environment variables + } + + try: + return req.marker.evaluate(env) # This returns False for extra == "socks" + except Exception as e: + logger.warning(f"Failed to evaluate marker for {req}: {e}") + return True +``` + +**Result**: Requirements like `PySocks!=1.5.7,>=1.5.6; extra == "socks"` were filtered out because `extra == "socks"` evaluated to `False` in an environment without the `extra` variable. + +## The Fix + +Added a check to exclude extra requirements from Python version filtering: + +```python +def _is_requirement_applicable(self, req: Requirement, python_version: Version) -> bool: + if not req.marker: + return True + + # If the marker contains 'extra ==', this is an extra dependency + # and should not be filtered by Python version. Extra dependencies + # are handled separately based on user selection. + marker_str = str(req.marker) + if "extra ==" in marker_str: + return True + + # Rest of the method unchanged... +``` + +### Why This Fix Works + +1. **Preserves extra requirements**: Extra dependencies are no longer filtered out by Python version filtering +2. **Maintains correct Python version filtering**: Non-extra requirements are still properly filtered by Python version +3. **Separates concerns**: Extra handling is kept separate from Python version filtering, as it should be +4. **Backwards compatible**: Doesn't break existing functionality + +## Testing Results + +### Before the Fix +```python +# Example: requests[socks] with Python 3.10 +result = await resolve_package_dependencies("requests", include_extras=["socks"], python_version="3.10") +print(result["summary"]["total_extra_dependencies"]) # Output: 0 ❌ +``` + +### After the Fix +```python +# Example: requests[socks] with Python 3.10 +result = await resolve_package_dependencies("requests", include_extras=["socks"], python_version="3.10") +print(result["summary"]["total_extra_dependencies"]) # Output: 1 ✅ +print(result["dependency_tree"]["requests"]["dependencies"]["extras"]) +# Output: {'socks': ['PySocks!=1.5.7,>=1.5.6; extra == "socks"']} ✅ +``` + +## Examples of Correct Usage + +### Basic Examples + +```python +# Requests with SOCKS proxy support +await resolve_package_dependencies("requests", include_extras=["socks"]) + +# Django with password hashing extras +await resolve_package_dependencies("django", include_extras=["argon2", "bcrypt"]) + +# Setuptools with testing tools +await resolve_package_dependencies("setuptools", include_extras=["test"]) + +# Flask with async and dotenv support +await resolve_package_dependencies("flask", include_extras=["async", "dotenv"]) +``` + +### How to Find Available Extras + +1. **Check the package's PyPI page** - Look for available extras in the description +2. **Use the `provides_extra` field** - Available in package metadata +3. **Check package documentation** - Often lists available extras +4. **Look for requirements with `extra ==`** - In the `requires_dist` field + +### Common Mistakes to Avoid + +❌ **Wrong**: Using generic extra names +```python +# These don't exist for requests +await resolve_package_dependencies("requests", include_extras=["dev", "test"]) +``` + +✅ **Right**: Using package-specific extra names +```python +# These are actual extras for requests +await resolve_package_dependencies("requests", include_extras=["socks", "use-chardet-on-py3"]) +``` + +## Files Modified + +1. **`pypi_query_mcp/core/dependency_parser.py`**: Fixed `_is_requirement_applicable` method +2. **`pypi_query_mcp/server.py`**: Updated documentation for `include_extras` parameter +3. **`pypi_query_mcp/tools/dependency_resolver.py`**: Updated docstring for `include_extras` parameter +4. **`tests/test_dependency_resolver.py`**: Added comprehensive test cases +5. **`examples/extras_usage_demo.py`**: Added demonstration of correct usage + +## Validation + +The fix has been validated with: + +- ✅ Real PyPI packages (requests, django, setuptools, flask) +- ✅ Various extra combinations +- ✅ Python version filtering still works for non-extra requirements +- ✅ Transitive dependency resolution with extras +- ✅ Edge cases (non-existent extras, extras with no dependencies) +- ✅ Backwards compatibility with existing code + +## Performance Impact + +- **Minimal**: Only adds a simple string check for `"extra =="` in markers +- **Positive**: Reduces unnecessary marker evaluations for extra requirements +- **No regression**: All existing functionality continues to work as before + +## Conclusion + +This fix resolves the core issue with `include_extras` parameter validation while maintaining all existing functionality. Users can now successfully resolve optional dependencies using the correct extra names as defined by each package. \ No newline at end of file diff --git a/examples/extras_usage_demo.py b/examples/extras_usage_demo.py new file mode 100644 index 0000000..b749a5c --- /dev/null +++ b/examples/extras_usage_demo.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +""" +Demonstration of PyPI Query MCP Server extras functionality. + +This script shows how to properly use the include_extras parameter +to resolve optional dependencies for Python packages. +""" + +import asyncio +import json +from pathlib import Path + +from pypi_query_mcp.tools.dependency_resolver import resolve_package_dependencies +from pypi_query_mcp.core.pypi_client import PyPIClient + + +async def show_available_extras(package_name: str): + """Show what extras are available for a package.""" + print(f"\n📦 Available extras for {package_name}:") + + async with PyPIClient() as client: + package_data = await client.get_package_info(package_name) + + info = package_data.get("info", {}) + provides_extra = info.get("provides_extra", []) + requires_dist = info.get("requires_dist", []) or [] + + if provides_extra: + print(f" Provides extras: {', '.join(provides_extra)}") + else: + print(" No provides_extra field found") + + # Find extras from requires_dist + extras_in_deps = set() + for req in requires_dist: + if "extra ==" in req: + # Extract extra name from requirement like: pytest>=6.0.0; extra=='test' + import re + match = re.search(r'extra\s*==\s*["\']([^"\']+)["\']', req) + if match: + extras_in_deps.add(match.group(1)) + + if extras_in_deps: + print(f" Extras with dependencies: {', '.join(sorted(extras_in_deps))}") + else: + print(" No extras with dependencies found") + + +async def demo_extras_resolution(): + """Demonstrate extras resolution with various packages.""" + + # Examples of packages with well-known extras + examples = [ + { + "package": "requests", + "extras": ["socks"], + "description": "HTTP library with SOCKS proxy support" + }, + { + "package": "django", + "extras": ["argon2", "bcrypt"], + "description": "Web framework with password hashing extras" + }, + { + "package": "setuptools", + "extras": ["test"], + "description": "Package development tools with testing extras" + }, + { + "package": "flask", + "extras": ["async", "dotenv"], + "description": "Web framework with async and dotenv support" + } + ] + + for example in examples: + package_name = example["package"] + extras = example["extras"] + description = example["description"] + + print(f"\n{'='*60}") + print(f"🔍 Example: {package_name}") + print(f"📋 Description: {description}") + print(f"🎯 Testing extras: {extras}") + + # Show available extras + await show_available_extras(package_name) + + try: + # Resolve without extras + print(f"\n📊 Resolving {package_name} WITHOUT extras...") + result_no_extras = await resolve_package_dependencies( + package_name=package_name, + python_version="3.10", + include_extras=[], + max_depth=1 # Limit depth for demo + ) + + # Resolve with extras + print(f"📊 Resolving {package_name} WITH extras {extras}...") + result_with_extras = await resolve_package_dependencies( + package_name=package_name, + python_version="3.10", + include_extras=extras, + max_depth=1 + ) + + # Compare results + print(f"\n📈 Results comparison:") + print(f" Without extras: {result_no_extras['summary']['total_extra_dependencies']} extra deps") + print(f" With extras: {result_with_extras['summary']['total_extra_dependencies']} extra deps") + + # Show actual extras resolved + main_pkg = next(iter(result_with_extras['dependency_tree'].values()), {}) + extras_resolved = main_pkg.get('dependencies', {}).get('extras', {}) + + if extras_resolved: + print(f" ✅ Extras resolved successfully:") + for extra_name, deps in extras_resolved.items(): + print(f" - {extra_name}: {len(deps)} dependencies") + for dep in deps[:2]: # Show first 2 + print(f" * {dep}") + if len(deps) > 2: + print(f" * ... and {len(deps) - 2} more") + else: + print(f" ⚠️ No extras resolved (may not exist or have no dependencies)") + + except Exception as e: + print(f" ❌ Error: {e}") + + +async def demo_incorrect_usage(): + """Demonstrate common mistakes with extras usage.""" + print(f"\n{'='*60}") + print("❌ Common Mistakes with Extras") + print("='*60") + + mistakes = [ + { + "package": "requests", + "extras": ["dev", "test"], # These don't exist for requests + "error": "Using generic extra names instead of package-specific ones" + }, + { + "package": "setuptools", + "extras": ["testing"], # Should be "test" not "testing" + "error": "Using similar but incorrect extra names" + } + ] + + for mistake in mistakes: + package_name = mistake["package"] + extras = mistake["extras"] + error_desc = mistake["error"] + + print(f"\n🚫 Mistake: {error_desc}") + print(f" Package: {package_name}") + print(f" Incorrect extras: {extras}") + + try: + result = await resolve_package_dependencies( + package_name=package_name, + python_version="3.10", + include_extras=extras, + max_depth=1 + ) + + total_extras = result['summary']['total_extra_dependencies'] + print(f" Result: {total_extras} extra dependencies resolved") + if total_extras == 0: + print(f" ⚠️ No extras resolved - these extras likely don't exist") + + except Exception as e: + print(f" ❌ Error: {e}") + + +async def main(): + """Main demonstration function.""" + print("🚀 PyPI Query MCP Server - Extras Usage Demo") + print("=" * 60) + print() + print("This demo shows how to properly use the include_extras parameter") + print("to resolve optional dependencies for Python packages.") + + await demo_extras_resolution() + await demo_incorrect_usage() + + print(f"\n{'='*60}") + print("✨ Demo completed!") + print() + print("💡 Key takeaways:") + print(" 1. Always check what extras are available for a package first") + print(" 2. Use the exact extra names defined by the package") + print(" 3. Check package documentation or PyPI page for available extras") + print(" 4. Not all packages have extras, and some extras may have no dependencies") + print() + print("📚 To find available extras:") + print(" - Check the package's PyPI page") + print(" - Look for 'provides_extra' in package metadata") + print(" - Check package documentation") + print(" - Look for requirements with 'extra ==' in requires_dist") + + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/pypi_query_mcp/core/dependency_parser.py b/pypi_query_mcp/core/dependency_parser.py index c1eea9d..021bafe 100644 --- a/pypi_query_mcp/core/dependency_parser.py +++ b/pypi_query_mcp/core/dependency_parser.py @@ -81,6 +81,13 @@ class DependencyParser: if not req.marker: return True + # If the marker contains 'extra ==', this is an extra dependency + # and should not be filtered by Python version. Extra dependencies + # are handled separately based on user selection. + marker_str = str(req.marker) + if "extra ==" in marker_str: + return True + # Create environment for marker evaluation env = { "python_version": str(python_version), diff --git a/pypi_query_mcp/server.py b/pypi_query_mcp/server.py index 2f52b84..9ad1031 100644 --- a/pypi_query_mcp/server.py +++ b/pypi_query_mcp/server.py @@ -305,7 +305,10 @@ async def resolve_dependencies( Args: package_name: The name of the PyPI package to analyze (e.g., 'pyside2', 'django') python_version: Target Python version for dependency filtering (e.g., '3.10', '3.11') - include_extras: List of extra dependency groups to include (e.g., ['dev', 'test']) + include_extras: List of extra dependency groups to include. These are optional + dependency groups defined by the package (e.g., ['socks'] for requests, + ['argon2', 'bcrypt'] for django, ['test', 'doc'] for setuptools). Check the + package's PyPI page or use the provides_extra field to see available extras. include_dev: Whether to include development dependencies (default: False) max_depth: Maximum recursion depth for dependency resolution (default: 5) @@ -373,7 +376,9 @@ async def download_package( package_name: The name of the PyPI package to download (e.g., 'pyside2', 'requests') download_dir: Local directory to download packages to (default: './downloads') python_version: Target Python version for compatibility (e.g., '3.10', '3.11') - include_extras: List of extra dependency groups to include (e.g., ['dev', 'test']) + include_extras: List of extra dependency groups to include. These are optional + dependency groups defined by the package (e.g., ['socks'] for requests, + ['argon2', 'bcrypt'] for django). Check the package's PyPI page to see available extras. include_dev: Whether to include development dependencies (default: False) prefer_wheel: Whether to prefer wheel files over source distributions (default: True) verify_checksums: Whether to verify downloaded file checksums (default: True) diff --git a/pypi_query_mcp/tools/dependency_resolver.py b/pypi_query_mcp/tools/dependency_resolver.py index 85cdd68..0047aff 100644 --- a/pypi_query_mcp/tools/dependency_resolver.py +++ b/pypi_query_mcp/tools/dependency_resolver.py @@ -35,7 +35,8 @@ class DependencyResolver: Args: package_name: Name of the package to resolve python_version: Target Python version (e.g., "3.10") - include_extras: List of extra dependencies to include + include_extras: List of extra dependency groups to include (e.g., ['socks'] for requests, + ['test', 'doc'] for setuptools). These are optional dependencies defined by the package. include_dev: Whether to include development dependencies max_depth: Maximum recursion depth (overrides instance default) @@ -242,7 +243,8 @@ async def resolve_package_dependencies( Args: package_name: Name of the package to resolve python_version: Target Python version (e.g., "3.10") - include_extras: List of extra dependencies to include + include_extras: List of extra dependency groups to include (e.g., ['socks'] for requests, + ['test', 'doc'] for setuptools). These are optional dependencies defined by the package. include_dev: Whether to include development dependencies max_depth: Maximum recursion depth diff --git a/tests/test_dependency_resolver.py b/tests/test_dependency_resolver.py index 3e9001b..b0b128f 100644 --- a/tests/test_dependency_resolver.py +++ b/tests/test_dependency_resolver.py @@ -83,24 +83,145 @@ class TestDependencyResolver: """Test dependency resolution with extra dependencies.""" mock_package_data = { "info": { - "name": "test-package", + "name": "mock-test-package-12345", "version": "1.0.0", "requires_python": ">=3.8", "requires_dist": ["requests>=2.25.0", "pytest>=6.0.0; extra=='test'"], } } + # Mock for transitive dependencies + mock_requests_data = { + "info": { + "name": "requests", + "version": "2.25.0", + "requires_python": ">=3.6", + "requires_dist": [], + } + } + + mock_pytest_data = { + "info": { + "name": "pytest", + "version": "6.0.0", + "requires_python": ">=3.6", + "requires_dist": [], + } + } + with patch("pypi_query_mcp.core.PyPIClient") as mock_client_class: mock_client = AsyncMock() mock_client_class.return_value.__aenter__.return_value = mock_client - mock_client.get_package_info.return_value = mock_package_data + + # Setup mock to return different data based on package name + def mock_get_package_info(package_name): + if package_name.lower() == "mock-test-package-12345": + return mock_package_data + elif package_name.lower() == "requests": + return mock_requests_data + elif package_name.lower() == "pytest": + return mock_pytest_data + else: + return {"info": {"name": package_name, "version": "1.0.0", "requires_dist": []}} + + mock_client.get_package_info.side_effect = mock_get_package_info result = await resolver.resolve_dependencies( - "test-package", include_extras=["test"] + "mock-test-package-12345", include_extras=["test"], max_depth=2 ) assert result["include_extras"] == ["test"] assert "dependency_tree" in result + + # Verify that extras are properly resolved and included + assert result["summary"]["total_extra_dependencies"] == 1 + main_pkg = result["dependency_tree"]["mock-test-package-12345"] + assert "test" in main_pkg["dependencies"]["extras"] + assert len(main_pkg["dependencies"]["extras"]["test"]) == 1 + assert "pytest" in main_pkg["dependencies"]["extras"]["test"][0] + + @pytest.mark.asyncio + async def test_resolve_dependencies_with_extras_and_python_version(self, resolver): + """Test that extras work correctly with Python version filtering.""" + mock_package_data = { + "info": { + "name": "test-package", + "version": "1.0.0", + "requires_python": ">=3.8", + "requires_dist": [ + "requests>=2.25.0", + "typing-extensions>=4.0.0; python_version<'3.10'", + "pytest>=6.0.0; extra=='test'", + "coverage>=5.0; extra=='test'", + ], + } + } + + # Mock for transitive dependencies + mock_requests_data = { + "info": { + "name": "requests", + "version": "2.25.0", + "requires_python": ">=3.6", + "requires_dist": [], + } + } + + mock_pytest_data = { + "info": { + "name": "pytest", + "version": "6.0.0", + "requires_python": ">=3.6", + "requires_dist": [], + } + } + + mock_coverage_data = { + "info": { + "name": "coverage", + "version": "5.0.0", + "requires_python": ">=3.6", + "requires_dist": [], + } + } + + with patch("pypi_query_mcp.core.PyPIClient") as mock_client_class: + mock_client = AsyncMock() + mock_client_class.return_value.__aenter__.return_value = mock_client + + # Setup mock to return different data based on package name + def mock_get_package_info(package_name): + if package_name.lower() == "test-package": + return mock_package_data + elif package_name.lower() == "requests": + return mock_requests_data + elif package_name.lower() == "pytest": + return mock_pytest_data + elif package_name.lower() == "coverage": + return mock_coverage_data + else: + return {"info": {"name": package_name, "version": "1.0.0", "requires_dist": []}} + + mock_client.get_package_info.side_effect = mock_get_package_info + + # Test with Python 3.11 - should not include typing-extensions but should include extras + result = await resolver.resolve_dependencies( + "test-package", python_version="3.11", include_extras=["test"], max_depth=2 + ) + + assert result["include_extras"] == ["test"] + assert result["python_version"] == "3.11" + + # Verify that extras are properly resolved + assert result["summary"]["total_extra_dependencies"] == 2 + main_pkg = result["dependency_tree"]["test-package"] + assert "test" in main_pkg["dependencies"]["extras"] + assert len(main_pkg["dependencies"]["extras"]["test"]) == 2 + + # Verify Python version filtering worked for runtime deps but not extras + runtime_deps = main_pkg["dependencies"]["runtime"] + assert len(runtime_deps) == 1 # Only requests, not typing-extensions + assert "requests" in runtime_deps[0] @pytest.mark.asyncio async def test_resolve_dependencies_max_depth(self, resolver):