From e6293da1b657af49556f6879372544c6981c7b1a Mon Sep 17 00:00:00 2001 From: Ryan Malloy Date: Sat, 31 Jan 2026 10:01:38 -0700 Subject: [PATCH] feat: add OOT module installer for building modules into Docker images Single MCP tool call builds any GNU Radio OOT module from a git repo into a reusable Docker image. Generates a Dockerfile from template, builds via Docker SDK, and tracks results in a persistent JSON registry. New tools: install_oot_module, list_oot_images, remove_oot_image Also changes launch_flowgraph default xmlrpc_port from 8080 to 0 (auto) --- src/gnuradio_mcp/middlewares/oot.py | 280 ++++++++++++++++++++++ src/gnuradio_mcp/models.py | 28 +++ src/gnuradio_mcp/providers/mcp_runtime.py | 15 +- src/gnuradio_mcp/providers/runtime.py | 60 ++++- tests/unit/test_oot_installer.py | 253 +++++++++++++++++++ 5 files changed, 632 insertions(+), 4 deletions(-) create mode 100644 src/gnuradio_mcp/middlewares/oot.py create mode 100644 tests/unit/test_oot_installer.py diff --git a/src/gnuradio_mcp/middlewares/oot.py b/src/gnuradio_mcp/middlewares/oot.py new file mode 100644 index 0000000..897e04f --- /dev/null +++ b/src/gnuradio_mcp/middlewares/oot.py @@ -0,0 +1,280 @@ +from __future__ import annotations + +import io +import json +import logging +import subprocess +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from gnuradio_mcp.models import OOTImageInfo, OOTInstallResult + +logger = logging.getLogger(__name__) + +DEFAULT_BASE_IMAGE = "gnuradio-runtime:latest" + +DOCKERFILE_TEMPLATE = """\ +FROM {base_image} + +# Build dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \\ + build-essential cmake git \\ + {extra_build_deps}\\ + && rm -rf /var/lib/apt/lists/* + +# Clone and build +WORKDIR /build +RUN git clone --depth 1 --branch {branch} {git_url} && \\ + cd {repo_dir} && mkdir build && cd build && \\ + cmake -DCMAKE_INSTALL_PREFIX=/usr {cmake_args}.. && \\ + make -j$(nproc) && make install && \\ + ldconfig && \\ + rm -rf /build + +WORKDIR /flowgraphs + +# Bridge Python site-packages (cmake installs to versioned path) +ENV PYTHONPATH="/usr/lib/python3.11/site-packages:${{PYTHONPATH}}" +""" + + +class OOTInstallerMiddleware: + """Builds OOT modules into Docker images from git repos. + + Each call to build_module() generates a Dockerfile matching the + pattern in docker/Dockerfile.gnuradio-lora-runtime, builds it, + and registers the result in a persistent JSON registry. + """ + + def __init__( + self, + docker_client: Any, + base_image: str = DEFAULT_BASE_IMAGE, + ): + self._client = docker_client + self._base_image = base_image + self._registry_path = Path.home() / ".gr-mcp" / "oot-registry.json" + self._registry: dict[str, OOTImageInfo] = self._load_registry() + + # ────────────────────────────────────────── + # Public API + # ────────────────────────────────────────── + + def build_module( + self, + git_url: str, + branch: str = "main", + build_deps: list[str] | None = None, + cmake_args: list[str] | None = None, + base_image: str | None = None, + force: bool = False, + ) -> OOTInstallResult: + """Build Docker image with an OOT module compiled in.""" + effective_base = base_image or self._base_image + + try: + module_name = self._module_name_from_url(git_url) + commit = self._get_remote_commit(git_url, branch) + image_tag = f"gr-oot-{module_name}:{branch}-{commit}" + + # Idempotent: skip if image already exists + if not force and self._image_exists(image_tag): + existing = self._registry.get(module_name) + return OOTInstallResult( + success=True, + image=existing, + skipped=True, + ) + + # Generate and build + dockerfile = self.generate_dockerfile( + git_url=git_url, + branch=branch, + base_image=effective_base, + build_deps=build_deps, + cmake_args=cmake_args, + ) + + log_lines = self._docker_build(dockerfile, image_tag) + build_log_tail = "\n".join(log_lines[-30:]) + + # Register + info = OOTImageInfo( + module_name=module_name, + image_tag=image_tag, + git_url=git_url, + branch=branch, + git_commit=commit, + base_image=effective_base, + built_at=datetime.now(timezone.utc).isoformat(), + ) + self._registry[module_name] = info + self._save_registry() + + return OOTInstallResult( + success=True, + image=info, + build_log_tail=build_log_tail, + ) + + except Exception as e: + logger.exception("OOT module build failed") + return OOTInstallResult( + success=False, + error=str(e), + ) + + def list_images(self) -> list[OOTImageInfo]: + """List all registered OOT module images.""" + return list(self._registry.values()) + + def remove_image(self, module_name: str) -> bool: + """Remove Docker image and registry entry.""" + info = self._registry.pop(module_name, None) + if info is None: + return False + + try: + self._client.images.remove(info.image_tag, force=True) + except Exception as e: + logger.warning("Failed to remove Docker image %s: %s", info.image_tag, e) + + self._save_registry() + return True + + # ────────────────────────────────────────── + # Dockerfile Generation + # ────────────────────────────────────────── + + def generate_dockerfile( + self, + git_url: str, + branch: str, + base_image: str, + build_deps: list[str] | None = None, + cmake_args: list[str] | None = None, + ) -> str: + """Generate a Dockerfile string for building an OOT module.""" + repo_dir = self._repo_dir_from_url(git_url) + + extra_deps = "" + if build_deps: + extra_deps = " ".join(build_deps) + " " + + cmake_flags = "" + if cmake_args: + cmake_flags = " ".join(cmake_args) + " " + + return DOCKERFILE_TEMPLATE.format( + base_image=base_image, + branch=branch, + git_url=git_url, + repo_dir=repo_dir, + extra_build_deps=extra_deps, + cmake_args=cmake_flags, + ) + + # ────────────────────────────────────────── + # Internal Helpers + # ────────────────────────────────────────── + + @staticmethod + def _module_name_from_url(url: str) -> str: + """Extract module name from git URL. + + "https://github.com/tapparelj/gr-lora_sdr.git" -> "lora_sdr" + "https://github.com/osmocom/gr-osmosdr" -> "osmosdr" + "https://github.com/gnuradio/volk.git" -> "volk" + """ + # Strip trailing .git and slashes + cleaned = url.rstrip("/") + if cleaned.endswith(".git"): + cleaned = cleaned[:-4] + # Get last path segment + name = cleaned.rsplit("/", 1)[-1] + # Strip gr- prefix if present + if name.startswith("gr-"): + name = name[3:] + return name + + @staticmethod + def _repo_dir_from_url(url: str) -> str: + """Get the directory name git clone will create. + + "https://github.com/tapparelj/gr-lora_sdr.git" -> "gr-lora_sdr" + """ + cleaned = url.rstrip("/") + if cleaned.endswith(".git"): + cleaned = cleaned[:-4] + return cleaned.rsplit("/", 1)[-1] + + @staticmethod + def _get_remote_commit(git_url: str, branch: str) -> str: + """Get latest commit hash from remote without cloning.""" + result = subprocess.run( + ["git", "ls-remote", git_url, f"refs/heads/{branch}"], + capture_output=True, + text=True, + timeout=30, + ) + if result.returncode != 0: + raise RuntimeError(f"git ls-remote failed: {result.stderr.strip()}") + output = result.stdout.strip() + if not output: + raise RuntimeError( + f"Branch '{branch}' not found in {git_url}" + ) + return output.split()[0][:7] + + def _image_exists(self, tag: str) -> bool: + """Check if a Docker image with this tag exists locally.""" + try: + self._client.images.get(tag) + return True + except Exception: + return False + + def _docker_build(self, dockerfile: str, tag: str) -> list[str]: + """Build a Docker image from a Dockerfile string. Returns log lines.""" + f = io.BytesIO(dockerfile.encode("utf-8")) + log_lines: list[str] = [] + try: + _image, build_log = self._client.images.build( + fileobj=f, + tag=tag, + rm=True, + forcerm=True, + ) + for chunk in build_log: + if "stream" in chunk: + line = chunk["stream"].rstrip("\n") + if line: + log_lines.append(line) + logger.debug("build: %s", line) + except Exception as e: + raise RuntimeError(f"Docker build failed: {e}") from e + return log_lines + + def _load_registry(self) -> dict[str, OOTImageInfo]: + """Load the OOT image registry from disk.""" + if not self._registry_path.exists(): + return {} + try: + data = json.loads(self._registry_path.read_text()) + return { + k: OOTImageInfo(**v) + for k, v in data.items() + } + except Exception as e: + logger.warning("Failed to load OOT registry: %s", e) + return {} + + def _save_registry(self) -> None: + """Persist the OOT image registry to disk.""" + self._registry_path.parent.mkdir(parents=True, exist_ok=True) + data = { + k: v.model_dump() + for k, v in self._registry.items() + } + self._registry_path.write_text(json.dumps(data, indent=2)) diff --git a/src/gnuradio_mcp/models.py b/src/gnuradio_mcp/models.py index e83fe0b..991b772 100644 --- a/src/gnuradio_mcp/models.py +++ b/src/gnuradio_mcp/models.py @@ -363,3 +363,31 @@ class BlockPathsModel(BaseModel): paths: list[str] block_count: int blocks_added: int = 0 + + +# ────────────────────────────────────────────── +# OOT Module Installer Models +# ────────────────────────────────────────────── + + +class OOTImageInfo(BaseModel): + """Metadata for a built OOT module image.""" + + module_name: str + image_tag: str + git_url: str + branch: str + git_commit: str + base_image: str + block_count: int = 0 + built_at: str # ISO-8601 + + +class OOTInstallResult(BaseModel): + """Result of OOT module installation.""" + + success: bool + image: OOTImageInfo | None = None + build_log_tail: str = "" # Last ~30 lines of build output + error: str | None = None + skipped: bool = False # True if image already existed diff --git a/src/gnuradio_mcp/providers/mcp_runtime.py b/src/gnuradio_mcp/providers/mcp_runtime.py index 1e9de46..e0b41b1 100644 --- a/src/gnuradio_mcp/providers/mcp_runtime.py +++ b/src/gnuradio_mcp/providers/mcp_runtime.py @@ -5,6 +5,7 @@ import logging from fastmcp import FastMCP from gnuradio_mcp.middlewares.docker import DockerMiddleware +from gnuradio_mcp.middlewares.oot import OOTInstallerMiddleware from gnuradio_mcp.providers.runtime import RuntimeProvider logger = logging.getLogger(__name__) @@ -71,7 +72,14 @@ class McpRuntimeProvider: self._mcp.tool(p.combine_coverage) self._mcp.tool(p.delete_coverage) - logger.info("Registered 29 runtime tools (Docker available)") + # OOT module installation + if p._has_oot: + self._mcp.tool(p.install_oot_module) + self._mcp.tool(p.list_oot_images) + self._mcp.tool(p.remove_oot_image) + logger.info("Registered 32 runtime tools (Docker + OOT available)") + else: + logger.info("Registered 29 runtime tools (Docker available)") else: logger.info( "Registered 17 runtime tools (Docker unavailable, " @@ -82,5 +90,8 @@ class McpRuntimeProvider: def create(cls, mcp_instance: FastMCP) -> McpRuntimeProvider: """Factory: create RuntimeProvider with optional Docker support.""" docker_mw = DockerMiddleware.create() - provider = RuntimeProvider(docker_mw=docker_mw) + oot_mw = None + if docker_mw is not None: + oot_mw = OOTInstallerMiddleware(docker_mw._client) + provider = RuntimeProvider(docker_mw=docker_mw, oot_mw=oot_mw) return cls(mcp_instance, provider) diff --git a/src/gnuradio_mcp/providers/runtime.py b/src/gnuradio_mcp/providers/runtime.py index a942a66..8036481 100644 --- a/src/gnuradio_mcp/providers/runtime.py +++ b/src/gnuradio_mcp/providers/runtime.py @@ -8,6 +8,7 @@ from pathlib import Path from typing import Any, Literal from gnuradio_mcp.middlewares.docker import HOST_COVERAGE_BASE, DockerMiddleware +from gnuradio_mcp.middlewares.oot import OOTInstallerMiddleware from gnuradio_mcp.middlewares.thrift import ThriftMiddleware from gnuradio_mcp.middlewares.xmlrpc import XmlRpcMiddleware from gnuradio_mcp.models import ( @@ -17,6 +18,8 @@ from gnuradio_mcp.models import ( CoverageReportModel, KnobModel, KnobPropertiesModel, + OOTImageInfo, + OOTInstallResult, PerfCounterModel, RuntimeStatusModel, ScreenshotModel, @@ -40,8 +43,10 @@ class RuntimeProvider: def __init__( self, docker_mw: DockerMiddleware | None = None, + oot_mw: OOTInstallerMiddleware | None = None, ): self._docker = docker_mw + self._oot = oot_mw self._xmlrpc: XmlRpcMiddleware | None = None self._thrift: ThriftMiddleware | None = None self._active_container: str | None = None @@ -50,6 +55,10 @@ class RuntimeProvider: def _has_docker(self) -> bool: return self._docker is not None + @property + def _has_oot(self) -> bool: + return self._oot is not None + def _require_docker(self) -> DockerMiddleware: if self._docker is None: raise RuntimeError( @@ -74,6 +83,14 @@ class RuntimeProvider: ) return self._thrift + def _require_oot(self) -> OOTInstallerMiddleware: + if self._oot is None: + raise RuntimeError( + "OOT installer requires Docker. Install the 'docker' package " + "and ensure the Docker daemon is running." + ) + return self._oot + # ────────────────────────────────────────── # Container Lifecycle # ────────────────────────────────────────── @@ -82,7 +99,7 @@ class RuntimeProvider: self, flowgraph_path: str, name: str | None = None, - xmlrpc_port: int = 8080, + xmlrpc_port: int = 0, enable_vnc: bool = False, enable_coverage: bool = False, enable_controlport: bool = False, @@ -96,7 +113,7 @@ class RuntimeProvider: Args: flowgraph_path: Path to the .py flowgraph file name: Container name (defaults to 'gr-{stem}') - xmlrpc_port: Port for XML-RPC variable control + xmlrpc_port: Port for XML-RPC variable control (0 = auto-allocate) enable_vnc: Enable VNC server for visual debugging enable_coverage: Enable Python code coverage collection enable_controlport: Enable ControlPort/Thrift for advanced control @@ -662,3 +679,42 @@ class RuntimeProvider: deleted += 1 return deleted + + # ────────────────────────────────────────── + # OOT Module Installation + # ────────────────────────────────────────── + + def install_oot_module( + self, + git_url: str, + branch: str = "main", + build_deps: list[str] | None = None, + cmake_args: list[str] | None = None, + base_image: str | None = None, + force: bool = False, + ) -> OOTInstallResult: + """Install an OOT module into a Docker image. + + Clones the git repo, compiles with cmake, and creates a reusable + Docker image. Use the returned image_tag with launch_flowgraph(). + + Args: + git_url: Git repository URL (e.g., "https://github.com/tapparelj/gr-lora_sdr") + branch: Git branch to build from + build_deps: Extra apt packages needed for compilation + cmake_args: Extra cmake flags (e.g., ["-DENABLE_TESTING=OFF"]) + base_image: Base image (default: gnuradio-runtime:latest) + force: Rebuild even if image exists + """ + oot = self._require_oot() + return oot.build_module(git_url, branch, build_deps, cmake_args, base_image, force) + + def list_oot_images(self) -> list[OOTImageInfo]: + """List all installed OOT module images.""" + oot = self._require_oot() + return oot.list_images() + + def remove_oot_image(self, module_name: str) -> bool: + """Remove an OOT module image and its registry entry.""" + oot = self._require_oot() + return oot.remove_image(module_name) diff --git a/tests/unit/test_oot_installer.py b/tests/unit/test_oot_installer.py new file mode 100644 index 0000000..c74ec5f --- /dev/null +++ b/tests/unit/test_oot_installer.py @@ -0,0 +1,253 @@ +"""Unit tests for OOT module installer middleware. + +Tests Dockerfile generation, module name extraction, registry persistence, +and image naming — all without requiring Docker. +""" + +import json +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from gnuradio_mcp.middlewares.oot import OOTInstallerMiddleware +from gnuradio_mcp.models import OOTImageInfo + + +@pytest.fixture +def mock_docker_client(): + return MagicMock() + + +@pytest.fixture +def oot(mock_docker_client, tmp_path): + mw = OOTInstallerMiddleware(mock_docker_client) + # Override registry path to use tmp_path + mw._registry_path = tmp_path / "oot-registry.json" + mw._registry = {} + return mw + + +# ────────────────────────────────────────── +# Module Name Extraction +# ────────────────────────────────────────── + + +class TestModuleNameFromUrl: + def test_gr_prefix_stripped(self): + name = OOTInstallerMiddleware._module_name_from_url( + "https://github.com/tapparelj/gr-lora_sdr.git" + ) + assert name == "lora_sdr" + + def test_gr_prefix_no_git_suffix(self): + name = OOTInstallerMiddleware._module_name_from_url( + "https://github.com/osmocom/gr-osmosdr" + ) + assert name == "osmosdr" + + def test_no_gr_prefix(self): + name = OOTInstallerMiddleware._module_name_from_url( + "https://github.com/gnuradio/volk.git" + ) + assert name == "volk" + + def test_trailing_slash(self): + name = OOTInstallerMiddleware._module_name_from_url( + "https://github.com/tapparelj/gr-lora_sdr/" + ) + assert name == "lora_sdr" + + def test_gr_satellites(self): + name = OOTInstallerMiddleware._module_name_from_url( + "https://github.com/daniestevez/gr-satellites.git" + ) + assert name == "satellites" + + +class TestRepoDirFromUrl: + def test_preserves_gr_prefix(self): + d = OOTInstallerMiddleware._repo_dir_from_url( + "https://github.com/tapparelj/gr-lora_sdr.git" + ) + assert d == "gr-lora_sdr" + + def test_no_git_suffix(self): + d = OOTInstallerMiddleware._repo_dir_from_url( + "https://github.com/osmocom/gr-osmosdr" + ) + assert d == "gr-osmosdr" + + +# ────────────────────────────────────────── +# Dockerfile Generation +# ────────────────────────────────────────── + + +class TestDockerfileGeneration: + def test_basic_dockerfile(self, oot): + dockerfile = oot.generate_dockerfile( + git_url="https://github.com/tapparelj/gr-lora_sdr.git", + branch="master", + base_image="gnuradio-runtime:latest", + ) + assert "FROM gnuradio-runtime:latest" in dockerfile + assert "git clone --depth 1 --branch master" in dockerfile + assert "https://github.com/tapparelj/gr-lora_sdr.git" in dockerfile + assert "cd gr-lora_sdr && mkdir build" in dockerfile + assert "cmake -DCMAKE_INSTALL_PREFIX=/usr" in dockerfile + assert "make -j$(nproc)" in dockerfile + assert "ldconfig" in dockerfile + assert "PYTHONPATH" in dockerfile + + def test_with_extra_build_deps(self, oot): + dockerfile = oot.generate_dockerfile( + git_url="https://github.com/tapparelj/gr-lora_sdr.git", + branch="master", + base_image="gnuradio-runtime:latest", + build_deps=["libvolk2-dev", "libboost-all-dev"], + ) + assert "libvolk2-dev libboost-all-dev" in dockerfile + + def test_with_cmake_args(self, oot): + dockerfile = oot.generate_dockerfile( + git_url="https://github.com/tapparelj/gr-lora_sdr.git", + branch="master", + base_image="gnuradio-runtime:latest", + cmake_args=["-DENABLE_TESTING=OFF", "-DBUILD_DOCS=OFF"], + ) + assert "-DENABLE_TESTING=OFF -DBUILD_DOCS=OFF" in dockerfile + + def test_custom_base_image(self, oot): + dockerfile = oot.generate_dockerfile( + git_url="https://github.com/tapparelj/gr-lora_sdr.git", + branch="main", + base_image="gnuradio-coverage:latest", + ) + assert "FROM gnuradio-coverage:latest" in dockerfile + + def test_no_extra_deps_no_trailing_space(self, oot): + dockerfile = oot.generate_dockerfile( + git_url="https://github.com/tapparelj/gr-lora_sdr.git", + branch="master", + base_image="gnuradio-runtime:latest", + ) + # With no extra deps, the apt-get line should still work + assert "build-essential cmake git" in dockerfile + + +# ────────────────────────────────────────── +# Registry Persistence +# ────────────────────────────────────────── + + +class TestRegistry: + def test_empty_on_fresh_start(self, oot): + assert oot._registry == {} + assert oot.list_images() == [] + + def test_save_and_load_roundtrip(self, oot): + info = OOTImageInfo( + module_name="lora_sdr", + image_tag="gr-oot-lora_sdr:master-862746d", + git_url="https://github.com/tapparelj/gr-lora_sdr.git", + branch="master", + git_commit="862746d", + base_image="gnuradio-runtime:latest", + built_at="2025-01-01T00:00:00+00:00", + ) + oot._registry["lora_sdr"] = info + oot._save_registry() + + # Reload from disk + loaded = oot._load_registry() + assert "lora_sdr" in loaded + assert loaded["lora_sdr"].image_tag == "gr-oot-lora_sdr:master-862746d" + assert loaded["lora_sdr"].git_commit == "862746d" + + def test_load_missing_file_returns_empty(self, tmp_path): + mw = OOTInstallerMiddleware(MagicMock()) + mw._registry_path = tmp_path / "nonexistent" / "registry.json" + result = mw._load_registry() + assert result == {} + + def test_load_corrupt_file_returns_empty(self, oot): + oot._registry_path.write_text("not valid json{{{") + result = oot._load_registry() + assert result == {} + + def test_save_creates_parent_dirs(self, tmp_path): + mw = OOTInstallerMiddleware(MagicMock()) + mw._registry_path = tmp_path / "nested" / "deep" / "registry.json" + mw._registry = {} + mw._save_registry() + assert mw._registry_path.exists() + + +# ────────────────────────────────────────── +# Image Naming +# ────────────────────────────────────────── + + +class TestImageTagFormat: + def test_standard_format(self): + """Image tags follow gr-oot-{name}:{branch}-{commit7}.""" + # This verifies the format used in build_module() + module_name = "lora_sdr" + branch = "master" + commit = "862746d" + tag = f"gr-oot-{module_name}:{branch}-{commit}" + assert tag == "gr-oot-lora_sdr:master-862746d" + + def test_different_branch(self): + tag = f"gr-oot-osmosdr:develop-abc1234" + assert "develop" in tag + assert "abc1234" in tag + + +# ────────────────────────────────────────── +# Remove Image +# ────────────────────────────────────────── + + +class TestRemoveImage: + def test_remove_existing(self, oot, mock_docker_client): + info = OOTImageInfo( + module_name="lora_sdr", + image_tag="gr-oot-lora_sdr:master-862746d", + git_url="https://github.com/tapparelj/gr-lora_sdr.git", + branch="master", + git_commit="862746d", + base_image="gnuradio-runtime:latest", + built_at="2025-01-01T00:00:00+00:00", + ) + oot._registry["lora_sdr"] = info + + result = oot.remove_image("lora_sdr") + assert result is True + assert "lora_sdr" not in oot._registry + mock_docker_client.images.remove.assert_called_once_with( + "gr-oot-lora_sdr:master-862746d", force=True + ) + + def test_remove_nonexistent(self, oot): + result = oot.remove_image("does_not_exist") + assert result is False + + def test_remove_survives_docker_error(self, oot, mock_docker_client): + """Registry entry is removed even if Docker image removal fails.""" + info = OOTImageInfo( + module_name="lora_sdr", + image_tag="gr-oot-lora_sdr:master-862746d", + git_url="https://github.com/tapparelj/gr-lora_sdr.git", + branch="master", + git_commit="862746d", + base_image="gnuradio-runtime:latest", + built_at="2025-01-01T00:00:00+00:00", + ) + oot._registry["lora_sdr"] = info + mock_docker_client.images.remove.side_effect = Exception("image not found") + + result = oot.remove_image("lora_sdr") + assert result is True + assert "lora_sdr" not in oot._registry