From 66f1b260f2b1825b1d4ad8cce7cb8e9e7fd3bb22 Mon Sep 17 00:00:00 2001 From: Ryan Malloy Date: Sun, 1 Feb 2026 22:45:14 -0700 Subject: [PATCH] feat: multi-OOT combo images via multi-stage Docker builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Combine multiple OOT modules into a single Docker image using multi-stage COPY from existing single-OOT images. No recompilation needed — fast and deterministic. New MCP tools: build_multi_oot_image, list_combo_images, remove_combo_image Also hardens _load_registry() to validate per-entry instead of all-or-nothing, preventing one corrupt entry from discarding the entire registry. --- src/gnuradio_mcp/middlewares/oot.py | 206 ++++++++++++++- src/gnuradio_mcp/models.py | 20 ++ src/gnuradio_mcp/providers/mcp_runtime.py | 6 +- src/gnuradio_mcp/providers/runtime.py | 32 +++ tests/unit/test_oot_installer.py | 305 +++++++++++++++++++++- 5 files changed, 559 insertions(+), 10 deletions(-) diff --git a/src/gnuradio_mcp/middlewares/oot.py b/src/gnuradio_mcp/middlewares/oot.py index 80ebf3c..ee169e5 100644 --- a/src/gnuradio_mcp/middlewares/oot.py +++ b/src/gnuradio_mcp/middlewares/oot.py @@ -9,7 +9,7 @@ from datetime import datetime, timezone from pathlib import Path from typing import Any -from gnuradio_mcp.models import OOTImageInfo, OOTInstallResult +from gnuradio_mcp.models import ComboImageInfo, ComboImageResult, OOTImageInfo, OOTInstallResult logger = logging.getLogger(__name__) @@ -101,6 +101,8 @@ class OOTInstallerMiddleware: self._base_image = base_image self._registry_path = Path.home() / ".gr-mcp" / "oot-registry.json" self._registry: dict[str, OOTImageInfo] = self._load_registry() + self._combo_registry_path = Path.home() / ".gr-mcp" / "oot-combo-registry.json" + self._combo_registry: dict[str, ComboImageInfo] = self._load_combo_registry() # ────────────────────────────────────────── # Public API @@ -319,18 +321,25 @@ class OOTInstallerMiddleware: return log_lines def _load_registry(self) -> dict[str, OOTImageInfo]: - """Load the OOT image registry from disk.""" + """Load the OOT image registry from disk. + + Validates entries individually so one corrupted entry + doesn't discard the entire registry. + """ if not self._registry_path.exists(): return {} try: data = json.loads(self._registry_path.read_text()) - return { - k: OOTImageInfo(**v) - for k, v in data.items() - } except Exception as e: - logger.warning("Failed to load OOT registry: %s", e) + logger.warning("Failed to parse OOT registry JSON: %s", e) return {} + registry: dict[str, OOTImageInfo] = {} + for k, v in data.items(): + try: + registry[k] = OOTImageInfo(**v) + except Exception as e: + logger.warning("Skipping corrupt registry entry '%s': %s", k, e) + return registry def _save_registry(self) -> None: """Persist the OOT image registry to disk.""" @@ -340,3 +349,186 @@ class OOTInstallerMiddleware: for k, v in self._registry.items() } self._registry_path.write_text(json.dumps(data, indent=2)) + + # ────────────────────────────────────────── + # Combo Image (Multi-OOT) Support + # ────────────────────────────────────────── + + @staticmethod + def _combo_key(module_names: list[str]) -> str: + """Deterministic key from module names: sorted, deduped, joined.""" + names = sorted(set(module_names)) + return "combo:" + "+".join(names) + + @staticmethod + def _combo_image_tag(module_names: list[str]) -> str: + """Deterministic image tag for a combo of modules.""" + names = sorted(set(module_names)) + return f"gr-combo-{'-'.join(names)}:latest" + + def generate_combo_dockerfile(self, module_names: list[str]) -> str: + """Generate multi-stage Dockerfile that COPYs from existing single-OOT images.""" + names = sorted(set(module_names)) + stages: list[str] = [] + copies: list[str] = [] + + for name in names: + info = self._registry.get(name) + if info is None: + raise ValueError( + f"Module '{name}' not found in OOT registry. " + f"Build it first with build_module()." + ) + stage_alias = f"stage_{name}" + stages.append(f"FROM {info.image_tag} AS {stage_alias}") + for path in ["/usr/lib/", "/usr/include/", "/usr/share/gnuradio/"]: + copies.append(f"COPY --from={stage_alias} {path} {path}") + + return "\n".join([ + *stages, + "", + f"FROM {self._base_image}", + "", + *copies, + "", + "RUN ldconfig", + "WORKDIR /flowgraphs", + 'ENV PYTHONPATH="/usr/lib/python3.11/site-packages:${PYTHONPATH}"', + "", + ]) + + def build_combo_image( + self, + module_names: list[str], + force: bool = False, + ) -> ComboImageResult: + """Build a combined Docker image with multiple OOT modules. + + Modules already in the registry are used as-is. Modules found + in the OOT catalog but not yet built are auto-built first. + """ + from gnuradio_mcp.oot_catalog import CATALOG + + names = sorted(set(module_names)) + if len(names) < 2: + return ComboImageResult( + success=False, + error="At least 2 distinct modules required for a combo image.", + ) + + combo_key = self._combo_key(names) + image_tag = self._combo_image_tag(names) + + try: + # Idempotent: skip if combo already exists + if not force and self._image_exists(image_tag): + existing = self._combo_registry.get(combo_key) + if existing is not None: + return ComboImageResult( + success=True, + image=existing, + skipped=True, + ) + + # Auto-build missing modules from catalog + modules_built: list[str] = [] + for name in names: + if name in self._registry: + continue + entry = CATALOG.get(name) + if entry is None: + return ComboImageResult( + success=False, + error=( + f"Module '{name}' is not in the OOT registry and " + f"not found in the catalog. Build it manually first " + f"with build_module()." + ), + ) + # Auto-build from catalog + logger.info("Auto-building '%s' from catalog for combo image", name) + result = self.build_module( + git_url=entry.git_url, + branch=entry.branch, + build_deps=entry.build_deps or None, + cmake_args=entry.cmake_args or None, + ) + if not result.success: + return ComboImageResult( + success=False, + error=f"Auto-build of '{name}' failed: {result.error}", + modules_built=modules_built, + ) + modules_built.append(name) + + # Generate and build combo + dockerfile = self.generate_combo_dockerfile(names) + log_lines = self._docker_build(dockerfile, image_tag) + build_log_tail = "\n".join(log_lines[-30:]) + + # Collect module infos for the combo record + module_infos = [self._registry[n] for n in names] + + info = ComboImageInfo( + combo_key=combo_key, + image_tag=image_tag, + modules=module_infos, + built_at=datetime.now(timezone.utc).isoformat(), + ) + self._combo_registry[combo_key] = info + self._save_combo_registry() + + return ComboImageResult( + success=True, + image=info, + build_log_tail=build_log_tail, + modules_built=modules_built, + ) + + except Exception as e: + logger.exception("Combo image build failed") + return ComboImageResult( + success=False, + error=str(e), + ) + + def list_combo_images(self) -> list[ComboImageInfo]: + """List all combined multi-OOT images.""" + return list(self._combo_registry.values()) + + def remove_combo_image(self, combo_key: str) -> bool: + """Remove a combo image by its key (e.g., 'combo:adsb+lora_sdr').""" + info = self._combo_registry.pop(combo_key, None) + if info is None: + return False + + try: + self._client.images.remove(info.image_tag, force=True) + except Exception as e: + logger.warning( + "Failed to remove combo Docker image %s: %s", info.image_tag, e + ) + + self._save_combo_registry() + return True + + # ────────────────────────────────────────── + # Combo Registry Persistence + # ────────────────────────────────────────── + + def _load_combo_registry(self) -> dict[str, ComboImageInfo]: + """Load the combo image registry from disk.""" + if not self._combo_registry_path.exists(): + return {} + try: + data = json.loads(self._combo_registry_path.read_text()) + return {k: ComboImageInfo(**v) for k, v in data.items()} + except Exception as e: + logger.warning("Failed to load combo registry: %s", e) + return {} + + def _save_combo_registry(self) -> None: + """Persist the combo image registry to disk.""" + self._combo_registry_path.parent.mkdir(parents=True, exist_ok=True) + data = {k: v.model_dump() for k, v in self._combo_registry.items()} + self._combo_registry_path.write_text(json.dumps(data, indent=2)) diff --git a/src/gnuradio_mcp/models.py b/src/gnuradio_mcp/models.py index 991b772..d6c412e 100644 --- a/src/gnuradio_mcp/models.py +++ b/src/gnuradio_mcp/models.py @@ -391,3 +391,23 @@ class OOTInstallResult(BaseModel): build_log_tail: str = "" # Last ~30 lines of build output error: str | None = None skipped: bool = False # True if image already existed + + +class ComboImageInfo(BaseModel): + """Metadata for a combined multi-OOT image.""" + + combo_key: str # "combo:adsb+lora_sdr" (sorted, deduped) + image_tag: str # "gr-combo-adsb-lora_sdr:latest" + modules: list[OOTImageInfo] # source images used + built_at: str # ISO-8601 + + +class ComboImageResult(BaseModel): + """Result of combo image build.""" + + success: bool + image: ComboImageInfo | None = None + build_log_tail: str = "" + error: str | None = None + skipped: bool = False # True if combo image already existed + modules_built: list[str] = [] # modules auto-built from catalog first diff --git a/src/gnuradio_mcp/providers/mcp_runtime.py b/src/gnuradio_mcp/providers/mcp_runtime.py index 4d09e1e..7c03c9c 100644 --- a/src/gnuradio_mcp/providers/mcp_runtime.py +++ b/src/gnuradio_mcp/providers/mcp_runtime.py @@ -78,7 +78,11 @@ class McpRuntimeProvider: self._mcp.tool(p.install_oot_module) self._mcp.tool(p.list_oot_images) self._mcp.tool(p.remove_oot_image) - logger.info("Registered 32 runtime tools (Docker + OOT available)") + # Multi-OOT combo images + self._mcp.tool(p.build_multi_oot_image) + self._mcp.tool(p.list_combo_images) + self._mcp.tool(p.remove_combo_image) + logger.info("Registered 35 runtime tools (Docker + OOT available)") else: logger.info("Registered 29 runtime tools (Docker available)") else: diff --git a/src/gnuradio_mcp/providers/runtime.py b/src/gnuradio_mcp/providers/runtime.py index 8036481..fb43857 100644 --- a/src/gnuradio_mcp/providers/runtime.py +++ b/src/gnuradio_mcp/providers/runtime.py @@ -12,6 +12,8 @@ from gnuradio_mcp.middlewares.oot import OOTInstallerMiddleware from gnuradio_mcp.middlewares.thrift import ThriftMiddleware from gnuradio_mcp.middlewares.xmlrpc import XmlRpcMiddleware from gnuradio_mcp.models import ( + ComboImageInfo, + ComboImageResult, ConnectionInfoModel, ContainerModel, CoverageDataModel, @@ -718,3 +720,33 @@ class RuntimeProvider: """Remove an OOT module image and its registry entry.""" oot = self._require_oot() return oot.remove_image(module_name) + + # ────────────────────────────────────────── + # Multi-OOT Combo Images + # ────────────────────────────────────────── + + def build_multi_oot_image( + self, + module_names: list[str], + force: bool = False, + ) -> ComboImageResult: + """Combine multiple OOT modules into a single Docker image. + + Modules are merged using multi-stage Docker builds from existing + single-OOT images. Missing modules that exist in the catalog + are auto-built first. + + Use the returned image_tag with launch_flowgraph(). + """ + oot = self._require_oot() + return oot.build_combo_image(module_names, force) + + def list_combo_images(self) -> list[ComboImageInfo]: + """List all combined multi-OOT images.""" + oot = self._require_oot() + return oot.list_combo_images() + + def remove_combo_image(self, combo_key: str) -> bool: + """Remove a combined image by its combo key (e.g., 'combo:adsb+lora_sdr').""" + oot = self._require_oot() + return oot.remove_combo_image(combo_key) diff --git a/tests/unit/test_oot_installer.py b/tests/unit/test_oot_installer.py index d2279f5..404198c 100644 --- a/tests/unit/test_oot_installer.py +++ b/tests/unit/test_oot_installer.py @@ -11,7 +11,7 @@ from unittest.mock import MagicMock import pytest from gnuradio_mcp.middlewares.oot import OOTInstallerMiddleware -from gnuradio_mcp.models import OOTImageInfo +from gnuradio_mcp.models import ComboImageInfo, OOTImageInfo @pytest.fixture @@ -22,9 +22,11 @@ def mock_docker_client(): @pytest.fixture def oot(mock_docker_client, tmp_path): mw = OOTInstallerMiddleware(mock_docker_client) - # Override registry path to use tmp_path + # Override registry paths to use tmp_path mw._registry_path = tmp_path / "oot-registry.json" mw._registry = {} + mw._combo_registry_path = tmp_path / "oot-combo-registry.json" + mw._combo_registry = {} return mw @@ -253,3 +255,302 @@ class TestRemoveImage: result = oot.remove_image("lora_sdr") assert result is True assert "lora_sdr" not in oot._registry + + +# ────────────────────────────────────────── +# Combo Key Generation +# ────────────────────────────────────────── + + +class TestComboKeyGeneration: + def test_sorted_and_deduped(self): + key = OOTInstallerMiddleware._combo_key(["lora_sdr", "adsb", "lora_sdr"]) + assert key == "combo:adsb+lora_sdr" + + def test_alphabetical_order(self): + key = OOTInstallerMiddleware._combo_key(["osmosdr", "adsb", "lora_sdr"]) + assert key == "combo:adsb+lora_sdr+osmosdr" + + def test_single_module(self): + key = OOTInstallerMiddleware._combo_key(["adsb"]) + assert key == "combo:adsb" + + def test_two_modules(self): + key = OOTInstallerMiddleware._combo_key(["lora_sdr", "adsb"]) + assert key == "combo:adsb+lora_sdr" + + +# ────────────────────────────────────────── +# Combo Image Tag +# ────────────────────────────────────────── + + +class TestComboImageTag: + def test_format(self): + tag = OOTInstallerMiddleware._combo_image_tag(["lora_sdr", "adsb"]) + assert tag == "gr-combo-adsb-lora_sdr:latest" + + def test_sorted_and_deduped(self): + tag = OOTInstallerMiddleware._combo_image_tag( + ["osmosdr", "adsb", "osmosdr"] + ) + assert tag == "gr-combo-adsb-osmosdr:latest" + + def test_three_modules(self): + tag = OOTInstallerMiddleware._combo_image_tag( + ["lora_sdr", "adsb", "osmosdr"] + ) + assert tag == "gr-combo-adsb-lora_sdr-osmosdr:latest" + + +# ────────────────────────────────────────── +# Combo Dockerfile Generation +# ────────────────────────────────────────── + + +def _make_oot_info(name: str, tag: str) -> OOTImageInfo: + """Helper to create a minimal OOTImageInfo for testing.""" + return OOTImageInfo( + module_name=name, + image_tag=tag, + git_url=f"https://example.com/gr-{name}.git", + branch="main", + git_commit="abc1234", + base_image="gnuradio-runtime:latest", + built_at="2025-01-01T00:00:00+00:00", + ) + + +class TestComboDockerfileGeneration: + def test_multi_stage_structure(self, oot): + oot._registry["adsb"] = _make_oot_info("adsb", "gr-oot-adsb:main-abc1234") + oot._registry["lora_sdr"] = _make_oot_info( + "lora_sdr", "gr-oot-lora_sdr:master-def5678" + ) + + dockerfile = oot.generate_combo_dockerfile(["lora_sdr", "adsb"]) + + # Stage aliases (sorted order: adsb first) + assert "FROM gr-oot-adsb:main-abc1234 AS stage_adsb" in dockerfile + assert "FROM gr-oot-lora_sdr:master-def5678 AS stage_lora_sdr" in dockerfile + + # Final base image + assert "FROM gnuradio-runtime:latest" in dockerfile + + # COPY directives for both modules + assert "COPY --from=stage_adsb /usr/lib/ /usr/lib/" in dockerfile + assert "COPY --from=stage_adsb /usr/include/ /usr/include/" in dockerfile + assert "COPY --from=stage_adsb /usr/share/gnuradio/ /usr/share/gnuradio/" in dockerfile + assert "COPY --from=stage_lora_sdr /usr/lib/ /usr/lib/" in dockerfile + assert "COPY --from=stage_lora_sdr /usr/include/ /usr/include/" in dockerfile + + # Runtime setup + assert "RUN ldconfig" in dockerfile + assert "WORKDIR /flowgraphs" in dockerfile + assert "PYTHONPATH" in dockerfile + + def test_missing_module_raises(self, oot): + oot._registry["adsb"] = _make_oot_info("adsb", "gr-oot-adsb:main-abc1234") + + with pytest.raises(ValueError, match="lora_sdr"): + oot.generate_combo_dockerfile(["adsb", "lora_sdr"]) + + def test_uses_configured_base_image(self, mock_docker_client, tmp_path): + mw = OOTInstallerMiddleware(mock_docker_client, base_image="my-custom:v2") + mw._registry_path = tmp_path / "oot-registry.json" + mw._registry = { + "adsb": _make_oot_info("adsb", "gr-oot-adsb:main-abc1234"), + "lora_sdr": _make_oot_info("lora_sdr", "gr-oot-lora_sdr:main-def5678"), + } + mw._combo_registry_path = tmp_path / "oot-combo-registry.json" + mw._combo_registry = {} + + dockerfile = mw.generate_combo_dockerfile(["adsb", "lora_sdr"]) + assert "FROM my-custom:v2" in dockerfile + + +# ────────────────────────────────────────── +# Combo Registry Persistence +# ────────────────────────────────────────── + + +class TestComboRegistry: + def test_separate_file(self, oot): + """Combo registry uses a different file from single-OOT registry.""" + assert oot._combo_registry_path != oot._registry_path + assert "combo" in str(oot._combo_registry_path) + + def test_empty_on_fresh_start(self, oot): + assert oot._combo_registry == {} + assert oot.list_combo_images() == [] + + def test_save_and_load_roundtrip(self, oot): + info = ComboImageInfo( + combo_key="combo:adsb+lora_sdr", + image_tag="gr-combo-adsb-lora_sdr:latest", + modules=[ + _make_oot_info("adsb", "gr-oot-adsb:main-abc1234"), + _make_oot_info("lora_sdr", "gr-oot-lora_sdr:master-def5678"), + ], + built_at="2025-01-01T00:00:00+00:00", + ) + oot._combo_registry["combo:adsb+lora_sdr"] = info + oot._save_combo_registry() + + loaded = oot._load_combo_registry() + assert "combo:adsb+lora_sdr" in loaded + assert loaded["combo:adsb+lora_sdr"].image_tag == "gr-combo-adsb-lora_sdr:latest" + assert len(loaded["combo:adsb+lora_sdr"].modules) == 2 + + def test_load_missing_file_returns_empty(self, oot): + oot._combo_registry_path = oot._combo_registry_path.parent / "nope" / "r.json" + result = oot._load_combo_registry() + assert result == {} + + def test_load_corrupt_file_returns_empty(self, oot): + oot._combo_registry_path.write_text("broken{{{") + result = oot._load_combo_registry() + assert result == {} + + def test_list_returns_values(self, oot): + info = ComboImageInfo( + combo_key="combo:adsb+lora_sdr", + image_tag="gr-combo-adsb-lora_sdr:latest", + modules=[], + built_at="2025-01-01T00:00:00+00:00", + ) + oot._combo_registry["combo:adsb+lora_sdr"] = info + result = oot.list_combo_images() + assert len(result) == 1 + assert result[0].combo_key == "combo:adsb+lora_sdr" + + def test_remove_existing(self, oot, mock_docker_client): + info = ComboImageInfo( + combo_key="combo:adsb+lora_sdr", + image_tag="gr-combo-adsb-lora_sdr:latest", + modules=[], + built_at="2025-01-01T00:00:00+00:00", + ) + oot._combo_registry["combo:adsb+lora_sdr"] = info + + result = oot.remove_combo_image("combo:adsb+lora_sdr") + assert result is True + assert "combo:adsb+lora_sdr" not in oot._combo_registry + mock_docker_client.images.remove.assert_called_once_with( + "gr-combo-adsb-lora_sdr:latest", force=True + ) + + def test_remove_nonexistent(self, oot): + result = oot.remove_combo_image("combo:nope+nada") + assert result is False + + def test_remove_survives_docker_error(self, oot, mock_docker_client): + info = ComboImageInfo( + combo_key="combo:adsb+lora_sdr", + image_tag="gr-combo-adsb-lora_sdr:latest", + modules=[], + built_at="2025-01-01T00:00:00+00:00", + ) + oot._combo_registry["combo:adsb+lora_sdr"] = info + mock_docker_client.images.remove.side_effect = Exception("gone") + + result = oot.remove_combo_image("combo:adsb+lora_sdr") + assert result is True + assert "combo:adsb+lora_sdr" not in oot._combo_registry + + +# ────────────────────────────────────────── +# Build Combo Image +# ────────────────────────────────────────── + + +class TestBuildComboImage: + def test_requires_at_least_two_modules(self, oot): + result = oot.build_combo_image(["adsb"]) + assert result.success is False + assert "2 distinct" in result.error + + def test_rejects_duplicate_as_single(self, oot): + result = oot.build_combo_image(["adsb", "adsb"]) + assert result.success is False + assert "2 distinct" in result.error + + def test_idempotent_skip(self, oot, mock_docker_client): + """Skips build if combo image already exists.""" + oot._registry["adsb"] = _make_oot_info("adsb", "gr-oot-adsb:main-abc") + oot._registry["lora_sdr"] = _make_oot_info("lora_sdr", "gr-oot-lora:m-def") + + existing = ComboImageInfo( + combo_key="combo:adsb+lora_sdr", + image_tag="gr-combo-adsb-lora_sdr:latest", + modules=[], + built_at="2025-01-01T00:00:00+00:00", + ) + oot._combo_registry["combo:adsb+lora_sdr"] = existing + + # Docker image exists + mock_docker_client.images.get.return_value = MagicMock() + + result = oot.build_combo_image(["lora_sdr", "adsb"]) + assert result.success is True + assert result.skipped is True + + def test_happy_path(self, oot, mock_docker_client): + """Builds combo from pre-existing single-OOT images.""" + oot._registry["adsb"] = _make_oot_info("adsb", "gr-oot-adsb:main-abc1234") + oot._registry["lora_sdr"] = _make_oot_info( + "lora_sdr", "gr-oot-lora_sdr:master-def5678" + ) + + # Docker image does not exist yet + mock_docker_client.images.get.side_effect = Exception("not found") + # Mock successful build + mock_docker_client.images.build.return_value = ( + MagicMock(), + [{"stream": "Step 1/5 : FROM ...\n"}], + ) + + result = oot.build_combo_image(["adsb", "lora_sdr"]) + assert result.success is True + assert result.skipped is False + assert result.image is not None + assert result.image.combo_key == "combo:adsb+lora_sdr" + assert result.image.image_tag == "gr-combo-adsb-lora_sdr:latest" + assert len(result.image.modules) == 2 + assert result.modules_built == [] + + # Verify persisted to combo registry + assert "combo:adsb+lora_sdr" in oot._combo_registry + + def test_unknown_module_not_in_catalog(self, oot): + """Fails if module not in registry and not in catalog.""" + oot._registry["adsb"] = _make_oot_info("adsb", "gr-oot-adsb:main-abc1234") + + result = oot.build_combo_image(["adsb", "totally_fake_module"]) + assert result.success is False + assert "totally_fake_module" in result.error + assert "not found in the catalog" in result.error + + def test_force_rebuilds(self, oot, mock_docker_client): + """force=True bypasses idempotency check.""" + oot._registry["adsb"] = _make_oot_info("adsb", "gr-oot-adsb:main-abc") + oot._registry["lora_sdr"] = _make_oot_info("lora_sdr", "gr-oot-lora:m-def") + + existing = ComboImageInfo( + combo_key="combo:adsb+lora_sdr", + image_tag="gr-combo-adsb-lora_sdr:latest", + modules=[], + built_at="2025-01-01T00:00:00+00:00", + ) + oot._combo_registry["combo:adsb+lora_sdr"] = existing + + # Docker image exists + mock_docker_client.images.get.return_value = MagicMock() + mock_docker_client.images.build.return_value = ( + MagicMock(), + [{"stream": "rebuilt\n"}], + ) + + result = oot.build_combo_image(["adsb", "lora_sdr"], force=True) + assert result.success is True + assert result.skipped is False