feat: multi-OOT combo images via multi-stage Docker builds

Combine multiple OOT modules into a single Docker image using
multi-stage COPY from existing single-OOT images. No recompilation
needed — fast and deterministic.

New MCP tools: build_multi_oot_image, list_combo_images, remove_combo_image

Also hardens _load_registry() to validate per-entry instead of
all-or-nothing, preventing one corrupt entry from discarding the
entire registry.
This commit is contained in:
Ryan Malloy 2026-02-01 22:45:14 -07:00
parent ca114fe2cb
commit 66f1b260f2
5 changed files with 559 additions and 10 deletions

View File

@ -9,7 +9,7 @@ from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from gnuradio_mcp.models import OOTImageInfo, OOTInstallResult
from gnuradio_mcp.models import ComboImageInfo, ComboImageResult, OOTImageInfo, OOTInstallResult
logger = logging.getLogger(__name__)
@ -101,6 +101,8 @@ class OOTInstallerMiddleware:
self._base_image = base_image
self._registry_path = Path.home() / ".gr-mcp" / "oot-registry.json"
self._registry: dict[str, OOTImageInfo] = self._load_registry()
self._combo_registry_path = Path.home() / ".gr-mcp" / "oot-combo-registry.json"
self._combo_registry: dict[str, ComboImageInfo] = self._load_combo_registry()
# ──────────────────────────────────────────
# Public API
@ -319,18 +321,25 @@ class OOTInstallerMiddleware:
return log_lines
def _load_registry(self) -> dict[str, OOTImageInfo]:
"""Load the OOT image registry from disk."""
"""Load the OOT image registry from disk.
Validates entries individually so one corrupted entry
doesn't discard the entire registry.
"""
if not self._registry_path.exists():
return {}
try:
data = json.loads(self._registry_path.read_text())
return {
k: OOTImageInfo(**v)
for k, v in data.items()
}
except Exception as e:
logger.warning("Failed to load OOT registry: %s", e)
logger.warning("Failed to parse OOT registry JSON: %s", e)
return {}
registry: dict[str, OOTImageInfo] = {}
for k, v in data.items():
try:
registry[k] = OOTImageInfo(**v)
except Exception as e:
logger.warning("Skipping corrupt registry entry '%s': %s", k, e)
return registry
def _save_registry(self) -> None:
"""Persist the OOT image registry to disk."""
@ -340,3 +349,186 @@ class OOTInstallerMiddleware:
for k, v in self._registry.items()
}
self._registry_path.write_text(json.dumps(data, indent=2))
# ──────────────────────────────────────────
# Combo Image (Multi-OOT) Support
# ──────────────────────────────────────────
@staticmethod
def _combo_key(module_names: list[str]) -> str:
"""Deterministic key from module names: sorted, deduped, joined."""
names = sorted(set(module_names))
return "combo:" + "+".join(names)
@staticmethod
def _combo_image_tag(module_names: list[str]) -> str:
"""Deterministic image tag for a combo of modules."""
names = sorted(set(module_names))
return f"gr-combo-{'-'.join(names)}:latest"
def generate_combo_dockerfile(self, module_names: list[str]) -> str:
"""Generate multi-stage Dockerfile that COPYs from existing single-OOT images."""
names = sorted(set(module_names))
stages: list[str] = []
copies: list[str] = []
for name in names:
info = self._registry.get(name)
if info is None:
raise ValueError(
f"Module '{name}' not found in OOT registry. "
f"Build it first with build_module()."
)
stage_alias = f"stage_{name}"
stages.append(f"FROM {info.image_tag} AS {stage_alias}")
for path in ["/usr/lib/", "/usr/include/", "/usr/share/gnuradio/"]:
copies.append(f"COPY --from={stage_alias} {path} {path}")
return "\n".join([
*stages,
"",
f"FROM {self._base_image}",
"",
*copies,
"",
"RUN ldconfig",
"WORKDIR /flowgraphs",
'ENV PYTHONPATH="/usr/lib/python3.11/site-packages:${PYTHONPATH}"',
"",
])
def build_combo_image(
self,
module_names: list[str],
force: bool = False,
) -> ComboImageResult:
"""Build a combined Docker image with multiple OOT modules.
Modules already in the registry are used as-is. Modules found
in the OOT catalog but not yet built are auto-built first.
"""
from gnuradio_mcp.oot_catalog import CATALOG
names = sorted(set(module_names))
if len(names) < 2:
return ComboImageResult(
success=False,
error="At least 2 distinct modules required for a combo image.",
)
combo_key = self._combo_key(names)
image_tag = self._combo_image_tag(names)
try:
# Idempotent: skip if combo already exists
if not force and self._image_exists(image_tag):
existing = self._combo_registry.get(combo_key)
if existing is not None:
return ComboImageResult(
success=True,
image=existing,
skipped=True,
)
# Auto-build missing modules from catalog
modules_built: list[str] = []
for name in names:
if name in self._registry:
continue
entry = CATALOG.get(name)
if entry is None:
return ComboImageResult(
success=False,
error=(
f"Module '{name}' is not in the OOT registry and "
f"not found in the catalog. Build it manually first "
f"with build_module()."
),
)
# Auto-build from catalog
logger.info("Auto-building '%s' from catalog for combo image", name)
result = self.build_module(
git_url=entry.git_url,
branch=entry.branch,
build_deps=entry.build_deps or None,
cmake_args=entry.cmake_args or None,
)
if not result.success:
return ComboImageResult(
success=False,
error=f"Auto-build of '{name}' failed: {result.error}",
modules_built=modules_built,
)
modules_built.append(name)
# Generate and build combo
dockerfile = self.generate_combo_dockerfile(names)
log_lines = self._docker_build(dockerfile, image_tag)
build_log_tail = "\n".join(log_lines[-30:])
# Collect module infos for the combo record
module_infos = [self._registry[n] for n in names]
info = ComboImageInfo(
combo_key=combo_key,
image_tag=image_tag,
modules=module_infos,
built_at=datetime.now(timezone.utc).isoformat(),
)
self._combo_registry[combo_key] = info
self._save_combo_registry()
return ComboImageResult(
success=True,
image=info,
build_log_tail=build_log_tail,
modules_built=modules_built,
)
except Exception as e:
logger.exception("Combo image build failed")
return ComboImageResult(
success=False,
error=str(e),
)
def list_combo_images(self) -> list[ComboImageInfo]:
"""List all combined multi-OOT images."""
return list(self._combo_registry.values())
def remove_combo_image(self, combo_key: str) -> bool:
"""Remove a combo image by its key (e.g., 'combo:adsb+lora_sdr')."""
info = self._combo_registry.pop(combo_key, None)
if info is None:
return False
try:
self._client.images.remove(info.image_tag, force=True)
except Exception as e:
logger.warning(
"Failed to remove combo Docker image %s: %s", info.image_tag, e
)
self._save_combo_registry()
return True
# ──────────────────────────────────────────
# Combo Registry Persistence
# ──────────────────────────────────────────
def _load_combo_registry(self) -> dict[str, ComboImageInfo]:
"""Load the combo image registry from disk."""
if not self._combo_registry_path.exists():
return {}
try:
data = json.loads(self._combo_registry_path.read_text())
return {k: ComboImageInfo(**v) for k, v in data.items()}
except Exception as e:
logger.warning("Failed to load combo registry: %s", e)
return {}
def _save_combo_registry(self) -> None:
"""Persist the combo image registry to disk."""
self._combo_registry_path.parent.mkdir(parents=True, exist_ok=True)
data = {k: v.model_dump() for k, v in self._combo_registry.items()}
self._combo_registry_path.write_text(json.dumps(data, indent=2))

View File

@ -391,3 +391,23 @@ class OOTInstallResult(BaseModel):
build_log_tail: str = "" # Last ~30 lines of build output
error: str | None = None
skipped: bool = False # True if image already existed
class ComboImageInfo(BaseModel):
"""Metadata for a combined multi-OOT image."""
combo_key: str # "combo:adsb+lora_sdr" (sorted, deduped)
image_tag: str # "gr-combo-adsb-lora_sdr:latest"
modules: list[OOTImageInfo] # source images used
built_at: str # ISO-8601
class ComboImageResult(BaseModel):
"""Result of combo image build."""
success: bool
image: ComboImageInfo | None = None
build_log_tail: str = ""
error: str | None = None
skipped: bool = False # True if combo image already existed
modules_built: list[str] = [] # modules auto-built from catalog first

View File

@ -78,7 +78,11 @@ class McpRuntimeProvider:
self._mcp.tool(p.install_oot_module)
self._mcp.tool(p.list_oot_images)
self._mcp.tool(p.remove_oot_image)
logger.info("Registered 32 runtime tools (Docker + OOT available)")
# Multi-OOT combo images
self._mcp.tool(p.build_multi_oot_image)
self._mcp.tool(p.list_combo_images)
self._mcp.tool(p.remove_combo_image)
logger.info("Registered 35 runtime tools (Docker + OOT available)")
else:
logger.info("Registered 29 runtime tools (Docker available)")
else:

View File

@ -12,6 +12,8 @@ from gnuradio_mcp.middlewares.oot import OOTInstallerMiddleware
from gnuradio_mcp.middlewares.thrift import ThriftMiddleware
from gnuradio_mcp.middlewares.xmlrpc import XmlRpcMiddleware
from gnuradio_mcp.models import (
ComboImageInfo,
ComboImageResult,
ConnectionInfoModel,
ContainerModel,
CoverageDataModel,
@ -718,3 +720,33 @@ class RuntimeProvider:
"""Remove an OOT module image and its registry entry."""
oot = self._require_oot()
return oot.remove_image(module_name)
# ──────────────────────────────────────────
# Multi-OOT Combo Images
# ──────────────────────────────────────────
def build_multi_oot_image(
self,
module_names: list[str],
force: bool = False,
) -> ComboImageResult:
"""Combine multiple OOT modules into a single Docker image.
Modules are merged using multi-stage Docker builds from existing
single-OOT images. Missing modules that exist in the catalog
are auto-built first.
Use the returned image_tag with launch_flowgraph().
"""
oot = self._require_oot()
return oot.build_combo_image(module_names, force)
def list_combo_images(self) -> list[ComboImageInfo]:
"""List all combined multi-OOT images."""
oot = self._require_oot()
return oot.list_combo_images()
def remove_combo_image(self, combo_key: str) -> bool:
"""Remove a combined image by its combo key (e.g., 'combo:adsb+lora_sdr')."""
oot = self._require_oot()
return oot.remove_combo_image(combo_key)

View File

@ -11,7 +11,7 @@ from unittest.mock import MagicMock
import pytest
from gnuradio_mcp.middlewares.oot import OOTInstallerMiddleware
from gnuradio_mcp.models import OOTImageInfo
from gnuradio_mcp.models import ComboImageInfo, OOTImageInfo
@pytest.fixture
@ -22,9 +22,11 @@ def mock_docker_client():
@pytest.fixture
def oot(mock_docker_client, tmp_path):
mw = OOTInstallerMiddleware(mock_docker_client)
# Override registry path to use tmp_path
# Override registry paths to use tmp_path
mw._registry_path = tmp_path / "oot-registry.json"
mw._registry = {}
mw._combo_registry_path = tmp_path / "oot-combo-registry.json"
mw._combo_registry = {}
return mw
@ -253,3 +255,302 @@ class TestRemoveImage:
result = oot.remove_image("lora_sdr")
assert result is True
assert "lora_sdr" not in oot._registry
# ──────────────────────────────────────────
# Combo Key Generation
# ──────────────────────────────────────────
class TestComboKeyGeneration:
def test_sorted_and_deduped(self):
key = OOTInstallerMiddleware._combo_key(["lora_sdr", "adsb", "lora_sdr"])
assert key == "combo:adsb+lora_sdr"
def test_alphabetical_order(self):
key = OOTInstallerMiddleware._combo_key(["osmosdr", "adsb", "lora_sdr"])
assert key == "combo:adsb+lora_sdr+osmosdr"
def test_single_module(self):
key = OOTInstallerMiddleware._combo_key(["adsb"])
assert key == "combo:adsb"
def test_two_modules(self):
key = OOTInstallerMiddleware._combo_key(["lora_sdr", "adsb"])
assert key == "combo:adsb+lora_sdr"
# ──────────────────────────────────────────
# Combo Image Tag
# ──────────────────────────────────────────
class TestComboImageTag:
def test_format(self):
tag = OOTInstallerMiddleware._combo_image_tag(["lora_sdr", "adsb"])
assert tag == "gr-combo-adsb-lora_sdr:latest"
def test_sorted_and_deduped(self):
tag = OOTInstallerMiddleware._combo_image_tag(
["osmosdr", "adsb", "osmosdr"]
)
assert tag == "gr-combo-adsb-osmosdr:latest"
def test_three_modules(self):
tag = OOTInstallerMiddleware._combo_image_tag(
["lora_sdr", "adsb", "osmosdr"]
)
assert tag == "gr-combo-adsb-lora_sdr-osmosdr:latest"
# ──────────────────────────────────────────
# Combo Dockerfile Generation
# ──────────────────────────────────────────
def _make_oot_info(name: str, tag: str) -> OOTImageInfo:
"""Helper to create a minimal OOTImageInfo for testing."""
return OOTImageInfo(
module_name=name,
image_tag=tag,
git_url=f"https://example.com/gr-{name}.git",
branch="main",
git_commit="abc1234",
base_image="gnuradio-runtime:latest",
built_at="2025-01-01T00:00:00+00:00",
)
class TestComboDockerfileGeneration:
def test_multi_stage_structure(self, oot):
oot._registry["adsb"] = _make_oot_info("adsb", "gr-oot-adsb:main-abc1234")
oot._registry["lora_sdr"] = _make_oot_info(
"lora_sdr", "gr-oot-lora_sdr:master-def5678"
)
dockerfile = oot.generate_combo_dockerfile(["lora_sdr", "adsb"])
# Stage aliases (sorted order: adsb first)
assert "FROM gr-oot-adsb:main-abc1234 AS stage_adsb" in dockerfile
assert "FROM gr-oot-lora_sdr:master-def5678 AS stage_lora_sdr" in dockerfile
# Final base image
assert "FROM gnuradio-runtime:latest" in dockerfile
# COPY directives for both modules
assert "COPY --from=stage_adsb /usr/lib/ /usr/lib/" in dockerfile
assert "COPY --from=stage_adsb /usr/include/ /usr/include/" in dockerfile
assert "COPY --from=stage_adsb /usr/share/gnuradio/ /usr/share/gnuradio/" in dockerfile
assert "COPY --from=stage_lora_sdr /usr/lib/ /usr/lib/" in dockerfile
assert "COPY --from=stage_lora_sdr /usr/include/ /usr/include/" in dockerfile
# Runtime setup
assert "RUN ldconfig" in dockerfile
assert "WORKDIR /flowgraphs" in dockerfile
assert "PYTHONPATH" in dockerfile
def test_missing_module_raises(self, oot):
oot._registry["adsb"] = _make_oot_info("adsb", "gr-oot-adsb:main-abc1234")
with pytest.raises(ValueError, match="lora_sdr"):
oot.generate_combo_dockerfile(["adsb", "lora_sdr"])
def test_uses_configured_base_image(self, mock_docker_client, tmp_path):
mw = OOTInstallerMiddleware(mock_docker_client, base_image="my-custom:v2")
mw._registry_path = tmp_path / "oot-registry.json"
mw._registry = {
"adsb": _make_oot_info("adsb", "gr-oot-adsb:main-abc1234"),
"lora_sdr": _make_oot_info("lora_sdr", "gr-oot-lora_sdr:main-def5678"),
}
mw._combo_registry_path = tmp_path / "oot-combo-registry.json"
mw._combo_registry = {}
dockerfile = mw.generate_combo_dockerfile(["adsb", "lora_sdr"])
assert "FROM my-custom:v2" in dockerfile
# ──────────────────────────────────────────
# Combo Registry Persistence
# ──────────────────────────────────────────
class TestComboRegistry:
def test_separate_file(self, oot):
"""Combo registry uses a different file from single-OOT registry."""
assert oot._combo_registry_path != oot._registry_path
assert "combo" in str(oot._combo_registry_path)
def test_empty_on_fresh_start(self, oot):
assert oot._combo_registry == {}
assert oot.list_combo_images() == []
def test_save_and_load_roundtrip(self, oot):
info = ComboImageInfo(
combo_key="combo:adsb+lora_sdr",
image_tag="gr-combo-adsb-lora_sdr:latest",
modules=[
_make_oot_info("adsb", "gr-oot-adsb:main-abc1234"),
_make_oot_info("lora_sdr", "gr-oot-lora_sdr:master-def5678"),
],
built_at="2025-01-01T00:00:00+00:00",
)
oot._combo_registry["combo:adsb+lora_sdr"] = info
oot._save_combo_registry()
loaded = oot._load_combo_registry()
assert "combo:adsb+lora_sdr" in loaded
assert loaded["combo:adsb+lora_sdr"].image_tag == "gr-combo-adsb-lora_sdr:latest"
assert len(loaded["combo:adsb+lora_sdr"].modules) == 2
def test_load_missing_file_returns_empty(self, oot):
oot._combo_registry_path = oot._combo_registry_path.parent / "nope" / "r.json"
result = oot._load_combo_registry()
assert result == {}
def test_load_corrupt_file_returns_empty(self, oot):
oot._combo_registry_path.write_text("broken{{{")
result = oot._load_combo_registry()
assert result == {}
def test_list_returns_values(self, oot):
info = ComboImageInfo(
combo_key="combo:adsb+lora_sdr",
image_tag="gr-combo-adsb-lora_sdr:latest",
modules=[],
built_at="2025-01-01T00:00:00+00:00",
)
oot._combo_registry["combo:adsb+lora_sdr"] = info
result = oot.list_combo_images()
assert len(result) == 1
assert result[0].combo_key == "combo:adsb+lora_sdr"
def test_remove_existing(self, oot, mock_docker_client):
info = ComboImageInfo(
combo_key="combo:adsb+lora_sdr",
image_tag="gr-combo-adsb-lora_sdr:latest",
modules=[],
built_at="2025-01-01T00:00:00+00:00",
)
oot._combo_registry["combo:adsb+lora_sdr"] = info
result = oot.remove_combo_image("combo:adsb+lora_sdr")
assert result is True
assert "combo:adsb+lora_sdr" not in oot._combo_registry
mock_docker_client.images.remove.assert_called_once_with(
"gr-combo-adsb-lora_sdr:latest", force=True
)
def test_remove_nonexistent(self, oot):
result = oot.remove_combo_image("combo:nope+nada")
assert result is False
def test_remove_survives_docker_error(self, oot, mock_docker_client):
info = ComboImageInfo(
combo_key="combo:adsb+lora_sdr",
image_tag="gr-combo-adsb-lora_sdr:latest",
modules=[],
built_at="2025-01-01T00:00:00+00:00",
)
oot._combo_registry["combo:adsb+lora_sdr"] = info
mock_docker_client.images.remove.side_effect = Exception("gone")
result = oot.remove_combo_image("combo:adsb+lora_sdr")
assert result is True
assert "combo:adsb+lora_sdr" not in oot._combo_registry
# ──────────────────────────────────────────
# Build Combo Image
# ──────────────────────────────────────────
class TestBuildComboImage:
def test_requires_at_least_two_modules(self, oot):
result = oot.build_combo_image(["adsb"])
assert result.success is False
assert "2 distinct" in result.error
def test_rejects_duplicate_as_single(self, oot):
result = oot.build_combo_image(["adsb", "adsb"])
assert result.success is False
assert "2 distinct" in result.error
def test_idempotent_skip(self, oot, mock_docker_client):
"""Skips build if combo image already exists."""
oot._registry["adsb"] = _make_oot_info("adsb", "gr-oot-adsb:main-abc")
oot._registry["lora_sdr"] = _make_oot_info("lora_sdr", "gr-oot-lora:m-def")
existing = ComboImageInfo(
combo_key="combo:adsb+lora_sdr",
image_tag="gr-combo-adsb-lora_sdr:latest",
modules=[],
built_at="2025-01-01T00:00:00+00:00",
)
oot._combo_registry["combo:adsb+lora_sdr"] = existing
# Docker image exists
mock_docker_client.images.get.return_value = MagicMock()
result = oot.build_combo_image(["lora_sdr", "adsb"])
assert result.success is True
assert result.skipped is True
def test_happy_path(self, oot, mock_docker_client):
"""Builds combo from pre-existing single-OOT images."""
oot._registry["adsb"] = _make_oot_info("adsb", "gr-oot-adsb:main-abc1234")
oot._registry["lora_sdr"] = _make_oot_info(
"lora_sdr", "gr-oot-lora_sdr:master-def5678"
)
# Docker image does not exist yet
mock_docker_client.images.get.side_effect = Exception("not found")
# Mock successful build
mock_docker_client.images.build.return_value = (
MagicMock(),
[{"stream": "Step 1/5 : FROM ...\n"}],
)
result = oot.build_combo_image(["adsb", "lora_sdr"])
assert result.success is True
assert result.skipped is False
assert result.image is not None
assert result.image.combo_key == "combo:adsb+lora_sdr"
assert result.image.image_tag == "gr-combo-adsb-lora_sdr:latest"
assert len(result.image.modules) == 2
assert result.modules_built == []
# Verify persisted to combo registry
assert "combo:adsb+lora_sdr" in oot._combo_registry
def test_unknown_module_not_in_catalog(self, oot):
"""Fails if module not in registry and not in catalog."""
oot._registry["adsb"] = _make_oot_info("adsb", "gr-oot-adsb:main-abc1234")
result = oot.build_combo_image(["adsb", "totally_fake_module"])
assert result.success is False
assert "totally_fake_module" in result.error
assert "not found in the catalog" in result.error
def test_force_rebuilds(self, oot, mock_docker_client):
"""force=True bypasses idempotency check."""
oot._registry["adsb"] = _make_oot_info("adsb", "gr-oot-adsb:main-abc")
oot._registry["lora_sdr"] = _make_oot_info("lora_sdr", "gr-oot-lora:m-def")
existing = ComboImageInfo(
combo_key="combo:adsb+lora_sdr",
image_tag="gr-combo-adsb-lora_sdr:latest",
modules=[],
built_at="2025-01-01T00:00:00+00:00",
)
oot._combo_registry["combo:adsb+lora_sdr"] = existing
# Docker image exists
mock_docker_client.images.get.return_value = MagicMock()
mock_docker_client.images.build.return_value = (
MagicMock(),
[{"stream": "rebuilt\n"}],
)
result = oot.build_combo_image(["adsb", "lora_sdr"], force=True)
assert result.success is True
assert result.skipped is False