Harden failure paths found in the reliability review

The success paths were fine; several failure paths drew a confident
conclusion without checking the thing they waited for.

Registry (was: any parse error killed the whole server, since load() runs
in the lifespan):
- quarantine an unreadable file and start empty instead of raising, so the
  tools that stop runaway VMs keep working when bookkeeping is damaged
- skip malformed or invalidly-named records rather than failing the load;
  report both through list_vms as registry_warnings
- read-modify-write under an exclusive flock so a second instance merges
  instead of clobbering, with a PID-unique temp file
- drop the lifespan shutdown save, which could resurrect deleted records
- version the schema and round-trip unknown record fields

sandbox_destroy (the only tool that deletes files):
- verify the process actually died, escalating to SIGKILL, and refuse to
  delete an overlay QEMU still holds open
- assert the target is inside the VM state tree before rmtree
- report cleanup errors instead of swallowing them; destroyed now reflects
  what happened

Launch races:
- reserve the name before the first await so two concurrent launches cannot
  race over one set of sockets
- refuse to unlink a QMP socket that is still accepting connections
- register the VM with a warning rather than orphaning it when the pidfile
  is unreadable but QEMU is up

Guest agent and QMP:
- bound every guest-agent call, not just the handshake; cap max_bytes and
  stop guest_file_read spinning on a zero-progress agent
- serialize QMP sessions per VM (the monitor is single-client) and say
  "another operation holds it" instead of "the VM has likely exited"
- poll liveness while waiting for SHUTDOWN so a crashed VM is reported as
  exited rather than as a guest ignoring ACPI
- default command timeout, with a longer bound for savevm/loadvm
- stricter snapshot tags; log destructive operations to stderr

Adds tests/test_reliability.py covering the conditions above.
This commit is contained in:
Ryan Malloy 2026-08-17 15:53:09 -06:00
parent 1dbe453731
commit f053762c4e
18 changed files with 996 additions and 175 deletions

View File

@ -10,7 +10,23 @@ def vm_not_found(name: str, known: list[str]) -> ToolError:
return ToolError(f"No VM named {name!r}. {hint} Use list_vms to see current state.")
def qmp_unreachable(name: str, socket_path: str, detail: str, qemu_log: str | None) -> ToolError:
def qmp_unreachable(
name: str,
socket_path: str,
detail: str,
qemu_log: str | None,
process_alive: bool = False,
) -> ToolError:
"""Connect failure. The advice differs sharply depending on whether the
process is still there, and guessing 'it exited' at a VM that is merely
busy sends the caller off to relaunch a VM that is still running."""
if process_alive:
return ToolError(
f"Could not open a QMP session to VM {name!r} at {socket_path} ({detail}), "
"but its process is still alive. QEMU's monitor accepts one client at a "
"time, so another tool call or an external qmp-shell may be holding it. "
"Do NOT relaunch the VM; retry in a moment."
)
log_hint = f" Check the QEMU log at {qemu_log} for crash details." if qemu_log else ""
return ToolError(
f"Cannot reach QMP socket for VM {name!r} at {socket_path} ({detail}). "

View File

@ -50,11 +50,33 @@ def pick_accel(arch: str) -> str:
def pick_free_port() -> int:
# The bind test closes before QEMU binds, so a racing process could still
# take the port; QEMU then fails to launch with a clear error. Accepted
# deliberately — holding the port open until exec is not possible here.
with socket.socket() as s:
s.bind(("", 0))
return s.getsockname()[1]
def socket_is_live(path: Path) -> bool:
"""True if something is still accepting connections on this unix socket.
A leftover socket *file* is normal after a crash; a socket that still
accepts means a process owns this VM's name and must not be trampled.
"""
if not path.exists():
return False
probe = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
try:
probe.settimeout(0.2)
probe.connect(str(path))
return True
except OSError:
return False
finally:
probe.close()
def resolve_port_forwards(forwards: list[str]) -> list[str]:
"""Turn user forward specs into concrete 'HOSTPORT:GUESTPORT' entries.
@ -196,7 +218,16 @@ async def spawn_vm(cfg: VMConfig, config: Config) -> VMRecord:
config.vm_runtime_dir(cfg.name).mkdir(parents=True, exist_ok=True)
config.vm_state_dir(cfg.name).mkdir(parents=True, exist_ok=True)
# Leftover sockets from a previous life of this name confuse liveness checks.
# Leftover sockets from a previous life of this name confuse liveness
# checks — but only delete them once we know nothing is listening. An
# unregistered-but-running QEMU (e.g. after forget_vm force=True) would
# otherwise lose its monitor socket and become unreachable forever.
if socket_is_live(paths.qmp_socket):
raise ToolError(
f"{paths.qmp_socket} is a LIVE QMP socket: an unregistered QEMU process is "
f"still using the name {cfg.name!r}. Re-register it with attach_vm and stop "
"it, or launch under a different name."
)
for stale in (paths.qmp_socket, paths.qga_socket, paths.pidfile):
stale.unlink(missing_ok=True)
@ -228,13 +259,23 @@ async def spawn_vm(cfg: VMConfig, config: Config) -> VMRecord:
detail = f"exit code {proc.returncode}"
raise ToolError(f"QEMU failed to launch VM {cfg.name!r}: {detail}")
try:
pid = int(paths.pidfile.read_text().strip())
except (OSError, ValueError) as e:
# QEMU exited 0, so the VM is up. If the pidfile is somehow unreadable we
# must still return a record: raising here would leave a running,
# unregistered VM that no tool could find or stop.
pid: int | None = None
for attempt in range(5):
try:
pid = int(paths.pidfile.read_text().strip())
break
except (OSError, ValueError):
if attempt < 4:
await asyncio.sleep(0.1)
if pid is None and not socket_is_live(paths.qmp_socket):
raise ToolError(
f"QEMU launched but pidfile {paths.pidfile} is unreadable ({e}) — "
"the VM state is unknown; check the QEMU log."
) from e
f"QEMU reported success for VM {cfg.name!r} but left neither a readable "
f"pidfile ({paths.pidfile}) nor a live QMP socket. Check {paths.qemu_log}; "
f'if a process is running, find it with: pgrep -af "qemu-system.*-name {cfg.name}"'
)
accel = pick_accel(cfg.arch)
return VMRecord(

View File

@ -41,11 +41,19 @@ class VMRecord:
qemu_log: str | None = None
serial_log: str | None = None
created_at: str | None = None
# Fields written by a newer mcqemu that this build doesn't know about.
# Carried through unchanged so an older instance can't strip them on save.
extra: dict[str, Any] = field(default_factory=dict)
def to_dict(self) -> dict[str, Any]:
return asdict(self)
data = asdict(self)
data.pop("extra", None)
data.update(self.extra)
return data
@classmethod
def from_dict(cls, data: dict[str, Any]) -> VMRecord:
known = {f for f in cls.__dataclass_fields__}
return cls(**{k: v for k, v in data.items() if k in known})
known = set(cls.__dataclass_fields__) - {"extra"}
fields = {k: v for k, v in data.items() if k in known}
extra = {k: v for k, v in data.items() if k not in known}
return cls(**fields, extra=extra)

View File

@ -4,65 +4,144 @@ QEMU's QMP unix socket accepts one client at a time, so every tool call
opens a short-lived session and disconnects the socket stays free for
qmp-shell or any other tooling between calls, and there is no stale
connection state to reconcile after either side restarts.
Because the socket is single-client, concurrent tool calls against the same
VM must not both try to hold it: the second would block until its connect
timeout and then be told the VM had exited. `_session_locks` serializes them
per VM name instead (a mutex table keyed by resource, not shared state).
"""
from __future__ import annotations
import asyncio
import contextlib
from collections import defaultdict
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from pathlib import Path
from typing import Any
from fastmcp.exceptions import ToolError
from qemu.qmp import ExecuteError, QMPClient
from .errors import qmp_command_failed, qmp_unreachable
from .models import VMRecord
from .registry import pid_alive
CONNECT_TIMEOUT = 5.0
COMMAND_TIMEOUT = 30.0
_session_locks: defaultdict[str, asyncio.Lock] = defaultdict(asyncio.Lock)
def _looks_alive(record: VMRecord) -> bool:
"""Best-effort liveness without the registry, for error messages only."""
if record.pid is not None:
return pid_alive(record.pid)
with contextlib.suppress(OSError):
return Path(record.qmp_socket).exists()
return False
@asynccontextmanager
async def qmp_session(record: VMRecord) -> AsyncIterator[QMPClient]:
client = QMPClient(record.name)
try:
async with _session_locks[record.name]:
client = QMPClient(record.name)
try:
await asyncio.wait_for(client.connect(record.qmp_socket), timeout=CONNECT_TIMEOUT)
except TimeoutError:
raise qmp_unreachable(
record.name, record.qmp_socket, "connect timed out", record.qemu_log
) from None
except OSError as e:
raise qmp_unreachable(record.name, record.qmp_socket, str(e), record.qemu_log) from e
except Exception as e: # qemu.qmp wraps failures in its own ConnectError
raise qmp_unreachable(record.name, record.qmp_socket, str(e), record.qemu_log) from e
yield client
finally:
# After a 'quit' the peer hangs up first; EOF on teardown is normal.
with contextlib.suppress(Exception):
await client.disconnect()
try:
await asyncio.wait_for(client.connect(record.qmp_socket), timeout=CONNECT_TIMEOUT)
except TimeoutError:
raise qmp_unreachable(
record.name,
record.qmp_socket,
"connect timed out",
record.qemu_log,
process_alive=_looks_alive(record),
) from None
except OSError as e:
raise qmp_unreachable(
record.name, record.qmp_socket, str(e), record.qemu_log
) from e
except Exception as e: # qemu.qmp wraps failures in its own ConnectError
raise qmp_unreachable(
record.name,
record.qmp_socket,
str(e),
record.qemu_log,
process_alive=_looks_alive(record),
) from e
yield client
finally:
# After a 'quit' the peer hangs up first; EOF on teardown is normal.
with contextlib.suppress(Exception):
await client.disconnect()
async def execute(client: QMPClient, command: str, arguments: dict[str, Any] | None = None) -> Any:
"""Run one QMP command, translating QMP-level errors to ToolError."""
async def execute(
client: QMPClient,
command: str,
arguments: dict[str, Any] | None = None,
timeout: float | None = COMMAND_TIMEOUT,
) -> Any:
"""Run one QMP command, translating QMP-level errors to ToolError.
Commands are bounded by default: a QEMU wedged inside a device-model
operation would otherwise hang the tool call (and this VM's session lock)
forever. Pass timeout=None for genuinely long operations like savevm.
"""
try:
return await client.execute(command, arguments or {})
call = client.execute(command, arguments or {})
return await (asyncio.wait_for(call, timeout) if timeout else call)
except ExecuteError as e:
error_class = getattr(e, "error_class", None) or "GenericError"
raise qmp_command_failed(command, error_class, str(e)) from e
async def wait_for_event(client: QMPClient, event_name: str, timeout: float) -> bool:
"""Drain the session's event stream until event_name arrives or time runs out."""
async def _drain() -> None:
while True:
event = await client.events.get()
if event.get("event") == event_name:
return
try:
await asyncio.wait_for(_drain(), timeout=timeout)
return True
except TimeoutError:
return False
raise ToolError(
f"QMP command {command!r} did not complete within {timeout}s. The VM may "
"be wedged or under heavy load; check list_vms and the QEMU log."
) from None
except Exception as e:
# 'quit' legitimately races the connection teardown: QEMU can drop the
# socket before acking. Anything else is a real transport failure.
if command == "quit" and type(e).__name__ in (
"ExecInterruptedError",
"StateError",
"EOFError",
):
return {}
raise
async def wait_for_event(
client: QMPClient,
event_name: str,
timeout: float,
record: VMRecord | None = None,
) -> str:
"""Wait for `event_name`, returning why the wait ended.
Returns "event", "process_exited", or "timeout". The liveness poll matters:
qemu.qmp's event queue never wakes on a dropped connection, so a VM that
crashes mid-wait would otherwise burn the caller's entire timeout and then
be misreported as a guest that ignores ACPI.
"""
loop = asyncio.get_event_loop()
deadline = loop.time() + timeout
while loop.time() < deadline:
try:
event = await asyncio.wait_for(client.events.get(), timeout=1.0)
except TimeoutError:
if record is not None and record.pid is not None and not pid_alive(record.pid):
return "process_exited"
continue
if event.get("event") == event_name:
return "event"
return "timeout"
def recent_events(client: QMPClient) -> list[str]:
"""Event names seen this session — context for a timeout error message."""
try:
return [e.get("event") for e in client.events.history if e.get("event")]
except Exception:
return []

View File

@ -1,15 +1,29 @@
"""Persistent VM registry with PID-liveness staleness detection."""
"""Persistent VM registry with PID-liveness staleness detection.
Two properties matter more than speed here. First, a damaged registry file
must never take the server down: VMs outlive this process, so the tools that
stop them have to keep working even when their bookkeeping is unreadable.
Second, several mcqemu instances may share one registry file (a second MCP
client, a stray `uvx mcqemu`), so every write is a read-modify-write under an
exclusive file lock otherwise the last writer silently orphans the other's
VMs.
"""
from __future__ import annotations
import contextlib
import fcntl
import json
import os
import time
from collections.abc import Callable, Iterator
from pathlib import Path
from .config import Config
from .config import Config, valid_vm_name
from .models import VMRecord
SCHEMA_VERSION = 1
def pid_alive(pid: int | None) -> bool:
"""True iff the PID exists AND is actually a qemu-system process.
@ -31,32 +45,132 @@ class VMRegistry:
def __init__(self, config: Config) -> None:
self._config = config
self._vms: dict[str, VMRecord] = {}
self._reserved: set[str] = set()
self.degraded: str | None = None
self.skipped: list[dict] = []
def load(self) -> None:
path = self._config.registry_path
if not path.exists():
self._vms = {}
return
data = json.loads(path.read_text())
self._vms = {name: VMRecord.from_dict(rec) for name, rec in data.items()}
# --- file primitives -------------------------------------------------
def save(self) -> None:
@contextlib.contextmanager
def _locked(self) -> Iterator[None]:
"""Exclusive lock over the registry for one read-modify-write cycle.
Blocking, but only ever held for a small local JSON read plus rename.
"""
path = self._config.registry_path
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_suffix(".tmp")
tmp.write_text(json.dumps({n: r.to_dict() for n, r in self._vms.items()}, indent=2))
fd = os.open(str(path) + ".lock", os.O_CREAT | os.O_RDWR, 0o600)
try:
fcntl.flock(fd, fcntl.LOCK_EX)
yield
finally:
os.close(fd) # closing releases the flock
def _read_disk(self) -> dict[str, VMRecord]:
"""Parse the registry, degrading instead of raising.
An unreadable file is quarantined and we continue with an empty
registry; individual malformed records are skipped. Both outcomes are
reported through `degraded` / `skipped` so list_vms can surface them.
"""
path = self._config.registry_path
if not path.exists():
return {}
try:
raw = json.loads(path.read_text())
except (OSError, ValueError) as e:
quarantine = path.with_name(f"vms.corrupt.{int(time.time())}.json")
with contextlib.suppress(OSError):
os.replace(path, quarantine)
self.degraded = (
f"Registry file was unreadable ({e}); it has been quarantined at "
f"{quarantine} and an empty registry was started. Any running VMs "
"are unaffected but no longer tracked — re-register them with "
"attach_vm (their QMP sockets are under the runtime directory)."
)
return {}
if isinstance(raw, dict) and "vms" in raw and "version" in raw:
version = raw.get("version")
if not isinstance(version, int) or version > SCHEMA_VERSION:
self.degraded = (
f"Registry schema version {version!r} is newer than this build "
f"understands (max {SCHEMA_VERSION}); refusing to load it so a "
"newer mcqemu's records are not damaged. Upgrade mcqemu."
)
return {}
entries = raw.get("vms") or {}
else:
entries = raw # legacy layout: a bare {name: record} mapping
if not isinstance(entries, dict):
self.degraded = "Registry contents were not an object; starting empty."
return {}
vms: dict[str, VMRecord] = {}
for key, rec in entries.items():
# Keys become filesystem paths and are what destructive tools look
# up by, so an invalid one is dropped rather than trusted.
if not isinstance(key, str) or not valid_vm_name(key):
self.skipped.append({"key": str(key)[:64], "reason": "invalid VM name"})
continue
if not isinstance(rec, dict):
self.skipped.append({"key": key, "reason": "record was not an object"})
continue
try:
record = VMRecord.from_dict(rec)
except TypeError as e:
self.skipped.append({"key": key, "reason": f"malformed record: {e}"})
continue
record.name = key # the key is authoritative; keep the record consistent
vms[key] = record
return vms
def _write_disk(self, vms: dict[str, VMRecord]) -> None:
path = self._config.registry_path
path.parent.mkdir(parents=True, exist_ok=True)
payload = {
"version": SCHEMA_VERSION,
"vms": {name: record.to_dict() for name, record in vms.items()},
}
# PID-unique temp name: a shared one can interleave between instances
# and os.replace would then publish the mixture.
tmp = path.with_name(f"{path.name}.tmp.{os.getpid()}")
tmp.write_text(json.dumps(payload, indent=2))
os.replace(tmp, path)
def _mutate(self, apply: Callable[[dict[str, VMRecord]], None]) -> None:
"""Re-read, apply, write — all under the lock, so a concurrent
instance's records are merged rather than clobbered."""
with self._locked():
vms = self._read_disk()
apply(vms)
self._write_disk(vms)
self._vms = vms
# --- public API ------------------------------------------------------
def load(self) -> None:
with self._locked():
self._vms = self._read_disk()
def refresh(self) -> None:
"""Re-read the registry so records written by another instance (or by
a tool call that raced this one) are visible."""
self.load()
def get(self, name: str) -> VMRecord | None:
return self._vms.get(name)
def add(self, record: VMRecord) -> None:
self._vms[record.name] = record
self.save()
self._mutate(lambda vms: vms.__setitem__(record.name, record))
def update(self, record: VMRecord) -> None:
"""Persist in-place changes to an existing record."""
self.add(record)
def remove(self, name: str) -> None:
self._vms.pop(name, None)
self.save()
self._mutate(lambda vms: vms.pop(name, None))
def all(self) -> list[VMRecord]:
return list(self._vms.values())
@ -64,6 +178,26 @@ class VMRegistry:
def names(self) -> list[str]:
return list(self._vms)
# --- in-process name reservations ------------------------------------
# launch_vm awaits (disk probing, spawning) between checking a name and
# registering it. Without a reservation two concurrent calls both pass the
# duplicate check and race over the same sockets and pidfile.
def reserve(self, name: str) -> bool:
"""Claim a name for an in-flight launch. False if already claimed."""
if name in self._reserved:
return False
self._reserved.add(name)
return True
def release(self, name: str) -> None:
self._reserved.discard(name)
def reserved(self) -> set[str]:
return set(self._reserved)
# --- liveness --------------------------------------------------------
def refresh_pid(self, record: VMRecord) -> int | None:
"""Re-read the pidfile for a spawned VM; returns the PID or None."""
if record.pidfile:
@ -80,7 +214,12 @@ class VMRegistry:
return pid_alive(record.pid)
def disks_in_use(self) -> dict[str, str]:
"""Map of absolute disk path -> VM name, for every live registered VM."""
"""Map of absolute disk path -> VM name, for every live registered VM.
Advisory only: attached VMs report no disks, and backing files are not
walked. QEMU's own image locks are the authoritative guard (qemu_img
translates them into an actionable error).
"""
used: dict[str, str] = {}
for rec in self._vms.values():
if not self.is_process_alive(rec):

View File

@ -1,5 +1,7 @@
"""mcqemu server — composition root and entry point."""
import logging
import os
import sys
from contextlib import asynccontextmanager
@ -11,6 +13,8 @@ from .registry import VMRegistry
from .state import AppContext
from .tools import register_all
log = logging.getLogger(__name__)
INSTRUCTIONS = """\
mcqemu manages QEMU virtual machines on this host.
@ -26,7 +30,10 @@ Typical flows:
screenshot -> act -> screenshot; guests need time to react, so re-check
rather than assuming.
- Inspect: list_vms / vm_info. VMs survive MCP server restarts (they are
daemonized QEMU processes).
daemonized QEMU processes). list_vms returns {"vms": [...],
"registry_warnings": [...]}; a non-empty registry_warnings means bookkeeping
was damaged and some VMs may be running untracked surface it to the user
rather than treating the list as complete.
- Inside the guest: guest_exec / guest_file_read / guest_file_write these
need qemu-guest-agent installed in the guest OS (guest_ping to check).
- Existing QEMU processes started outside this server can be managed after
@ -47,8 +54,14 @@ async def lifespan(mcp: FastMCP):
config.runtime_dir.mkdir(parents=True, exist_ok=True)
registry = VMRegistry(config)
registry.load()
if registry.degraded:
log.warning("registry degraded: %s", registry.degraded)
for entry in registry.skipped:
log.warning("skipped registry entry %r: %s", entry["key"], entry["reason"])
yield AppContext(config=config, registry=registry)
registry.save() # VMs keep running by design — they are daemonized.
# No save on shutdown: every mutation already persisted itself, and writing
# this instance's in-memory view here would clobber records another
# instance created while we were running. VMs keep running by design.
mcp = FastMCP("mcqemu", lifespan=lifespan, instructions=INSTRUCTIONS)
@ -58,6 +71,13 @@ prompts.register(mcp)
def main() -> None:
# stderr only: stdout is the JSON-RPC transport. Destructive operations log
# here so there is a record when something goes wrong unattended.
logging.basicConfig(
stream=sys.stderr,
level=os.environ.get("MCQEMU_LOG_LEVEL", "INFO").upper(),
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
)
print(f"mcqemu v{__version__}", file=sys.stderr)
mcp.run()

View File

@ -2,6 +2,7 @@
import asyncio
import base64
import contextlib
from fastmcp import Context, FastMCP
from fastmcp.exceptions import ToolError
@ -10,6 +11,39 @@ from ..qga import qga_session
from ._common import require_vm
_READ_CHUNK = 48 * 1024 # QGA caps single reads at 48 MiB of b64; stay modest.
MAX_READ_BYTES = 8 * 1024 * 1024
CALL_TIMEOUT = 10.0
# Guest agents on RHEL-family distros ship with these RPCs disabled by default.
_BLOCKED_HINT = (
"The guest agent refused the command ({desc}). On RHEL/CentOS/Fedora guests "
"these RPCs are disabled by default — remove them from BLOCK_RPCS in "
"/etc/sysconfig/qemu-ga and restart qemu-guest-agent."
)
async def _qga(client, name: str, command: str, args: dict | None = None, timeout=CALL_TIMEOUT):
"""One guest-agent call, bounded and with failures translated.
The timeout is the whole point: qga_session only bounds the handshake, so
a guest that freezes (panic, OOM, or another tool pausing its vCPUs) after
the sync would otherwise hang this call forever.
"""
try:
return await asyncio.wait_for(client.execute(command, args or {}), timeout)
except TimeoutError:
raise ToolError(
f"The guest agent in VM {name!r} stopped responding to {command!r} after "
f"{timeout}s. The guest may be frozen or paused — check vm_screenshot "
"and list_vms (a paused VM cannot run its agent)."
) from None
except ToolError:
raise
except Exception as e:
desc = str(e) or type(e).__name__
if "CommandNotFound" in type(e).__name__ or "not found" in desc.lower():
raise ToolError(_BLOCKED_HINT.format(desc=desc)) from e
raise ToolError(f"Guest agent command {command!r} failed: {desc}") from e
async def guest_ping(name: str, ctx: Context = None) -> dict:
@ -18,7 +52,7 @@ async def guest_ping(name: str, ctx: Context = None) -> dict:
or running the other guest_* tools won't work until it does."""
_, record = require_vm(ctx, name)
async with qga_session(record) as client:
await asyncio.wait_for(client.execute("guest-ping"), timeout=3.0)
await _qga(client, name, "guest-ping", timeout=3.0)
return {"name": name, "guest_agent": "responding"}
@ -27,11 +61,10 @@ async def guest_info(name: str, ctx: Context = None) -> dict:
agent's version and supported commands."""
_, record = require_vm(ctx, name)
async with qga_session(record) as client:
agent = await client.execute("guest-info")
try:
osinfo = await client.execute("guest-get-osinfo")
except Exception:
osinfo = None
agent = await _qga(client, name, "guest-info")
osinfo = None
with contextlib.suppress(ToolError):
osinfo = await _qga(client, name, "guest-get-osinfo")
supported = sorted(c["name"] for c in agent.get("supported_commands", []) if c.get("enabled"))
return {
"name": name,
@ -55,27 +88,46 @@ async def guest_exec(
command="/bin/sh", args=["-c", "your | pipeline"]). Requires
qemu-guest-agent in the guest."""
_, record = require_vm(ctx, name)
if timeout < 1 or timeout > 3600:
raise ToolError("timeout must be between 1 and 3600 seconds.")
exec_args: dict = {"path": command, "capture-output": True}
if args:
exec_args["arg"] = args
if stdin is not None:
exec_args["input-data"] = base64.b64encode(stdin.encode()).decode()
async with qga_session(record) as client:
reply = await client.execute("guest-exec", exec_args)
pid = reply["pid"]
deadline = asyncio.get_event_loop().time() + timeout
async def _run(client):
reply = await _qga(client, name, "guest-exec", exec_args)
try:
pid = reply["pid"]
except (KeyError, TypeError) as e:
raise ToolError(
f"Guest agent returned an unexpected guest-exec reply: {reply!r}"
) from e
loop = asyncio.get_event_loop()
deadline = loop.time() + timeout
while True:
status = await client.execute("guest-exec-status", {"pid": pid})
status = await _qga(client, name, "guest-exec-status", {"pid": pid})
if status.get("exited"):
break
if asyncio.get_event_loop().time() > deadline:
return status
if loop.time() > deadline:
raise ToolError(
f"Command still running in guest after {timeout}s (guest pid {pid}). "
"It continues in the background; raise timeout for long commands."
)
await asyncio.sleep(0.2)
async with qga_session(record) as client:
# Outer bound as well: the poll loop checks the clock between awaits,
# so without this a single hung call could outlive the deadline.
try:
status = await asyncio.wait_for(_run(client), timeout + CALL_TIMEOUT + 5)
except TimeoutError:
raise ToolError(
f"guest_exec on VM {name!r} exceeded its overall deadline; the guest "
"agent may have stopped responding mid-command."
) from None
def _decode(field: str) -> str:
data = status.get(field)
return base64.b64decode(data).decode(errors="replace") if data else ""
@ -92,28 +144,42 @@ async def guest_exec(
async def guest_file_read(
name: str, path: str, max_bytes: int = 1_048_576, ctx: Context = None
) -> dict:
"""Read a text file from inside the guest (up to max_bytes, default 1 MiB).
Requires qemu-guest-agent in the guest."""
"""Read a text file from inside the guest (up to max_bytes, default 1 MiB,
8 MiB ceiling). Requires qemu-guest-agent in the guest."""
_, record = require_vm(ctx, name)
if max_bytes < 1 or max_bytes > MAX_READ_BYTES:
raise ToolError(f"max_bytes must be between 1 and {MAX_READ_BYTES}.")
async with qga_session(record) as client:
handle = await client.execute("guest-file-open", {"path": path, "mode": "r"})
handle = await _qga(client, name, "guest-file-open", {"path": path, "mode": "r"})
chunks: list[bytes] = []
total = 0
truncated = False
empty_reads = 0
try:
chunks: list[bytes] = []
total = 0
truncated = False
while total < max_bytes:
count = min(_READ_CHUNK, max_bytes - total)
reply = await client.execute("guest-file-read", {"handle": handle, "count": count})
if reply.get("count", 0) > 0:
reply = await _qga(
client, name, "guest-file-read", {"handle": handle, "count": count}
)
got = reply.get("count", 0)
if got > 0:
chunk = base64.b64decode(reply["buf-b64"])
chunks.append(chunk)
total += len(chunk)
empty_reads = 0
else:
# FIFOs, ttys and some /proc files return 0 without EOF
# forever; give up rather than spin on the agent.
empty_reads += 1
if empty_reads >= 3 and not reply.get("eof"):
break
if reply.get("eof"):
break
else:
truncated = True
finally:
await client.execute("guest-file-close", {"handle": handle})
with contextlib.suppress(Exception):
await _qga(client, name, "guest-file-close", {"handle": handle})
return {
"path": path,
"content": b"".join(chunks).decode(errors="replace"),
@ -130,13 +196,16 @@ async def guest_file_write(
_, record = require_vm(ctx, name)
payload = base64.b64encode(content.encode()).decode()
async with qga_session(record) as client:
handle = await client.execute(
"guest-file-open", {"path": path, "mode": "a" if append else "w"}
handle = await _qga(
client, name, "guest-file-open", {"path": path, "mode": "a" if append else "w"}
)
try:
reply = await client.execute("guest-file-write", {"handle": handle, "buf-b64": payload})
reply = await _qga(
client, name, "guest-file-write", {"handle": handle, "buf-b64": payload}
)
finally:
await client.execute("guest-file-close", {"handle": handle})
with contextlib.suppress(Exception):
await _qga(client, name, "guest-file-close", {"handle": handle})
return {"path": path, "bytes_written": reply.get("count", 0), "appended": append}

View File

@ -10,7 +10,7 @@ from fastmcp.exceptions import ToolError
from ..config import resolve_image_path, valid_vm_name
from ..launcher import resolve_port_forwards, spawn_vm
from ..models import VMConfig
from ..qmp import execute, qmp_session, wait_for_event
from ..qmp import execute, qmp_session, recent_events, wait_for_event
from ..registry import pid_alive
from ._common import app, require_vm
@ -46,6 +46,9 @@ async def launch_vm(
f"Invalid VM name {name!r}: use letters, digits, '.', '_', '-' (max 48 chars)."
)
# Pick up records written by another instance (or another in-flight call)
# before deciding this name is free.
state.registry.refresh()
existing = state.registry.get(name)
if existing is not None:
if state.registry.is_process_alive(existing):
@ -54,38 +57,46 @@ async def launch_vm(
)
state.registry.remove(name) # dead leftover — reuse the name
resolved_disks: list[str] = []
in_use = state.registry.disks_in_use()
for disk in disks or []:
try:
p = resolve_image_path(disk)
except ValueError as e:
raise ToolError(f"Bad disk path: {e}. Create images with image_create.") from e
if str(p) in in_use:
raise ToolError(f"Disk {p} is already attached to running VM {in_use[str(p)]!r}.")
resolved_disks.append(str(p))
# Claim the name before the first await. Everything below suspends, and
# two concurrent launches that both passed the check above would otherwise
# race over the same sockets and pidfile.
if not state.registry.reserve(name):
raise ToolError(f"A launch of VM {name!r} is already in progress.")
try:
resolved_disks: list[str] = []
in_use = state.registry.disks_in_use()
for disk in disks or []:
try:
p = resolve_image_path(disk)
except ValueError as e:
raise ToolError(f"Bad disk path: {e}. Create images with image_create.") from e
if str(p) in in_use:
raise ToolError(f"Disk {p} is already attached to running VM {in_use[str(p)]!r}.")
resolved_disks.append(str(p))
resolved_iso: str | None = None
if iso:
try:
resolved_iso = str(resolve_image_path(iso))
except ValueError as e:
raise ToolError(f"Bad ISO path: {e}") from e
resolved_iso: str | None = None
if iso:
try:
resolved_iso = str(resolve_image_path(iso))
except ValueError as e:
raise ToolError(f"Bad ISO path: {e}") from e
cfg = VMConfig(
name=name,
arch=arch,
machine=machine,
memory_mb=memory_mb,
cpus=cpus,
disks=resolved_disks,
iso=resolved_iso,
firmware=firmware,
port_forwards=resolve_port_forwards(port_forwards or []),
no_net=no_net,
extra_args=extra_args or [],
)
record = await spawn_vm(cfg, state.config)
cfg = VMConfig(
name=name,
arch=arch,
machine=machine,
memory_mb=memory_mb,
cpus=cpus,
disks=resolved_disks,
iso=resolved_iso,
firmware=firmware,
port_forwards=resolve_port_forwards(port_forwards or []),
no_net=no_net,
extra_args=extra_args or [],
)
record = await spawn_vm(cfg, state.config)
finally:
state.registry.release(name)
state.registry.add(record)
return {
"name": name,
@ -122,12 +133,21 @@ async def stop_vm(name: str, force: bool = False, timeout: int = 30, ctx: Contex
async with qmp_session(record) as client:
await execute(client, "system_powerdown")
if not await wait_for_event(client, "SHUTDOWN", timeout=timeout):
outcome = await wait_for_event(client, "SHUTDOWN", timeout=timeout, record=record)
if outcome == "timeout":
seen = recent_events(client)
context = f" Events seen meanwhile: {', '.join(seen)}." if seen else ""
raise ToolError(
f"VM {name!r} did not shut down within {timeout}s after the ACPI "
"power signal — the guest may have no OS or ignores ACPI. "
"Retry with force=True to terminate it immediately."
f"Retry with force=True to terminate it immediately.{context}"
)
if outcome == "process_exited":
return {
"name": name,
"status": "stopped",
"method": "process exited during shutdown",
}
deadline = asyncio.get_event_loop().time() + 10
while pid_alive(record.pid) and asyncio.get_event_loop().time() < deadline:
await asyncio.sleep(0.1)

View File

@ -18,15 +18,22 @@ async def _live_status(state, record) -> str:
return "unreachable"
async def list_vms(ctx: Context = None) -> list[dict]:
async def list_vms(ctx: Context = None) -> dict:
"""List every registered VM with its live status (running, paused,
shutdown, stopped, or unreachable). Includes both VMs launched by this
server and externally attached ones."""
server and externally attached ones.
Returns {"vms": [...], "registry_warnings": [...]}. A non-empty
registry_warnings means bookkeeping was damaged and some VMs may be
running but untracked report it rather than assuming the list is
complete."""
state = app(ctx)
out = []
# Pick up anything another mcqemu instance registered since we loaded.
state.registry.refresh()
vms = []
for record in state.registry.all():
status = await _live_status(state, record)
out.append(
vms.append(
{
"name": record.name,
"status": status,
@ -37,7 +44,13 @@ async def list_vms(ctx: Context = None) -> list[dict]:
"created_at": record.created_at,
}
)
return out
warnings = []
if state.registry.degraded:
warnings.append(state.registry.degraded)
warnings += [
f"skipped registry entry {e['key']!r}: {e['reason']}" for e in state.registry.skipped
]
return {"vms": vms, "registry_warnings": warnings}
async def vm_info(name: str, ctx: Context = None) -> dict:

View File

@ -2,7 +2,11 @@
the matching destroy that cleans up everything the sandbox created."""
import asyncio
import contextlib
import logging
import os
import shutil
import signal
from pathlib import Path
from fastmcp import Context, FastMCP
@ -16,6 +20,8 @@ from ..registry import pid_alive
from ._common import app, require_vm
from .lifecycle import launch_vm
log = logging.getLogger(__name__)
async def _agent_responding(record) -> bool:
try:
@ -25,6 +31,26 @@ async def _agent_responding(record) -> bool:
return False
async def _await_exit(pid: int | None, seconds: float) -> bool:
"""Poll until the PID is gone. Returns True if it exited in time."""
loop = asyncio.get_event_loop()
deadline = loop.time() + seconds
while loop.time() < deadline:
if not pid_alive(pid):
return True
await asyncio.sleep(0.1)
return not pid_alive(pid)
def _tail_log(path: str | None, lines: int = 3) -> str:
if not path:
return "(no log)"
try:
return " | ".join(Path(path).read_text(errors="replace").splitlines()[-lines:])
except OSError:
return "(log unreadable)"
def _auto_name(registry) -> str:
"""First 'sandbox[-N]' name that is unregistered or belongs to a dead VM
(launch_vm reclaims dead names, so reusing them is safe and keeps names
@ -80,19 +106,26 @@ async def sandbox_vm(
overlay.unlink(missing_ok=True)
await run_qemu_img("create", "-f", "qcow2", "-b", str(base), "-F", base_fmt, str(overlay))
result = await launch_vm(
name=name,
disks=[str(overlay)],
memory_mb=memory_mb,
cpus=cpus,
port_forwards=["auto:22"] if port_forwards is None else port_forwards,
ctx=ctx,
)
try:
result = await launch_vm(
name=name,
disks=[str(overlay)],
memory_mb=memory_mb,
cpus=cpus,
port_forwards=["auto:22"] if port_forwards is None else port_forwards,
ctx=ctx,
)
except Exception:
# Nothing was registered, so sandbox_destroy could never reach this
# overlay — clean it up here or it leaks with no owner.
with contextlib.suppress(OSError):
overlay.unlink(missing_ok=True)
raise
record = state.registry.get(name)
record.config["sandbox_overlay"] = str(overlay)
record.config["sandbox_base"] = str(base)
state.registry.save()
state.registry.update(record)
agent = "unavailable"
loop = asyncio.get_event_loop()
@ -101,6 +134,14 @@ async def sandbox_vm(
if await _agent_responding(record):
agent = "responding"
break
# A VM that died during boot will never answer; say so now rather than
# blaming a missing guest agent after the full timeout.
if not state.registry.is_process_alive(record):
raise ToolError(
f"VM {name!r} exited while booting. Last QEMU log lines: "
f"{_tail_log(record.qemu_log)}. The overlay was kept at {overlay}; "
"remove it with sandbox_destroy."
)
await asyncio.sleep(2)
result.update(
@ -124,7 +165,8 @@ async def sandbox_destroy(name: str, ctx: Context = None) -> dict:
"""Destroy a sandbox created with sandbox_vm: force-stop the VM, remove it
from the registry, and delete its overlay disk and logs. The base image is
untouched. Refuses to operate on VMs that were not created by sandbox_vm
use stop_vm / forget_vm for those (they never delete disks)."""
use stop_vm / forget_vm for those (they never delete disks). If the VM
cannot be killed, nothing is deleted and the call fails."""
state, record = require_vm(ctx, name)
overlay = record.config.get("sandbox_overlay")
if not overlay:
@ -132,23 +174,58 @@ async def sandbox_destroy(name: str, ctx: Context = None) -> dict:
f"VM {name!r} was not created by sandbox_vm; refusing to delete its "
"disks. Use stop_vm and forget_vm instead."
)
# This is the only tool that deletes files, so it re-validates the name it
# was handed rather than trusting the registry key it came from.
if not valid_vm_name(name):
raise ToolError(f"Refusing to destroy {name!r}: not a valid VM name.")
targets = [state.config.vm_state_dir(name), state.config.vm_runtime_dir(name)]
roots = [state.config.state_dir / "vms", state.config.runtime_dir]
for target, root in zip(targets, roots, strict=True):
resolved, root_resolved = target.resolve(), root.resolve()
if resolved == root_resolved or not resolved.is_relative_to(root_resolved):
raise ToolError(
f"Refusing to delete {resolved}: it is not inside {root_resolved}. "
"The registry entry looks corrupt; inspect it before retrying."
)
# Verify death before deleting: an overlay unlinked while QEMU still holds
# it open keeps consuming space and the guest keeps writing into a file
# nobody can find.
if state.registry.is_process_alive(record):
async with qmp_session(record) as client:
await execute(client, "quit")
deadline = asyncio.get_event_loop().time() + 10
while pid_alive(record.pid) and asyncio.get_event_loop().time() < deadline:
await asyncio.sleep(0.1)
log.info("sandbox_destroy: quitting VM %s (pid %s)", name, record.pid)
with contextlib.suppress(ToolError):
async with qmp_session(record) as client:
await execute(client, "quit")
if not await _await_exit(record.pid, 10):
log.warning("sandbox_destroy: %s ignored quit, sending SIGKILL", name)
with contextlib.suppress(OSError):
os.kill(record.pid, signal.SIGKILL)
await _await_exit(record.pid, 5)
if pid_alive(record.pid):
raise ToolError(
f"VM {name!r} (pid {record.pid}) survived both 'quit' and SIGKILL. "
"Nothing was deleted — its overlay is still open by that process. "
"Investigate the process before retrying."
)
state.registry.remove(name)
shutil.rmtree(state.config.vm_state_dir(name), ignore_errors=True)
shutil.rmtree(state.config.vm_runtime_dir(name), ignore_errors=True)
return {
errors: list[str] = []
for target in targets:
if not target.exists():
continue # nothing to remove is success, not a failure
shutil.rmtree(target, onexc=lambda _f, path, exc: errors.append(f"{path}: {exc}"))
overlay_gone = not Path(overlay).exists()
log.info("sandbox_destroy: removed %s (overlay_deleted=%s)", name, overlay_gone)
result = {
"name": name,
"destroyed": True,
"overlay_deleted": not Path(overlay).exists(),
"destroyed": overlay_gone and not errors,
"overlay_deleted": overlay_gone,
"base_image_untouched": record.config.get("sandbox_base"),
}
if errors or not overlay_gone:
result["cleanup_errors"] = errors or [f"{overlay} still exists after rmtree"]
return result
def register(mcp: FastMCP) -> None:

View File

@ -6,15 +6,26 @@ job polling for the same v1 outcome. Internal snapshots require all writable
disks to be qcow2.
"""
import re
from fastmcp import Context, FastMCP
from fastmcp.exceptions import ToolError
from ..qmp import execute, qmp_session
from ._common import require_vm
# savevm/loadvm write or read the whole guest RAM and can legitimately take
# minutes on a large VM, so they get a far longer bound than ordinary QMP
# commands — but still a bound, so a wedged one cannot hang the tool forever.
SNAPSHOT_TIMEOUT = 900.0
async def _hmp(client, command_line: str) -> str:
return await execute(client, "human-monitor-command", {"command-line": command_line})
_TAG_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$")
async def _hmp(client, command_line: str, timeout: float = SNAPSHOT_TIMEOUT) -> str:
return await execute(
client, "human-monitor-command", {"command-line": command_line}, timeout=timeout
)
async def _hmp_expect_silence(client, command_line: str, what: str) -> None:
@ -24,8 +35,13 @@ async def _hmp_expect_silence(client, command_line: str, what: str) -> None:
def _check_tag(tag: str) -> None:
if not tag or any(c.isspace() for c in tag) or tag.startswith("-"):
raise ToolError(f"Invalid snapshot tag {tag!r}: no whitespace, must not start with '-'.")
"""HMP has no statement separator, but quotes and spaces still confuse its
tokenizer, so a tag that survives here means what it says."""
if not _TAG_RE.match(tag or ""):
raise ToolError(
f"Invalid snapshot tag {tag!r}: use letters, digits, '.', '_', '-' "
"(must start alphanumeric, max 64 chars)."
)
async def vm_snapshot_create(name: str, tag: str, ctx: Context = None) -> dict:

View File

@ -1,6 +1,7 @@
"""Shared fixtures: temp XDG dirs, scriptable fake QMP client, in-memory MCP client."""
import asyncio
import inspect
import json
from types import SimpleNamespace
@ -81,7 +82,10 @@ class FakeQMPClient:
if isinstance(reply, Exception):
raise reply
if callable(reply):
return reply(arguments)
reply = reply(arguments)
# Awaitable replies let a test simulate an agent that stops answering.
if inspect.isawaitable(reply):
reply = await reply
return reply
@property
@ -97,13 +101,23 @@ def fake_qmp(monkeypatch):
return FakeQMPClient
# Every module that imported pid_alive into its own namespace needs patching,
# or a "dead" fixture leaves one module still seeing live processes.
_PID_ALIVE_REFS = (
"mcqemu.registry.pid_alive",
"mcqemu.qmp.pid_alive",
"mcqemu.tools.lifecycle.pid_alive",
"mcqemu.tools.sandbox.pid_alive",
)
@pytest.fixture
def all_pids_dead(monkeypatch):
monkeypatch.setattr("mcqemu.registry.pid_alive", lambda pid: False)
monkeypatch.setattr("mcqemu.tools.lifecycle.pid_alive", lambda pid: False)
for ref in _PID_ALIVE_REFS:
monkeypatch.setattr(ref, lambda pid: False)
@pytest.fixture
def all_pids_alive(monkeypatch):
monkeypatch.setattr("mcqemu.registry.pid_alive", lambda pid: bool(pid))
monkeypatch.setattr("mcqemu.tools.lifecycle.pid_alive", lambda pid: bool(pid))
for ref in _PID_ALIVE_REFS:
monkeypatch.setattr(ref, lambda pid: bool(pid))

View File

@ -34,16 +34,16 @@ async def test_real_vm_lifecycle(dirs):
assert data["status"] == "running"
assert pid_alive(pid)
vms = result_data(await client.call_tool("list_vms", {}))
assert [(v["name"], v["status"]) for v in vms] == [("itest", "running")]
listing = result_data(await client.call_tool("list_vms", {}))
assert [(v["name"], v["status"]) for v in listing["vms"]] == [("itest", "running")]
info = result_data(await client.call_tool("vm_info", {"name": "itest"}))
assert info["vcpus"] == 1
data = result_data(await client.call_tool("pause_vm", {"name": "itest"}))
assert data["status"] == "paused"
vms = result_data(await client.call_tool("list_vms", {}))
assert vms[0]["status"] == "paused"
listing = result_data(await client.call_tool("list_vms", {}))
assert listing["vms"][0]["status"] == "paused"
await client.call_tool("resume_vm", {"name": "itest"})

View File

@ -59,8 +59,8 @@ async def test_launch_happy_path_registers(dirs, monkeypatch):
data = result_data(await client.call_tool("launch_vm", {"name": "fresh", "no_net": True}))
assert data["status"] == "running"
assert data["accel"] == "kvm"
vms = result_data(await client.call_tool("list_vms", {}))
assert [v["name"] for v in vms] == ["fresh"]
result = result_data(await client.call_tool("list_vms", {}))
assert [v["name"] for v in result["vms"]] == ["fresh"]
async def test_stop_force_sends_quit(dirs, fake_qmp, all_pids_dead, monkeypatch):
@ -84,15 +84,27 @@ async def test_stop_graceful_waits_for_shutdown_event(dirs, fake_qmp, all_pids_d
assert ("system_powerdown", {}) in FakeQMPClient.calls
async def test_stop_graceful_timeout_advises_force(dirs, fake_qmp, all_pids_dead, monkeypatch):
async def test_stop_graceful_timeout_advises_force(dirs, fake_qmp, all_pids_alive, monkeypatch):
# Guest ignores ACPI: no SHUTDOWN event AND the process stays alive.
monkeypatch.setattr("mcqemu.tools.lifecycle.Path.exists", lambda self: True)
FakeQMPClient.events_to_emit = [] # guest ignores ACPI
FakeQMPClient.events_to_emit = []
write_registry(dirs, seeded_record(dirs, "vm1"))
async with Client(mcp) as client:
with pytest.raises(ToolError, match="force=True"):
await client.call_tool("stop_vm", {"name": "vm1", "timeout": 1})
async def test_stop_graceful_reports_process_exit(dirs, fake_qmp, all_pids_dead, monkeypatch):
"""A VM that dies mid-shutdown must not be misreported as ignoring ACPI."""
monkeypatch.setattr("mcqemu.tools.lifecycle.Path.exists", lambda self: True)
FakeQMPClient.events_to_emit = [] # crashed before emitting SHUTDOWN
write_registry(dirs, seeded_record(dirs, "vm1"))
async with Client(mcp) as client:
data = result_data(await client.call_tool("stop_vm", {"name": "vm1", "timeout": 30}))
assert data["status"] == "stopped"
assert "exited" in data["method"]
async def test_stop_already_stopped(dirs, all_pids_dead):
write_registry(dirs, seeded_record(dirs, "vm1"))
async with Client(mcp) as client:
@ -139,8 +151,8 @@ async def test_attach_and_forget(dirs, fake_qmp, tmp_path):
assert data["status"] == "running"
data = result_data(await client.call_tool("forget_vm", {"name": "ext"}))
assert data["forgotten"] is True
vms = result_data(await client.call_tool("list_vms", {}))
assert vms == []
result = result_data(await client.call_tool("list_vms", {}))
assert result["vms"] == []
async def test_forget_refuses_running_spawned_vm(dirs, all_pids_alive):

View File

@ -10,24 +10,25 @@ from test_lifecycle import seeded_record
async def test_list_vms_stopped(dirs, all_pids_dead):
write_registry(dirs, seeded_record(dirs, "vm1"))
async with Client(mcp) as client:
vms = result_data(await client.call_tool("list_vms", {}))
assert vms[0]["status"] == "stopped"
result = result_data(await client.call_tool("list_vms", {}))
assert result["vms"][0]["status"] == "stopped"
assert result["registry_warnings"] == []
async def test_list_vms_running(dirs, fake_qmp, all_pids_alive):
FakeQMPClient.responses["query-status"] = {"status": "running"}
write_registry(dirs, seeded_record(dirs, "vm1"))
async with Client(mcp) as client:
vms = result_data(await client.call_tool("list_vms", {}))
assert vms[0]["status"] == "running"
result = result_data(await client.call_tool("list_vms", {}))
assert result["vms"][0]["status"] == "running"
async def test_list_vms_unreachable_when_qmp_fails(dirs, fake_qmp, all_pids_alive):
FakeQMPClient.connect_error = OSError("connection refused")
write_registry(dirs, seeded_record(dirs, "vm1"))
async with Client(mcp) as client:
vms = result_data(await client.call_tool("list_vms", {}))
assert vms[0]["status"] == "unreachable"
result = result_data(await client.call_tool("list_vms", {}))
assert result["vms"][0]["status"] == "unreachable"
async def test_vm_info_running_includes_block_devices(dirs, fake_qmp, all_pids_alive):

261
tests/test_reliability.py Normal file
View File

@ -0,0 +1,261 @@
"""Failure-path tests for the conditions found in the reliability review.
Every test here corresponds to a way the server used to fail badly: a corrupt
state file taking down every tool, a destructive tool deleting outside its
tree or while QEMU still held the file, a guest agent that stops answering
mid-call, two instances clobbering each other's records.
"""
import asyncio
import json
import socket
import pytest
from fastmcp import Client
from fastmcp.exceptions import ToolError
from conftest import FakeQMPClient, result_data, write_registry
from mcqemu.config import Config
from mcqemu.models import VMRecord
from mcqemu.registry import VMRegistry
from mcqemu.server import mcp
from test_lifecycle import seeded_record
def config_for(dirs) -> Config:
return Config(state_dir=dirs.state, runtime_dir=dirs.run)
# --- registry survives damage ----------------------------------------------
async def test_corrupt_registry_does_not_kill_the_server(dirs):
"""A truncated state file must not make every tool unreachable."""
(dirs.state / "vms.json").write_text('{"broken": ')
async with Client(mcp) as client:
listing = result_data(await client.call_tool("list_vms", {}))
assert listing["vms"] == []
assert any("quarantined" in w for w in listing["registry_warnings"])
assert list(dirs.state.glob("vms.corrupt.*.json")), "bad file should be preserved"
async def test_malformed_record_is_skipped_not_fatal(dirs):
good = seeded_record(dirs, "good")
(dirs.state / "vms.json").write_text(
json.dumps({"good": good.to_dict(), "broken": {"no": "required fields"}})
)
async with Client(mcp) as client:
listing = result_data(await client.call_tool("list_vms", {}))
assert [v["name"] for v in listing["vms"]] == ["good"]
assert any("broken" in w for w in listing["registry_warnings"])
async def test_traversal_key_is_dropped_on_load(dirs):
"""A registry key that is not a valid VM name never reaches the tools."""
rec = seeded_record(dirs, "ok")
payload = {"../../escape": rec.to_dict(), "ok": rec.to_dict()}
(dirs.state / "vms.json").write_text(json.dumps(payload))
async with Client(mcp) as client:
listing = result_data(await client.call_tool("list_vms", {}))
assert [v["name"] for v in listing["vms"]] == ["ok"]
assert any("invalid VM name" in w for w in listing["registry_warnings"])
def test_unknown_schema_version_refuses_rather_than_stripping(dirs):
(dirs.state / "vms.json").write_text(json.dumps({"version": 99, "vms": {}}))
registry = VMRegistry(config_for(dirs))
registry.load()
assert registry.all() == []
assert "newer than this build" in registry.degraded
def test_unknown_record_fields_round_trip(dirs):
"""An older build must not strip a newer one's fields when it saves."""
registry = VMRegistry(config_for(dirs))
registry.load()
raw = seeded_record(dirs, "vm1").to_dict() | {"future_field": {"keep": "me"}}
(dirs.state / "vms.json").write_text(json.dumps({"vm1": raw}))
registry2 = VMRegistry(config_for(dirs))
registry2.load()
registry2.add(seeded_record(dirs, "vm2")) # triggers a full rewrite
on_disk = json.loads((dirs.state / "vms.json").read_text())["vms"]
assert on_disk["vm1"]["future_field"] == {"keep": "me"}
def test_concurrent_instances_merge_instead_of_clobbering(dirs):
"""Two mcqemu processes sharing a registry must not orphan each other."""
a, b = VMRegistry(config_for(dirs)), VMRegistry(config_for(dirs))
a.load()
b.load() # both start from an empty view
a.add(seeded_record(dirs, "from-a"))
b.add(seeded_record(dirs, "from-b")) # stale in-memory view, must still merge
fresh = VMRegistry(config_for(dirs))
fresh.load()
assert sorted(fresh.names()) == ["from-a", "from-b"]
def test_name_reservation_blocks_a_second_in_flight_launch(dirs):
registry = VMRegistry(config_for(dirs))
registry.load()
assert registry.reserve("vm1") is True
assert registry.reserve("vm1") is False
registry.release("vm1")
assert registry.reserve("vm1") is True
# --- destructive tool refuses to guess -------------------------------------
async def test_sandbox_destroy_refuses_path_outside_state_dir(dirs, all_pids_dead, monkeypatch):
"""Even with a hostile record, deletion stays inside the VM state tree."""
victim = dirs.state / "IMPORTANT"
victim.mkdir()
(victim / "precious.txt").write_text("keep me")
record = seeded_record(dirs, "sb", config={"disks": [], "sandbox_overlay": "/tmp/x.qcow2"})
write_registry(dirs, record)
# Simulate a corrupted config whose state dir escapes the vms/ root.
monkeypatch.setattr(Config, "vm_state_dir", lambda self, name: victim)
async with Client(mcp) as client:
with pytest.raises(ToolError, match="not inside"):
await client.call_tool("sandbox_destroy", {"name": "sb"})
assert (victim / "precious.txt").exists()
async def test_sandbox_destroy_refuses_when_vm_survives_kill(dirs, all_pids_alive, monkeypatch):
"""If the process will not die, its overlay must not be deleted."""
overlay = dirs.state / "vms" / "sb" / "overlay.qcow2"
overlay.parent.mkdir(parents=True)
overlay.write_bytes(b"disk")
write_registry(
dirs,
seeded_record(dirs, "sb", config={"disks": [], "sandbox_overlay": str(overlay)}),
)
monkeypatch.setattr("mcqemu.tools.sandbox.os.kill", lambda pid, sig: None)
monkeypatch.setattr("mcqemu.tools.sandbox._await_exit", _never_exits)
async with Client(mcp) as client:
with pytest.raises(ToolError, match="survived both"):
await client.call_tool("sandbox_destroy", {"name": "sb"})
assert overlay.exists(), "overlay must survive a failed kill"
async with Client(mcp) as client:
listing = result_data(await client.call_tool("list_vms", {}))
assert [v["name"] for v in listing["vms"]] == ["sb"], "record must not be dropped"
async def _never_exits(pid, seconds):
return False
async def test_sandbox_destroy_reports_cleanup_failure_honestly(dirs, all_pids_dead, monkeypatch):
overlay = dirs.state / "vms" / "sb" / "overlay.qcow2"
overlay.parent.mkdir(parents=True)
overlay.write_bytes(b"disk")
write_registry(
dirs,
seeded_record(dirs, "sb", config={"disks": [], "sandbox_overlay": str(overlay)}),
)
def boom(target, onexc=None):
onexc(None, str(target), PermissionError("read-only filesystem"))
monkeypatch.setattr("mcqemu.tools.sandbox.shutil.rmtree", boom)
async with Client(mcp) as client:
data = result_data(await client.call_tool("sandbox_destroy", {"name": "sb"}))
assert data["destroyed"] is False
assert data["cleanup_errors"], "a failed delete must be reported, not swallowed"
# --- launch does not trample a live VM -------------------------------------
async def test_launch_refuses_to_reuse_a_live_qmp_socket(dirs):
"""An unregistered but running QEMU must not lose its monitor socket."""
run_dir = dirs.run / "ghost"
run_dir.mkdir(parents=True)
sock_path = run_dir / "qmp.sock"
listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
listener.bind(str(sock_path))
listener.listen(1)
try:
async with Client(mcp) as client:
with pytest.raises(ToolError, match="LIVE QMP socket"):
await client.call_tool("launch_vm", {"name": "ghost", "no_net": True})
assert sock_path.exists(), "the live socket must not be unlinked"
finally:
listener.close()
# --- guest agent cannot hang the server ------------------------------------
async def test_guest_exec_times_out_when_agent_stops_answering(dirs, fake_qmp, monkeypatch):
monkeypatch.setattr("mcqemu.tools.guest.CALL_TIMEOUT", 1.0)
FakeQMPClient.responses["guest-sync"] = lambda args: args["id"]
FakeQMPClient.responses["guest-exec"] = {"pid": 7}
FakeQMPClient.responses["guest-exec-status"] = lambda args: asyncio.sleep(3600)
write_registry(dirs, seeded_record(dirs, "vm1"))
async with Client(mcp) as client:
with pytest.raises(ToolError, match="stopped responding"):
await asyncio.wait_for(
client.call_tool("guest_exec", {"name": "vm1", "command": "sleep", "timeout": 2}),
timeout=30,
)
async def test_guest_file_read_gives_up_on_a_zero_progress_agent(dirs, fake_qmp):
"""A FIFO or tty returns count=0 without EOF forever; don't spin on it."""
FakeQMPClient.responses["guest-sync"] = lambda args: args["id"]
FakeQMPClient.responses["guest-file-open"] = 3
FakeQMPClient.responses["guest-file-read"] = {"count": 0, "eof": False}
FakeQMPClient.responses["guest-file-close"] = {}
write_registry(dirs, seeded_record(dirs, "vm1"))
async with Client(mcp) as client:
data = result_data(
await asyncio.wait_for(
client.call_tool("guest_file_read", {"name": "vm1", "path": "/dev/fifo"}),
timeout=20,
)
)
assert data["bytes_read"] == 0
reads = [c for c, _ in FakeQMPClient.calls if c == "guest-file-read"]
assert len(reads) < 10, "should stop after a few empty reads, not loop"
async def test_guest_file_read_rejects_absurd_max_bytes(dirs, fake_qmp):
FakeQMPClient.responses["guest-sync"] = lambda args: args["id"]
write_registry(dirs, seeded_record(dirs, "vm1"))
async with Client(mcp) as client:
with pytest.raises(ToolError, match="max_bytes"):
await client.call_tool(
"guest_file_read", {"name": "vm1", "path": "/x", "max_bytes": 10**12}
)
# --- untrusted bytes --------------------------------------------------------
async def test_screenshot_rejects_non_png_output(dirs, fake_qmp):
def write_junk(args):
with open(args["filename"], "wb") as f:
f.write(b"not a png at all")
return {}
FakeQMPClient.responses["screendump"] = write_junk
write_registry(dirs, seeded_record(dirs, "vm1"))
async with Client(mcp) as client:
with pytest.raises(ToolError, match="valid PNG"):
await client.call_tool("vm_click", {"name": "vm1", "x": 1, "y": 1})
def test_record_name_follows_the_registry_key(dirs):
"""Tools look records up by key, so a mismatched inner name is corrected."""
rec = VMRecord(name="lies", source="spawned", qmp_socket="/tmp/q.sock")
(dirs.state / "vms.json").write_text(json.dumps({"truth": rec.to_dict()}))
registry = VMRegistry(config_for(dirs))
registry.load()
assert registry.get("truth").name == "truth"

View File

@ -145,8 +145,8 @@ async def test_sandbox_destroy_removes_overlay(dirs, all_pids_dead):
assert data["destroyed"] is True
assert data["overlay_deleted"] is True
assert not overlay.exists()
vms = result_data(await client.call_tool("list_vms", {}))
assert vms == []
result = result_data(await client.call_tool("list_vms", {}))
assert result["vms"] == []
async def test_sandbox_destroy_refuses_non_sandbox(dirs, all_pids_dead):

View File

@ -62,3 +62,38 @@ async def test_snapshot_list_raw(dirs, fake_qmp):
async with Client(mcp) as client:
data = result_data(await client.call_tool("vm_snapshot_list", {"name": "vm1"}))
assert "clean" in data["snapshots_raw"]
async def test_snapshot_tag_rejects_quotes_and_specials(dirs, fake_qmp):
"""HMP's tokenizer strips quotes, so 'x' and '\"x\"' would silently be the
same snapshot reject anything that isn't literal."""
write_registry(dirs, seeded_record(dirs, "vm1"))
async with Client(mcp) as client:
for tag in ['"quoted"', "semi;colon", "tab\tsep", "a" * 80]:
with pytest.raises(ToolError, match="Invalid snapshot tag"):
await client.call_tool("vm_snapshot_create", {"name": "vm1", "tag": tag})
async def test_savevm_gets_a_long_but_finite_timeout(dirs, fake_qmp):
"""A RAM-sized savevm must not be killed by the ordinary command timeout."""
from mcqemu.qmp import COMMAND_TIMEOUT
from mcqemu.tools.snapshots import SNAPSHOT_TIMEOUT
assert SNAPSHOT_TIMEOUT > COMMAND_TIMEOUT * 10
seen = {}
async def spy(client, command, arguments=None, timeout=None):
seen["timeout"] = timeout
return ""
import mcqemu.tools.snapshots as snapshots_mod
original = snapshots_mod.execute
snapshots_mod.execute = spy
try:
write_registry(dirs, seeded_record(dirs, "vm1"))
async with Client(mcp) as client:
await client.call_tool("vm_snapshot_create", {"name": "vm1", "tag": "clean"})
finally:
snapshots_mod.execute = original
assert seen["timeout"] == SNAPSHOT_TIMEOUT