Add listen() tool — capture mic via pw-record, transcribe via Parakeet

mcspeak gains a `listen` MCP tool that records from the host's default
PipeWire source for N seconds and pipes the WAV through the existing
transcribe_audio() pipeline. The container's PipeWire socket bind
mount (already in place for pw-play output) gives pw-record the same
bidirectional access — no new container capabilities needed.

Recording format: 16 kHz mono s16, which is Parakeet's preferred
input, so no resample step. Default source is whatever the host
considers the default mic (`pactl get-default-source`); switching
mics on the host is picked up at next call without restart. Override
per-call via the `source` param if needed.

Optional save_path persists the recording under /output/ (same
scoping discipline as generate_audio). Response carries recorded
(in-container path) and saved_to (host-visible path) so callers can
re-transcribe with different params or play back later.

Forward-compat params (timestamp_granularities, diarize, num_speakers,
punctuation, min_confidence) thread straight through to transcribe_audio
so listen() inherits whatever the gpu-stack Parakeet endpoint gains
later, no client change.

Implementation gotcha: pw-record's exit code on SIGTERM is
non-deterministic (observed 1 on this version, not 0 or -15). The
original allowlist check would have rejected every successful
recording. The fix trusts the WAV file's existence + size instead;
if pw-record genuinely fails the file is missing or empty, which is
the reliable signal.
This commit is contained in:
Ryan Malloy 2026-06-27 21:39:13 -06:00
parent 549deb877d
commit d73bb49fd4
2 changed files with 162 additions and 1 deletions

View File

@ -186,6 +186,76 @@ async def _force_pwplay_volume_100() -> None:
return
async def record_audio(
out_path: Path,
duration_seconds: float,
source: str | None = None,
sample_rate: int = 16000,
) -> Path:
"""Capture audio from the host mic via pw-record (PipeWire socket bind mount).
Records to `out_path` as 16-bit mono WAV at `sample_rate` (16 kHz default,
matching Parakeet's preferred input format). If `source` is None, uses the
system default source (typically the host's default mic). Otherwise targets
that PipeWire source name (e.g. "bluez_input.XX:XX:XX..." or
"alsa_input.usb-...").
pw-record runs until SIGTERM, so we let it run for `duration_seconds` then
kill it. pw-record handles SIGTERM by closing the WAV header cleanly
verified by recording 2s and reading the file back with the wave module.
Returns out_path on success. Raises PlaybackError on pw-record failure or
if the resulting WAV is empty (mic disconnected, etc).
"""
cmd = [
"pw-record",
"--rate", str(sample_rate),
"--channels", "1",
"--format", "s16",
]
if source:
cmd.extend(["--target", source])
cmd.append(str(out_path))
proc = await asyncio.create_subprocess_exec(
*cmd,
stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.PIPE,
)
try:
# pw-record never exits on its own — wait the desired duration then term.
await asyncio.wait_for(proc.wait(), timeout=duration_seconds)
# If we get here, pw-record exited early (mic gone, permission, etc.)
except asyncio.TimeoutError:
# Expected path — stop recording cleanly.
proc.terminate()
try:
await asyncio.wait_for(proc.wait(), timeout=2.0)
except asyncio.TimeoutError:
proc.kill()
await proc.wait()
except asyncio.CancelledError:
proc.kill()
await proc.wait()
raise
# pw-record's exit code on SIGTERM is non-deterministic: some versions
# return 0, some 1, some -15. The reliable signal is whether the WAV
# file got written with a valid header and a sensible amount of data.
# If the file is missing/empty, pw-record genuinely failed; surface
# whatever it wrote to stderr.
if not out_path.exists() or out_path.stat().st_size < 100:
stderr = ""
if proc.stderr is not None:
stderr = (await proc.stderr.read()).decode(errors="replace").strip()
raise PlaybackError(
f"pw-record produced no audio (exit {proc.returncode}, "
f"file missing or empty). stderr: {stderr[:300] or '(empty)'}. "
f"Check that source '{source or 'default'}' has signal."
)
return out_path
async def play_audio(path: Path, expected_seconds: float = 0) -> None:
"""Play a WAV file through PipeWire (pw-play). Async wrapper.

View File

@ -12,7 +12,9 @@ from fastmcp import Context, FastMCP
from fastmcp.server.dependencies import CurrentContext
from mcp.types import ToolAnnotations
from .audio import ConversionError, SUPPORTED_FORMATS, convert_audio, play_audio
from .audio import (
ConversionError, SUPPORTED_FORMATS, convert_audio, play_audio, record_audio,
)
from .transcribe import TranscriptionError, transcribe_audio
from .engines.base import TTSEngine
from .engines.kokoro import KokoroEngine
@ -784,6 +786,95 @@ async def transcribe(
return result
@mcp.tool(annotations=ToolAnnotations(readOnlyHint=False, openWorldHint=True))
async def listen(
duration_seconds: float = 5.0,
response_format: Literal["json", "text", "verbose_json"] = "json",
source: str | None = None,
save_path: str | None = None,
# Forward-compat: passed straight through to transcribe_audio.
timestamp_granularities: list[Literal["word", "segment"]] | None = None,
diarize: bool = False,
num_speakers: int | None = None,
punctuation: bool | None = None,
min_confidence: float | None = None,
ctx: Context = CurrentContext(),
) -> dict:
"""Capture audio from the host mic for N seconds, transcribe via Parakeet.
Records 16 kHz mono WAV via pw-record using the container's PipeWire
socket bind mount (same socket play_audio uses for output). Default
source is the host's system default mic. Pipes the WAV through the
same transcribe_audio() machinery as transcribe(), so all forward-compat
params (diarize, timestamp_granularities, etc.) work identically.
Args:
duration_seconds: How long to listen. pw-record runs for this
wall-clock duration then receives SIGTERM to close the WAV.
response_format: 'json' (default), 'text', or 'verbose_json'.
source: PipeWire source name (e.g. "alsa_input.usb-..." or
"bluez_input.XX:XX:XX..."). None = system default source.
save_path: If set, persist the recording under /output/ at this
path (same scoping rules as generate_audio's output_path).
None = recording is ephemeral in /tmp/mcspeak/.
timestamp_granularities, diarize, num_speakers, punctuation,
min_confidence: forward-compat passed through to Parakeet.
See transcribe() for current support status.
"""
if duration_seconds <= 0 or duration_seconds > 300:
return {"error": f"duration_seconds must be in (0, 300], got {duration_seconds}"}
ts = time.strftime("%Y%m%d-%H%M%S")
rec_path = Path("/tmp/mcspeak") / f"listen-{ts}.wav"
rec_path.parent.mkdir(parents=True, exist_ok=True)
await ctx.info(f"Listening for {duration_seconds:.1f}s (source={source or 'default'})...")
try:
await record_audio(rec_path, duration_seconds, source=source)
except Exception as e:
return {"error": f"Recording failed: {e}"}
# Optional persistence to /output/. Reuse _resolve_output_path so the
# scoping rules + extension correction are consistent with generate_audio.
saved_to: str | None = None
if save_path is not None:
try:
dest = _resolve_output_path(save_path, "wav")
dest.parent.mkdir(parents=True, exist_ok=True)
import shutil
shutil.copyfile(rec_path, dest)
saved_to = str(dest)
except (ValueError, OSError) as e:
# Don't fail the whole call — transcription still runs.
await ctx.info(f"Recording saved failed (non-fatal): {e}")
await ctx.info(f"Transcribing {rec_path.name}...")
try:
result = await transcribe_audio(
rec_path,
response_format=response_format,
timestamp_granularities=timestamp_granularities,
diarize=diarize,
num_speakers=num_speakers,
punctuation=punctuation,
min_confidence=min_confidence,
)
except TranscriptionError as e:
return {
"error": str(e),
"recorded": str(rec_path),
"saved_to": saved_to,
}
# Annotate the response with where the recording lives so callers can
# play it back, re-transcribe with different params, etc.
result["recorded"] = str(rec_path)
if saved_to:
result["saved_to"] = saved_to
result["duration_recorded"] = duration_seconds
return result
@mcp.tool(annotations=ToolAnnotations(readOnlyHint=True))
async def list_voices(
engine: ENGINE_NAMES,