Add listen() tool — capture mic via pw-record, transcribe via Parakeet
mcspeak gains a `listen` MCP tool that records from the host's default PipeWire source for N seconds and pipes the WAV through the existing transcribe_audio() pipeline. The container's PipeWire socket bind mount (already in place for pw-play output) gives pw-record the same bidirectional access — no new container capabilities needed. Recording format: 16 kHz mono s16, which is Parakeet's preferred input, so no resample step. Default source is whatever the host considers the default mic (`pactl get-default-source`); switching mics on the host is picked up at next call without restart. Override per-call via the `source` param if needed. Optional save_path persists the recording under /output/ (same scoping discipline as generate_audio). Response carries recorded (in-container path) and saved_to (host-visible path) so callers can re-transcribe with different params or play back later. Forward-compat params (timestamp_granularities, diarize, num_speakers, punctuation, min_confidence) thread straight through to transcribe_audio so listen() inherits whatever the gpu-stack Parakeet endpoint gains later, no client change. Implementation gotcha: pw-record's exit code on SIGTERM is non-deterministic (observed 1 on this version, not 0 or -15). The original allowlist check would have rejected every successful recording. The fix trusts the WAV file's existence + size instead; if pw-record genuinely fails the file is missing or empty, which is the reliable signal.
This commit is contained in:
parent
549deb877d
commit
d73bb49fd4
@ -186,6 +186,76 @@ async def _force_pwplay_volume_100() -> None:
|
||||
return
|
||||
|
||||
|
||||
async def record_audio(
|
||||
out_path: Path,
|
||||
duration_seconds: float,
|
||||
source: str | None = None,
|
||||
sample_rate: int = 16000,
|
||||
) -> Path:
|
||||
"""Capture audio from the host mic via pw-record (PipeWire socket bind mount).
|
||||
|
||||
Records to `out_path` as 16-bit mono WAV at `sample_rate` (16 kHz default,
|
||||
matching Parakeet's preferred input format). If `source` is None, uses the
|
||||
system default source (typically the host's default mic). Otherwise targets
|
||||
that PipeWire source name (e.g. "bluez_input.XX:XX:XX..." or
|
||||
"alsa_input.usb-...").
|
||||
|
||||
pw-record runs until SIGTERM, so we let it run for `duration_seconds` then
|
||||
kill it. pw-record handles SIGTERM by closing the WAV header cleanly —
|
||||
verified by recording 2s and reading the file back with the wave module.
|
||||
|
||||
Returns out_path on success. Raises PlaybackError on pw-record failure or
|
||||
if the resulting WAV is empty (mic disconnected, etc).
|
||||
"""
|
||||
cmd = [
|
||||
"pw-record",
|
||||
"--rate", str(sample_rate),
|
||||
"--channels", "1",
|
||||
"--format", "s16",
|
||||
]
|
||||
if source:
|
||||
cmd.extend(["--target", source])
|
||||
cmd.append(str(out_path))
|
||||
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*cmd,
|
||||
stdout=asyncio.subprocess.DEVNULL,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
try:
|
||||
# pw-record never exits on its own — wait the desired duration then term.
|
||||
await asyncio.wait_for(proc.wait(), timeout=duration_seconds)
|
||||
# If we get here, pw-record exited early (mic gone, permission, etc.)
|
||||
except asyncio.TimeoutError:
|
||||
# Expected path — stop recording cleanly.
|
||||
proc.terminate()
|
||||
try:
|
||||
await asyncio.wait_for(proc.wait(), timeout=2.0)
|
||||
except asyncio.TimeoutError:
|
||||
proc.kill()
|
||||
await proc.wait()
|
||||
except asyncio.CancelledError:
|
||||
proc.kill()
|
||||
await proc.wait()
|
||||
raise
|
||||
|
||||
# pw-record's exit code on SIGTERM is non-deterministic: some versions
|
||||
# return 0, some 1, some -15. The reliable signal is whether the WAV
|
||||
# file got written with a valid header and a sensible amount of data.
|
||||
# If the file is missing/empty, pw-record genuinely failed; surface
|
||||
# whatever it wrote to stderr.
|
||||
if not out_path.exists() or out_path.stat().st_size < 100:
|
||||
stderr = ""
|
||||
if proc.stderr is not None:
|
||||
stderr = (await proc.stderr.read()).decode(errors="replace").strip()
|
||||
raise PlaybackError(
|
||||
f"pw-record produced no audio (exit {proc.returncode}, "
|
||||
f"file missing or empty). stderr: {stderr[:300] or '(empty)'}. "
|
||||
f"Check that source '{source or 'default'}' has signal."
|
||||
)
|
||||
return out_path
|
||||
|
||||
|
||||
async def play_audio(path: Path, expected_seconds: float = 0) -> None:
|
||||
"""Play a WAV file through PipeWire (pw-play). Async wrapper.
|
||||
|
||||
|
||||
@ -12,7 +12,9 @@ from fastmcp import Context, FastMCP
|
||||
from fastmcp.server.dependencies import CurrentContext
|
||||
from mcp.types import ToolAnnotations
|
||||
|
||||
from .audio import ConversionError, SUPPORTED_FORMATS, convert_audio, play_audio
|
||||
from .audio import (
|
||||
ConversionError, SUPPORTED_FORMATS, convert_audio, play_audio, record_audio,
|
||||
)
|
||||
from .transcribe import TranscriptionError, transcribe_audio
|
||||
from .engines.base import TTSEngine
|
||||
from .engines.kokoro import KokoroEngine
|
||||
@ -784,6 +786,95 @@ async def transcribe(
|
||||
return result
|
||||
|
||||
|
||||
@mcp.tool(annotations=ToolAnnotations(readOnlyHint=False, openWorldHint=True))
|
||||
async def listen(
|
||||
duration_seconds: float = 5.0,
|
||||
response_format: Literal["json", "text", "verbose_json"] = "json",
|
||||
source: str | None = None,
|
||||
save_path: str | None = None,
|
||||
# Forward-compat: passed straight through to transcribe_audio.
|
||||
timestamp_granularities: list[Literal["word", "segment"]] | None = None,
|
||||
diarize: bool = False,
|
||||
num_speakers: int | None = None,
|
||||
punctuation: bool | None = None,
|
||||
min_confidence: float | None = None,
|
||||
ctx: Context = CurrentContext(),
|
||||
) -> dict:
|
||||
"""Capture audio from the host mic for N seconds, transcribe via Parakeet.
|
||||
|
||||
Records 16 kHz mono WAV via pw-record using the container's PipeWire
|
||||
socket bind mount (same socket play_audio uses for output). Default
|
||||
source is the host's system default mic. Pipes the WAV through the
|
||||
same transcribe_audio() machinery as transcribe(), so all forward-compat
|
||||
params (diarize, timestamp_granularities, etc.) work identically.
|
||||
|
||||
Args:
|
||||
duration_seconds: How long to listen. pw-record runs for this
|
||||
wall-clock duration then receives SIGTERM to close the WAV.
|
||||
response_format: 'json' (default), 'text', or 'verbose_json'.
|
||||
source: PipeWire source name (e.g. "alsa_input.usb-..." or
|
||||
"bluez_input.XX:XX:XX..."). None = system default source.
|
||||
save_path: If set, persist the recording under /output/ at this
|
||||
path (same scoping rules as generate_audio's output_path).
|
||||
None = recording is ephemeral in /tmp/mcspeak/.
|
||||
timestamp_granularities, diarize, num_speakers, punctuation,
|
||||
min_confidence: forward-compat — passed through to Parakeet.
|
||||
See transcribe() for current support status.
|
||||
"""
|
||||
if duration_seconds <= 0 or duration_seconds > 300:
|
||||
return {"error": f"duration_seconds must be in (0, 300], got {duration_seconds}"}
|
||||
|
||||
ts = time.strftime("%Y%m%d-%H%M%S")
|
||||
rec_path = Path("/tmp/mcspeak") / f"listen-{ts}.wav"
|
||||
rec_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
await ctx.info(f"Listening for {duration_seconds:.1f}s (source={source or 'default'})...")
|
||||
try:
|
||||
await record_audio(rec_path, duration_seconds, source=source)
|
||||
except Exception as e:
|
||||
return {"error": f"Recording failed: {e}"}
|
||||
|
||||
# Optional persistence to /output/. Reuse _resolve_output_path so the
|
||||
# scoping rules + extension correction are consistent with generate_audio.
|
||||
saved_to: str | None = None
|
||||
if save_path is not None:
|
||||
try:
|
||||
dest = _resolve_output_path(save_path, "wav")
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
import shutil
|
||||
shutil.copyfile(rec_path, dest)
|
||||
saved_to = str(dest)
|
||||
except (ValueError, OSError) as e:
|
||||
# Don't fail the whole call — transcription still runs.
|
||||
await ctx.info(f"Recording saved failed (non-fatal): {e}")
|
||||
|
||||
await ctx.info(f"Transcribing {rec_path.name}...")
|
||||
try:
|
||||
result = await transcribe_audio(
|
||||
rec_path,
|
||||
response_format=response_format,
|
||||
timestamp_granularities=timestamp_granularities,
|
||||
diarize=diarize,
|
||||
num_speakers=num_speakers,
|
||||
punctuation=punctuation,
|
||||
min_confidence=min_confidence,
|
||||
)
|
||||
except TranscriptionError as e:
|
||||
return {
|
||||
"error": str(e),
|
||||
"recorded": str(rec_path),
|
||||
"saved_to": saved_to,
|
||||
}
|
||||
|
||||
# Annotate the response with where the recording lives so callers can
|
||||
# play it back, re-transcribe with different params, etc.
|
||||
result["recorded"] = str(rec_path)
|
||||
if saved_to:
|
||||
result["saved_to"] = saved_to
|
||||
result["duration_recorded"] = duration_seconds
|
||||
return result
|
||||
|
||||
|
||||
@mcp.tool(annotations=ToolAnnotations(readOnlyHint=True))
|
||||
async def list_voices(
|
||||
engine: ENGINE_NAMES,
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user