From e65a1207478f93bccd9ce72d435870a4ad5f6b37 Mon Sep 17 00:00:00 2001 From: Ryan Malloy Date: Sat, 4 Jul 2026 13:02:05 -0600 Subject: [PATCH] Trim mcspeak context footprint (docstrings + listen json result) Two pure-context wins, no behavior change: - Tighten the speak() and listen() docstrings, which are permanent per-turn context. speak 480->197 tok, listen 548->250 tok; all-tools fixed cost ~1783->~1203 tok/turn. Keeps the load-bearing cues (call speak in parallel; speak-then-listen sequentially) and drops the progress/ducking/chunking prose the calling model doesn't need. - Slim the default listen() 'json' result to just {text} + recording/vad metadata. The gateway returns a null-heavy segments[]/usage/words blob that only burned context; 'verbose_json' still returns the full segments. --- src/mcspeak/server.py | 106 ++++++++++++++---------------------------- 1 file changed, 35 insertions(+), 71 deletions(-) diff --git a/src/mcspeak/server.py b/src/mcspeak/server.py index 5bc9bb2..5ee6986 100644 --- a/src/mcspeak/server.py +++ b/src/mcspeak/server.py @@ -566,47 +566,24 @@ async def speak( project: str | None = None, ctx: Context = CurrentContext(), ) -> dict: - """Synthesize text and play it through the host speakers. + """Speak text through the host speakers. - Two ways to use this: + Two modes: + - Milestone ping (default): a status update the person overhears. Call it + IN PARALLEL with your next tool; don't block on it. The return value is + informational only. + - Conversation: if you want a spoken reply, let speak() finish, THEN call + listen() (sequentially, never parallel, or the mic records your voice). - 1. MILESTONE PING (default) — a status update the person overhears ("build - done", "tests green"). CALL THIS IN PARALLEL with your next tool; don't - block on it alone. The return value (speech_id, duration) is informational - and never needed for subsequent reasoning. - - 2. VOICE CONVERSATION — you asked the person something and want their spoken - answer. Then speak() and listen() form a turn: let speak() finish, THEN - call listen() to capture the reply. Do NOT run them in parallel here — the - mic must open after playback ends or it records your own voice. If your - text ends in a question or otherwise invites a response, reach for - listen(wait_for_silence=True) right after this returns. - - Audio plays through physical speakers, not into your context. Progress - notifications track playback either way. - - Progress notifications arrive every ~0.5s during playback: - - 5%: Entry tone played (audible acknowledgement) - - 30%: Synthesis complete - - 35-99%: Playing — progress tracks elapsed time vs duration - - 100%: Playback complete - - External media (YouTube, Spotify) is automatically ducked during speech - and fades back in after the exit tone. - - Long texts are automatically split into sentences and pipelined — the - first sentence plays within ~1-2s while remaining sentences synthesize - in the background. - - Audio is queued so agents don't talk over each other. Urgent messages - jump ahead of normal-priority items. + Audio is queued (urgent messages jump ahead) and plays through physical + speakers, not into your context. Args: - text: Text to speak. Orpheus supports emotion tags like , , etc. - engine: TTS engine to use. kokoro is fastest, orpheus is most expressive. - voice: Voice name (use list_voices to see options). None = auto-assigned by project. - urgent: If True, this message jumps ahead of normal-priority items. - project: Project name for voice identity (auto-detected from MCP roots if omitted). + text: Text to speak (Orpheus supports , tags). + engine: kokoro (fastest), piper, or orpheus (most expressive). + voice: Voice name; None = auto-assigned per project. + urgent: Jump ahead of normal-priority items. + project: Project name for voice identity (auto-detected if omitted). """ engines, queue, voice_cache, entry_tone, ducker = _get_state(ctx) @@ -823,44 +800,25 @@ async def listen( min_confidence: float | None = None, ctx: Context = CurrentContext(), ) -> dict: - """Capture audio from the host mic, transcribe via Parakeet, return the text. + """Capture the host mic, transcribe via Parakeet, and return the text. - This is how you HEAR THE PERSON BACK. Pair it with speak() to hold a - voice conversation: speak your question (let it finish), then call listen() - to capture their spoken answer. The transcribed text comes back to you in - the result, so unlike speak() you DO use the return value. For a natural - turn, set wait_for_silence=True so recording ends when they stop talking - instead of running the full duration. - - Records 16 kHz mono WAV via pw-record using the container's PipeWire - socket bind mount (same socket play_audio uses for output). Default - source is the host's system default mic. Pipes the WAV through the - same transcribe_audio() machinery as transcribe(), so all forward-compat - params (diarize, timestamp_granularities, etc.) work identically. - - A "ready to talk" tone plays before recording opens and a "got it" tone - after it closes, so the person knows exactly when to speak. + This is how you HEAR THE PERSON BACK: unlike speak(), you DO use the + return value. Pair with speak() for a voice turn: speak your question (let + it finish), then listen() for the answer. Empty text or a timeout means + re-prompt; the recording is saved under /tmp/mcspeak/ for retry via + transcribe(). Args: - duration_seconds: Max seconds to listen. With wait_for_silence this is - an upper bound; otherwise pw-record runs the full duration then - receives SIGTERM to close the WAV. - wait_for_silence: Stop as soon as the person stops talking (voice- - activity detection) instead of recording the whole duration. Best - for conversational turns — recommended when capturing a reply. - silence_threshold_ms: With wait_for_silence, how long a pause counts as - "they're done" (default 2200ms). - vad_aggressiveness: webrtcvad mode 0-3 (0 lax, 3 strict). 2 balances - rejecting background noise against catching soft speech. - response_format: 'json' (default), 'text', or 'verbose_json'. - source: PipeWire source name (e.g. "alsa_input.usb-..." or - "bluez_input.XX:XX:XX..."). None = system default source. - save_path: If set, persist the recording under /output/ at this - path (same scoping rules as generate_audio's output_path). - None = recording is ephemeral in /tmp/mcspeak/. + duration_seconds: Max seconds (upper bound with wait_for_silence). + wait_for_silence: Stop when the person stops talking (VAD). Default on. + silence_threshold_ms: Pause that counts as "done" (default 2200). + vad_aggressiveness: webrtcvad 0-3 (default 3, strict). + response_format: 'json' (default: text + vad only), 'text', or + 'verbose_json' (adds segments[]). + source: PipeWire source name; None = default mic. + save_path: Persist the recording under /output/ (else ephemeral). timestamp_granularities, diarize, num_speakers, punctuation, - min_confidence: forward-compat — passed through to Parakeet. - See transcribe() for current support status. + min_confidence: forward-compat, passed to Parakeet. """ if duration_seconds <= 0 or duration_seconds > 300: return {"error": f"duration_seconds must be in (0, 300], got {duration_seconds}"} @@ -952,6 +910,12 @@ async def listen( "saved_to": saved_to, } + # Slim the default 'json' result: the gateway returns a null-heavy + # segments[]/usage/words blob that just burns context. Keep only the text; + # 'verbose_json' callers who explicitly want segments still get everything. + if response_format == "json" and isinstance(result, dict): + result = {"text": result.get("text", "")} + # Annotate the response with where the recording lives so callers can # play it back, re-transcribe with different params, etc. result["recorded"] = str(rec_path)