From 5db7876dac16294933dc0b6882d78749dcd849be Mon Sep 17 00:00:00 2001 From: Ryan Malloy Date: Fri, 3 Jul 2026 20:08:04 -0600 Subject: [PATCH 1/3] Stream utterances through the speech queue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the per-chunk queue model with whole streaming utterances so concurrent projects never interleave. Each speak() reserves one ordering slot up front and streams its synthesized chunks through an internal channel; the consumer stays locked to that utterance from entry tone to exit tone, preserving synth/playback pipelining without letting another project's audio wedge between sentences. Also folded in: - Reap synthesized WAVs after playback (and on cancel/shutdown/queue-full), fixing an unbounded /tmp/mcspeak leak — one file per spoken sentence was never deleted. - Secretary announcements: announce a project by name on speaker-change or after a lull, synthesized in a reserved voice (Kokoro) that is excluded from the project auto-assignment pool. Decision made at play time so urgent reordering is respected. - Call-waiting: a brief blip mixed over the current message when a different project queues up, gated to once per playing turn so it never spams. - Unify _speak_single/_speak_chunked into one _speak path. - Status-aware progress: a queued item now reports "waiting in line" instead of racing to a false "playing 99%". - Synchronous abort flag closes a cancel-race where a chunk synthesized during the cancel-tone window could leak past the reap. --- src/mcspeak/queue.py | 530 ++++++++++++++++++++++++---------- src/mcspeak/server.py | 475 +++++++++++++++++------------- src/mcspeak/voice_identity.py | 4 + 3 files changed, 652 insertions(+), 357 deletions(-) diff --git a/src/mcspeak/queue.py b/src/mcspeak/queue.py index e6b6b35..aec38dc 100644 --- a/src/mcspeak/queue.py +++ b/src/mcspeak/queue.py @@ -1,12 +1,20 @@ -"""Speech queue — producer-consumer pattern with priority and cancellation. +"""Speech queue — producer-consumer with priority, streaming utterances, cancellation. -Replaces the original asyncio.Lock approach. A dedicated consumer coroutine -pulls work items from a bounded PriorityQueue and plays them one at a time. -Producers call enqueue() which returns immediately with a speech_id. -Use get_status() to check playback outcomes asynchronously. +A dedicated consumer coroutine pulls whole *utterances* from a bounded +PriorityQueue and plays them one at a time. Each utterance streams its +synthesized chunks in through an internal channel, so playback of chunk N +overlaps synthesis of chunk N+1 (pipelining) WITHOUT letting a second +project's audio interleave: the consumer stays locked to a single utterance +from its entry tone through its exit tone. + +This is what makes the queue behave like a secretary when several projects +speak at once — each message is played whole, in order, one at a time, and +a project is announced by name when the speaker changes (or returns after a +lull). Priority tiers: - 0 = urgent (preempts normal items in the queue) + 0 = urgent (jumps ahead of *queued* normal utterances; never chops the + one already playing) 1 = normal (default) """ @@ -31,30 +39,99 @@ class Priority(IntEnum): MAX_QUEUE_DEPTH = 20 MAX_OUTCOMES = 100 +# Sentinel pushed onto an utterance's chunk channel to mark "no more chunks". +_END = object() + + +def _unlink_quietly(path: Path | str | None) -> None: + """Best-effort delete of a synthesized WAV. Swallows all OS errors. + + Speech WAVs (chunks + the project preamble) are ephemeral — once pw-play + has read them off disk they're never needed again. Deleting them here is + what stops /tmp/mcspeak from growing one file per spoken sentence for the + container's whole lifetime. Shared tone WAVs (_tone-*.wav) are never + passed here, so they survive. + """ + if path is None: + return + try: + Path(path).unlink(missing_ok=True) + except OSError: + pass + @dataclass(order=True) -class _WorkItem: - """A prioritized playback request.""" +class _Utterance: + """One speak() call — an ordered stream of synthesized chunks played as a unit. + + Ordering (priority, sequence) is fixed the moment the utterance is created, + so all of its chunks travel together: nothing another project enqueues can + sort between them. Chunks arrive lazily through `chunks` (a channel closed + by an `_END` sentinel), which is what preserves synth/playback pipelining + without giving up per-message coherence. + """ priority: int - sequence: int # tiebreaker for same-priority items (FIFO) - result: TTSResult = field(compare=False) - future: asyncio.Future = field(compare=False) + sequence: int speech_id: str = field(default="", compare=False) - enqueued_at: float = field(default_factory=time.time, compare=False) - suppress_exit_tone: bool = field(default=False, compare=False) - # Entry tone deferred from speak() handler when the queue was busy at - # call time. Consumer plays this right before the item's audio so the - # tone doesn't overlap a previously-playing item. None = no entry tone - # (either no tone configured, or already played immediately by the handler). + project: str | None = field(default=None, compare=False) + engine: str = field(default="", compare=False) + voice: str = field(default="", compare=False) + # Entry tone deferred from the speak() handler when the queue was busy at + # call time. Consumer plays it right before this utterance so it doesn't + # overlap a previously-playing item. Shared tone WAV — never unlinked. entry_tone: Path | None = field(default=None, compare=False) + # Short "." preamble in this utterance's own voice. The consumer + # decides at play time whether to actually play it (see _should_announce). + # Per-message WAV — unlinked whether or not it ends up played. + announce_audio: Path | None = field(default=None, compare=False) + future: asyncio.Future | None = field(default=None, compare=False) + enqueued_at: float = field(default_factory=time.time, compare=False) + chunks: asyncio.Queue = field(default_factory=asyncio.Queue, compare=False) + # Running total of chunk durations — grows as chunks stream in. Used for + # progress/shutdown ETA only, never for control flow. + duration_estimate: float = field(default=0.0, compare=False) + # Set synchronously the instant this utterance is cancelled/abandoned, so a + # producer mid-synthesis sees it BEFORE the consumer has finished resolving + # the future (which can lag behind a ~0.3s cancel tone). Without this, a + # chunk synthesized during that window would slip into an already-reaped + # queue and leak. Distinct from future.done() for exactly that timing reason. + _aborted: bool = field(default=False, compare=False) + + def abort(self) -> None: + """Signal — synchronously — that nothing more should be synthesized.""" + self._aborted = True + + async def add_chunk(self, result: TTSResult) -> None: + """Producer: hand a synthesized chunk to the consumer. + + If we've been aborted, drop and reap the chunk instead of enqueuing it — + the consumer has already moved on and won't reap anything added late. + """ + if self.aborted: + _unlink_quietly(result.audio_path) + return + self.duration_estimate += result.duration_seconds + await self.chunks.put(result) + + async def close(self) -> None: + """Producer: signal no more chunks. MUST be called (even on error).""" + await self.chunks.put(_END) + + @property + def aborted(self) -> bool: + """True once cancelled/abandoned — producers poll this between chunks. + + Either the explicit abort flag (set the moment cancel fires) or the + future resolving early trips it. + """ + return self._aborted or (self.future is not None and self.future.done()) class SpeechQueue: """Bounded priority queue with a dedicated playback consumer. - Call start() to launch the consumer, stop() to cancel it and - drain pending items. + Call start() to launch the consumer, stop() to gracefully drain it. """ def __init__( @@ -65,12 +142,15 @@ class SpeechQueue: cancel_tone: Path | None = None, shutdown_timeout: float = 30.0, ducker: MediaDucker | None = None, + announce_mode: str = "secretary", + reintroduce_after: float = 120.0, + call_waiting_tone: Path | None = None, ) -> None: - self._queue: asyncio.PriorityQueue[_WorkItem] = asyncio.PriorityQueue( + self._queue: asyncio.PriorityQueue[_Utterance] = asyncio.PriorityQueue( maxsize=max_depth ) self._consumer_task: asyncio.Task | None = None - self._current: _WorkItem | None = None + self._current: _Utterance | None = None self._item_cancelled = False self._counter = 0 self._max_depth = max_depth @@ -82,6 +162,17 @@ class SpeechQueue: self._ducker = ducker self._outcomes: OrderedDict[str, dict] = OrderedDict() self._futures: dict[str, asyncio.Future] = {} + # Secretary state — who spoke last and when, for announcement decisions. + self._announce_mode = announce_mode + self._reintroduce_after = reintroduce_after + self._last_project: str | None = None + self._last_spoke_at: float = 0.0 + # Call-waiting: a brief blip mixed over the current message when another + # project queues up. _call_waiting_fired gates it to once per playing + # turn (reset when a new utterance starts playing) so it never spams. + self._call_waiting_tone = call_waiting_tone + self._call_waiting_fired = False + self._bg_tasks: set[asyncio.Task] = set() def _next_speech_id(self) -> str: self._counter += 1 @@ -109,11 +200,37 @@ class SpeechQueue: """ return self._current is None and self._queue.empty() + def should_prepare_announcement(self, project: str | None) -> bool: + """Cheap producer-side hint: is it worth synthesizing a preamble? + + The real decision is made at play time in _should_announce (order isn't + final until then). This just lets the common case — one project talking + to itself with an empty queue — skip the wasted preamble synthesis. + Errs toward preparing whenever anything could reorder or the speaker + might have changed. + """ + if project is None or self._announce_mode == "off": + return False + if self._announce_mode == "always": + return True + # secretary: only skip if we're confident the same project is + # continuing and nothing queued/playing could slip in front. + if ( + self._last_project == project + and (time.time() - self._last_spoke_at) <= self._reintroduce_after + and self._current is None + and self._queue.empty() + ): + return False + return True + def status(self) -> dict: return { "current_speaker": self.current_speaker, + "current_project": self._current.project if self._current else None, "queue_depth": self._queue.qsize(), "max_depth": self._max_depth, + "last_project": self._last_project, } def start(self) -> None: @@ -125,21 +242,39 @@ class SpeechQueue: ) def _drain_remaining(self) -> None: - """Drain pending items, recording shutdown outcomes. Sync — safe for finally.""" + """Drain pending utterances, recording shutdown outcomes. Sync — safe for finally.""" while not self._queue.empty(): try: - item = self._queue.get_nowait() - outcome = { - "played": False, - "error": "Queue shut down", - "engine": item.result.engine, - "voice": item.result.voice, - } - if not item.future.done(): - item.future.set_result(outcome) - self._record_outcome(item.speech_id, outcome) + utt = self._queue.get_nowait() except asyncio.QueueEmpty: break + outcome = { + "played": False, + "error": "Queue shut down", + "engine": utt.engine, + "voice": utt.voice, + } + if utt.future is not None and not utt.future.done(): + utt.future.set_result(outcome) + self._record_outcome(utt.speech_id, outcome) + # Never played, so the consumer's cleanup never ran — reap files. + utt.abort() + self._reap_utterance_files(utt) + + def _reap_utterance_files(self, utt: _Utterance) -> None: + """Delete an utterance's per-message WAVs that never got played. + + The preamble plus any chunks already sitting in the channel. Best-effort + and synchronous — the entry tone is a shared WAV and is left alone. + """ + _unlink_quietly(utt.announce_audio) + while True: + try: + chunk = utt.chunks.get_nowait() + except asyncio.QueueEmpty: + break + if chunk is not _END: + _unlink_quietly(chunk.audio_path) def _force_cancel_consumer(self) -> None: """Cancel consumer task if still running.""" @@ -149,17 +284,16 @@ class SpeechQueue: async def stop(self) -> None: """Gracefully stop the consumer — let current speech finish, then drain. - Waits up to shutdown_timeout seconds for the currently-playing item to - complete. If it doesn't finish in time, falls back to hard cancel. + Waits up to shutdown_timeout seconds for the currently-playing utterance + to complete. If it doesn't finish in time, falls back to hard cancel. Drain always runs, even if stop() itself is cancelled. """ self._stopped = True try: if self._consumer_task and not self._consumer_task.done(): - # Log what we're waiting for if self._current: elapsed = time.time() - self._current.enqueued_at - remaining = max(0, self._current.result.duration_seconds - elapsed) + remaining = max(0, self._current.duration_estimate - elapsed) print( f"Speech queue: waiting for current audio " f"(~{remaining:.0f}s remaining, budget {self._shutdown_timeout:.0f}s)", @@ -168,17 +302,14 @@ class SpeechQueue: else: print("Speech queue: consumer idle, shutting down", file=sys.stderr) - # Consumer polls _stopped every 0.5s between items, so it will - # exit the while loop after finishing the current item. + # Consumer polls _stopped every 0.5s between utterances, so it + # exits the while loop after finishing the current one. try: await asyncio.wait_for( asyncio.shield(self._consumer_task), timeout=self._shutdown_timeout, ) - print( - "Speech queue: graceful shutdown complete", - file=sys.stderr, - ) + print("Speech queue: graceful shutdown complete", file=sys.stderr) except asyncio.TimeoutError: print( f"Speech queue: shutdown timeout ({self._shutdown_timeout:.0f}s) " @@ -191,11 +322,9 @@ class SpeechQueue: except asyncio.CancelledError: pass except asyncio.CancelledError: - # stop() itself was cancelled — force-kill consumer before draining self._force_cancel_consumer() raise finally: - # Always restore media volumes on shutdown if self._ducker: try: await self._ducker.unduck() @@ -203,186 +332,269 @@ class SpeechQueue: pass # Non-fatal self._drain_remaining() - def _resolve_outcome(self, item: _WorkItem, outcome: dict) -> None: + def _resolve_outcome(self, utt: _Utterance, outcome: dict) -> None: """Set the future result and record the outcome.""" - if not item.future.done(): - item.future.set_result(outcome) - self._record_outcome(item.speech_id, outcome) + if utt.future is not None and not utt.future.done(): + utt.future.set_result(outcome) + self._record_outcome(utt.speech_id, outcome) + + def _should_announce(self, utt: _Utterance) -> bool: + """Play-time decision: introduce this utterance's project by name? + + Made here (not at enqueue) because urgent reordering means the real + speaker sequence isn't known until we pull the next utterance. + """ + if utt.announce_audio is None or utt.project is None: + return False + if self._announce_mode == "off": + return False + if self._announce_mode == "always": + return True + # secretary: announce on speaker change, or same speaker after a lull. + if self._last_project != utt.project: + return True + return (time.time() - self._last_spoke_at) > self._reintroduce_after + + async def _play_utterance(self, utt: _Utterance) -> None: + """Play one utterance whole: entry tone, announcement, chunks, exit tone. + + Blocks on the chunk channel between chunks — that's the pipelining seam + (consumer plays chunk N while the producer synthesizes chunk N+1). No + other utterance can be pulled until this returns, so nothing interleaves. + """ + # Deferred entry tone (queue was busy when speak() was called). + if utt.entry_tone: + try: + await play_audio(utt.entry_tone, expected_seconds=0.3) + except PlaybackError: + pass # Non-fatal — shared tone WAV, don't unlink + + # Secretary announcement — decided now that play order is final. + if utt.announce_audio: + if self._should_announce(utt): + try: + await play_audio(utt.announce_audio, expected_seconds=0.6) + except PlaybackError: + pass + _unlink_quietly(utt.announce_audio) # reap whether played or skipped + + # Stream chunks until the producer closes the channel. + while True: + chunk = await utt.chunks.get() + if chunk is _END: + break + try: + await play_audio( + chunk.audio_path, expected_seconds=chunk.duration_seconds + ) + finally: + _unlink_quietly(chunk.audio_path) + + # Exit tone once, at the very end — "roger" if the queue is now empty, + # "standby" if more utterances are waiting. + exit_t = ( + self._exit_tone if self._queue.qsize() == 0 else self._standby_tone + ) + if exit_t: + try: + await play_audio(exit_t, expected_seconds=0.3) + except PlaybackError: + pass + + # Remember who just spoke — drives the next utterance's announcement. + if utt.project is not None: + self._last_project = utt.project + self._last_spoke_at = time.time() + + self._resolve_outcome(utt, { + "played": True, + "engine": utt.engine, + "voice": utt.voice, + "project": utt.project, + "duration_seconds": utt.duration_estimate, + }) async def _consumer(self) -> None: - """Pull work items and play them sequentially. + """Pull utterances and play them sequentially, one whole message at a time. The _item_cancelled flag distinguishes cancel(speech_id) (item-level, continue loop) from shutdown cancellation (exit loop). """ while not self._stopped: try: - item = await asyncio.wait_for(self._queue.get(), timeout=0.5) + utt = await asyncio.wait_for(self._queue.get(), timeout=0.5) except asyncio.TimeoutError: continue # Re-check _stopped flag except asyncio.CancelledError: break - if item.future.cancelled(): + if utt.future is not None and utt.future.cancelled(): + # Cancelled while queued — skip it, reap its files. self._queue.task_done() + utt.abort() + self._reap_utterance_files(utt) continue - self._current = item + self._current = utt self._item_cancelled = False + # New turn — allow one call-waiting blip if others queue during it. + self._call_waiting_fired = False try: - # Deferred entry tone — speak() handler deferred this when the - # queue was busy at call time. Play it now so it lands right - # before this item's audio (no overlap with prior playback). - if item.entry_tone: - try: - await play_audio(item.entry_tone, expected_seconds=0.3) - except PlaybackError: - pass # Non-fatal - - await play_audio( - item.result.audio_path, - expected_seconds=item.result.duration_seconds, - ) - - # Play exit tone — "standby" if more queued, "roger" if done - # suppress_exit_tone skips tones between chunks of the same message - if not item.suppress_exit_tone: - exit_t = ( - self._exit_tone if self._queue.qsize() == 0 - else self._standby_tone - ) - if exit_t: - try: - await play_audio(exit_t, expected_seconds=0.3) - except PlaybackError: - pass # Non-fatal - - self._resolve_outcome(item, { - "played": True, - "file": str(item.result.audio_path), - "duration_seconds": item.result.duration_seconds, - "engine": item.result.engine, - "voice": item.result.voice, - }) - - except PlaybackError as e: - self._resolve_outcome(item, { - "played": False, - "error": str(e), - "file": str(item.result.audio_path), - "engine": item.result.engine, - "voice": item.result.voice, - }) + await self._play_utterance(utt) except asyncio.CancelledError: if self._item_cancelled: - # Item-level cancel via cancel() — play cancel tone, continue + # Item-level cancel via cancel() — cancel tone, then continue. self._item_cancelled = False + utt.abort() + self._reap_utterance_files(utt) if self._cancel_tone: try: await play_audio(self._cancel_tone, expected_seconds=0.3) except (PlaybackError, asyncio.CancelledError): pass - self._resolve_outcome(item, { + self._resolve_outcome(utt, { "played": False, "error": "Cancelled by client", - "engine": item.result.engine, - "voice": item.result.voice, + "engine": utt.engine, + "voice": utt.voice, }) - # Don't break — continue to next item + # Fall through to finally, keep looping. else: - # Shutdown — record and exit loop - self._resolve_outcome(item, { + # Shutdown — record and exit loop. + utt.abort() + self._reap_utterance_files(utt) + self._resolve_outcome(utt, { "played": False, "error": "Playback cancelled", - "engine": item.result.engine, - "voice": item.result.voice, + "engine": utt.engine, + "voice": utt.voice, }) break except Exception as e: - self._resolve_outcome(item, { + utt.abort() + self._reap_utterance_files(utt) + self._resolve_outcome(utt, { "played": False, "error": f"Unexpected error: {e}", - "engine": item.result.engine, - "voice": item.result.voice, + "engine": utt.engine, + "voice": utt.voice, }) finally: self._current = None self._queue.task_done() - # Unduck media when queue is empty — covers ALL exit paths - # (normal completion, PlaybackError, cancel, generic exception) + # Unduck media when queue is empty — covers ALL exit paths. if self._queue.qsize() == 0 and self._ducker and not self._stopped: try: await self._ducker.unduck() except Exception: pass # Non-fatal - async def enqueue( + def create_utterance( self, - result: TTSResult, priority: Priority = Priority.NORMAL, - suppress_exit_tone: bool = False, + project: str | None = None, + engine: str = "", + voice: str = "", entry_tone: Path | None = None, - ) -> dict: - """Enqueue a TTSResult for playback. Returns immediately with enqueue metadata. + announce_audio: Path | None = None, + ) -> tuple[_Utterance | None, dict]: + """Reserve a queue slot for a new utterance and register it for playback. - Does NOT block until playback finishes — use get_status(speech_id) to - check the outcome later. + Returns (utterance, meta). On success `utterance` is the _Utterance to + feed chunks into (add_chunk/close) and `meta` carries the speech_id and + queue position. On failure (shut down / full) `utterance` is None and + `meta` is an error dict — the caller should reap any announce_audio it + pre-synthesized. - If entry_tone is given, the consumer plays it immediately before this - item's audio (used when speak() handler deferred the tone because the - queue was busy at call time). - - Raises QueueFull if the queue is at max capacity (backpressure). + The utterance goes into the priority queue immediately (before its + chunks exist), which fixes its ordering vs. concurrent producers. The + consumer that pulls it then blocks on the chunk channel until chunks + arrive, so a slow synth just delays that one message — it never lets a + faster producer's audio interleave. """ if self._stopped: - return { + _unlink_quietly(announce_audio) + return None, { "queued": False, "error": "Queue is shut down", - "engine": result.engine, - "voice": result.voice, + "engine": engine, + "voice": voice, } speech_id = self._next_speech_id() loop = asyncio.get_running_loop() future: asyncio.Future = loop.create_future() - item = _WorkItem( + utt = _Utterance( priority=priority, sequence=self._counter, - result=result, - future=future, speech_id=speech_id, - suppress_exit_tone=suppress_exit_tone, + project=project, + engine=engine, + voice=voice, entry_tone=entry_tone, + announce_audio=announce_audio, + future=future, ) try: - self._queue.put_nowait(item) + self._queue.put_nowait(utt) except asyncio.QueueFull: - return { + _unlink_quietly(announce_audio) + return None, { "queued": False, "error": f"Queue full ({self._max_depth} items). Try again later.", "speech_id": speech_id, - "engine": result.engine, - "voice": result.voice, + "engine": engine, + "voice": voice, } self._futures[speech_id] = future - - return { + self._maybe_fire_call_waiting(utt) + return utt, { "speech_id": speech_id, "queued": True, "position": self._queue.qsize(), - "file": str(result.audio_path), - "duration_seconds": result.duration_seconds, - "engine": result.engine, - "voice": result.voice, + "engine": engine, + "voice": voice, } + def _maybe_fire_call_waiting(self, new_utt: _Utterance) -> None: + """Blip once over the current message when a DIFFERENT project queues up. + + Gated to once per playing turn (_call_waiting_fired, reset when a new + utterance starts playing) so a burst of queued messages doesn't + machine-gun the listener. Fires as a background task — a second pw-play + stream that PipeWire mixes over the currently-playing audio. + """ + cur = self._current + if ( + self._call_waiting_tone is None + or self._call_waiting_fired + or cur is None + or new_utt.project is None + or cur.project is None + or new_utt.project == cur.project + ): + return + self._call_waiting_fired = True + task = asyncio.create_task(self._play_call_waiting()) + self._bg_tasks.add(task) + task.add_done_callback(self._bg_tasks.discard) + + async def _play_call_waiting(self) -> None: + """Play the call-waiting blip over the current audio. Non-fatal.""" + if self._call_waiting_tone is None: + return + try: + await play_audio(self._call_waiting_tone, expected_seconds=0.2) + except Exception: + pass + async def wait_for_completion(self, speech_id: str) -> dict: """Await playback completion for a specific speech_id. @@ -391,25 +603,28 @@ class SpeechQueue: """ future = self._futures.get(speech_id) if future is None: - # Already completed and cleaned up, or unknown return self._outcomes.get(speech_id, {"status": "unknown", "speech_id": speech_id}) return await future def cancel(self, speech_id: str) -> dict: - """Cancel a queued or playing speech item. + """Cancel a queued or playing utterance. - If the item is currently playing, kills pw-play and the consumer plays - the cancel tone. If queued, marks the future as cancelled so the - consumer skips it. Returns cancellation status. + If it's currently playing, kills pw-play and the consumer plays the + cancel tone. If queued, marks the future cancelled so the consumer skips + it (and the producer stops synthesizing once it sees `aborted`). """ - # Currently playing — cancel via the consumer task + # Currently playing — cancel via the consumer task. if self._current and self._current.speech_id == speech_id: if self._consumer_task and not self._consumer_task.done(): + # Flag abort NOW (synchronously) so a producer still synthesizing + # stops before the consumer's cancel-tone delay lets a late chunk + # leak past the reap. + self._current.abort() self._item_cancelled = True self._consumer_task.cancel() return {"cancelled": True, "was": "playing", "speech_id": speech_id} - # Queued — cancel the future (consumer skips cancelled futures) + # Queued — cancel the future (consumer skips cancelled utterances). future = self._futures.get(speech_id) if future and not future.done(): future.cancel() @@ -427,36 +642,35 @@ class SpeechQueue: } def get_status(self, speech_id: str) -> dict: - """Check the status of a speech item by its ID. + """Check the status of an utterance by its ID. - Returns a dict with 'status' key: 'completed', 'playing', 'queued', or 'unknown'. + Returns a dict with 'status': 'completed', 'playing', 'queued', or 'unknown'. """ - # Check completed outcomes if speech_id in self._outcomes: outcome = self._outcomes[speech_id] return {"status": "completed", **outcome} - # Check currently playing if self._current and self._current.speech_id == speech_id: return { "status": "playing", "speech_id": speech_id, - "engine": self._current.result.engine, - "voice": self._current.result.voice, - "file": str(self._current.result.audio_path), - "duration_seconds": self._current.result.duration_seconds, + "project": self._current.project, + "engine": self._current.engine, + "voice": self._current.voice, + "duration_seconds": self._current.duration_estimate, } - # Scan queue (PriorityQueue._queue is the underlying heap list) + # Scan queue (PriorityQueue._queue is the underlying heap list). try: - for pos, item in enumerate(self._queue._queue, start=1): - if item.speech_id == speech_id: + for pos, utt in enumerate(self._queue._queue, start=1): + if utt.speech_id == speech_id: return { "status": "queued", "speech_id": speech_id, "position": pos, - "engine": item.result.engine, - "voice": item.result.voice, + "project": utt.project, + "engine": utt.engine, + "voice": utt.voice, } except AttributeError: pass # Internal API — degrade gracefully diff --git a/src/mcspeak/server.py b/src/mcspeak/server.py index 91d307e..3342177 100644 --- a/src/mcspeak/server.py +++ b/src/mcspeak/server.py @@ -13,9 +13,12 @@ from fastmcp.server.dependencies import CurrentContext from mcp.types import ToolAnnotations from .audio import ( - ConversionError, SUPPORTED_FORMATS, convert_audio, play_audio, record_audio, + ConversionError, + convert_audio, + play_audio, + record_audio, + record_audio_until_silence, ) -from .transcribe import TranscriptionError, transcribe_audio from .engines.base import TTSEngine from .engines.kokoro import KokoroEngine from .engines.orpheus import OrpheusEngine @@ -24,6 +27,7 @@ from .media_duck import MediaDucker from .queue import Priority, SpeechQueue from .settings import settings from .tones import generate_tones, resolve_tone +from .transcribe import TranscriptionError, transcribe_audio from .voice_identity import VoiceIdentityCache, get_project_name, resolve_voice ENGINE_NAMES = Literal["piper", "kokoro", "orpheus"] @@ -145,10 +149,22 @@ async def app_lifespan(server: FastMCP): # Generate tones and resolve which ones to use tone_paths = generate_tones(settings.output_dir) - entry_tone = resolve_tone(settings.entry_tone, tone_paths, "entry_tone", default="chirp") - exit_tone = resolve_tone(settings.exit_tone, tone_paths, "exit_tone", default="roger") + entry_tone = resolve_tone(settings.entry_tone, tone_paths, "entry_tone", default="heartbeat") + exit_tone = resolve_tone(settings.exit_tone, tone_paths, "exit_tone", default="soft-chord") cancel_tone = resolve_tone(settings.cancel_tone, tone_paths, "cancel_tone", default="scratch") standby_tone = tone_paths.get("standby") + # Call-waiting blip mixed over the current message when another project queues. + call_waiting_tone = resolve_tone( + settings.call_waiting_tone, tone_paths, "call_waiting_tone", default="call-waiting", + ) + # listen() bookends — distinct from speak's chirp/roger so the user can + # tell capture mode from playback mode by ear alone. + listen_start_tone = resolve_tone( + settings.listen_start_tone, tone_paths, "listen_start_tone", default="mf-dial", + ) + listen_end_tone = resolve_tone( + settings.listen_end_tone, tone_paths, "listen_end_tone", default="machine", + ) # --- Media ducker --- ducker: MediaDucker | None = None @@ -158,12 +174,18 @@ async def app_lifespan(server: FastMCP): fade_in_ms=settings.duck_fade_in_ms, ) + # Legacy announce_project=True maps to the "always" mode; otherwise use + # announce_mode (default "secretary": announce on speaker change / after a lull). + announce_mode = "always" if settings.announce_project else settings.announce_mode queue = SpeechQueue( exit_tone=exit_tone, standby_tone=standby_tone, cancel_tone=cancel_tone, shutdown_timeout=settings.shutdown_timeout, ducker=ducker, + announce_mode=announce_mode, + reintroduce_after=settings.reintroduce_after_seconds, + call_waiting_tone=call_waiting_tone, ) queue.start() @@ -181,7 +203,7 @@ async def app_lifespan(server: FastMCP): voice_cache = VoiceIdentityCache(persist_path=settings.voice_identity_file) identity_label = "on" if settings.voice_identity else "off" - announce_label = "+announce" if settings.announce_project else "" + announce_label = f"+announce:{announce_mode}" if announce_mode != "off" else "" print( f" Voice identity: {identity_label}{announce_label}", file=sys.stderr, @@ -199,6 +221,8 @@ async def app_lifespan(server: FastMCP): "voice_cache": voice_cache, "entry_tone": entry_tone, "ducker": ducker, + "listen_start_tone": listen_start_tone, + "listen_end_tone": listen_end_tone, } finally: print("McSpeak shutting down", file=sys.stderr) @@ -227,6 +251,14 @@ mcp = FastMCP( "~0.5s during playback so clients can show a progress indicator.\n\n" "If you need to check whether speech finished, use speech_status(speech_id) " "after the fact. Use generate_audio to synthesize without playing.\n\n" + "TALKING WITH THE PERSON (voice conversation): when you speak something " + "that expects a spoken reply — a question, a confirmation prompt, 'anything " + "else?' — follow the speak() with a listen() call to capture their answer. " + "Do this SEQUENTIALLY, not in parallel: let speak() finish (it blocks until " + "playback is done), THEN call listen(), so the mic doesn't record your own " + "voice. Prefer listen(wait_for_silence=True) so recording stops when they " + "stop talking. Rule of thumb: if you'd expect a human to answer out loud, " + "listen for it — don't just speak and move on.\n\n" "Engines: kokoro (fast ONNX, ~50 voices), piper (Wyoming/Docker), " "orpheus (LLM via llama-server, supports etc.). " "Always pass project= with the current project directory name (last path component) " @@ -300,59 +332,74 @@ async def _resolve_project_voice( engine_name: str, eng: TTSEngine, voice: str | None, - text: str, voice_cache: VoiceIdentityCache, project: str | None = None, -) -> tuple[str | None, str]: - """Auto-assign voice from project hint or MCP roots, optionally prefix text.""" - if voice is not None or not settings.voice_identity: - return voice, text +) -> tuple[str | None, str | None]: + """Resolve (voice, project_name) for this call. - # Explicit project param takes priority, then try MCP roots - project_name = project or await get_project_name(ctx) - if not project_name: - return voice, text + The project name is resolved regardless of whether a voice was passed + explicitly, because it drives the consumer's spoken announcements. A voice + is auto-assigned only when none was given and voice identity is enabled. + Announcements are handled downstream as a synthesized preamble (not by + prefixing the text), so this no longer mutates `text`. + """ + # Resolve the project name: explicit param wins, else try MCP roots. + project_name = project + if project_name is None and settings.voice_identity: + project_name = await get_project_name(ctx) - voice = await resolve_voice(engine_name, eng, project_name, voice_cache) - if settings.announce_project and voice is not None: - text = f"{project_name}. {text}" + # Auto-assign a voice only if the caller didn't pin one. + if voice is None and settings.voice_identity and project_name: + voice = await resolve_voice(engine_name, eng, project_name, voice_cache) - return voice, text + return voice, project_name # --------------------------------------------------------------------------- # Tools # --------------------------------------------------------------------------- -async def _speak_single( +async def _speak( eng: TTSEngine, engine: str, text: str, voice: str | None, + project: str | None, queue: SpeechQueue, entry_tone: Path | None, priority: Priority, ctx: Context, ducker: MediaDucker | None = None, + announce_eng: TTSEngine | None = None, + secretary_voice: str = "", ) -> dict: - """Single-shot speak path — synthesize full text, then enqueue. + """Stream one utterance's chunks into the queue as a single coherent unit. - Used for short texts (< 20 words) and as the fallback when text has - no sentence boundaries. + Short texts are one chunk; long texts split into sentence chunks that + pipeline (synth of chunk N+1 overlaps playback of chunk N). Either way the + whole message is ONE queue utterance, so when several projects speak at + once nothing interleaves: each message plays whole, in order. + + A short "." preamble is synthesized in this utterance's own voice + and handed to the consumer, which decides at play time whether to actually + announce (speaker changed, or same speaker after a lull). """ - # Entry tone gating — idle queue plays tone now (hides synth latency); - # busy queue defers tone to the consumer so it doesn't overlap whatever's - # already playing. Capture idle state before duck/synth changes it. + chunks = split_text(text) + n = len(chunks) + + # Entry tone gating: idle queue plays the tone now (hides synth latency); + # busy queue defers it to the consumer so it lands right before this + # utterance instead of over whatever's already playing. queue_was_idle = queue.is_idle() deferred_entry_tone: Path | None = ( None if queue_was_idle or entry_tone is None else entry_tone ) try: - # Duck external media with crossfade into entry tone + # Duck external media with crossfade into the entry tone. if ducker: duck_task = asyncio.create_task(ducker.duck()) - await asyncio.sleep(0.15) # Let fade start — media dips to ~70% + await asyncio.sleep(0.15) # Let fade start, media dips to ~70% if entry_tone and queue_was_idle: await _play_tone(entry_tone) # Tone crossfades over fading media await duck_task # Ensure duck completes @@ -360,155 +407,80 @@ async def _speak_single( await _play_tone(entry_tone) await ctx.report_progress(progress=5, total=100, message="Entry tone") - await ctx.info(f"Synthesizing with {engine}...") - result = await eng.synthesize(text, voice) - await ctx.report_progress(progress=30, total=100, message="Synthesizing...") + # Secretary preamble: a short "." spoken in the reserved + # secretary voice (always via Kokoro, so it sounds identical no matter + # which engine the project uses). Synthesized here, played (or skipped) + # by the consumer. Skip the synthesis entirely when we're confident it + # won't be needed (same project talking to itself, empty queue). + announce_audio: Path | None = None + if project and secretary_voice and queue.should_prepare_announcement(project): + try: + display = project.replace("-", " ").replace("_", " ") + announce_res = await (announce_eng or eng).synthesize( + f"{display}.", secretary_voice, + ) + announce_audio = announce_res.audio_path + except Exception as e: + await ctx.info(f"Announcement synth failed (non-fatal): {e}") - await ctx.info(f"Synthesized {result.duration_seconds:.1f}s audio, enqueueing...") - enqueue_result = await queue.enqueue( - result, priority=priority, entry_tone=deferred_entry_tone, + # Reserve the queue slot NOW: this fixes ordering vs. concurrent + # producers before any chunk exists. The consumer that pulls it blocks + # on the chunk channel until chunks arrive, so a slow synth only delays + # this one message; it never lets another project's audio slip in. + utt, meta = queue.create_utterance( + priority=priority, + project=project, + engine=engine, + voice=voice or eng.default_voice, + entry_tone=deferred_entry_tone, + announce_audio=announce_audio, ) + if utt is None: + # Queue full / shut down: create_utterance reaped the preamble. + return meta - if not enqueue_result.get("queued"): - return enqueue_result + # Stream chunks: synthesize each and hand it straight to the consumer. + try: + for i, chunk in enumerate(chunks): + if utt.aborted: + break # cancelled or shut down mid-synthesis, stop early + pct_synth = 5 + int(25 * (i + 1) / n) # 5-30% across all chunks + label = ( + f"Synthesizing with {engine}..." if n == 1 + else f"Synthesizing chunk {i + 1}/{n} with {engine}..." + ) + await ctx.info(label) + result = await eng.synthesize(chunk, voice) + await utt.add_chunk(result) + await ctx.report_progress( + progress=pct_synth, total=100, + message=("Synthesizing..." if n == 1 + else f"Synthesizing chunk {i + 1}/{n}..."), + ) + finally: + # ALWAYS close the channel, otherwise the consumer blocks forever + # waiting for a next chunk that never comes (synth error, or this + # handler was cancelled mid-loop). + await utt.close() - speech_id = enqueue_result["speech_id"] - duration = result.duration_seconds - await ctx.report_progress( - progress=35, total=100, - message="Playing audio \u2014 you can continue working", + await ctx.info("Enqueued for playback") + # NOTE: Do NOT use asyncio.wait_for() - see CLAUDE.md Python 3.13 pitfall. + # _await_with_progress owns 35→100 and distinguishes "queued/waiting" + # from "actually playing" so a queued item doesn't falsely report 99%. + outcome = await _await_with_progress( + queue, utt.speech_id, utt.duration_estimate, ctx, 35, ) - - await ctx.info("Playing...") - # NOTE: Do NOT use asyncio.wait_for() — see CLAUDE.md Python 3.13 pitfall - outcome = await _await_with_progress(queue, speech_id, duration, ctx, 35) - await ctx.report_progress(progress=100, total=100, message="Playback complete") + if n > 1: + outcome["chunks"] = n return outcome except asyncio.CancelledError: # speak() was cancelled (server shutdown or MCP client disconnect). - # Do NOT cancel the consumer — let it finish the current audio. - # Shutdown: queue.stop() in the lifespan finalizer handles graceful drain. - # Client cancel: use cancel_speech(speech_id) for explicit mid-playback stop. - raise - - -async def _speak_chunked( - eng: TTSEngine, - engine: str, - chunks: list[str], - voice: str | None, - queue: SpeechQueue, - entry_tone: Path | None, - priority: Priority, - ctx: Context, - ducker: MediaDucker | None = None, -) -> dict: - """Chunked speak path — pipeline synthesis with playback. - - Synthesizes sentence chunks one at a time and enqueues each immediately. - Playback of chunk N overlaps synthesis of chunk N+1. Non-final chunks - suppress exit/standby tones for seamless playback. - """ - n_chunks = len(chunks) - total_duration = 0.0 - all_speech_ids: list[str] = [] - first_enqueue_time: float | None = None - - # Entry tone gating (see _speak_single for full rationale). Only the FIRST - # chunk carries the deferred tone — subsequent chunks are continuations of - # the same message and don't get their own tone. - queue_was_idle = queue.is_idle() - deferred_entry_tone: Path | None = ( - None if queue_was_idle or entry_tone is None else entry_tone - ) - - try: - # Duck external media with crossfade into entry tone - if ducker: - duck_task = asyncio.create_task(ducker.duck()) - await asyncio.sleep(0.15) # Let fade start — media dips to ~70% - if entry_tone and queue_was_idle: - await _play_tone(entry_tone) # Tone crossfades over fading media - await duck_task # Ensure duck completes - elif entry_tone and queue_was_idle: - await _play_tone(entry_tone) - await ctx.report_progress(progress=5, total=100, message="Entry tone") - - for i, chunk in enumerate(chunks): - is_last = i == n_chunks - 1 - pct_synth = 5 + int(25 * (i + 1) / n_chunks) # 5-30% across all chunks - - await ctx.info(f"Synthesizing chunk {i + 1}/{n_chunks} with {engine}...") - result = await eng.synthesize(chunk, voice) - total_duration += result.duration_seconds - await ctx.report_progress( - progress=pct_synth, total=100, - message=f"Synthesizing chunk {i + 1}/{n_chunks}...", - ) - - enqueue_result = await queue.enqueue( - result, - priority=priority, - suppress_exit_tone=not is_last, - entry_tone=deferred_entry_tone if i == 0 else None, - ) - - if not enqueue_result.get("queued"): - # Queue full mid-message — already-enqueued chunks play through - if all_speech_ids: - msg = f"Queue full at chunk {i + 1}/{n_chunks}, waiting for enqueued chunks..." - await ctx.info(msg) - elapsed = time.time() - first_enqueue_time if first_enqueue_time else 0 - remaining = max(total_duration - elapsed, 0.1) - outcome = await _await_with_progress( - queue, all_speech_ids[-1], remaining, ctx, 35 - ) - outcome["chunks_enqueued"] = len(all_speech_ids) - outcome["chunks_total"] = n_chunks - return outcome - return enqueue_result - - all_speech_ids.append(enqueue_result["speech_id"]) - if first_enqueue_time is None: - first_enqueue_time = time.time() - - # All chunks enqueued — wait for final chunk to finish playing - if not all_speech_ids: - return {"error": "No chunks were successfully enqueued", "chunks_total": n_chunks} - - await ctx.report_progress( - progress=35, total=100, - message="Playing audio \u2014 you can continue working", - ) - await ctx.info(f"Playing {n_chunks} chunks ({total_duration:.1f}s total)...") - - # Subtract time already elapsed since first chunk started playing - elapsed = time.time() - first_enqueue_time if first_enqueue_time else 0 - remaining = max(total_duration - elapsed, 0.1) - outcome = await _await_with_progress(queue, all_speech_ids[-1], remaining, ctx, 35) - await ctx.report_progress(progress=100, total=100, message="Playback complete") - - # Check for partial failures in earlier chunks - failed_chunks = [] - for idx, sid in enumerate(all_speech_ids[:-1]): - chunk_status = queue.get_status(sid) - if chunk_status.get("status") == "completed" and not chunk_status.get("played", True): - failed_chunks.append(idx + 1) - - outcome["chunks"] = n_chunks - outcome["total_duration_seconds"] = total_duration - if failed_chunks: - outcome["partial_failure"] = True - outcome["failed_chunks"] = failed_chunks - return outcome - - except asyncio.CancelledError: - # speak() was cancelled (server shutdown or MCP client disconnect). - # Do NOT cancel the consumer — let already-enqueued chunks play through. - # Shutdown: queue.stop() in the lifespan finalizer handles graceful drain. - # Client cancel: use cancel_speech(speech_id) for explicit mid-playback stop. + # The finally above already closed the utterance, so the consumer drains + # the chunks it has and finishes cleanly. Do NOT cancel the consumer: + # already-synthesized audio should play through. For explicit + # mid-playback stop, use cancel_speech(speech_id). raise @@ -519,21 +491,45 @@ async def _await_with_progress( ctx: Context, base_pct: int, ) -> dict: - """Await playback completion with a progress ticker. + """Await playback completion with a status-aware progress ticker. - Uses a background task for progress — NOT asyncio.wait_for polling + Uses a background task for progress \u2014 NOT asyncio.wait_for polling (see CLAUDE.md for the Python 3.13 pitfall). + + The ticker distinguishes a WAITING item from a PLAYING one by polling + queue.get_status(): while the item is still queued behind others it holds + at base_pct with a "waiting in line" message; the playing percentage only + advances from the moment it actually becomes the current speaker. Without + this, an item queued behind a long message raced to "Playing audio 99%" + while it was silent and just waiting its turn. """ - async def _progress_ticker(): - t0 = time.time() - while True: - await asyncio.sleep(0.5) - elapsed = time.time() - t0 + async def _report(play_started): + status = queue.get_status(speech_id) + st = status.get("status") + if st == "queued": + pos = status.get("position") + msg = ( + f"Waiting in the queue (about {pos} ahead) \u2014 playback hasn't started" + if pos else "Waiting in the queue \u2014 playback hasn't started" + ) + await ctx.report_progress(progress=base_pct, total=100, message=msg) + elif st == "playing": + if play_started is None: + play_started = time.time() # clock starts when audio actually starts + elapsed = time.time() - play_started pct = min(base_pct + int((99 - base_pct) * elapsed / max(duration, 0.1)), 99) await ctx.report_progress( progress=pct, total=100, message="Playing audio \u2014 you can continue working", ) + # completed/unknown: nothing to report; wait_for_completion resolves it. + return play_started + + async def _progress_ticker(): + play_started = await _report(None) # immediate first report + while True: + await asyncio.sleep(0.5) + play_started = await _report(play_started) ticker = asyncio.create_task(_progress_ticker()) try: @@ -562,11 +558,22 @@ async def speak( ) -> dict: """Synthesize text and play it through the host speakers. - CALL THIS IN PARALLEL with other tools — don't block on it alone. - Audio plays through physical speakers, not into your context. The return - value (speech_id, duration) is informational only and never needed for - subsequent reasoning. Call speak alongside your next tool in the same - message and let progress notifications track playback. + Two ways to use this: + + 1. MILESTONE PING (default) — a status update the person overhears ("build + done", "tests green"). CALL THIS IN PARALLEL with your next tool; don't + block on it alone. The return value (speech_id, duration) is informational + and never needed for subsequent reasoning. + + 2. VOICE CONVERSATION — you asked the person something and want their spoken + answer. Then speak() and listen() form a turn: let speak() finish, THEN + call listen() to capture the reply. Do NOT run them in parallel here — the + mic must open after playback ends or it records your own voice. If your + text ends in a question or otherwise invites a response, reach for + listen(wait_for_silence=True) right after this returns. + + Audio plays through physical speakers, not into your context. Progress + notifications track playback either way. Progress notifications arrive every ~0.5s during playback: - 5%: Entry tone played (audible acknowledgement) @@ -597,19 +604,20 @@ async def speak( return {"error": f"Unknown engine: {engine}. Available: {list(engines.keys())}"} eng = engines[engine] - voice, text = await _resolve_project_voice( - ctx, engine, eng, voice, text, voice_cache, project, + voice, project_name = await _resolve_project_voice( + ctx, engine, eng, voice, voice_cache, project, ) priority = Priority.URGENT if urgent else Priority.NORMAL - chunks = split_text(text) - if len(chunks) <= 1: - return await _speak_single( - eng, engine, text, voice, queue, entry_tone, priority, ctx, ducker, - ) - return await _speak_chunked( - eng, engine, chunks, voice, queue, entry_tone, priority, ctx, ducker, + # Announcements always go through Kokoro so the secretary sounds identical + # regardless of the project's engine; fall back to the speaking engine if + # Kokoro somehow isn't loaded. + announce_eng = engines.get("kokoro", eng) + + return await _speak( + eng, engine, text, voice, project_name, queue, entry_tone, priority, ctx, ducker, + announce_eng=announce_eng, secretary_voice=settings.secretary_voice, ) @@ -692,8 +700,10 @@ async def generate_audio( return {"error": f"Unknown engine: {engine}. Available: {list(engines.keys())}"} eng = engines[engine] - voice, text = await _resolve_project_voice( - ctx, engine, eng, voice, text, voice_cache, project, + # generate_audio bypasses the queue, so it never announces — it only needs + # the resolved voice, not the project name. + voice, _project_name = await _resolve_project_voice( + ctx, engine, eng, voice, voice_cache, project, ) await ctx.info(f"Generating audio with {engine}...") @@ -788,7 +798,10 @@ async def transcribe( @mcp.tool(annotations=ToolAnnotations(readOnlyHint=False, openWorldHint=True)) async def listen( - duration_seconds: float = 5.0, + duration_seconds: float = 30.0, + wait_for_silence: bool = True, + silence_threshold_ms: int = 2200, + vad_aggressiveness: int = 3, response_format: Literal["json", "text", "verbose_json"] = "json", source: str | None = None, save_path: str | None = None, @@ -800,7 +813,14 @@ async def listen( min_confidence: float | None = None, ctx: Context = CurrentContext(), ) -> dict: - """Capture audio from the host mic for N seconds, transcribe via Parakeet. + """Capture audio from the host mic, transcribe via Parakeet, return the text. + + This is how you HEAR THE PERSON BACK. Pair it with speak() to hold a + voice conversation: speak your question (let it finish), then call listen() + to capture their spoken answer. The transcribed text comes back to you in + the result, so unlike speak() you DO use the return value. For a natural + turn, set wait_for_silence=True so recording ends when they stop talking + instead of running the full duration. Records 16 kHz mono WAV via pw-record using the container's PipeWire socket bind mount (same socket play_audio uses for output). Default @@ -808,9 +828,20 @@ async def listen( same transcribe_audio() machinery as transcribe(), so all forward-compat params (diarize, timestamp_granularities, etc.) work identically. + A "ready to talk" tone plays before recording opens and a "got it" tone + after it closes, so the person knows exactly when to speak. + Args: - duration_seconds: How long to listen. pw-record runs for this - wall-clock duration then receives SIGTERM to close the WAV. + duration_seconds: Max seconds to listen. With wait_for_silence this is + an upper bound; otherwise pw-record runs the full duration then + receives SIGTERM to close the WAV. + wait_for_silence: Stop as soon as the person stops talking (voice- + activity detection) instead of recording the whole duration. Best + for conversational turns — recommended when capturing a reply. + silence_threshold_ms: With wait_for_silence, how long a pause counts as + "they're done" (default 2200ms). + vad_aggressiveness: webrtcvad mode 0-3 (0 lax, 3 strict). 2 balances + rejecting background noise against catching soft speech. response_format: 'json' (default), 'text', or 'verbose_json'. source: PipeWire source name (e.g. "alsa_input.usb-..." or "bluez_input.XX:XX:XX..."). None = system default source. @@ -828,11 +859,52 @@ async def listen( rec_path = Path("/tmp/mcspeak") / f"listen-{ts}.wav" rec_path.parent.mkdir(parents=True, exist_ok=True) - await ctx.info(f"Listening for {duration_seconds:.1f}s (source={source or 'default'})...") - try: - await record_audio(rec_path, duration_seconds, source=source) - except Exception as e: - return {"error": f"Recording failed: {e}"} + state = ctx.lifespan_context + start_tone = state.get("listen_start_tone") + end_tone = state.get("listen_end_tone") + warmup_ms = settings.listen_warmup_ms + + # The "ready to talk" tone is handed to the recorder rather than played + # here: it fires AFTER the mic goes live (post-warmup) so the person's first + # word isn't clipped by pw-record's stream startup. See audio.py for the + # warmup/no-clip rationale. + vad_info: dict | None = None + if wait_for_silence: + await ctx.info( + f"Listening (VAD; up to {duration_seconds:.1f}s, stop after " + f"{silence_threshold_ms}ms silence)..." + ) + try: + vad_info = await record_audio_until_silence( + rec_path, + silence_threshold_ms=silence_threshold_ms, + max_seconds=duration_seconds, + aggressiveness=vad_aggressiveness, + source=source, + ready_tone=start_tone, + warmup_ms=warmup_ms, + ) + except Exception as e: + return {"error": f"VAD recording failed: {e}"} + else: + await ctx.info(f"Listening for {duration_seconds:.1f}s (source={source or 'default'})...") + try: + await record_audio( + rec_path, duration_seconds, source=source, + ready_tone=start_tone, warmup_ms=warmup_ms, + ) + except Exception as e: + return {"error": f"Recording failed: {e}"} + + # "Capture done" tone AFTER pw-record stops. Audible "over and out" + # so the user knows they can stop talking and the system is now + # working on transcription. + if end_tone: + try: + await asyncio.sleep(0.05) # let record buffers flush first + await _play_tone(end_tone) + except Exception: + pass # Optional persistence to /output/. Reuse _resolve_output_path so the # scoping rules + extension correction are consistent with generate_audio. @@ -871,7 +943,12 @@ async def listen( result["recorded"] = str(rec_path) if saved_to: result["saved_to"] = saved_to - result["duration_recorded"] = duration_seconds + if vad_info: + # VAD path: real recorded duration came from VAD, not the cap. + result["duration_recorded"] = vad_info["duration_ms"] / 1000.0 + result["vad"] = vad_info + else: + result["duration_recorded"] = duration_seconds return result diff --git a/src/mcspeak/voice_identity.py b/src/mcspeak/voice_identity.py index f35f8c8..c62fd9b 100644 --- a/src/mcspeak/voice_identity.py +++ b/src/mcspeak/voice_identity.py @@ -51,6 +51,10 @@ def _build_identity_pool(voices: list[str]) -> list[str]: for v in settings.voice_identity_exclude.split(",") if v.strip() } + # The secretary's voice is reserved — never auto-assign it to a project, so + # no project can be confused for the receptionist. (Explicit voice= still works.) + if settings.secretary_voice: + excludes.add(settings.secretary_voice.strip().lower()) eligible = [ v for v in voices if v.startswith(prefixes) and v.lower() not in excludes From 3c4e06aa647830cf3f75def57cf629faa0c58cfa Mon Sep 17 00:00:00 2001 From: Ryan Malloy Date: Fri, 3 Jul 2026 20:08:14 -0600 Subject: [PATCH 2/3] Add Pink Floyd tone kit and conversation-first listen A telephone-themed bookend set, tuned by ear: - speak: heartbeat entry ("Speak to Me" woven with warm MF tones), soft-chord exit (pure sines, easy on the ears on repetition) - listen: mf-dial start (the genuine Young Lust R1 operator routing sequence, KP 0-4-4-1-8-3-1 ST) and machine end (Welcome to the Machine throb) - mf-listen/mf-done gentler alternatives, and a call-waiting blip listen() is now conversation-first: wait_for_silence defaults on with a 30s cap, and the "go" tone plays only after the mic is live (warmup) so the first word isn't clipped. VAD defaults tuned (aggressiveness 3, 2200ms silence, 400ms min-speech) after live testing showed the old values cut replies off on brief background transients. Adds webrtcvad-wheels for the VAD path. --- pyproject.toml | 4 + src/mcspeak/audio.py | 151 ++++++++++++++ src/mcspeak/settings.py | 52 ++++- src/mcspeak/tones.py | 432 ++++++++++++++++++++++++++++++++++++++++ uv.lock | 41 +++- 5 files changed, 675 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 11cd0ae..626ab74 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,10 @@ dependencies = [ "snac>=1.2.1", "soundfile", "torch", + # webrtcvad-wheels is a drop-in fork of webrtcvad that ships pre-built + # wheels for Python 3.12/3.13 — the original requires gcc to build from + # source, which the slim Docker image doesn't have. + "webrtcvad-wheels", "wyoming>=1.8.0", ] diff --git a/src/mcspeak/audio.py b/src/mcspeak/audio.py index 12ada71..18f3ef1 100644 --- a/src/mcspeak/audio.py +++ b/src/mcspeak/audio.py @@ -186,11 +186,153 @@ async def _force_pwplay_volume_100() -> None: return +async def record_audio_until_silence( + out_path: Path, + silence_threshold_ms: int = 1500, + max_seconds: float = 30.0, + aggressiveness: int = 3, + source: str | None = None, + min_speech_ms: int = 400, + ready_tone: Path | None = None, + warmup_ms: int = 150, +) -> dict: + """Record from mic, terminate when N ms of consecutive silence detected. + + Streams pw-record's raw s16 PCM stdout in 30 ms frames (480 samples = + 960 bytes at 16 kHz mono), feeds each frame to webrtcvad.is_speech(). + Tracks consecutive non-speech frames; when their cumulative duration + exceeds silence_threshold_ms (AND at least min_speech_ms of speech has + been observed first), terminates pw-record and writes the WAV. + + The min_speech_ms guard prevents premature termination from the initial + "user hasn't started yet" silence — we only count silence as end-of-speech + AFTER speech has been seen. + + Args: + out_path: where to write the final WAV. + silence_threshold_ms: pause length that counts as "user stopped". + max_seconds: absolute cap to prevent runaway recordings. + aggressiveness: webrtcvad mode 0-3 (0 lax, 3 strict). 2 is the + balance between rejecting noise and accepting soft speech. + source: PipeWire source name, None = system default. + min_speech_ms: minimum speech observed before silence can end recording. + + Returns dict with: bytes_written, speech_ms, silence_ms, terminated_by + ("silence" | "max_duration" | "no_speech"). + """ + import webrtcvad + + SAMPLE_RATE = 16000 # webrtcvad requires 8000, 16000, 32000, or 48000 + FRAME_MS = 30 # webrtcvad accepts 10/20/30 ms; 30 = lowest CPU + FRAME_BYTES = SAMPLE_RATE * 2 * FRAME_MS // 1000 # 960 bytes at 16k mono s16 + + vad = webrtcvad.Vad(aggressiveness) + + cmd = [ + "pw-record", + "--rate", str(SAMPLE_RATE), + "--channels", "1", + "--format", "s16", + ] + if source: + cmd.extend(["--target", source]) + cmd.append("-") # stdout + + proc = await asyncio.create_subprocess_exec( + *cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.DEVNULL, + ) + + # Cue the person only AFTER the mic is live (see warmup rationale above), so + # the first word isn't clipped by pw-record's stream startup. The beep is + # captured as leading audio but VAD skips it (pure tone != speech) and the + # min_speech_ms guard below keeps it from ending the recording. + if ready_tone is not None: + await asyncio.sleep(warmup_ms / 1000) + try: + await play_audio(ready_tone, expected_seconds=0.3) + except Exception: + pass # tone failure is non-fatal — keep recording + + pcm_buf = bytearray() + leftover = b"" + speech_frames = 0 + silence_frames = 0 + silence_run_frames = 0 # consecutive silence frames since last speech + terminated_by = "max_duration" + + silence_frame_threshold = silence_threshold_ms // FRAME_MS + min_speech_frames = min_speech_ms // FRAME_MS + max_frames = int(max_seconds * 1000) // FRAME_MS + + try: + for _ in range(max_frames): + # Read enough bytes for one VAD frame. pw-record may give us + # smaller chunks; accumulate leftover across reads. + while len(leftover) < FRAME_BYTES: + chunk = await proc.stdout.read(FRAME_BYTES - len(leftover)) + if not chunk: + terminated_by = "no_speech" + break + leftover += chunk + else: + # Got a complete frame + frame = bytes(leftover[:FRAME_BYTES]) + leftover = leftover[FRAME_BYTES:] + pcm_buf.extend(frame) + if vad.is_speech(frame, SAMPLE_RATE): + speech_frames += 1 + silence_run_frames = 0 + else: + silence_frames += 1 + silence_run_frames += 1 + # End-of-speech: enough silence run AND we've heard speech + if (silence_run_frames >= silence_frame_threshold + and speech_frames >= min_speech_frames): + terminated_by = "silence" + break + continue + # The inner read returned no data — break outer loop + break + finally: + proc.terminate() + try: + await asyncio.wait_for(proc.wait(), timeout=2.0) + except asyncio.TimeoutError: + proc.kill() + await proc.wait() + + if not pcm_buf: + raise PlaybackError( + f"VAD recording captured no audio. Check source '{source or 'default'}'." + ) + + # Write the accumulated PCM as a WAV file. + write_wav_from_pcm( + bytes(pcm_buf), + sample_rate=SAMPLE_RATE, + sample_width=2, + channels=1, + path=out_path, + ) + + return { + "bytes_written": len(pcm_buf), + "speech_ms": speech_frames * FRAME_MS, + "silence_ms": silence_frames * FRAME_MS, + "terminated_by": terminated_by, + "duration_ms": (speech_frames + silence_frames) * FRAME_MS, + } + + async def record_audio( out_path: Path, duration_seconds: float, source: str | None = None, sample_rate: int = 16000, + ready_tone: Path | None = None, + warmup_ms: int = 150, ) -> Path: """Capture audio from the host mic via pw-record (PipeWire socket bind mount). @@ -222,6 +364,15 @@ async def record_audio( stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.PIPE, ) + # Cue the person only AFTER the mic is live so the first word isn't clipped + # by pw-record's stream startup (the beep bleeds harmlessly into the head of + # the WAV; Parakeet ignores it). The duration timeout starts after the cue. + if ready_tone is not None: + await asyncio.sleep(warmup_ms / 1000) + try: + await play_audio(ready_tone, expected_seconds=0.3) + except Exception: + pass # tone failure is non-fatal — keep recording try: # pw-record never exits on its own — wait the desired duration then term. await asyncio.wait_for(proc.wait(), timeout=duration_seconds) diff --git a/src/mcspeak/settings.py b/src/mcspeak/settings.py index 4a546ea..4708056 100644 --- a/src/mcspeak/settings.py +++ b/src/mcspeak/settings.py @@ -42,7 +42,29 @@ class Settings(BaseSettings): # Voice identity (project-aware voice assignment) voice_identity: bool = True + # Legacy flag — when True, forces announce_mode="always" (kept for back-compat). announce_project: bool = False + # Secretary-style project announcements before an utterance plays: + # "secretary" — announce only when the speaker changes from the last + # project that spoke, or when the same project returns after + # a lull (see reintroduce_after_seconds). The default. + # "always" — announce every utterance with its project name. + # "off" — never announce. + # The announcement is a short "." preamble synthesized in the + # project's own assigned voice and played by the queue consumer right + # before the utterance, so the listener learns which voice maps to which + # project. The decision is made at play time (after urgent reordering). + announce_mode: str = "secretary" + # Same project speaking again after this many seconds of overall silence + # gets re-introduced under "secretary" mode. Speaker *changes* always + # announce regardless of this value. + reintroduce_after_seconds: float = 120.0 + # Dedicated voice for the secretary's project announcements. It is reserved: + # excluded from the project auto-assignment pool so no project ever sounds + # like the secretary. Announcements always synthesize through Kokoro (fast, + # and identical every time) regardless of the speaking engine, so this must + # be a Kokoro voice name. An explicit voice= request can still use it. + secretary_voice: str = "bf_emma" # Comma-separated prefixes for auto-assignment pool (English voices) voice_identity_prefixes: str = "af_,am_,bf_,bm_,ef_,em_" # Voices available explicitly but excluded from auto-assignment (e.g. whispery) @@ -50,16 +72,38 @@ class Settings(BaseSettings): # JSON file to persist project->voice assignments across restarts voice_identity_file: str = "" - # Entry tone before speech: "chirp", "apollo", "none", or path to custom WAV - entry_tone: str = "chirp" + # Entry tone before speech: "heartbeat" (Speak to Me heartbeat woven with + # warm MF telephone tones), or "chirp"/"apollo"/"none"/path to custom WAV + entry_tone: str = "heartbeat" - # Exit tone after speech: "roger", "quindar-out", "none", or path to custom WAV + # Exit tone after speech: "soft-chord" (mellow resolving dyad, ear-friendly), + # or "roger"/"quindar-out"/"none"/path to custom WAV. # When more items are queued, plays "standby" (ascending blip) instead - exit_tone: str = "roger" + exit_tone: str = "soft-chord" # Cancel tone: "scratch", "reverse-roger", "none", or path to custom WAV cancel_tone: str = "scratch" + # Call-waiting tone: a brief blip mixed OVER the currently-playing message + # when a DIFFERENT project's message joins the queue, so the listener knows + # someone else is waiting. Fires at most once per playing turn (not once per + # arriving message). "none" disables; any tone name or path to a custom WAV. + call_waiting_tone: str = "call-waiting" + + # Bookend tones for listen() — a Pink Floyd telephone arc: mf-dial (the + # genuine Young Lust R1 operator routing sequence) once the mic is live, as + # if placing a call to the user; machine (the Welcome to the Machine throb) + # after recording stops, a warm "connected / got it". "none" disables; any + # tone name from tones.py or a path to a custom WAV. (mf-listen / mf-done + # remain available as the gentler alternative.) + listen_start_tone: str = "mf-dial" + listen_end_tone: str = "machine" + # Mic warm-up before the "go" tone sounds. The recorder is spawned, given + # this long for its PipeWire stream to go live, THEN the person is cued — so + # the first word isn't clipped by capture-stream startup. The beep is leading + # non-speech that VAD and Parakeet both ignore. + listen_warmup_ms: int = 150 + # Media ducking duck_media: bool = True duck_fade_out_ms: int = 500 diff --git a/src/mcspeak/tones.py b/src/mcspeak/tones.py index de93dce..45f9d98 100644 --- a/src/mcspeak/tones.py +++ b/src/mcspeak/tones.py @@ -110,6 +110,412 @@ def _build_reverse_roger() -> np.ndarray: return np.concatenate(segments) +# ----------------------------------------------------------------------------- +# Natural-feeling tones — softer alternatives to the telephony beeps above. +# Tuned for in-ear / headset use where the radio-style tones feel aggressive. +# ----------------------------------------------------------------------------- + +def _build_bell_soft() -> np.ndarray: + """Struck bell — 880 Hz fundamental + slightly-inharmonic partials, + exponential decay, ~400ms. Like a small finger cymbal or triangle. + """ + duration_ms = 400 + n = int(SAMPLE_RATE * duration_ms / 1000) + t = np.arange(n, dtype=np.float32) / SAMPLE_RATE + f0 = 880.0 + # Slight inharmonicity (the 2.01x, 3.02x) gives "real bell" warmth vs + # pure integer harmonics which sound synthesized. + samples = ( + 1.00 * np.sin(2 * np.pi * f0 * t) + + 0.50 * np.sin(2 * np.pi * f0 * 2.01 * t) + + 0.30 * np.sin(2 * np.pi * f0 * 3.02 * t) + + 0.15 * np.sin(2 * np.pi * f0 * 4.10 * t) + ) + envelope = np.exp(-3.5 * t / (duration_ms / 1000)) + # 1ms attack ramp to avoid the initial click + ramp_n = int(SAMPLE_RATE * 0.001) + envelope[:ramp_n] *= np.linspace(0, 1, ramp_n) + return samples * envelope + + +def _build_bell_deep() -> np.ndarray: + """Deeper bell — 440 Hz fundamental, slower decay, ~500ms. + Resolving "done" feel; pairs well with bell-soft as start/end. + """ + duration_ms = 500 + n = int(SAMPLE_RATE * duration_ms / 1000) + t = np.arange(n, dtype=np.float32) / SAMPLE_RATE + f0 = 440.0 + samples = ( + 1.00 * np.sin(2 * np.pi * f0 * t) + + 0.50 * np.sin(2 * np.pi * f0 * 2.01 * t) + + 0.30 * np.sin(2 * np.pi * f0 * 3.02 * t) + ) + envelope = np.exp(-3.0 * t / (duration_ms / 1000)) + ramp_n = int(SAMPLE_RATE * 0.001) + envelope[:ramp_n] *= np.linspace(0, 1, ramp_n) + return samples * envelope + + +def _build_tap_wood() -> np.ndarray: + """Wood block tap — 20ms bandpassed noise burst with sharp attack/decay. + Brief, percussive, almost zero sustain. Good "got it" close. + """ + duration_ms = 20 + n = int(SAMPLE_RATE * duration_ms / 1000) + rng = np.random.default_rng(123) + noise = rng.standard_normal(n).astype(np.float32) + # Crude bandpass: high-pass via 1-sample diff, then short MA low-pass. + # Centers energy around 1.5-3 kHz, which reads as "wood" not "snare". + hp = np.diff(noise, prepend=0) + kernel = np.ones(4, dtype=np.float32) / 4 + bandpass = np.convolve(hp, kernel, mode="same") + # Sharp exponential decay + envelope = np.exp(-80 * np.arange(n, dtype=np.float32) / SAMPLE_RATE) + return bandpass * envelope + + +def _build_water_drop() -> np.ndarray: + """Single water drop — physically-informed "plink". + + Real water drops produce two acoustic events: + 1. Splash impact: 2-4ms broadband click (mid-high frequencies). + 2. Air-bubble Helmholtz oscillation: a pitched tone whose frequency + RISES as the entrained bubble shrinks/escapes — typically + 700→1100 Hz for a kitchen-sink-sized drop. NOT a descending + sweep (a common synth-tone mistake). + + Total ~100ms with sharp decay. The combination of "splash click + + rising pitched ring" is what makes the brain hear it as water vs. + just a beep with reverb. + """ + SR = SAMPLE_RATE + rng = np.random.default_rng(789) + + # ---- 1. splash impact: 3ms broadband click, mostly mid-high band ---- + splash_n = int(SR * 0.003) + noise = rng.standard_normal(splash_n).astype(np.float32) + # Simple high-pass via 1-sample diff to push energy upward + splash = np.diff(noise, prepend=0.0) + splash_env = np.exp(-300 * np.arange(splash_n, dtype=np.float32) / SR) + splash = splash * splash_env * 0.6 # half-volume vs the ring + + # ---- 2. bubble ring: 700→1100 Hz rising over 90ms ---- + ring_ms = 90 + ring_n = int(SR * ring_ms / 1000) + t = np.arange(ring_n, dtype=np.float32) / SR + f0, f1 = 700.0, 1100.0 + # Quadratic chirp: phase = 2π·(f0·t + (f1-f0)/(2·T)·t²) + phase = 2 * np.pi * (f0 * t + (f1 - f0) / (2 * (ring_ms / 1000)) * t**2) + ring = np.sin(phase) + # Slight 2nd harmonic for body + ring += 0.25 * np.sin(2 * phase) + # Exponential decay + ring_env = np.exp(-25 * t) + # 1ms attack so the splice with splash doesn't click + ramp_n = int(SR * 0.001) + ring_env[:ramp_n] *= np.linspace(0, 1, ramp_n) + ring = ring * ring_env + + return np.concatenate([splash, ring]) + + +def _build_soft_pulse() -> np.ndarray: + """Breath-like soft pulse — 440 Hz sine with raised-cosine envelope, ~80ms. + No attack click, no decay tail. Feels like a "heartbeat blip" or a quiet + ambient cue. Gentlest tone in the catalog, ideal for repeated triggers. + """ + duration_ms = 80 + n = int(SAMPLE_RATE * duration_ms / 1000) + t = np.arange(n, dtype=np.float32) / SAMPLE_RATE + tone = np.sin(2 * np.pi * 440.0 * t) + # Raised-cosine envelope: 0→1→0 smoothly, no clicks at either end. + envelope = 0.5 * (1.0 - np.cos(2.0 * np.pi * np.arange(n, dtype=np.float32) / n)) + return tone * envelope + + +def _build_hmm_up() -> np.ndarray: + """Vocal-feeling "hm?" — two sine waves at vowel formants with rising + pitch. Reads as a soft questioning acknowledgement, like the system + just opened its ears. ~180ms. + """ + SR = SAMPLE_RATE + duration_ms = 180 + n = int(SR * duration_ms / 1000) + t = np.arange(n, dtype=np.float32) / SR + # Pitch contour: starts at 220 Hz, rises to 280 Hz (questioning intonation) + pitch = 220.0 + (280.0 - 220.0) * t / (duration_ms / 1000) + # Cumulative phase for time-varying frequency + phase = 2 * np.pi * np.cumsum(pitch) / SR + # Vowel /ʌ/ approximated: fundamental + F1≈700Hz + F2≈1300Hz formants + # Approximated by adding harmonics at those frequencies. + fundamental = np.sin(phase) + # F1 and F2 as modulated by the pitch contour (rough formant approximation) + f1 = 0.4 * np.sin(2 * np.pi * 700.0 * t) + f2 = 0.2 * np.sin(2 * np.pi * 1300.0 * t) + samples = fundamental + f1 + f2 + # Soft attack + slow decay (vocal envelope) + envelope = np.ones(n, dtype=np.float32) + attack_n = int(SR * 0.020) # 20ms attack + envelope[:attack_n] = np.linspace(0, 1, attack_n) + decay_n = int(SR * 0.080) # 80ms decay tail + envelope[-decay_n:] *= np.linspace(1, 0, decay_n) + return samples * envelope + + +def _build_shutter_click() -> np.ndarray: + """Camera shutter — two brief noise bursts 35ms apart (mirror up, mirror + down). Mechanical, definite, satisfying. ~60ms total. + """ + SR = SAMPLE_RATE + rng = np.random.default_rng(456) + + # Two 8ms noise bursts. Second is slightly lower-energy. + def burst(samples_n: int, energy: float) -> np.ndarray: + noise = rng.standard_normal(samples_n).astype(np.float32) + # High-pass via diff to emphasize mechanical click character + clicky = np.diff(noise, prepend=0.0) + env = np.exp(-150 * np.arange(samples_n, dtype=np.float32) / SR) + return clicky * env * energy + + burst_n = int(SR * 0.008) + gap_n = int(SR * 0.027) + return np.concatenate([ + burst(burst_n, 1.0), + np.zeros(gap_n, dtype=np.float32), + burst(burst_n, 0.75), + ]) + + +def _build_chime_tube() -> np.ndarray: + """Wind chime tube — single struck metal tube, long decay (~600ms). + More resonant + longer-lived than bell-soft. Multiple inharmonic partials. + """ + SR = SAMPLE_RATE + duration_ms = 600 + n = int(SR * duration_ms / 1000) + t = np.arange(n, dtype=np.float32) / SR + # Tube modes: not harmonic, ratios roughly 1, 2.76, 5.40, 8.93 for + # transverse vibrations of a free-free metal bar (Chladni's formula). + f0 = 520.0 + samples = ( + 1.00 * np.sin(2 * np.pi * f0 * 1.00 * t) * np.exp(-2.0 * t) + + 0.55 * np.sin(2 * np.pi * f0 * 2.76 * t) * np.exp(-3.5 * t) + + 0.25 * np.sin(2 * np.pi * f0 * 5.40 * t) * np.exp(-5.0 * t) + + 0.10 * np.sin(2 * np.pi * f0 * 8.93 * t) * np.exp(-7.0 * t) + ) + # 1ms attack ramp + ramp_n = int(SR * 0.001) + samples[:ramp_n] *= np.linspace(0, 1, ramp_n) + return samples + + +def _build_drop_deep() -> np.ndarray: + """Deeper "plop" into a basin or bowl — same physics as water-drop but + larger air cavity, so lower starting pitch and longer decay (~200ms). + """ + SR = SAMPLE_RATE + rng = np.random.default_rng(790) + + # Bigger splash for the bigger drop + splash_n = int(SR * 0.005) + noise = rng.standard_normal(splash_n).astype(np.float32) + splash = np.diff(noise, prepend=0.0) + splash_env = np.exp(-200 * np.arange(splash_n, dtype=np.float32) / SR) + splash = splash * splash_env * 0.5 + + # Lower bubble: 320→520 Hz rising over 180ms, slower decay + ring_ms = 180 + ring_n = int(SR * ring_ms / 1000) + t = np.arange(ring_n, dtype=np.float32) / SR + f0, f1 = 320.0, 520.0 + phase = 2 * np.pi * (f0 * t + (f1 - f0) / (2 * (ring_ms / 1000)) * t**2) + ring = np.sin(phase) + 0.30 * np.sin(2 * phase) + 0.10 * np.sin(3 * phase) + ring_env = np.exp(-12 * t) + ramp_n = int(SR * 0.001) + ring_env[:ramp_n] *= np.linspace(0, 1, ramp_n) + ring = ring * ring_env + + return np.concatenate([splash, ring]) + + +def _build_mf_listen() -> np.ndarray: + """Warm analog "I'm listening" swell — a stacked-fifths chord (D-A-E) that + blooms in with detuned chorus and a slow vibrato, like a Pink Floyd VCS3 pad + opening up. Stacked fifths are the open, unresolved "waiting for you" voicing + Floyd leans on. A short portamento glides into pitch for the analog bloom. + Signals "the mic is live, go ahead." ~700ms. + """ + SR = SAMPLE_RATE + dur_ms = 700 + n = int(SR * dur_ms / 1000) + t = np.arange(n, dtype=np.float32) / SR + + notes = [146.83, 220.00, 329.63] # D3, A3, E4 — stacked fifths + amps = [1.0, 0.75, 0.55] + + # Shared subtle vibrato (~5 Hz) + a portamento swell into pitch over the + # first ~45ms tau (starts 4% flat, glides up) for the analog bloom. + vib = 1.0 + 0.003 * np.sin(2 * np.pi * 5.0 * t) + glide = 1.0 - 0.04 * np.exp(-t / 0.045) + + samples = np.zeros(n, dtype=np.float32) + for f, a in zip(notes, amps): + for detune in (0.997, 1.003): # dual-osc chorus per note + inst = f * detune * vib * glide + phase = 2 * np.pi * np.cumsum(inst) / SR + samples += a * np.sin(phase) + # Faint octave shimmer up top — air without harshness. + samples += 0.12 * np.sin(2 * np.pi * 587.33 * t) # D5 + + # Bloom envelope: 110ms raised-cosine attack, 300ms raised-cosine release. + env = np.ones(n, dtype=np.float32) + a_n = int(SR * 0.110) + env[:a_n] = 0.5 * (1 - np.cos(np.pi * np.arange(a_n, dtype=np.float32) / a_n)) + r_n = int(SR * 0.300) + env[-r_n:] *= 0.5 * (1 + np.cos(np.pi * np.arange(r_n, dtype=np.float32) / r_n)) + return samples * env + + +def _build_mf_done() -> np.ndarray: + """Warm resolving "got it" chord — a low D-major triad (D-F#-A) that settles + with a slight downward drift, the consonant answer to mf-listen's open + fifths. Same tonal center (D), so the two read as a matched pair of + bookends. "Captured, closing." ~520ms. + """ + SR = SAMPLE_RATE + dur_ms = 520 + n = int(SR * dur_ms / 1000) + t = np.arange(n, dtype=np.float32) / SR + + notes = [146.83, 185.00, 220.00] # D3, F#3, A3 — low, warm resolution + amps = [1.0, 0.7, 0.7] + + vib = 1.0 + 0.002 * np.sin(2 * np.pi * 4.5 * t) + settle = 1.0 - 0.015 * (t / (dur_ms / 1000)) # gentle 1.5% downward drift + + samples = np.zeros(n, dtype=np.float32) + for f, a in zip(notes, amps): + for detune in (0.998, 1.002): + inst = f * detune * vib * settle + phase = 2 * np.pi * np.cumsum(inst) / SR + samples += a * np.sin(phase) + + env = np.ones(n, dtype=np.float32) + a_n = int(SR * 0.050) + env[:a_n] = 0.5 * (1 - np.cos(np.pi * np.arange(a_n, dtype=np.float32) / a_n)) + r_n = int(SR * 0.280) + env[-r_n:] *= 0.5 * (1 + np.cos(np.pi * np.arange(r_n, dtype=np.float32) / r_n)) + return samples * env + + +def _build_mf_dial() -> np.ndarray: + """Young Lust telephone dial — the genuine Bell-System R1 multi-frequency + operator routing sequence from the end of the track: KP, 0-4-4-1-8-3-1, ST + (the 44 is the UK country code). Real MF pairs drawn from + {700,900,1100,1300,1500,1700}, ~67ms digits with ~50ms gaps, KP/ST longer. + Used as the listen "I'm dialing you, go ahead" cue. ~1.1s. + """ + pairs = { + "0": (1300, 1500), "1": (700, 900), "3": (900, 1100), "4": (700, 1300), + "8": (900, 1500), "KP": (1100, 1700), "ST": (1500, 1700), + } + + def mf(pair: tuple[int, int], ms: float) -> np.ndarray: + # Sum the two MF frequencies (equal length → direct add), soft edges. + return _sine_segment(pair[0], ms, fade_ms=4.0) + _sine_segment(pair[1], ms, fade_ms=4.0) + + seq = ["KP", "0", "4", "4", "1", "8", "3", "1", "ST"] + out: list[np.ndarray] = [] + for d in seq: + out.append(mf(pairs[d], 100 if d in ("KP", "ST") else 67)) + out.append(_silence(50)) + return np.concatenate(out) + + +def _build_machine() -> np.ndarray: + """Welcome to the Machine throb — a low pulsing VCS3-style drone (~55 Hz + fundamental + harmonics), amplitude-pulsed by a ~3 Hz LFO with a bright + upper-harmonic sheen. Warm, mechanical "connected / got it" close. ~900ms. + """ + n = int(SAMPLE_RATE * 0.9) + t = np.arange(n, dtype=np.float32) / SAMPLE_RATE + b = 55.0 + tone = ( + np.sin(2 * np.pi * b * t) + + 0.5 * np.sin(2 * np.pi * 2 * b * t) + + 0.3 * np.sin(2 * np.pi * 3 * b * t) + ) + lfo = 0.5 * (1 + np.sin(2 * np.pi * 3.0 * t - np.pi / 2)) # ~3 Hz pulse + tone = tone * (0.3 + 0.7 * lfo) + tone += 0.15 * np.sin(2 * np.pi * 8 * b * t) * lfo # sheen on the pulses + a = int(SAMPLE_RATE * 0.02) + tone[:a] *= np.linspace(0, 1, a) + r = int(SAMPLE_RATE * 0.15) + tone[-r:] *= np.linspace(1, 0, r) + return tone + + +def _build_heartbeat() -> np.ndarray: + """speak() entry — a "Speak to Me" heartbeat woven with warm MF telephone + tones, tying the speak bookends back to the listen phone theme. A mellow MF + pair leads in, then a lub-dub beat, a small rising MF figure, and a softer + second beat. Low and warm, not harsh. ~1.5s. + """ + def thump(f0: float, dur_ms: float, amp: float) -> np.ndarray: + n = int(SAMPLE_RATE * dur_ms / 1000) + t = np.arange(n, dtype=np.float32) / SAMPLE_RATE + freq = f0 * (0.6 + 0.4 * np.exp(-t / 0.03)) # quick downward pitch drop + body = np.sin(2 * np.pi * np.cumsum(freq) / SAMPLE_RATE) + env = np.exp(-t / 0.055) + a = int(SAMPLE_RATE * 0.003) + env[:a] *= np.linspace(0, 1, a) + return body * env * amp + + def beat(amp: float) -> np.ndarray: + return np.concatenate([thump(55, 140, amp), _silence(110), thump(66, 120, 0.7 * amp)]) + + def mf(pair: tuple[int, int], ms: float, amp: float = 1.0) -> np.ndarray: + return ( + _sine_segment(pair[0], ms, fade_ms=4.0) + + _sine_segment(pair[1], ms, fade_ms=4.0) + ) * amp + + d1, d4, d0 = (700, 900), (700, 1300), (1300, 1500) # warm MF pairs + return np.concatenate([ + mf(d1, 90, 0.8), _silence(80), + beat(0.9), _silence(150), + mf(d4, 55, 0.6), _silence(40), mf(d0, 60, 0.6), _silence(150), + beat(0.7), + ]) + + +def _build_soft_chord() -> np.ndarray: + """speak() exit — a mellow low resolving dyad (A3 + E4 + soft octave), pure + sines under a raised-cosine envelope so there are no clicks and no hiss. + Gentle "over and out" that's easy on the ears on every message. ~700ms. + """ + n = int(SAMPLE_RATE * 0.7) + t = np.arange(n, dtype=np.float32) / SAMPLE_RATE + s = ( + np.sin(2 * np.pi * 220.0 * t) + + 0.7 * np.sin(2 * np.pi * 329.63 * t) + + 0.2 * np.sin(2 * np.pi * 440.0 * t) + ) + env = 0.5 * (1 - np.cos(2 * np.pi * np.arange(n, dtype=np.float32) / n)) + return s * env + + +def _build_call_waiting() -> np.ndarray: + """Call-waiting alert — two very short soft blips at 1047 Hz (C6), ~125ms + total. Deliberately brief and high so it rides cleanly over ongoing speech + (it's mixed OVER the currently-playing message, phone-style) to signal that + another project's message just joined the queue. + """ + seg = _sine_segment(1047.0, 45, fade_ms=4.0) # soft edges, no click + return np.concatenate([seg, _silence(35), seg]) + + def _write_tone(samples: np.ndarray, path: Path) -> Path: """Write float32 samples to 16-bit PCM WAV.""" peak = max(abs(samples.max()), abs(samples.min()), 1e-8) @@ -132,6 +538,32 @@ _TONE_BUILDERS = { "standby": _build_standby, "scratch": _build_scratch, "reverse-roger": _build_reverse_roger, + # Natural-feeling alternatives — designed for in-ear/headset listen() use. + "bell-soft": _build_bell_soft, + "bell-deep": _build_bell_deep, + "tap-wood": _build_tap_wood, + "water-drop": _build_water_drop, + "drop-deep": _build_drop_deep, + "soft-pulse": _build_soft_pulse, + "hmm-up": _build_hmm_up, + "shutter-click": _build_shutter_click, + "chime-tube": _build_chime_tube, + # Pink Floyd-esque multi-frequency pads for listen() bookends — warm, + # chorused, musical. mf-listen ("go, I'm listening") / mf-done ("got it"). + "mf-listen": _build_mf_listen, + "mf-done": _build_mf_done, + # Pink Floyd telephone set for listen() — mf-dial is the genuine Young Lust + # R1 operator routing sequence ("dialing you, go ahead"); machine is the + # Welcome to the Machine throb ("connected / got it"). + "mf-dial": _build_mf_dial, + "machine": _build_machine, + # speak() bookends — heartbeat+MF entry ("Speak to Me" woven with telephone + # tones) and a soft resolving chord exit. Warm, ear-friendly on repetition. + "heartbeat": _build_heartbeat, + "soft-chord": _build_soft_chord, + # Brief blip mixed over ongoing speech when another project's message + # joins the queue — phone-style call-waiting. + "call-waiting": _build_call_waiting, } diff --git a/uv.lock b/uv.lock index 618aecc..5e52fd5 100644 --- a/uv.lock +++ b/uv.lock @@ -893,7 +893,7 @@ wheels = [ [[package]] name = "mcspeak" -version = "2026.3.4" +version = "2026.3.4.1" source = { editable = "." } dependencies = [ { name = "fastmcp" }, @@ -905,6 +905,7 @@ dependencies = [ { name = "snac" }, { name = "soundfile" }, { name = "torch" }, + { name = "webrtcvad-wheels" }, { name = "wyoming" }, ] @@ -919,6 +920,7 @@ requires-dist = [ { name = "snac", specifier = ">=1.2.1" }, { name = "soundfile" }, { name = "torch" }, + { name = "webrtcvad-wheels" }, { name = "wyoming", specifier = ">=1.8.0" }, ] @@ -1962,6 +1964,11 @@ dependencies = [ wheels = [ { url = "https://files.pythonhosted.org/packages/d3/54/a2ba279afcca44bbd320d4e73675b282fcee3d81400ea1b53934efca6462/torch-2.10.0-2-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:13ec4add8c3faaed8d13e0574f5cd4a323c11655546f91fbe6afa77b57423574", size = 79498202, upload-time = "2026-02-10T21:44:52.603Z" }, { url = "https://files.pythonhosted.org/packages/ec/23/2c9fe0c9c27f7f6cb865abcea8a4568f29f00acaeadfc6a37f6801f84cb4/torch-2.10.0-2-cp313-none-macosx_11_0_arm64.whl", hash = "sha256:e521c9f030a3774ed770a9c011751fb47c4d12029a3d6522116e48431f2ff89e", size = 79498254, upload-time = "2026-02-10T21:44:44.095Z" }, + { url = "https://files.pythonhosted.org/packages/b3/7a/abada41517ce0011775f0f4eacc79659bc9bc6c361e6bfe6f7052a6b9363/torch-2.10.0-3-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:98c01b8bb5e3240426dcde1446eed6f40c778091c8544767ef1168fc663a05a6", size = 915622781, upload-time = "2026-03-11T14:17:11.354Z" }, + { url = "https://files.pythonhosted.org/packages/ab/c6/4dfe238342ffdcec5aef1c96c457548762d33c40b45a1ab7033bb26d2ff2/torch-2.10.0-3-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:80b1b5bfe38eb0e9f5ff09f206dcac0a87aadd084230d4a36eea5ec5232c115b", size = 915627275, upload-time = "2026-03-11T14:16:11.325Z" }, + { url = "https://files.pythonhosted.org/packages/d8/f0/72bf18847f58f877a6a8acf60614b14935e2f156d942483af1ffc081aea0/torch-2.10.0-3-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:46b3574d93a2a8134b3f5475cfb98e2eb46771794c57015f6ad1fb795ec25e49", size = 915523474, upload-time = "2026-03-11T14:17:44.422Z" }, + { url = "https://files.pythonhosted.org/packages/f4/39/590742415c3030551944edc2ddc273ea1fdfe8ffb2780992e824f1ebee98/torch-2.10.0-3-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:b1d5e2aba4eb7f8e87fbe04f86442887f9167a35f092afe4c237dfcaaef6e328", size = 915632474, upload-time = "2026-03-11T14:15:13.666Z" }, + { url = "https://files.pythonhosted.org/packages/b6/8e/34949484f764dde5b222b7fe3fede43e4a6f0da9d7f8c370bb617d629ee2/torch-2.10.0-3-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:0228d20b06701c05a8f978357f657817a4a63984b0c90745def81c18aedfa591", size = 915523882, upload-time = "2026-03-11T14:14:46.311Z" }, { url = "https://files.pythonhosted.org/packages/cc/af/758e242e9102e9988969b5e621d41f36b8f258bb4a099109b7a4b4b50ea4/torch-2.10.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:5fd4117d89ffd47e3dcc71e71a22efac24828ad781c7e46aaaf56bf7f2796acf", size = 145996088, upload-time = "2026-01-21T16:24:44.171Z" }, { url = "https://files.pythonhosted.org/packages/23/8e/3c74db5e53bff7ed9e34c8123e6a8bfef718b2450c35eefab85bb4a7e270/torch-2.10.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:787124e7db3b379d4f1ed54dd12ae7c741c16a4d29b49c0226a89bea50923ffb", size = 915711952, upload-time = "2026-01-21T16:23:53.503Z" }, { url = "https://files.pythonhosted.org/packages/6e/01/624c4324ca01f66ae4c7cd1b74eb16fb52596dce66dbe51eff95ef9e7a4c/torch-2.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:2c66c61f44c5f903046cc696d088e21062644cbe541c7f1c4eaae88b2ad23547", size = 113757972, upload-time = "2026-01-21T16:24:39.516Z" }, @@ -2157,6 +2164,38 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e3/bd/fa9bb053192491b3867ba07d2343d9f2252e00811567d30ae8d0f78136fe/watchfiles-1.1.1-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:a916a2932da8f8ab582f242c065f5c81bed3462849ca79ee357dd9551b0e9b01", size = 622112, upload-time = "2025-10-14T15:05:50.941Z" }, ] +[[package]] +name = "webrtcvad-wheels" +version = "2.0.14" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/28/ba/3a8ce2cff3eee72a39ed190e5f9dac792da1526909c97a11589590b21739/webrtcvad_wheels-2.0.14.tar.gz", hash = "sha256:5f59c8e291c6ef102d9f39532982fbf26a52ce2de6328382e2654b0960fea397", size = 70607, upload-time = "2024-09-05T10:17:02.471Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dd/1f/af4af5d30228d46bbc3ad85cd12aac670b930d94107d8c755c191e5454f3/webrtcvad_wheels-2.0.14-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fc5d5a5bb13e5f25e095859d7a0018101a6563434781f27965f38ee75da45f9e", size = 31016, upload-time = "2024-09-05T10:15:08.799Z" }, + { url = "https://files.pythonhosted.org/packages/df/7f/8ff8af528b0a8db20d28356dc5486ecf779a7d1bd9ff379d8b3d921a2ab3/webrtcvad_wheels-2.0.14-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:323aa89df721377f053923fe1ea5c14e748c85314a977e2fcc3c920c4f474d40", size = 29515, upload-time = "2024-09-05T10:15:09.829Z" }, + { url = "https://files.pythonhosted.org/packages/24/16/8e21cfabd0cd9b9d01231e3b65716ab0cf1a6bd0b9115e2ed038996fe2c1/webrtcvad_wheels-2.0.14-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42310d3bb6cdb46a79a219d14fe97ae272b8af629d84321e5cfa0764556109a6", size = 86036, upload-time = "2024-09-05T10:15:11.122Z" }, + { url = "https://files.pythonhosted.org/packages/8a/c3/51e1f87610b55823367f08253fba50659beda2807e2ec4cf08360aff6c25/webrtcvad_wheels-2.0.14-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:09fdc138ce3519e2f4ba2f74640bc6f72aff162fa0c9ae2941da8068f3d22ba6", size = 95471, upload-time = "2024-09-05T10:15:12.652Z" }, + { url = "https://files.pythonhosted.org/packages/84/59/a6cd0eeae17640e43215843ea6176645b65f58dd6d15a38c77c253116e87/webrtcvad_wheels-2.0.14-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63fdad42c0ca6248b9a5c8ca34ff17a5c43bb63ffd8997a5902ad7237a05ca68", size = 82895, upload-time = "2024-09-05T10:15:13.738Z" }, + { url = "https://files.pythonhosted.org/packages/7b/cd/fd784f552a32d1b44be9ffe8b8f243c8c9a9eb69a61ef92a8b6611d38349/webrtcvad_wheels-2.0.14-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7aa4cfbf0c816d2b9ee3100d3008f5acf8bd1a634d6058fa6c1604a1e2ba264", size = 80808, upload-time = "2024-09-05T10:15:15.166Z" }, + { url = "https://files.pythonhosted.org/packages/7b/0c/28846038f5de2872b91a5b91925792d056477cd42a6d639b6a3ba84bd0e8/webrtcvad_wheels-2.0.14-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6056a2559d83836bf4b6af4993448202bfd080efd5e799d2153118e4a91d3fd1", size = 86266, upload-time = "2024-09-05T10:15:16.382Z" }, + { url = "https://files.pythonhosted.org/packages/17/38/8b20fb52c14bc316089a7486b559412c785968b17f97b5486ad17e115b0e/webrtcvad_wheels-2.0.14-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:db9d5e6ae07cc82277eacac2b747afa7db53e9effab0259055116f5f77a28bf3", size = 82609, upload-time = "2024-09-05T10:15:17.707Z" }, + { url = "https://files.pythonhosted.org/packages/3c/7c/3a8abe5cc3d2b65072b24c4122ceae29de0ac5d4d5df45ebf7f3f101ef74/webrtcvad_wheels-2.0.14-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:407c29a1d577dff8c2ef746bbd78151688f23ca763d97f3334d6e310ffa68aaa", size = 93359, upload-time = "2024-09-05T10:15:19.021Z" }, + { url = "https://files.pythonhosted.org/packages/7a/e6/14069e350e5cbd78f7c16add5613f03378d2ee36c43d5dc96d20ce9bc8e6/webrtcvad_wheels-2.0.14-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fe2ced612732a41aefa14ea2ea52f84de5afd43665ebed092e1d6df93641e239", size = 86940, upload-time = "2024-09-05T10:15:20.301Z" }, + { url = "https://files.pythonhosted.org/packages/ec/7c/ac5a40ca80a09e978968b7065528ce340295115d1af2b1e0a98aa0ac0244/webrtcvad_wheels-2.0.14-cp312-cp312-win32.whl", hash = "sha256:68e2041d1a30dc619a0e7ff5adaa948a6b862fa6e6d4d85337f235240707eab5", size = 17364, upload-time = "2024-09-05T10:15:21.162Z" }, + { url = "https://files.pythonhosted.org/packages/24/1d/9a8f4c842ca880253821acf165080077b245577719e5e03b13e3c45853c5/webrtcvad_wheels-2.0.14-cp312-cp312-win_amd64.whl", hash = "sha256:4010b95b31b9b360fd1bf47a8344b06b946ec866cf3d14fb39fc692307ddf312", size = 19901, upload-time = "2024-09-05T10:15:21.96Z" }, + { url = "https://files.pythonhosted.org/packages/51/ab/08750672514f4d9d0304d8648ebdf7827c38e266263953629b22a370c68f/webrtcvad_wheels-2.0.14-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:984f412292bd6edb3487911a5b2e36d1690a6afb8fc3fab18286b52fdf20eea0", size = 32768, upload-time = "2024-09-05T10:15:22.797Z" }, + { url = "https://files.pythonhosted.org/packages/ba/e0/ab4624a30c59abeaa6824cda455fd7842b7feafa94c7e52f35933926be88/webrtcvad_wheels-2.0.14-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:578151f6d1d58a3735fc4c4b7d47db3d42503a49587f84d9c42c2cd8aee93867", size = 29519, upload-time = "2024-09-05T10:15:23.867Z" }, + { url = "https://files.pythonhosted.org/packages/0e/cb/b1f1890e863e6adf0f6d3d9aabe1951169b7f514eae2fd8bd266752e923b/webrtcvad_wheels-2.0.14-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:792ce66f32b6d3ffa7569ef467b4ec1a5a45eb95cc466d204c04c120f4169015", size = 85987, upload-time = "2024-09-05T10:15:26.021Z" }, + { url = "https://files.pythonhosted.org/packages/cc/3d/78c1f47214bf75682aba75435a9762451f1d1392d810fae1b3a5ece1aecb/webrtcvad_wheels-2.0.14-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de5d6d71cbe2b92ec8411abd9c40d86e32f7d65f92a41029c0387d59c642810b", size = 95426, upload-time = "2024-09-05T10:15:27.231Z" }, + { url = "https://files.pythonhosted.org/packages/07/07/4b9eff8e3eb64e7017e6b28d7d3881290f19ece015ad182824f52ae088d0/webrtcvad_wheels-2.0.14-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c67055236f1ffcf160825bb87dbe2c1d3846ad51b54602906b1c7ea37a0b4ac", size = 82864, upload-time = "2024-09-05T10:15:29.73Z" }, + { url = "https://files.pythonhosted.org/packages/9e/32/067431c5313722860f37602d6c37a53f5f9250a0c46c70e918a8afce0293/webrtcvad_wheels-2.0.14-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2f2ae19589d89e1e67ad08f62fa11d5edce1a05a9a0e61d74ac3af6762da4caf", size = 80776, upload-time = "2024-09-05T10:15:31.092Z" }, + { url = "https://files.pythonhosted.org/packages/06/73/d297e6ea1493005e1b05c9ce56638f097bfd665a71045dad57000224a4fd/webrtcvad_wheels-2.0.14-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7de1df623d148052a61bd151545344dfafcf85dfb7075ea0e855009c080eb0ec", size = 86313, upload-time = "2024-09-05T10:15:32.014Z" }, + { url = "https://files.pythonhosted.org/packages/23/08/26f15a04aefeade474148ebf32ecc9aa68bb001fcea24dc8a3c299677e3d/webrtcvad_wheels-2.0.14-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:bcdffafce093f72ccea028698163b33c85267481900f8389fc2ecc1d6d1b57ba", size = 82633, upload-time = "2024-09-05T10:15:32.876Z" }, + { url = "https://files.pythonhosted.org/packages/ac/f9/02a13c7dd6113ecc3f2e02d4a2ef54586e55a0bb4ea3772b54df9e08eb49/webrtcvad_wheels-2.0.14-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:1a7a3cb39b4d33cf4895fceea9a12d4e30a44b98a9512e661ffb0a520b3f6bfb", size = 93403, upload-time = "2024-09-05T10:15:34.044Z" }, + { url = "https://files.pythonhosted.org/packages/a7/6e/0cd49b425ee9ccba2acd8451cbd96e6318081abcde6d9d6caa2083e53fab/webrtcvad_wheels-2.0.14-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:34b315326dd6c8529221492ebf28fc649d96b1a49ce49bf649eca60aabc70a2d", size = 86979, upload-time = "2024-09-05T10:15:35.045Z" }, + { url = "https://files.pythonhosted.org/packages/77/36/52e8998f8421c746defa8f8b509cab4aac984ef39a2cd5818f664901d245/webrtcvad_wheels-2.0.14-cp313-cp313-win32.whl", hash = "sha256:ff2a7eb4fe2189a7766726d6122b319df3e727c4cfed86409e2802fed1b459d3", size = 17368, upload-time = "2024-09-05T10:15:35.973Z" }, + { url = "https://files.pythonhosted.org/packages/f9/7c/73b9dd022c835e19180aad8aafd2f33863c02cbbafd5bc599dd54f6c1111/webrtcvad_wheels-2.0.14-cp313-cp313-win_amd64.whl", hash = "sha256:561f87975930833a5b77135fb6f15e6df430bfc000c327dc517a175aef15a707", size = 19901, upload-time = "2024-09-05T10:15:36.877Z" }, +] + [[package]] name = "websockets" version = "16.0" From fc5057366ce9c03922adeb13eb9dd7b8d5fd30d4 Mon Sep 17 00:00:00 2001 From: Ryan Malloy Date: Fri, 3 Jul 2026 20:08:20 -0600 Subject: [PATCH 3/3] docs: utterance queue, secretary, tone kit, listen Update CLAUDE.md for the streaming-utterance model (drop the stale _WorkItem/suppress_exit_tone references), the secretary announcements + reserved voice, call-waiting, the telephone tone kit, and the conversation-first listen() defaults. --- CLAUDE.md | 72 +++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 57 insertions(+), 15 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 1aca469..af2df3a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -36,17 +36,55 @@ Queued speech playback (`speak()`) is bookended by short alert tones. `generate_ | `scratch` | 2000-300 Hz sweep + noise | ~120 ms | Vinyl record scratch — needle yanked off the platter | | `reverse-roger` | 1000-1400 Hz | ~100 ms | Ascending two-tone — mathematical inverse of roger beep | +**Pink Floyd voice kit (current defaults).** The bookends now follow a telephone theme: + +| Name | Used as | Inspired by | +|------|---------|-------------| +| `heartbeat` | speak entry | "Speak to Me" heartbeat (DSotM opener) woven with warm MF telephone tones | +| `soft-chord` | speak exit | Mellow resolving A3+E4 dyad, pure sines, ear-friendly on repetition | +| `mf-dial` | listen start | The **genuine Young Lust R1 MF operator dial** — KP, 0-4-4-1-8-3-1, ST (the 44 is the UK country code), real Bell-System MF pairs | +| `machine` | listen end | "Welcome to the Machine" pulsing VCS3 throb — a warm "connected / got it" | +| `mf-listen` / `mf-done` | listen (alt) | Gentler stacked-fifths swell / resolving D-major (previous listen defaults) | +| `call-waiting` | over ongoing speech | Brief C6 double-blip mixed over the current message when another project queues | + +There are also softer "natural" tones (`bell-soft`, `chime-tube`, `water-drop`, `soft-pulse`, `hmm-up`, …) in `tones.py` for custom use. + ### Configuration ```env -TTS_ENTRY_TONE=chirp # before speech (chirp, apollo, none, or /path/to/custom.wav) -TTS_EXIT_TONE=roger # after speech, queue empty (roger, quindar-out, none, or path) -TTS_CANCEL_TONE=scratch # on cancel (scratch, reverse-roger, none, or /path/to/custom.wav) -TTS_SHUTDOWN_TIMEOUT=30 # max seconds to wait for current speech on container stop +TTS_ENTRY_TONE=heartbeat # before speech (heartbeat, chirp, apollo, none, or /path/to/custom.wav) +TTS_EXIT_TONE=soft-chord # after speech, queue empty (soft-chord, roger, quindar-out, none, or path) +TTS_CANCEL_TONE=scratch # on cancel (scratch, reverse-roger, none, or /path/to/custom.wav) +TTS_CALL_WAITING_TONE=call-waiting # mixed over current speech when a DIFFERENT project queues (once/turn) +TTS_LISTEN_START_TONE=mf-dial # played once the mic is live (see listen()) +TTS_LISTEN_END_TONE=machine # played after recording stops +TTS_SHUTDOWN_TIMEOUT=30 # max seconds to wait for current speech on container stop ``` The standby tone is always the built-in ascending blip. It plays instead of the exit tone when more items are queued. +## Secretary Announcements & Call-Waiting + +When several projects speak concurrently, the queue behaves like a **secretary**: each message plays whole and in order (never interleaved — see the streaming-utterance model below), and a project is announced by name when the speaker changes or returns after a lull. + +- The announcement is a short `"."` preamble synthesized in a **reserved secretary voice** (`TTS_SECRETARY_VOICE`, default `bf_emma`) — always via Kokoro so it sounds identical regardless of the speaking engine. That voice is excluded from the project auto-assignment pool so no project ever sounds like the secretary. +- The play-or-skip decision is made at **play time** in the consumer (`queue.py:_should_announce`), not enqueue time, because urgent reordering means the real speaker order isn't final until then. +- **Call-waiting**: when a *different* project's message joins the queue while one is playing, a brief `call-waiting` blip is mixed over the current audio (a second `pw-play` stream — PipeWire mixes it). Fires **at most once per playing turn** (`_call_waiting_fired`, reset when a new utterance starts) so a burst of queued messages never spams the listener. + +```env +TTS_ANNOUNCE_MODE=secretary # secretary | always | off (legacy TTS_ANNOUNCE_PROJECT=true → always) +TTS_REINTRODUCE_AFTER_SECONDS=120 # same project after this much silence gets re-introduced +TTS_SECRETARY_VOICE=bf_emma # reserved; excluded from project auto-assignment +``` + +## listen() — Voice Conversations + +`listen()` captures the host mic (`pw-record`), transcribes via Parakeet on the gpu.supported.systems gateway, and returns the text. Pair it with `speak()` for turn-taking: speak a question (let it finish), then `listen()` for the reply — **sequentially, never in parallel**, or the mic records the TTS. + +- **Defaults are conversation-first**: `wait_for_silence=True` (stop when the person stops), `duration_seconds=30` cap, `vad_aggressiveness=3`, `silence_threshold_ms=2200`. The aggressiveness/threshold defaults were tuned live to stop brief background transients from ending the turn before the real reply. +- **No first-word clip**: the "go" tone is played *after* the mic is live (a `warmup_ms` lead, default 150ms). The beep bleeds harmlessly into the head of the recording — VAD treats a pure tone as non-speech and Parakeet ignores it. See `audio.py:record_audio_until_silence`. +- Empty transcription (`text == ""`) or a gateway timeout means re-prompt rather than proceed; the recording is saved under `/tmp/mcspeak/` and can be retried with `transcribe()`. + Tones are generated programmatically at startup (48kHz, 16-bit PCM, -3 dB headroom) in `tones.py` using numpy. No bundled audio assets. ## Voice Identity (Project-Aware Voices) @@ -132,8 +170,8 @@ All pactl errors are caught and logged. If PulseAudio is unavailable (no socket, ## Architecture - `server.py` — FastMCP lifespan, tool definitions, engine setup -- `queue.py` — Producer-consumer speech queue with priority tiers and outcome tracking -- `tones.py` — Tone WAV generator (entry/exit/standby) +- `queue.py` — Producer-consumer queue of streaming **utterances** (one per `speak()`): priority tiers, secretary announcements + reserved voice, call-waiting, outcome tracking. Synthesized WAVs are reaped after playback (no /tmp leak). +- `tones.py` — Tone WAV generator (speak + listen bookends, call-waiting, natural set) - `media_duck.py` — Async PulseAudio volume control for media ducking - `audio.py` — WAV writing and `pw-play` async wrapper - `settings.py` — Pydantic settings from env vars (prefix: `TTS_`) @@ -185,16 +223,18 @@ Kokoro synthesizes ~4x realtime on CPU, so synthesis always outpaces playback. ` Texts under 20 words or without sentence boundaries (`.!?` followed by whitespace) take the single-shot path — zero overhead, identical to pre-chunking behavior. -### Tone behavior +### Tone behavior & per-message coherence + +Each `speak()` call is **one queue utterance** (not one queue item per chunk). The utterance reserves its ordering slot up front and streams its synthesized chunks in through an internal channel closed by an `_END` sentinel; the consumer stays locked to it from entry tone to exit tone. So when several projects speak at once, a message plays **whole and in order** — chunks never interleave with another project's audio (the old per-chunk `suppress_exit_tone`/`_WorkItem` model is gone). - Entry tone plays once at the start (before first chunk synthesis) -- Exit/standby tones are suppressed between chunks (`suppress_exit_tone` flag on `_WorkItem`) -- Final chunk plays the normal exit tone (roger) or standby tone +- Exit/standby tone plays once, at the very end of the utterance +- Pipelining is preserved: synthesis of chunk N+1 overlaps playback of chunk N, but nothing else can be pulled until this utterance finishes ### Cancellation in chunked mode -- **Explicit `cancel_speech(speech_id)`** — cancels the returned speech_id (the final chunk). Already-playing earlier chunks finish naturally. -- **MCP disconnect** — the synthesis loop stops (remaining chunks aren't synthesized). Already-enqueued chunks play through. +- **Explicit `cancel_speech(speech_id)`** — one speech_id now covers the whole utterance; cancelling it flags an abort (synchronously), reaps un-played chunk WAVs, and plays the cancel tone. +- **MCP disconnect** — the `speak()` handler is cancelled; its `finally` closes the utterance channel so the consumer drains the chunks it already has and finishes cleanly. ### Progress lifecycle (chunked) @@ -208,8 +248,8 @@ Texts under 20 words or without sentence boundaries (`.!?` followed by whitespac ### Files -- `server.py` — `split_text()`, `_speak_single()`, `_speak_chunked()`, `_await_with_progress()` -- `queue.py` — `suppress_exit_tone` field on `_WorkItem` +- `server.py` — `split_text()`, unified `_speak()` (short + chunked share one path), status-aware `_await_with_progress()` +- `queue.py` — `_Utterance` (streaming chunk channel + `_END` sentinel), `create_utterance()`, `_play_utterance()`, `_should_announce()`, call-waiting ## Cancellation @@ -235,8 +275,10 @@ Docker's `stop_grace_period` must exceed the total: `3s + shutdown_timeout + 5s ## Key Design Decisions -- Speech queue is serialized (one playback at a time) but synthesis is parallel -- `speak()` blocks until playback finishes with live progress (5% → 30% → 35-99% → 100%) +- Speech queue is serialized (one playback at a time) but synthesis is parallel; each `speak()` is one **streaming utterance** so concurrent projects never interleave +- Secretary behavior: a project is announced by name on speaker-change / after a lull, in a **reserved voice** excluded from the project pool; a different project queuing mid-playback fires a **once-per-turn** call-waiting blip mixed over the current audio +- Synthesized WAVs are reaped after playback (and on cancel/shutdown) so `/tmp/mcspeak` doesn't grow unbounded +- `speak()` blocks until playback finishes with **status-aware** progress — a queued item reports "waiting in line", not a false "playing" percentage - Progress uses a background ticker task, NOT `asyncio.wait_for` polling (see below) - Entry tone is awaited in `speak()` before synthesis — covers latency gap - Explicit `cancel_speech()` kills pw-play + plays cancel tone; MCP disconnect lets playback finish