diff --git a/PLATFORMS.md b/PLATFORMS.md new file mode 100644 index 0000000..1af3130 --- /dev/null +++ b/PLATFORMS.md @@ -0,0 +1,62 @@ +# Platform support + +McSpeak's engines (Kokoro/Piper/Orpheus) are cross-platform; only the +**audio I/O** is OS-specific. Everything the mic/speaker touches goes through +four subprocess touch-points in `platform_audio.py`, so porting to a new OS is +adding a branch there — the queue, VAD, secretary, tones, and progress logic +never change. Every backend's recorder emits the **same wire format** (raw +`s16`, mono, 16 kHz, streamed to stdout), so the VAD frame loop in `audio.py` +is byte-for-byte identical everywhere. + +| Touch-point | Linux (default) | macOS | Windows (planned) | +|---|---|---|---| +| Play WAV | `pw-play` | `afplay` | `winsound.PlaySound` (stdlib, in-process) | +| Record → s16 PCM/stdout | `pw-record -` | Swift/AVFoundation binary | `sounddevice` (PortAudio) or `ffmpeg -f dshow` | +| Transcode / resample | `ffmpeg` | `afconvert` (m4a) / `ffmpeg` | `ffmpeg` or in-process `soundfile` | +| Duck other apps | `pactl` (per-app) | none — deliberate no-op¹ | `pycaw` (per-app) | + +¹ macOS ducking is intentionally a no-op: the only built-in volume control +(`osascript`) is system-wide and would dim our own `afplay` voice. Per-app +ducking needs CoreAudio, a future native-helper task. + +## Linux (current production) + +Runs in Docker with the host PipeWire socket bind-mounted +(`/run/user/1000/pulse`). `make up` builds + starts. This is the tested, +shipping path — unchanged by the cross-platform work. + +## macOS (audio backend done; native run TBD) + +Status: the four audio touch-points are implemented. Headless mic capture is +**verified** on an Apple-Silicon Mac over SSH (TCC does not block it). Not yet +done: running the server natively + a live speech test. + +Requirements (all present on the target Mac, all built-in except the venv): +- **Xcode command-line tools** (`xcode-select --install`) for `swiftc` — the + recorder is compiled once, cached in `~/.cache/mcspeak/`. +- `afplay`, `afconvert`, `osascript` ship with macOS. +- `uv` for the venv (no Docker — Docker Desktop containers can't reach + CoreAudio, which is why the Mac already runs its ML services natively). + +Deployment sketch (native, as the console user who owns the audio session): +```bash +uv sync # or a minimal env for listen-only testing: + # numpy webrtcvad-wheels httpx pydantic-settings +uv run mcspeak # server picks the macOS backend automatically +``` +`generate_audio` in mp3/ogg/flac needs `brew install ffmpeg`; wav and m4a work +with built-ins alone. + +## Windows (designed, not implemented) + +The abstraction has the slots; the work is filling the four touch-points: +- **Play**: `winsound.PlaySound` — Python stdlib, zero deps, WAV only. +- **Record**: the real gap (same as macOS had). `sounddevice` (PortAudio pip + wheel) is the cleanest — it streams numpy frames that feed `webrtcvad` + directly; `ffmpeg -f dshow` is the subprocess alternative. +- **Transcode**: `ffmpeg`, or do it in-process with `soundfile`/`numpy`. +- **Duck**: `pycaw` (per-app WASAPI), or degrade to the no-op. + +`sounddevice` could cover both Windows *and* macOS recording with one library +if we ever want to drop the Swift compile step — macOS stays built-ins-only +for now to keep it zero-dependency. diff --git a/src/mcspeak/_macos/mcspeak_record.swift b/src/mcspeak/_macos/mcspeak_record.swift new file mode 100644 index 0000000..11a9683 --- /dev/null +++ b/src/mcspeak/_macos/mcspeak_record.swift @@ -0,0 +1,64 @@ +// mcspeak macOS mic recorder — streams 16 kHz mono s16 PCM to stdout until +// SIGTERM, mirroring `pw-record --rate 16000 --channels 1 --format s16 -`. +// The VAD loop in audio.py reads this stream frame-by-frame, unchanged, so the +// rest of mcspeak is oblivious to which OS captured the audio. +// +// Built once by platform_audio.ensure_macos_recorder() via `swiftc -O`. +// Uses only the AVFoundation framework that ships with macOS — no dependencies. +import AVFoundation +import Foundation + +let targetRate = 16000.0 + +let engine = AVAudioEngine() +let input = engine.inputNode +let inputFormat = input.outputFormat(forBus: 0) +FileHandle.standardError.write( + "mcspeak-record: input \(inputFormat.sampleRate)Hz \(inputFormat.channelCount)ch -> 16000Hz mono s16\n" + .data(using: .utf8)!) + +guard let targetFormat = AVAudioFormat( + commonFormat: .pcmFormatInt16, sampleRate: targetRate, + channels: 1, interleaved: true +) else { + FileHandle.standardError.write("mcspeak-record: bad target format\n".data(using: .utf8)!) + exit(2) +} +guard let converter = AVAudioConverter(from: inputFormat, to: targetFormat) else { + FileHandle.standardError.write("mcspeak-record: no converter\n".data(using: .utf8)!) + exit(2) +} + +let out = FileHandle.standardOutput + +input.installTap(onBus: 0, bufferSize: 2048, format: inputFormat) { buffer, _ in + // Resample + downmix each hardware buffer to 16k mono s16 and write raw + // little-endian samples straight to stdout. + let cap = AVAudioFrameCount(Double(buffer.frameLength) * targetRate / inputFormat.sampleRate) + 32 + guard let outBuf = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: cap) else { return } + var fed = false + var err: NSError? + let status = converter.convert(to: outBuf, error: &err) { _, outStatus in + if fed { outStatus.pointee = .noDataNow; return nil } + fed = true + outStatus.pointee = .haveData + return buffer + } + if status == .error { return } + if let ch = outBuf.int16ChannelData, outBuf.frameLength > 0 { + out.write(Data(bytes: ch[0], count: Int(outBuf.frameLength) * 2)) + } +} + +// Terminate cleanly on SIGTERM/SIGINT so the WAV/stream closes like pw-record. +signal(SIGTERM) { _ in exit(0) } +signal(SIGINT) { _ in exit(0) } + +do { + try engine.start() +} catch { + FileHandle.standardError.write("mcspeak-record: engine start failed: \(error)\n".data(using: .utf8)!) + exit(3) +} + +RunLoop.current.run() diff --git a/src/mcspeak/audio.py b/src/mcspeak/audio.py index 18f3ef1..4574649 100644 --- a/src/mcspeak/audio.py +++ b/src/mcspeak/audio.py @@ -10,6 +10,7 @@ from pathlib import Path import numpy as np +from . import platform_audio from .settings import settings _counter = itertools.count(1) @@ -93,13 +94,19 @@ class ConversionError(RuntimeError): """Raised when format conversion fails.""" +# macOS built-in afconvert can only encode a subset (AAC/m4a) — no mp3/ogg/flac +# encoders ship with it. Those still need ffmpeg (brew install ffmpeg). +_AFCONVERT_ARGS: dict[str, list[str]] = { + "m4a": ["-f", "m4af", "-d", "aac"], # AAC in an MPEG-4 container +} + + async def convert_audio(src_wav: Path, dest: Path, fmt: str) -> Path: """Convert a source WAV to the requested format at dest. - `fmt == "wav"` shells out to a plain copy (no ffmpeg invocation). - Other formats use ffmpeg with codec args from _FFMPEG_CODEC_ARGS. - - Raises ConversionError on ffmpeg non-zero exit or unsupported format. + `fmt == "wav"` is a plain copy. Otherwise ffmpeg is used when available; on + macOS without ffmpeg, afconvert covers m4a (mp3/ogg/flac still need ffmpeg). + Raises ConversionError on failure or unsupported format. """ if fmt not in SUPPORTED_FORMATS: raise ConversionError( @@ -112,20 +119,44 @@ async def convert_audio(src_wav: Path, dest: Path, fmt: str) -> Path: shutil.copyfile(src_wav, dest) return dest - proc = await asyncio.create_subprocess_exec( - "ffmpeg", "-y", "-loglevel", "error", "-i", str(src_wav), - *_FFMPEG_CODEC_ARGS[fmt], - str(dest), - stdout=asyncio.subprocess.DEVNULL, - stderr=asyncio.subprocess.PIPE, - ) - _, stderr = await proc.communicate() - if proc.returncode != 0: - raise ConversionError( - f"ffmpeg failed ({proc.returncode}) converting to {fmt}: " - f"{stderr.decode(errors='replace').strip()[:500]}" + if platform_audio.has_ffmpeg(): + proc = await asyncio.create_subprocess_exec( + "ffmpeg", "-y", "-loglevel", "error", "-i", str(src_wav), + *_FFMPEG_CODEC_ARGS[fmt], + str(dest), + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.PIPE, ) - return dest + _, stderr = await proc.communicate() + if proc.returncode != 0: + raise ConversionError( + f"ffmpeg failed ({proc.returncode}) converting to {fmt}: " + f"{stderr.decode(errors='replace').strip()[:500]}" + ) + return dest + + # No ffmpeg — fall back to the macOS built-in afconvert for what it can do. + if platform_audio.IS_MACOS: + args = _AFCONVERT_ARGS.get(fmt) + if args is None: + raise ConversionError( + f"macOS afconvert can't encode {fmt!r} (only m4a/wav without ffmpeg). " + f"Install ffmpeg (brew install ffmpeg) for mp3/ogg/flac." + ) + proc = await asyncio.create_subprocess_exec( + "afconvert", *args, str(src_wav), str(dest), + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.PIPE, + ) + _, stderr = await proc.communicate() + if proc.returncode != 0: + raise ConversionError( + f"afconvert failed ({proc.returncode}) converting to {fmt}: " + f"{stderr.decode(errors='replace').strip()[:500]}" + ) + return dest + + raise ConversionError(f"ffmpeg not found — needed to convert to {fmt!r}. Install ffmpeg.") class PlaybackError(RuntimeError): @@ -228,15 +259,10 @@ async def record_audio_until_silence( vad = webrtcvad.Vad(aggressiveness) - cmd = [ - "pw-record", - "--rate", str(SAMPLE_RATE), - "--channels", "1", - "--format", "s16", - ] - if source: - cmd.extend(["--target", source]) - cmd.append("-") # stdout + # Platform-appropriate recorder — always streams s16 mono PCM to stdout, so + # the VAD frame loop below is identical on Linux (pw-record) and macOS (the + # Swift/AVFoundation recorder). + cmd = platform_audio.record_stream_command(SAMPLE_RATE, 1, source) proc = await asyncio.create_subprocess_exec( *cmd, @@ -346,9 +372,17 @@ async def record_audio( kill it. pw-record handles SIGTERM by closing the WAV header cleanly — verified by recording 2s and reading the file back with the wave module. - Returns out_path on success. Raises PlaybackError on pw-record failure or + Returns out_path on success. Raises PlaybackError on recorder failure or if the resulting WAV is empty (mic disconnected, etc). + + On macOS the Swift recorder only streams to stdout, so the fixed-duration + path collects PCM for `duration_seconds` and writes the WAV itself. """ + if platform_audio.IS_MACOS: + return await _record_fixed_stream( + out_path, duration_seconds, sample_rate, source, ready_tone, warmup_ms + ) + cmd = [ "pw-record", "--rate", str(sample_rate), @@ -407,8 +441,71 @@ async def record_audio( return out_path +async def _record_fixed_stream( + out_path: Path, + duration_seconds: float, + sample_rate: int, + source: str | None, + ready_tone: Path | None, + warmup_ms: int, +) -> Path: + """Fixed-duration capture for stdout-streaming backends (macOS). + + Runs the streaming recorder, plays the ready tone once the mic is live, + collects s16 PCM for `duration_seconds`, then writes the WAV. Mirrors + record_audio's contract (returns out_path; raises PlaybackError on no audio). + """ + cmd = platform_audio.record_stream_command(sample_rate, 1, source) + proc = await asyncio.create_subprocess_exec( + *cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.DEVNULL, + ) + + if ready_tone is not None: + await asyncio.sleep(warmup_ms / 1000) + try: + await play_audio(ready_tone, expected_seconds=0.3) + except Exception: + pass # tone failure is non-fatal — keep recording + + pcm = bytearray() + + async def _drain() -> None: + while True: + chunk = await proc.stdout.read(8192) + if not chunk: + break + pcm.extend(chunk) + + try: + await asyncio.wait_for(_drain(), timeout=duration_seconds) + except asyncio.TimeoutError: + proc.terminate() + try: + await asyncio.wait_for(proc.wait(), timeout=2.0) + except asyncio.TimeoutError: + proc.kill() + await proc.wait() + except asyncio.CancelledError: + proc.kill() + await proc.wait() + raise + + if len(pcm) < 100: + raise PlaybackError( + "macOS recorder produced no audio. Check that the launching process " + "has Microphone permission (System Settings > Privacy > Microphone)." + ) + write_wav_from_pcm( + bytes(pcm), sample_rate=sample_rate, sample_width=2, channels=1, path=out_path + ) + return out_path + + async def play_audio(path: Path, expected_seconds: float = 0) -> None: - """Play a WAV file through PipeWire (pw-play). Async wrapper. + """Play a WAV file to the default output device (pw-play on Linux, afplay on + macOS). Async wrapper. Args: path: Path to the WAV file. @@ -417,35 +514,42 @@ async def play_audio(path: Path, expected_seconds: float = 0) -> None: """ if expected_seconds <= 0: expected_seconds = wav_duration(path) - # Generous margin: 2x duration + 10s for PipeWire startup overhead + # Generous margin: 2x duration + 10s for player startup overhead timeout = max(expected_seconds * 2, 10) + 10 proc = await asyncio.create_subprocess_exec( - "pw-play", str(path), + *platform_audio.play_command(path), stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.PIPE, ) - # Defensive volume override — guards against module-stream-restore drift. - vol_task = asyncio.create_task(_force_pwplay_volume_100()) + # Defensive volume override — guards against PulseAudio module-stream-restore + # drift. Linux/PipeWire only; other backends don't need it. + vol_task = ( + asyncio.create_task(_force_pwplay_volume_100()) + if platform_audio.volume_workaround_applies() else None + ) try: _, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout) except asyncio.TimeoutError: proc.kill() await proc.wait() - vol_task.cancel() + if vol_task: + vol_task.cancel() raise PlaybackError( - f"pw-play timed out after {timeout:.0f}s " + f"playback timed out after {timeout:.0f}s " f"(expected {expected_seconds:.1f}s audio)" ) except asyncio.CancelledError: # Consumer was cancelled (shutdown or cancel) — don't orphan the subprocess proc.kill() await proc.wait() - vol_task.cancel() + if vol_task: + vol_task.cancel() raise - if not vol_task.done(): + if vol_task and not vol_task.done(): vol_task.cancel() if proc.returncode != 0: raise PlaybackError( - f"pw-play failed ({proc.returncode}): {stderr.decode().strip()}" + f"{platform_audio.play_command(path)[0]} failed ({proc.returncode}): " + f"{stderr.decode(errors='replace').strip()}" ) diff --git a/src/mcspeak/media_duck.py b/src/mcspeak/media_duck.py index 0124751..9ecd4d1 100644 --- a/src/mcspeak/media_duck.py +++ b/src/mcspeak/media_duck.py @@ -11,6 +11,13 @@ protocol connection, while pactl (using libpulse) works reliably. If pactl is not installed or the PulseAudio socket is unavailable, all operations silently no-op — TTS still works, just without ducking. + +macOS: intentionally left as that no-op. The only built-in volume control +(`osascript ... set volume output volume`) is SYSTEM-WIDE, and our TTS plays +through afplay as system audio — so ducking would dim our own voice, the +opposite of the goal. Per-app ducking on macOS needs CoreAudio (not a +built-in), so proper ducking is a future native-helper task, not an osascript +one. Don't "fix" this by adding osascript volume control. """ import asyncio diff --git a/src/mcspeak/platform_audio.py b/src/mcspeak/platform_audio.py new file mode 100644 index 0000000..01140af --- /dev/null +++ b/src/mcspeak/platform_audio.py @@ -0,0 +1,134 @@ +"""Platform audio backend — the four OS-specific subprocess touch-points. + +The rest of mcspeak reaches the speaker/mic only through here, so porting to a +new OS means adding a branch, not touching the queue/VAD/tone/secretary logic. +Every backend's recorder emits the SAME wire format (raw s16, mono, 16 kHz, to +stdout), so the VAD reader in audio.py is identical on every platform. + +- Linux : PipeWire CLIs — `pw-play`, `pw-record` (the proven Docker path). +- macOS : built-ins only — `afplay` (play), a compiled Swift/AVFoundation + recorder (`_macos/mcspeak_record.swift`), `afconvert` (transcode), + `osascript` (system-volume duck). +- Windows: PLANNED — `winsound.PlaySound` (stdlib WAV play), `sounddevice` + (PortAudio) or `ffmpeg -f dshow` (capture), `pycaw` (per-app duck). +""" + +from __future__ import annotations + +import hashlib +import shutil +import subprocess +import sys +from pathlib import Path + +IS_LINUX = sys.platform.startswith("linux") +IS_MACOS = sys.platform == "darwin" +IS_WINDOWS = sys.platform.startswith("win") + +_MACOS_DIR = Path(__file__).parent / "_macos" +_RECORDER_SRC = _MACOS_DIR / "mcspeak_record.swift" +# Compiled recorder lives in a user cache dir (the install dir may be read-only). +_CACHE_DIR = Path.home() / ".cache" / "mcspeak" + + +class AudioBackendError(RuntimeError): + """Raised when the platform backend can't satisfy a request.""" + + +def platform_name() -> str: + if IS_MACOS: + return "macos" + if IS_WINDOWS: + return "windows" + if IS_LINUX: + return "linux" + return sys.platform + + +# --------------------------------------------------------------------------- +# Playback +# --------------------------------------------------------------------------- + +def play_command(path: Path | str) -> list[str]: + """Command to play a WAV to the default output device, blocking until done.""" + if IS_MACOS: + return ["afplay", str(path)] + if IS_WINDOWS: + # Windows plays via winsound in-process (see play_audio), not a subprocess. + raise AudioBackendError("Windows playback is in-process; don't call play_command") + return ["pw-play", str(path)] # Linux / PipeWire + + +def volume_workaround_applies() -> bool: + """Only PulseAudio/PipeWire needs the stream-restore 100% volume override.""" + return IS_LINUX + + +# --------------------------------------------------------------------------- +# Recording — every backend streams raw s16 mono PCM at `sample_rate` to stdout +# --------------------------------------------------------------------------- + +def record_stream_command( + sample_rate: int = 16000, channels: int = 1, source: str | None = None +) -> list[str]: + """Command that streams raw s16 PCM to stdout until it receives SIGTERM.""" + if IS_MACOS: + # The Swift recorder is fixed at 16 kHz mono (Parakeet's format); it + # ignores sample_rate/source for now (default input device only). + return [str(ensure_macos_recorder())] + if IS_WINDOWS: + raise AudioBackendError( + "Windows capture not implemented yet — planned via sounddevice/ffmpeg" + ) + cmd = ["pw-record", "--rate", str(sample_rate), "--channels", str(channels), "--format", "s16"] + if source: + cmd.extend(["--target", source]) + cmd.append("-") # stdout + return cmd + + +def ensure_macos_recorder() -> Path: + """Compile the Swift recorder once (cached in ~/.cache/mcspeak), return its path. + + Recompiles only when the source changes (tracked by a content-hash stamp). + """ + if not IS_MACOS: + raise AudioBackendError("macOS recorder requested on a non-macOS host") + swiftc = shutil.which("swiftc") + if not swiftc: + raise AudioBackendError( + "swiftc not found — install the Xcode command-line tools (xcode-select --install)" + ) + _CACHE_DIR.mkdir(parents=True, exist_ok=True) + binary = _CACHE_DIR / "mcspeak-record" + src_hash = hashlib.sha256(_RECORDER_SRC.read_bytes()).hexdigest()[:12] + stamp = _CACHE_DIR / f".record-{src_hash}" + if binary.exists() and stamp.exists(): + return binary + + proc = subprocess.run( + [swiftc, "-O", str(_RECORDER_SRC), "-o", str(binary)], + capture_output=True, text=True, + ) + if proc.returncode != 0: + raise AudioBackendError( + f"swiftc failed compiling the recorder: {proc.stderr.strip()[:500]}" + ) + # Clear any stale stamps, then mark this build good. + for old in _CACHE_DIR.glob(".record-*"): + old.unlink(missing_ok=True) + stamp.write_text("ok") + return binary + + +# --------------------------------------------------------------------------- +# Transcode / resample (used by convert_audio and any 16k-mono normalization) +# --------------------------------------------------------------------------- + +def has_ffmpeg() -> bool: + return shutil.which("ffmpeg") is not None + + +def prefers_afconvert() -> bool: + """On macOS without ffmpeg, use the built-in afconvert.""" + return IS_MACOS and not has_ffmpeg() diff --git a/src/mcspeak/server.py b/src/mcspeak/server.py index 3342177..a1fc7c0 100644 --- a/src/mcspeak/server.py +++ b/src/mcspeak/server.py @@ -147,6 +147,16 @@ async def app_lifespan(server: FastMCP): health = await eng.check_health() print(f" {name}: {health['status']}", file=sys.stderr) + # On macOS, pre-compile the Swift mic recorder now so the first listen() + # isn't slowed by swiftc and any build error surfaces here, not mid-call. + from . import platform_audio + if platform_audio.IS_MACOS: + try: + rec = platform_audio.ensure_macos_recorder() + print(f" macOS mic recorder ready: {rec}", file=sys.stderr) + except platform_audio.AudioBackendError as e: + print(f" macOS mic recorder unavailable (listen will fail): {e}", file=sys.stderr) + # Generate tones and resolve which ones to use tone_paths = generate_tones(settings.output_dir) entry_tone = resolve_tone(settings.entry_tone, tone_paths, "entry_tone", default="heartbeat")