macOS audio backend groundwork (capture proven, deploy TBD)

Introduce a platform audio backend so the four OS-specific subprocess
touch-points (play, record-stream, transcode, duck) are swappable without
touching the queue/VAD/tone logic. The macOS recorder emits the same
s16/mono/16k PCM-to-stdout wire format as `pw-record -`, so the VAD frame
loop is byte-for-byte identical across platforms.

- platform_audio.py: backend selector (Linux + macOS implemented, Windows
  documented). Compiles the Swift recorder once, cached in ~/.cache/mcspeak.
- _macos/mcspeak_record.swift: AVFoundation recorder, built-ins only (no brew).
  Verified capturing on the target Mac over SSH — TCC does not block headless
  capture.
- audio.py: play_audio uses afplay on macOS (pw-play + volume workaround stay
  Linux-only); record_audio_until_silence streams via the backend command;
  record_audio gains a stream-and-collect path for stdout-only backends.

Linux behavior is unchanged (commands verified, queue tests green, ruff clean).

Not yet done: afconvert transcode branch (generate_audio non-WAV only),
osascript ducking (currently a graceful no-op), and running the server
natively on the Mac + a live speech test.
This commit is contained in:
Ryan Malloy 2026-07-03 22:22:58 -06:00
parent f13278da64
commit 2711e90dce
3 changed files with 292 additions and 20 deletions

View File

@ -0,0 +1,64 @@
// mcspeak macOS mic recorder streams 16 kHz mono s16 PCM to stdout until
// SIGTERM, mirroring `pw-record --rate 16000 --channels 1 --format s16 -`.
// The VAD loop in audio.py reads this stream frame-by-frame, unchanged, so the
// rest of mcspeak is oblivious to which OS captured the audio.
//
// Built once by platform_audio.ensure_macos_recorder() via `swiftc -O`.
// Uses only the AVFoundation framework that ships with macOS no dependencies.
import AVFoundation
import Foundation
let targetRate = 16000.0
let engine = AVAudioEngine()
let input = engine.inputNode
let inputFormat = input.outputFormat(forBus: 0)
FileHandle.standardError.write(
"mcspeak-record: input \(inputFormat.sampleRate)Hz \(inputFormat.channelCount)ch -> 16000Hz mono s16\n"
.data(using: .utf8)!)
guard let targetFormat = AVAudioFormat(
commonFormat: .pcmFormatInt16, sampleRate: targetRate,
channels: 1, interleaved: true
) else {
FileHandle.standardError.write("mcspeak-record: bad target format\n".data(using: .utf8)!)
exit(2)
}
guard let converter = AVAudioConverter(from: inputFormat, to: targetFormat) else {
FileHandle.standardError.write("mcspeak-record: no converter\n".data(using: .utf8)!)
exit(2)
}
let out = FileHandle.standardOutput
input.installTap(onBus: 0, bufferSize: 2048, format: inputFormat) { buffer, _ in
// Resample + downmix each hardware buffer to 16k mono s16 and write raw
// little-endian samples straight to stdout.
let cap = AVAudioFrameCount(Double(buffer.frameLength) * targetRate / inputFormat.sampleRate) + 32
guard let outBuf = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: cap) else { return }
var fed = false
var err: NSError?
let status = converter.convert(to: outBuf, error: &err) { _, outStatus in
if fed { outStatus.pointee = .noDataNow; return nil }
fed = true
outStatus.pointee = .haveData
return buffer
}
if status == .error { return }
if let ch = outBuf.int16ChannelData, outBuf.frameLength > 0 {
out.write(Data(bytes: ch[0], count: Int(outBuf.frameLength) * 2))
}
}
// Terminate cleanly on SIGTERM/SIGINT so the WAV/stream closes like pw-record.
signal(SIGTERM) { _ in exit(0) }
signal(SIGINT) { _ in exit(0) }
do {
try engine.start()
} catch {
FileHandle.standardError.write("mcspeak-record: engine start failed: \(error)\n".data(using: .utf8)!)
exit(3)
}
RunLoop.current.run()

View File

@ -10,6 +10,7 @@ from pathlib import Path
import numpy as np import numpy as np
from . import platform_audio
from .settings import settings from .settings import settings
_counter = itertools.count(1) _counter = itertools.count(1)
@ -228,15 +229,10 @@ async def record_audio_until_silence(
vad = webrtcvad.Vad(aggressiveness) vad = webrtcvad.Vad(aggressiveness)
cmd = [ # Platform-appropriate recorder — always streams s16 mono PCM to stdout, so
"pw-record", # the VAD frame loop below is identical on Linux (pw-record) and macOS (the
"--rate", str(SAMPLE_RATE), # Swift/AVFoundation recorder).
"--channels", "1", cmd = platform_audio.record_stream_command(SAMPLE_RATE, 1, source)
"--format", "s16",
]
if source:
cmd.extend(["--target", source])
cmd.append("-") # stdout
proc = await asyncio.create_subprocess_exec( proc = await asyncio.create_subprocess_exec(
*cmd, *cmd,
@ -346,9 +342,17 @@ async def record_audio(
kill it. pw-record handles SIGTERM by closing the WAV header cleanly kill it. pw-record handles SIGTERM by closing the WAV header cleanly
verified by recording 2s and reading the file back with the wave module. verified by recording 2s and reading the file back with the wave module.
Returns out_path on success. Raises PlaybackError on pw-record failure or Returns out_path on success. Raises PlaybackError on recorder failure or
if the resulting WAV is empty (mic disconnected, etc). if the resulting WAV is empty (mic disconnected, etc).
On macOS the Swift recorder only streams to stdout, so the fixed-duration
path collects PCM for `duration_seconds` and writes the WAV itself.
""" """
if platform_audio.IS_MACOS:
return await _record_fixed_stream(
out_path, duration_seconds, sample_rate, source, ready_tone, warmup_ms
)
cmd = [ cmd = [
"pw-record", "pw-record",
"--rate", str(sample_rate), "--rate", str(sample_rate),
@ -407,8 +411,71 @@ async def record_audio(
return out_path return out_path
async def _record_fixed_stream(
out_path: Path,
duration_seconds: float,
sample_rate: int,
source: str | None,
ready_tone: Path | None,
warmup_ms: int,
) -> Path:
"""Fixed-duration capture for stdout-streaming backends (macOS).
Runs the streaming recorder, plays the ready tone once the mic is live,
collects s16 PCM for `duration_seconds`, then writes the WAV. Mirrors
record_audio's contract (returns out_path; raises PlaybackError on no audio).
"""
cmd = platform_audio.record_stream_command(sample_rate, 1, source)
proc = await asyncio.create_subprocess_exec(
*cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.DEVNULL,
)
if ready_tone is not None:
await asyncio.sleep(warmup_ms / 1000)
try:
await play_audio(ready_tone, expected_seconds=0.3)
except Exception:
pass # tone failure is non-fatal — keep recording
pcm = bytearray()
async def _drain() -> None:
while True:
chunk = await proc.stdout.read(8192)
if not chunk:
break
pcm.extend(chunk)
try:
await asyncio.wait_for(_drain(), timeout=duration_seconds)
except asyncio.TimeoutError:
proc.terminate()
try:
await asyncio.wait_for(proc.wait(), timeout=2.0)
except asyncio.TimeoutError:
proc.kill()
await proc.wait()
except asyncio.CancelledError:
proc.kill()
await proc.wait()
raise
if len(pcm) < 100:
raise PlaybackError(
"macOS recorder produced no audio. Check that the launching process "
"has Microphone permission (System Settings > Privacy > Microphone)."
)
write_wav_from_pcm(
bytes(pcm), sample_rate=sample_rate, sample_width=2, channels=1, path=out_path
)
return out_path
async def play_audio(path: Path, expected_seconds: float = 0) -> None: async def play_audio(path: Path, expected_seconds: float = 0) -> None:
"""Play a WAV file through PipeWire (pw-play). Async wrapper. """Play a WAV file to the default output device (pw-play on Linux, afplay on
macOS). Async wrapper.
Args: Args:
path: Path to the WAV file. path: Path to the WAV file.
@ -417,35 +484,42 @@ async def play_audio(path: Path, expected_seconds: float = 0) -> None:
""" """
if expected_seconds <= 0: if expected_seconds <= 0:
expected_seconds = wav_duration(path) expected_seconds = wav_duration(path)
# Generous margin: 2x duration + 10s for PipeWire startup overhead # Generous margin: 2x duration + 10s for player startup overhead
timeout = max(expected_seconds * 2, 10) + 10 timeout = max(expected_seconds * 2, 10) + 10
proc = await asyncio.create_subprocess_exec( proc = await asyncio.create_subprocess_exec(
"pw-play", str(path), *platform_audio.play_command(path),
stdout=asyncio.subprocess.DEVNULL, stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE,
) )
# Defensive volume override — guards against module-stream-restore drift. # Defensive volume override — guards against PulseAudio module-stream-restore
vol_task = asyncio.create_task(_force_pwplay_volume_100()) # drift. Linux/PipeWire only; other backends don't need it.
vol_task = (
asyncio.create_task(_force_pwplay_volume_100())
if platform_audio.volume_workaround_applies() else None
)
try: try:
_, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout) _, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout)
except asyncio.TimeoutError: except asyncio.TimeoutError:
proc.kill() proc.kill()
await proc.wait() await proc.wait()
vol_task.cancel() if vol_task:
vol_task.cancel()
raise PlaybackError( raise PlaybackError(
f"pw-play timed out after {timeout:.0f}s " f"playback timed out after {timeout:.0f}s "
f"(expected {expected_seconds:.1f}s audio)" f"(expected {expected_seconds:.1f}s audio)"
) )
except asyncio.CancelledError: except asyncio.CancelledError:
# Consumer was cancelled (shutdown or cancel) — don't orphan the subprocess # Consumer was cancelled (shutdown or cancel) — don't orphan the subprocess
proc.kill() proc.kill()
await proc.wait() await proc.wait()
vol_task.cancel() if vol_task:
vol_task.cancel()
raise raise
if not vol_task.done(): if vol_task and not vol_task.done():
vol_task.cancel() vol_task.cancel()
if proc.returncode != 0: if proc.returncode != 0:
raise PlaybackError( raise PlaybackError(
f"pw-play failed ({proc.returncode}): {stderr.decode().strip()}" f"{platform_audio.play_command(path)[0]} failed ({proc.returncode}): "
f"{stderr.decode(errors='replace').strip()}"
) )

View File

@ -0,0 +1,134 @@
"""Platform audio backend — the four OS-specific subprocess touch-points.
The rest of mcspeak reaches the speaker/mic only through here, so porting to a
new OS means adding a branch, not touching the queue/VAD/tone/secretary logic.
Every backend's recorder emits the SAME wire format (raw s16, mono, 16 kHz, to
stdout), so the VAD reader in audio.py is identical on every platform.
- Linux : PipeWire CLIs `pw-play`, `pw-record` (the proven Docker path).
- macOS : built-ins only `afplay` (play), a compiled Swift/AVFoundation
recorder (`_macos/mcspeak_record.swift`), `afconvert` (transcode),
`osascript` (system-volume duck).
- Windows: PLANNED `winsound.PlaySound` (stdlib WAV play), `sounddevice`
(PortAudio) or `ffmpeg -f dshow` (capture), `pycaw` (per-app duck).
"""
from __future__ import annotations
import hashlib
import shutil
import subprocess
import sys
from pathlib import Path
IS_LINUX = sys.platform.startswith("linux")
IS_MACOS = sys.platform == "darwin"
IS_WINDOWS = sys.platform.startswith("win")
_MACOS_DIR = Path(__file__).parent / "_macos"
_RECORDER_SRC = _MACOS_DIR / "mcspeak_record.swift"
# Compiled recorder lives in a user cache dir (the install dir may be read-only).
_CACHE_DIR = Path.home() / ".cache" / "mcspeak"
class AudioBackendError(RuntimeError):
"""Raised when the platform backend can't satisfy a request."""
def platform_name() -> str:
if IS_MACOS:
return "macos"
if IS_WINDOWS:
return "windows"
if IS_LINUX:
return "linux"
return sys.platform
# ---------------------------------------------------------------------------
# Playback
# ---------------------------------------------------------------------------
def play_command(path: Path | str) -> list[str]:
"""Command to play a WAV to the default output device, blocking until done."""
if IS_MACOS:
return ["afplay", str(path)]
if IS_WINDOWS:
# Windows plays via winsound in-process (see play_audio), not a subprocess.
raise AudioBackendError("Windows playback is in-process; don't call play_command")
return ["pw-play", str(path)] # Linux / PipeWire
def volume_workaround_applies() -> bool:
"""Only PulseAudio/PipeWire needs the stream-restore 100% volume override."""
return IS_LINUX
# ---------------------------------------------------------------------------
# Recording — every backend streams raw s16 mono PCM at `sample_rate` to stdout
# ---------------------------------------------------------------------------
def record_stream_command(
sample_rate: int = 16000, channels: int = 1, source: str | None = None
) -> list[str]:
"""Command that streams raw s16 PCM to stdout until it receives SIGTERM."""
if IS_MACOS:
# The Swift recorder is fixed at 16 kHz mono (Parakeet's format); it
# ignores sample_rate/source for now (default input device only).
return [str(ensure_macos_recorder())]
if IS_WINDOWS:
raise AudioBackendError(
"Windows capture not implemented yet — planned via sounddevice/ffmpeg"
)
cmd = ["pw-record", "--rate", str(sample_rate), "--channels", str(channels), "--format", "s16"]
if source:
cmd.extend(["--target", source])
cmd.append("-") # stdout
return cmd
def ensure_macos_recorder() -> Path:
"""Compile the Swift recorder once (cached in ~/.cache/mcspeak), return its path.
Recompiles only when the source changes (tracked by a content-hash stamp).
"""
if not IS_MACOS:
raise AudioBackendError("macOS recorder requested on a non-macOS host")
swiftc = shutil.which("swiftc")
if not swiftc:
raise AudioBackendError(
"swiftc not found — install the Xcode command-line tools (xcode-select --install)"
)
_CACHE_DIR.mkdir(parents=True, exist_ok=True)
binary = _CACHE_DIR / "mcspeak-record"
src_hash = hashlib.sha256(_RECORDER_SRC.read_bytes()).hexdigest()[:12]
stamp = _CACHE_DIR / f".record-{src_hash}"
if binary.exists() and stamp.exists():
return binary
proc = subprocess.run(
[swiftc, "-O", str(_RECORDER_SRC), "-o", str(binary)],
capture_output=True, text=True,
)
if proc.returncode != 0:
raise AudioBackendError(
f"swiftc failed compiling the recorder: {proc.stderr.strip()[:500]}"
)
# Clear any stale stamps, then mark this build good.
for old in _CACHE_DIR.glob(".record-*"):
old.unlink(missing_ok=True)
stamp.write_text("ok")
return binary
# ---------------------------------------------------------------------------
# Transcode / resample (used by convert_audio and any 16k-mono normalization)
# ---------------------------------------------------------------------------
def has_ffmpeg() -> bool:
return shutil.which("ffmpeg") is not None
def prefers_afconvert() -> bool:
"""On macOS without ffmpeg, use the built-in afconvert."""
return IS_MACOS and not has_ffmpeg()