macOS audio backend groundwork (capture proven, deploy TBD)

Introduce a platform audio backend so the four OS-specific subprocess
touch-points (play, record-stream, transcode, duck) are swappable without
touching the queue/VAD/tone logic. The macOS recorder emits the same
s16/mono/16k PCM-to-stdout wire format as `pw-record -`, so the VAD frame
loop is byte-for-byte identical across platforms.

- platform_audio.py: backend selector (Linux + macOS implemented, Windows
  documented). Compiles the Swift recorder once, cached in ~/.cache/mcspeak.
- _macos/mcspeak_record.swift: AVFoundation recorder, built-ins only (no brew).
  Verified capturing on the target Mac over SSH — TCC does not block headless
  capture.
- audio.py: play_audio uses afplay on macOS (pw-play + volume workaround stay
  Linux-only); record_audio_until_silence streams via the backend command;
  record_audio gains a stream-and-collect path for stdout-only backends.

Linux behavior is unchanged (commands verified, queue tests green, ruff clean).

Not yet done: afconvert transcode branch (generate_audio non-WAV only),
osascript ducking (currently a graceful no-op), and running the server
natively on the Mac + a live speech test.
This commit is contained in:
Ryan Malloy 2026-07-03 22:22:58 -06:00
parent f13278da64
commit 2711e90dce
3 changed files with 292 additions and 20 deletions

View File

@ -0,0 +1,64 @@
// mcspeak macOS mic recorder streams 16 kHz mono s16 PCM to stdout until
// SIGTERM, mirroring `pw-record --rate 16000 --channels 1 --format s16 -`.
// The VAD loop in audio.py reads this stream frame-by-frame, unchanged, so the
// rest of mcspeak is oblivious to which OS captured the audio.
//
// Built once by platform_audio.ensure_macos_recorder() via `swiftc -O`.
// Uses only the AVFoundation framework that ships with macOS no dependencies.
import AVFoundation
import Foundation
let targetRate = 16000.0
let engine = AVAudioEngine()
let input = engine.inputNode
let inputFormat = input.outputFormat(forBus: 0)
FileHandle.standardError.write(
"mcspeak-record: input \(inputFormat.sampleRate)Hz \(inputFormat.channelCount)ch -> 16000Hz mono s16\n"
.data(using: .utf8)!)
guard let targetFormat = AVAudioFormat(
commonFormat: .pcmFormatInt16, sampleRate: targetRate,
channels: 1, interleaved: true
) else {
FileHandle.standardError.write("mcspeak-record: bad target format\n".data(using: .utf8)!)
exit(2)
}
guard let converter = AVAudioConverter(from: inputFormat, to: targetFormat) else {
FileHandle.standardError.write("mcspeak-record: no converter\n".data(using: .utf8)!)
exit(2)
}
let out = FileHandle.standardOutput
input.installTap(onBus: 0, bufferSize: 2048, format: inputFormat) { buffer, _ in
// Resample + downmix each hardware buffer to 16k mono s16 and write raw
// little-endian samples straight to stdout.
let cap = AVAudioFrameCount(Double(buffer.frameLength) * targetRate / inputFormat.sampleRate) + 32
guard let outBuf = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: cap) else { return }
var fed = false
var err: NSError?
let status = converter.convert(to: outBuf, error: &err) { _, outStatus in
if fed { outStatus.pointee = .noDataNow; return nil }
fed = true
outStatus.pointee = .haveData
return buffer
}
if status == .error { return }
if let ch = outBuf.int16ChannelData, outBuf.frameLength > 0 {
out.write(Data(bytes: ch[0], count: Int(outBuf.frameLength) * 2))
}
}
// Terminate cleanly on SIGTERM/SIGINT so the WAV/stream closes like pw-record.
signal(SIGTERM) { _ in exit(0) }
signal(SIGINT) { _ in exit(0) }
do {
try engine.start()
} catch {
FileHandle.standardError.write("mcspeak-record: engine start failed: \(error)\n".data(using: .utf8)!)
exit(3)
}
RunLoop.current.run()

View File

@ -10,6 +10,7 @@ from pathlib import Path
import numpy as np
from . import platform_audio
from .settings import settings
_counter = itertools.count(1)
@ -228,15 +229,10 @@ async def record_audio_until_silence(
vad = webrtcvad.Vad(aggressiveness)
cmd = [
"pw-record",
"--rate", str(SAMPLE_RATE),
"--channels", "1",
"--format", "s16",
]
if source:
cmd.extend(["--target", source])
cmd.append("-") # stdout
# Platform-appropriate recorder — always streams s16 mono PCM to stdout, so
# the VAD frame loop below is identical on Linux (pw-record) and macOS (the
# Swift/AVFoundation recorder).
cmd = platform_audio.record_stream_command(SAMPLE_RATE, 1, source)
proc = await asyncio.create_subprocess_exec(
*cmd,
@ -346,9 +342,17 @@ async def record_audio(
kill it. pw-record handles SIGTERM by closing the WAV header cleanly
verified by recording 2s and reading the file back with the wave module.
Returns out_path on success. Raises PlaybackError on pw-record failure or
Returns out_path on success. Raises PlaybackError on recorder failure or
if the resulting WAV is empty (mic disconnected, etc).
On macOS the Swift recorder only streams to stdout, so the fixed-duration
path collects PCM for `duration_seconds` and writes the WAV itself.
"""
if platform_audio.IS_MACOS:
return await _record_fixed_stream(
out_path, duration_seconds, sample_rate, source, ready_tone, warmup_ms
)
cmd = [
"pw-record",
"--rate", str(sample_rate),
@ -407,8 +411,71 @@ async def record_audio(
return out_path
async def _record_fixed_stream(
out_path: Path,
duration_seconds: float,
sample_rate: int,
source: str | None,
ready_tone: Path | None,
warmup_ms: int,
) -> Path:
"""Fixed-duration capture for stdout-streaming backends (macOS).
Runs the streaming recorder, plays the ready tone once the mic is live,
collects s16 PCM for `duration_seconds`, then writes the WAV. Mirrors
record_audio's contract (returns out_path; raises PlaybackError on no audio).
"""
cmd = platform_audio.record_stream_command(sample_rate, 1, source)
proc = await asyncio.create_subprocess_exec(
*cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.DEVNULL,
)
if ready_tone is not None:
await asyncio.sleep(warmup_ms / 1000)
try:
await play_audio(ready_tone, expected_seconds=0.3)
except Exception:
pass # tone failure is non-fatal — keep recording
pcm = bytearray()
async def _drain() -> None:
while True:
chunk = await proc.stdout.read(8192)
if not chunk:
break
pcm.extend(chunk)
try:
await asyncio.wait_for(_drain(), timeout=duration_seconds)
except asyncio.TimeoutError:
proc.terminate()
try:
await asyncio.wait_for(proc.wait(), timeout=2.0)
except asyncio.TimeoutError:
proc.kill()
await proc.wait()
except asyncio.CancelledError:
proc.kill()
await proc.wait()
raise
if len(pcm) < 100:
raise PlaybackError(
"macOS recorder produced no audio. Check that the launching process "
"has Microphone permission (System Settings > Privacy > Microphone)."
)
write_wav_from_pcm(
bytes(pcm), sample_rate=sample_rate, sample_width=2, channels=1, path=out_path
)
return out_path
async def play_audio(path: Path, expected_seconds: float = 0) -> None:
"""Play a WAV file through PipeWire (pw-play). Async wrapper.
"""Play a WAV file to the default output device (pw-play on Linux, afplay on
macOS). Async wrapper.
Args:
path: Path to the WAV file.
@ -417,35 +484,42 @@ async def play_audio(path: Path, expected_seconds: float = 0) -> None:
"""
if expected_seconds <= 0:
expected_seconds = wav_duration(path)
# Generous margin: 2x duration + 10s for PipeWire startup overhead
# Generous margin: 2x duration + 10s for player startup overhead
timeout = max(expected_seconds * 2, 10) + 10
proc = await asyncio.create_subprocess_exec(
"pw-play", str(path),
*platform_audio.play_command(path),
stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.PIPE,
)
# Defensive volume override — guards against module-stream-restore drift.
vol_task = asyncio.create_task(_force_pwplay_volume_100())
# Defensive volume override — guards against PulseAudio module-stream-restore
# drift. Linux/PipeWire only; other backends don't need it.
vol_task = (
asyncio.create_task(_force_pwplay_volume_100())
if platform_audio.volume_workaround_applies() else None
)
try:
_, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout)
except asyncio.TimeoutError:
proc.kill()
await proc.wait()
vol_task.cancel()
if vol_task:
vol_task.cancel()
raise PlaybackError(
f"pw-play timed out after {timeout:.0f}s "
f"playback timed out after {timeout:.0f}s "
f"(expected {expected_seconds:.1f}s audio)"
)
except asyncio.CancelledError:
# Consumer was cancelled (shutdown or cancel) — don't orphan the subprocess
proc.kill()
await proc.wait()
vol_task.cancel()
if vol_task:
vol_task.cancel()
raise
if not vol_task.done():
if vol_task and not vol_task.done():
vol_task.cancel()
if proc.returncode != 0:
raise PlaybackError(
f"pw-play failed ({proc.returncode}): {stderr.decode().strip()}"
f"{platform_audio.play_command(path)[0]} failed ({proc.returncode}): "
f"{stderr.decode(errors='replace').strip()}"
)

View File

@ -0,0 +1,134 @@
"""Platform audio backend — the four OS-specific subprocess touch-points.
The rest of mcspeak reaches the speaker/mic only through here, so porting to a
new OS means adding a branch, not touching the queue/VAD/tone/secretary logic.
Every backend's recorder emits the SAME wire format (raw s16, mono, 16 kHz, to
stdout), so the VAD reader in audio.py is identical on every platform.
- Linux : PipeWire CLIs `pw-play`, `pw-record` (the proven Docker path).
- macOS : built-ins only `afplay` (play), a compiled Swift/AVFoundation
recorder (`_macos/mcspeak_record.swift`), `afconvert` (transcode),
`osascript` (system-volume duck).
- Windows: PLANNED `winsound.PlaySound` (stdlib WAV play), `sounddevice`
(PortAudio) or `ffmpeg -f dshow` (capture), `pycaw` (per-app duck).
"""
from __future__ import annotations
import hashlib
import shutil
import subprocess
import sys
from pathlib import Path
IS_LINUX = sys.platform.startswith("linux")
IS_MACOS = sys.platform == "darwin"
IS_WINDOWS = sys.platform.startswith("win")
_MACOS_DIR = Path(__file__).parent / "_macos"
_RECORDER_SRC = _MACOS_DIR / "mcspeak_record.swift"
# Compiled recorder lives in a user cache dir (the install dir may be read-only).
_CACHE_DIR = Path.home() / ".cache" / "mcspeak"
class AudioBackendError(RuntimeError):
"""Raised when the platform backend can't satisfy a request."""
def platform_name() -> str:
if IS_MACOS:
return "macos"
if IS_WINDOWS:
return "windows"
if IS_LINUX:
return "linux"
return sys.platform
# ---------------------------------------------------------------------------
# Playback
# ---------------------------------------------------------------------------
def play_command(path: Path | str) -> list[str]:
"""Command to play a WAV to the default output device, blocking until done."""
if IS_MACOS:
return ["afplay", str(path)]
if IS_WINDOWS:
# Windows plays via winsound in-process (see play_audio), not a subprocess.
raise AudioBackendError("Windows playback is in-process; don't call play_command")
return ["pw-play", str(path)] # Linux / PipeWire
def volume_workaround_applies() -> bool:
"""Only PulseAudio/PipeWire needs the stream-restore 100% volume override."""
return IS_LINUX
# ---------------------------------------------------------------------------
# Recording — every backend streams raw s16 mono PCM at `sample_rate` to stdout
# ---------------------------------------------------------------------------
def record_stream_command(
sample_rate: int = 16000, channels: int = 1, source: str | None = None
) -> list[str]:
"""Command that streams raw s16 PCM to stdout until it receives SIGTERM."""
if IS_MACOS:
# The Swift recorder is fixed at 16 kHz mono (Parakeet's format); it
# ignores sample_rate/source for now (default input device only).
return [str(ensure_macos_recorder())]
if IS_WINDOWS:
raise AudioBackendError(
"Windows capture not implemented yet — planned via sounddevice/ffmpeg"
)
cmd = ["pw-record", "--rate", str(sample_rate), "--channels", str(channels), "--format", "s16"]
if source:
cmd.extend(["--target", source])
cmd.append("-") # stdout
return cmd
def ensure_macos_recorder() -> Path:
"""Compile the Swift recorder once (cached in ~/.cache/mcspeak), return its path.
Recompiles only when the source changes (tracked by a content-hash stamp).
"""
if not IS_MACOS:
raise AudioBackendError("macOS recorder requested on a non-macOS host")
swiftc = shutil.which("swiftc")
if not swiftc:
raise AudioBackendError(
"swiftc not found — install the Xcode command-line tools (xcode-select --install)"
)
_CACHE_DIR.mkdir(parents=True, exist_ok=True)
binary = _CACHE_DIR / "mcspeak-record"
src_hash = hashlib.sha256(_RECORDER_SRC.read_bytes()).hexdigest()[:12]
stamp = _CACHE_DIR / f".record-{src_hash}"
if binary.exists() and stamp.exists():
return binary
proc = subprocess.run(
[swiftc, "-O", str(_RECORDER_SRC), "-o", str(binary)],
capture_output=True, text=True,
)
if proc.returncode != 0:
raise AudioBackendError(
f"swiftc failed compiling the recorder: {proc.stderr.strip()[:500]}"
)
# Clear any stale stamps, then mark this build good.
for old in _CACHE_DIR.glob(".record-*"):
old.unlink(missing_ok=True)
stamp.write_text("ok")
return binary
# ---------------------------------------------------------------------------
# Transcode / resample (used by convert_audio and any 16k-mono normalization)
# ---------------------------------------------------------------------------
def has_ffmpeg() -> bool:
return shutil.which("ffmpeg") is not None
def prefers_afconvert() -> bool:
"""On macOS without ffmpeg, use the built-in afconvert."""
return IS_MACOS and not has_ffmpeg()