macOS audio backend groundwork (capture proven, deploy TBD)
Introduce a platform audio backend so the four OS-specific subprocess touch-points (play, record-stream, transcode, duck) are swappable without touching the queue/VAD/tone logic. The macOS recorder emits the same s16/mono/16k PCM-to-stdout wire format as `pw-record -`, so the VAD frame loop is byte-for-byte identical across platforms. - platform_audio.py: backend selector (Linux + macOS implemented, Windows documented). Compiles the Swift recorder once, cached in ~/.cache/mcspeak. - _macos/mcspeak_record.swift: AVFoundation recorder, built-ins only (no brew). Verified capturing on the target Mac over SSH — TCC does not block headless capture. - audio.py: play_audio uses afplay on macOS (pw-play + volume workaround stay Linux-only); record_audio_until_silence streams via the backend command; record_audio gains a stream-and-collect path for stdout-only backends. Linux behavior is unchanged (commands verified, queue tests green, ruff clean). Not yet done: afconvert transcode branch (generate_audio non-WAV only), osascript ducking (currently a graceful no-op), and running the server natively on the Mac + a live speech test.
This commit is contained in:
parent
f13278da64
commit
2711e90dce
64
src/mcspeak/_macos/mcspeak_record.swift
Normal file
64
src/mcspeak/_macos/mcspeak_record.swift
Normal file
@ -0,0 +1,64 @@
|
|||||||
|
// mcspeak macOS mic recorder — streams 16 kHz mono s16 PCM to stdout until
|
||||||
|
// SIGTERM, mirroring `pw-record --rate 16000 --channels 1 --format s16 -`.
|
||||||
|
// The VAD loop in audio.py reads this stream frame-by-frame, unchanged, so the
|
||||||
|
// rest of mcspeak is oblivious to which OS captured the audio.
|
||||||
|
//
|
||||||
|
// Built once by platform_audio.ensure_macos_recorder() via `swiftc -O`.
|
||||||
|
// Uses only the AVFoundation framework that ships with macOS — no dependencies.
|
||||||
|
import AVFoundation
|
||||||
|
import Foundation
|
||||||
|
|
||||||
|
let targetRate = 16000.0
|
||||||
|
|
||||||
|
let engine = AVAudioEngine()
|
||||||
|
let input = engine.inputNode
|
||||||
|
let inputFormat = input.outputFormat(forBus: 0)
|
||||||
|
FileHandle.standardError.write(
|
||||||
|
"mcspeak-record: input \(inputFormat.sampleRate)Hz \(inputFormat.channelCount)ch -> 16000Hz mono s16\n"
|
||||||
|
.data(using: .utf8)!)
|
||||||
|
|
||||||
|
guard let targetFormat = AVAudioFormat(
|
||||||
|
commonFormat: .pcmFormatInt16, sampleRate: targetRate,
|
||||||
|
channels: 1, interleaved: true
|
||||||
|
) else {
|
||||||
|
FileHandle.standardError.write("mcspeak-record: bad target format\n".data(using: .utf8)!)
|
||||||
|
exit(2)
|
||||||
|
}
|
||||||
|
guard let converter = AVAudioConverter(from: inputFormat, to: targetFormat) else {
|
||||||
|
FileHandle.standardError.write("mcspeak-record: no converter\n".data(using: .utf8)!)
|
||||||
|
exit(2)
|
||||||
|
}
|
||||||
|
|
||||||
|
let out = FileHandle.standardOutput
|
||||||
|
|
||||||
|
input.installTap(onBus: 0, bufferSize: 2048, format: inputFormat) { buffer, _ in
|
||||||
|
// Resample + downmix each hardware buffer to 16k mono s16 and write raw
|
||||||
|
// little-endian samples straight to stdout.
|
||||||
|
let cap = AVAudioFrameCount(Double(buffer.frameLength) * targetRate / inputFormat.sampleRate) + 32
|
||||||
|
guard let outBuf = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: cap) else { return }
|
||||||
|
var fed = false
|
||||||
|
var err: NSError?
|
||||||
|
let status = converter.convert(to: outBuf, error: &err) { _, outStatus in
|
||||||
|
if fed { outStatus.pointee = .noDataNow; return nil }
|
||||||
|
fed = true
|
||||||
|
outStatus.pointee = .haveData
|
||||||
|
return buffer
|
||||||
|
}
|
||||||
|
if status == .error { return }
|
||||||
|
if let ch = outBuf.int16ChannelData, outBuf.frameLength > 0 {
|
||||||
|
out.write(Data(bytes: ch[0], count: Int(outBuf.frameLength) * 2))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Terminate cleanly on SIGTERM/SIGINT so the WAV/stream closes like pw-record.
|
||||||
|
signal(SIGTERM) { _ in exit(0) }
|
||||||
|
signal(SIGINT) { _ in exit(0) }
|
||||||
|
|
||||||
|
do {
|
||||||
|
try engine.start()
|
||||||
|
} catch {
|
||||||
|
FileHandle.standardError.write("mcspeak-record: engine start failed: \(error)\n".data(using: .utf8)!)
|
||||||
|
exit(3)
|
||||||
|
}
|
||||||
|
|
||||||
|
RunLoop.current.run()
|
||||||
@ -10,6 +10,7 @@ from pathlib import Path
|
|||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
|
|
||||||
|
from . import platform_audio
|
||||||
from .settings import settings
|
from .settings import settings
|
||||||
|
|
||||||
_counter = itertools.count(1)
|
_counter = itertools.count(1)
|
||||||
@ -228,15 +229,10 @@ async def record_audio_until_silence(
|
|||||||
|
|
||||||
vad = webrtcvad.Vad(aggressiveness)
|
vad = webrtcvad.Vad(aggressiveness)
|
||||||
|
|
||||||
cmd = [
|
# Platform-appropriate recorder — always streams s16 mono PCM to stdout, so
|
||||||
"pw-record",
|
# the VAD frame loop below is identical on Linux (pw-record) and macOS (the
|
||||||
"--rate", str(SAMPLE_RATE),
|
# Swift/AVFoundation recorder).
|
||||||
"--channels", "1",
|
cmd = platform_audio.record_stream_command(SAMPLE_RATE, 1, source)
|
||||||
"--format", "s16",
|
|
||||||
]
|
|
||||||
if source:
|
|
||||||
cmd.extend(["--target", source])
|
|
||||||
cmd.append("-") # stdout
|
|
||||||
|
|
||||||
proc = await asyncio.create_subprocess_exec(
|
proc = await asyncio.create_subprocess_exec(
|
||||||
*cmd,
|
*cmd,
|
||||||
@ -346,9 +342,17 @@ async def record_audio(
|
|||||||
kill it. pw-record handles SIGTERM by closing the WAV header cleanly —
|
kill it. pw-record handles SIGTERM by closing the WAV header cleanly —
|
||||||
verified by recording 2s and reading the file back with the wave module.
|
verified by recording 2s and reading the file back with the wave module.
|
||||||
|
|
||||||
Returns out_path on success. Raises PlaybackError on pw-record failure or
|
Returns out_path on success. Raises PlaybackError on recorder failure or
|
||||||
if the resulting WAV is empty (mic disconnected, etc).
|
if the resulting WAV is empty (mic disconnected, etc).
|
||||||
|
|
||||||
|
On macOS the Swift recorder only streams to stdout, so the fixed-duration
|
||||||
|
path collects PCM for `duration_seconds` and writes the WAV itself.
|
||||||
"""
|
"""
|
||||||
|
if platform_audio.IS_MACOS:
|
||||||
|
return await _record_fixed_stream(
|
||||||
|
out_path, duration_seconds, sample_rate, source, ready_tone, warmup_ms
|
||||||
|
)
|
||||||
|
|
||||||
cmd = [
|
cmd = [
|
||||||
"pw-record",
|
"pw-record",
|
||||||
"--rate", str(sample_rate),
|
"--rate", str(sample_rate),
|
||||||
@ -407,8 +411,71 @@ async def record_audio(
|
|||||||
return out_path
|
return out_path
|
||||||
|
|
||||||
|
|
||||||
|
async def _record_fixed_stream(
|
||||||
|
out_path: Path,
|
||||||
|
duration_seconds: float,
|
||||||
|
sample_rate: int,
|
||||||
|
source: str | None,
|
||||||
|
ready_tone: Path | None,
|
||||||
|
warmup_ms: int,
|
||||||
|
) -> Path:
|
||||||
|
"""Fixed-duration capture for stdout-streaming backends (macOS).
|
||||||
|
|
||||||
|
Runs the streaming recorder, plays the ready tone once the mic is live,
|
||||||
|
collects s16 PCM for `duration_seconds`, then writes the WAV. Mirrors
|
||||||
|
record_audio's contract (returns out_path; raises PlaybackError on no audio).
|
||||||
|
"""
|
||||||
|
cmd = platform_audio.record_stream_command(sample_rate, 1, source)
|
||||||
|
proc = await asyncio.create_subprocess_exec(
|
||||||
|
*cmd,
|
||||||
|
stdout=asyncio.subprocess.PIPE,
|
||||||
|
stderr=asyncio.subprocess.DEVNULL,
|
||||||
|
)
|
||||||
|
|
||||||
|
if ready_tone is not None:
|
||||||
|
await asyncio.sleep(warmup_ms / 1000)
|
||||||
|
try:
|
||||||
|
await play_audio(ready_tone, expected_seconds=0.3)
|
||||||
|
except Exception:
|
||||||
|
pass # tone failure is non-fatal — keep recording
|
||||||
|
|
||||||
|
pcm = bytearray()
|
||||||
|
|
||||||
|
async def _drain() -> None:
|
||||||
|
while True:
|
||||||
|
chunk = await proc.stdout.read(8192)
|
||||||
|
if not chunk:
|
||||||
|
break
|
||||||
|
pcm.extend(chunk)
|
||||||
|
|
||||||
|
try:
|
||||||
|
await asyncio.wait_for(_drain(), timeout=duration_seconds)
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
proc.terminate()
|
||||||
|
try:
|
||||||
|
await asyncio.wait_for(proc.wait(), timeout=2.0)
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
proc.kill()
|
||||||
|
await proc.wait()
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
proc.kill()
|
||||||
|
await proc.wait()
|
||||||
|
raise
|
||||||
|
|
||||||
|
if len(pcm) < 100:
|
||||||
|
raise PlaybackError(
|
||||||
|
"macOS recorder produced no audio. Check that the launching process "
|
||||||
|
"has Microphone permission (System Settings > Privacy > Microphone)."
|
||||||
|
)
|
||||||
|
write_wav_from_pcm(
|
||||||
|
bytes(pcm), sample_rate=sample_rate, sample_width=2, channels=1, path=out_path
|
||||||
|
)
|
||||||
|
return out_path
|
||||||
|
|
||||||
|
|
||||||
async def play_audio(path: Path, expected_seconds: float = 0) -> None:
|
async def play_audio(path: Path, expected_seconds: float = 0) -> None:
|
||||||
"""Play a WAV file through PipeWire (pw-play). Async wrapper.
|
"""Play a WAV file to the default output device (pw-play on Linux, afplay on
|
||||||
|
macOS). Async wrapper.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
path: Path to the WAV file.
|
path: Path to the WAV file.
|
||||||
@ -417,35 +484,42 @@ async def play_audio(path: Path, expected_seconds: float = 0) -> None:
|
|||||||
"""
|
"""
|
||||||
if expected_seconds <= 0:
|
if expected_seconds <= 0:
|
||||||
expected_seconds = wav_duration(path)
|
expected_seconds = wav_duration(path)
|
||||||
# Generous margin: 2x duration + 10s for PipeWire startup overhead
|
# Generous margin: 2x duration + 10s for player startup overhead
|
||||||
timeout = max(expected_seconds * 2, 10) + 10
|
timeout = max(expected_seconds * 2, 10) + 10
|
||||||
|
|
||||||
proc = await asyncio.create_subprocess_exec(
|
proc = await asyncio.create_subprocess_exec(
|
||||||
"pw-play", str(path),
|
*platform_audio.play_command(path),
|
||||||
stdout=asyncio.subprocess.DEVNULL,
|
stdout=asyncio.subprocess.DEVNULL,
|
||||||
stderr=asyncio.subprocess.PIPE,
|
stderr=asyncio.subprocess.PIPE,
|
||||||
)
|
)
|
||||||
# Defensive volume override — guards against module-stream-restore drift.
|
# Defensive volume override — guards against PulseAudio module-stream-restore
|
||||||
vol_task = asyncio.create_task(_force_pwplay_volume_100())
|
# drift. Linux/PipeWire only; other backends don't need it.
|
||||||
|
vol_task = (
|
||||||
|
asyncio.create_task(_force_pwplay_volume_100())
|
||||||
|
if platform_audio.volume_workaround_applies() else None
|
||||||
|
)
|
||||||
try:
|
try:
|
||||||
_, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout)
|
_, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout)
|
||||||
except asyncio.TimeoutError:
|
except asyncio.TimeoutError:
|
||||||
proc.kill()
|
proc.kill()
|
||||||
await proc.wait()
|
await proc.wait()
|
||||||
vol_task.cancel()
|
if vol_task:
|
||||||
|
vol_task.cancel()
|
||||||
raise PlaybackError(
|
raise PlaybackError(
|
||||||
f"pw-play timed out after {timeout:.0f}s "
|
f"playback timed out after {timeout:.0f}s "
|
||||||
f"(expected {expected_seconds:.1f}s audio)"
|
f"(expected {expected_seconds:.1f}s audio)"
|
||||||
)
|
)
|
||||||
except asyncio.CancelledError:
|
except asyncio.CancelledError:
|
||||||
# Consumer was cancelled (shutdown or cancel) — don't orphan the subprocess
|
# Consumer was cancelled (shutdown or cancel) — don't orphan the subprocess
|
||||||
proc.kill()
|
proc.kill()
|
||||||
await proc.wait()
|
await proc.wait()
|
||||||
vol_task.cancel()
|
if vol_task:
|
||||||
|
vol_task.cancel()
|
||||||
raise
|
raise
|
||||||
if not vol_task.done():
|
if vol_task and not vol_task.done():
|
||||||
vol_task.cancel()
|
vol_task.cancel()
|
||||||
if proc.returncode != 0:
|
if proc.returncode != 0:
|
||||||
raise PlaybackError(
|
raise PlaybackError(
|
||||||
f"pw-play failed ({proc.returncode}): {stderr.decode().strip()}"
|
f"{platform_audio.play_command(path)[0]} failed ({proc.returncode}): "
|
||||||
|
f"{stderr.decode(errors='replace').strip()}"
|
||||||
)
|
)
|
||||||
|
|||||||
134
src/mcspeak/platform_audio.py
Normal file
134
src/mcspeak/platform_audio.py
Normal file
@ -0,0 +1,134 @@
|
|||||||
|
"""Platform audio backend — the four OS-specific subprocess touch-points.
|
||||||
|
|
||||||
|
The rest of mcspeak reaches the speaker/mic only through here, so porting to a
|
||||||
|
new OS means adding a branch, not touching the queue/VAD/tone/secretary logic.
|
||||||
|
Every backend's recorder emits the SAME wire format (raw s16, mono, 16 kHz, to
|
||||||
|
stdout), so the VAD reader in audio.py is identical on every platform.
|
||||||
|
|
||||||
|
- Linux : PipeWire CLIs — `pw-play`, `pw-record` (the proven Docker path).
|
||||||
|
- macOS : built-ins only — `afplay` (play), a compiled Swift/AVFoundation
|
||||||
|
recorder (`_macos/mcspeak_record.swift`), `afconvert` (transcode),
|
||||||
|
`osascript` (system-volume duck).
|
||||||
|
- Windows: PLANNED — `winsound.PlaySound` (stdlib WAV play), `sounddevice`
|
||||||
|
(PortAudio) or `ffmpeg -f dshow` (capture), `pycaw` (per-app duck).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
IS_LINUX = sys.platform.startswith("linux")
|
||||||
|
IS_MACOS = sys.platform == "darwin"
|
||||||
|
IS_WINDOWS = sys.platform.startswith("win")
|
||||||
|
|
||||||
|
_MACOS_DIR = Path(__file__).parent / "_macos"
|
||||||
|
_RECORDER_SRC = _MACOS_DIR / "mcspeak_record.swift"
|
||||||
|
# Compiled recorder lives in a user cache dir (the install dir may be read-only).
|
||||||
|
_CACHE_DIR = Path.home() / ".cache" / "mcspeak"
|
||||||
|
|
||||||
|
|
||||||
|
class AudioBackendError(RuntimeError):
|
||||||
|
"""Raised when the platform backend can't satisfy a request."""
|
||||||
|
|
||||||
|
|
||||||
|
def platform_name() -> str:
|
||||||
|
if IS_MACOS:
|
||||||
|
return "macos"
|
||||||
|
if IS_WINDOWS:
|
||||||
|
return "windows"
|
||||||
|
if IS_LINUX:
|
||||||
|
return "linux"
|
||||||
|
return sys.platform
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Playback
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def play_command(path: Path | str) -> list[str]:
|
||||||
|
"""Command to play a WAV to the default output device, blocking until done."""
|
||||||
|
if IS_MACOS:
|
||||||
|
return ["afplay", str(path)]
|
||||||
|
if IS_WINDOWS:
|
||||||
|
# Windows plays via winsound in-process (see play_audio), not a subprocess.
|
||||||
|
raise AudioBackendError("Windows playback is in-process; don't call play_command")
|
||||||
|
return ["pw-play", str(path)] # Linux / PipeWire
|
||||||
|
|
||||||
|
|
||||||
|
def volume_workaround_applies() -> bool:
|
||||||
|
"""Only PulseAudio/PipeWire needs the stream-restore 100% volume override."""
|
||||||
|
return IS_LINUX
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Recording — every backend streams raw s16 mono PCM at `sample_rate` to stdout
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def record_stream_command(
|
||||||
|
sample_rate: int = 16000, channels: int = 1, source: str | None = None
|
||||||
|
) -> list[str]:
|
||||||
|
"""Command that streams raw s16 PCM to stdout until it receives SIGTERM."""
|
||||||
|
if IS_MACOS:
|
||||||
|
# The Swift recorder is fixed at 16 kHz mono (Parakeet's format); it
|
||||||
|
# ignores sample_rate/source for now (default input device only).
|
||||||
|
return [str(ensure_macos_recorder())]
|
||||||
|
if IS_WINDOWS:
|
||||||
|
raise AudioBackendError(
|
||||||
|
"Windows capture not implemented yet — planned via sounddevice/ffmpeg"
|
||||||
|
)
|
||||||
|
cmd = ["pw-record", "--rate", str(sample_rate), "--channels", str(channels), "--format", "s16"]
|
||||||
|
if source:
|
||||||
|
cmd.extend(["--target", source])
|
||||||
|
cmd.append("-") # stdout
|
||||||
|
return cmd
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_macos_recorder() -> Path:
|
||||||
|
"""Compile the Swift recorder once (cached in ~/.cache/mcspeak), return its path.
|
||||||
|
|
||||||
|
Recompiles only when the source changes (tracked by a content-hash stamp).
|
||||||
|
"""
|
||||||
|
if not IS_MACOS:
|
||||||
|
raise AudioBackendError("macOS recorder requested on a non-macOS host")
|
||||||
|
swiftc = shutil.which("swiftc")
|
||||||
|
if not swiftc:
|
||||||
|
raise AudioBackendError(
|
||||||
|
"swiftc not found — install the Xcode command-line tools (xcode-select --install)"
|
||||||
|
)
|
||||||
|
_CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
|
binary = _CACHE_DIR / "mcspeak-record"
|
||||||
|
src_hash = hashlib.sha256(_RECORDER_SRC.read_bytes()).hexdigest()[:12]
|
||||||
|
stamp = _CACHE_DIR / f".record-{src_hash}"
|
||||||
|
if binary.exists() and stamp.exists():
|
||||||
|
return binary
|
||||||
|
|
||||||
|
proc = subprocess.run(
|
||||||
|
[swiftc, "-O", str(_RECORDER_SRC), "-o", str(binary)],
|
||||||
|
capture_output=True, text=True,
|
||||||
|
)
|
||||||
|
if proc.returncode != 0:
|
||||||
|
raise AudioBackendError(
|
||||||
|
f"swiftc failed compiling the recorder: {proc.stderr.strip()[:500]}"
|
||||||
|
)
|
||||||
|
# Clear any stale stamps, then mark this build good.
|
||||||
|
for old in _CACHE_DIR.glob(".record-*"):
|
||||||
|
old.unlink(missing_ok=True)
|
||||||
|
stamp.write_text("ok")
|
||||||
|
return binary
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Transcode / resample (used by convert_audio and any 16k-mono normalization)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def has_ffmpeg() -> bool:
|
||||||
|
return shutil.which("ffmpeg") is not None
|
||||||
|
|
||||||
|
|
||||||
|
def prefers_afconvert() -> bool:
|
||||||
|
"""On macOS without ffmpeg, use the built-in afconvert."""
|
||||||
|
return IS_MACOS and not has_ffmpeg()
|
||||||
Loading…
x
Reference in New Issue
Block a user