macOS audio backend groundwork (capture proven, deploy TBD)
Introduce a platform audio backend so the four OS-specific subprocess touch-points (play, record-stream, transcode, duck) are swappable without touching the queue/VAD/tone logic. The macOS recorder emits the same s16/mono/16k PCM-to-stdout wire format as `pw-record -`, so the VAD frame loop is byte-for-byte identical across platforms. - platform_audio.py: backend selector (Linux + macOS implemented, Windows documented). Compiles the Swift recorder once, cached in ~/.cache/mcspeak. - _macos/mcspeak_record.swift: AVFoundation recorder, built-ins only (no brew). Verified capturing on the target Mac over SSH — TCC does not block headless capture. - audio.py: play_audio uses afplay on macOS (pw-play + volume workaround stay Linux-only); record_audio_until_silence streams via the backend command; record_audio gains a stream-and-collect path for stdout-only backends. Linux behavior is unchanged (commands verified, queue tests green, ruff clean). Not yet done: afconvert transcode branch (generate_audio non-WAV only), osascript ducking (currently a graceful no-op), and running the server natively on the Mac + a live speech test.
This commit is contained in:
parent
f13278da64
commit
2711e90dce
64
src/mcspeak/_macos/mcspeak_record.swift
Normal file
64
src/mcspeak/_macos/mcspeak_record.swift
Normal file
@ -0,0 +1,64 @@
|
||||
// mcspeak macOS mic recorder — streams 16 kHz mono s16 PCM to stdout until
|
||||
// SIGTERM, mirroring `pw-record --rate 16000 --channels 1 --format s16 -`.
|
||||
// The VAD loop in audio.py reads this stream frame-by-frame, unchanged, so the
|
||||
// rest of mcspeak is oblivious to which OS captured the audio.
|
||||
//
|
||||
// Built once by platform_audio.ensure_macos_recorder() via `swiftc -O`.
|
||||
// Uses only the AVFoundation framework that ships with macOS — no dependencies.
|
||||
import AVFoundation
|
||||
import Foundation
|
||||
|
||||
let targetRate = 16000.0
|
||||
|
||||
let engine = AVAudioEngine()
|
||||
let input = engine.inputNode
|
||||
let inputFormat = input.outputFormat(forBus: 0)
|
||||
FileHandle.standardError.write(
|
||||
"mcspeak-record: input \(inputFormat.sampleRate)Hz \(inputFormat.channelCount)ch -> 16000Hz mono s16\n"
|
||||
.data(using: .utf8)!)
|
||||
|
||||
guard let targetFormat = AVAudioFormat(
|
||||
commonFormat: .pcmFormatInt16, sampleRate: targetRate,
|
||||
channels: 1, interleaved: true
|
||||
) else {
|
||||
FileHandle.standardError.write("mcspeak-record: bad target format\n".data(using: .utf8)!)
|
||||
exit(2)
|
||||
}
|
||||
guard let converter = AVAudioConverter(from: inputFormat, to: targetFormat) else {
|
||||
FileHandle.standardError.write("mcspeak-record: no converter\n".data(using: .utf8)!)
|
||||
exit(2)
|
||||
}
|
||||
|
||||
let out = FileHandle.standardOutput
|
||||
|
||||
input.installTap(onBus: 0, bufferSize: 2048, format: inputFormat) { buffer, _ in
|
||||
// Resample + downmix each hardware buffer to 16k mono s16 and write raw
|
||||
// little-endian samples straight to stdout.
|
||||
let cap = AVAudioFrameCount(Double(buffer.frameLength) * targetRate / inputFormat.sampleRate) + 32
|
||||
guard let outBuf = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: cap) else { return }
|
||||
var fed = false
|
||||
var err: NSError?
|
||||
let status = converter.convert(to: outBuf, error: &err) { _, outStatus in
|
||||
if fed { outStatus.pointee = .noDataNow; return nil }
|
||||
fed = true
|
||||
outStatus.pointee = .haveData
|
||||
return buffer
|
||||
}
|
||||
if status == .error { return }
|
||||
if let ch = outBuf.int16ChannelData, outBuf.frameLength > 0 {
|
||||
out.write(Data(bytes: ch[0], count: Int(outBuf.frameLength) * 2))
|
||||
}
|
||||
}
|
||||
|
||||
// Terminate cleanly on SIGTERM/SIGINT so the WAV/stream closes like pw-record.
|
||||
signal(SIGTERM) { _ in exit(0) }
|
||||
signal(SIGINT) { _ in exit(0) }
|
||||
|
||||
do {
|
||||
try engine.start()
|
||||
} catch {
|
||||
FileHandle.standardError.write("mcspeak-record: engine start failed: \(error)\n".data(using: .utf8)!)
|
||||
exit(3)
|
||||
}
|
||||
|
||||
RunLoop.current.run()
|
||||
@ -10,6 +10,7 @@ from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
from . import platform_audio
|
||||
from .settings import settings
|
||||
|
||||
_counter = itertools.count(1)
|
||||
@ -228,15 +229,10 @@ async def record_audio_until_silence(
|
||||
|
||||
vad = webrtcvad.Vad(aggressiveness)
|
||||
|
||||
cmd = [
|
||||
"pw-record",
|
||||
"--rate", str(SAMPLE_RATE),
|
||||
"--channels", "1",
|
||||
"--format", "s16",
|
||||
]
|
||||
if source:
|
||||
cmd.extend(["--target", source])
|
||||
cmd.append("-") # stdout
|
||||
# Platform-appropriate recorder — always streams s16 mono PCM to stdout, so
|
||||
# the VAD frame loop below is identical on Linux (pw-record) and macOS (the
|
||||
# Swift/AVFoundation recorder).
|
||||
cmd = platform_audio.record_stream_command(SAMPLE_RATE, 1, source)
|
||||
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*cmd,
|
||||
@ -346,9 +342,17 @@ async def record_audio(
|
||||
kill it. pw-record handles SIGTERM by closing the WAV header cleanly —
|
||||
verified by recording 2s and reading the file back with the wave module.
|
||||
|
||||
Returns out_path on success. Raises PlaybackError on pw-record failure or
|
||||
Returns out_path on success. Raises PlaybackError on recorder failure or
|
||||
if the resulting WAV is empty (mic disconnected, etc).
|
||||
|
||||
On macOS the Swift recorder only streams to stdout, so the fixed-duration
|
||||
path collects PCM for `duration_seconds` and writes the WAV itself.
|
||||
"""
|
||||
if platform_audio.IS_MACOS:
|
||||
return await _record_fixed_stream(
|
||||
out_path, duration_seconds, sample_rate, source, ready_tone, warmup_ms
|
||||
)
|
||||
|
||||
cmd = [
|
||||
"pw-record",
|
||||
"--rate", str(sample_rate),
|
||||
@ -407,8 +411,71 @@ async def record_audio(
|
||||
return out_path
|
||||
|
||||
|
||||
async def _record_fixed_stream(
|
||||
out_path: Path,
|
||||
duration_seconds: float,
|
||||
sample_rate: int,
|
||||
source: str | None,
|
||||
ready_tone: Path | None,
|
||||
warmup_ms: int,
|
||||
) -> Path:
|
||||
"""Fixed-duration capture for stdout-streaming backends (macOS).
|
||||
|
||||
Runs the streaming recorder, plays the ready tone once the mic is live,
|
||||
collects s16 PCM for `duration_seconds`, then writes the WAV. Mirrors
|
||||
record_audio's contract (returns out_path; raises PlaybackError on no audio).
|
||||
"""
|
||||
cmd = platform_audio.record_stream_command(sample_rate, 1, source)
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*cmd,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.DEVNULL,
|
||||
)
|
||||
|
||||
if ready_tone is not None:
|
||||
await asyncio.sleep(warmup_ms / 1000)
|
||||
try:
|
||||
await play_audio(ready_tone, expected_seconds=0.3)
|
||||
except Exception:
|
||||
pass # tone failure is non-fatal — keep recording
|
||||
|
||||
pcm = bytearray()
|
||||
|
||||
async def _drain() -> None:
|
||||
while True:
|
||||
chunk = await proc.stdout.read(8192)
|
||||
if not chunk:
|
||||
break
|
||||
pcm.extend(chunk)
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(_drain(), timeout=duration_seconds)
|
||||
except asyncio.TimeoutError:
|
||||
proc.terminate()
|
||||
try:
|
||||
await asyncio.wait_for(proc.wait(), timeout=2.0)
|
||||
except asyncio.TimeoutError:
|
||||
proc.kill()
|
||||
await proc.wait()
|
||||
except asyncio.CancelledError:
|
||||
proc.kill()
|
||||
await proc.wait()
|
||||
raise
|
||||
|
||||
if len(pcm) < 100:
|
||||
raise PlaybackError(
|
||||
"macOS recorder produced no audio. Check that the launching process "
|
||||
"has Microphone permission (System Settings > Privacy > Microphone)."
|
||||
)
|
||||
write_wav_from_pcm(
|
||||
bytes(pcm), sample_rate=sample_rate, sample_width=2, channels=1, path=out_path
|
||||
)
|
||||
return out_path
|
||||
|
||||
|
||||
async def play_audio(path: Path, expected_seconds: float = 0) -> None:
|
||||
"""Play a WAV file through PipeWire (pw-play). Async wrapper.
|
||||
"""Play a WAV file to the default output device (pw-play on Linux, afplay on
|
||||
macOS). Async wrapper.
|
||||
|
||||
Args:
|
||||
path: Path to the WAV file.
|
||||
@ -417,35 +484,42 @@ async def play_audio(path: Path, expected_seconds: float = 0) -> None:
|
||||
"""
|
||||
if expected_seconds <= 0:
|
||||
expected_seconds = wav_duration(path)
|
||||
# Generous margin: 2x duration + 10s for PipeWire startup overhead
|
||||
# Generous margin: 2x duration + 10s for player startup overhead
|
||||
timeout = max(expected_seconds * 2, 10) + 10
|
||||
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
"pw-play", str(path),
|
||||
*platform_audio.play_command(path),
|
||||
stdout=asyncio.subprocess.DEVNULL,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
# Defensive volume override — guards against module-stream-restore drift.
|
||||
vol_task = asyncio.create_task(_force_pwplay_volume_100())
|
||||
# Defensive volume override — guards against PulseAudio module-stream-restore
|
||||
# drift. Linux/PipeWire only; other backends don't need it.
|
||||
vol_task = (
|
||||
asyncio.create_task(_force_pwplay_volume_100())
|
||||
if platform_audio.volume_workaround_applies() else None
|
||||
)
|
||||
try:
|
||||
_, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout)
|
||||
except asyncio.TimeoutError:
|
||||
proc.kill()
|
||||
await proc.wait()
|
||||
vol_task.cancel()
|
||||
if vol_task:
|
||||
vol_task.cancel()
|
||||
raise PlaybackError(
|
||||
f"pw-play timed out after {timeout:.0f}s "
|
||||
f"playback timed out after {timeout:.0f}s "
|
||||
f"(expected {expected_seconds:.1f}s audio)"
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
# Consumer was cancelled (shutdown or cancel) — don't orphan the subprocess
|
||||
proc.kill()
|
||||
await proc.wait()
|
||||
vol_task.cancel()
|
||||
if vol_task:
|
||||
vol_task.cancel()
|
||||
raise
|
||||
if not vol_task.done():
|
||||
if vol_task and not vol_task.done():
|
||||
vol_task.cancel()
|
||||
if proc.returncode != 0:
|
||||
raise PlaybackError(
|
||||
f"pw-play failed ({proc.returncode}): {stderr.decode().strip()}"
|
||||
f"{platform_audio.play_command(path)[0]} failed ({proc.returncode}): "
|
||||
f"{stderr.decode(errors='replace').strip()}"
|
||||
)
|
||||
|
||||
134
src/mcspeak/platform_audio.py
Normal file
134
src/mcspeak/platform_audio.py
Normal file
@ -0,0 +1,134 @@
|
||||
"""Platform audio backend — the four OS-specific subprocess touch-points.
|
||||
|
||||
The rest of mcspeak reaches the speaker/mic only through here, so porting to a
|
||||
new OS means adding a branch, not touching the queue/VAD/tone/secretary logic.
|
||||
Every backend's recorder emits the SAME wire format (raw s16, mono, 16 kHz, to
|
||||
stdout), so the VAD reader in audio.py is identical on every platform.
|
||||
|
||||
- Linux : PipeWire CLIs — `pw-play`, `pw-record` (the proven Docker path).
|
||||
- macOS : built-ins only — `afplay` (play), a compiled Swift/AVFoundation
|
||||
recorder (`_macos/mcspeak_record.swift`), `afconvert` (transcode),
|
||||
`osascript` (system-volume duck).
|
||||
- Windows: PLANNED — `winsound.PlaySound` (stdlib WAV play), `sounddevice`
|
||||
(PortAudio) or `ffmpeg -f dshow` (capture), `pycaw` (per-app duck).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
IS_LINUX = sys.platform.startswith("linux")
|
||||
IS_MACOS = sys.platform == "darwin"
|
||||
IS_WINDOWS = sys.platform.startswith("win")
|
||||
|
||||
_MACOS_DIR = Path(__file__).parent / "_macos"
|
||||
_RECORDER_SRC = _MACOS_DIR / "mcspeak_record.swift"
|
||||
# Compiled recorder lives in a user cache dir (the install dir may be read-only).
|
||||
_CACHE_DIR = Path.home() / ".cache" / "mcspeak"
|
||||
|
||||
|
||||
class AudioBackendError(RuntimeError):
|
||||
"""Raised when the platform backend can't satisfy a request."""
|
||||
|
||||
|
||||
def platform_name() -> str:
|
||||
if IS_MACOS:
|
||||
return "macos"
|
||||
if IS_WINDOWS:
|
||||
return "windows"
|
||||
if IS_LINUX:
|
||||
return "linux"
|
||||
return sys.platform
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Playback
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def play_command(path: Path | str) -> list[str]:
|
||||
"""Command to play a WAV to the default output device, blocking until done."""
|
||||
if IS_MACOS:
|
||||
return ["afplay", str(path)]
|
||||
if IS_WINDOWS:
|
||||
# Windows plays via winsound in-process (see play_audio), not a subprocess.
|
||||
raise AudioBackendError("Windows playback is in-process; don't call play_command")
|
||||
return ["pw-play", str(path)] # Linux / PipeWire
|
||||
|
||||
|
||||
def volume_workaround_applies() -> bool:
|
||||
"""Only PulseAudio/PipeWire needs the stream-restore 100% volume override."""
|
||||
return IS_LINUX
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Recording — every backend streams raw s16 mono PCM at `sample_rate` to stdout
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def record_stream_command(
|
||||
sample_rate: int = 16000, channels: int = 1, source: str | None = None
|
||||
) -> list[str]:
|
||||
"""Command that streams raw s16 PCM to stdout until it receives SIGTERM."""
|
||||
if IS_MACOS:
|
||||
# The Swift recorder is fixed at 16 kHz mono (Parakeet's format); it
|
||||
# ignores sample_rate/source for now (default input device only).
|
||||
return [str(ensure_macos_recorder())]
|
||||
if IS_WINDOWS:
|
||||
raise AudioBackendError(
|
||||
"Windows capture not implemented yet — planned via sounddevice/ffmpeg"
|
||||
)
|
||||
cmd = ["pw-record", "--rate", str(sample_rate), "--channels", str(channels), "--format", "s16"]
|
||||
if source:
|
||||
cmd.extend(["--target", source])
|
||||
cmd.append("-") # stdout
|
||||
return cmd
|
||||
|
||||
|
||||
def ensure_macos_recorder() -> Path:
|
||||
"""Compile the Swift recorder once (cached in ~/.cache/mcspeak), return its path.
|
||||
|
||||
Recompiles only when the source changes (tracked by a content-hash stamp).
|
||||
"""
|
||||
if not IS_MACOS:
|
||||
raise AudioBackendError("macOS recorder requested on a non-macOS host")
|
||||
swiftc = shutil.which("swiftc")
|
||||
if not swiftc:
|
||||
raise AudioBackendError(
|
||||
"swiftc not found — install the Xcode command-line tools (xcode-select --install)"
|
||||
)
|
||||
_CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
binary = _CACHE_DIR / "mcspeak-record"
|
||||
src_hash = hashlib.sha256(_RECORDER_SRC.read_bytes()).hexdigest()[:12]
|
||||
stamp = _CACHE_DIR / f".record-{src_hash}"
|
||||
if binary.exists() and stamp.exists():
|
||||
return binary
|
||||
|
||||
proc = subprocess.run(
|
||||
[swiftc, "-O", str(_RECORDER_SRC), "-o", str(binary)],
|
||||
capture_output=True, text=True,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
raise AudioBackendError(
|
||||
f"swiftc failed compiling the recorder: {proc.stderr.strip()[:500]}"
|
||||
)
|
||||
# Clear any stale stamps, then mark this build good.
|
||||
for old in _CACHE_DIR.glob(".record-*"):
|
||||
old.unlink(missing_ok=True)
|
||||
stamp.write_text("ok")
|
||||
return binary
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Transcode / resample (used by convert_audio and any 16k-mono normalization)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def has_ffmpeg() -> bool:
|
||||
return shutil.which("ffmpeg") is not None
|
||||
|
||||
|
||||
def prefers_afconvert() -> bool:
|
||||
"""On macOS without ffmpeg, use the built-in afconvert."""
|
||||
return IS_MACOS and not has_ffmpeg()
|
||||
Loading…
x
Reference in New Issue
Block a user