Ryan Malloy
bbdcb243dc
Apply .gitattributes normalization to convert all CRLF line endings inherited from Windows-origin source files to Unix LF. 175 files, zero content changes.
441 lines
16 KiB
Python
441 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Automated carrier survey engine -- six-stage pipeline.
|
|
|
|
Orchestrates spectrum sweep, peak detection, blind scan, and TS
|
|
sampling to build a complete carrier catalog from the IF band.
|
|
"""
|
|
|
|
import sys
|
|
import time
|
|
import io
|
|
|
|
from skywalker_lib import SkyWalker1, MODULATIONS, MOD_FEC_GROUP, FEC_RATES
|
|
from signal_analysis import (
|
|
adaptive_noise_floor,
|
|
detect_peaks_enhanced,
|
|
estimate_carrier_bw,
|
|
classify_carrier,
|
|
)
|
|
from carrier_catalog import CarrierEntry, CarrierCatalog
|
|
from ts_analyze import TSReader, PSIParser, parse_pat, parse_pmt, parse_sdt
|
|
|
|
|
|
# Modulation index table for reverse lookup
|
|
_MOD_BY_INDEX = {}
|
|
for name, (idx, desc) in MODULATIONS.items():
|
|
_MOD_BY_INDEX[idx] = name
|
|
|
|
|
|
class SurveyEngine:
|
|
"""
|
|
Six-stage carrier survey pipeline:
|
|
|
|
1. Coarse sweep -- full IF range at configurable step size
|
|
2. Peak detection -- adaptive noise floor, peak merging
|
|
3. Fine sweep -- +/-10 MHz around each peak at 1 MHz steps
|
|
4. Blind scan -- try symbol rate range at each refined peak
|
|
5. TS sample -- for locked carriers, short capture + PAT/PMT/SDT
|
|
6. Catalog assembly -- aggregate everything into a CarrierCatalog
|
|
"""
|
|
|
|
STAGE_COARSE = "coarse_sweep"
|
|
STAGE_PEAKS = "peak_detection"
|
|
STAGE_FINE = "fine_sweep"
|
|
STAGE_BLIND = "blind_scan"
|
|
STAGE_TS = "ts_sample"
|
|
STAGE_CATALOG = "catalog_assembly"
|
|
|
|
def __init__(self, device: SkyWalker1, callback=None):
|
|
"""
|
|
device -- open SkyWalker1 instance
|
|
callback -- optional function(stage, progress_pct, message)
|
|
called at each major step for progress reporting
|
|
"""
|
|
self.dev = device
|
|
self.callback = callback
|
|
|
|
def _report(self, stage: str, pct: float, msg: str) -> None:
|
|
if self.callback:
|
|
self.callback(stage, pct, msg)
|
|
|
|
# ------------------------------------------------------------------
|
|
# Public entry points
|
|
# ------------------------------------------------------------------
|
|
|
|
def run_full_scan(self, start_mhz: float = 950, stop_mhz: float = 2150,
|
|
coarse_step: float = 5.0, fine_step: float = 1.0,
|
|
sr_min: int = 1_000_000, sr_max: int = 30_000_000,
|
|
sr_step: int = 1_000_000,
|
|
ts_capture_secs: float = 3.0) -> CarrierCatalog:
|
|
"""
|
|
Run all six stages and return a populated CarrierCatalog.
|
|
"""
|
|
# Stage 1: coarse sweep
|
|
self._report(self.STAGE_COARSE, 0, "Starting coarse sweep")
|
|
freqs, powers = self._coarse_sweep(start_mhz, stop_mhz, coarse_step)
|
|
self._report(self.STAGE_COARSE, 100, f"Coarse sweep done: {len(freqs)} points")
|
|
|
|
# Stage 2: peak detection
|
|
self._report(self.STAGE_PEAKS, 0, "Detecting peaks")
|
|
peaks = self._detect_peaks(freqs, powers)
|
|
self._report(self.STAGE_PEAKS, 100, f"Found {len(peaks)} candidate peaks")
|
|
|
|
if not peaks:
|
|
self._report(self.STAGE_CATALOG, 100, "No peaks found, empty catalog")
|
|
return self._assemble_catalog([], start_mhz, stop_mhz,
|
|
coarse_step, fine_step)
|
|
|
|
# Stage 3: fine sweep around each peak
|
|
self._report(self.STAGE_FINE, 0, "Starting fine sweeps")
|
|
refined = self._fine_sweep(peaks, fine_step)
|
|
self._report(self.STAGE_FINE, 100, f"Refined to {len(refined)} carriers")
|
|
|
|
# Stage 4: blind scan at each refined peak
|
|
self._report(self.STAGE_BLIND, 0, "Starting blind scan")
|
|
scanned = self._blind_scan_peaks(refined, sr_min, sr_max, sr_step)
|
|
self._report(self.STAGE_BLIND, 100,
|
|
f"Blind scan done: {sum(1 for s in scanned if s.get('locked'))} locked")
|
|
|
|
# Stage 5: TS sample for locked carriers
|
|
locked = [s for s in scanned if s.get("locked")]
|
|
self._report(self.STAGE_TS, 0, f"Sampling TS from {len(locked)} locked carriers")
|
|
sampled = self._sample_ts(locked, capture_secs=ts_capture_secs)
|
|
self._report(self.STAGE_TS, 100, "TS sampling done")
|
|
|
|
# Stage 6: assemble catalog
|
|
self._report(self.STAGE_CATALOG, 0, "Assembling catalog")
|
|
catalog = self._assemble_catalog(sampled, start_mhz, stop_mhz,
|
|
coarse_step, fine_step)
|
|
self._report(self.STAGE_CATALOG, 100,
|
|
f"Catalog ready: {len(catalog.carriers)} carriers")
|
|
return catalog
|
|
|
|
def run_quick_scan(self, start_mhz: float = 950, stop_mhz: float = 2150,
|
|
step: float = 5.0) -> list:
|
|
"""
|
|
Quick scan: coarse sweep + peak detection only.
|
|
Returns list of peak dicts from detect_peaks_enhanced.
|
|
No blind scan or TS capture.
|
|
"""
|
|
self._report(self.STAGE_COARSE, 0, "Quick scan: coarse sweep")
|
|
freqs, powers = self._coarse_sweep(start_mhz, stop_mhz, step)
|
|
self._report(self.STAGE_COARSE, 100, f"Sweep done: {len(freqs)} points")
|
|
|
|
self._report(self.STAGE_PEAKS, 0, "Quick scan: peak detection")
|
|
peaks = self._detect_peaks(freqs, powers)
|
|
self._report(self.STAGE_PEAKS, 100, f"Found {len(peaks)} peaks")
|
|
return peaks
|
|
|
|
# ------------------------------------------------------------------
|
|
# Internal stage methods
|
|
# ------------------------------------------------------------------
|
|
|
|
def _coarse_sweep(self, start_mhz: float, stop_mhz: float,
|
|
step: float) -> tuple:
|
|
"""
|
|
Stage 1: sweep the IF band and collect power measurements.
|
|
Returns (freqs_mhz[], powers_db[]).
|
|
"""
|
|
total_steps = int((stop_mhz - start_mhz) / step) + 1
|
|
|
|
def sweep_cb(freq, step_num, total, result):
|
|
pct = (step_num / max(total, 1)) * 100
|
|
self._report(self.STAGE_COARSE, pct,
|
|
f"{freq:.0f} MHz {result['power_db']:+.1f} dB")
|
|
|
|
freqs, powers, _ = self.dev.sweep_spectrum(
|
|
start_mhz, stop_mhz, step_mhz=step,
|
|
dwell_ms=15, callback=sweep_cb
|
|
)
|
|
return freqs, powers
|
|
|
|
def _detect_peaks(self, freqs: list, powers: list) -> list:
|
|
"""
|
|
Stage 2: enhanced peak detection with adaptive noise floor.
|
|
Returns list of peak dicts.
|
|
"""
|
|
noise_floor, mad = adaptive_noise_floor(powers)
|
|
self._report(self.STAGE_PEAKS, 50,
|
|
f"Noise floor: {noise_floor:.1f} dB, MAD: {mad:.2f} dB")
|
|
|
|
peaks = detect_peaks_enhanced(freqs, powers, threshold_db=6.0)
|
|
|
|
# Annotate each peak with classification
|
|
for p in peaks:
|
|
p["classification"] = classify_carrier(p["width_mhz"], p["power"])
|
|
|
|
return peaks
|
|
|
|
def _fine_sweep(self, peaks: list, fine_step: float = 1.0) -> list:
|
|
"""
|
|
Stage 3: sweep +/-10 MHz around each peak at fine resolution.
|
|
Returns list of refined peak dicts with updated freq/power/width.
|
|
"""
|
|
refined = []
|
|
for i, peak in enumerate(peaks):
|
|
pct = (i / max(len(peaks), 1)) * 100
|
|
center = peak["freq"]
|
|
margin = max(peak["width_mhz"] * 1.5, 10.0)
|
|
fine_start = max(950.0, center - margin)
|
|
fine_stop = min(2150.0, center + margin)
|
|
|
|
self._report(self.STAGE_FINE, pct,
|
|
f"Fine sweep {center:.0f} MHz ({fine_start:.0f}-{fine_stop:.0f})")
|
|
|
|
freqs, powers, _ = self.dev.sweep_spectrum(
|
|
fine_start, fine_stop, step_mhz=fine_step,
|
|
dwell_ms=20
|
|
)
|
|
|
|
# Re-detect peaks in the fine data
|
|
fine_peaks = detect_peaks_enhanced(freqs, powers, threshold_db=4.0)
|
|
if fine_peaks:
|
|
# Take the strongest peak from the fine sweep
|
|
best = max(fine_peaks, key=lambda p: p["power"])
|
|
best["classification"] = classify_carrier(
|
|
best["width_mhz"], best["power"]
|
|
)
|
|
refined.append(best)
|
|
else:
|
|
# Keep the coarse peak if fine sweep didn't improve it
|
|
refined.append(peak)
|
|
|
|
return refined
|
|
|
|
def _blind_scan_peaks(self, refined_peaks: list,
|
|
sr_min: int, sr_max: int,
|
|
sr_step: int) -> list:
|
|
"""
|
|
Stage 4: attempt blind scan at each refined peak frequency.
|
|
Returns list of result dicts, each with the peak info plus
|
|
blind scan results (locked, sr_sps, etc).
|
|
"""
|
|
results = []
|
|
for i, peak in enumerate(refined_peaks):
|
|
pct = (i / max(len(refined_peaks), 1)) * 100
|
|
freq_khz = int(peak["freq"] * 1000)
|
|
|
|
self._report(self.STAGE_BLIND, pct,
|
|
f"Blind scan {peak['freq']:.1f} MHz")
|
|
|
|
# Use classification to narrow SR range if possible
|
|
cls = peak.get("classification", {})
|
|
sr_range = cls.get("estimated_sr_range", (sr_min, sr_max))
|
|
scan_min = max(sr_min, sr_range[0])
|
|
scan_max = min(sr_max, sr_range[1])
|
|
|
|
result = {
|
|
"freq_mhz": peak["freq"],
|
|
"freq_khz": freq_khz,
|
|
"power_db": peak["power"],
|
|
"width_mhz": peak["width_mhz"],
|
|
"prominence_db": peak.get("prominence_db", 0),
|
|
"classification": cls,
|
|
"locked": False,
|
|
"sr_sps": 0,
|
|
"mod_index": -1,
|
|
"fec_index": -1,
|
|
}
|
|
|
|
# Try adaptive blind scan first (firmware-assisted)
|
|
try:
|
|
lock = self.dev.adaptive_blind_scan(
|
|
freq_khz, scan_min, scan_max, sr_step
|
|
)
|
|
if lock and lock.get("locked"):
|
|
result["locked"] = True
|
|
result["sr_sps"] = lock["sr_sps"]
|
|
result["freq_khz"] = lock.get("freq_khz", freq_khz)
|
|
# Read signal quality
|
|
time.sleep(0.1)
|
|
sig = self.dev.signal_monitor()
|
|
result["snr_db"] = sig.get("snr_db", 0)
|
|
result["agc1"] = sig.get("agc1", 0)
|
|
except Exception as e:
|
|
self._report(self.STAGE_BLIND, pct,
|
|
f"Blind scan error at {peak['freq']:.1f} MHz: {e}")
|
|
|
|
results.append(result)
|
|
|
|
return results
|
|
|
|
def _sample_ts(self, locked_carriers: list,
|
|
capture_secs: float = 3.0) -> list:
|
|
"""
|
|
Stage 5: for each locked carrier, tune + arm + capture TS data,
|
|
then parse PAT/PMT/SDT for service information.
|
|
"""
|
|
results = []
|
|
for i, carrier in enumerate(locked_carriers):
|
|
pct = (i / max(len(locked_carriers), 1)) * 100
|
|
freq_khz = carrier["freq_khz"]
|
|
sr_sps = carrier["sr_sps"]
|
|
|
|
self._report(self.STAGE_TS, pct,
|
|
f"Sampling {carrier['freq_mhz']:.1f} MHz "
|
|
f"SR={sr_sps / 1e6:.3f} Msps")
|
|
|
|
carrier["services"] = []
|
|
carrier["pat"] = None
|
|
carrier["pmt"] = {}
|
|
|
|
if sr_sps <= 0:
|
|
results.append(carrier)
|
|
continue
|
|
|
|
try:
|
|
# Tune with QPSK auto-FEC as a safe default
|
|
self.dev.tune(sr_sps, freq_khz, 0, 5)
|
|
time.sleep(0.3)
|
|
|
|
# Verify lock
|
|
sig = self.dev.signal_monitor()
|
|
if not sig.get("locked"):
|
|
results.append(carrier)
|
|
continue
|
|
|
|
carrier["snr_db"] = sig.get("snr_db", 0)
|
|
|
|
# Arm and capture TS data
|
|
self.dev.arm_transfer(True)
|
|
ts_data = bytearray()
|
|
deadline = time.time() + capture_secs
|
|
|
|
while time.time() < deadline:
|
|
chunk = self.dev.read_stream(timeout=500)
|
|
if chunk:
|
|
ts_data.extend(chunk)
|
|
|
|
self.dev.arm_transfer(False)
|
|
|
|
# Parse the captured TS
|
|
if ts_data:
|
|
services = _parse_ts_services(bytes(ts_data))
|
|
carrier["services"] = services.get("service_names", [])
|
|
carrier["pat"] = services.get("pat")
|
|
carrier["pmt"] = services.get("pmts", {})
|
|
carrier["sdt"] = services.get("sdt")
|
|
|
|
except Exception as e:
|
|
self._report(self.STAGE_TS, pct,
|
|
f"TS capture error at {carrier['freq_mhz']:.1f} MHz: {e}")
|
|
try:
|
|
self.dev.arm_transfer(False)
|
|
except Exception:
|
|
pass
|
|
|
|
results.append(carrier)
|
|
|
|
return results
|
|
|
|
def _assemble_catalog(self, all_results: list,
|
|
start_mhz: float = 950,
|
|
stop_mhz: float = 2150,
|
|
coarse_step: float = 5.0,
|
|
fine_step: float = 1.0) -> CarrierCatalog:
|
|
"""
|
|
Stage 6: build a CarrierCatalog from the collected results.
|
|
"""
|
|
catalog = CarrierCatalog()
|
|
catalog.sweep_params = {
|
|
"start_mhz": start_mhz,
|
|
"stop_mhz": stop_mhz,
|
|
"coarse_step_mhz": coarse_step,
|
|
"fine_step_mhz": fine_step,
|
|
}
|
|
|
|
for r in all_results:
|
|
mod_name = ""
|
|
if r.get("mod_index", -1) >= 0:
|
|
mod_name = _MOD_BY_INDEX.get(r["mod_index"], "")
|
|
|
|
entry = CarrierEntry(
|
|
freq_khz=r.get("freq_khz", int(r.get("freq_mhz", 0) * 1000)),
|
|
sr_sps=r.get("sr_sps", 0),
|
|
modulation=mod_name,
|
|
fec="",
|
|
power_db=r.get("power_db", 0),
|
|
snr_db=r.get("snr_db", 0),
|
|
locked=r.get("locked", False),
|
|
services=r.get("services", []),
|
|
bw_mhz=r.get("width_mhz", 0),
|
|
classification=r.get("classification", {}),
|
|
)
|
|
catalog.add_carrier(entry)
|
|
|
|
return catalog
|
|
|
|
|
|
def _parse_ts_services(ts_data: bytes) -> dict:
|
|
"""
|
|
Parse PAT, PMT, and SDT from a chunk of TS data.
|
|
|
|
Returns dict with:
|
|
pat - parsed PAT or None
|
|
pmts - {pmt_pid: parsed PMT}
|
|
sdt - parsed SDT or None
|
|
service_names - list of service name strings from SDT
|
|
"""
|
|
result = {
|
|
"pat": None,
|
|
"pmts": {},
|
|
"sdt": None,
|
|
"service_names": [],
|
|
}
|
|
|
|
source = io.BytesIO(ts_data)
|
|
reader = TSReader(source)
|
|
psi_pat = PSIParser()
|
|
psi_pmt = PSIParser()
|
|
psi_sdt = PSIParser()
|
|
|
|
pat = None
|
|
pmt_pids = set()
|
|
pmts_found = {}
|
|
|
|
try:
|
|
for pkt in reader.iter_packets(max_packets=50000):
|
|
# PAT on PID 0x0000
|
|
if pkt.pid == 0x0000 and pat is None:
|
|
section = psi_pat.feed(pkt)
|
|
if section is not None:
|
|
pat = parse_pat(section)
|
|
if pat:
|
|
result["pat"] = pat
|
|
for prog, pid in pat["programs"].items():
|
|
if prog != 0:
|
|
pmt_pids.add(pid)
|
|
|
|
# PMT sections
|
|
if pkt.pid in pmt_pids and pkt.pid not in pmts_found:
|
|
section = psi_pmt.feed(pkt)
|
|
if section is not None:
|
|
pmt = parse_pmt(section)
|
|
if pmt:
|
|
pmts_found[pkt.pid] = pmt
|
|
|
|
# SDT on PID 0x0011
|
|
if pkt.pid == 0x0011 and result["sdt"] is None:
|
|
section = psi_sdt.feed(pkt)
|
|
if section is not None:
|
|
sdt = parse_sdt(section)
|
|
if sdt:
|
|
result["sdt"] = sdt
|
|
for svc in sdt.get("services", []):
|
|
name = svc.get("service_name", "")
|
|
if name:
|
|
result["service_names"].append(name)
|
|
|
|
# Stop early once we have everything
|
|
if (pat is not None
|
|
and len(pmts_found) >= len(pmt_pids)
|
|
and result["sdt"] is not None):
|
|
break
|
|
|
|
except Exception:
|
|
pass
|
|
|
|
result["pmts"] = pmts_found
|
|
return result
|