Ryan Malloy
c905032a39
Specctra's SpecCharASCII includes / and *, and an Identifier may start with /, so KiCad emits hierarchical net names such as /*52 and /53. The tokenizer treated any /* as a block-comment start and raised 'unterminated comment' when no */ followed — rejecting real KiCad DSN. Match FreeRouting's JFlex rule-order resolution: /* is a comment only when a closing */ exists; otherwise it is an ordinary name run. Validated against a KiCad 10.0.4 pcbnew-exported DSN (78 nets incl. /*52, /53).
134 lines
3.8 KiB
Python
134 lines
3.8 KiB
Python
"""Tests for the DSN tokenizer."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from freeroute.dsn.tokenizer import DsnSyntaxError, TokenKind, tokenize
|
|
|
|
|
|
def kinds(text: str) -> list[TokenKind]:
|
|
return [t.kind for t in tokenize(text)]
|
|
|
|
|
|
def texts(text: str) -> list[str]:
|
|
return [t.text for t in tokenize(text)]
|
|
|
|
|
|
def test_brackets_and_atoms():
|
|
toks = tokenize("(pcb board)")
|
|
assert [t.kind for t in toks] == [
|
|
TokenKind.LPAREN,
|
|
TokenKind.ATOM,
|
|
TokenKind.ATOM,
|
|
TokenKind.RPAREN,
|
|
]
|
|
assert texts("(pcb board)") == ["(", "pcb", "board", ")"]
|
|
|
|
|
|
def test_numbers_stay_atoms_but_expose_values():
|
|
toks = tokenize("12 -3 4.5 -0.25 1e3")
|
|
assert all(t.kind is TokenKind.ATOM for t in toks)
|
|
assert toks[0].as_int() == 12
|
|
assert toks[1].as_int() == -3
|
|
assert toks[2].as_int() is None
|
|
assert toks[2].as_float() == 4.5
|
|
assert toks[3].as_float() == -0.25
|
|
assert toks[4].as_float() == 1000.0
|
|
|
|
|
|
def test_double_quoted_string():
|
|
toks = tokenize('(host_cad "KiCad EDA")')
|
|
assert toks[2].kind is TokenKind.STRING
|
|
assert toks[2].text == "KiCad EDA"
|
|
|
|
|
|
def test_single_quoted_string():
|
|
toks = tokenize("(x 'a b c')")
|
|
assert toks[2].kind is TokenKind.STRING
|
|
assert toks[2].text == "a b c"
|
|
|
|
|
|
def test_string_quote_directive_reads_quote_as_atom():
|
|
# `(string_quote ")` — FreeRouting's IGNORE_QUOTE state reads the lone quote
|
|
# char after `string_quote` as a literal atom, not as an (unterminated)
|
|
# string opener.
|
|
toks = tokenize('(string_quote ")')
|
|
assert [t.kind for t in toks] == [
|
|
TokenKind.LPAREN,
|
|
TokenKind.ATOM,
|
|
TokenKind.ATOM,
|
|
TokenKind.RPAREN,
|
|
]
|
|
assert toks[2].text == '"'
|
|
|
|
|
|
def test_quotes_still_open_strings_normally():
|
|
toks = tokenize('(host_cad "KiCad")')
|
|
assert toks[2].kind is TokenKind.STRING
|
|
assert toks[2].text == "KiCad"
|
|
|
|
|
|
def test_atom_may_contain_special_chars():
|
|
toks = tokenize("Via[0-1]_600:300_um")
|
|
assert len(toks) == 1
|
|
assert toks[0].text == "Via[0-1]_600:300_um"
|
|
|
|
|
|
def test_quote_stops_a_bareword_and_opens_a_string():
|
|
# `space_in_quoted_tokens` pin ref: `"J3"-"GND"` is three tokens.
|
|
toks = tokenize('"J3"-"GND"')
|
|
assert [t.kind for t in toks] == [
|
|
TokenKind.STRING,
|
|
TokenKind.ATOM,
|
|
TokenKind.STRING,
|
|
]
|
|
assert [t.text for t in toks] == ["J3", "-", "GND"]
|
|
|
|
|
|
def test_hash_inside_quoted_string_is_not_a_comment():
|
|
toks = tokenize('"_VBUS #22"')
|
|
assert toks[0].kind is TokenKind.STRING
|
|
assert toks[0].text == "_VBUS #22"
|
|
|
|
|
|
def test_line_comment_ignored():
|
|
assert texts("# a comment\n(a)") == ["(", "a", ")"]
|
|
|
|
|
|
def test_block_comment_ignored():
|
|
assert texts("(a /* skip me */ b)") == ["(", "a", "b", ")"]
|
|
|
|
|
|
def test_hash_glued_to_name_is_an_atom_not_a_comment():
|
|
# `#WLTXD` is a net name, not a comment (FreeRouting reads it in NAME state).
|
|
assert texts("(net #WLTXD)") == ["(", "net", "#WLTXD", ")"]
|
|
assert texts("#22") == ["#22"]
|
|
|
|
|
|
def test_line_tracking():
|
|
toks = tokenize("(a\n b\n c)")
|
|
lines = {t.text: t.line for t in toks if t.kind is TokenKind.ATOM}
|
|
assert lines == {"a": 1, "b": 2, "c": 3}
|
|
|
|
|
|
def test_unterminated_string_raises():
|
|
with pytest.raises(DsnSyntaxError):
|
|
tokenize('(a "no end')
|
|
|
|
|
|
def test_unclosed_block_comment_is_name_token():
|
|
# KiCad emits hierarchical net names like `/*52` (SpecCharASCII includes
|
|
# `/` and `*`). With no closing `*/`, `/*...` is a name run, not an
|
|
# unterminated comment — matching FreeRouting's JFlex rule-order resolution.
|
|
assert texts("(net /*52)") == ["(", "net", "/*52", ")"]
|
|
|
|
|
|
def test_closed_block_comment_is_stripped():
|
|
# A properly closed `/* ... */` is still a comment and gets dropped.
|
|
assert texts("(a /* c */ b)") == ["(", "a", "b", ")"]
|
|
|
|
|
|
def test_hyphenated_pin_ref_is_single_atom():
|
|
assert texts("U1-1 R2-2") == ["U1-1", "R2-2"]
|