"""Tests for the DSN tokenizer.""" from __future__ import annotations import pytest from freeroute.dsn.tokenizer import DsnSyntaxError, TokenKind, tokenize def kinds(text: str) -> list[TokenKind]: return [t.kind for t in tokenize(text)] def texts(text: str) -> list[str]: return [t.text for t in tokenize(text)] def test_brackets_and_atoms(): toks = tokenize("(pcb board)") assert [t.kind for t in toks] == [ TokenKind.LPAREN, TokenKind.ATOM, TokenKind.ATOM, TokenKind.RPAREN, ] assert texts("(pcb board)") == ["(", "pcb", "board", ")"] def test_numbers_stay_atoms_but_expose_values(): toks = tokenize("12 -3 4.5 -0.25 1e3") assert all(t.kind is TokenKind.ATOM for t in toks) assert toks[0].as_int() == 12 assert toks[1].as_int() == -3 assert toks[2].as_int() is None assert toks[2].as_float() == 4.5 assert toks[3].as_float() == -0.25 assert toks[4].as_float() == 1000.0 def test_double_quoted_string(): toks = tokenize('(host_cad "KiCad EDA")') assert toks[2].kind is TokenKind.STRING assert toks[2].text == "KiCad EDA" def test_single_quoted_string(): toks = tokenize("(x 'a b c')") assert toks[2].kind is TokenKind.STRING assert toks[2].text == "a b c" def test_string_quote_directive_reads_quote_as_atom(): # `(string_quote ")` — FreeRouting's IGNORE_QUOTE state reads the lone quote # char after `string_quote` as a literal atom, not as an (unterminated) # string opener. toks = tokenize('(string_quote ")') assert [t.kind for t in toks] == [ TokenKind.LPAREN, TokenKind.ATOM, TokenKind.ATOM, TokenKind.RPAREN, ] assert toks[2].text == '"' def test_quotes_still_open_strings_normally(): toks = tokenize('(host_cad "KiCad")') assert toks[2].kind is TokenKind.STRING assert toks[2].text == "KiCad" def test_atom_may_contain_special_chars(): toks = tokenize("Via[0-1]_600:300_um") assert len(toks) == 1 assert toks[0].text == "Via[0-1]_600:300_um" def test_quote_stops_a_bareword_and_opens_a_string(): # `space_in_quoted_tokens` pin ref: `"J3"-"GND"` is three tokens. toks = tokenize('"J3"-"GND"') assert [t.kind for t in toks] == [ TokenKind.STRING, TokenKind.ATOM, TokenKind.STRING, ] assert [t.text for t in toks] == ["J3", "-", "GND"] def test_hash_inside_quoted_string_is_not_a_comment(): toks = tokenize('"_VBUS #22"') assert toks[0].kind is TokenKind.STRING assert toks[0].text == "_VBUS #22" def test_line_comment_ignored(): assert texts("# a comment\n(a)") == ["(", "a", ")"] def test_block_comment_ignored(): assert texts("(a /* skip me */ b)") == ["(", "a", "b", ")"] def test_hash_glued_to_name_is_an_atom_not_a_comment(): # `#WLTXD` is a net name, not a comment (FreeRouting reads it in NAME state). assert texts("(net #WLTXD)") == ["(", "net", "#WLTXD", ")"] assert texts("#22") == ["#22"] def test_line_tracking(): toks = tokenize("(a\n b\n c)") lines = {t.text: t.line for t in toks if t.kind is TokenKind.ATOM} assert lines == {"a": 1, "b": 2, "c": 3} def test_unterminated_string_raises(): with pytest.raises(DsnSyntaxError): tokenize('(a "no end') def test_unclosed_block_comment_is_name_token(): # KiCad emits hierarchical net names like `/*52` (SpecCharASCII includes # `/` and `*`). With no closing `*/`, `/*...` is a name run, not an # unterminated comment — matching FreeRouting's JFlex rule-order resolution. assert texts("(net /*52)") == ["(", "net", "/*52", ")"] def test_closed_block_comment_is_stripped(): # A properly closed `/* ... */` is still a comment and gets dropped. assert texts("(a /* c */ b)") == ["(", "a", "b", ")"] def test_hyphenated_pin_ref_is_single_atom(): assert texts("U1-1 R2-2") == ["U1-1", "R2-2"]