Coverage for trlc/lexer

1#!/usr/bin/env python3

3# TRLC - Treat Requirements Like Code

6# This file is part of the TRLC Python Reference Implementation.

8# TRLC is free software: you can redistribute it and/or modify it

9# under the terms of the GNU General Public License as published by

10# the Free Software Foundation, either version 3 of the License, or

11# (at your option) any later version.

12#

13# TRLC is distributed in the hope that it will be useful, but WITHOUT

14# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY

15# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public

16# License for more details.

17#

18# You should have received a copy of the GNU General Public License

19# along with TRLC. If not, see <https://www.gnu.org/licenses/>.

21"""Lexer for Markdown TRLC (.trlc.md) files.

23Converts a Markdown representation of a TRLC requirements file into a

24token stream compatible with the TRLC parser.

26Markdown format

27---------------

28The file structure maps to TRLC constructs as follows:

30 # PackageName

31 → ``package PackageName``

33 import pkg

34 → ``import pkg``

36 ## Section name

37 → ``section "Section name" {``

39 <hr>

40 → Record separator (closes any open record within the section)

42 ### Record heading

43 → Starts a record; the heading text becomes the record identifier

44 (spaces and non-alphanumeric characters are replaced with ``_``).

46 Property table (under the ``###`` heading):

48 | Property | Value |

49 |----------|-------|

50 | type | Foo | ← ``type`` row gives the record type name

51 | field | value | ← subsequent rows are ``field = value`` assignments

53 #### Field heading

54 → Starts a free-text / string field whose name is the (lowercased)

55 heading identifier. All content that follows—including any Markdown

56 tables—is collected verbatim and emitted as a single STRING token.

57 The string field ends when the next heading (``##``/``###``/``####``),

58 ``<hr>``, or end-of-file is reached.

60Value inference

61---------------

62Values in the property table are interpreted as follows (in order):

64 * ``true`` / ``false`` / ``null`` → KEYWORD token

65 * Decimal integer (``42``) → INTEGER token

66 * Decimal number (``3.14``) → DECIMAL token

67 * Hex integer (``0x1A``) → INTEGER token

68 * Binary integer (``0b101``) → INTEGER token

69 * Dot-qualified identifier → chain of IDENTIFIER + DOT tokens

70 * Anything else → STRING token

71"""

73import re

74from fractions import Fraction

76from trlc import ast as trlc_ast

77from trlc.lexer import Token, TRLC_Lexer

78from trlc.errors import Location, Message_Handler

79from trlc.location_md import MD_Location

82class MD_Source_Reference(Location):

83 """A Location subclass that carries the source line and caret position

84 so Message_Handler can render the same visual caret output as TRLC."""

86 def __init__(self, file_name, line_no, col_no, source_line):

87 super().__init__(file_name, line_no, col_no)

88 self._source_line = source_line

90 def context_lines(self):

91 # Mirror Source_Reference.context_lines() from trlc/lexer.py:

92 # return [source_line_stripped, caret_string]

93 col = self.col_no if self.col_no else 1

94 stripped = self._source_line.lstrip()

95 leading = len(self._source_line) - len(stripped)

96 caret_col = max(col - 1 - leading, 0)

97 return [stripped, " " * caret_col + "^"]

100class MD_Lexer(TRLC_Lexer):

101 """Lexer that converts a ``.trlc.md`` file to a TRLC token stream.

102

103 The resulting token stream is compatible with :class:`trlc.parser.Parser`

104 when the parser is instantiated with a custom *lexer* argument.

105

106 Usage::

107

108 mh = Message_Handler()

109 lexer = MD_Lexer(mh, "path/to/file.trlc.md")

110 # pass lexer to Parser(mh, stab, file_name, ..., lexer=lexer)

111 """

112

113 # Boolean / null keywords (mirrors TRLC_Lexer.KEYWORDS)

114 KEYWORDS = frozenset(["true", "false", "null", "#", "##", "import"])

115

116 # Markdown-friendly aliases that map to TRLC block tokens.

117 MD_SECTION_START_TOKEN = "C_BRA"

118 MD_SECTION_END_TOKEN = "C_KET"

119

120 # ------------------------------------------------------------------ #

121 # Character classification (mirrors Lexer_Base / TRLC_Lexer helpers) #

122 # ------------------------------------------------------------------ #

123

124 @staticmethod

125 def _is_alpha(c):

126 return c.isascii() and c.isalpha()

127

128 @staticmethod

129 def _is_numeric(c):

130 return c.isascii() and c.isdigit()

131

132 @staticmethod

133 def _is_alnum(c):

134 return c.isascii() and c.isalnum()

135

136 @staticmethod

137 def _is_ident_start(c):

138 return c == "_" or (c.isascii() and c.isalpha())

139

140 @staticmethod

141 def _is_ident_cont(c):

142 return c == "_" or (c.isascii() and c.isalnum())

143

144 @staticmethod

145 def _is_hex_digit(c):

146 return c in "0123456789abcdefABCDEF"

147

148 # ------------------------------------------------------------------ #

149 # Value-token scanning helpers #

150 # ------------------------------------------------------------------ #

151

152 @staticmethod

153 def _scan_integer(text, start=0):

154 """Scan a run of decimal digits (and underscores) from *start*.

155

156 Returns the index one past the last scanned character, or ``-1``

157 when no digit is present at *start*.

158 """

159 i = start

160 n = len(text)

161 if i >= n or not text[i].isdigit():

162 return -1

163 while i < n and (text[i].isdigit() or text[i] == "_"):

164 i += 1

165 return i

166

167 @staticmethod

168 def _scan_hex(text, start=2):

169 """Scan hex digits from *start* (caller has consumed the ``0x`` prefix).

170

171 Returns the end index or ``-1`` if no valid hex digit at *start*.

172 """

173 i = start

174 n = len(text)

175 if i >= n or not MD_Lexer._is_hex_digit(text[i]): 175 ↛ 176line 175 didn't jump to line 176 because the condition on line 175 was never true

176 return -1

177 while i < n and (MD_Lexer._is_hex_digit(text[i]) or text[i] == "_"):

178 i += 1

179 return i

180

181 @staticmethod

182 def _scan_binary(text, start=2):

183 """Scan binary digits from *start* (caller has consumed the ``0b`` prefix).

184

185 Returns the end index or ``-1`` if no valid binary digit at *start*.

186 """

187 i = start

188 n = len(text)

189 if i >= n or text[i] not in "01": 189 ↛ 190line 189 didn't jump to line 190 because the condition on line 189 was never true

190 return -1

191 while i < n and (text[i] in "01" or text[i] == "_"):

192 i += 1

193 return i

194

195 @staticmethod

196 def _scan_ident(text, start=0):

197 """Scan one identifier segment from *text[start]*.

198

199 Returns the end index or ``-1`` when no identifier can start here.

200 """

201 i = start

202 n = len(text)

203 if i >= n or not MD_Lexer._is_ident_start(text[i]): 203 ↛ 204line 203 didn't jump to line 204 because the condition on line 203 was never true

204 return -1

205 i += 1

206 while i < n and MD_Lexer._is_ident_cont(text[i]):

207 i += 1

208 return i

209

210 # ------------------------------------------------------------------ #

211 # Heading / structural helpers #

212 # ------------------------------------------------------------------ #

213

214 @staticmethod

215 def _parse_heading(line):

216 """Return ``(level, content)`` for a Markdown heading, or ``(0, None)``.

217

218 *level* is the number of leading ``#`` characters; *content* is the

219 stripped heading text. Levels 1–4 are meaningful to this lexer.

220 """

221 i = 0

222 while i < len(line) and line[i] == "#":

223 i += 1

224 if i == 0 or i >= len(line) or not line[i].isspace():

225 return 0, None

226 content = line[i:].strip()

227 if not content: 227 ↛ 228line 227 didn't jump to line 228 because the condition on line 227 was never true

228 return 0, None

229 return i, content

230

231 @staticmethod

232 def _is_hr(stripped):

233 """Return True if *stripped* is a markdown record separator line.

234

235 Accepts ``<hr>``, ``<hr/>``, `` ``, `` `` combinations used in

236 exported markdown such as ``<hr> <hr>``.

237 """

238 lower = stripped.lower().replace(" ", "")

239 if not lower:

240 return False

241

242 idx = 0

243 saw_hr = False

244 while idx < len(lower):

245 if lower.startswith("<hr>", idx):

246 idx += 4

247 saw_hr = True

248 elif lower.startswith("<hr/>", idx):

249 idx += 5

250 saw_hr = True

251 elif lower.startswith(" ", idx):

252 idx += 4

253 elif lower.startswith(" ", idx):

254 idx += 5

255 else:

256 return False

257

258 return saw_hr

259

260 # ------------------------------------------------------------------ #

261 # Construction #

262 # ------------------------------------------------------------------ #

263

264 def __init__(self, mh, file_name, file_content=None):

265 assert isinstance(mh, Message_Handler)

266 assert isinstance(file_name, str)

267

268 self.file_name = file_name

269

270 if file_content is None:

271 with open(file_name, "r", encoding="UTF-8") as fd:

272 content = fd.read()

273 else:

274 assert isinstance(file_content, str)

275 content = file_content

276

277 # Initialise TRLC_Lexer to satisfy Parser lexer type checks.

278 super().__init__(mh, file_name, "")

279

280 self._raw_content = content # kept for Phase 2 reprocessing

281 self._stab = None # set by prepare_phase2() after RSL

282 self._md_tokens = []

283 self._tok_index = 0

284

285 # Phase 1: emit only preamble tokens (# PackageName, import lines).

286 # Phase 2 is triggered by Source_Manager after parse_rsl_files().

287 self._process_preamble(content)

288 self._preamble_end_index = len(self._md_tokens)

289 self.tokens = self._md_tokens

290

291 # ------------------------------------------------------------------ #

292 # Lexer_Base interface #

293 # ------------------------------------------------------------------ #

294

295 def file_location(self):

296 return Location(self.file_name, 1, 1)

297

298 def token(self):

299 if self._tok_index < len(self._md_tokens):

300 tok = self._md_tokens[self._tok_index]

301 self._tok_index += 1

302 # print("MD_Lexer: end of token stream reached", tok)

303 return tok

304 return None

305

306 def _process_preamble(self, content):

307 """Phase 1: emit only preamble tokens (# PackageName, import lines).

308

309 Body processing is deferred to prepare_phase2() so that RSL types

310 are available when field values are tokenised.

311 """

312 lines = content.splitlines()

313 preamble = []

314 for line in lines:

315 stripped = line.strip()

316 if not stripped:

317 preamble.append(line)

318 continue

319 if stripped.startswith("# ") or stripped == "#":

320 preamble.append(line)

321 continue

322 if stripped.startswith("import "):

323 preamble.append(line)

324 continue

325 break # first non-preamble line stops Phase 1

326 self._process("\n".join(preamble))

327

328 def prepare_phase2(self, stab):

329 """Phase 2: reprocess the full content with RSL types available.

330

331 Called by Source_Manager after parse_rsl_files() so the stab is

332 fully populated. After this returns, the caller must re-prime the

333 parser token cursor (set ct=None and call advance() once).

334 """

335 self._stab = stab

336 self._md_tokens = []

337 self._tok_index = 0

338 self._process(self._raw_content) # full reprocessing with types

339 self._tok_index = self._preamble_end_index # skip preamble

340 self.tokens = self._md_tokens

341

342 def _resolve_record_type(self, type_name, package_name=None):

343 """Look up the Record_Type AST node in the stab for *type_name*."""

344 if self._stab is None or not type_name: 344 ↛ 345line 344 didn't jump to line 345 because the condition on line 344 was never true

345 return None

346 if "." in type_name:

347 pkg_name, local_name = type_name.split(".", 1)

348 elif package_name: 348 ↛ 351line 348 didn't jump to line 351 because the condition on line 348 was always true

349 pkg_name, local_name = package_name, type_name

350 else:

351 return None

352 pkg_simple = trlc_ast.Symbol_Table.simplified_name(pkg_name)

353 pkg = self._stab.table.get(pkg_simple)

354 if not isinstance(pkg, trlc_ast.Package):

355 return None

356 type_simple = trlc_ast.Symbol_Table.simplified_name(local_name)

357 record_type = pkg.symbols.table.get(type_simple)

358 if isinstance(record_type, trlc_ast.Record_Type):

359 return record_type

360 return None

361

362 @staticmethod

363 def _get_field_type(record_type_ast, field_name):

364 """Return the declared type of *field_name*, or None."""

365 if record_type_ast is None: 365 ↛ 366line 365 didn't jump to line 366 because the condition on line 365 was never true

366 return None

367 simple = trlc_ast.Symbol_Table.simplified_name(field_name)

368 component = record_type_ast.components.table.get(simple)

369 return component.n_typ if component is not None else None

370

371 @staticmethod

372 def _is_tuple_array_field(record_type_ast, field_name):

373 """Return True when *field_name* is declared as an array of tuples."""

374 typ = MD_Lexer._get_field_type(record_type_ast, field_name)

375 return isinstance(typ, trlc_ast.Array_Type) and isinstance(

376 typ.element_type, trlc_ast.Tuple_Type

377 )

378

379 @staticmethod

380 def _is_string_field(record_type_ast, field_name):

381 """Return True when *field_name* is declared as a plain String."""

382 typ = MD_Lexer._get_field_type(record_type_ast, field_name)

383 return typ is not None and isinstance(typ, trlc_ast.Builtin_String)

384

385 def _emit_field_value(

386 self, raw_value, location, record_type_ast=None, field_name=None

387 ):

388 """Type-aware field value emission (used in Phase 2).

389

390 Dispatch rules when RSL type info is available:

391 - String field → emit STRING directly (never try array/tuple)

392 - Tuple-array field → type-confirmed array tokenisation

393 - Any other field → existing _emit_value() heuristics

394

395 Falls back to _emit_value() heuristics when type is unknown.

396 """

397 if record_type_ast is not None:

398 if self._is_string_field(record_type_ast, field_name):

399 self._emit(location, "STRING", raw_value.strip())

400 return

401 if self._is_tuple_array_field(record_type_ast, field_name):

402 if self._check_bracket_array(raw_value, location):

403 return

404 if not self._maybe_emit_array( 404 ↛ 407line 404 didn't jump to line 407 because the condition on line 404 was never true

405 raw_value, location, allow_unqualified=True

406 ):

407 self._emit(location, "STRING", raw_value.strip())

408 return

409 self._emit_value(raw_value, location)

410

411 # ------------------------------------------------------------------ #

412 # Internal helpers #

413 # ------------------------------------------------------------------ #

414

415 def _loc(self, line_no, col_no=1):

416 return Location(self.file_name, line_no, col_no)

417

418 def _source_ref(self, line_no, col_no, source_line):

419 """Return a location that renders a caret line, like TRLC Source_Reference."""

420 return MD_Source_Reference(self.file_name, line_no, col_no, source_line)

421

422 def _emit(self, location, kind, value=None):

423 if kind == "STRING" and not hasattr(location, "text"):

424 location = MD_Location(

425 file_name=location.file_name,

426 line_no=location.line_no,

427 col_no=location.col_no,

428 token_text=value,

429 mh=self.mh,

430 )

431 self._md_tokens.append(Token(location, kind, value))

432

433 @staticmethod

434 def _heading_to_identifier(text):

435 """Convert arbitrary heading text to a valid TRLC identifier.

436

437 Replaces any run of non-alphanumeric characters with a single

438 underscore and strips leading/trailing underscores.

439 """

440 result = []

441 in_sep = False

442 for c in text.strip():

443 if c.isascii() and c.isalnum():

444 result.append(c)

445 in_sep = False

446 else:

447 if not in_sep:

448 result.append("_")

449 in_sep = True

450 return "".join(result).strip("_")

451

452 @staticmethod

453 def _is_valid_identifier(name):

454 """Return True when *name* matches TRLC identifier syntax."""

455 if not name:

456 return False

457 if not MD_Lexer._is_alpha(name[0]):

458 return False

459 return all(MD_Lexer._is_alnum(ch) or ch == "_" for ch in name[1:])

460

461 def _validate_identifier(self, name, loc_line, heading_prefix_len, source_line=""):

462 """Validate name against TRLC identifier rules.

463

464 Raises lex_error pointing at the first offending character,

465 with the same caret-line visual as TRLC's 'unexpected character X'.

466 """

467 if not name: 467 ↛ 468line 467 didn't jump to line 468 because the condition on line 467 was never true

468 self.mh.lex_error(

469 self._source_ref(loc_line, heading_prefix_len + 1, source_line),

470 "expected identifier",

471 )

472 if not MD_Lexer._is_alpha(name[0]): 472 ↛ 473line 472 didn't jump to line 473 because the condition on line 472 was never true

473 self.mh.lex_error(

474 self._source_ref(loc_line, heading_prefix_len + 1, source_line),

475 f"unexpected character '{name[0]}'",

476 )

477 for i, ch in enumerate(name[1:], 1):

478 if not (MD_Lexer._is_alnum(ch) or ch == "_"): 478 ↛ 479line 478 didn't jump to line 479 because the condition on line 478 was never true

479 self.mh.lex_error(

480 self._source_ref(loc_line, heading_prefix_len + 1 + i, source_line),

481 f"unexpected character '{ch}'",

482 )

483

484 @staticmethod

485 def _is_separator_row(row):

486 """Return True if *row* is a Markdown table separator (``|---|``)."""

487 stripped = row.strip()

488 if not stripped.startswith("|"): 488 ↛ 489line 488 didn't jump to line 489 because the condition on line 488 was never true

489 return False

490 for c in stripped:

491 if c not in "|-: ":

492 return False

493 return True

494

495 @staticmethod

496 def _parse_table_row(row):

497 """Split a ``| key | value |`` row into (key, value) strings.

498

499 Returns ``None`` when the row cannot be parsed as a two-column

500 table entry.

501 """

502 stripped = row.strip()

503 if not stripped.startswith("|"):

504 return None

505 # Split on "|", ignore the empty strings at both ends

506 parts = [p.strip() for p in stripped.split("|")]

507 # After split: ["", cell0, cell1, ..., ""]

508 cells = parts[1:-1]

509 if len(cells) >= 2: 509 ↛ 511line 509 didn't jump to line 511 because the condition on line 509 was always true

510 return cells[0], cells[1]

511 return None

512

513 def _emit_qualified_identifier(self, location, value):

514 """Emit IDENTIFIER[/DOT/IDENTIFIER...] tokens for a type name."""

515 parts = [p for p in value.split(".") if p]

516 for idx, part in enumerate(parts):

517 self._emit(location, "IDENTIFIER", part)

518 if idx < len(parts) - 1:

519 self._emit(location, "DOT")

520

521 # Separator symbol token kinds (mirrors TRLC_Lexer.PUNCTUATION).

522 _SEPARATOR_PUNCTUATION = {"@": "AT", ":": "COLON", ";": "SEMICOLON"}

523

524 @staticmethod

525 def _normalize_array_value(raw_value):

526 """Normalize array value by converting various separators to comma format.

527

528 Handles:

529 - tags (converted to newlines, then normalized)

530 - Multiple spaces around @, :, ; separators (normalized to single space)

531 - Identifier separators (multiple spaces collapsed to single space)

532 - Newlines (preserved initially for multiline detection, then normalized)

533

534 Returns normalized string with tuple refs as

535 ``"identifier <sep> integer, ..."``.

536 """

537 # Replace and tags with newlines for uniform processing

538 normalized = re.sub(r"<[Bb][Rr]\s*/?>", "\n", raw_value)

539

540 # Split on newlines and commas, trim each part

541 parts = []

542 for line in normalized.split("\n"):

543 for item in line.split(","):

544 trimmed = item.strip()

545 if trimmed: 545 ↛ 543line 545 didn't jump to line 543 because the condition on line 545 was always true

546 parts.append(trimmed)

547

548 # Normalize whitespace around each part:

549 # - punctuation separators (@, :, ;) get exactly one space each side

550 # - identifier separators: collapse multiple spaces to one

551 normalized_parts = []

552 for part in parts:

553 part = re.sub(r"\s*([@:;])\s*", r" \1 ", part)

554 part = re.sub(r" +", " ", part.strip())

555 normalized_parts.append(part)

556

557 return ", ".join(normalized_parts)

558

559 # Tuple-reference pattern: package-qualified identifier, separator, integer.

560 # The reference MUST contain at least one dot (Package.name) so that plain

561 # free-text values are never falsely matched.

562 # Separator is @, :, ; or a plain identifier.

563 _TUPLE_REF_RE = re.compile(

564 r"^"

565 r"(?P<ref>[a-zA-Z_]\w*\.[a-zA-Z_]\w*(?:\.[a-zA-Z_]\w*)*)"

566 r" "

567 r"(?P<sep>[@:;]|[a-zA-Z_]\w*)"

568 r" "

569 r"(?P<ver>\d+)"

570 r"$"

571 )

572

573 # Like _TUPLE_REF_RE but the dot is optional, so unqualified same-package

574 # references (e.g. ``item @ 1``) are also accepted. Only safe in the

575 # Phase 2 type-aware path where the field type is already known.

576 _TUPLE_REF_FLEXIBLE_RE = re.compile(

577 r"^"

578 r"(?P<ref>[a-zA-Z_]\w*(?:\.[a-zA-Z_]\w*)*)"

579 r" "

580 r"(?P<sep>[@:;]|[a-zA-Z_]\w*)"

581 r" "

582 r"(?P<ver>\d+)"

583 r"$"

584 )

585

586 # Chunk patterns for the multi-separator tuple scanner.

587 # A numeric chunk: hex, binary, or decimal (integer or float).

588 _TUPLE_NUM_RE = re.compile(

589 r"0[xX][0-9a-fA-F][0-9a-fA-F_]*"

590 r"|0[bB][01][01_]*"

591 r"|\d+(?:\.\d+)?"

592 )

593 # A separator chunk: @, :, ; or a word identifier.

594 _TUPLE_SEP_RE = re.compile(r"[@:;]|[a-zA-Z_]\w*")

595

596 @staticmethod

597 def _looks_like_array(raw_value, allow_unqualified=False):

598 """Check if value looks like tuple-reference array without emitting.

599

600 Expected format (separator may be @, :, ;, or any identifier)::

601

602 identifier[.identifier]* sep integer

603 [, identifier[.identifier]* sep integer]*

604

605 When *allow_unqualified* is True, unqualified same-package references

606 (e.g. ``item @ 1``) are also accepted. Only pass True when the field

607 type is confirmed as a tuple-reference array.

608 """

609 normalized = MD_Lexer._normalize_array_value(raw_value)

610 parts = [p.strip() for p in normalized.split(",") if p.strip()]

611 pattern = (

612 MD_Lexer._TUPLE_REF_FLEXIBLE_RE

613 if allow_unqualified

614 else MD_Lexer._TUPLE_REF_RE

615 )

616 return bool(parts) and all(pattern.match(part) for part in parts)

617

618 @staticmethod

619 def _looks_like_simple_tuple(raw_value):

620 """Check if value looks like a separator-form tuple: num [sep num]+.

621

622 Handles multi-separator patterns like 0x500:12345@6.1 or 1@2:3;4.

623 Returns True only when there is at least one separator (bare numbers

624 are handled by the scalar integer/decimal paths in _emit_value).

625 """

626 value = raw_value.strip()

627 pos = 0

628 n = len(value)

629

630 def skip_ws():

631 nonlocal pos

632 while pos < n and value[pos] in (" ", "\t"): 632 ↛ 633line 632 didn't jump to line 633 because the condition on line 632 was never true

633 pos += 1

634

635 skip_ws()

636 m = MD_Lexer._TUPLE_NUM_RE.match(value, pos)

637 if not m: 637 ↛ 639line 637 didn't jump to line 639 because the condition on line 637 was always true

638 return False

639 pos = m.end()

640 has_sep = False

641 while pos < n:

642 skip_ws()

643 if pos >= n:

644 break

645 m = MD_Lexer._TUPLE_SEP_RE.match(value, pos)

646 if not m:

647 return False

648 pos = m.end()

649 has_sep = True

650 skip_ws()

651 m = MD_Lexer._TUPLE_NUM_RE.match(value, pos)

652 if not m:

653 return False

654 pos = m.end()

655 return pos == n and has_sep

656

657 def _maybe_emit_simple_tuple(self, raw_value, location):

658 """Try to emit tokens for a separator-form tuple: num [sep num]+.

659

660 Handles multi-separator patterns such as::

661

662 12345@42 → INTEGER AT INTEGER

663 0x500:12345@6.1 → INTEGER COLON INTEGER AT DECIMAL

664 1@2:3;4 → INTEGER AT INTEGER COLON INTEGER SEMICOLON INTEGER

665

666 Returns True if at least one separator was found and all tokens were

667 emitted, False to let _emit_value fall through to STRING.

668 """

669 value = raw_value.strip()

670 pos = 0

671 n = len(value)

672 chunks = [] # list of ('num', text) | ('sep', text)

673

674 def skip_ws():

675 nonlocal pos

676 while pos < n and value[pos] in (" ", "\t"): 676 ↛ 677line 676 didn't jump to line 677 because the condition on line 676 was never true

677 pos += 1

678

679 skip_ws()

680 m = MD_Lexer._TUPLE_NUM_RE.match(value, pos)

681 if not m:

682 return False

683 chunks.append(("num", m.group()))

684 pos = m.end()

685

686 while pos < n:

687 skip_ws()

688 if pos >= n: 688 ↛ 689line 688 didn't jump to line 689 because the condition on line 688 was never true

689 break

690 m = MD_Lexer._TUPLE_SEP_RE.match(value, pos)

691 if not m: 691 ↛ 692line 691 didn't jump to line 692 because the condition on line 691 was never true

692 return False

693 chunks.append(("sep", m.group()))

694 pos = m.end()

695 skip_ws()

696 m = MD_Lexer._TUPLE_NUM_RE.match(value, pos)

697 if not m: 697 ↛ 698line 697 didn't jump to line 698 because the condition on line 697 was never true

698 return False

699 chunks.append(("num", m.group()))

700 pos = m.end()

701

702 if pos != n: 702 ↛ 703line 702 didn't jump to line 703 because the condition on line 702 was never true

703 return False

704

705 # Must have at least one separator so bare numbers fall through to

706 # the integer/decimal scalar checks in _emit_value.

707 has_sep = any(kind == "sep" for kind, _ in chunks)

708 if not has_sep: 708 ↛ 709line 708 didn't jump to line 709 because the condition on line 708 was never true

709 return False

710

711 for kind, text in chunks:

712 if kind == "num":

713 if "." in text:

714 self._emit(location, "DECIMAL", Fraction(text.replace("_", "")))

715 else:

716 self._emit(location, "INTEGER", self._parse_number(text))

717 else: # sep

718 sep_kind = MD_Lexer._SEPARATOR_PUNCTUATION.get(text, "IDENTIFIER")

719 sep_val = text if sep_kind == "IDENTIFIER" else None

720 self._emit(location, sep_kind, sep_val)

721

722 return True

723

724 def _maybe_emit_parentheses_tuple(self, raw_value, location):

725 """Try to emit tokens for a tuple in parentheses form: (val1, val2, val3).

726

727 Returns True if a tuple was emitted, False for fallback to STRING.

728 """

729 value = raw_value.strip()

730 if not (value.startswith("(") and value.endswith(")")):

731 return False

732

733 inner = value[1:-1].strip()

734 if not inner: 734 ↛ 736line 734 didn't jump to line 736 because the condition on line 734 was never true

735 # Empty tuple () - treat as error or skip

736 return False

737

738 # Split by commas and parse each value

739 self._emit(location, "BRA")

740

741 parts = [p.strip() for p in inner.split(",") if p.strip()]

742 for idx, part in enumerate(parts):

743 if idx > 0:

744 self._emit(location, "COMMA")

745

746 # Try to infer the value type for each part

747 self._emit_value(part, location)

748

749 self._emit(location, "KET")

750 return True

751

752 @staticmethod

753 def _parse_number(text):

754 """Parse a number string (decimal, hex, or binary).

755

756 Returns the integer value or None if parsing fails.

757 """

758 try:

759 text_clean = text.replace("_", "")

760 if text_clean.startswith("0x") or text_clean.startswith("0X"):

761 return int(text_clean, 16)

762 elif text_clean.startswith("0b") or text_clean.startswith("0B"): 762 ↛ 763line 762 didn't jump to line 763 because the condition on line 762 was never true

763 return int(text_clean, 2)

764 else:

765 return int(text_clean, 10)

766 except ValueError:

767 return None

768

769 def _check_bracket_array(self, raw_value, location):

770 """Detect bracket notation for tuple arrays and emit a clear error.

771

772 Bracket syntax like ``[Pkg.item @ 1, Pkg.item @ 2]`` is not supported

773 because brackets conflict with Markdown URL syntax.

774

775 Returns True when bracket array syntax was detected (error was emitted),

776 so the caller can skip further processing of this value.

777 """

778 value = raw_value.strip()

779 if not (value.startswith("[") and value.endswith("]")):

780 return False

781 inner = value[1:-1].strip()

782 # Only reject if the content inside the brackets actually looks like

783 # tuple references – avoids false positives on URLs containing '@'

784 if not MD_Lexer._looks_like_array(inner): 784 ↛ 785line 784 didn't jump to line 785 because the condition on line 784 was never true

785 return False

786 self.mh.error(

787 location,

788 "bracket notation for tuple-reference arrays is not supported",

789 explanation=(

790 "Use comma-separated or newline-separated tuple references "

791 "without brackets instead, e.g. "

792 "'Pkg.item_a @ 1, Pkg.item_b @ 2'"

793 ),

794 fatal=False,

795 )

796 return True

797

798 def _maybe_emit_array(self, raw_value, location, allow_unqualified=False):

799 """Try to emit array tokens for tuple-reference format.

800

801 When *allow_unqualified* is True, unqualified same-package references

802 (e.g. ``item @ 1``) are also accepted. Only safe when the field type

803 is confirmed as a tuple-reference array (Phase 2 path).

804

805 Recognises values of the form::

806

807 Package.item sep integer [, Package.item sep integer]*

808

809 The reference must be package-qualified (contain at least one dot)

810 so that plain free-text values (e.g. ``Revision : 12``) are never

811 misinterpreted as tuple-reference arrays.

812

813 Returns True if array was emitted, False for STRING fallback.

814 """

815 if not self._looks_like_array(raw_value, allow_unqualified=allow_unqualified):

816 return False

817

818 normalized = self._normalize_array_value(raw_value)

819 parts = [p.strip() for p in normalized.split(",")]

820 self._emit(location, "S_BRA")

821 pattern = (

822 MD_Lexer._TUPLE_REF_FLEXIBLE_RE

823 if allow_unqualified

824 else MD_Lexer._TUPLE_REF_RE

825 )

826

827 for idx, part in enumerate(parts):

828 if idx > 0:

829 self._emit(location, "COMMA")

830

831 match = pattern.match(part)

832 if match: 832 ↛ 827line 832 didn't jump to line 827 because the condition on line 832 was always true

833 qual_ident = match.group("ref")

834 sep = match.group("sep")

835 integer = int(match.group("ver"))

836

837 # Emit qualified identifier (Pkg.item → IDENTIFIER DOT IDENTIFIER)

838 ident_parts = qual_ident.split(".")

839 for i, ident_part in enumerate(ident_parts):

840 self._emit(location, "IDENTIFIER", ident_part)

841 if i < len(ident_parts) - 1:

842 self._emit(location, "DOT")

843

844 # Emit separator: @→AT, :→COLON, ;→SEMICOLON, word→IDENTIFIER

845 sep_kind = MD_Lexer._SEPARATOR_PUNCTUATION.get(sep, "IDENTIFIER")

846 sep_val = sep if sep_kind == "IDENTIFIER" else None

847 self._emit(location, sep_kind, sep_val)

848

849 # Emit version integer

850 self._emit(location, "INTEGER", integer)

851

852 self._emit(location, "S_KET")

853 return True

854

855 def _emit_value(self, raw_value, location):

856 """Emit one or more tokens representing a property value.

857

858 Inference rules are applied in the order documented in the

859 module docstring.

860 """

861 value = raw_value.strip()

862

863 # Bracket array notation is not supported – emit a clear error

864 if self._check_bracket_array(raw_value, location):

865 return

866

867 # Parentheses tuple form: (val1, val2, val3)

868 if self._maybe_emit_parentheses_tuple(raw_value, location):

869 return

870

871 # Array of tuple references (before other checks)

872 if self._maybe_emit_array(raw_value, location):

873 return

874

875 # Boolean / null keywords

876 if value in MD_Lexer.KEYWORDS:

877 self._emit(location, "KEYWORD", value)

878 return

879

880 # Positive integers

881 end = MD_Lexer._scan_integer(value)

882 if end == len(value) and end > 0:

883 self._emit(location, "INTEGER", int(value.replace("_", "")))

884 return

885

886 # Positive decimals

887 dot_pos = value.find(".")

888 if dot_pos > 0:

889 int_end = MD_Lexer._scan_integer(value, 0)

890 if int_end == dot_pos:

891 dec_end = MD_Lexer._scan_integer(value, dot_pos + 1)

892 if dec_end == len(value) and dec_end > dot_pos + 1: 892 ↛ 897line 892 didn't jump to line 897 because the condition on line 892 was always true

893 self._emit(location, "DECIMAL", Fraction(value.replace("_", "")))

894 return

895

896 # Hexadecimal 0x…

897 if value.startswith("0x") and len(value) > 2:

898 end = MD_Lexer._scan_hex(value)

899 if end == len(value):

900 self._emit(location, "INTEGER", int(value.replace("_", ""), 16))

901 return

902

903 # Binary 0b…

904 if value.startswith("0b") and len(value) > 2:

905 end = MD_Lexer._scan_binary(value)

906 if end == len(value): 906 ↛ 911line 906 didn't jump to line 911 because the condition on line 906 was always true

907 self._emit(location, "INTEGER", int(value[2:].replace("_", ""), 2))

908 return

909

910 # Simple tuple syntax (e.g., 12345@42, 0x500:6.1)

911 if self._maybe_emit_simple_tuple(raw_value, location):

912 return

913

914 # Dot-qualified identifier or plain identifier

915 if value and MD_Lexer._is_ident_start(value[0]): 915 ↛ 941line 915 didn't jump to line 941 because the condition on line 915 was always true

916 parts = []

917 i = 0

918 valid = True

919 while i <= len(value): 919 ↛ 933line 919 didn't jump to line 933 because the condition on line 919 was always true

920 end = MD_Lexer._scan_ident(value, i)

921 if end < 0: 921 ↛ 922line 921 didn't jump to line 922 because the condition on line 921 was never true

922 valid = False

923 break

924 parts.append(value[i:end])

925 i = end

926 if i == len(value):

927 break

928 if value[i] == ".":

929 i += 1

930 else:

931 valid = False

932 break

933 if valid and parts:

934 for idx, part in enumerate(parts):

935 self._emit(location, "IDENTIFIER", part)

936 if idx < len(parts) - 1:

937 self._emit(location, "DOT")

938 return

939

940 # Fall-back: treat the value as a plain string

941 self._emit(location, "STRING", value)

942

943 @staticmethod

944 def _looks_like_scalar_value(value):

945 """Return True when *value* should be inferred via _emit_value.

946

947 This is used for single-line ``####`` field bodies so short scalar

948 values (e.g. enum members, booleans, numbers) behave like table

949 properties, while free text remains a STRING.

950 """

951 text = value.strip()

952 if not text: 952 ↛ 953line 952 didn't jump to line 953 because the condition on line 952 was never true

953 return False

954

955 if text in ("true", "false", "null"): 955 ↛ 956line 955 didn't jump to line 956 because the condition on line 955 was never true

956 return True

957

958 # Parentheses tuple form (simple detection)

959 if text.startswith("(") and text.endswith(")"): 959 ↛ 960line 959 didn't jump to line 960 because the condition on line 959 was never true

960 return True

961

962 # Integer / decimal

963 end = MD_Lexer._scan_integer(text)

964 if end == len(text) and end > 0: 964 ↛ 965line 964 didn't jump to line 965 because the condition on line 964 was never true

965 return True

966

967 dot_pos = text.find(".")

968 if dot_pos > 0:

969 int_end = MD_Lexer._scan_integer(text, 0)

970 if int_end == dot_pos: 970 ↛ 971line 970 didn't jump to line 971 because the condition on line 970 was never true

971 dec_end = MD_Lexer._scan_integer(text, dot_pos + 1)

972 if dec_end == len(text) and dec_end > dot_pos + 1:

973 return True

974

975 # Simple tuple syntax (e.g., 12345@42)

976 if MD_Lexer._looks_like_simple_tuple(text): 976 ↛ 977line 976 didn't jump to line 977 because the condition on line 976 was never true

977 return True

978

979 # Dot-qualified identifier (used e.g. for enum values)

980 if "." in text and MD_Lexer._is_ident_start(text[0]):

981 i = 0

982 while i <= len(text): 982 ↛ 994line 982 didn't jump to line 994 because the condition on line 982 was always true

983 end = MD_Lexer._scan_ident(text, i)

984 if end < 0: 984 ↛ 985line 984 didn't jump to line 985 because the condition on line 984 was never true

985 return False

986 i = end

987 if i == len(text): 987 ↛ 988line 987 didn't jump to line 988 because the condition on line 987 was never true

988 return True

989 if text[i] == ".":

990 i += 1

991 else:

992 return False

993

994 return False

995

996 # ------------------------------------------------------------------ #

997 # Main processing #

998 # ------------------------------------------------------------------ #

999

1000 def _process(self, content):

1001 """Transform *content* into the ``_md_tokens`` list."""

1002

1003 lines = content.splitlines()

1004 total_lines = len(lines)

1005

1006 # ── Section tracking ─────────────────────────────────────────── #

1007 in_section = False

1008 imported_packages = []

1009

1010 # ── Type tracking for Phase 2 ─────────────────────────────────── #

1011 current_package_name = None # from # PackageName heading

1012 current_record_type_ast = None # resolved after type row found

1013

1014 # ── Record tracking ───────────────────────────────────────────── #

1015 # When a ### heading is seen we buffer the name and then wait for

1016 # the properties table to discover the record type.

1017 in_record = False

1018 pending_name = None # identifier string for the record

1019 pending_name_loc = None # Location of the ### line

1020 pending_props = [] # [(key, value, line_no)] before "type" row

1021 record_type_found = False

1022 props_first_row = False # True while we should skip the header row

1023

1024 # ── String-field tracking ─────────────────────────────────────── #

1025 in_string_field = False

1026 str_field_name = None

1027 str_field_loc = None

1028 str_field_lines = []

1029 str_field_first_content_line = None

1030 str_field_first_content_col = 1

1031 # #### fields seen before the "type" row are buffered here, then

1032 # emitted inside the record after C_BRA (mirrors pending_props).

1033 # Tuple: (name, loc, text, emit_as_scalar, is_array, is_bracket)

1034 pending_string_fields = []

1035

1036 # ── Helpers (closures) ────────────────────────────────────────── #

1037

1038 def flush_string_field():

1039 nonlocal in_string_field, str_field_name, str_field_lines

1040 nonlocal str_field_first_content_line, str_field_first_content_col

1041 if not in_string_field:

1042 return

1043 # Strip leading and trailing blank lines

1044 while str_field_lines and not str_field_lines[0].strip(): 1044 ↛ 1045line 1044 didn't jump to line 1045 because the condition on line 1044 was never true

1045 str_field_lines.pop(0)

1046 while str_field_lines and not str_field_lines[-1].strip():

1047 str_field_lines.pop()

1048 text = "\n".join(str_field_lines)

1049

1050 emit_as_scalar = "\n" not in text and MD_Lexer._looks_like_scalar_value(

1051 text

1052 )

1053

1054 value_loc = str_field_loc

1055 if str_field_first_content_line is not None: 1055 ↛ 1061line 1055 didn't jump to line 1061 because the condition on line 1055 was always true

1056 value_loc = self._loc(

1057 str_field_first_content_line,

1058 str_field_first_content_col,

1059 )

1060

1061 if in_record: 1061 ↛ 1071line 1061 didn't jump to line 1071 because the condition on line 1061 was always true

1062 # Record already open – emit directly inside the block.

1063 self._emit(str_field_loc, "IDENTIFIER", str_field_name)

1064 self._emit(str_field_loc, "ASSIGN")

1065 self._emit_field_value(

1066 text, value_loc, current_record_type_ast, str_field_name

1067 )

1068 else:

1069 # "type" row not yet seen – buffer until the record opens.

1070 # Check if it looks like an array (but don't emit yet)

1071 is_bracket = (

1072 not emit_as_scalar

1073 and text.strip().startswith("[")

1074 and text.strip().endswith("]")

1075 and "@" in text

1076 )

1077 is_array = (

1078 not emit_as_scalar

1079 and not is_bracket

1080 and MD_Lexer._looks_like_array(text)

1081 )

1082 pending_string_fields.append(

1083 (

1084 str_field_name,

1085 value_loc,

1086 text,

1087 emit_as_scalar,

1088 is_array,

1089 is_bracket,

1090 )

1091 )

1092 in_string_field = False

1093 str_field_name = None

1094 str_field_lines = []

1095 str_field_first_content_line = None

1096 str_field_first_content_col = 1

1097

1098 def flush_record(loc):

1099 nonlocal in_record, pending_name, pending_props

1100 nonlocal record_type_found, props_first_row

1101 if not in_record:

1102 # Nothing open – check for incomplete pending record

1103 if pending_name is not None and not record_type_found: 1103 ↛ 1104line 1103 didn't jump to line 1104 because the condition on line 1103 was never true

1104 self.mh.error(

1105 pending_name_loc,

1106 f"record heading '{pending_name}' has no 'type' property"

1107 " in its property table; record will be skipped",

1108 fatal=False,

1109 )

1110 pending_name = None

1111 pending_props = []

1112 pending_string_fields.clear()

1113 record_type_found = False

1114 props_first_row = False

1115 return

1116 flush_string_field()

1117 self._emit(loc, "C_KET")

1118 in_record = False

1119 pending_name = None

1120 pending_props = []

1121 pending_string_fields.clear()

1122 record_type_found = False

1123 props_first_row = False

1124

1125 def open_section(name, loc):

1126 nonlocal in_section

1127 self._emit(loc, "KEYWORD", "##")

1128 self._emit(loc, "STRING", name)

1129 self._emit(loc, self.MD_SECTION_START_TOKEN)

1130 in_section = True

1131

1132 def close_section(loc):

1133 nonlocal in_section

1134 if in_section:

1135 self._emit(loc, self.MD_SECTION_END_TOKEN)

1136 in_section = False

1137

1138 # ── Line-by-line scan ─────────────────────────────────────────── #

1139

1140 for i, line in enumerate(lines):

1141 line_no = i + 1

1142 loc = self._loc(line_no)

1143 stripped = line.strip()

1144

1145 # ── While inside a string field, only headings / <hr> break out ─

1146

1147 if in_string_field:

1148 level, _ = MD_Lexer._parse_heading(line)

1149 if level in (2, 3, 4) or MD_Lexer._is_hr(stripped):

1150 flush_string_field()

1151 # fall through so the line is processed normally

1152 elif not in_record and stripped.startswith("|"): 1152 ↛ 1157line 1152 didn't jump to line 1157 because the condition on line 1152 was never true

1153 # A property table row arrived while collecting a ####

1154 # string field but before "type" is known. Close the

1155 # string field (buffering its content) so the table row

1156 # is processed normally below.

1157 flush_string_field()

1158 # fall through to table-row handling

1159 else:

1160 if str_field_first_content_line is None and stripped:

1161 line_stripped = line.lstrip()

1162 str_field_first_content_line = line_no

1163 str_field_first_content_col = len(line) - len(line_stripped) + 1

1164 str_field_lines.append(line)

1165 continue

1166

1167 # ── Heading dispatch (H1–H4) ─────────────────────────────────

1168

1169 _h_level, _h_content = MD_Lexer._parse_heading(line)

1170

1171 # ── H1: package declaration ──────────────────────────────────

1172

1173 if _h_level == 1:

1174 parts = _h_content.split()

1175 if len(parts) != 1: 1175 ↛ 1176line 1175 didn't jump to line 1176 because the condition on line 1175 was never true

1176 self.mh.lex_error(loc, "package heading must be '# <PackageName>'")

1177 package_name = parts[0]

1178 if ( 1178 ↛ 1186line 1178 didn't jump to line 1186 because the condition on line 1178 was never true

1179 not package_name

1180 or not MD_Lexer._is_alpha(package_name[0])

1181 or any(

1182 not (MD_Lexer._is_alnum(ch) or ch == "_")

1183 for ch in package_name[1:]

1184 )

1185 ):

1186 self.mh.lex_error(loc, "invalid package name in markdown heading")

1187 self._emit(loc, "KEYWORD", "#")

1188 self._emit(loc, "IDENTIFIER", package_name)

1189 current_package_name = package_name

1190 continue

1191

1192 # ── H2: section ──────────────────────────────────────────────

1193 # _parse_heading already distinguishes the levels by exact count

1194 # of leading "#" characters, so no prefix-collision is possible.

1195

1196 if _h_level == 2:

1197 flush_record(loc)

1198 close_section(loc)

1199 open_section(_h_content, loc)

1200 continue

1201

1202 # ── H3: record heading ───────────────────────────────────────

1203

1204 if _h_level == 3:

1205 flush_record(loc)

1206 pending_name = _h_content.strip()

1207 # heading_prefix_len = level(3) + 1 space

1208 self._validate_identifier(pending_name, line_no, _h_level + 1, line)

1209 pending_name_loc = loc

1210 pending_props = []

1211 record_type_found = False

1212 props_first_row = True # skip the column-header row

1213 continue

1214

1215 # ── H4: string field heading ─────────────────────────────────

1216

1217 if _h_level == 4:

1218 # flush_string_field already called at the top of the loop

1219 str_field_name = _h_content.strip()

1220 # heading_prefix_len = level(4) + 1 space

1221 self._validate_identifier(str_field_name, line_no, _h_level + 1, line)

1222 str_field_loc = loc

1223 str_field_lines = []

1224 str_field_first_content_line = None

1225 str_field_first_content_col = 1

1226 in_string_field = True

1227 continue

1228

1229 # ── Horizontal rule ──────────────────────────────────────────

1230

1231 if MD_Lexer._is_hr(stripped):

1232 flush_record(loc)

1233 continue

1234

1235 # ── Table rows ───────────────────────────────────────────────

1236

1237 if stripped.startswith("|") and pending_name is not None:

1238 # Skip the column-header row (first row after ###)

1239 if props_first_row:

1240 props_first_row = False

1241 continue

1242

1243 # Skip Markdown separator rows (|---|---|)

1244 if self._is_separator_row(stripped):

1245 continue

1246

1247 row = self._parse_table_row(stripped)

1248 if row is None: 1248 ↛ 1249line 1248 didn't jump to line 1249 because the condition on line 1248 was never true

1249 continue

1250 key, value = row

1251 if not key: 1251 ↛ 1252line 1251 didn't jump to line 1252 because the condition on line 1251 was never true

1252 continue

1253

1254 if key == "type":

1255 # Emit: RecordType RecordName {

1256 type_name = value.strip()

1257 if "." not in type_name and len(imported_packages) == 1:

1258 type_name = imported_packages[0] + "." + type_name

1259 self._emit_qualified_identifier(loc, type_name)

1260 self._emit(pending_name_loc, "IDENTIFIER", pending_name)

1261 self._emit(pending_name_loc, "C_BRA")

1262 in_record = True

1263 record_type_found = True

1264 current_record_type_ast = self._resolve_record_type(

1265 type_name, package_name=current_package_name

1266 )

1267

1268 # Flush any properties that arrived before "type"

1269 for bkey, bval, bline in pending_props: 1269 ↛ 1270line 1269 didn't jump to line 1270 because the loop on line 1269 never started

1270 bloc = self._loc(bline)

1271 self._emit(bloc, "IDENTIFIER", bkey)

1272 self._emit(bloc, "ASSIGN")

1273 self._emit_field_value(

1274 bval, bloc, current_record_type_ast, bkey

1275 )

1276 pending_props = []

1277

1278 # Flush any #### string fields that arrived

1279 # before "type"

1280 for ( 1280 ↛ 1288line 1280 didn't jump to line 1288 because the loop on line 1280 never started

1281 fname,

1282 floc,

1283 ftext,

1284 _fis_scalar,

1285 _fis_array,

1286 _fis_bracket,

1287 ) in pending_string_fields:

1288 self._emit(floc, "IDENTIFIER", fname)

1289 self._emit(floc, "ASSIGN")

1290 self._emit_field_value(

1291 ftext, floc, current_record_type_ast, fname

1292 )

1293 pending_string_fields.clear()

1294

1295 elif record_type_found: 1295 ↛ 1302line 1295 didn't jump to line 1302 because the condition on line 1295 was always true

1296 self._emit(loc, "IDENTIFIER", key)

1297 self._emit(loc, "ASSIGN")

1298 self._emit_field_value(value, loc, current_record_type_ast, key)

1299

1300 else:

1301 # Buffer: "type" has not appeared yet

1302 pending_props.append((key, value, line_no))

1303

1304 continue

1305

1306 # ── Import statement ─────────────────────────────────────────

1307

1308 if stripped.startswith("import "):

1309 parts = stripped.split()

1310 if len(parts) == 2: 1310 ↛ 1321line 1310 didn't jump to line 1321 because the condition on line 1310 was always true

1311 self._emit(loc, "KEYWORD", "import")

1312 self._emit(loc, "IDENTIFIER", parts[1])

1313 imported_packages.append(parts[1])

1314 continue

1315

1316 # ── Everything else: delegate to TRLC_Lexer ──────────────────

1317 # Tokenize the raw line so the parser receives real tokens and

1318 # can report a meaningful error (e.g. "expected keyword #,

1319 # encountered OPERATOR instead" for "* foo", or "encountered

1320 # IDENTIFIER instead" for "sadsad foo").

1321 if stripped: 1321 ↛ 1322line 1321 didn't jump to line 1322 because the condition on line 1321 was never true

1322 trlc_lex = TRLC_Lexer(self.mh, self.file_name, line)

1323 tok = trlc_lex.token()

1324 while tok is not None:

1325 self._emit(

1326 self._loc(line_no, tok.location.col_no),

1327 tok.kind,

1328 tok.value,

1329 )

1330 tok = trlc_lex.token()

1331

1332 # ── End-of-file cleanup ──────────────────────────────────────────

1333

1334 eof_loc = self._loc(total_lines + 1)

1335 flush_record(eof_loc)

1336 close_section(eof_loc)

Coverage for trlc/lexer_md.py: 84%

690 statements